[ { "path": ".clang-format", "content": "Language: Cpp\nBasedOnStyle: WebKit\nAccessModifierOffset: -2\nAlignAfterOpenBracket: Align\nAlignArrayOfStructures: None\nAlignConsecutiveAssignments: None\nAlignConsecutiveBitFields: Consecutive\nAlignConsecutiveDeclarations: None\nAlignConsecutiveMacros: None\nAlignEscapedNewlines: Left\nAlignOperands: Align\nAlignTrailingComments: true\nAllowAllParametersOfDeclarationOnNextLine: false\nAllowShortCaseLabelsOnASingleLine: true\nAllowShortEnumsOnASingleLine: true\nAllowShortFunctionsOnASingleLine: Empty\nAllowShortIfStatementsOnASingleLine: WithoutElse\nAllowShortLambdasOnASingleLine: Inline\nAlwaysBreakAfterDefinitionReturnType: None\nAlwaysBreakAfterReturnType: None\nAlwaysBreakBeforeMultilineStrings: false\nAlwaysBreakTemplateDeclarations: true\nAttributeMacros:\n - JEMALLOC_NOTHROW\n - FEX_ALIGNED\n - FEX_ANNOTATE\n - FEX_DEFAULT_VISIBILITY\n - FEX_NAKED\n - FEX_PACKED\n - FEXCORE_PRESERVE_ALL_ATTR\n - GLIBC_ALIAS_FUNCTION\nBinPackArguments: true\nBinPackParameters: true\nBitFieldColonSpacing: Both\nBreakAfterAttributes: Leave\nBreakBeforeBraces: Attach\nBreakBeforeBinaryOperators: None\nBreakBeforeInlineASMColon: OnlyMultiline # clang 16 required\nBreakBeforeTernaryOperators: false\nBreakConstructorInitializers: BeforeComma\nBreakInheritanceList: BeforeColon\nColumnLimit: 140\nCompactNamespaces: false\nConstructorInitializerIndentWidth: 2\nContinuationIndentWidth: 2\nCpp11BracedListStyle: true\nDerivePointerAlignment: false\nEmptyLineAfterAccessModifier: Leave\nEmptyLineBeforeAccessModifier: Leave\nExperimentalAutoDetectBinPacking: false\nFixNamespaceComments: true\nIncludeBlocks: Preserve\nIndentAccessModifiers: false\nIndentCaseBlocks: false\nIndentCaseLabels: false\nIndentExternBlock: AfterExternBlock\nIndentGotoLabels: false\nIndentPPDirectives: None\nIndentRequires: false\nIndentWidth: 2\nInsertBraces: true\nKeepEmptyLinesAtTheStartOfBlocks: true\nLambdaBodyIndentation: Signature\nLineEnding: LF # clang 16 required\nMaxEmptyLinesToKeep: 2\nNamespaceIndentation: Inner\nQualifierAlignment: Left\nPackConstructorInitializers: Never\nPenaltyBreakAssignment: 2\nPenaltyBreakBeforeFirstCallParameter: 2\nPenaltyBreakOpenParenthesis: 2\nPenaltyBreakString: 10\nPenaltyBreakTemplateDeclaration: 8\nPenaltyExcessCharacter: 2\nPenaltyReturnTypeOnItsOwnLine: 16\nPointerAlignment: Left\nRemoveBracesLLVM: false\nReferenceAlignment: Left\nReflowComments: true\nRequiresClausePosition: WithPreceding\nSeparateDefinitionBlocks: Leave\nSortIncludes: Never\nSpaceAfterCStyleCast: false\nSpaceAfterLogicalNot: false\nSpaceAfterTemplateKeyword: false\nSpaceAroundPointerQualifiers: Default\nSpaceBeforeAssignmentOperators: true\nSpaceBeforeCaseColon: false\nSpaceBeforeCpp11BracedList: true\nSpaceBeforeInheritanceColon: true\nSpaceBeforeParens: Custom\nSpaceBeforeParensOptions:\n AfterControlStatements: true\n AfterFunctionDeclarationName: false\n AfterFunctionDefinitionName: false\n AfterOverloadedOperator: false\n AfterRequiresInClause: true\n BeforeNonEmptyParentheses: false\nSpaceBeforeRangeBasedForLoopColon: true\nSpaceBeforeSquareBrackets: false\nSpaceInEmptyBlock: false\nSpaceInEmptyParentheses: false\nSpacesBeforeTrailingComments: 1\nSpacesInAngles: Leave\nSpacesInCStyleCastParentheses: false\nSpacesInConditionalStatement: false\nSpacesInParentheses: false\nStandard: c++20\nUseTab: Never\n" }, { "path": ".clang-format-ignore", "content": "# This file is used to ignore files and directories from clang-format\nSource/Common/cpp-optparse/*\n\n# Files with human-indented tables for readability - don't mess with these\nFEXCore/Source/Interface/Core/X86Tables/*.cpp\n\n# Inline headers with list-like content that can't be processed individually\nSource/Tools/LinuxEmulation/LinuxSyscalls/x*/SyscallsNames.inl\nSource/Tools/LinuxEmulation/LinuxSyscalls/x*/Ioctl/*.inl\n\n# Include files in unittests\nunittests/*ASM/Includes/*.inc\n" }, { "path": ".git-blame-ignore-revs", "content": "# Since version 2.23 (released in August 2019), git-blame has a feature\n# to ignore or bypass certain commits.\n#\n# This file contains a list of commits that are not likely what you\n# are looking for in a blame, such as mass reformatting or renaming.\n# You can set this file as a default ignore file for blame by running\n# the following command.\n#\n# $ git config blame.ignoreRevsFile .git-blame-ignore-revs\n\n# Whole tree reformat PR#3571\n2b4ec88daebd35fefb5bf5c73d7fc2b4155771ed\n\n# Second reformat to find fixed point PR#3577\n905aa935f5ce344a48ef4d5edab3c31efa8d793e\n\n# Reformat of CodeEmitter inl files\n8760c593ece92d7e9fa94c40da0368fd367c9cad\n\n# Whole-tree reformat with clang-format-19\n5267cde60e7642852d18f20ae8568643bb5293d5\n\n# Minor reformat with clang-format-19\n9fdd96af61c969cb5732471223f00eda64b7a069\n\n# Reformat of X86Tables.h\nba2b0ef809f66f1a6d334f000798fa2ceafab26f\n" }, { "path": ".github/ISSUE_TEMPLATE/potential-game-bug.md", "content": "---\nname: Potential Game Bug\nabout: A bug in FEX-Emu that causes a problem in a game\ntitle: \"[Game]: [Short Problem Description]\"\nlabels: Game related\nassignees: ''\n\n---\n\n**What Game**\nThe game name.\nA link to the storefront where to get the game. GOG, Steam, Itch.io, etc\n\n**Describe the bug**\nA clear and concise description of what the bug is.\n\n**To Reproduce**\nSteps to reproduce the behavior:\n1. Go to '...'\n2. Click on '....'\n3. Scroll down to '....'\n4. See error\n\n**Expected behavior**\nA clear and concise description of what you expected to happen.\n\n**Screenshots and Video**\nIf applicable, add screenshots and video to help explain your problem.\n\n**System information:**\n - OS: [eg: Ubuntu 21.10]\n - CPU/SoC: [eg: Snapdragon 888, Intel Core i8-12900k]\n - Video driver version: [eg: OpenGL ES 3.2 Mesa 22.0.0-devel (git-9ff086052a)]\n - RootFS used: [eg: Ubuntu 21.10 Official Rootfs]\n - FEX version: (FEXGetConfig --version) [eg: FEX-2112-155-gc691d709]\n - Thunks Enabled: [Yes/No]\n\n**Additional context**\n - Is this an x86 or x86-64 game: [x86/x86-64/Both]\n - Does this reproduce on AArch64 with Radeon/Intel/Nvidia: [Yes/No/Untested]\n - Is this a Vulkan game: [Yes/No/Unknown]\n - If Yes, What is your Vulkan driver:\n\nAdd any other context about the problem here.\n" }, { "path": ".github/workflows/ccpp.yml", "content": "name: Build + Test\n\non:\n push:\n branches:\n - main\n pull_request:\n branches:\n - main\n\nenv:\n # Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.)\n BUILD_TYPE: Release\n CC: clang\n CXX: clang++\n FEX_PORTABLE: 1\n\njobs:\n build_plus_test:\n runs-on: ${{ matrix.arch }}\n strategy:\n matrix:\n arch: [[self-hosted, ARMv8.0], [self-hosted, ARMv8.2], [self-hosted, ARMv8.4]]\n fail-fast: false\n\n steps:\n - uses: actions/checkout@v6\n with:\n fetch-depth: '0'\n fetch-tags: 'true'\n\n - name: Set runner info\n run: |\n echo \"runner_label=${{ matrix.arch[1] }}\" >> $GITHUB_ENV\n echo \"runner_name=$(hostname)\" >> $GITHUB_ENV\n\n - name: Setup Build Environment\n uses: ./.github/workflows/setup-env\n\n - name: Configure CMake\n run: |\n cmake -S . -B build -DCMAKE_BUILD_TYPE=$BUILD_TYPE -G Ninja -DENABLE_LTO=False -DENABLE_ASSERTIONS=True \\\n -DENABLE_X86_HOST_DEBUG=True -DBUILD_FEX_LINUX_TESTS=True -DBUILD_THUNKS=True \\\n -DCMAKE_INSTALL_PREFIX=\"$PWD\"/build/install\n\n # These steps make a lot of noise but rarely fail.\n # Put them in a separate step to make normal build logs easier to parse\n - name: Noisy Build Targets\n run: cmake --build build --target asm_files 32bit_asm_files JemallocLibs Catch2 vixl cephes_128bit\n\n - name: Build\n id: build\n run: cmake --build build\n\n - name: Install\n run: cmake --build build --target install\n\n # GCC tests\n - name: GCC64 Target Tests\n if: steps.build.outcome == 'success'\n uses: ./.github/workflows/test\n with:\n target: gcc_target_tests_64\n\n - name: GCC32 Target Tests\n if: steps.build.outcome == 'success'\n uses: ./.github/workflows/test\n with:\n target: gcc_target_tests_32\n\n # API tests\n - name: API Tests\n if: steps.build.outcome == 'success'\n uses: ./.github/workflows/test\n with:\n target: api_tests\n\n - name: FEXCore API Tests\n if: steps.build.outcome == 'success'\n uses: ./.github/workflows/test\n with:\n target: fexcore_apitests\n\n # ARM emission tests\n - name: ARM Emitter Tests\n if: steps.build.outcome == 'success'\n uses: ./.github/workflows/test\n with:\n target: emitter_tests\n\n # Linux tests\n - name: FEX Linux Tests\n if: steps.build.outcome == 'success'\n uses: ./.github/workflows/test\n with:\n target: fex_linux_tests_all\n env:\n FEX_PORTABLE: 0\n\n # Thunking\n - name: Thunkgen tests\n if: steps.build.outcome == 'success'\n uses: ./.github/workflows/test\n with:\n target: thunkgen_tests\n\n - name: Test GL No-Thunks\n if: ${{ steps.build.outcome == 'success' && matrix.arch[1] == 'x64' }}\n uses: ./.github/workflows/test\n with:\n target: thunk_functional_tests_nothunks\n env:\n DISPLAY: ':0'\n\n - name: Test GL Thunks\n if: ${{ steps.build.outcome == 'success' && matrix.arch[1] == 'x64' }}\n uses: ./.github/workflows/test\n with:\n target: thunk_functional_tests_thunks\n env:\n DISPLAY: ':0'\n\n # ASM tests\n - name: ASM Tests\n if: steps.build.outcome == 'success'\n uses: ./.github/workflows/test\n with:\n target: asm_tests\n\n # POSIX tests\n - name: POSIX Tests\n if: steps.build.outcome == 'success'\n uses: ./.github/workflows/test\n with:\n target: posix_tests\n\n # GVisor tests\n - name: GVisor Tests\n if: steps.build.outcome == 'success'\n uses: ./.github/workflows/test\n with:\n target: gvisor_tests\n\n # Struct verifier tests\n - name: Struct verifier tests\n if: steps.build.outcome == 'success'\n uses: ./.github/workflows/test\n with:\n target: struct_verifier\n\n - name: Remove old SHM regions\n if: ${{ always() }}\n run: cmake --build build --target remove_old_shm_regions\n\n - name: Upload results\n if: ${{ always() }}\n uses: actions/upload-artifact@v6\n timeout-minutes: 1\n with:\n name: Results-${{ env.runner_name }}-${{ env.runner_label }}\n path: results/*.log\n retention-days: 3\n" }, { "path": ".github/workflows/glibc_fault.yml", "content": "name: GLIBC fault test\n# This workflow file is the same as the `Build + Test` with some key differences\n# - Runs on any x86 and ARM64 runner\n# - Disables the glibc jemalloc compile option\n# - Enables the glibc allocator fault option\n# - Disables gvisor tests to reduce stress on CI machines (tmp/shm tests overwhelm them)\n# - Disables thunk tests since they are incompatible with glibc fault allocator\n# - Disables ARMEmitter tests (We don't want to fault test vixl's disassembler)\n\non:\n push:\n branches:\n - main\n pull_request:\n branches:\n - main\n\nenv:\n # Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.)\n BUILD_TYPE: Release\n CC: clang\n CXX: clang++\n FEX_PORTABLE: 1\n\njobs:\n glibc_fault_test:\n runs-on: ${{ matrix.arch }}\n strategy:\n matrix:\n arch: [[self-hosted, ARM64]]\n fail-fast: false\n\n steps:\n - uses: actions/checkout@v6\n with:\n fetch-depth: '0'\n fetch-tags: 'true'\n\n - name: Set runner info\n run: |\n echo \"runner_label=${{ matrix.arch[1] }}\" >> $GITHUB_ENV\n echo \"runner_name=$(hostname)\" >> $GITHUB_ENV\n\n - name: Setup Build Environment\n uses: ./.github/workflows/setup-env\n\n - name: Configure CMake\n run: |\n cmake -S . -B build -DCMAKE_BUILD_TYPE=$BUILD_TYPE -G Ninja -DENABLE_LTO=False \\\n -DENABLE_ASSERTIONS=True -DENABLE_X86_HOST_DEBUG=True -DBUILD_FEX_LINUX_TESTS=True \\\n -DENABLE_GLIBC_ALLOCATOR_HOOK_FAULT=True -DENABLE_JEMALLOC_GLIBC_ALLOC=False \\\n -DCMAKE_INSTALL_PREFIX=\"$PWD\"/build/install\n\n # These steps make a lot of noise but rarely fail.\n # Put them in a separate step to make normal build logs easier to parse\n - name: Noisy Build Targets\n run: cmake --build build --target asm_files 32bit_asm_files JemallocLibs Catch2 vixl cephes_128bit\n\n - name: Build\n run: cmake --build build\n\n - name: Install\n run: cmake --build build --target install\n\n # GCC tests\n - name: GCC64 Target Tests\n if: ${{ always() }}\n uses: ./.github/workflows/test\n with:\n target: gcc_target_tests_64\n\n - name: GCC32 Target Tests\n if: ${{ always() }}\n uses: ./.github/workflows/test\n with:\n target: gcc_target_tests_32\n\n # API Tests\n - name: API Tests\n if: ${{ always() }}\n uses: ./.github/workflows/test\n with:\n target: api_tests\n\n - name: FEXCore API Tests\n if: ${{ always() }}\n uses: ./.github/workflows/test\n with:\n target: fexcore_apitests\n\n # Linux tests\n - name: FEX Linux Tests\n if: ${{ always() }}\n uses: ./.github/workflows/test\n with:\n target: fex_linux_tests_all\n\n # ASM Tests\n - name: ASM Tests\n if: ${{ always() }}\n uses: ./.github/workflows/test\n with:\n target: asm_tests\n\n # POSIX Tests\n - name: POSIX Tests\n if: ${{ always() }}\n uses: ./.github/workflows/test\n with:\n target: posix_tests\n\n - name: Remove old SHM regions\n if: ${{ always() }}\n run: cmake --build build --target remove_old_shm_regions\n\n - name: Upload results\n if: ${{ always() }}\n uses: actions/upload-artifact@v6\n timeout-minutes: 1\n with:\n name: Results-${{ env.runner_name }}-${{ env.runner_label }}\n path: results/*.log\n retention-days: 3\n\n" }, { "path": ".github/workflows/hostrunner.yml", "content": "name: Hostrunner tests\n\non:\n push:\n branches:\n - main\n pull_request:\n branches:\n - main\n\nenv:\n # Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.)\n BUILD_TYPE: Release\n CC: clang\n CXX: clang++\n FEX_PORTABLE: 1\n\njobs:\n hostrunner_tests:\n runs-on: ${{ matrix.arch }}\n strategy:\n matrix:\n arch: [[self-hosted, x64]]\n fail-fast: false\n\n steps:\n - uses: actions/checkout@v6\n with:\n fetch-depth: '0'\n fetch-tags: 'true'\n\n - name: Set runner info\n run: |\n echo \"runner_label=${{ matrix.arch[1] }}\" >> $GITHUB_ENV\n echo \"runner_name=$(hostname)\" >> $GITHUB_ENV\n\n - name: Setup Build Environment\n uses: ./.github/workflows/setup-env\n\n - name: Configure CMake\n run: |\n cmake -S . -B build -DCMAKE_BUILD_TYPE=$BUILD_TYPE -G Ninja -DENABLE_LTO=False \\\n -DENABLE_ASSERTIONS=True -DENABLE_X86_HOST_DEBUG=True\n\n # These steps make a lot of noise but rarely fail.\n # Put them in a separate step to make normal build logs easier to parse\n - name: Noisy Build Targets\n run: cmake --build build --target asm_files 32bit_asm_files JemallocLibs Catch2 vixl cephes_128bit\n\n - name: Build\n run: cmake --build build\n\n # ASM tests\n - name: ASM Tests\n if: ${{ always() }}\n uses: ./.github/workflows/test\n with:\n target: asm_tests\n\n - name: Upload results\n if: ${{ always() }}\n uses: actions/upload-artifact@v6\n timeout-minutes: 1\n with:\n name: Results-${{ env.runner_name }}-${{ env.runner_label }}\n path: results/*.log\n retention-days: 3\n\n" }, { "path": ".github/workflows/instcountci.yml", "content": "name: Instruction Count CI run\n\non:\n push:\n branches:\n - main\n pull_request:\n branches:\n - main\n\nenv:\n # Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.)\n BUILD_TYPE: Release\n CC: clang\n CXX: clang++\n\njobs:\n instcountci_tests:\n runs-on: ${{ matrix.arch }}\n strategy:\n matrix:\n arch: [[self-hosted, x64], [self-hosted, ARM64]]\n fail-fast: false\n\n steps:\n - uses: actions/checkout@v6\n with:\n fetch-depth: '0'\n fetch-tags: 'true'\n\n - name: Set runner info\n run: |\n echo \"runner_label=${{ matrix.arch[1] }}\" >> $GITHUB_ENV\n echo \"runner_name=$(hostname)\" >> $GITHUB_ENV\n\n - name: Setup Build Environment\n uses: ./.github/workflows/setup-env\n\n - name: Set VIXL_SIM_ENABLED\n run: |\n case '${{ matrix.arch[1] }}' in\n x64) _sim=True ;;\n ARM64) _sim=False ;;\n esac\n echo \"VIXL_SIM_ENABLED=$_sim\" >> $GITHUB_ENV\n\n - name: Configure CMake\n run: |\n cmake -S . -B build -DCMAKE_BUILD_TYPE=$BUILD_TYPE -G Ninja -DENABLE_VIXL_SIMULATOR=$VIXL_SIM_ENABLED \\\n -DENABLE_VIXL_DISASSEMBLER=True -DENABLE_LTO=False -DENABLE_ASSERTIONS=True -DENABLE_X86_HOST_DEBUG=True\n\n - name: Build\n env:\n FEX_DISABLETELEMETRY: 1\n run: cmake --build build --target CodeSizeValidation instcountci_test_files\n\n - name: Instruction Count Tests\n if: ${{ always() }}\n uses: ./.github/workflows/test\n with:\n target: instcountci_tests\n\n - name: Update local repo instcount\n if: ${{ always() }}\n run: cmake --build build --target instcountci_update_tests\n\n - name: Check InstCountCI diff\n if: ${{ always() }}\n run: git --no-pager diff --exit-code HEAD\n\n - name: Upload results\n if: ${{ always() }}\n uses: actions/upload-artifact@v6\n timeout-minutes: 1\n with:\n name: Results-${{ env.runner_name }}-${{ env.runner_label }}\n path: results/*.log\n retention-days: 3\n" }, { "path": ".github/workflows/mingw_build.yml", "content": "name: Mingw build\n\non:\n push:\n branches:\n - main\n pull_request:\n branches:\n - main\n\nenv:\n BUILD_TYPE: Debug\n\njobs:\n mingw_build:\n runs-on: ${{ matrix.arch }}\n strategy:\n matrix:\n arch: [[self-hosted, ARM64, mingw], [self-hosted, ARM64EC, mingw, ARM64]]\n fail-fast: false\n\n steps:\n - uses: actions/checkout@v6\n with:\n fetch-depth: '0'\n fetch-tags: 'true'\n\n - name: Set runner label\n run: echo \"runner_label=${{ matrix.arch[1] }}\" >> $GITHUB_ENV\n\n - name: Add MingGW to PATH\n run: echo \"$HOME/llvm-mingw/build/bin/\" >> $GITHUB_PATH\n\n - name: Set CC\n run: |\n case '${{ matrix.arch[1] }}' in\n x64) _cpu=x86_64 ;;\n ARM64) _cpu=aarch64 ;;\n ARM64EC) _cpu=arm64ec ;;\n esac\n echo \"MINGW_TRIPLE=${_cpu}-w64-mingw32\" >> $GITHUB_ENV\n\n - name: Setup Build Environment\n uses: ./.github/workflows/setup-env\n\n - name: Configure CMake\n run: |\n cmake -S . -B build -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DCMAKE_TOOLCHAIN_FILE=$GITHUB_WORKSPACE/Data/CMake/toolchain_mingw.cmake \\\n -DMINGW_TRIPLE=$MINGW_TRIPLE -G Ninja -DENABLE_LTO=False -DENABLE_ASSERTIONS=True -DENABLE_X86_HOST_DEBUG=True -DBUILD_TESTING=False \\\n -DCMAKE_INSTALL_PREFIX=\"$PWD\"/build/install\n\n - name: Build\n run: cmake --build build\n" }, { "path": ".github/workflows/pr-code-format.yml", "content": "# Inspired by LLVM's pr-code-format.yml at\n# https://github.com/llvm/llvm-project/blob/main/.github/workflows/pr-code-format.yml\n\nname: Check code formatting\non:\n pull_request:\n branches:\n - main\n\njobs:\n code_formatter:\n runs-on: [self-hosted, X64]\n if: github.repository == 'FEX-Emu/FEX'\n\n steps:\n - name: Checkout\n uses: actions/checkout@v4\n with:\n ref: ${{ github.event.pull_request.head.sha }}\n\n - name: Checkout through merge base\n uses: rmacklin/fetch-through-merge-base@v0\n timeout-minutes: 3\n with:\n base_ref: ${{ github.event.pull_request.base.ref }}\n head_ref: ${{ github.event.pull_request.head.sha }}\n deepen_length: 500\n\n - name: Get changed files\n run: |\n BASE=$(git merge-base main HEAD)\n FILES=$(git diff --name-only \"$BASE\" | tr '\\n' ',' | sed 's/,$//')\n echo \"CHANGED_FILES=$FILES\" >> $GITHUB_ENV\n\n echo \"Changed files:\"\n echo \"$FILES\"\n\n - name: Check git-clang-format-19 exists\n run: which git-clang-format-19\n\n - name: Setup Python env\n uses: actions/setup-python@v4\n with:\n python-version: 3.11\n cache: pip\n cache-dependency-path: ./External/code-format-helper/requirements_formatting.txt\n\n - name: Install python dependencies\n run: pip install -r ./External/code-format-helper/requirements_formatting.txt\n\n - name: Run code formatter\n env:\n CLANG_FORMAT_PATH: git-clang-format-19\n GITHUB_PR_NUMBER: ${{ github.event.pull_request.number }}\n START_REV: ${{ github.event.pull_request.base.sha }}\n END_REV: ${{ github.event.pull_request.head.sha }}\n run: |\n python ./External/code-format-helper/code-format-helper.py \\\n --repo \"FEX-Emu/FEX\" \\\n --issue-number \"$GITHUB_PR_NUMBER\" \\\n --start-rev \"$START_REV\" \\\n --end-rev \"$END_REV\" \\\n --changed-files \"$CHANGED_FILES\"\n" }, { "path": ".github/workflows/setup-env/action.yml", "content": "name: Setup Build Environment\ndescription: Setup RootFS and build environment\n\ninputs:\n setup-rootfs:\n description: 'Whether or not to set up the rootfs'\n default: true\n\nruns:\n using: composite\n steps:\n - name: Set rootfs paths\n if: ${{ inputs.setup-rootfs == 'true' }}\n shell: bash\n run: |\n echo \"FEX_ROOTFS_MOUNT=/mnt/AutoNFS/rootfs/\" >> $GITHUB_ENV\n echo \"FEX_ROOTFS_PATH=$HOME/Rootfs/\" >> $GITHUB_ENV\n echo \"FEX_ROOTFS=$HOME/Rootfs/\" >> $GITHUB_ENV\n\n - name: Update RootFS cache\n if: ${{ inputs.setup-rootfs == 'true' }}\n shell: bash\n run: python3 Scripts/CI_FetchRootFS.py\n\n - name: Checkout Submodules\n shell: bash\n run: |\n git submodule sync --recursive\n git submodule update --init --depth 1\n\n - name: Clean Build Environment\n shell: bash\n run: rm -Rf build\n" }, { "path": ".github/workflows/steamrt4.yml", "content": "name: steamrt4 build\n\non:\n push:\n branches:\n - main\n pull_request:\n branches:\n - main\n\nenv:\n DEBIAN_FRONTEND: noninteractive\n BUILD_TYPE: Release\n CC: clang\n CXX: clang++\n\njobs:\n steamrt4_build:\n runs-on: ${{ matrix.arch }}\n strategy:\n matrix:\n arch: [[self-hosted, ARM64, distrobox]]\n fail-fast: false\n\n steps:\n - uses: actions/checkout@v6\n with:\n fetch-depth: '0'\n fetch-tags: 'true'\n\n - name: Set runner label\n run: echo \"runner_label=${{ matrix.arch[1] }}\" >> $GITHUB_ENV\n\n - name: Setup Build Environment\n uses: ./.github/workflows/setup-env\n with:\n setup-rootfs: false\n\n # Setup everything required.\n - name : distrobox setup\n run: |\n distrobox create -Y -i registry.gitlab.steamos.cloud/steamrt/steamrt4/sdk/arm64:4.0.20251117.183306 steamrt4 || true\n distrobox upgrade steamrt4\n distrobox enter --name steamrt4 -- sudo apt-get install -y \\\n git cmake ninja-build ccache \\\n lld clang \\\n libclang-dev llvm-dev \\\n libstdc++-14-dev-i386-cross libgcc-14-dev-i386-cross \\\n libstdc++-14-dev-amd64-cross libgcc-14-dev-amd64-cross\n\n - name: Configure CMake\n run: |\n distrobox enter --name steamrt4 -- cmake -S . -B build -DCMAKE_BUILD_TYPE=$BUILD_TYPE \\\n -G Ninja -DBUILD_STEAM_SUPPORT=True -DENABLE_LTO=True -DENABLE_ASSERTIONS=False -DBUILD_THUNKS=True \\\n -DBUILD_FEXCONFIG=False -DBUILD_TESTING=False -DENABLE_CLANG_THUNKS=True -DUSE_LINKER=lld \\\n -DCMAKE_INSTALL_PREFIX=/usr\n\n - name: Build\n run: distrobox enter --name steamrt4 -- cmake --build build\n\n - name: install\n run: DESTDIR=\"$PWD\"/install distrobox enter --name steamrt4 -- cmake --build build -t install\n\n - name: Upload libraries\n uses: actions/upload-artifact@v6\n timeout-minutes: 1\n with:\n overwrite: true\n name: steamrt4_steampipe_depot\n path: ${{ github.workspace }}/install/*\n retention-days: 60\n compression-level: 9\n" }, { "path": ".github/workflows/test/action.yml", "content": "name: Run Test and Store Logs\ndescription: Run a test and store the log.\ninputs:\n target:\n description: 'The test target to run'\n required: true\n\nruns:\n using: composite\n steps:\n - name: Run Tests\n shell: bash\n run: cmake --build build --target ${{ inputs.target }}\n\n - name: Move and Truncate Results\n if: ${{ always() }}\n shell: bash\n run: |\n mkdir -p results\n mv build/Testing/Temporary/LastTest.log results/${{ inputs.target }}.log || true\n truncate --size=\"<20M\" results/${{ inputs.target }}.log || true\n" }, { "path": ".github/workflows/vixl_simulator.yml", "content": "name: Vixl Simulator run\n\non:\n push:\n branches:\n - main\n pull_request:\n branches:\n - main\n\nenv:\n # Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.)\n BUILD_TYPE: Release\n CC: clang\n CXX: clang++\n FEX_PORTABLE: 1\n\njobs:\n vixl_simulator:\n runs-on: ${{ matrix.arch }}\n strategy:\n matrix:\n # Only the x86-64 runner is fast enough to run this\n arch: [[self-hosted, x64], [self-hosted, ARMv8.4]]\n fail-fast: false\n\n steps:\n - uses: actions/checkout@v6\n with:\n fetch-depth: '0'\n fetch-tags: 'true'\n\n - name: Set runner info\n run: |\n echo \"runner_label=${{ matrix.arch[1] }}\" >> $GITHUB_ENV\n echo \"runner_name=$(hostname)\" >> $GITHUB_ENV\n\n - name: Setup Build Environment\n uses: ./.github/workflows/setup-env\n\n - name: Configure CMake\n run: |\n cmake -S . -B build -DCMAKE_BUILD_TYPE=$BUILD_TYPE -G Ninja -DENABLE_VIXL_SIMULATOR=True -DENABLE_LTO=False \\\n -DENABLE_VIXL_DISASSEMBLER=True -DENABLE_ASSERTIONS=True -DENABLE_X86_HOST_DEBUG=True\n\n # These steps make a lot of noise but rarely fail.\n # Put them in a separate step to make normal build logs easier to parse\n - name: Noisy Build Targets\n run: cmake --build build --target asm_files 32bit_asm_files JemallocLibs Catch2 vixl cephes_128bit\n\n - name: Build\n run: cmake --build build\n\n - name: ASM Tests - SVE256\n if: ${{ always() }}\n uses: ./.github/workflows/test\n with:\n target: asm_tests\n\n - name: ASM Tests - SVE128\n if: ${{ always() }}\n uses: ./.github/workflows/test\n env:\n FEX_FORCESVEWIDTH: \"128\"\n with:\n target: asm_tests\n\n - name: ASM Tests - ASIMD\n if: ${{ always() }}\n uses: ./.github/workflows/test\n env:\n FEX_HOSTFEATURES: \"disablesve\"\n with:\n target: asm_tests\n\n - name: Upload results\n if: ${{ always() }}\n uses: actions/upload-artifact@v6\n timeout-minutes: 1\n with:\n name: Results-${{ env.runner_name }}-${{ env.runner_label }}\n path: results/*.log\n retention-days: 3\n" }, { "path": ".github/workflows/wine_build/action.yml", "content": "name: Wine DLL Build\ndescription: Build a wow64 or arm64ec Wine DLL\n\ninputs:\n target:\n description: 'The target (arm64ec or wow64)'\n required: true\n\nruns:\n using: composite\n steps:\n - name: Clean Build Environment\n shell: bash\n run: rm -Rf build_${{ inputs.target }}\n\n - name: Configure CMake\n shell: bash\n run: |\n case \"${{ inputs.target }}\" in\n wow64) _cc=aarch64 ;;\n arm64ec) _cc=arm64ec ;;\n esac\n\n cmake -S . -B build_${{ inputs.target }} -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DCMAKE_TOOLCHAIN_FILE=Data/CMake/toolchain_mingw.cmake \\\n -DMINGW_TRIPLE=${_cc}-w64-mingw32 -DCMAKE_INSTALL_LIBDIR=/usr/lib/wine/aarch64-windows -G Ninja \\\n -DENABLE_LTO=False -DENABLE_ASSERTIONS=False -DENABLE_JEMALLOC_GLIBC_ALLOC=False \\\n -DBUILD_TESTING=False -DCMAKE_INSTALL_PREFIX=/usr -DTUNE_ARCH=generic -DTUNE_CPU=none\n\n - name: Build\n shell: bash\n run: cmake --build build_${{ inputs.target }}\n\n - name: Install\n shell: bash\n run: DESTDIR=\"$PWD\"/install cmake --build build_${{ inputs.target }} -t install\n" }, { "path": ".github/workflows/wine_dll_artifacts.yml", "content": "name: Wine DLL artifacts\n\non:\n push:\n branches:\n - main\n\nenv:\n BUILD_TYPE: Release\n\njobs:\n wine_dll_artifacts:\n runs-on: ${{ matrix.arch }}\n strategy:\n matrix:\n arch: [[self-hosted, ARM64, mingw]]\n fail-fast: false\n\n steps:\n - uses: actions/checkout@v6\n with:\n fetch-depth: '0'\n fetch-tags: 'true'\n\n - name: Add MingGW to PATH\n run: echo \"$HOME/llvm-mingw/build/bin/\" >> $GITHUB_PATH\n\n - name: Checkout Submodules\n # Need to update submodules\n run: |\n git submodule sync --recursive\n git submodule update --init --depth 1\n\n - name: Clean install directory\n run: rm -Rf install\n\n - name: Build (wow64)\n uses: ./.github/workflows/wine_build\n with:\n target: wow64\n\n - name: Build (arm64ec)\n uses: ./.github/workflows/wine_build\n with:\n target: arm64ec\n\n - name: Upload libraries\n uses: actions/upload-artifact@v6\n timeout-minutes: 1\n with:\n overwrite: true\n name: wine_dll_artifacts\n path: ${{ github.workspace }}/install/usr/lib/wine/aarch64-windows/lib*.dll\n retention-days: 60\n compression-level: 9\n" }, { "path": ".gitignore", "content": "# Existing\n\ncompile_commands.json\nvim_rc\nConfig.json\n\n[Bb]uild*\n[Bb]in/\nout/\n.vscode/\n.vs/\n*.pyc\n.cache\n.idea/\nCMakeLists.txt.user\n" }, { "path": ".gitlab-ci.yml", "content": "spec:\n inputs:\n PROMOTE_BRANCH:\n description: \"Branch to promote the build to. Empty means no promotion.\"\n default: \"bleeding-edge\"\n\n---\n\nworkflow:\n rules:\n - when: always\n variables:\n PROMOTE_BRANCH: $[[ inputs.PROMOTE_BRANCH ]]\n\nvariables:\n DEBIAN_FRONTEND: noninteractive\n GIT_SUBMODULE_STRATEGY: recursive\n GIT_DEPTH: 0\n CC: clang\n CXX: clang++\n\nbuild:\n stage: build\n image: registry.gitlab.steamos.cloud/steamrt/steamrt4/sdk/arm64:4.0.20251117.183306\n tags:\n - docker\n - linux\n - arm64\n - aarch64\n script:\n - apt-get -y update\n - apt-get install -y\n git cmake ninja-build ccache\n lld clang\n libclang-dev llvm-dev\n libstdc++-14-dev-i386-cross libgcc-14-dev-i386-cross\n libstdc++-14-dev-amd64-cross libgcc-14-dev-amd64-cross\n - cmake -E make_directory build/\n - cmake -DCMAKE_BUILD_TYPE=Release -G Ninja -DBUILD_STEAM_SUPPORT=True -DENABLE_LTO=True -DENABLE_ASSERTIONS=False -DBUILD_THUNKS=True -DBUILD_FEXCONFIG=False -DBUILD_TESTING=False -DENABLE_CLANG_THUNKS=True -DUSE_LINKER=lld -DCMAKE_INSTALL_PREFIX=/usr -DTUNE_ARCH=armv8.2-a -DTUNE_CPU=none . -B build/\n - cmake --build build/ --config Release\n - DESTDIR=$(pwd)/install/ cmake --build build/ --config Release -t install\n\n artifacts:\n name: \"steamrt artifacts\"\n untracked: false\n paths:\n - install/\n\npromote:\n stage: deploy\n variables:\n GIT_STRATEGY: none\n image: registry.gitlab.steamos.cloud/steamrt/steamrt4/sdk/arm64:4.0.20251117.183306\n tags:\n - docker\n - linux\n - arm64\n - aarch64\n rules:\n - if: '$PROMOTE_BRANCH'\n before_script:\n - apt-get -y update\n - apt-get install -y tmux curl\n script:\n # comment out to debug: SSH in via GCP, go down the container and attach to the session (with `tmux attach -t debug`)\n# - tmux new-session -d -s debug\n# - while tmux has-session -t debug 2>/dev/null; do sleep 1; done\n\n # ref controls which fex-depot code runs the pipeline, while VERSION_PARAM controls which fex branch's artifacts that pipeline downloads.\n - >\n curl --fail --location --request POST --form token=${FEX_DEPOT_TRIGGER_TOKEN} --form ref=master --form \"variables[PROMOTE_BRANCH]=${PROMOTE_BRANCH}\" --form \"variables[VERSION_PARAM]=${CI_COMMIT_REF_NAME}\" \"${CI_API_V4_URL}/projects/fex%2Ffex-depot/trigger/pipeline\"\n" }, { "path": ".gitmodules", "content": "[submodule \"External/vixl\"]\n\tshallow = true\n\tpath = External/vixl\n\turl = https://github.com/FEX-Emu/vixl.git\n[submodule \"External/cpp-optparse\"]\n\tpath = Source/Common/cpp-optparse\n\turl = https://github.com/Sonicadvance1/cpp-optparse\n[submodule \"External/fex-posixtest-bins\"]\n shallow = true\n\tpath = External/fex-posixtest-bins\n\turl = https://github.com/FEX-Emu/fex-posixtest-bins.git\n[submodule \"External/fex-gvisor-tests-bins\"]\n shallow = true\n\tpath = External/fex-gvisor-tests-bins\n\turl = https://github.com/FEX-Emu/fex-gvisor-tests-bins.git\n[submodule \"External/fex-gcc-target-tests-bins\"]\n shallow = true\n\tpath = External/fex-gcc-target-tests-bins\n\turl = https://github.com/FEX-Emu/fex-gcc-target-tests-bins.git\n[submodule \"External/fmt\"]\n\tpath = External/fmt\n\turl = https://github.com/fmtlib/fmt.git\n[submodule \"External/drm-headers\"]\n\tpath = External/drm-headers\n\turl = https://github.com/FEX-Emu/drm-headers.git\n[submodule \"External/xxhash\"]\n\tpath = External/xxhash\n\turl = https://github.com/Cyan4973/xxHash.git\n[submodule \"External/Catch2\"]\n\tpath = External/Catch2\n\turl = https://github.com/catchorg/Catch2.git\n[submodule \"External/Vulkan-Headers\"]\n\tshallow = true\n\tpath = External/Vulkan-Headers\n\turl = https://github.com/KhronosGroup/Vulkan-Headers.git\n[submodule \"External/jemalloc_glibc\"]\n\tpath = External/jemalloc_glibc\n\turl = https://github.com/FEX-Emu/jemalloc.git\n[submodule \"External/tracy\"]\n\tpath = External/tracy\n\turl = https://github.com/wolfpld/tracy\n[submodule \"External/range-v3\"]\n\tpath = External/range-v3\n\turl = https://github.com/ericniebler/range-v3.git\n[submodule \"External/zydis\"]\n\tshallow = true\n\tpath = External/zydis\n\turl = https://github.com/zyantific/zydis.git\n[submodule \"External/unordered_dense\"]\n\tpath = External/unordered_dense\n\turl = https://github.com/martinus/unordered_dense.git\n[submodule \"External/rpmalloc\"]\n\tpath = External/rpmalloc\n\turl = https://github.com/FEX-Emu/rpmalloc.git\n" }, { "path": "CMakeLists.txt", "content": "cmake_minimum_required(VERSION 3.14)\nproject(FEX C CXX ASM)\n\ninclude(CheckIncludeFiles)\ncheck_include_files(\"gdb/jit-reader.h\" HAVE_GDB_JIT_READER_H)\n\noption(BUILD_FEX_LINUX_TESTS \"Build FEXLinuxTests (requires x86 compiler)\" FALSE)\noption(BUILD_THUNKS \"Build thunks\" FALSE)\noption(BUILD_FEXCONFIG \"Build FEXConfig\" TRUE)\noption(ENABLE_CLANG_THUNKS \"Build thunks with clang\" TRUE)\noption(ENABLE_IWYU \"Enable the Include What You Use sanitizer\" FALSE)\noption(ENABLE_LTO \"Enable LTO with compilation\" TRUE)\noption(ENABLE_XRAY \"Enable building with LLVM X-Ray\" FALSE)\nset(USE_LINKER \"\" CACHE STRING \"Path to a custom linker program\")\noption(ENABLE_UBSAN \"Enable the Clang Undefined Behavior Sanitizer\" FALSE)\noption(ENABLE_ASAN \"Enable the Clang Address Sanitizer\" FALSE)\noption(ENABLE_TSAN \"Enable the Clang Thread Sanitizer\" FALSE)\noption(ENABLE_COVERAGE \"Enable Code Coverage\" FALSE)\noption(ENABLE_ASSERTIONS \"Enable debug assertions\" FALSE)\noption(ENABLE_GDB_SYMBOLS \"Enable GDBSymbols integration support\" ${HAVE_GDB_JIT_READER_H})\noption(ENABLE_STRICT_WERROR \"Enable stricter -Werror\" FALSE)\noption(ENABLE_WERROR \"Enable -Werror\" FALSE)\noption(ENABLE_FEX_ALLOCATOR \"Enable allocator for FEX\" TRUE)\noption(ENABLE_JEMALLOC_GLIBC_ALLOC \"Enable jemalloc glibc allocator\" TRUE)\noption(ENABLE_OFFLINE_TELEMETRY \"Enable FEX offline telemetry\" TRUE)\noption(ENABLE_COMPILE_TIME_TRACE \"Enable time trace compile option\" FALSE)\noption(ENABLE_LIBCXX \"Use LLVM's libc++ instead of the GNU libstdc++\" FALSE)\noption(ENABLE_CCACHE \"Enable ccache for build caching\" TRUE)\noption(ENABLE_VIXL_SIMULATOR \"Use the VIXL simulator for emulation (only useful for CI testing)\" FALSE)\noption(ENABLE_VIXL_DISASSEMBLER \"Enable debug disassembler output with VIXL\" FALSE)\noption(ENABLE_ZYDIS \"Enable x86/x86-64 guest disassembler output with Zydis\" FALSE)\noption(USE_LEGACY_BINFMTMISC \"Use legacy method of setting up binfmt_misc\" FALSE)\noption(ENABLE_FEXCORE_PROFILER \"Enable FEXCore's timeline profiling capabilities\" FALSE)\nset(FEXCORE_PROFILER_BACKEND \"gpuvis\" CACHE STRING \"Set which backend to use for FEXCore's profiler\")\nset_property(CACHE FEXCORE_PROFILER_BACKEND PROPERTY STRINGS gpuvis tracy)\noption(ENABLE_GLIBC_ALLOCATOR_HOOK_FAULT \"Enables glibc memory allocation hooking with fault for CI testing\")\noption(USE_PDB_DEBUGINFO \"Build debug info in PDB format\" FALSE)\noption(BUILD_STEAM_SUPPORT \"Enable Steam integration\" FALSE)\n\nset(X86_32_TOOLCHAIN_FILE \"${CMAKE_CURRENT_SOURCE_DIR}/Data/CMake/toolchain_x86_32.cmake\" CACHE FILEPATH \"Toolchain file for the (cross-)compiler targeting i686\")\nset(X86_64_TOOLCHAIN_FILE \"${CMAKE_CURRENT_SOURCE_DIR}/Data/CMake/toolchain_x86_64.cmake\" CACHE FILEPATH \"Toolchain file for the (cross-)compiler targeting x86_64\")\nset(X86_DEV_ROOTFS \"/\" CACHE FILEPATH \"Path to the sysroot used for cross-compiling for i686 and x86_64\")\nset(DATA_DIRECTORY \"\" CACHE PATH \"Global data directory (override)\")\nset(HOSTLIBS_DATA_DIRECTORY \"\" CACHE PATH \"Global data directory (override)\")\n\nif (NOT DATA_DIRECTORY)\n set(DATA_DIRECTORY \"${CMAKE_INSTALL_PREFIX}/share/fex-emu\")\nendif()\n\ninclude(GNUInstallDirs)\nif (NOT HOSTLIBS_DATA_DIRECTORY)\n set(HOSTLIBS_DATA_DIRECTORY \"${CMAKE_INSTALL_FULL_LIBDIR}/fex-emu\")\nendif()\n\n## Platform Checks ##\n# Only 64-bit Linux and Windows are supported\n\n# NB: SIZEOF_VOID_P is in bytes, not bits\n# On 32-bit systems this is set to 4\nif (NOT CMAKE_SIZEOF_VOID_P EQUAL 8)\n message(FATAL_ERROR \"Unsupported pointer size ${CMAKE_SIZEOF_VOID_P}.\"\n \" FEX only supports 64-bit (8-byte pointer) systems.\"\n \" If you believe this is in error, file an issue.\")\nelseif (NOT (WIN32 OR CMAKE_SYSTEM_NAME STREQUAL \"Linux\"))\n message(FATAL_ERROR \"Unsupported system type ${CMAKE_SYSTEM_NAME}.\"\n \" FEX only supports Linux and Windows.\"\n \" If you believe this is in error, file an issue.\")\nendif()\n\n## Compiler Checks ##\n# GCC and MSVC are unsupported\nif (CMAKE_CXX_COMPILER_ID STREQUAL \"GNU\")\n message(FATAL_ERROR \"FEX doesn't support GCC! Use Clang instead.\")\nelseif (MSVC)\n message(FATAL_ERROR \"FEX doesn't support MSVC! Use Clang on MinGW instead.\")\nelseif (MINGW)\n message(STATUS \"Building for MinGW\")\n set(ENABLE_FEX_ALLOCATOR TRUE)\n set(ENABLE_JEMALLOC_GLIBC_ALLOC FALSE)\nelse ()\n message(STATUS \"Clang version ${CMAKE_CXX_COMPILER_VERSION}\")\n set(CLANG_MINIMUM_VERSION 13.0)\n if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS ${CLANG_MINIMUM_VERSION})\n message(FATAL_ERROR \"Clang version too old for FEX. Need at least ${CLANG_MINIMUM_VERSION} but has ${CMAKE_CXX_COMPILER_VERSION}\")\n endif()\nendif()\n\n## Architecture Handling ##\nstring(TOLOWER ${CMAKE_SYSTEM_PROCESSOR} processor)\nif (processor MATCHES \"x86|amd64\")\n option(ENABLE_X86_HOST_DEBUG \"Enables compiling on x86_64 host\" FALSE)\n if (NOT ENABLE_X86_HOST_DEBUG)\n message(FATAL_ERROR\n \" FEX doesn't support compiling for x86-64 hosts!\"\n \" This is /only/ a supported configuration for FEX CI and nothing else!\")\n else()\n message(STATUS \"x86_64 debug build\")\n endif()\n\n set(ARCHITECTURE_x86_64 1)\n add_compile_definitions(ARCHITECTURE_x86_64=1)\n set(CMAKE_CXX_FLAGS \"${CMAKE_CXX_FLAGS} -mcx16\")\nelseif (processor MATCHES \"^aarch64|^arm64|^armv8\\.*\")\n set(ARCHITECTURE_arm64 1)\n add_compile_definitions(ARCHITECTURE_arm64=1)\n\n # arm64ec needs to define both arm64 and arm64ec\n if (processor MATCHES \"^arm64ec\")\n set(ARCHITECTURE_arm64ec 1)\n add_compile_definitions(ARCHITECTURE_arm64ec=1)\n endif()\nendif()\n\nif (NOT (ARCHITECTURE_arm64 OR ARCHITECTURE_arm64ec OR ARCHITECTURE_x86_64))\n message(FATAL_ERROR \"Unsupported processor type ${processor}.\"\n \" If you believe this is in error, file an issue.\")\nendif()\n\nif (BUILD_STEAM_SUPPORT)\n add_compile_definitions(FEX_STEAM_SUPPORT=1)\nendif()\n\nif (ENABLE_FEXCORE_PROFILER)\n add_compile_definitions(ENABLE_FEXCORE_PROFILER=1)\n string(TOUPPER \"${FEXCORE_PROFILER_BACKEND}\" FEXCORE_PROFILER_BACKEND)\n\n if (FEXCORE_PROFILER_BACKEND STREQUAL \"GPUVIS\")\n add_compile_definitions(FEXCORE_PROFILER_BACKEND=1)\n elseif (FEXCORE_PROFILER_BACKEND STREQUAL \"TRACY\")\n add_compile_definitions(FEXCORE_PROFILER_BACKEND=2)\n add_compile_definitions(TRACY_ENABLE=1)\n # Required so that Tracy will only start in the selected guest application\n add_compile_definitions(TRACY_MANUAL_LIFETIME=1)\n add_compile_definitions(TRACY_DELAYED_INIT=1)\n # This interferes with FEX's signal handling\n add_compile_definitions(TRACY_NO_CRASH_HANDLER=1)\n # Tracy can gather call stack samples in regular intervals, but this\n # isn't useful for us since it would usually sample opaque JIT code\n add_compile_definitions(TRACY_NO_SAMPLING=1)\n # This pulls in libbacktrace which allocators in global constructors (before FEX can set up its allocator hooks)\n add_compile_definitions(TRACY_NO_CALLSTACK=1)\n if (MINGW)\n message(FATAL_ERROR \"Tracy profiler not supported on MinGW\")\n endif()\n else()\n message(FATAL_ERROR \"Unknown FEXCore profiler backend ${FEXCORE_PROFILER_BACKEND}\")\n endif()\nendif()\n\nif (ENABLE_JEMALLOC_GLIBC_ALLOC AND ENABLE_GLIBC_ALLOCATOR_HOOK_FAULT)\n message(FATAL_ERROR \"Can't have both glibc fault allocator and jemalloc glibc allocator enabled at the same time\")\nendif()\n\nif (ENABLE_GLIBC_ALLOCATOR_HOOK_FAULT)\n add_compile_definitions(GLIBC_ALLOCATOR_FAULT=1)\nendif()\n\n# uninstall target\nif(NOT TARGET uninstall)\n configure_file(\n \"${CMAKE_CURRENT_SOURCE_DIR}/Data/CMake/cmake_uninstall.cmake.in\"\n \"${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/cmake_uninstall.cmake\"\n IMMEDIATE @ONLY)\n\n add_custom_target(uninstall\n COMMAND ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/cmake_uninstall.cmake)\nendif()\n\n# These options are meant for package management\nset(TUNE_CPU \"native\" CACHE STRING \"Override the CPU the build is tuned for\")\nset(TUNE_ARCH \"generic\" CACHE STRING \"Override the Arch the build is tuned for\")\nset(OVERRIDE_VERSION \"detect\" CACHE STRING \"Override the FEX version\")\nset(OVERRIDE_HASH \"detect\" CACHE STRING \"Override the FEX git hash\")\n\nget_property(IS_MULTI_CONFIG GLOBAL PROPERTY GENERATOR_IS_MULTI_CONFIG)\nif (NOT IS_MULTI_CONFIG AND NOT CMAKE_BUILD_TYPE)\n set(CMAKE_BUILD_TYPE Release\n CACHE STRING \"Choose the type of build.\" FORCE)\n message(STATUS \"No build type set, defaulting to a Release build\")\nendif()\n\nstring(TOUPPER \"${CMAKE_BUILD_TYPE}\" CMAKE_BUILD_TYPE)\nif (CMAKE_BUILD_TYPE MATCHES \"DEBUG\")\n set(ENABLE_ASSERTIONS TRUE)\nendif()\n\nif (ENABLE_ASSERTIONS)\n message(STATUS \"Assertions enabled\")\n add_compile_definitions(ASSERTIONS_ENABLED=1)\nendif()\n\nif (ENABLE_GDB_SYMBOLS)\n message(STATUS \"GDBSymbols support enabled\")\n add_compile_definitions(GDB_SYMBOLS_ENABLED=1)\nendif()\n\nset(CMAKE_CXX_STANDARD 20)\nset(CMAKE_EXPORT_COMPILE_COMMANDS ON)\nset(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/Bin)\nset(CMAKE_INCLUDE_CURRENT_DIR ON)\nset(CMAKE_POSITION_INDEPENDENT_CODE ON)\ncmake_policy(SET CMP0083 NEW) # Follow new PIE policy\ninclude(CheckPIESupported)\ncheck_pie_supported()\n\nset(CMAKE_INTERPROCEDURAL_OPTIMIZATION ${ENABLE_LTO})\n\ninclude(CheckCXXSourceCompiles)\nset(CMAKE_REQUIRED_FLAGS \"-std=c++11 -Wattributes -Werror=attributes\")\ncheck_cxx_source_compiles(\n \"\n __attribute__((preserve_all))\n int Testy(int a, int b, int c, int d, int e, int f) {\n return a + b + c + d + e + f;\n }\n int main() {\n return Testy(0, 1, 2, 3, 4, 5);\n }\"\n HAS_CLANG_PRESERVE_ALL)\nunset(CMAKE_REQUIRED_FLAGS)\nif (HAS_CLANG_PRESERVE_ALL)\n if (MINGW)\n message(STATUS \"Ignoring broken clang::preserve_all support\")\n set(HAS_CLANG_PRESERVE_ALL FALSE)\n else()\n message(STATUS \"Has clang::preserve_all\")\n endif()\nendif()\n\nif (ARCHITECTURE_arm64 AND HAS_CLANG_PRESERVE_ALL)\n add_compile_definitions(\"FEX_PRESERVE_ALL_ATTR=__attribute__((preserve_all))\" \"FEX_HAS_PRESERVE_ALL_ATTR=1\")\nelse()\n add_compile_definitions(\"FEX_PRESERVE_ALL_ATTR=\" \"FEX_HAS_PRESERVE_ALL_ATTR=0\")\nendif()\n\ncheck_cxx_source_compiles(\n \"\n #define _GNU_SOURCE\n #include \n int main() {\n return program_invocation_name == nullptr;\n }\"\n HAS_PROGRAM_INVOCATION_NAME)\nadd_compile_definitions(\"HAS_PROGRAM_INVOCATION_NAME=${HAS_PROGRAM_INVOCATION_NAME}\")\n\nif (ENABLE_VIXL_SIMULATOR)\n # We can run the simulator on both x86-64 or AArch64 hosts\n add_compile_definitions(VIXL_SIMULATOR=1 VIXL_INCLUDE_SIMULATOR_AARCH64=1)\nendif()\n\nif (ENABLE_CCACHE)\n find_program(CCACHE_PROGRAM ccache)\n if(CCACHE_PROGRAM)\n message(STATUS \"CCache enabled\")\n set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE \"${CCACHE_PROGRAM}\")\n endif()\nendif()\n\nif (ENABLE_XRAY)\n add_compile_options(-fxray-instrument)\n link_libraries(-fxray-instrument)\nendif()\n\nif (ENABLE_COMPILE_TIME_TRACE)\n add_compile_options(-ftime-trace)\n link_libraries(-ftime-trace)\nendif()\n\nset(PTHREAD_LIB pthread)\n\nif (USE_LINKER)\n message(STATUS \"Overriding linker to: ${USE_LINKER}\")\n add_link_options(\"-fuse-ld=${USE_LINKER}\")\nendif()\n\nif (ENABLE_LIBCXX)\n message(WARNING \"This is an unsupported configuration and should only be used for testing\")\n set(CMAKE_CXX_FLAGS \"${CMAKE_CXX_FLAGS} -std=c++11 -stdlib=libc++\")\n set(CMAKE_EXE_LINKER_FLAGS \"${CMAKE_EXE_LINKER_FLAGS} -stdlib=libc++ -lc++abi\")\nendif()\n\nif (NOT ENABLE_OFFLINE_TELEMETRY)\n # Disable FEX offline telemetry entirely if asked\n add_compile_definitions(FEX_DISABLE_TELEMETRY=1)\nendif()\n\nif (ENABLE_UBSAN)\n # See https://github.com/FEX-Emu/FEX/pull/4494#issuecomment-2800608944\n # and related discussion for the use of -fno-sanitize=alignment -fno-sanitize=function\n # with UBSAN.\n # alignment: we don't follow a strict alignment policy, for example IR uses packed structs\n # that are regularly access unaligned.\n # function: syscalls cast function pointers to void (*)(unsigned long...), causing warnings\n # related to this access.\n add_compile_definitions(ENABLE_UBSAN=1)\n add_compile_options(-fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize=alignment -fno-sanitize=function -fno-sanitize-recover=undefined)\n link_libraries(-fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize=alignment -fno-sanitize=function -fno-sanitize-recover=undefined)\nendif()\n\nif (ENABLE_ASAN)\n add_compile_definitions(ENABLE_ASAN=1)\n add_compile_options(-fno-omit-frame-pointer -fsanitize=address -fsanitize-address-use-after-scope)\n link_libraries(-fno-omit-frame-pointer -fsanitize=address -fsanitize-address-use-after-scope)\nendif()\n\nif (ENABLE_TSAN)\n add_compile_options(-fno-omit-frame-pointer -fsanitize=thread)\n link_libraries(-fno-omit-frame-pointer -fsanitize=thread)\nendif()\n\nif (ENABLE_COVERAGE)\n add_compile_options(-fprofile-instr-generate -fcoverage-mapping)\n link_libraries(-fprofile-instr-generate -fcoverage-mapping)\nendif()\n\nif (ENABLE_JEMALLOC_GLIBC_ALLOC)\n # The glibc jemalloc subproject which hooks the glibc allocator.\n # Required for thunks to work.\n # All host native libraries will use this allocator, while *most* other FEX internal allocations will use the other jemalloc allocator.\n add_subdirectory(External/jemalloc_glibc/)\nelseif (NOT MINGW)\n message(STATUS\n \" jemalloc glibc allocator disabled!\\n\"\n \" This is not a recommended configuration!\\n\"\n \" This will very explicitly break thunk execution!\\n\"\n \" Use at your own risk!\")\nendif()\n\nif (ENABLE_FEX_ALLOCATOR)\n # The rpmalloc subproject that all FEXCore fextl objects allocate through.\n add_subdirectory(External/rpmalloc/)\nelseif (NOT MINGW)\n message (STATUS\n \" FEX allocator is disabled!\\n\"\n \" This is not a recommended configuration!\\n\"\n \" This will very explicitly break 32-bit application execution!\\n\"\n \" Use at your own risk!\")\nendif()\n\nif (USE_PDB_DEBUGINFO)\n add_compile_options(-g -gcodeview)\n add_link_options(-g -Wl,--pdb=)\nendif()\n\nset(CMAKE_CXX_FLAGS_RELWITHDEBINFO \"${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -fno-omit-frame-pointer\")\nset(CMAKE_LINKER_FLAGS_RELWITHDEBINFO \"${CMAKE_LINKER_FLAGS_RELWITHDEBINFO} -fno-omit-frame-pointer\")\n\nset(CMAKE_CXX_FLAGS_RELEASE \"${CMAKE_CXX_FLAGS_RELEASE} -fomit-frame-pointer\")\nset(CMAKE_LINKER_FLAGS_RELEASE \"${CMAKE_LINKER_FLAGS_RELEASE} -fomit-frame-pointer\")\n\n## Modules ##\nlist(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/Data/CMake/)\n\ninclude(LinkerGC)\n\n## Externals ##\n\nfind_package(unordered_dense QUIET CONFIG)\nif (NOT unordered_dense_FOUND)\n add_subdirectory(External/unordered_dense)\nendif()\n\ninclude(CTest)\nif (BUILD_TESTING OR ENABLE_VIXL_DISASSEMBLER OR ENABLE_VIXL_SIMULATOR)\n add_subdirectory(External/vixl/)\nendif()\n\nif (ENABLE_ZYDIS)\n find_package(Zycore 1.5 MODULE QUIET)\n find_package(Zydis 4.0 MODULE QUIET)\n\n if (TARGET Zydis::Zydis AND TARGET Zycore::Zycore)\n message(STATUS \"Using system Zydis\")\n else()\n set(ZYDIS_BUILD_TOOLS OFF CACHE BOOL \"\" FORCE)\n set(ZYDIS_BUILD_EXAMPLES OFF CACHE BOOL \"\" FORCE)\n\n message(STATUS \"Using bundled Zydis\")\n add_subdirectory(External/zydis/)\n endif()\nendif()\n\nif (ENABLE_FEXCORE_PROFILER AND FEXCORE_PROFILER_BACKEND STREQUAL \"TRACY\")\n add_subdirectory(External/tracy)\nendif()\n\nfind_package(Python 3.9 REQUIRED COMPONENTS Interpreter)\n\nset(BUILD_SHARED_LIBS OFF)\n\nif (NOT CMAKE_CROSSCOMPILING)\n find_package(xxhash MODULE QUIET)\nendif()\n\nif (NOT TARGET xxHash::xxhash)\n set(XXHASH_BUNDLED_MODE TRUE)\n set(XXHASH_BUILD_XXHSUM FALSE)\n add_subdirectory(External/xxhash/cmake_unofficial/)\nendif()\n\nadd_compile_options(-Wno-trigraphs)\nadd_compile_definitions(GLOBAL_DATA_DIRECTORY=\"${DATA_DIRECTORY}/\")\n\nif (BUILD_TESTING)\n find_package(Catch2 3 QUIET)\n if (NOT Catch2_FOUND)\n add_subdirectory(External/Catch2/)\n\n # Pull in catch_discover_tests definition\n list(APPEND CMAKE_MODULE_PATH \"${CMAKE_CURRENT_SOURCE_DIR}/External/Catch2/contrib/\")\n endif()\n\n include(Catch)\nelse ()\n # Override any previously generated test list to avoid running stale test binaries\n file(GENERATE OUTPUT CTestTestfile.cmake CONTENT \"# No tests since BUILD_TESTING is disabled\")\nendif()\n\nfind_package(fmt QUIET)\nif (NOT fmt_FOUND)\n # Disable fmt install\n set(FMT_INSTALL OFF)\n add_subdirectory(External/fmt/)\nendif()\n\nfind_package(range-v3 QUIET)\nif (NOT range-v3_FOUND)\n add_subdirectory(External/range-v3/)\n target_compile_definitions(range-v3 INTERFACE RANGES_DISABLE_DEPRECATED_WARNINGS)\nendif()\n\nadd_subdirectory(External/tiny-json/)\n\ninclude_directories(Source/)\ninclude_directories(\"${CMAKE_BINARY_DIR}/Source/\")\n\ninclude(CheckCXXCompilerFlag)\n\n# Add in diagnostic colours if the option is available.\n# Ninja code generator will kill colours if this isn't here\ncheck_cxx_compiler_flag(-fdiagnostics-color=always GCC_COLOR)\ncheck_cxx_compiler_flag(-fcolor-diagnostics CLANG_COLOR)\ncheck_cxx_compiler_flag(-Wno-deprecated-enum-enum-conversion ENUM_ENUM_WARNING)\n\nif (GCC_COLOR)\n add_compile_options(-fdiagnostics-color=always)\nendif()\nif (CLANG_COLOR)\n add_compile_options(-fcolor-diagnostics)\nendif()\n\nif(ENUM_ENUM_WARNING)\n add_compile_options(-Wno-deprecated-enum-enum-conversion)\nendif()\n\n# GCC enables -Wchanges-meaning by default and treats some cases as an error\nif(CMAKE_CXX_COMPILER_ID STREQUAL \"GNU\")\n add_compile_options(-Wno-error=changes-meaning)\nendif()\n\nif(ENABLE_WERROR OR ENABLE_STRICT_WERROR)\n add_compile_options(-Werror)\n if (NOT ENABLE_STRICT_WERROR)\n # Disable some Werror that can add frustration when developing\n add_compile_options(-Wno-error=unused-variable)\n endif()\nendif()\n\nset(FEX_TUNE_COMPILE_FLAGS)\nif (NOT TUNE_ARCH STREQUAL \"generic\")\n check_cxx_compiler_flag(\"-march=${TUNE_ARCH}\" COMPILER_SUPPORTS_ARCH_TYPE)\n if(COMPILER_SUPPORTS_ARCH_TYPE)\n list(APPEND FEX_TUNE_COMPILE_FLAGS \"-march=${TUNE_ARCH}\")\n else()\n message(FATAL_ERROR \"Trying to compile arch type '${TUNE_ARCH}' but the compiler doesn't support this\")\n endif()\nendif()\n\nif (TUNE_CPU STREQUAL \"native\")\n if(ARCHITECTURE_arm64)\n if (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 999999.0)\n # Clang 12.0 fixed the -mcpu=native bug with mixed big.little implementers\n # Clang can not currently check for native Apple M1 type in hypervisor. Currently disabled\n check_cxx_compiler_flag(\"-mcpu=native\" COMPILER_SUPPORTS_CPU_TYPE)\n if(COMPILER_SUPPORTS_CPU_TYPE)\n list(APPEND FEX_TUNE_COMPILE_FLAGS \"-mcpu=native\")\n endif()\n else()\n execute_process(COMMAND python3 \"${PROJECT_SOURCE_DIR}/Scripts/aarch64_fit_native.py\" \"/proc/cpuinfo\" \"${CMAKE_CXX_COMPILER_VERSION}\"\n OUTPUT_VARIABLE AARCH64_CPU)\n\n string(STRIP ${AARCH64_CPU} AARCH64_CPU)\n\n execute_process(COMMAND python3 \"${PROJECT_SOURCE_DIR}/Scripts/NeedDisabledSVE.py\"\n RESULT_VARIABLE NEEDS_SVE_DISABLED)\n if (NEEDS_SVE_DISABLED)\n message(STATUS \"Platform has bugged SVE. Disabling\")\n set(AARCH64_CPU \"cortex-a78\")\n endif()\n\n check_cxx_compiler_flag(\"-mcpu=${AARCH64_CPU}\" COMPILER_SUPPORTS_CPU_TYPE)\n if(COMPILER_SUPPORTS_CPU_TYPE)\n list(APPEND FEX_TUNE_COMPILE_FLAGS \"-mcpu=${AARCH64_CPU}\")\n endif()\n endif()\n else()\n check_cxx_compiler_flag(\"-march=native\" COMPILER_SUPPORTS_MARCH_NATIVE)\n if(COMPILER_SUPPORTS_MARCH_NATIVE)\n list(APPEND FEX_TUNE_COMPILE_FLAGS \"-march=native\")\n endif()\n endif()\nelseif (NOT TUNE_CPU STREQUAL \"none\")\n check_cxx_compiler_flag(\"-mcpu=${TUNE_CPU}\" COMPILER_SUPPORTS_CPU_TYPE)\n if(COMPILER_SUPPORTS_CPU_TYPE)\n list(APPEND FEX_TUNE_COMPILE_FLAGS \"-mcpu=${TUNE_CPU}\")\n else()\n message(FATAL_ERROR \"Trying to compile cpu type '${TUNE_CPU}' but the compiler doesn't support this\")\n endif()\nendif()\n\nset(GIT_DESCRIBE_STRING \"FEX-Unknown\")\n\nif (OVERRIDE_VERSION STREQUAL \"detect\")\n find_package(Git)\n\n if (GIT_FOUND)\n execute_process(\n COMMAND ${GIT_EXECUTABLE} describe --abbrev=7\n WORKING_DIRECTORY \"${CMAKE_SOURCE_DIR}\"\n OUTPUT_VARIABLE GIT_DESCRIBE_STRING\n ERROR_QUIET\n OUTPUT_STRIP_TRAILING_WHITESPACE)\n endif()\nelse()\n set(GIT_DESCRIBE_STRING \"${OVERRIDE_VERSION}\")\nendif()\n\nset(GIT_HASH \"Unknown\")\n\nif (OVERRIDE_HASH STREQUAL \"detect\")\n find_package(Git)\n\n if (GIT_FOUND)\n execute_process(\n COMMAND ${GIT_EXECUTABLE} rev-parse HEAD\n WORKING_DIRECTORY \"${CMAKE_SOURCE_DIR}\"\n OUTPUT_VARIABLE GIT_HASH\n ERROR_QUIET\n OUTPUT_STRIP_TRAILING_WHITESPACE)\n endif()\nelse()\n set(GIT_HASH \"${OVERRIDE_HASH}\")\nendif()\n\nmessage(STATUS \"FEX version: ${GIT_DESCRIBE_STRING}\")\nmessage(STATUS \"FEX commit: ${GIT_HASH}\")\n\n# Prepends 0x to every two-character sequence in the hash,\n# OR the final character of the hash, to plumb it for C++ usage. e.g.:\n# -DOVERRIDE_HASH=123456aa => 0x12, 0x34, 0x56, 0xaa,\n# -DOVERRIDE_HASH=12345678a => 0x12, 0x34, 0x56, 0x78, 0xa,\nstring(REGEX\n REPLACE \"(..|.$)\" \"0x\\\\1, \"\n GIT_HASH_ARRAY \"${GIT_HASH}\")\n\nif (ENABLE_IWYU)\n find_program(IWYU_EXE\n NAMES iwyu include-what-you-use)\n if (IWYU_EXE)\n message(STATUS \"IWYU enabled\")\n set(CMAKE_CXX_INCLUDE_WHAT_YOU_USE \"${IWYU_EXE}\")\n endif()\nendif()\n\nadd_compile_options(-Wall)\n\nif (BUILD_TESTING)\n message(STATUS \"Unit tests are enabled\")\n\n set(TEST_JOB_COUNT \"\" CACHE STRING \"Override number of parallel jobs to use while running tests\")\n if (TEST_JOB_COUNT)\n message(STATUS \"Running tests with ${TEST_JOB_COUNT} jobs\")\n elseif(CMAKE_VERSION VERSION_LESS \"3.29\")\n execute_process(COMMAND \"nproc\" OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE TEST_JOB_COUNT)\n endif()\n set(TEST_JOB_FLAG \"-j${TEST_JOB_COUNT}\")\nendif()\n\nadd_subdirectory(External/SoftFloat-3e/)\nadd_subdirectory(External/cephes/)\nadd_subdirectory(FEXHeaderUtils/)\nadd_subdirectory(CodeEmitter/)\nadd_subdirectory(FEXCore/)\n\nif (ARCHITECTURE_arm64 AND NOT MINGW AND NOT BUILD_STEAM_SUPPORT)\n # Binfmt_misc files must be installed prior to Source/ installs\n add_subdirectory(Data/binfmts/)\nendif()\n\nadd_subdirectory(Source/)\n\nif (NOT BUILD_STEAM_SUPPORT)\n add_subdirectory(Data/AppConfig/)\nendif()\n\n# Install the ThunksDB file\nfile(GLOB CONFIG_SOURCES CONFIGURE_DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/Data/*.json)\n\n# Any application configuration json file gets installed\nforeach(CONFIG_SRC ${CONFIG_SOURCES})\n install(FILES ${CONFIG_SRC}\n DESTINATION ${DATA_DIRECTORY}/\n COMPONENT Runtime)\nendforeach()\n\nif (BUILD_TESTING)\n add_subdirectory(unittests/)\nendif()\n\nif (BUILD_THUNKS)\n set(FEX_PROJECT_SOURCE_DIR ${PROJECT_SOURCE_DIR})\n add_subdirectory(ThunkLibs/Generator)\n\n # Thunk targets for both host libraries and IDE integration\n add_subdirectory(ThunkLibs/HostLibs)\n\n # Thunk targets for IDE integration of guest code, only\n add_subdirectory(ThunkLibs/GuestLibs)\n\n # Thunk targets for guest libraries\n include(ExternalProject)\n ExternalProject_Add(guest-libs\n PREFIX guest-libs\n SOURCE_DIR \"${CMAKE_CURRENT_SOURCE_DIR}/ThunkLibs/GuestLibs\"\n BINARY_DIR \"Guest\"\n CMAKE_ARGS\n \"-DBITNESS=64\"\n \"-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}\"\n \"-DBUILD_FEX_LINUX_TESTS=${BUILD_FEX_LINUX_TESTS}\"\n \"-DENABLE_CLANG_THUNKS=${ENABLE_CLANG_THUNKS}\"\n \"-DCMAKE_TOOLCHAIN_FILE:FILEPATH=${X86_64_TOOLCHAIN_FILE}\"\n \"-DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}\"\n \"-DFEX_PROJECT_SOURCE_DIR=${FEX_PROJECT_SOURCE_DIR}\"\n \"-DGENERATOR_EXE=$\"\n \"-DX86_DEV_ROOTFS=${X86_DEV_ROOTFS}\"\n INSTALL_COMMAND \"\"\n BUILD_ALWAYS ON\n DEPENDS thunkgen)\n\n ExternalProject_Add(guest-libs-32\n PREFIX guest-libs-32\n SOURCE_DIR \"${CMAKE_CURRENT_SOURCE_DIR}/ThunkLibs/GuestLibs\"\n BINARY_DIR \"Guest_32\"\n CMAKE_ARGS\n \"-DBITNESS=32\"\n \"-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}\"\n \"-DBUILD_FEX_LINUX_TESTS=${BUILD_FEX_LINUX_TESTS}\"\n \"-DENABLE_CLANG_THUNKS=${ENABLE_CLANG_THUNKS}\"\n \"-DCMAKE_TOOLCHAIN_FILE:FILEPATH=${X86_32_TOOLCHAIN_FILE}\"\n \"-DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}\"\n \"-DFEX_PROJECT_SOURCE_DIR=${FEX_PROJECT_SOURCE_DIR}\"\n \"-DGENERATOR_EXE=$\"\n \"-DX86_DEV_ROOTFS=${X86_DEV_ROOTFS}\"\n INSTALL_COMMAND \"\"\n BUILD_ALWAYS ON\n DEPENDS thunkgen)\n\n install(\n CODE \"message(\\\"-- Installing: guest-libs\\\")\"\n CODE \"\n execute_process(COMMAND ${CMAKE_COMMAND} --build . --target install\n WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/Guest)\"\n DEPENDS guest-libs\n COMPONENT Runtime)\n\n install(\n CODE \"message(\\\"-- Installing: guest-libs-32\\\")\"\n CODE \"\n execute_process(COMMAND ${CMAKE_COMMAND} --build . --target install\n WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/Guest_32)\"\n DEPENDS guest-libs-32\n COMPONENT Runtime)\n\n add_custom_target(uninstall_guest-libs\n COMMAND ${CMAKE_COMMAND} \"--build\" \".\" \"--target\" \"uninstall\"\n WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/Guest)\n\n add_custom_target(uninstall_guest-libs-32\n COMMAND ${CMAKE_COMMAND} \"--build\" \".\" \"--target\" \"uninstall\"\n WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/Guest_32)\n\n add_dependencies(uninstall uninstall_guest-libs)\n add_dependencies(uninstall uninstall_guest-libs-32)\nendif()\n\nif (NOT MINGW AND BUILD_STEAM_SUPPORT)\n add_subdirectory(Source/Steam/)\nendif()\n" }, { "path": "CMakeSettings.json", "content": "{\r\n\t\"environments\": [\r\n\t\t{\r\n\t\t\t\"BuildPath\": \"${projectDir}\\\\out\\\\build\\\\${name}\",\r\n\t\t\t\"InstallPath\": \"${projectDir}\\\\out\\\\install\\\\${name}\",\r\n\t\t\t\"clangcl\": \"clang-cl.exe\",\r\n\t\t\t\"cc\": \"clang\",\r\n\t\t\t\"cxx\": \"clang++\"\r\n\t\t}\r\n\t],\r\n\t\"configurations\": [\r\n\t\t{\r\n\t\t\t\"name\": \"WSL-Clang-Debug\",\r\n\t\t\t\"generator\": \"Ninja\",\r\n\t\t\t\"configurationType\": \"Debug\",\r\n\t\t\t\"buildRoot\": \"${env.BuildPath}\",\r\n\t\t\t\"installRoot\": \"${env.InstallPath}\",\r\n\t\t\t\"cmakeExecutable\": \"/usr/bin/cmake\",\r\n\t\t\t\"cmakeCommandArgs\": \"\",\r\n\t\t\t\"buildCommandArgs\": \"-v\",\r\n\t\t\t\"ctestCommandArgs\": \"\",\r\n\t\t\t\"wslPath\": \"${defaultWSLPath}\",\r\n\t\t\t\"inheritEnvironments\": [ \"linux_clang_x64\" ],\r\n\t\t\t\"addressSanitizerRuntimeFlags\": \"detect_leaks=0\",\r\n\t\t\t\"variables\": [\r\n\t\t\t\t{\r\n\t\t\t\t\t\"name\": \"WSL\",\r\n\t\t\t\t\t\"value\": \"TRUE\",\r\n\t\t\t\t\t\"type\": \"BOOL\"\r\n\t\t\t\t}\r\n\t\t\t]\r\n\t\t},\r\n\t\t{\r\n\t\t\t\"name\": \"WSL-Clang-Release\",\r\n\t\t\t\"generator\": \"Ninja\",\r\n\t\t\t\"configurationType\": \"RelWithDebInfo\",\r\n\t\t\t\"buildRoot\": \"${env.BuildPath}\",\r\n\t\t\t\"installRoot\": \"${env.InstallPath}\",\r\n\t\t\t\"cmakeExecutable\": \"/usr/bin/cmake\",\r\n\t\t\t\"cmakeCommandArgs\": \"\",\r\n\t\t\t\"buildCommandArgs\": \"-v\",\r\n\t\t\t\"ctestCommandArgs\": \"\",\r\n\t\t\t\"wslPath\": \"${defaultWSLPath}\",\r\n\t\t\t\"inheritEnvironments\": [ \"linux_clang_x64\" ],\r\n\t\t\t\"addressSanitizerRuntimeFlags\": \"detect_leaks=0\",\r\n\t\t\t\"variables\": [\r\n\t\t\t\t{\r\n\t\t\t\t\t\"name\": \"WSL\",\r\n\t\t\t\t\t\"value\": \"TRUE\",\r\n\t\t\t\t\t\"type\": \"BOOL\"\r\n\t\t\t\t}\r\n\t\t\t]\r\n\t\t},\r\n\t\t{\r\n\t\t\t\"name\": \"x86-Clang-Cross-Debug\",\r\n\t\t\t\"generator\": \"Ninja\",\r\n\t\t\t\"configurationType\": \"Debug\",\r\n\t\t\t\"buildRoot\": \"${env.BuildPath}\",\r\n\t\t\t\"installRoot\": \"${env.InstallPath}\",\r\n\t\t\t\"cmakeCommandArgs\": \"\",\r\n\t\t\t\"buildCommandArgs\": \"-v\",\r\n\t\t\t\"ctestCommandArgs\": \"\",\r\n\t\t\t\"inheritEnvironments\": [ \"clang_cl_x86\" ],\r\n\t\t\t\"variables\": [\r\n\t\t\t\t{\r\n\t\t\t\t\t\"name\": \"CMAKE_C_COMPILER\",\r\n\t\t\t\t\t\"value\": \"${env.cc}\",\r\n\t\t\t\t\t\"type\": \"STRING\"\r\n\t\t\t\t},\r\n\t\t\t\t{\r\n\t\t\t\t\t\"name\": \"CMAKE_CXX_COMPILER\",\r\n\t\t\t\t\t\"value\": \"${env.cxx}\",\r\n\t\t\t\t\t\"type\": \"STRING\"\r\n\t\t\t\t},\r\n\t\t\t\t{\r\n\t\t\t\t\t\"name\": \"CMAKE_SYSROOT\",\r\n\t\t\t\t\t\"value\": \"${env.fexsysroot}\",\r\n\t\t\t\t\t\"type\": \"STRING\"\r\n\t\t\t\t}\r\n\t\t\t]\r\n\t\t},\r\n\t\t{\r\n\t\t\t\"name\": \"x64-Clang-Cross-Release\",\r\n\t\t\t\"generator\": \"Ninja\",\r\n\t\t\t\"configurationType\": \"RelWithDebInfo\",\r\n\t\t\t\"buildRoot\": \"${env.BuildPath}\",\r\n\t\t\t\"installRoot\": \"${env.InstallPath}\",\r\n\t\t\t\"cmakeCommandArgs\": \"\",\r\n\t\t\t\"buildCommandArgs\": \"-v\",\r\n\t\t\t\"ctestCommandArgs\": \"\",\r\n\t\t\t\"inheritEnvironments\": [ \"clang_cl_x86\" ],\r\n\t\t\t\"variables\": [\r\n\t\t\t\t{\r\n\t\t\t\t\t\"name\": \"CMAKE_C_COMPILER\",\r\n\t\t\t\t\t\"value\": \"${env.cc}\",\r\n\t\t\t\t\t\"type\": \"STRING\"\r\n\t\t\t\t},\r\n\t\t\t\t{\r\n\t\t\t\t\t\"name\": \"CMAKE_CXX_COMPILER\",\r\n\t\t\t\t\t\"value\": \"${env.cxx}\",\r\n\t\t\t\t\t\"type\": \"STRING\"\r\n\t\t\t\t},\r\n\t\t\t\t{\r\n\t\t\t\t\t\"name\": \"CMAKE_SYSROOT\",\r\n\t\t\t\t\t\"value\": \"${env.fexsysroot}\",\r\n\t\t\t\t\t\"type\": \"STRING\"\r\n\t\t\t\t}\r\n\t\t\t]\r\n\t\t},\r\n\t\t{\r\n\t\t\t\"name\": \"Linux-Clang-Remote-Debug\",\r\n\t\t\t\"generator\": \"Ninja\",\r\n\t\t\t\"configurationType\": \"Debug\",\r\n\t\t\t\"cmakeExecutable\": \"/usr/bin/cmake\",\r\n\t\t\t\"remoteCopySourcesExclusionList\": [ \".vs\", \".vscode\", \".git\", \".github\", \"build\", \"out\", \"bin\" ],\r\n\t\t\t\"cmakeCommandArgs\": \"\",\r\n\t\t\t\"buildCommandArgs\": \"-v\",\r\n\t\t\t\"ctestCommandArgs\": \"\",\r\n\t\t\t\"inheritEnvironments\": [ \"linux_clang_x64\" ],\r\n\t\t\t\"remoteMachineName\": \"${env.fexremote}\",\r\n\t\t\t\"remoteCMakeListsRoot\": \"$HOME/projects/.vs/${projectDirName}/src\",\r\n\t\t\t\"remoteBuildRoot\": \"$HOME/projects/.vs/${projectDirName}/build/${name}\",\r\n\t\t\t\"remoteInstallRoot\": \"$HOME/projects/.vs/${projectDirName}/install/${name}\",\r\n\t\t\t\"remoteCopySources\": true,\r\n\t\t\t\"rsyncCommandArgs\": \"-t --delete --delete-excluded\",\r\n\t\t\t\"remoteCopyBuildOutput\": false,\r\n\t\t\t\"remoteCopySourcesMethod\": \"rsync\",\r\n\t\t\t\"addressSanitizerRuntimeFlags\": \"detect_leaks=0\",\r\n\t\t\t\"variables\": []\r\n\t\t}\r\n\t]\r\n}\r\n" }, { "path": "CODE_OF_CONDUCT.md", "content": "# Contributor Covenant Code of Conduct\n\n## Our Pledge\n\nIn the interest of fostering an open and welcoming environment, we as\ncontributors and maintainers pledge to making participation in our project and\nour community a harassment-free experience for everyone, regardless of age, body\nsize, disability, ethnicity, sex characteristics, gender identity and expression,\nlevel of experience, education, socio-economic status, nationality, personal\nappearance, race, religion, or sexual identity and orientation.\n\n## Our Standards\n\nExamples of behavior that contributes to creating a positive environment\ninclude:\n\n* Using welcoming and inclusive language\n* Being respectful of differing viewpoints and experiences\n* Gracefully accepting constructive criticism\n* Focusing on what is best for the community\n* Showing empathy towards other community members\n\nExamples of unacceptable behavior by participants include:\n\n* The use of sexualized language or imagery and unwelcome sexual attention or\n advances\n* Trolling, insulting/derogatory comments, and personal or political attacks\n* Public or private harassment\n* Publishing others' private information, such as a physical or electronic\n address, without explicit permission\n* Other conduct which could reasonably be considered inappropriate in a\n professional setting\n\n## Our Responsibilities\n\nProject maintainers are responsible for clarifying the standards of acceptable\nbehavior and are expected to take appropriate and fair corrective action in\nresponse to any instances of unacceptable behavior.\n\nProject maintainers have the right and responsibility to remove, edit, or\nreject comments, commits, code, wiki edits, issues, and other contributions\nthat are not aligned to this Code of Conduct, or to ban temporarily or\npermanently any contributor for other behaviors that they deem inappropriate,\nthreatening, offensive, or harmful.\n\n## Scope\n\nThis Code of Conduct applies both within project spaces and in public spaces\nwhen an individual is representing the project or its community. Examples of\nrepresenting a project or community include using an official project e-mail\naddress, posting via an official social media account, or acting as an appointed\nrepresentative at an online or offline event. Representation of a project may be\nfurther defined and clarified by project maintainers.\n\n## Enforcement\n\nInstances of abusive, harassing, or otherwise unacceptable behavior may be\nreported by contacting the project team at team@fex-emu.com. All\ncomplaints will be reviewed and investigated and will result in a response that\nis deemed necessary and appropriate to the circumstances. The project team is\nobligated to maintain confidentiality with regard to the reporter of an incident.\nFurther details of specific enforcement policies may be posted separately.\n\nProject maintainers who do not follow or enforce the Code of Conduct in good\nfaith may face temporary or permanent repercussions as determined by other\nmembers of the project's leadership.\n\n## Attribution\n\nThis Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,\navailable at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html\n\n[homepage]: https://www.contributor-covenant.org\n\nFor answers to common questions about this code of conduct, see\nhttps://www.contributor-covenant.org/faq\n" }, { "path": "CodeEmitter/CMakeLists.txt", "content": "add_library(CodeEmitter INTERFACE)\ntarget_include_directories(CodeEmitter INTERFACE .)\n" }, { "path": "CodeEmitter/CodeEmitter/ALUOps.inl", "content": "// SPDX-License-Identifier: MIT\n/* ALU instruction emitters.\n *\n * Almost all of these operations have `ARMEmitter::Size` as their first argument.\n * This allows both 32-bit and 64-bit selection of how that instruction is going to operate.\n *\n * Some emitter operations explicitly use `XRegister` or `WRegister`.\n * This is usually due to the instruction only supporting one operating size.\n * Although in some cases is a minor convenience without any performance implications.\n *\n * FEX-Emu ALU operations usually have a 32-bit or 64-bit operating size encoded in the IR operation,\n * This allows FEX to use a single helper function which decodes to both handlers.\n */\n\n#pragma once\n#ifndef INCLUDED_BY_EMITTER\n#include \nnamespace ARMEmitter {\nstruct EmitterOps : Emitter {\n#endif\n\nprivate:\n static bool IsADRRange(int64_t Imm) {\n return Imm >= -1048576 && Imm <= 1048575;\n }\n static bool IsADRPRange(int64_t Imm) {\n return Imm >= -4294967296 && Imm <= 4294963200;\n }\n static bool IsADRPAligned(int64_t Imm) {\n return (Imm & 0xFFF) == 0;\n }\npublic:\n // PC relative\n void adr(ARMEmitter::Register rd, uint32_t Imm) {\n constexpr uint32_t Op = 0b0001'0000 << 24;\n DataProcessing_PCRel_Imm(Op, rd, Imm);\n }\n\n [[nodiscard]] BranchEncodeSucceeded adr(ARMEmitter::Register rd, const BackwardLabel* Label) {\n int32_t Imm = static_cast(Label->Location - GetCursorAddress());\n if (IsADRRange(Imm)) {\n constexpr uint32_t Op = 0b0001'0000 << 24;\n DataProcessing_PCRel_Imm(Op, rd, Imm);\n return BranchEncodeSucceeded::Success;\n }\n\n // Can't encode.\n return BranchEncodeSucceeded::Failure;\n }\n [[nodiscard]] BranchEncodeSucceeded adr(ARMEmitter::Register rd, ForwardLabel* Label) {\n AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress(), .Type = ForwardLabel::InstType::ADR});\n constexpr uint32_t Op = 0b0001'0000 << 24;\n DataProcessing_PCRel_Imm(Op, rd, 0);\n\n // Forward label doesn't know if it can encode until Bind.\n return BranchEncodeSucceeded::Success;\n }\n\n [[nodiscard]] BranchEncodeSucceeded adr(ARMEmitter::Register rd, BiDirectionalLabel* Label) {\n if (Label->Backward.Location) {\n return adr(rd, &Label->Backward);\n } else {\n return adr(rd, &Label->Forward);\n }\n }\n\n void adrp(ARMEmitter::Register rd, uint32_t Imm) {\n constexpr uint32_t Op = 0b1001'0000 << 24;\n DataProcessing_PCRel_Imm(Op, rd, Imm);\n }\n\n [[nodiscard]] BranchEncodeSucceeded adrp(ARMEmitter::Register rd, const BackwardLabel* Label) {\n int64_t Imm = reinterpret_cast(Label->Location) - (GetCursorAddress() & ~0xFFFLL);\n\n if (IsADRPRange(Imm) && IsADRPAligned(Imm)) {\n constexpr uint32_t Op = 0b1001'0000 << 24;\n DataProcessing_PCRel_Imm(Op, rd, Imm);\n return BranchEncodeSucceeded::Success;\n }\n\n // Can't encode.\n return BranchEncodeSucceeded::Failure;\n }\n\n [[nodiscard]] BranchEncodeSucceeded adrp(ARMEmitter::Register rd, ForwardLabel* Label) {\n AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress(), .Type = ForwardLabel::InstType::ADRP});\n constexpr uint32_t Op = 0b1001'0000 << 24;\n DataProcessing_PCRel_Imm(Op, rd, 0);\n\n // Forward label doesn't know if it can encode until Bind.\n return BranchEncodeSucceeded::Success;\n }\n\n [[nodiscard]] BranchEncodeSucceeded adrp(ARMEmitter::Register rd, BiDirectionalLabel* Label) {\n if (Label->Backward.Location) {\n return adrp(rd, &Label->Backward);\n } else {\n return adrp(rd, &Label->Forward);\n }\n }\n\n [[nodiscard]] BranchEncodeSucceeded LongAddressGen(ARMEmitter::Register rd, const BackwardLabel* Label) {\n const auto SLocation = reinterpret_cast(Label->Location);\n const auto ULocation = std::bit_cast(SLocation);\n\n const int64_t Imm = SLocation - (GetCursorAddress());\n const auto UImm = std::bit_cast(Imm);\n\n if (IsADRRange(Imm)) {\n // If the range is in ADR range then we can just use ADR.\n return adr(rd, Label);\n }\n if (IsADRPRange(Imm)) {\n const int64_t ADRPImm = (SLocation & ~0xFFFLL) - (GetCursorAddress() & ~0xFFFLL);\n\n // If the range is in the ADRP range then we can use ADRP.\n const bool NeedsOffset = !IsADRPAligned(ULocation);\n const uint64_t AlignedOffset = ULocation & 0xFFFULL;\n\n // First emit ADRP\n adrp(rd, ADRPImm >> 12);\n\n if (NeedsOffset) {\n // Now even an add\n add(ARMEmitter::Size::i64Bit, rd, rd, AlignedOffset);\n }\n\n return BranchEncodeSucceeded::Success;\n }\n\n // Stinky path, we need to load the address as a sequence of movz+movk+movk\n movz(ARMEmitter::Size::i64Bit, rd, (UImm >> 32) & 0xFFFF, 32);\n movk(ARMEmitter::Size::i64Bit, rd, (UImm >> 16) & 0xFFFF, 16);\n movk(ARMEmitter::Size::i64Bit, rd, UImm & 0xFFFF);\n\n return BranchEncodeSucceeded::Success;\n }\n [[nodiscard]] BranchEncodeSucceeded LongAddressGen(ARMEmitter::Register rd, ForwardLabel* Label) {\n AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress(), .Type = ForwardLabel::InstType::LONG_ADDRESS_GEN});\n // Emit a register index and two nops. These will be backpatched.\n dc32(rd.Idx());\n nop();\n nop();\n\n // Forward label doesn't know if it can encode until Bind.\n return BranchEncodeSucceeded::Success;\n }\n\n [[nodiscard]] BranchEncodeSucceeded LongAddressGen(ARMEmitter::Register rd, BiDirectionalLabel* Label) {\n if (Label->Backward.Location) {\n return LongAddressGen(rd, &Label->Backward);\n } else {\n return LongAddressGen(rd, &Label->Forward);\n }\n }\n\n // Add/subtract immediate\n void add(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, uint32_t Imm, bool LSL12 = false) {\n constexpr uint32_t Op = 0b0001'0001'0 << 23;\n DataProcessing_AddSub_Imm(Op, s, rd, rn, Imm, LSL12);\n }\n\n void adds(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, uint32_t Imm, bool LSL12 = false) {\n constexpr uint32_t Op = 0b0011'0001'0 << 23;\n DataProcessing_AddSub_Imm(Op, s, rd, rn, Imm, LSL12);\n }\n void cmn(ARMEmitter::Size s, ARMEmitter::Register rn, uint32_t Imm, bool LSL12 = false) {\n adds(s, ARMEmitter::Reg::zr, rn, Imm, LSL12);\n }\n void sub(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, uint32_t Imm, bool LSL12 = false) {\n constexpr uint32_t Op = 0b0101'0001'0 << 23;\n DataProcessing_AddSub_Imm(Op, s, rd, rn, Imm, LSL12);\n }\n\n void cmp(ARMEmitter::Size s, ARMEmitter::Register rn, uint32_t Imm, bool LSL12 = false) {\n constexpr uint32_t Op = 0b0111'0001'0 << 23;\n DataProcessing_AddSub_Imm(Op, s, ARMEmitter::Reg::rsp, rn, Imm, LSL12);\n }\n\n void subs(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, uint32_t Imm, bool LSL12 = false) {\n constexpr uint32_t Op = 0b0111'0001'0 << 23;\n DataProcessing_AddSub_Imm(Op, s, rd, rn, Imm, LSL12);\n }\n\n // Min/max immediate\n void smax(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, int64_t Imm) {\n LOGMAN_THROW_A_FMT(Imm >= -128 && Imm <= 127, \"{} Immediate too large\", __func__);\n MinMaxImmediate(0b0000, s, rd, rn, Imm);\n }\n\n void umax(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, uint64_t Imm) {\n LOGMAN_THROW_A_FMT(Imm <= 255, \"{} Immediate too large\", __func__);\n MinMaxImmediate(0b0001, s, rd, rn, Imm);\n }\n\n void smin(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, int64_t Imm) {\n LOGMAN_THROW_A_FMT(Imm >= -128 && Imm <= 127, \"{} Immediate too large\", __func__);\n MinMaxImmediate(0b0010, s, rd, rn, Imm);\n }\n\n void umin(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, uint64_t Imm) {\n LOGMAN_THROW_A_FMT(Imm <= 255, \"{} Immediate too large\", __func__);\n MinMaxImmediate(0b0011, s, rd, rn, Imm);\n }\n\n // Logical immediate\n void and_(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, uint64_t Imm) {\n uint32_t n, immr, imms;\n const auto IsImm = IsImmLogical(Imm, RegSizeInBits(s), &n, &imms, &immr);\n LOGMAN_THROW_A_FMT(IsImm, \"Couldn't encode immediate to logical op\");\n and_(s, rd, rn, n, immr, imms);\n }\n\n void bic(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, uint64_t Imm) {\n and_(s, rd, rn, ~Imm);\n }\n\n void ands(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, uint64_t Imm) {\n uint32_t n, immr, imms;\n const auto IsImm = IsImmLogical(Imm, RegSizeInBits(s), &n, &imms, &immr);\n LOGMAN_THROW_A_FMT(IsImm, \"Couldn't encode immediate to logical op\");\n ands(s, rd, rn, n, immr, imms);\n }\n\n void bics(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, uint64_t Imm) {\n ands(s, rd, rn, ~Imm);\n }\n\n void orr(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, uint64_t Imm) {\n uint32_t n, immr, imms;\n const auto IsImm = IsImmLogical(Imm, RegSizeInBits(s), &n, &imms, &immr);\n LOGMAN_THROW_A_FMT(IsImm, \"Couldn't encode immediate to logical op\");\n orr(s, rd, rn, n, immr, imms);\n }\n\n void eor(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, uint64_t Imm) {\n uint32_t n, immr, imms;\n const auto IsImm = IsImmLogical(Imm, RegSizeInBits(s), &n, &imms, &immr);\n LOGMAN_THROW_A_FMT(IsImm, \"Couldn't encode immediate to logical op\");\n eor(s, rd, rn, n, immr, imms);\n }\n\n void tst(ARMEmitter::Size s, Register rn, uint64_t imm) {\n ands(s, Reg::zr, rn, imm);\n }\n\n // Move wide immediate\n void movn(ARMEmitter::Size s, ARMEmitter::Register rd, uint32_t Imm, uint32_t Offset = 0) {\n LOGMAN_THROW_A_FMT((Imm & 0xFFFF0000U) == 0, \"Upper bits of move wide not valid\");\n LOGMAN_THROW_A_FMT((Offset % 16) == 0, \"Offset must be 16bit aligned\");\n\n constexpr uint32_t Op = 0b001'0010'100 << 21;\n DataProcessing_MoveWide(Op, s, rd, Imm, Offset >> 4);\n }\n void mov(ARMEmitter::Size s, ARMEmitter::Register rd, uint32_t Imm) {\n movz(s, rd, Imm, 0);\n }\n void mov(ARMEmitter::XRegister rd, uint32_t Imm) {\n movz(ARMEmitter::Size::i64Bit, rd.R(), Imm, 0);\n }\n void mov(ARMEmitter::WRegister rd, uint32_t Imm) {\n movz(ARMEmitter::Size::i32Bit, rd.R(), Imm, 0);\n }\n\n void movz(ARMEmitter::Size s, ARMEmitter::Register rd, uint32_t Imm, uint32_t Offset = 0) {\n LOGMAN_THROW_A_FMT((Imm & 0xFFFF0000U) == 0, \"Upper bits of move wide not valid\");\n LOGMAN_THROW_A_FMT((Offset % 16) == 0, \"Offset must be 16bit aligned\");\n\n constexpr uint32_t Op = 0b101'0010'100 << 21;\n DataProcessing_MoveWide(Op, s, rd, Imm, Offset >> 4);\n }\n void movk(ARMEmitter::Size s, ARMEmitter::Register rd, uint32_t Imm, uint32_t Offset = 0) {\n LOGMAN_THROW_A_FMT((Imm & 0xFFFF0000U) == 0, \"Upper bits of move wide not valid\");\n LOGMAN_THROW_A_FMT((Offset % 16) == 0, \"Offset must be 16bit aligned\");\n\n constexpr uint32_t Op = 0b111'0010'100 << 21;\n DataProcessing_MoveWide(Op, s, rd, Imm, Offset >> 4);\n }\n\n void movn(ARMEmitter::XRegister rd, uint32_t Imm, uint32_t Offset = 0) {\n movn(ARMEmitter::Size::i64Bit, rd.R(), Imm, Offset);\n }\n void movz(ARMEmitter::XRegister rd, uint32_t Imm, uint32_t Offset = 0) {\n movz(ARMEmitter::Size::i64Bit, rd.R(), Imm, Offset);\n }\n void movk(ARMEmitter::XRegister rd, uint32_t Imm, uint32_t Offset = 0) {\n movk(ARMEmitter::Size::i64Bit, rd.R(), Imm, Offset);\n }\n void movn(ARMEmitter::WRegister rd, uint32_t Imm, uint32_t Offset = 0) {\n movn(ARMEmitter::Size::i32Bit, rd.R(), Imm, Offset);\n }\n void movz(ARMEmitter::WRegister rd, uint32_t Imm, uint32_t Offset = 0) {\n movz(ARMEmitter::Size::i32Bit, rd.R(), Imm, Offset);\n }\n void movk(ARMEmitter::WRegister rd, uint32_t Imm, uint32_t Offset = 0) {\n movk(ARMEmitter::Size::i32Bit, rd.R(), Imm, Offset);\n }\n\n // Bitfield\n void sxtb(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn) {\n sbfm(s, rd, rn, 0, 7);\n }\n void sxth(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn) {\n sbfm(s, rd, rn, 0, 15);\n }\n void sxtw(ARMEmitter::XRegister rd, ARMEmitter::WRegister rn) {\n sbfm(ARMEmitter::Size::i64Bit, rd, rn.X(), 0, 31);\n }\n void sbfx(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, uint32_t lsb, uint32_t width) {\n LOGMAN_THROW_A_FMT(width > 0, \"sbfx needs width > 0\");\n LOGMAN_THROW_A_FMT((lsb + width) <= RegSizeInBits(s), \"Tried to sbfx a region larger than the register\");\n sbfm(s, rd, rn, lsb, lsb + width - 1);\n }\n void sbfiz(ARMEmitter::Size s, Register rd, Register rn, uint32_t lsb, uint32_t width) {\n xbfiz_helper(true, s, rd, rn, lsb, width);\n }\n void asr(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, uint32_t shift) {\n const auto RegSize_m1 = RegSizeInBits(s) - 1;\n shift &= RegSize_m1;\n sbfm(s, rd, rn, shift, RegSize_m1);\n }\n\n void uxtb(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn) {\n ubfm(s, rd, rn, 0, 7);\n }\n void uxth(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn) {\n ubfm(s, rd, rn, 0, 15);\n }\n void uxtw(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn) {\n ubfm(s, rd, rn, 0, 31);\n }\n\n void ubfm(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, uint32_t immr, uint32_t imms) {\n constexpr uint32_t Op = 0b0101'0011'00 << 22;\n DataProcessing_Logical_Imm(Op, s, rd, rn, s == ARMEmitter::Size::i64Bit, immr, imms);\n }\n\n void ubfiz(ARMEmitter::Size s, Register rd, Register rn, uint32_t lsb, uint32_t width) {\n xbfiz_helper(false, s, rd, rn, lsb, width);\n }\n\n void lsl(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, uint32_t shift) {\n const auto RegSize_m1 = RegSizeInBits(s) - 1;\n shift &= RegSize_m1;\n ubfm(s, rd, rn, (RegSizeInBits(s) - shift) & RegSize_m1, RegSize_m1 - shift);\n }\n void lsr(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, uint32_t shift) {\n const auto RegSize_m1 = RegSizeInBits(s) - 1;\n shift &= RegSize_m1;\n ubfm(s, rd, rn, shift, RegSize_m1);\n }\n void ubfx(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, uint32_t lsb, uint32_t width) {\n LOGMAN_THROW_A_FMT(width > 0, \"ubfx needs width > 0\");\n LOGMAN_THROW_A_FMT((lsb + width) <= RegSizeInBits(s), \"Tried to ubfx a region larger than the register\");\n ubfm(s, rd, rn, lsb, lsb + width - 1);\n }\n\n void bfi(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, uint32_t lsb, uint32_t width) {\n const auto RegSize = RegSizeInBits(s);\n LOGMAN_THROW_A_FMT(width > 0, \"bfc/bfi needs width > 0\");\n LOGMAN_THROW_A_FMT((lsb + width) <= RegSize, \"Tried to bfc/bfi a region larger than the register\");\n bfm(s, rd, rn, (RegSize - lsb) & (RegSize - 1), width - 1);\n }\n void bfc(ARMEmitter::Size s, Register rd, uint32_t lsb, uint32_t width) {\n bfi(s, rd, Reg::zr, lsb, width);\n }\n void bfxil(ARMEmitter::Size s, Register rd, Register rn, uint32_t lsb, uint32_t width) {\n const auto reg_size_bits = RegSizeInBits(s);\n const auto lsb_p_width = lsb + width;\n\n LOGMAN_THROW_A_FMT(width >= 1, \"bfxil needs width >= 1\");\n LOGMAN_THROW_A_FMT(lsb_p_width <= reg_size_bits, \"bfxil lsb + width ({}) must be <= {}. lsb={}, width={}\", lsb_p_width, reg_size_bits,\n lsb, width);\n\n bfm(s, rd, rn, lsb, lsb_p_width - 1);\n }\n\n // Extract\n void extr(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, ARMEmitter::Register rm, uint32_t Imm) {\n constexpr uint32_t Op = 0b001'0011'100 << 21;\n LOGMAN_THROW_A_FMT(Imm < RegSizeInBits(s), \"Tried to extr a region larger than the register\");\n DataProcessing_Extract(Op, s, rd, rn, rm, Imm);\n }\n\n void ror(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, uint32_t Imm) {\n Imm &= RegSizeInBits(s) - 1;\n extr(s, rd, rn, rn, Imm);\n }\n\n // Data processing - 2 source\n void udiv(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, ARMEmitter::Register rm) {\n constexpr uint32_t Op = (0b001'1010'110U << 21) | (0b0000'10U << 10);\n DataProcessing_2Source(Op, s, rd, rn, rm);\n }\n void sdiv(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, ARMEmitter::Register rm) {\n constexpr uint32_t Op = (0b001'1010'110U << 21) | (0b0000'11U << 10);\n DataProcessing_2Source(Op, s, rd, rn, rm);\n }\n\n void lslv(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, ARMEmitter::Register rm) {\n constexpr uint32_t Op = (0b001'1010'110U << 21) | (0b0010'00U << 10);\n DataProcessing_2Source(Op, s, rd, rn, rm);\n }\n void lsrv(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, ARMEmitter::Register rm) {\n constexpr uint32_t Op = (0b001'1010'110U << 21) | (0b0010'01U << 10);\n DataProcessing_2Source(Op, s, rd, rn, rm);\n }\n void asrv(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, ARMEmitter::Register rm) {\n constexpr uint32_t Op = (0b001'1010'110U << 21) | (0b0010'10U << 10);\n DataProcessing_2Source(Op, s, rd, rn, rm);\n }\n void rorv(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, ARMEmitter::Register rm) {\n constexpr uint32_t Op = (0b001'1010'110U << 21) | (0b0010'11U << 10);\n DataProcessing_2Source(Op, s, rd, rn, rm);\n }\n void crc32b(ARMEmitter::WRegister rd, ARMEmitter::WRegister rn, ARMEmitter::WRegister rm) {\n constexpr uint32_t Op = (0b001'1010'110U << 21) | (0b0100'00U << 10);\n DataProcessing_2Source(Op, ARMEmitter::Size::i32Bit, rd, rn, rm);\n }\n void crc32h(ARMEmitter::WRegister rd, ARMEmitter::WRegister rn, ARMEmitter::WRegister rm) {\n constexpr uint32_t Op = (0b001'1010'110U << 21) | (0b0100'01U << 10);\n DataProcessing_2Source(Op, ARMEmitter::Size::i32Bit, rd, rn, rm);\n }\n void crc32w(ARMEmitter::WRegister rd, ARMEmitter::WRegister rn, ARMEmitter::WRegister rm) {\n constexpr uint32_t Op = (0b001'1010'110U << 21) | (0b0100'10U << 10);\n DataProcessing_2Source(Op, ARMEmitter::Size::i32Bit, rd, rn, rm);\n }\n void crc32cb(ARMEmitter::WRegister rd, ARMEmitter::WRegister rn, ARMEmitter::WRegister rm) {\n constexpr uint32_t Op = (0b001'1010'110U << 21) | (0b0101'00U << 10);\n DataProcessing_2Source(Op, ARMEmitter::Size::i32Bit, rd, rn, rm);\n }\n void crc32ch(ARMEmitter::WRegister rd, ARMEmitter::WRegister rn, ARMEmitter::WRegister rm) {\n constexpr uint32_t Op = (0b001'1010'110U << 21) | (0b0101'01U << 10);\n DataProcessing_2Source(Op, ARMEmitter::Size::i32Bit, rd, rn, rm);\n }\n void crc32cw(ARMEmitter::WRegister rd, ARMEmitter::WRegister rn, ARMEmitter::WRegister rm) {\n constexpr uint32_t Op = (0b001'1010'110U << 21) | (0b0101'10U << 10);\n DataProcessing_2Source(Op, ARMEmitter::Size::i32Bit, rd, rn, rm);\n }\n void smax(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, ARMEmitter::Register rm) {\n constexpr uint32_t Op = (0b001'1010'110U << 21) | (0b0110'00U << 10);\n DataProcessing_2Source(Op, s, rd, rn, rm);\n }\n void umax(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, ARMEmitter::Register rm) {\n constexpr uint32_t Op = (0b001'1010'110U << 21) | (0b0110'01U << 10);\n DataProcessing_2Source(Op, s, rd, rn, rm);\n }\n void smin(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, ARMEmitter::Register rm) {\n constexpr uint32_t Op = (0b001'1010'110U << 21) | (0b0110'10U << 10);\n DataProcessing_2Source(Op, s, rd, rn, rm);\n }\n void umin(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, ARMEmitter::Register rm) {\n constexpr uint32_t Op = (0b001'1010'110U << 21) | (0b0110'11U << 10);\n DataProcessing_2Source(Op, s, rd, rn, rm);\n }\n void subp(ARMEmitter::XRegister rd, ARMEmitter::XRegister rn, ARMEmitter::XRegister rm) {\n constexpr uint32_t Op = (0b001'1010'110U << 21) | (0b0000'00U << 10);\n DataProcessing_2Source(Op, ARMEmitter::Size::i64Bit, rd, rn, rm);\n }\n void irg(ARMEmitter::XRegister rd, ARMEmitter::XRegister rn, ARMEmitter::XRegister rm) {\n constexpr uint32_t Op = (0b001'1010'110U << 21) | (0b0001'00U << 10);\n DataProcessing_2Source(Op, ARMEmitter::Size::i64Bit, rd, rn, rm);\n }\n void gmi(ARMEmitter::XRegister rd, ARMEmitter::XRegister rn, ARMEmitter::XRegister rm) {\n constexpr uint32_t Op = (0b001'1010'110U << 21) | (0b0001'01U << 10);\n DataProcessing_2Source(Op, ARMEmitter::Size::i64Bit, rd, rn, rm);\n }\n void pacga(ARMEmitter::XRegister rd, ARMEmitter::XRegister rn, ARMEmitter::XRegister rm) {\n constexpr uint32_t Op = (0b001'1010'110U << 21) | (0b0011'00U << 10);\n DataProcessing_2Source(Op, ARMEmitter::Size::i64Bit, rd, rn, rm);\n }\n void crc32x(ARMEmitter::XRegister rd, ARMEmitter::XRegister rn, ARMEmitter::XRegister rm) {\n constexpr uint32_t Op = (0b001'1010'110U << 21) | (0b0100'11U << 10);\n DataProcessing_2Source(Op, ARMEmitter::Size::i64Bit, rd, rn, rm);\n }\n void crc32cx(ARMEmitter::XRegister rd, ARMEmitter::XRegister rn, ARMEmitter::XRegister rm) {\n constexpr uint32_t Op = (0b001'1010'110U << 21) | (0b0101'11U << 10);\n DataProcessing_2Source(Op, ARMEmitter::Size::i64Bit, rd, rn, rm);\n }\n void subps(ARMEmitter::XRegister rd, ARMEmitter::XRegister rn, ARMEmitter::XRegister rm) {\n constexpr uint32_t Op = (0b011'1010'110U << 21) | (0b0000'00U << 10);\n DataProcessing_2Source(Op, ARMEmitter::Size::i64Bit, rd, rn, rm);\n }\n\n // Data processing - 1 source\n void rbit(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn) {\n constexpr uint32_t Op = (0b101'1010'110U << 21) | (0b0'0000U << 16) | (0b0000'00U << 10);\n DataProcessing_1Source(Op, s, rd, rn);\n }\n void rev16(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn) {\n constexpr uint32_t Op = (0b101'1010'110U << 21) | (0b0'0000U << 16) | (0b0000'01U << 10);\n DataProcessing_1Source(Op, s, rd, rn);\n }\n void rev(ARMEmitter::WRegister rd, ARMEmitter::WRegister rn) {\n constexpr uint32_t Op = (0b101'1010'110U << 21) | (0b0'0000U << 16) | (0b0000'10U << 10);\n DataProcessing_1Source(Op, ARMEmitter::Size::i32Bit, rd, rn);\n }\n void rev32(ARMEmitter::XRegister rd, ARMEmitter::XRegister rn) {\n constexpr uint32_t Op = (0b101'1010'110U << 21) | (0b0'0000U << 16) | (0b0000'10U << 10);\n DataProcessing_1Source(Op, ARMEmitter::Size::i64Bit, rd, rn);\n }\n void clz(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn) {\n constexpr uint32_t Op = (0b101'1010'110U << 21) | (0b0'0000U << 16) | (0b0001'00U << 10);\n DataProcessing_1Source(Op, s, rd, rn);\n }\n void cls(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn) {\n constexpr uint32_t Op = (0b101'1010'110U << 21) | (0b0'0000U << 16) | (0b0001'01U << 10);\n DataProcessing_1Source(Op, s, rd, rn);\n }\n void rev(ARMEmitter::XRegister rd, ARMEmitter::XRegister rn) {\n constexpr uint32_t Op = (0b101'1010'110U << 21) | (0b0'0000U << 16) | (0b0000'11U << 10);\n DataProcessing_1Source(Op, ARMEmitter::Size::i64Bit, rd, rn);\n }\n void rev(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn) {\n uint32_t Op = (0b101'1010'110U << 21) | (0b0'0000U << 16) | (0b0000'10U << 10) | (s == ARMEmitter::Size::i64Bit ? (1U << 10) : 0);\n DataProcessing_1Source(Op, s, rd, rn);\n }\n void ctz(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn) {\n constexpr uint32_t Op = (0b101'1010'110U << 21) | (0b0'0000U << 16) | (0b0001'10U << 10);\n DataProcessing_1Source(Op, s, rd, rn);\n }\n void cnt(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn) {\n constexpr uint32_t Op = (0b101'1010'110U << 21) | (0b0'0000U << 16) | (0b0001'11U << 10);\n DataProcessing_1Source(Op, s, rd, rn);\n }\n void abs(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn) {\n constexpr uint32_t Op = (0b101'1010'110U << 21) | (0b0'0000U << 16) | (0b0010'00U << 10);\n DataProcessing_1Source(Op, s, rd, rn);\n }\n\n // TODO: PAUTH\n\n // Logical - shifted register\n void mov(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn) {\n orr(s, rd, ARMEmitter::Reg::zr, rn, ARMEmitter::ShiftType::LSL, 0);\n }\n void mov(ARMEmitter::XRegister rd, ARMEmitter::XRegister rn) {\n orr(ARMEmitter::Size::i64Bit, rd.R(), ARMEmitter::Reg::zr, rn.R(), ARMEmitter::ShiftType::LSL, 0);\n }\n void mov(ARMEmitter::WRegister rd, ARMEmitter::WRegister rn) {\n orr(ARMEmitter::Size::i32Bit, rd.R(), ARMEmitter::Reg::zr, rn.R(), ARMEmitter::ShiftType::LSL, 0);\n }\n\n void mvn(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, ARMEmitter::ShiftType Shift = ARMEmitter::ShiftType::LSL,\n uint32_t amt = 0) {\n orn(s, rd, ARMEmitter::Reg::zr, rn, Shift, amt);\n }\n\n void and_(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, ARMEmitter::Register rm,\n ARMEmitter::ShiftType Shift = ARMEmitter::ShiftType::LSL, uint32_t amt = 0) {\n constexpr uint32_t Op = 0b000'1010'000U << 21;\n DataProcessing_Shifted_Reg(Op, s, rd, rn, rm, Shift, amt);\n }\n void ands(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, ARMEmitter::Register rm,\n ARMEmitter::ShiftType Shift = ARMEmitter::ShiftType::LSL, uint32_t amt = 0) {\n constexpr uint32_t Op = 0b110'1010'000U << 21;\n DataProcessing_Shifted_Reg(Op, s, rd, rn, rm, Shift, amt);\n }\n void bic(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, ARMEmitter::Register rm,\n ARMEmitter::ShiftType Shift = ARMEmitter::ShiftType::LSL, uint32_t amt = 0) {\n constexpr uint32_t Op = 0b000'1010'001U << 21;\n DataProcessing_Shifted_Reg(Op, s, rd, rn, rm, Shift, amt);\n }\n void bics(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, ARMEmitter::Register rm,\n ARMEmitter::ShiftType Shift = ARMEmitter::ShiftType::LSL, uint32_t amt = 0) {\n constexpr uint32_t Op = 0b110'1010'001U << 21;\n DataProcessing_Shifted_Reg(Op, s, rd, rn, rm, Shift, amt);\n }\n void orr(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, ARMEmitter::Register rm,\n ARMEmitter::ShiftType Shift = ARMEmitter::ShiftType::LSL, uint32_t amt = 0) {\n constexpr uint32_t Op = 0b010'1010'000U << 21;\n DataProcessing_Shifted_Reg(Op, s, rd, rn, rm, Shift, amt);\n }\n void tst(ARMEmitter::Size s, Register rn, Register rm, ShiftType shift = ShiftType::LSL, uint32_t amt = 0) {\n ands(s, Reg::zr, rn, rm, shift, amt);\n }\n\n void orn(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, ARMEmitter::Register rm,\n ARMEmitter::ShiftType Shift = ARMEmitter::ShiftType::LSL, uint32_t amt = 0) {\n constexpr uint32_t Op = 0b010'1010'001U << 21;\n DataProcessing_Shifted_Reg(Op, s, rd, rn, rm, Shift, amt);\n }\n void eor(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, ARMEmitter::Register rm,\n ARMEmitter::ShiftType Shift = ARMEmitter::ShiftType::LSL, uint32_t amt = 0) {\n constexpr uint32_t Op = 0b100'1010'000U << 21;\n DataProcessing_Shifted_Reg(Op, s, rd, rn, rm, Shift, amt);\n }\n void eon(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, ARMEmitter::Register rm,\n ARMEmitter::ShiftType Shift = ARMEmitter::ShiftType::LSL, uint32_t amt = 0) {\n constexpr uint32_t Op = 0b100'1010'001U << 21;\n DataProcessing_Shifted_Reg(Op, s, rd, rn, rm, Shift, amt);\n }\n\n // AddSub - shifted register\n void add(ARMEmitter::XRegister rd, ARMEmitter::XRegister rn, ARMEmitter::XRegister rm,\n ARMEmitter::ShiftType Shift = ARMEmitter::ShiftType::LSL, uint32_t amt = 0) {\n add(ARMEmitter::Size::i64Bit, rd.R(), rn.R(), rm.R(), Shift, amt);\n }\n void adds(ARMEmitter::XRegister rd, ARMEmitter::XRegister rn, ARMEmitter::XRegister rm,\n ARMEmitter::ShiftType Shift = ARMEmitter::ShiftType::LSL, uint32_t amt = 0) {\n adds(ARMEmitter::Size::i64Bit, rd.R(), rn.R(), rm.R(), Shift, amt);\n }\n void cmn(ARMEmitter::XRegister rn, ARMEmitter::XRegister rm, ARMEmitter::ShiftType Shift = ARMEmitter::ShiftType::LSL, uint32_t amt = 0) {\n adds(ARMEmitter::Size::i64Bit, ARMEmitter::XReg::zr, rn.R(), rm.R(), Shift, amt);\n }\n void sub(ARMEmitter::XRegister rd, ARMEmitter::XRegister rn, ARMEmitter::XRegister rm,\n ARMEmitter::ShiftType Shift = ARMEmitter::ShiftType::LSL, uint32_t amt = 0) {\n sub(ARMEmitter::Size::i64Bit, rd.R(), rn.R(), rm.R(), Shift, amt);\n }\n void neg(ARMEmitter::XRegister rd, ARMEmitter::XRegister rm, ARMEmitter::ShiftType Shift = ARMEmitter::ShiftType::LSL, uint32_t amt = 0) {\n sub(rd, ARMEmitter::XReg::zr, rm, Shift, amt);\n }\n void cmp(ARMEmitter::XRegister rn, ARMEmitter::XRegister rm, ARMEmitter::ShiftType Shift = ARMEmitter::ShiftType::LSL, uint32_t amt = 0) {\n subs(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::rsp, rn.R(), rm.R(), Shift, amt);\n }\n void subs(ARMEmitter::XRegister rd, ARMEmitter::XRegister rn, ARMEmitter::XRegister rm,\n ARMEmitter::ShiftType Shift = ARMEmitter::ShiftType::LSL, uint32_t amt = 0) {\n subs(ARMEmitter::Size::i64Bit, rd.R(), rn.R(), rm.R(), Shift, amt);\n }\n void negs(ARMEmitter::XRegister rd, ARMEmitter::XRegister rm, ARMEmitter::ShiftType Shift = ARMEmitter::ShiftType::LSL, uint32_t amt = 0) {\n subs(rd, ARMEmitter::XReg::zr, rm, Shift, amt);\n }\n\n void add(ARMEmitter::WRegister rd, ARMEmitter::WRegister rn, ARMEmitter::WRegister rm,\n ARMEmitter::ShiftType Shift = ARMEmitter::ShiftType::LSL, uint32_t amt = 0) {\n add(ARMEmitter::Size::i32Bit, rd.R(), rn.R(), rm.R(), Shift, amt);\n }\n void adds(ARMEmitter::WRegister rd, ARMEmitter::WRegister rn, ARMEmitter::WRegister rm,\n ARMEmitter::ShiftType Shift = ARMEmitter::ShiftType::LSL, uint32_t amt = 0) {\n adds(ARMEmitter::Size::i32Bit, rd.R(), rn.R(), rm.R(), Shift, amt);\n }\n void cmn(ARMEmitter::WRegister rn, ARMEmitter::WRegister rm, ARMEmitter::ShiftType Shift = ARMEmitter::ShiftType::LSL, uint32_t amt = 0) {\n adds(ARMEmitter::Size::i32Bit, ARMEmitter::WReg::zr, rn.R(), rm.R(), Shift, amt);\n }\n void sub(ARMEmitter::WRegister rd, ARMEmitter::WRegister rn, ARMEmitter::WRegister rm,\n ARMEmitter::ShiftType Shift = ARMEmitter::ShiftType::LSL, uint32_t amt = 0) {\n sub(ARMEmitter::Size::i32Bit, rd.R(), rn.R(), rm.R(), Shift, amt);\n }\n void neg(ARMEmitter::WRegister rd, ARMEmitter::WRegister rm, ARMEmitter::ShiftType Shift = ARMEmitter::ShiftType::LSL, uint32_t amt = 0) {\n sub(rd, ARMEmitter::WReg::zr, rm, Shift, amt);\n }\n void cmp(ARMEmitter::WRegister rn, ARMEmitter::WRegister rm, ARMEmitter::ShiftType Shift = ARMEmitter::ShiftType::LSL, uint32_t amt = 0) {\n subs(ARMEmitter::Size::i32Bit, ARMEmitter::Reg::rsp, rn.R(), rm.R(), Shift, amt);\n }\n void subs(ARMEmitter::WRegister rd, ARMEmitter::WRegister rn, ARMEmitter::WRegister rm,\n ARMEmitter::ShiftType Shift = ARMEmitter::ShiftType::LSL, uint32_t amt = 0) {\n subs(ARMEmitter::Size::i32Bit, rd.R(), rn.R(), rm.R(), Shift, amt);\n }\n void negs(ARMEmitter::WRegister rd, ARMEmitter::WRegister rm, ARMEmitter::ShiftType Shift = ARMEmitter::ShiftType::LSL, uint32_t amt = 0) {\n subs(rd, ARMEmitter::WReg::zr, rm, Shift, amt);\n }\n\n void add(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, ARMEmitter::Register rm,\n ARMEmitter::ShiftType Shift = ARMEmitter::ShiftType::LSL, uint32_t amt = 0) {\n LOGMAN_THROW_A_FMT(Shift != ARMEmitter::ShiftType::ROR, \"Doesn't support ROR\");\n constexpr uint32_t Op = 0b000'1011'000U << 21;\n DataProcessing_Shifted_Reg(Op, s, rd, rn, rm, Shift, amt);\n }\n void adds(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, ARMEmitter::Register rm,\n ARMEmitter::ShiftType Shift = ARMEmitter::ShiftType::LSL, uint32_t amt = 0) {\n LOGMAN_THROW_A_FMT(Shift != ARMEmitter::ShiftType::ROR, \"Doesn't support ROR\");\n constexpr uint32_t Op = 0b010'1011'000U << 21;\n DataProcessing_Shifted_Reg(Op, s, rd, rn, rm, Shift, amt);\n }\n void cmn(ARMEmitter::Size s, ARMEmitter::Register rn, ARMEmitter::Register rm, ARMEmitter::ShiftType Shift = ARMEmitter::ShiftType::LSL,\n uint32_t amt = 0) {\n adds(s, ARMEmitter::Reg::zr, rn, rm, Shift, amt);\n }\n void sub(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, ARMEmitter::Register rm,\n ARMEmitter::ShiftType Shift = ARMEmitter::ShiftType::LSL, uint32_t amt = 0) {\n LOGMAN_THROW_A_FMT(Shift != ARMEmitter::ShiftType::ROR, \"Doesn't support ROR\");\n constexpr uint32_t Op = 0b100'1011'000U << 21;\n DataProcessing_Shifted_Reg(Op, s, rd, rn, rm, Shift, amt);\n }\n void neg(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rm, ARMEmitter::ShiftType Shift = ARMEmitter::ShiftType::LSL,\n uint32_t amt = 0) {\n sub(s, rd, ARMEmitter::Reg::zr, rm, Shift, amt);\n }\n void cmp(ARMEmitter::Size s, ARMEmitter::Register rn, ARMEmitter::Register rm, ARMEmitter::ShiftType Shift = ARMEmitter::ShiftType::LSL,\n uint32_t amt = 0) {\n subs(s, ARMEmitter::Reg::zr, rn, rm, Shift, amt);\n }\n\n void subs(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, ARMEmitter::Register rm,\n ARMEmitter::ShiftType Shift = ARMEmitter::ShiftType::LSL, uint32_t amt = 0) {\n LOGMAN_THROW_A_FMT(Shift != ARMEmitter::ShiftType::ROR, \"Doesn't support ROR\");\n constexpr uint32_t Op = 0b110'1011'000U << 21;\n DataProcessing_Shifted_Reg(Op, s, rd, rn, rm, Shift, amt);\n }\n void negs(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rm, ARMEmitter::ShiftType Shift = ARMEmitter::ShiftType::LSL,\n uint32_t amt = 0) {\n subs(s, rd, ARMEmitter::Reg::zr, rm, Shift, amt);\n }\n\n // AddSub - extended register\n void add(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, ARMEmitter::Register rm, ARMEmitter::ExtendedType Option,\n uint32_t Shift = 0) {\n LOGMAN_THROW_A_FMT(Shift <= 4, \"Shift amount is too large\");\n constexpr uint32_t Op = 0b000'1011'001U << 21;\n DataProcessing_Extended_Reg(Op, s, rd, rn, rm, Option, Shift);\n }\n void adds(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, ARMEmitter::Register rm, ARMEmitter::ExtendedType Option,\n uint32_t Shift = 0) {\n constexpr uint32_t Op = 0b010'1011'001U << 21;\n DataProcessing_Extended_Reg(Op, s, rd, rn, rm, Option, Shift);\n }\n void cmn(ARMEmitter::Size s, ARMEmitter::Register rn, ARMEmitter::Register rm, ARMEmitter::ExtendedType Option, uint32_t Shift = 0) {\n adds(s, ARMEmitter::Reg::zr, rn, rm, Option, Shift);\n }\n void sub(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, ARMEmitter::Register rm, ARMEmitter::ExtendedType Option,\n uint32_t Shift = 0) {\n constexpr uint32_t Op = 0b100'1011'001U << 21;\n DataProcessing_Extended_Reg(Op, s, rd, rn, rm, Option, Shift);\n }\n void subs(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, ARMEmitter::Register rm, ARMEmitter::ExtendedType Option,\n uint32_t Shift = 0) {\n constexpr uint32_t Op = 0b110'1011'001U << 21;\n DataProcessing_Extended_Reg(Op, s, rd, rn, rm, Option, Shift);\n }\n void cmp(ARMEmitter::Size s, ARMEmitter::Register rn, ARMEmitter::Register rm, ARMEmitter::ExtendedType Option, uint32_t Shift = 0) {\n constexpr uint32_t Op = 0b110'1011'001U << 21;\n DataProcessing_Extended_Reg(Op, s, ARMEmitter::Reg::zr, rn, rm, Option, Shift);\n }\n\n // AddSub - with carry\n void adc(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, ARMEmitter::Register rm) {\n constexpr uint32_t Op = 0b0001'1010'000U << 21;\n DataProcessing_Extended_Reg(Op, s, rd, rn, rm, ARMEmitter::ExtendedType::UXTB, 0);\n }\n void adcs(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, ARMEmitter::Register rm) {\n constexpr uint32_t Op = 0b0011'1010'000U << 21;\n DataProcessing_Extended_Reg(Op, s, rd, rn, rm, ARMEmitter::ExtendedType::UXTB, 0);\n }\n void sbc(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, ARMEmitter::Register rm) {\n constexpr uint32_t Op = 0b0101'1010'000U << 21;\n DataProcessing_Extended_Reg(Op, s, rd, rn, rm, ARMEmitter::ExtendedType::UXTB, 0);\n }\n void sbcs(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, ARMEmitter::Register rm) {\n constexpr uint32_t Op = 0b0111'1010'000U << 21;\n DataProcessing_Extended_Reg(Op, s, rd, rn, rm, ARMEmitter::ExtendedType::UXTB, 0);\n }\n void ngc(ARMEmitter::Size s, Register rd, Register rm) {\n sbc(s, rd, Reg::zr, rm);\n }\n void ngcs(ARMEmitter::Size s, Register rd, Register rm) {\n sbcs(s, rd, Reg::zr, rm);\n }\n\n // Rotate right into flags\n void rmif(XRegister rn, uint32_t shift, uint32_t mask) {\n LOGMAN_THROW_A_FMT(shift <= 63, \"Shift must be within 0-63. Shift: {}\", shift);\n LOGMAN_THROW_A_FMT(mask <= 15, \"Mask must be within 0-15. Mask: {}\", mask);\n\n uint32_t Op = 0b1011'1010'0000'0000'0000'0100'0000'0000;\n Op |= rn.Idx() << 5;\n Op |= shift << 15;\n Op |= mask;\n\n dc32(Op);\n }\n\n // Evaluate into flags\n void setf8(WRegister rn) {\n constexpr uint32_t Op = 0b0011'1010'0000'0000'0000'1000'0000'1101;\n EvaluateIntoFlags(Op, 0, rn);\n }\n void setf16(WRegister rn) {\n constexpr uint32_t Op = 0b0011'1010'0000'0000'0000'1000'0000'1101;\n EvaluateIntoFlags(Op, 1, rn);\n }\n\n void cfinv() {\n constexpr uint32_t Op = 0b1101'0101'0000'0000'0100'0000'0001'1111;\n dc32(Op);\n }\n\n void axflag() {\n constexpr uint32_t Op = 0b1101'0101'0000'0000'0100'0000'0101'1111;\n dc32(Op);\n }\n\n void xaflag() {\n constexpr uint32_t Op = 0b1101'0101'0000'0000'0100'0000'0011'1111;\n dc32(Op);\n }\n\n // Conditional compare - register\n void ccmn(ARMEmitter::Size s, ARMEmitter::Register rn, ARMEmitter::Register rm, ARMEmitter::StatusFlags flags, ARMEmitter::Condition Cond) {\n constexpr uint32_t Op = 0b0011'1010'010 << 21;\n ConditionalCompare(Op, 0, 0b00, 0, s, rn, rm, flags, Cond);\n }\n void ccmp(ARMEmitter::Size s, ARMEmitter::Register rn, ARMEmitter::Register rm, ARMEmitter::StatusFlags flags, ARMEmitter::Condition Cond) {\n constexpr uint32_t Op = 0b0011'1010'010 << 21;\n ConditionalCompare(Op, 1, 0b00, 0, s, rn, rm, flags, Cond);\n }\n\n // Conditional compare - immediate\n void ccmn(ARMEmitter::Size s, ARMEmitter::Register rn, uint32_t rm, ARMEmitter::StatusFlags flags, ARMEmitter::Condition Cond) {\n LOGMAN_THROW_A_FMT((rm & ~0b1'1111) == 0, \"Comparison imm too large\");\n constexpr uint32_t Op = 0b0011'1010'010 << 21;\n ConditionalCompare(Op, 0, 0b10, 0, s, rn, rm, flags, Cond);\n }\n void ccmp(ARMEmitter::Size s, ARMEmitter::Register rn, uint32_t rm, ARMEmitter::StatusFlags flags, ARMEmitter::Condition Cond) {\n LOGMAN_THROW_A_FMT((rm & ~0b1'1111) == 0, \"Comparison imm too large\");\n constexpr uint32_t Op = 0b0011'1010'010 << 21;\n ConditionalCompare(Op, 1, 0b10, 0, s, rn, rm, flags, Cond);\n }\n\n // Conditional select\n void csel(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, ARMEmitter::Register rm, ARMEmitter::Condition Cond) {\n constexpr uint32_t Op = 0b0001'1010'100 << 21;\n ConditionalCompare(Op, 0, 0b00, s, rd, rn, rm, Cond);\n }\n void cset(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Condition Cond) {\n constexpr uint32_t Op = 0b0001'1010'100 << 21;\n ConditionalCompare(Op, 0, 0b01, s, rd, ARMEmitter::Reg::zr, ARMEmitter::Reg::zr,\n static_cast(FEXCore::ToUnderlying(Cond) ^ FEXCore::ToUnderlying(ARMEmitter::Condition::CC_NE)));\n }\n void csinc(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, ARMEmitter::Register rm, ARMEmitter::Condition Cond) {\n constexpr uint32_t Op = 0b0001'1010'100 << 21;\n ConditionalCompare(Op, 0, 0b01, s, rd, rn, rm, Cond);\n }\n void csinv(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, ARMEmitter::Register rm, ARMEmitter::Condition Cond) {\n constexpr uint32_t Op = 0b0001'1010'100 << 21;\n ConditionalCompare(Op, 1, 0b00, s, rd, rn, rm, Cond);\n }\n void csneg(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, ARMEmitter::Register rm, ARMEmitter::Condition Cond) {\n constexpr uint32_t Op = 0b0001'1010'100 << 21;\n ConditionalCompare(Op, 1, 0b01, s, rd, rn, rm, Cond);\n }\n void cneg(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, ARMEmitter::Condition Cond) {\n csneg(s, rd, rn, rn, InvertCondition(Cond));\n }\n void cinc(ARMEmitter::Size s, Register rd, Register rn, Condition cond) {\n csinc(s, rd, rn, rn, InvertCondition(cond));\n }\n void cinv(ARMEmitter::Size s, Register rd, Register rn, Condition cond) {\n csinv(s, rd, rn, rn, InvertCondition(cond));\n }\n void csetm(ARMEmitter::Size s, Register rd, Condition cond) {\n csinv(s, rd, Reg::zr, Reg::zr, InvertCondition(cond));\n }\n\n // Data processing - 3 source\n void madd(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, ARMEmitter::Register rm, ARMEmitter::Register ra) {\n constexpr uint32_t Op = 0b001'1011'000U << 21;\n DataProcessing_3Source(Op, 0, s, rd, rn, rm, ra);\n }\n void mul(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, ARMEmitter::Register rm) {\n madd(s, rd, rn, rm, XReg::zr);\n }\n void msub(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, ARMEmitter::Register rm, ARMEmitter::Register ra) {\n constexpr uint32_t Op = 0b001'1011'000U << 21;\n DataProcessing_3Source(Op, 1, s, rd, rn, rm, ra);\n }\n void mneg(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, ARMEmitter::Register rm) {\n msub(s, rd, rn, rm, XReg::zr);\n }\n void smaddl(ARMEmitter::XRegister rd, ARMEmitter::WRegister rn, ARMEmitter::WRegister rm, ARMEmitter::XRegister ra) {\n constexpr uint32_t Op = 0b001'1011'001U << 21;\n DataProcessing_3Source(Op, 0, ARMEmitter::Size::i64Bit, rd, rn, rm, ra);\n }\n void smull(ARMEmitter::XRegister rd, ARMEmitter::WRegister rn, ARMEmitter::WRegister rm) {\n smaddl(rd, rn, rm, XReg::zr);\n }\n void smsubl(ARMEmitter::XRegister rd, ARMEmitter::WRegister rn, ARMEmitter::WRegister rm, ARMEmitter::XRegister ra) {\n constexpr uint32_t Op = 0b001'1011'001U << 21;\n DataProcessing_3Source(Op, 1, ARMEmitter::Size::i64Bit, rd, rn, rm, ra);\n }\n void smnegl(ARMEmitter::XRegister rd, ARMEmitter::WRegister rn, ARMEmitter::WRegister rm) {\n smsubl(rd, rn, rm, XReg::zr);\n }\n void smulh(ARMEmitter::XRegister rd, ARMEmitter::XRegister rn, ARMEmitter::XRegister rm) {\n constexpr uint32_t Op = 0b001'1011'010U << 21;\n DataProcessing_3Source(Op, 0, ARMEmitter::Size::i64Bit, rd, rn, rm, ARMEmitter::Reg::zr);\n }\n void umaddl(ARMEmitter::XRegister rd, ARMEmitter::WRegister rn, ARMEmitter::WRegister rm, ARMEmitter::XRegister ra) {\n constexpr uint32_t Op = 0b001'1011'101U << 21;\n DataProcessing_3Source(Op, 0, ARMEmitter::Size::i64Bit, rd, rn, rm, ra);\n }\n void umull(ARMEmitter::XRegister rd, ARMEmitter::WRegister rn, ARMEmitter::WRegister rm) {\n umaddl(rd, rn, rm, XReg::zr);\n }\n void umsubl(ARMEmitter::XRegister rd, ARMEmitter::WRegister rn, ARMEmitter::WRegister rm, ARMEmitter::XRegister ra) {\n constexpr uint32_t Op = 0b001'1011'101U << 21;\n DataProcessing_3Source(Op, 1, ARMEmitter::Size::i64Bit, rd, rn, rm, ra);\n }\n void umnegl(ARMEmitter::XRegister rd, ARMEmitter::WRegister rn, ARMEmitter::WRegister rm) {\n umsubl(rd, rn, rm, XReg::zr);\n }\n void umulh(ARMEmitter::XRegister rd, ARMEmitter::XRegister rn, ARMEmitter::XRegister rm) {\n constexpr uint32_t Op = 0b001'1011'110U << 21;\n DataProcessing_3Source(Op, 0, ARMEmitter::Size::i64Bit, rd, rn, rm, ARMEmitter::Reg::zr);\n }\n\nprivate:\n void and_(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, uint32_t n, uint32_t immr, uint32_t imms) {\n constexpr uint32_t Op = 0b001'0010'00 << 22;\n DataProcessing_Logical_Imm(Op, s, rd, rn, n, immr, imms);\n }\n void ands(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, uint32_t n, uint32_t immr, uint32_t imms) {\n constexpr uint32_t Op = 0b111'0010'00 << 22;\n DataProcessing_Logical_Imm(Op, s, rd, rn, n, immr, imms);\n }\n void orr(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, uint32_t n, uint32_t immr, uint32_t imms) {\n constexpr uint32_t Op = 0b011'0010'00 << 22;\n DataProcessing_Logical_Imm(Op, s, rd, rn, n, immr, imms);\n }\n void eor(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, uint32_t n, uint32_t immr, uint32_t imms) {\n constexpr uint32_t Op = 0b101'0010'00 << 22;\n DataProcessing_Logical_Imm(Op, s, rd, rn, n, immr, imms);\n }\n\n void sbfm(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, uint32_t immr, uint32_t imms) {\n constexpr uint32_t Op = 0b0001'0011'00 << 22;\n DataProcessing_Logical_Imm(Op, s, rd, rn, s == ARMEmitter::Size::i64Bit, immr, imms);\n }\n void bfm(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, uint32_t immr, uint32_t imms) {\n constexpr uint32_t Op = 0b0011'0011'00 << 22;\n DataProcessing_Logical_Imm(Op, s, rd, rn, s == ARMEmitter::Size::i64Bit, immr, imms);\n }\n // 4.1.64 - Data processing - Immediate\n void DataProcessing_PCRel_Imm(uint32_t Op, ARMEmitter::Register rd, uint32_t Imm) {\n // Ensure the immediate is masked.\n Imm &= 0b1'1111'1111'1111'1111'1111U;\n\n uint32_t Instr = Op;\n\n Instr |= (Imm & 0b11) << 29;\n Instr |= (Imm >> 2) << 5;\n Instr |= Encode_rd(rd);\n\n dc32(Instr);\n }\n\n void DataProcessing_AddSub_Imm(uint32_t Op, ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, uint32_t Imm, bool LSL12) {\n bool TooLarge = (Imm & ~0b1111'1111'1111U) != 0;\n if (TooLarge && !LSL12 && ((Imm >> 12) & ~0b1111'1111'1111U) == 0) {\n // We can convert an immediate\n TooLarge = false;\n LSL12 = true;\n Imm >>= 12;\n }\n LOGMAN_THROW_A_FMT(TooLarge == false, \"Imm amount too large: 0x{:x}\", Imm);\n\n const uint32_t SF = s == ARMEmitter::Size::i64Bit ? (1U << 31) : 0;\n\n uint32_t Instr = Op;\n\n Instr |= SF;\n Instr |= LSL12 << 22;\n Instr |= Imm << 10;\n Instr |= Encode_rn(rn);\n Instr |= Encode_rd(rd);\n\n dc32(Instr);\n }\n\n // Min/max immediate\n void MinMaxImmediate(uint32_t opc, ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, uint64_t Imm) {\n const uint32_t SF = s == ARMEmitter::Size::i64Bit ? (1U << 31) : 0;\n\n uint32_t Instr = 0b1'0001'11U << 22;\n\n Instr |= SF;\n Instr |= opc << 18;\n Instr |= (Imm & 0xFF) << 10;\n Instr |= Encode_rn(rn);\n Instr |= Encode_rd(rd);\n\n dc32(Instr);\n }\n\n // Move Wide\n void DataProcessing_MoveWide(uint32_t Op, ARMEmitter::Size s, ARMEmitter::Register rd, uint32_t Imm, uint32_t Offset) {\n const uint32_t SF = s == ARMEmitter::Size::i64Bit ? (1U << 31) : 0;\n\n uint32_t Instr = Op;\n\n Instr |= SF;\n Instr |= Imm << 5;\n Instr |= Offset << 21;\n Instr |= Encode_rd(rd);\n\n dc32(Instr);\n }\n\n // Logical immediate\n void DataProcessing_Logical_Imm(uint32_t Op, ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, uint32_t n,\n uint32_t immr, uint32_t imms) {\n const uint32_t SF = s == ARMEmitter::Size::i64Bit ? (1U << 31) : 0;\n\n uint32_t Instr = Op;\n\n Instr |= SF;\n Instr |= n << 22;\n Instr |= immr << 16;\n Instr |= imms << 10;\n Instr |= Encode_rn(rn);\n Instr |= Encode_rd(rd);\n\n dc32(Instr);\n }\n\n void xbfiz_helper(bool is_signed, ARMEmitter::Size s, Register rd, Register rn, uint32_t lsb, uint32_t width) {\n const auto lsb_p_width = lsb + width;\n const auto reg_size_bits = RegSizeInBits(s);\n\n LOGMAN_THROW_A_FMT(lsb_p_width <= reg_size_bits, \"lsb + width ({}) must be <= {}. lsb={}, width={}\", lsb_p_width, reg_size_bits, lsb, width);\n LOGMAN_THROW_A_FMT(width >= 1, \"xbfiz width must be >= 1\");\n\n const auto immr = (reg_size_bits - lsb) & (reg_size_bits - 1);\n const auto imms = width - 1;\n\n if (is_signed) {\n sbfm(s, rd, rn, immr, imms);\n } else {\n ubfm(s, rd, rn, immr, imms);\n }\n }\n\n void DataProcessing_Extract(uint32_t Op, ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, ARMEmitter::Register rm,\n uint32_t Imm) {\n const uint32_t SF = s == ARMEmitter::Size::i64Bit ? (1U << 31) : 0;\n\n // Current ARMv8 spec hardcodes SF == N for this class of instructions.\n // Anythign else is undefined behaviour.\n const uint32_t N = s == ARMEmitter::Size::i64Bit ? (1U << 22) : 0;\n\n uint32_t Instr = Op;\n\n Instr |= SF;\n Instr |= N;\n Instr |= Encode_rm(rm);\n Instr |= Imm << 10;\n Instr |= Encode_rn(rn);\n Instr |= Encode_rd(rd);\n\n dc32(Instr);\n }\n\n // Data-processing - 2 source\n void DataProcessing_2Source(uint32_t Op, ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, ARMEmitter::Register rm) {\n const uint32_t SF = s == ARMEmitter::Size::i64Bit ? (1U << 31) : 0;\n\n uint32_t Instr = Op;\n\n Instr |= SF;\n Instr |= Encode_rm(rm);\n Instr |= Encode_rn(rn);\n Instr |= Encode_rd(rd);\n\n dc32(Instr);\n }\n\n // Data processing - 1 source\n template\n void DataProcessing_1Source(uint32_t Op, ARMEmitter::Size s, T rd, T rn) {\n const uint32_t SF = s == ARMEmitter::Size::i64Bit ? (1U << 31) : 0;\n\n uint32_t Instr = Op;\n\n Instr |= SF;\n Instr |= Encode_rn(rn);\n Instr |= Encode_rd(rd);\n\n dc32(Instr);\n }\n\n // AddSub - shifted register\n void DataProcessing_Shifted_Reg(uint32_t Op, ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn,\n ARMEmitter::Register rm, ARMEmitter::ShiftType Shift, uint32_t amt) {\n LOGMAN_THROW_A_FMT((amt & ~0b11'1111U) == 0, \"Shift amount too large\");\n if (s == ARMEmitter::Size::i32Bit) {\n LOGMAN_THROW_A_FMT(amt < 32, \"Shift amount for 32-bit must be below 32\");\n }\n\n const uint32_t SF = s == ARMEmitter::Size::i64Bit ? (1U << 31) : 0;\n\n uint32_t Instr = Op;\n\n Instr |= SF;\n Instr |= FEXCore::ToUnderlying(Shift) << 22;\n Instr |= Encode_rm(rm);\n Instr |= static_cast(amt) << 10;\n Instr |= Encode_rn(rn);\n Instr |= Encode_rd(rd);\n\n dc32(Instr);\n }\n\n // AddSub - extended register\n void DataProcessing_Extended_Reg(uint32_t Op, ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn,\n ARMEmitter::Register rm, ARMEmitter::ExtendedType Option, uint32_t Shift) {\n const uint32_t SF = s == ARMEmitter::Size::i64Bit ? (1U << 31) : 0;\n\n uint32_t Instr = Op;\n\n Instr |= SF;\n Instr |= Encode_rm(rm);\n Instr |= FEXCore::ToUnderlying(Option) << 13;\n Instr |= static_cast(Shift) << 10;\n Instr |= Encode_rn(rn);\n Instr |= Encode_rd(rd);\n\n dc32(Instr);\n }\n // Conditional compare - register\n template\n void ConditionalCompare(uint32_t Op, uint32_t o1, uint32_t o2, uint32_t o3, ARMEmitter::Size s, ARMEmitter::Register rn, T rm,\n ARMEmitter::StatusFlags flags, ARMEmitter::Condition Cond) {\n const uint32_t SF = s == ARMEmitter::Size::i64Bit ? (1U << 31) : 0;\n\n uint32_t Instr = Op;\n\n Instr |= SF;\n Instr |= o1 << 30;\n Instr |= Encode_rm(rm);\n Instr |= FEXCore::ToUnderlying(Cond) << 12;\n Instr |= o2 << 10;\n Instr |= Encode_rn(rn);\n Instr |= o3 << 4;\n Instr |= FEXCore::ToUnderlying(flags);\n\n dc32(Instr);\n }\n\n template\n void ConditionalCompare(uint32_t Op, uint32_t o1, uint32_t o2, ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, T rm,\n ARMEmitter::Condition Cond) {\n const uint32_t SF = s == ARMEmitter::Size::i64Bit ? (1U << 31) : 0;\n\n uint32_t Instr = Op;\n\n Instr |= SF;\n Instr |= o1 << 30;\n Instr |= Encode_rm(rm);\n Instr |= FEXCore::ToUnderlying(Cond) << 12;\n Instr |= o2 << 10;\n Instr |= Encode_rn(rn);\n Instr |= Encode_rd(rd);\n\n dc32(Instr);\n }\n\n // Data-processing - 3 source\n void DataProcessing_3Source(uint32_t Op, uint32_t Op0, ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn,\n ARMEmitter::Register rm, ARMEmitter::Register ra) {\n const uint32_t SF = s == ARMEmitter::Size::i64Bit ? (1U << 31) : 0;\n\n uint32_t Instr = Op;\n\n Instr |= SF;\n Instr |= Encode_rm(rm);\n Instr |= Op0 << 15;\n Instr |= Encode_ra(ra);\n Instr |= Encode_rn(rn);\n Instr |= Encode_rd(rd);\n\n dc32(Instr);\n }\n\n void EvaluateIntoFlags(uint32_t op, uint32_t size, WRegister rn) {\n uint32_t Instr = op;\n Instr |= size << 14;\n Instr |= rn.Idx() << 5;\n dc32(Instr);\n }\n\n#ifndef INCLUDED_BY_EMITTER\n}; // struct LoadstoreEmitterOps\n} // namespace ARMEmitter\n#endif\n" }, { "path": "CodeEmitter/CodeEmitter/ASIMDOps.inl", "content": "// SPDX-License-Identifier: MIT\n/* ASIMD instruction emitters.\n *\n * This contains emitters for vector operations explicitly.\n * Most instructions have a `SubRegSize` as their first argument to select element size while operating.\n * Additionally most emitters accept templated vector register arguments of both `QRegister` and `DRegister` types.\n * Based on the combination of those two arguments, it will emit an instruction operating on a 64-bit or 128-bit wide register\n * with the selected element size.\n *\n * Some vector operations are unsized and only operate at the one width. In these cases the instruction only\n * operates at one size, the width depends on the instruction.\n * The arguments for these instructions are usually `VRegister` but might be one of the other sized types as well.\n *\n * Only two instructions support the `i128Bit` ElementSize.\n */\n\n#pragma once\n#ifndef INCLUDED_BY_EMITTER\n#include \nnamespace ARMEmitter {\nstruct EmitterOps : Emitter {\n#endif\n\npublic:\n // Data Processing -- Scalar Floating-Point and Advanced SIMD\n // Cryptographic AES\n void aese(VRegister rd, VRegister rn) {\n CryptoAES(0b00100, rd, rn);\n }\n void aesd(VRegister rd, VRegister rn) {\n CryptoAES(0b00101, rd, rn);\n }\n void aesmc(VRegister rd, VRegister rn) {\n CryptoAES(0b00110, rd, rn);\n }\n void aesimc(VRegister rd, VRegister rn) {\n CryptoAES(0b00111, rd, rn);\n }\n\n // Cryptographic three-register SHA\n void sha1c(VRegister rd, SRegister rn, VRegister rm) {\n Crypto3RegSHA(0b000, rd, rn.V(), rm);\n }\n void sha1p(VRegister rd, SRegister rn, VRegister rm) {\n Crypto3RegSHA(0b001, rd, rn.V(), rm);\n }\n void sha1m(VRegister rd, SRegister rn, VRegister rm) {\n Crypto3RegSHA(0b010, rd, rn.V(), rm);\n }\n void sha1su0(VRegister rd, VRegister rn, VRegister rm) {\n Crypto3RegSHA(0b011, rd, rn, rm);\n }\n void sha256h(VRegister rd, VRegister rn, VRegister rm) {\n Crypto3RegSHA(0b100, rd, rn, rm);\n }\n void sha256h2(VRegister rd, VRegister rn, VRegister rm) {\n Crypto3RegSHA(0b101, rd, rn, rm);\n }\n void sha256su1(VRegister rd, VRegister rn, VRegister rm) {\n Crypto3RegSHA(0b110, rd, rn, rm);\n }\n\n // Cryptographic two-register SHA\n void sha1h(SRegister rd, SRegister rn) {\n Crypto2RegSHA(0b00000, rd.V(), rn.V());\n }\n void sha1su1(VRegister rd, VRegister rn) {\n Crypto2RegSHA(0b00001, rd, rn);\n }\n void sha256su0(VRegister rd, VRegister rn) {\n Crypto2RegSHA(0b00010, rd, rn);\n }\n // Advanced SIMD table lookup\n void tbl(QRegister rd, QRegister rn, QRegister rm) {\n ASIMDTable(1, 0b00, 0b00, 0b0, rd.V(), rn.V(), rm.V());\n }\n void tbl(DRegister rd, QRegister rn, DRegister rm) {\n ASIMDTable(0, 0b00, 0b00, 0b0, rd.V(), rn.V(), rm.V());\n }\n void tbx(QRegister rd, QRegister rn, QRegister rm) {\n ASIMDTable(1, 0b00, 0b00, 0b1, rd.V(), rn.V(), rm.V());\n }\n void tbx(DRegister rd, QRegister rn, DRegister rm) {\n ASIMDTable(0, 0b00, 0b00, 0b1, rd.V(), rn.V(), rm.V());\n }\n\n void tbl(QRegister rd, QRegister rn, QRegister rn2, QRegister rm) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rn, rn2), \"rn and rn2 must be sequential\");\n ASIMDTable(1, 0b00, 0b01, 0b0, rd.V(), rn.V(), rm.V());\n }\n void tbl(DRegister rd, QRegister rn, QRegister rn2, DRegister rm) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rn, rn2), \"rn and rn2 must be sequential\");\n ASIMDTable(0, 0b00, 0b01, 0b0, rd.V(), rn.V(), rm.V());\n }\n void tbx(QRegister rd, QRegister rn, QRegister rn2, QRegister rm) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rn, rn2), \"rn and rn2 must be sequential\");\n ASIMDTable(1, 0b00, 0b01, 0b1, rd.V(), rn.V(), rm.V());\n }\n void tbx(DRegister rd, QRegister rn, QRegister rn2, DRegister rm) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rn, rn2), \"rn and rn2 must be sequential\");\n ASIMDTable(0, 0b00, 0b01, 0b1, rd.V(), rn.V(), rm.V());\n }\n\n void tbl(QRegister rd, QRegister rn, QRegister rn2, QRegister rn3, QRegister rm) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rn, rn2, rn3), \"rn, rn2, and rn3 must be sequential\");\n ASIMDTable(1, 0b00, 0b10, 0b0, rd.V(), rn.V(), rm.V());\n }\n void tbl(DRegister rd, QRegister rn, QRegister rn2, QRegister rn3, DRegister rm) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rn, rn2, rn3), \"rn, rn2, and rn3 must be sequential\");\n ASIMDTable(0, 0b00, 0b10, 0b0, rd.V(), rn.V(), rm.V());\n }\n void tbx(QRegister rd, QRegister rn, QRegister rn2, QRegister rn3, QRegister rm) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rn, rn2, rn3), \"rn, rn2, and rn3 must be sequential\");\n ASIMDTable(1, 0b00, 0b10, 0b1, rd.V(), rn.V(), rm.V());\n }\n void tbx(DRegister rd, QRegister rn, QRegister rn2, QRegister rn3, DRegister rm) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rn, rn2, rn3), \"rn, rn2, and rn3 must be sequential\");\n ASIMDTable(0, 0b00, 0b10, 0b1, rd.V(), rn.V(), rm.V());\n }\n\n void tbl(QRegister rd, QRegister rn, QRegister rn2, QRegister rn3, QRegister rn4, QRegister rm) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rn, rn2, rn3, rn4), \"rn, rn2, rn3, and rn4 must be sequential\");\n ASIMDTable(1, 0b00, 0b11, 0b0, rd.V(), rn.V(), rm.V());\n }\n void tbl(DRegister rd, QRegister rn, QRegister rn2, QRegister rn3, QRegister rn4, DRegister rm) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rn, rn2, rn3, rn4), \"rn, rn2, rn3, and rn4 must be sequential\");\n ASIMDTable(0, 0b00, 0b11, 0b0, rd.V(), rn.V(), rm.V());\n }\n void tbx(QRegister rd, QRegister rn, QRegister rn2, QRegister rn3, QRegister rn4, QRegister rm) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rn, rn2, rn3, rn4), \"rn, rn2, rn3, and rn4 must be sequential\");\n ASIMDTable(1, 0b00, 0b11, 0b1, rd.V(), rn.V(), rm.V());\n }\n void tbx(DRegister rd, QRegister rn, QRegister rn2, QRegister rn3, QRegister rn4, DRegister rm) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rn, rn2, rn3, rn4), \"rn, rn2, rn3, and rn4 must be sequential\");\n ASIMDTable(0, 0b00, 0b11, 0b1, rd.V(), rn.V(), rm.V());\n }\n\n // Advanced SIMD permute\n void uzp1(SubRegSize size, QRegister rd, QRegister rn, QRegister rm) {\n ASIMDPermute(1, size, 0b001, rd.V(), rn.V(), rm.V());\n }\n void uzp1(SubRegSize size, DRegister rd, DRegister rn, DRegister rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid 64-bit size on 64-bit permute\");\n ASIMDPermute(0, size, 0b001, rd.V(), rn.V(), rm.V());\n }\n void trn1(SubRegSize size, QRegister rd, QRegister rn, QRegister rm) {\n ASIMDPermute(1, size, 0b010, rd.V(), rn.V(), rm.V());\n }\n void trn1(SubRegSize size, DRegister rd, DRegister rn, DRegister rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid 64-bit size on 64-bit permute\");\n ASIMDPermute(0, size, 0b010, rd.V(), rn.V(), rm.V());\n }\n void zip1(SubRegSize size, QRegister rd, QRegister rn, QRegister rm) {\n ASIMDPermute(1, size, 0b011, rd.V(), rn.V(), rm.V());\n }\n void zip1(SubRegSize size, DRegister rd, DRegister rn, DRegister rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid 64-bit size on 64-bit permute\");\n ASIMDPermute(0, size, 0b011, rd.V(), rn.V(), rm.V());\n }\n void uzp2(SubRegSize size, QRegister rd, QRegister rn, QRegister rm) {\n ASIMDPermute(1, size, 0b101, rd.V(), rn.V(), rm.V());\n }\n void uzp2(SubRegSize size, DRegister rd, DRegister rn, DRegister rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid 64-bit size on 64-bit permute\");\n ASIMDPermute(0, size, 0b101, rd.V(), rn.V(), rm.V());\n }\n void trn2(SubRegSize size, QRegister rd, QRegister rn, QRegister rm) {\n ASIMDPermute(1, size, 0b110, rd.V(), rn.V(), rm.V());\n }\n void trn2(SubRegSize size, DRegister rd, DRegister rn, DRegister rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid 64-bit size on 64-bit permute\");\n ASIMDPermute(0, size, 0b110, rd.V(), rn.V(), rm.V());\n }\n void zip2(SubRegSize size, QRegister rd, QRegister rn, QRegister rm) {\n ASIMDPermute(1, size, 0b111, rd.V(), rn.V(), rm.V());\n }\n void zip2(SubRegSize size, DRegister rd, DRegister rn, DRegister rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid 64-bit size on 64-bit permute\");\n ASIMDPermute(0, size, 0b111, rd.V(), rn.V(), rm.V());\n }\n\n // Advanced SIMD extract\n void ext(QRegister rd, QRegister rn, QRegister rm, uint32_t Index) {\n LOGMAN_THROW_A_FMT(Index < 16, \"Index can't be more than 15\");\n ASIMDExtract(1, 0b00, Index, rd.V(), rn.V(), rm.V());\n }\n void ext(DRegister rd, DRegister rn, DRegister rm, uint32_t Index) {\n LOGMAN_THROW_A_FMT(Index < 8, \"Index can't be more than 7\");\n ASIMDExtract(0, 0b00, Index, rd.V(), rn.V(), rm.V());\n }\n\n // Advanced SIMD copy\n template\n void dup(SubRegSize size, T rd, T rn, uint32_t Index) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid element size with 64-bit dup\");\n }\n\n constexpr uint32_t Q = std::is_same_v ? 1 : 0;\n\n const uint32_t SizeImm = FEXCore::ToUnderlying(size);\n const uint32_t IndexShift = SizeImm + 1;\n const uint32_t ElementSize = 1U << SizeImm;\n [[maybe_unused]] const uint32_t MaxIndex = 128U / (ElementSize * 8);\n\n LOGMAN_THROW_A_FMT(Index < MaxIndex, \"Index too large. Index={}, Max Index: {}\", Index, MaxIndex);\n\n const uint32_t imm5 = (Index << IndexShift) | ElementSize;\n\n ASIMDScalarCopy(Q, 0, imm5, 0b0000, rd.V(), rn.V());\n }\n\n template\n void dup(SubRegSize size, T rd, Register rn) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid element size with 64-bit dup\");\n }\n\n constexpr uint32_t Q = std::is_same_v ? 1 : 0;\n\n // Upper bits of imm5 are ignored for GPR dup\n const uint32_t imm5 = 1U << FEXCore::ToUnderlying(size);\n\n ASIMDScalarCopy(Q, 0, imm5, 0b0001, rd, ToVReg(rn));\n }\n\n template\n requires (size == SubRegSize::i8Bit || size == SubRegSize::i16Bit || size == SubRegSize::i32Bit)\n void smov(XRegister rd, VRegister rn, uint32_t Index) {\n static_assert(size == SubRegSize::i8Bit || size == SubRegSize::i16Bit || size == SubRegSize::i32Bit, \"Unsupported smov size\");\n\n constexpr uint32_t SizeImm = FEXCore::ToUnderlying(size);\n constexpr uint32_t IndexShift = SizeImm + 1;\n constexpr uint32_t ElementSize = 1U << SizeImm;\n [[maybe_unused]] constexpr uint32_t MaxIndex = 128U / (ElementSize * 8);\n\n LOGMAN_THROW_A_FMT(Index < MaxIndex, \"Index too large. Index={}, Max Index: {}\", Index, MaxIndex);\n\n const uint32_t imm5 = (Index << IndexShift) | ElementSize;\n\n ASIMDScalarCopy(1, 0, imm5, 0b0101, ToVReg(rd), rn);\n }\n template\n requires (size == SubRegSize::i8Bit || size == SubRegSize::i16Bit)\n void smov(WRegister rd, VRegister rn, uint32_t Index) {\n static_assert(size == SubRegSize::i8Bit || size == SubRegSize::i16Bit, \"Unsupported smov size\");\n\n constexpr uint32_t SizeImm = FEXCore::ToUnderlying(size);\n constexpr uint32_t IndexShift = SizeImm + 1;\n constexpr uint32_t ElementSize = 1U << SizeImm;\n [[maybe_unused]] constexpr uint32_t MaxIndex = 128U / (ElementSize * 8);\n\n LOGMAN_THROW_A_FMT(Index < MaxIndex, \"Index too large. Index={}, Max Index: {}\", Index, MaxIndex);\n\n const uint32_t imm5 = (Index << IndexShift) | ElementSize;\n\n ASIMDScalarCopy(0, 0, imm5, 0b0101, ToVReg(rd), rn);\n }\n\n template\n void umov(Register rd, VRegister rn, uint32_t Index) {\n static_assert(size == SubRegSize::i8Bit || size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit,\n \"Unsupported umov size\");\n\n constexpr uint32_t Q = size == SubRegSize::i64Bit ? 1 : 0;\n\n constexpr uint32_t SizeImm = FEXCore::ToUnderlying(size);\n constexpr uint32_t IndexShift = SizeImm + 1;\n constexpr uint32_t ElementSize = 1U << SizeImm;\n [[maybe_unused]] constexpr uint32_t MaxIndex = 128U / (ElementSize * 8);\n\n LOGMAN_THROW_A_FMT(Index < MaxIndex, \"Index too large. Index={}, Max Index: {}\", Index, MaxIndex);\n\n const uint32_t imm5 = (Index << IndexShift) | ElementSize;\n\n ASIMDScalarCopy(Q, 0, imm5, 0b0111, ToVReg(rd), rn);\n }\n\n void ins(SubRegSize size, VRegister rd, uint32_t Index, Register rn) {\n const uint32_t SizeImm = FEXCore::ToUnderlying(size);\n const uint32_t IndexShift = SizeImm + 1;\n const uint32_t ElementSize = 1U << SizeImm;\n [[maybe_unused]] const uint32_t MaxIndex = 128U / (ElementSize * 8);\n\n LOGMAN_THROW_A_FMT(Index < MaxIndex, \"Index too large. Index={}, Max Index: {}\", Index, MaxIndex);\n\n const uint32_t imm5 = (Index << IndexShift) | ElementSize;\n\n ASIMDScalarCopy(1, 0, imm5, 0b0011, rd, ToVReg(rn));\n }\n\n void ins(SubRegSize size, VRegister rd, uint32_t Index, VRegister rn, uint32_t Index2) {\n const uint32_t SizeImm = FEXCore::ToUnderlying(size);\n const uint32_t IndexShift = SizeImm + 1;\n const uint32_t ElementSize = 1U << SizeImm;\n [[maybe_unused]] const uint32_t MaxIndex = 128U / (ElementSize * 8);\n\n LOGMAN_THROW_A_FMT(Index < MaxIndex, \"Index too large. Index={}, Max Index: {}\", Index, MaxIndex);\n LOGMAN_THROW_A_FMT(Index2 < MaxIndex, \"Index2 too large. Index2={}, Max Index: {}\", Index2, MaxIndex);\n\n const uint32_t imm5 = (Index << IndexShift) | ElementSize;\n const uint32_t imm4 = Index2 << SizeImm;\n\n ASIMDScalarCopy(1, 0b10, imm5, imm4, rd, rn);\n }\n\n // Advanced SIMD three-register extension\n template\n void sdot(ARMEmitter::SubRegSize size, T rd, T rn, T rm) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != ARMEmitter::SubRegSize::i64Bit, \"64-bit subregsize not supported\");\n }\n ASIMDThreeRegisterExt(0, 0b0010, size, rm, rn, rd);\n }\n\n template\n void usdot(T rd, T rn, T rm) {\n ASIMDThreeRegisterExt(0, 0b0011, ARMEmitter::SubRegSize::i32Bit, rm, rn, rd);\n }\n\n template\n void sqrdmlah(ARMEmitter::SubRegSize size, T rd, T rn, T rm) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != ARMEmitter::SubRegSize::i64Bit, \"64-bit subregsize not supported\");\n }\n ASIMDThreeRegisterExt(1, 0b0000, size, rm, rn, rd);\n }\n template\n void sqrdmlsh(ARMEmitter::SubRegSize size, T rd, T rn, T rm) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != ARMEmitter::SubRegSize::i64Bit, \"64-bit subregsize not supported\");\n }\n ASIMDThreeRegisterExt(1, 0b0001, size, rm, rn, rd);\n }\n template\n void udot(ARMEmitter::SubRegSize size, T rd, T rn, T rm) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != ARMEmitter::SubRegSize::i64Bit, \"64-bit subregsize not supported\");\n }\n ASIMDThreeRegisterExt(1, 0b0010, size, rm, rn, rd);\n }\n\n template\n void fcmla(ARMEmitter::SubRegSize size, T rd, T rn, T rm, ARMEmitter::Rotation Rot) {\n LOGMAN_THROW_A_FMT(size != ARMEmitter::SubRegSize::i8Bit, \"8-bit subregsize not supported\");\n\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != ARMEmitter::SubRegSize::i64Bit, \"64-bit subregsize not supported\");\n }\n ASIMDThreeRegisterExt(1, 0b1000 | FEXCore::ToUnderlying(Rot), size, rm, rn, rd);\n }\n template\n void fcadd(ARMEmitter::SubRegSize size, T rd, T rn, T rm, ARMEmitter::Rotation Rot) {\n LOGMAN_THROW_A_FMT(size != ARMEmitter::SubRegSize::i8Bit, \"8-bit subregsize not supported\");\n\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != ARMEmitter::SubRegSize::i64Bit, \"64-bit subregsize not supported\");\n }\n LOGMAN_THROW_A_FMT(Rot == ARMEmitter::Rotation::ROTATE_90 || Rot == ARMEmitter::Rotation::ROTATE_270, \"Invalid rotation\");\n const uint32_t ConvertedRotation = Rot == ARMEmitter::Rotation::ROTATE_90 ? 0b00 : 0b10;\n ASIMDThreeRegisterExt(1, 0b1100 | ConvertedRotation, size, rm, rn, rd);\n }\n template\n void bfdot(T rd, T rn, T rm) {\n ASIMDThreeRegisterExt(1, 0b1111, ARMEmitter::SubRegSize::i16Bit, rm, rn, rd);\n }\n void bfmlalb(ARMEmitter::VRegister rd, ARMEmitter::VRegister rn, ARMEmitter::VRegister rm) {\n ASIMDThreeRegisterExt(1, 0b1111, ARMEmitter::SubRegSize::i64Bit, rm.D(), rn.D(), rd.D());\n }\n void bfmlalt(ARMEmitter::VRegister rd, ARMEmitter::VRegister rn, ARMEmitter::VRegister rm) {\n ASIMDThreeRegisterExt(1, 0b1111, ARMEmitter::SubRegSize::i64Bit, rm.Q(), rn.Q(), rd.Q());\n }\n void smmla(ARMEmitter::VRegister rd, ARMEmitter::VRegister rn, ARMEmitter::VRegister rm) {\n ASIMDThreeRegisterExt(0, 0b0100, ARMEmitter::SubRegSize::i32Bit, rm.Q(), rn.Q(), rd.Q());\n }\n void usmmla(ARMEmitter::VRegister rd, ARMEmitter::VRegister rn, ARMEmitter::VRegister rm) {\n ASIMDThreeRegisterExt(0, 0b0101, ARMEmitter::SubRegSize::i32Bit, rm.Q(), rn.Q(), rd.Q());\n }\n void bfmmla(ARMEmitter::VRegister rd, ARMEmitter::VRegister rn, ARMEmitter::VRegister rm) {\n ASIMDThreeRegisterExt(1, 0b1101, ARMEmitter::SubRegSize::i16Bit, rm.Q(), rn.Q(), rd.Q());\n }\n void ummla(ARMEmitter::VRegister rd, ARMEmitter::VRegister rn, ARMEmitter::VRegister rm) {\n ASIMDThreeRegisterExt(1, 0b0100, ARMEmitter::SubRegSize::i32Bit, rm.Q(), rn.Q(), rd.Q());\n }\n\n // Advanced SIMD two-register miscellaneous\n template\n void rev64(SubRegSize size, T rd, T rn) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"64-bit subregsize not supported\");\n ASIMD2RegMisc(0, size, 0b00000, rd, rn);\n }\n\n template\n void rev16(SubRegSize size, T rd, T rn) {\n LOGMAN_THROW_A_FMT(size == SubRegSize::i8Bit, \"Only 8-bit subregsize supported\");\n ASIMD2RegMisc(0, size, 0b00001, rd, rn);\n }\n\n ///< size is the destination size.\n ///< source size is the next size up.\n template\n void saddlp(SubRegSize size, T rd, T rn) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i8Bit, \"Destination 8-bit subregsize unsupported\");\n const auto ConvertedSize = SubRegSize {FEXCore::ToUnderlying(size) - 1};\n\n ASIMD2RegMisc(0, ConvertedSize, 0b00010, rd, rn);\n }\n\n template\n void suqadd(SubRegSize size, T rd, T rn) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"64-bit subregsize not supported\");\n }\n ASIMD2RegMisc(0, size, 0b00011, rd, rn);\n }\n\n template\n void cls(SubRegSize size, T rd, T rn) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"64-bit subregsize not supported\");\n ASIMD2RegMisc(0, size, 0b00100, rd, rn);\n }\n template\n void cnt(SubRegSize size, T rd, T rn) {\n LOGMAN_THROW_A_FMT(size == SubRegSize::i8Bit, \"Only 8-bit subregsize supported\");\n ASIMD2RegMisc(0, size, 0b00101, rd, rn);\n }\n\n ///< size is the destination size.\n ///< source size is the next size up.\n template\n void sadalp(SubRegSize size, T rd, T rn) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i8Bit, \"Destination 8-bit subregsize unsupported\");\n const auto ConvertedSize = SubRegSize {FEXCore::ToUnderlying(size) - 1};\n\n ASIMD2RegMisc(0, ConvertedSize, 0b00110, rd, rn);\n }\n\n template\n void sqabs(SubRegSize size, T rd, T rn) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"64-bit subregsize not supported\");\n }\n ASIMD2RegMisc(0, size, 0b00111, rd, rn);\n }\n // Comparison against zero\n template\n void cmgt(SubRegSize size, T rd, T rn) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"64-bit subregsize not supported\");\n }\n ASIMD2RegMisc(0, size, 0b01000, rd, rn);\n }\n // Comparison against zero\n template\n void cmeq(SubRegSize size, T rd, T rn) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"64-bit subregsize not supported\");\n }\n ASIMD2RegMisc(0, size, 0b01001, rd, rn);\n }\n // Comparison against zero\n template\n void cmlt(SubRegSize size, T rd, T rn) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"64-bit subregsize not supported\");\n }\n ASIMD2RegMisc(0, size, 0b01010, rd, rn);\n }\n template\n void abs(SubRegSize size, T rd, T rn) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"64-bit subregsize not supported\");\n }\n ASIMD2RegMisc(0, size, 0b01011, rd, rn);\n }\n\n ///< size is the destination size.\n ///< source size is the next size up.\n void xtn(SubRegSize size, VRegister rd, VRegister rn) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"64-bit destination subregsize not supported\");\n ASIMD2RegMisc(0, size, 0b10010, rd.D(), rn.D());\n }\n ///< size is the destination size.\n ///< source size is the next size up.\n void xtn2(SubRegSize size, VRegister rd, VRegister rn) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"64-bit destination subregsize not supported\");\n ASIMD2RegMisc(0, size, 0b10010, rd.Q(), rn.Q());\n }\n\n ///< size is the destination size.\n ///< source size is the next size up.\n void sqxtn(SubRegSize size, VRegister rd, VRegister rn) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"64-bit destination subregsize not supported\");\n ASIMD2RegMisc(0, size, 0b10100, rd.D(), rn.D());\n }\n ///< size is the destination size.\n ///< source size is the next size up.\n void sqxtn2(SubRegSize size, VRegister rd, VRegister rn) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"64-bit destination subregsize not supported\");\n ASIMD2RegMisc(0, size, 0b10100, rd.Q(), rn.Q());\n }\n\n ///< size is the destination size.\n ///< source size is the next size up.\n void fcvtn(SubRegSize size, VRegister rd, VRegister rn) {\n LOGMAN_THROW_A_FMT(size == SubRegSize::i32Bit || size == SubRegSize::i16Bit, \"Only 16-bit & 32-bit subregsize supported\");\n const auto ConvertedSize = size == SubRegSize::i32Bit ? SubRegSize::i16Bit : SubRegSize::i8Bit;\n\n ASIMD2RegMisc(0, ConvertedSize, 0b10110, rd.D(), rn.D());\n }\n ///< size is the destination size.\n ///< source size is the next size up.\n void fcvtn2(SubRegSize size, VRegister rd, VRegister rn) {\n LOGMAN_THROW_A_FMT(size == SubRegSize::i32Bit || size == SubRegSize::i16Bit, \"Only 16-bit & 32-bit subregsize supported\");\n const auto ConvertedSize = size == SubRegSize::i32Bit ? SubRegSize::i16Bit : SubRegSize::i8Bit;\n\n ASIMD2RegMisc(0, ConvertedSize, 0b10110, rd.Q(), rn.Q());\n }\n\n ///< size is the destination size.\n ///< source size is the next size up.\n void fcvtl(SubRegSize size, VRegister rd, VRegister rn) {\n LOGMAN_THROW_A_FMT(size == SubRegSize::i64Bit || size == SubRegSize::i32Bit, \"Only 32-bit & 64-bit subregsize supported\");\n const auto ConvertedSize = size == SubRegSize::i64Bit ? SubRegSize::i16Bit : SubRegSize::i8Bit;\n\n ASIMD2RegMisc(0, ConvertedSize, 0b10111, rd.D(), rn.D());\n }\n ///< size is the destination size.\n ///< source size is the next size up.\n void fcvtl2(SubRegSize size, VRegister rd, VRegister rn) {\n LOGMAN_THROW_A_FMT(size == SubRegSize::i64Bit || size == SubRegSize::i32Bit, \"Only 32-bit & 64-bit subregsize supported\");\n const auto ConvertedSize = size == SubRegSize::i64Bit ? SubRegSize::i16Bit : SubRegSize::i8Bit;\n\n ASIMD2RegMisc(0, ConvertedSize, 0b10111, rd.Q(), rn.Q());\n }\n\n template\n void frintn(SubRegSize size, T rd, T rn) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"64-bit subregsize not supported\");\n }\n LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), \"Only 16/32/64-bit subregsize supported\");\n\n if (size == SubRegSize::i16Bit) {\n ASIMDTwoRegMiscFP16(0, 0, 0b11000, rn, rd);\n } else {\n const auto ConvertedSize = size == SubRegSize::i64Bit ? SubRegSize::i16Bit : SubRegSize::i8Bit;\n ASIMD2RegMisc(0, ConvertedSize, 0b11000, rd, rn);\n }\n }\n template\n void frintm(SubRegSize size, T rd, T rn) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"64-bit subregsize not supported\");\n }\n LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), \"Only 16/32/64-bit subregsize supported\");\n\n if (size == SubRegSize::i16Bit) {\n ASIMDTwoRegMiscFP16(0, 0, 0b11001, rn, rd);\n } else {\n const auto ConvertedSize = size == SubRegSize::i64Bit ? SubRegSize::i16Bit : SubRegSize::i8Bit;\n ASIMD2RegMisc(0, ConvertedSize, 0b11001, rd, rn);\n }\n }\n\n template\n void fcvtns(SubRegSize size, T rd, T rn) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"64-bit subregsize not supported\");\n }\n LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), \"Only 16/32/64-bit subregsize supported\");\n\n if (size == SubRegSize::i16Bit) {\n ASIMDTwoRegMiscFP16(0, 0, 0b11010, rn, rd);\n } else {\n const auto ConvertedSize = size == SubRegSize::i64Bit ? SubRegSize::i16Bit : SubRegSize::i8Bit;\n ASIMD2RegMisc(0, ConvertedSize, 0b11010, rd, rn);\n }\n }\n template\n void fcvtms(SubRegSize size, T rd, T rn) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"64-bit subregsize not supported\");\n }\n LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), \"Only 16/32/64-bit subregsize supported\");\n\n if (size == SubRegSize::i16Bit) {\n ASIMDTwoRegMiscFP16(0, 0, 0b11011, rn, rd);\n } else {\n const auto ConvertedSize = size == SubRegSize::i64Bit ? SubRegSize::i16Bit : SubRegSize::i8Bit;\n ASIMD2RegMisc(0, ConvertedSize, 0b11011, rd, rn);\n }\n }\n template\n void fcvtas(SubRegSize size, T rd, T rn) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"64-bit subregsize not supported\");\n }\n LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), \"Only 16/32/64-bit subregsize supported\");\n\n if (size == SubRegSize::i16Bit) {\n ASIMDTwoRegMiscFP16(0, 0, 0b11100, rn, rd);\n } else {\n const auto ConvertedSize = size == SubRegSize::i64Bit ? SubRegSize::i16Bit : SubRegSize::i8Bit;\n ASIMD2RegMisc(0, ConvertedSize, 0b11100, rd, rn);\n }\n }\n template\n void scvtf(SubRegSize size, T rd, T rn) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"64-bit subregsize not supported\");\n }\n LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), \"Only 16/32/64-bit subregsize supported\");\n\n if (size == SubRegSize::i16Bit) {\n ASIMDTwoRegMiscFP16(0, 0, 0b11101, rn, rd);\n } else {\n const auto ConvertedSize = size == SubRegSize::i64Bit ? SubRegSize::i16Bit : SubRegSize::i8Bit;\n ASIMD2RegMisc(0, ConvertedSize, 0b11101, rd, rn);\n }\n }\n template\n void frint32z(SubRegSize size, T rd, T rn) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"64-bit subregsize not supported\");\n }\n LOGMAN_THROW_A_FMT(size == SubRegSize::i64Bit || size == SubRegSize::i32Bit, \"Only 32-bit & 64-bit subregsize supported\");\n const auto ConvertedSize = size == SubRegSize::i64Bit ? SubRegSize::i16Bit : SubRegSize::i8Bit;\n\n ASIMD2RegMisc(0, ConvertedSize, 0b11110, rd, rn);\n }\n template\n void frint64z(SubRegSize size, T rd, T rn) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"64-bit subregsize not supported\");\n }\n LOGMAN_THROW_A_FMT(size == SubRegSize::i64Bit || size == SubRegSize::i32Bit, \"Only 32-bit & 64-bit subregsize supported\");\n const auto ConvertedSize = size == SubRegSize::i64Bit ? SubRegSize::i16Bit : SubRegSize::i8Bit;\n\n ASIMD2RegMisc(0, ConvertedSize, 0b11111, rd, rn);\n }\n\n template\n void fcmgt(SubRegSize size, T rd, T rn) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"64-bit subregsize not supported\");\n }\n LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), \"Only 16/32/64-bit subregsize supported\");\n\n if (size == SubRegSize::i16Bit) {\n ASIMDTwoRegMiscFP16(0, 1, 0b01100, rn, rd);\n } else {\n ASIMD2RegMisc(0, size, 0b01100, rd, rn);\n }\n }\n\n template\n void fcmeq(SubRegSize size, T rd, T rn) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"64-bit subregsize not supported\");\n }\n LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), \"Only 16/32/64-bit subregsize supported\");\n\n if (size == SubRegSize::i16Bit) {\n ASIMDTwoRegMiscFP16(0, 1, 0b01101, rn, rd);\n } else {\n ASIMD2RegMisc(0, size, 0b01101, rd, rn);\n }\n }\n\n template\n void fcmlt(SubRegSize size, T rd, T rn) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"64-bit subregsize not supported\");\n }\n LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), \"Only 16/32/64-bit subregsize supported\");\n\n if (size == SubRegSize::i16Bit) {\n ASIMDTwoRegMiscFP16(0, 1, 0b01110, rn, rd);\n } else {\n ASIMD2RegMisc(0, size, 0b01110, rd, rn);\n }\n }\n\n template\n void fabs(SubRegSize size, T rd, T rn) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"64-bit subregsize not supported\");\n }\n LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), \"Only 16/32/64-bit subregsize supported\");\n\n if (size == SubRegSize::i16Bit) {\n ASIMDTwoRegMiscFP16(0, 1, 0b01111, rn, rd);\n } else {\n ASIMD2RegMisc(0, size, 0b01111, rd, rn);\n }\n }\n\n template\n void frintp(SubRegSize size, T rd, T rn) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"64-bit subregsize not supported\");\n }\n LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), \"Only 16/32/64-bit subregsize supported\");\n\n if (size == SubRegSize::i16Bit) {\n ASIMDTwoRegMiscFP16(0, 1, 0b11000, rn, rd);\n } else {\n ASIMD2RegMisc(0, size, 0b11000, rd, rn);\n }\n }\n\n template\n void frintz(SubRegSize size, T rd, T rn) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"64-bit subregsize not supported\");\n }\n LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), \"Only 16/32/64-bit subregsize supported\");\n\n if (size == SubRegSize::i16Bit) {\n ASIMDTwoRegMiscFP16(0, 1, 0b11001, rn, rd);\n } else {\n ASIMD2RegMisc(0, size, 0b11001, rd, rn);\n }\n }\n\n template\n void fcvtps(SubRegSize size, T rd, T rn) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"64-bit subregsize not supported\");\n }\n LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), \"Only 16/32/64-bit subregsize supported\");\n\n if (size == SubRegSize::i16Bit) {\n ASIMDTwoRegMiscFP16(0, 1, 0b11010, rn, rd);\n } else {\n ASIMD2RegMisc(0, size, 0b11010, rd, rn);\n }\n }\n\n template\n void fcvtzs(SubRegSize size, T rd, T rn) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"64-bit subregsize not supported\");\n }\n LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), \"Only 16/32/64-bit subregsize supported\");\n\n if (size == SubRegSize::i16Bit) {\n ASIMDTwoRegMiscFP16(0, 1, 0b11011, rn, rd);\n } else {\n ASIMD2RegMisc(0, size, 0b11011, rd, rn);\n }\n }\n\n template\n void urecpe(SubRegSize size, T rd, T rn) {\n LOGMAN_THROW_A_FMT(size == SubRegSize::i32Bit, \"Only 32-bit subregsize supported\");\n ASIMD2RegMisc(0, size, 0b11100, rd, rn);\n }\n\n template\n void frecpe(SubRegSize size, T rd, T rn) {\n LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), \"Only 16/32/64-bit subregsize supported\");\n\n if (size == SubRegSize::i16Bit) {\n ASIMDTwoRegMiscFP16(0, 1, 0b11101, rn, rd);\n } else {\n ASIMD2RegMisc(0, size, 0b11101, rd, rn);\n }\n }\n\n template\n void rev32(SubRegSize size, T rd, T rn) {\n LOGMAN_THROW_A_FMT(size == SubRegSize::i8Bit || size == SubRegSize::i16Bit, \"Only 8-bit & 16-bit subregsize supported\");\n ASIMD2RegMisc(1, size, 0b00000, rd, rn);\n }\n\n ///< size is the destination size.\n ///< source size is the next size up.\n template\n void uaddlp(SubRegSize size, T rd, T rn) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i8Bit, \"Destination 8-bit subregsize unsupported\");\n const auto ConvertedSize = SubRegSize {FEXCore::ToUnderlying(size) - 1};\n\n ASIMD2RegMisc(1, ConvertedSize, 0b00010, rd, rn);\n }\n\n template\n void usqadd(SubRegSize size, T rd, T rn) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"64-bit subregsize not supported\");\n }\n ASIMD2RegMisc(1, size, 0b00011, rd, rn);\n }\n\n template\n void clz(SubRegSize size, T rd, T rn) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"64-bit subregsize not supported\");\n ASIMD2RegMisc(1, size, 0b00100, rd, rn);\n }\n\n ///< size is the destination size.\n ///< source size is the next size up.\n template\n void uadalp(SubRegSize size, T rd, T rn) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i8Bit, \"Destination 8-bit subregsize unsupported\");\n const auto ConvertedSize = SubRegSize {FEXCore::ToUnderlying(size) - 1};\n\n ASIMD2RegMisc(1, ConvertedSize, 0b00110, rd, rn);\n }\n\n template\n void sqneg(SubRegSize size, T rd, T rn) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"64-bit subregsize not supported\");\n }\n ASIMD2RegMisc(1, size, 0b00111, rd, rn);\n }\n\n // Comparison against zero\n template\n void cmge(SubRegSize size, T rd, T rn) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"64-bit subregsize not supported\");\n }\n ASIMD2RegMisc(1, size, 0b01000, rd, rn);\n }\n // Comparison against zero\n template\n void cmle(SubRegSize size, T rd, T rn) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"64-bit subregsize not supported\");\n }\n ASIMD2RegMisc(1, size, 0b01001, rd, rn);\n }\n template\n void neg(SubRegSize size, T rd, T rn) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"64-bit subregsize not supported\");\n }\n ASIMD2RegMisc(1, size, 0b01011, rd, rn);\n }\n ///< size is the destination size.\n ///< source size is the next size up.\n void sqxtun(SubRegSize size, VRegister rd, VRegister rn) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"64-bit destination subregsize not supported\");\n ASIMD2RegMisc(1, size, 0b10010, rd.D(), rn.D());\n }\n ///< size is the destination size.\n ///< source size is the next size up.\n void sqxtun2(SubRegSize size, VRegister rd, VRegister rn) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"64-bit destination subregsize not supported\");\n ASIMD2RegMisc(1, size, 0b10010, rd.Q(), rn.Q());\n }\n\n ///< size is the destination size.\n ///< source size is the next size up.\n void shll(SubRegSize size, DRegister rd, DRegister rn) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i8Bit, \"Destination 8-bit subregsize unsupported\");\n const auto ConvertedSize = SubRegSize {FEXCore::ToUnderlying(size) - 1};\n\n ASIMD2RegMisc(1, ConvertedSize, 0b10011, rd, rn);\n }\n ///< size is the destination size.\n ///< source size is the next size up.\n void shll2(SubRegSize size, QRegister rd, QRegister rn) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i8Bit, \"Destination 8-bit subregsize unsupported\");\n const auto ConvertedSize = SubRegSize {FEXCore::ToUnderlying(size) - 1};\n\n ASIMD2RegMisc(1, ConvertedSize, 0b10011, rd, rn);\n }\n ///< size is the destination size.\n ///< source size is the next size up.\n void uqxtn(SubRegSize size, VRegister rd, VRegister rn) {\n ASIMD2RegMisc(1, size, 0b10100, rd.D(), rn.D());\n }\n ///< size is the destination size.\n ///< source size is the next size up.\n void uqxtn2(SubRegSize size, VRegister rd, VRegister rn) {\n ASIMD2RegMisc(1, size, 0b10100, rd.Q(), rn.Q());\n }\n ///< size is the destination size.\n ///< source size is the next size up.\n void fcvtxn(SubRegSize size, VRegister rd, VRegister rn) {\n LOGMAN_THROW_A_FMT(size == SubRegSize::i32Bit, \"Only 32-bit subregsize supported\");\n const auto ConvertedSize = size == SubRegSize::i32Bit ? SubRegSize::i16Bit : SubRegSize::i8Bit;\n\n ASIMD2RegMisc(1, ConvertedSize, 0b10110, rd.D(), rn.D());\n }\n ///< size is the destination size.\n ///< source size is the next size up.\n void fcvtxn2(SubRegSize size, VRegister rd, VRegister rn) {\n LOGMAN_THROW_A_FMT(size == SubRegSize::i32Bit, \"Only 32-bit subregsize supported\");\n const auto ConvertedSize = size == SubRegSize::i32Bit ? SubRegSize::i16Bit : SubRegSize::i8Bit;\n\n ASIMD2RegMisc(1, ConvertedSize, 0b10110, rd.Q(), rn.Q());\n }\n template\n void frinta(SubRegSize size, T rd, T rn) {\n LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), \"Only 16/32/64-bit subregsize supported\");\n\n if (size == SubRegSize::i16Bit) {\n ASIMDTwoRegMiscFP16(1, 0, 0b11000, rn, rd);\n } else {\n const auto ConvertedSize = size == SubRegSize::i64Bit ? SubRegSize::i16Bit : SubRegSize::i8Bit;\n ASIMD2RegMisc(1, ConvertedSize, 0b11000, rd, rn);\n }\n }\n template\n void frintx(SubRegSize size, T rd, T rn) {\n LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), \"Only 16/32/64-bit subregsize supported\");\n\n if (size == SubRegSize::i16Bit) {\n ASIMDTwoRegMiscFP16(1, 0, 0b11001, rn, rd);\n } else {\n const auto ConvertedSize = size == SubRegSize::i64Bit ? SubRegSize::i16Bit : SubRegSize::i8Bit;\n ASIMD2RegMisc(1, ConvertedSize, 0b11001, rd, rn);\n }\n }\n\n template\n void fcvtnu(SubRegSize size, T rd, T rn) {\n LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), \"Only 16/32/64-bit subregsize supported\");\n\n if (size == SubRegSize::i16Bit) {\n ASIMDTwoRegMiscFP16(1, 0, 0b11010, rn, rd);\n } else {\n const auto ConvertedSize = size == SubRegSize::i64Bit ? SubRegSize::i16Bit : SubRegSize::i8Bit;\n ASIMD2RegMisc(1, ConvertedSize, 0b11010, rd, rn);\n }\n }\n template\n void fcvtmu(SubRegSize size, T rd, T rn) {\n LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), \"Only 16/32/64-bit subregsize supported\");\n\n if (size == SubRegSize::i16Bit) {\n ASIMDTwoRegMiscFP16(1, 0, 0b11011, rn, rd);\n } else {\n const auto ConvertedSize = size == SubRegSize::i64Bit ? SubRegSize::i16Bit : SubRegSize::i8Bit;\n ASIMD2RegMisc(1, ConvertedSize, 0b11011, rd, rn);\n }\n }\n template\n void fcvtau(SubRegSize size, T rd, T rn) {\n LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), \"Only 16/32/64-bit subregsize supported\");\n\n if (size == SubRegSize::i16Bit) {\n ASIMDTwoRegMiscFP16(1, 0, 0b11100, rn, rd);\n } else {\n const auto ConvertedSize = size == SubRegSize::i64Bit ? SubRegSize::i16Bit : SubRegSize::i8Bit;\n ASIMD2RegMisc(1, ConvertedSize, 0b11100, rd, rn);\n }\n }\n template\n void ucvtf(SubRegSize size, T rd, T rn) {\n LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), \"Only 16/32/64-bit subregsize supported\");\n\n if (size == SubRegSize::i16Bit) {\n ASIMDTwoRegMiscFP16(1, 0, 0b11101, rn, rd);\n } else {\n const auto ConvertedSize = size == SubRegSize::i64Bit ? SubRegSize::i16Bit : SubRegSize::i8Bit;\n ASIMD2RegMisc(1, ConvertedSize, 0b11101, rd, rn);\n }\n }\n template\n void frint32x(SubRegSize size, T rd, T rn) {\n LOGMAN_THROW_A_FMT(size == SubRegSize::i64Bit || size == SubRegSize::i32Bit, \"Only 32-bit & 64-bit subregsize supported\");\n const auto ConvertedSize = size == SubRegSize::i64Bit ? SubRegSize::i16Bit : SubRegSize::i8Bit;\n\n ASIMD2RegMisc(1, ConvertedSize, 0b11110, rd, rn);\n }\n template\n void frint64x(SubRegSize size, T rd, T rn) {\n LOGMAN_THROW_A_FMT(size == SubRegSize::i64Bit || size == SubRegSize::i32Bit, \"Only 32-bit & 64-bit subregsize supported\");\n const auto ConvertedSize = size == SubRegSize::i64Bit ? SubRegSize::i16Bit : SubRegSize::i8Bit;\n\n ASIMD2RegMisc(1, ConvertedSize, 0b11111, rd, rn);\n }\n\n template\n void not_(SubRegSize size, T rd, T rn) {\n LOGMAN_THROW_A_FMT(size == SubRegSize::i8Bit, \"Only 8-bit subregsize supported\");\n ASIMD2RegMisc(1, SubRegSize::i8Bit, 0b00101, rd, rn);\n }\n template\n void mvn(SubRegSize size, T rd, T rn) {\n LOGMAN_THROW_A_FMT(size == SubRegSize::i8Bit, \"Only 8-bit subregsize supported\");\n ASIMD2RegMisc(1, SubRegSize::i8Bit, 0b00101, rd, rn);\n }\n\n template\n void rbit(SubRegSize size, T rd, T rn) {\n LOGMAN_THROW_A_FMT(size == SubRegSize::i8Bit, \"Only 8-bit subregsize supported\");\n ASIMD2RegMisc(1, SubRegSize::i16Bit, 0b00101, rd, rn);\n }\n\n template\n void fcmge(SubRegSize size, T rd, T rn) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"64-bit subregsize not supported\");\n }\n LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), \"Only 16/32/64-bit subregsize supported\");\n\n if (size == SubRegSize::i16Bit) {\n ASIMDTwoRegMiscFP16(1, 1, 0b01100, rn, rd);\n } else {\n ASIMD2RegMisc(1, size, 0b01100, rd, rn);\n }\n }\n template\n void fcmle(SubRegSize size, T rd, T rn) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"64-bit subregsize not supported\");\n }\n LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), \"Only 16/32/64-bit subregsize supported\");\n\n if (size == SubRegSize::i16Bit) {\n ASIMDTwoRegMiscFP16(1, 1, 0b01101, rn, rd);\n } else {\n ASIMD2RegMisc(1, size, 0b01101, rd, rn);\n }\n }\n\n template\n void fneg(SubRegSize size, T rd, T rn) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"64-bit subregsize not supported\");\n }\n LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), \"Only 16/32/64-bit subregsize supported\");\n\n if (size == SubRegSize::i16Bit) {\n ASIMDTwoRegMiscFP16(1, 1, 0b01111, rn, rd);\n } else {\n ASIMD2RegMisc(1, size, 0b01111, rd, rn);\n }\n }\n\n template\n void frinti(SubRegSize size, T rd, T rn) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"64-bit subregsize not supported\");\n }\n LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), \"Only 16/32/64-bit subregsize supported\");\n\n if (size == SubRegSize::i16Bit) {\n ASIMDTwoRegMiscFP16(1, 1, 0b11001, rn, rd);\n } else {\n ASIMD2RegMisc(1, size, 0b11001, rd, rn);\n }\n }\n\n template\n void fcvtpu(SubRegSize size, T rd, T rn) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"64-bit subregsize not supported\");\n }\n LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), \"Only 16/32/64-bit subregsize supported\");\n\n if (size == SubRegSize::i16Bit) {\n ASIMDTwoRegMiscFP16(1, 1, 0b11010, rn, rd);\n } else {\n ASIMD2RegMisc(1, size, 0b11010, rd, rn);\n }\n }\n template\n void fcvtzu(SubRegSize size, T rd, T rn) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"64-bit subregsize not supported\");\n }\n LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), \"Only 16/32/64-bit subregsize supported\");\n\n if (size == SubRegSize::i16Bit) {\n ASIMDTwoRegMiscFP16(1, 1, 0b11011, rn, rd);\n } else {\n ASIMD2RegMisc(1, size, 0b11011, rd, rn);\n }\n }\n\n template\n void ursqrte(SubRegSize size, T rd, T rn) {\n LOGMAN_THROW_A_FMT(size == SubRegSize::i32Bit, \"Only 32-bit & 64-bit subregsize supported\");\n ASIMD2RegMisc(1, size, 0b11100, rd, rn);\n }\n template\n void frsqrte(SubRegSize size, T rd, T rn) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"64-bit subregsize not supported\");\n }\n LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), \"Only 16/32/64-bit subregsize supported\");\n\n if (size == SubRegSize::i16Bit) {\n ASIMDTwoRegMiscFP16(1, 1, 0b11101, rn, rd);\n } else {\n ASIMD2RegMisc(1, size, 0b11101, rd, rn);\n }\n }\n\n template\n void fsqrt(SubRegSize size, T rd, T rn) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"64-bit subregsize not supported\");\n }\n LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), \"Only 16/32/64-bit subregsize supported\");\n\n if (size == SubRegSize::i16Bit) {\n ASIMDTwoRegMiscFP16(1, 1, 0b11111, rn, rd);\n } else {\n ASIMD2RegMisc(1, size, 0b11111, rd, rn);\n }\n }\n\n // Advanced SIMD across lanes\n ///< size is the destination size.\n ///< source size is the next size up.\n template\n void saddlv(SubRegSize size, T rd, T rn) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"64-bit subregsize not supported\");\n }\n LOGMAN_THROW_A_FMT(size != SubRegSize::i8Bit, \"Destination 8-bit subregsize unsupported\");\n const auto ConvertedSize = SubRegSize {FEXCore::ToUnderlying(size) - 1};\n\n ASIMDAcrossLanes(0, ConvertedSize, 0b00011, rd, rn);\n }\n\n template\n void smaxv(SubRegSize size, T rd, T rn) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i32Bit && size != SubRegSize::i64Bit, \"32/64-bit subregsize not supported\");\n }\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Destination 64-bit subregsize unsupported\");\n ASIMDAcrossLanes(0, size, 0b01010, rd, rn);\n }\n template\n void sminv(SubRegSize size, T rd, T rn) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i32Bit && size != SubRegSize::i64Bit, \"32/64-bit subregsize not supported\");\n }\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Destination 64-bit subregsize unsupported\");\n ASIMDAcrossLanes(0, size, 0b11010, rd, rn);\n }\n template\n void addv(SubRegSize size, T rd, T rn) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i32Bit && size != SubRegSize::i64Bit, \"32/64-bit subregsize not supported\");\n }\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Destination 64-bit subregsize unsupported\");\n ASIMDAcrossLanes(0, size, 0b11011, rd, rn);\n }\n template\n void uaddlv(SubRegSize size, T rd, T rn) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"64-bit subregsize not supported\");\n }\n LOGMAN_THROW_A_FMT(size != SubRegSize::i8Bit, \"Destination 8-bit subregsize unsupported\");\n const auto ConvertedSize = SubRegSize {FEXCore::ToUnderlying(size) - 1};\n\n ASIMDAcrossLanes(1, ConvertedSize, 0b00011, rd, rn);\n }\n template\n void umaxv(SubRegSize size, T rd, T rn) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i32Bit && size != SubRegSize::i64Bit, \"32/64-bit subregsize not supported\");\n }\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Destination 64-bit subregsize unsupported\");\n ASIMDAcrossLanes(1, size, 0b01010, rd, rn);\n }\n template\n void uminv(SubRegSize size, T rd, T rn) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i32Bit && size != SubRegSize::i64Bit, \"32/64-bit subregsize not supported\");\n }\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Destination 64-bit subregsize unsupported\");\n ASIMDAcrossLanes(1, size, 0b11010, rd, rn);\n }\n template\n void fmaxnmv(SubRegSize size, T rd, T rn) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i32Bit, \"32-bit subregsize not supported\");\n }\n LOGMAN_THROW_A_FMT(size != SubRegSize::i8Bit && size != SubRegSize::i64Bit, \"Destination 8/64-bit subregsize unsupported\");\n const auto ConvertedSize = size == SubRegSize::i64Bit ? SubRegSize::i16Bit : SubRegSize::i8Bit;\n const auto U = size == SubRegSize::i16Bit ? 0 : 1;\n\n ASIMDAcrossLanes(U, ConvertedSize, 0b01100, rd, rn);\n }\n template\n void fmaxv(SubRegSize size, T rd, T rn) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i32Bit, \"32-bit subregsize not supported\");\n }\n LOGMAN_THROW_A_FMT(size != SubRegSize::i8Bit && size != SubRegSize::i64Bit, \"Destination 8/64-bit subregsize unsupported\");\n const auto ConvertedSize = size == SubRegSize::i64Bit ? SubRegSize::i16Bit : SubRegSize::i8Bit;\n const auto U = size == ARMEmitter::SubRegSize::i16Bit ? 0 : 1;\n\n ASIMDAcrossLanes(U, ConvertedSize, 0b01111, rd, rn);\n }\n template\n void fminnmv(SubRegSize size, T rd, T rn) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i32Bit, \"32-bit subregsize not supported\");\n }\n LOGMAN_THROW_A_FMT(size != SubRegSize::i8Bit && size != SubRegSize::i64Bit, \"Destination 8/64-bit subregsize unsupported\");\n const auto ConvertedSize = size == SubRegSize::i64Bit ? SubRegSize::i64Bit : SubRegSize::i32Bit;\n const auto U = size == SubRegSize::i16Bit ? 0 : 1;\n\n ASIMDAcrossLanes(U, ConvertedSize, 0b01100, rd, rn);\n }\n template\n void fminv(SubRegSize size, T rd, T rn) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i32Bit, \"32-bit subregsize not supported\");\n }\n LOGMAN_THROW_A_FMT(size != SubRegSize::i8Bit && size != SubRegSize::i64Bit, \"Destination 8/64-bit subregsize unsupported\");\n const auto ConvertedSize = size == SubRegSize::i64Bit ? SubRegSize::i64Bit : SubRegSize::i32Bit;\n const auto U = size == SubRegSize::i16Bit ? 0 : 1;\n\n ASIMDAcrossLanes(U, ConvertedSize, 0b01111, rd, rn);\n }\n\n // Advanced SIMD three different\n // TODO: Double check narrowing op size limits.\n // TODO: Don't enforce DRegister/QRegister for Q check\n ///< Size is dest size\n void saddl(SubRegSize size, DRegister rd, DRegister rn, DRegister rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i8Bit, \"No 8-bit dest support.\");\n const auto ConvertedSize = SubRegSize {FEXCore::ToUnderlying(size) - 1};\n\n ASIMD3Different(0, 0b0000, ConvertedSize, rd, rn, rm);\n }\n ///< Size is dest size\n void saddl2(SubRegSize size, QRegister rd, QRegister rn, QRegister rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i8Bit, \"No 8-bit dest support.\");\n const auto ConvertedSize = SubRegSize {FEXCore::ToUnderlying(size) - 1};\n\n ASIMD3Different(0, 0b0000, ConvertedSize, rd, rn, rm);\n }\n ///< Size is dest size\n void saddw(SubRegSize size, DRegister rd, DRegister rn, DRegister rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i8Bit, \"No 8-bit dest support.\");\n const auto ConvertedSize = SubRegSize {FEXCore::ToUnderlying(size) - 1};\n\n ASIMD3Different(0, 0b0001, ConvertedSize, rd, rn, rm);\n }\n ///< Size is dest size\n void saddw2(SubRegSize size, QRegister rd, QRegister rn, QRegister rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i8Bit, \"No 8-bit dest support.\");\n const auto ConvertedSize = SubRegSize {FEXCore::ToUnderlying(size) - 1};\n\n ASIMD3Different(0, 0b0001, ConvertedSize, rd, rn, rm);\n }\n ///< Size is dest size\n void ssubl(SubRegSize size, DRegister rd, DRegister rn, DRegister rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i8Bit, \"No 8-bit dest support.\");\n const auto ConvertedSize = SubRegSize {FEXCore::ToUnderlying(size) - 1};\n\n ASIMD3Different(0, 0b0010, ConvertedSize, rd, rn, rm);\n }\n ///< Size is dest size\n void ssubl2(SubRegSize size, QRegister rd, QRegister rn, QRegister rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i8Bit, \"No 8-bit dest support.\");\n const auto ConvertedSize = SubRegSize {FEXCore::ToUnderlying(size) - 1};\n\n ASIMD3Different(0, 0b0010, ConvertedSize, rd, rn, rm);\n }\n ///< Size is dest size\n void ssubw(SubRegSize size, DRegister rd, DRegister rn, DRegister rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i8Bit, \"No 8-bit dest support.\");\n const auto ConvertedSize = SubRegSize {FEXCore::ToUnderlying(size) - 1};\n\n ASIMD3Different(0, 0b0011, ConvertedSize, rd, rn, rm);\n }\n ///< Size is dest size\n void ssubw2(SubRegSize size, QRegister rd, QRegister rn, QRegister rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i8Bit, \"No 8-bit dest support.\");\n const auto ConvertedSize = SubRegSize {FEXCore::ToUnderlying(size) - 1};\n\n ASIMD3Different(0, 0b0011, ConvertedSize, rd, rn, rm);\n }\n void addhn(SubRegSize size, DRegister rd, DRegister rn, DRegister rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"No 64-bit dest support.\");\n ASIMD3Different(0, 0b0100, size, rd, rn, rm);\n }\n void addhn2(SubRegSize size, QRegister rd, QRegister rn, QRegister rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"No 64-bit dest support.\");\n ASIMD3Different(0, 0b0100, size, rd, rn, rm);\n }\n ///< Size is dest size\n void sabal(SubRegSize size, DRegister rd, DRegister rn, DRegister rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i8Bit, \"No 8-bit dest support.\");\n const auto ConvertedSize = SubRegSize {FEXCore::ToUnderlying(size) - 1};\n\n ASIMD3Different(0, 0b0101, ConvertedSize, rd, rn, rm);\n }\n ///< Size is dest size\n void sabal2(SubRegSize size, QRegister rd, QRegister rn, QRegister rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i8Bit, \"No 8-bit dest support.\");\n\n const auto ConvertedSize = SubRegSize {FEXCore::ToUnderlying(size) - 1};\n ASIMD3Different(0, 0b0101, ConvertedSize, rd, rn, rm);\n }\n void subhn(SubRegSize size, DRegister rd, DRegister rn, DRegister rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"No 64-bit dest support.\");\n ASIMD3Different(0, 0b0110, size, rd, rn, rm);\n }\n void subhn2(SubRegSize size, QRegister rd, QRegister rn, QRegister rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"No 64-bit dest support.\");\n ASIMD3Different(0, 0b0110, size, rd, rn, rm);\n }\n ///< Size is dest size\n void sabdl(SubRegSize size, DRegister rd, DRegister rn, DRegister rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i8Bit, \"No 8-bit dest support.\");\n const auto ConvertedSize = SubRegSize {FEXCore::ToUnderlying(size) - 1};\n\n ASIMD3Different(0, 0b0111, ConvertedSize, rd, rn, rm);\n }\n ///< Size is dest size\n void sabdl2(SubRegSize size, QRegister rd, QRegister rn, QRegister rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i8Bit, \"No 8-bit dest support.\");\n const auto ConvertedSize = SubRegSize {FEXCore::ToUnderlying(size) - 1};\n\n ASIMD3Different(0, 0b0111, ConvertedSize, rd, rn, rm);\n }\n ///< Size is dest size\n void smlal(SubRegSize size, DRegister rd, DRegister rn, DRegister rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i8Bit, \"No 8-bit dest support.\");\n const auto ConvertedSize = SubRegSize {FEXCore::ToUnderlying(size) - 1};\n\n ASIMD3Different(0, 0b1000, ConvertedSize, rd, rn, rm);\n }\n ///< Size is dest size\n void smlal2(SubRegSize size, QRegister rd, QRegister rn, QRegister rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i8Bit, \"No 8-bit dest support.\");\n const auto ConvertedSize = SubRegSize {FEXCore::ToUnderlying(size) - 1};\n\n ASIMD3Different(0, 0b1000, ConvertedSize, rd, rn, rm);\n }\n ///< Size is dest size\n void sqdmlal(SubRegSize size, DRegister rd, DRegister rn, DRegister rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i8Bit && size != SubRegSize::i16Bit, \"No 8/16-bit dest support.\");\n const auto ConvertedSize = SubRegSize {FEXCore::ToUnderlying(size) - 1};\n\n ASIMD3Different(0, 0b1001, ConvertedSize, rd, rn, rm);\n }\n ///< Size is dest size\n void sqdmlal2(SubRegSize size, QRegister rd, QRegister rn, QRegister rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i8Bit && size != SubRegSize::i16Bit, \"No 8/16-bit dest support.\");\n const auto ConvertedSize = SubRegSize {FEXCore::ToUnderlying(size) - 1};\n\n ASIMD3Different(0, 0b1001, ConvertedSize, rd, rn, rm);\n }\n ///< Size is dest size\n void smlsl(SubRegSize size, DRegister rd, DRegister rn, DRegister rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i8Bit, \"No 8-bit dest support.\");\n const auto ConvertedSize = SubRegSize {FEXCore::ToUnderlying(size) - 1};\n\n ASIMD3Different(0, 0b1010, ConvertedSize, rd, rn, rm);\n }\n ///< Size is dest size\n void smlsl2(SubRegSize size, QRegister rd, QRegister rn, QRegister rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i8Bit, \"No 8-bit dest support.\");\n const auto ConvertedSize = SubRegSize {FEXCore::ToUnderlying(size) - 1};\n\n ASIMD3Different(0, 0b1010, ConvertedSize, rd, rn, rm);\n }\n ///< Size is dest size\n void sqdmlsl(SubRegSize size, DRegister rd, DRegister rn, DRegister rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i8Bit && size != SubRegSize::i16Bit, \"No 8/16-bit dest support.\");\n const auto ConvertedSize = SubRegSize {FEXCore::ToUnderlying(size) - 1};\n\n ASIMD3Different(0, 0b1011, ConvertedSize, rd, rn, rm);\n }\n ///< Size is dest size\n void sqdmlsl2(SubRegSize size, QRegister rd, QRegister rn, QRegister rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i8Bit && size != SubRegSize::i16Bit, \"No 8/16-bit dest support.\");\n const auto ConvertedSize = SubRegSize {FEXCore::ToUnderlying(size) - 1};\n\n ASIMD3Different(0, 0b1011, ConvertedSize, rd, rn, rm);\n }\n ///< Size is dest size\n void smull(SubRegSize size, DRegister rd, DRegister rn, DRegister rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i8Bit, \"No 8-bit dest support.\");\n const auto ConvertedSize = SubRegSize {FEXCore::ToUnderlying(size) - 1};\n\n ASIMD3Different(0, 0b1100, ConvertedSize, rd, rn, rm);\n }\n ///< Size is dest size\n void smull2(SubRegSize size, QRegister rd, QRegister rn, QRegister rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i8Bit, \"No 8-bit dest support.\");\n const auto ConvertedSize = SubRegSize {FEXCore::ToUnderlying(size) - 1};\n\n ASIMD3Different(0, 0b1100, ConvertedSize, rd, rn, rm);\n }\n ///< Size is dest size\n void sqdmull(SubRegSize size, DRegister rd, DRegister rn, DRegister rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i8Bit && size != SubRegSize::i16Bit, \"No 8/16-bit dest support.\");\n const auto ConvertedSize = SubRegSize {FEXCore::ToUnderlying(size) - 1};\n\n ASIMD3Different(0, 0b1101, ConvertedSize, rd, rn, rm);\n }\n ///< Size is dest size\n void sqdmull2(SubRegSize size, QRegister rd, QRegister rn, QRegister rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i8Bit && size != SubRegSize::i16Bit, \"No 8/16-bit dest support.\");\n const auto ConvertedSize = SubRegSize {FEXCore::ToUnderlying(size) - 1};\n\n ASIMD3Different(0, 0b1101, ConvertedSize, rd, rn, rm);\n }\n ///< Size is dest size\n void pmull(SubRegSize size, DRegister rd, DRegister rn, DRegister rm) {\n LOGMAN_THROW_A_FMT(size == SubRegSize::i16Bit || size == SubRegSize::i128Bit, \"Only 16-bit and 128-bit destination supported\");\n const auto ConvertedSize = SubRegSize {FEXCore::ToUnderlying(size) - 1};\n\n ASIMD3Different(0, 0b1110, ConvertedSize, rd, rn, rm);\n }\n ///< Size is dest size\n void pmull2(SubRegSize size, QRegister rd, QRegister rn, QRegister rm) {\n LOGMAN_THROW_A_FMT(size == SubRegSize::i16Bit || size == SubRegSize::i128Bit, \"Only 16-bit and 128-bit destination supported\");\n const auto ConvertedSize = SubRegSize {FEXCore::ToUnderlying(size) - 1};\n\n ASIMD3Different(0, 0b1110, ConvertedSize, rd, rn, rm);\n }\n ///< Size is dest size\n void uaddl(SubRegSize size, DRegister rd, DRegister rn, DRegister rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i8Bit, \"No 8-bit dest support.\");\n const auto ConvertedSize = SubRegSize {FEXCore::ToUnderlying(size) - 1};\n\n ASIMD3Different(1, 0b0000, ConvertedSize, rd, rn, rm);\n }\n ///< Size is dest size\n void uaddl2(SubRegSize size, QRegister rd, QRegister rn, QRegister rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i8Bit, \"No 8-bit dest support.\");\n const auto ConvertedSize = SubRegSize {FEXCore::ToUnderlying(size) - 1};\n\n ASIMD3Different(1, 0b0000, ConvertedSize, rd, rn, rm);\n }\n ///< Size is dest size\n void uaddw(SubRegSize size, DRegister rd, DRegister rn, DRegister rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i8Bit, \"No 8-bit dest support.\");\n const auto ConvertedSize = SubRegSize {FEXCore::ToUnderlying(size) - 1};\n\n ASIMD3Different(1, 0b0001, ConvertedSize, rd, rn, rm);\n }\n ///< Size is dest size\n void uaddw2(SubRegSize size, QRegister rd, QRegister rn, QRegister rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i8Bit, \"No 8-bit dest support.\");\n const auto ConvertedSize = SubRegSize {FEXCore::ToUnderlying(size) - 1};\n\n ASIMD3Different(1, 0b0001, ConvertedSize, rd, rn, rm);\n }\n ///< Size is dest size\n void usubl(SubRegSize size, DRegister rd, DRegister rn, DRegister rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i8Bit, \"No 8-bit dest support.\");\n const auto ConvertedSize = SubRegSize {FEXCore::ToUnderlying(size) - 1};\n\n ASIMD3Different(1, 0b0010, ConvertedSize, rd, rn, rm);\n }\n ///< Size is dest size\n void usubl2(SubRegSize size, QRegister rd, QRegister rn, QRegister rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i8Bit, \"No 8-bit dest support.\");\n const auto ConvertedSize = SubRegSize {FEXCore::ToUnderlying(size) - 1};\n\n ASIMD3Different(1, 0b0010, ConvertedSize, rd, rn, rm);\n }\n ///< Size is dest size\n void usubw(SubRegSize size, DRegister rd, DRegister rn, DRegister rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i8Bit, \"No 8-bit dest support.\");\n const auto ConvertedSize = SubRegSize {FEXCore::ToUnderlying(size) - 1};\n\n ASIMD3Different(1, 0b0011, ConvertedSize, rd, rn, rm);\n }\n ///< Size is dest size\n void usubw2(SubRegSize size, QRegister rd, QRegister rn, QRegister rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i8Bit, \"No 8-bit dest support.\");\n const auto ConvertedSize = SubRegSize {FEXCore::ToUnderlying(size) - 1};\n\n ASIMD3Different(1, 0b0011, ConvertedSize, rd, rn, rm);\n }\n ///< Size is dest size\n void raddhn(SubRegSize size, DRegister rd, DRegister rn, DRegister rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"No 64-bit dest support.\");\n ASIMD3Different(1, 0b0100, size, rd, rn, rm);\n }\n ///< Size is dest size\n void raddhn2(SubRegSize size, QRegister rd, QRegister rn, QRegister rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"No 64-bit dest support.\");\n ASIMD3Different(1, 0b0100, size, rd, rn, rm);\n }\n ///< Size is dest size\n void uabal(SubRegSize size, DRegister rd, DRegister rn, DRegister rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i8Bit, \"No 8-bit dest support.\");\n const auto ConvertedSize = SubRegSize {FEXCore::ToUnderlying(size) - 1};\n\n ASIMD3Different(1, 0b0101, ConvertedSize, rd, rn, rm);\n }\n ///< Size is dest size\n void uabal2(SubRegSize size, QRegister rd, QRegister rn, QRegister rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i8Bit, \"No 8-bit dest support.\");\n const auto ConvertedSize = SubRegSize {FEXCore::ToUnderlying(size) - 1};\n\n ASIMD3Different(1, 0b0101, ConvertedSize, rd, rn, rm);\n }\n ///< Size is dest size\n void rsubhn(SubRegSize size, DRegister rd, DRegister rn, DRegister rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"No 64-bit dest support.\");\n ASIMD3Different(1, 0b0110, size, rd, rn, rm);\n }\n ///< Size is dest size\n void rsubhn2(SubRegSize size, QRegister rd, QRegister rn, QRegister rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"No 64-bit dest support.\");\n ASIMD3Different(1, 0b0110, size, rd, rn, rm);\n }\n ///< Size is dest size\n void uabdl(SubRegSize size, DRegister rd, DRegister rn, DRegister rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i8Bit, \"No 8-bit dest support.\");\n const auto ConvertedSize = SubRegSize {FEXCore::ToUnderlying(size) - 1};\n\n ASIMD3Different(1, 0b0111, ConvertedSize, rd, rn, rm);\n }\n ///< Size is dest size\n void uabdl2(SubRegSize size, QRegister rd, QRegister rn, QRegister rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i8Bit, \"No 8-bit dest support.\");\n const auto ConvertedSize = SubRegSize {FEXCore::ToUnderlying(size) - 1};\n\n ASIMD3Different(1, 0b0111, ConvertedSize, rd, rn, rm);\n }\n ///< Size is dest size\n void umlal(SubRegSize size, DRegister rd, DRegister rn, DRegister rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i8Bit, \"No 8-bit dest support.\");\n const auto ConvertedSize = SubRegSize {FEXCore::ToUnderlying(size) - 1};\n\n ASIMD3Different(1, 0b1000, ConvertedSize, rd, rn, rm);\n }\n ///< Size is dest size\n void umlal2(SubRegSize size, QRegister rd, QRegister rn, QRegister rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i8Bit, \"No 8-bit dest support.\");\n const auto ConvertedSize = SubRegSize {FEXCore::ToUnderlying(size) - 1};\n\n ASIMD3Different(1, 0b1000, ConvertedSize, rd, rn, rm);\n }\n ///< Size is dest size\n void umlsl(SubRegSize size, DRegister rd, DRegister rn, DRegister rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i8Bit, \"No 8-bit dest support.\");\n const auto ConvertedSize = SubRegSize {FEXCore::ToUnderlying(size) - 1};\n\n ASIMD3Different(1, 0b1010, ConvertedSize, rd, rn, rm);\n }\n ///< Size is dest size\n void umlsl2(SubRegSize size, QRegister rd, QRegister rn, QRegister rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i8Bit, \"No 8-bit dest support.\");\n const auto ConvertedSize = SubRegSize {FEXCore::ToUnderlying(size) - 1};\n\n ASIMD3Different(1, 0b1010, ConvertedSize, rd, rn, rm);\n }\n ///< Size is dest size\n void umull(SubRegSize size, DRegister rd, DRegister rn, DRegister rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i8Bit, \"No 8-bit dest support.\");\n const auto ConvertedSize = SubRegSize {FEXCore::ToUnderlying(size) - 1};\n\n ASIMD3Different(1, 0b1100, ConvertedSize, rd, rn, rm);\n }\n ///< Size is dest size\n void umull2(SubRegSize size, QRegister rd, QRegister rn, QRegister rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i8Bit, \"No 8-bit dest support.\");\n const auto ConvertedSize = SubRegSize {FEXCore::ToUnderlying(size) - 1};\n\n ASIMD3Different(1, 0b1100, ConvertedSize, rd, rn, rm);\n }\n\n // Advanced SIMD three same\n template\n void shadd(SubRegSize size, T rd, T rn, T rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"64-bit subregsize not supported\");\n ASIMD3Same(0, size, 0b00000, rd, rn, rm);\n }\n\n template\n void sqadd(SubRegSize size, T rd, T rn, T rm) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid element size with 64-bit sqadd\");\n }\n ASIMD3Same(0, size, 0b00001, rd, rn, rm);\n }\n\n template\n void srhadd(SubRegSize size, T rd, T rn, T rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"64-bit subregsize not supported\");\n ASIMD3Same(0, size, 0b00010, rd, rn, rm);\n }\n template\n void shsub(SubRegSize size, T rd, T rn, T rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"64-bit subregsize not supported\");\n ASIMD3Same(0, size, 0b00100, rd, rn, rm);\n }\n template\n void sqsub(SubRegSize size, T rd, T rn, T rm) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid element size with 64-bit sqsub\");\n }\n ASIMD3Same(0, size, 0b00101, rd, rn, rm);\n }\n template\n void cmgt(SubRegSize size, T rd, T rn, T rm) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid element size with 64-bit cmgt\");\n }\n ASIMD3Same(0, size, 0b00110, rd, rn, rm);\n }\n template\n void cmge(SubRegSize size, T rd, T rn, T rm) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid element size with 64-bit cmge\");\n }\n ASIMD3Same(0, size, 0b00111, rd, rn, rm);\n }\n template\n void sshl(SubRegSize size, T rd, T rn, T rm) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid element size with 64-bit sshl\");\n }\n ASIMD3Same(0, size, 0b01000, rd, rn, rm);\n }\n template\n void sqshl(SubRegSize size, T rd, T rn, T rm) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid element size with 64-bit sqshl\");\n }\n ASIMD3Same(0, size, 0b01001, rd, rn, rm);\n }\n template\n void srshl(SubRegSize size, T rd, T rn, T rm) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid element size with 64-bit srshl\");\n }\n ASIMD3Same(0, size, 0b01010, rd, rn, rm);\n }\n template\n void sqrshl(SubRegSize size, T rd, T rn, T rm) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid element size with 64-bit sqrshl\");\n }\n ASIMD3Same(0, size, 0b01011, rd, rn, rm);\n }\n template\n void smax(SubRegSize size, T rd, T rn, T rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"64-bit subregsize not supported\");\n ASIMD3Same(0, size, 0b01100, rd, rn, rm);\n }\n template\n void smin(SubRegSize size, T rd, T rn, T rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"64-bit subregsize not supported\");\n ASIMD3Same(0, size, 0b01101, rd, rn, rm);\n }\n template\n void sabd(SubRegSize size, T rd, T rn, T rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"64-bit subregsize not supported\");\n ASIMD3Same(0, size, 0b01110, rd, rn, rm);\n }\n template\n void saba(SubRegSize size, T rd, T rn, T rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"64-bit subregsize not supported\");\n ASIMD3Same(0, size, 0b01111, rd, rn, rm);\n }\n template\n void add(SubRegSize size, T rd, T rn, T rm) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid element size with 64-bit add\");\n }\n ASIMD3Same(0, size, 0b10000, rd, rn, rm);\n }\n template\n void cmtst(SubRegSize size, T rd, T rn, T rm) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid element size with 64-bit cmtst\");\n }\n ASIMD3Same(0, size, 0b10001, rd, rn, rm);\n }\n template\n void mla(SubRegSize size, T rd, T rn, T rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"64-bit subregsize not supported\");\n ASIMD3Same(0, size, 0b10010, rd, rn, rm);\n }\n template\n void mul(SubRegSize size, T rd, T rn, T rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"64-bit subregsize not supported\");\n ASIMD3Same(0, size, 0b10011, rd, rn, rm);\n }\n template\n void smaxp(SubRegSize size, T rd, T rn, T rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"64-bit subregsize not supported\");\n ASIMD3Same(0, size, 0b10100, rd, rn, rm);\n }\n template\n void sminp(SubRegSize size, T rd, T rn, T rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"64-bit subregsize not supported\");\n ASIMD3Same(0, size, 0b10101, rd, rn, rm);\n }\n template\n void sqdmulh(SubRegSize size, T rd, T rn, T rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i8Bit, \"No 8-bit dest support.\");\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"64-bit subregsize not supported\");\n ASIMD3Same(0, size, 0b10110, rd, rn, rm);\n }\n template\n void addp(SubRegSize size, T rd, T rn, T rm) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid element size with 64-bit {}\", __func__);\n }\n ASIMD3Same(0, size, 0b10111, rd, rn, rm);\n }\n template\n void fmaxnm(SubRegSize size, T rd, T rn, T rm) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid element size with 64-bit {}\", __func__);\n }\n LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), \"Only 16/32/64-bit subregsize supported\");\n\n if (size == SubRegSize::i16Bit) {\n ASIMDThreeSameFP16(0, 0, 0b000, rm, rn, rd);\n } else {\n const auto ConvertedSize = size == SubRegSize::i64Bit ? SubRegSize::i16Bit : SubRegSize::i8Bit;\n ASIMD3Same(0, ConvertedSize, 0b11000, rd, rn, rm);\n }\n }\n template\n void fmla(SubRegSize size, T rd, T rn, T rm) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid element size with 64-bit {}\", __func__);\n }\n LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), \"Only 16/32/64-bit subregsize supported\");\n\n if (size == SubRegSize::i16Bit) {\n ASIMDThreeSameFP16(0, 0, 0b001, rm, rn, rd);\n } else {\n const auto ConvertedSize = size == SubRegSize::i64Bit ? SubRegSize::i16Bit : SubRegSize::i8Bit;\n ASIMD3Same(0, ConvertedSize, 0b11001, rd, rn, rm);\n }\n }\n template\n void fadd(SubRegSize size, T rd, T rn, T rm) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid element size with 64-bit {}\", __func__);\n }\n LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), \"Only 16/32/64-bit subregsize supported\");\n\n if (size == SubRegSize::i16Bit) {\n ASIMDThreeSameFP16(0, 0, 0b010, rm, rn, rd);\n } else {\n const auto ConvertedSize = size == SubRegSize::i64Bit ? SubRegSize::i16Bit : SubRegSize::i8Bit;\n ASIMD3Same(0, ConvertedSize, 0b11010, rd, rn, rm);\n }\n }\n template\n void fmulx(SubRegSize size, T rd, T rn, T rm) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid element size with 64-bit {}\", __func__);\n }\n LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), \"Only 16/32/64-bit subregsize supported\");\n\n if (size == SubRegSize::i16Bit) {\n ASIMDThreeSameFP16(0, 0, 0b011, rm, rn, rd);\n } else {\n const auto ConvertedSize = size == SubRegSize::i64Bit ? SubRegSize::i16Bit : SubRegSize::i8Bit;\n ASIMD3Same(0, ConvertedSize, 0b11011, rd, rn, rm);\n }\n }\n template\n void fcmeq(SubRegSize size, T rd, T rn, T rm) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid element size with 64-bit {}\", __func__);\n }\n LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), \"Only 16/32/64-bit subregsize supported\");\n\n if (size == SubRegSize::i16Bit) {\n ASIMDThreeSameFP16(0, 0, 0b100, rm, rn, rd);\n } else {\n const auto ConvertedSize = size == SubRegSize::i64Bit ? SubRegSize::i16Bit : SubRegSize::i8Bit;\n ASIMD3Same(0, ConvertedSize, 0b11100, rd, rn, rm);\n }\n }\n template\n void fmax(SubRegSize size, T rd, T rn, T rm) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid element size with 64-bit {}\", __func__);\n }\n LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), \"Only 16/32/64-bit subregsize supported\");\n\n if (size == SubRegSize::i16Bit) {\n ASIMDThreeSameFP16(0, 0, 0b110, rm, rn, rd);\n } else {\n const auto ConvertedSize = size == SubRegSize::i64Bit ? SubRegSize::i16Bit : SubRegSize::i8Bit;\n ASIMD3Same(0, ConvertedSize, 0b11110, rd, rn, rm);\n }\n }\n template\n void frecps(SubRegSize size, T rd, T rn, T rm) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid element size with 64-bit {}\", __func__);\n }\n LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), \"Only 16/32/64-bit subregsize supported\");\n\n if (size == SubRegSize::i16Bit) {\n ASIMDThreeSameFP16(0, 0, 0b111, rm, rn, rd);\n } else {\n const auto ConvertedSize = size == SubRegSize::i64Bit ? SubRegSize::i16Bit : SubRegSize::i8Bit;\n ASIMD3Same(0, ConvertedSize, 0b11111, rd, rn, rm);\n }\n }\n template\n void and_(T rd, T rn, T rm) {\n ASIMD3Same(0, SubRegSize::i8Bit, 0b00011, rd, rn, rm);\n }\n template\n void fmlal(T rd, T rn, T rm) {\n ASIMD3Same(0, SubRegSize::i8Bit, 0b11101, rd, rn, rm);\n }\n template\n void fmlal2(T rd, T rn, T rm) {\n ASIMD3Same(1, SubRegSize::i8Bit, 0b11001, rd, rn, rm);\n }\n template\n void bic(T rd, T rn, T rm) {\n ASIMD3Same(0, SubRegSize::i16Bit, 0b00011, rd, rn, rm);\n }\n template\n void fminnm(SubRegSize size, T rd, T rn, T rm) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid element size with 64-bit {}\", __func__);\n }\n LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), \"Only 16/32/64-bit subregsize supported\");\n\n if (size == SubRegSize::i16Bit) {\n ASIMDThreeSameFP16(0, 1, 0b000, rm, rn, rd);\n } else {\n ASIMD3Same(0, size, 0b11000, rd, rn, rm);\n }\n }\n template\n void fmls(SubRegSize size, T rd, T rn, T rm) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid element size with 64-bit {}\", __func__);\n }\n LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), \"Only 16/32/64-bit subregsize supported\");\n\n if (size == SubRegSize::i16Bit) {\n ASIMDThreeSameFP16(0, 1, 0b001, rm, rn, rd);\n } else {\n ASIMD3Same(0, size, 0b11001, rd, rn, rm);\n }\n }\n template\n void fsub(SubRegSize size, T rd, T rn, T rm) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid element size with 64-bit {}\", __func__);\n }\n LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), \"Only 16/32/64-bit subregsize supported\");\n\n if (size == SubRegSize::i16Bit) {\n ASIMDThreeSameFP16(0, 1, 0b010, rm, rn, rd);\n } else {\n ASIMD3Same(0, size, 0b11010, rd, rn, rm);\n }\n }\n template\n void fmin(SubRegSize size, T rd, T rn, T rm) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid element size with 64-bit {}\", __func__);\n }\n LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), \"Only 16/32/64-bit subregsize supported\");\n\n if (size == SubRegSize::i16Bit) {\n ASIMDThreeSameFP16(0, 1, 0b110, rm, rn, rd);\n } else {\n ASIMD3Same(0, size, 0b11110, rd, rn, rm);\n }\n }\n template\n void frsqrts(SubRegSize size, T rd, T rn, T rm) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid element size with 64-bit {}\", __func__);\n }\n LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), \"Only 16/32/64-bit subregsize supported\");\n\n if (size == SubRegSize::i16Bit) {\n ASIMDThreeSameFP16(0, 1, 0b111, rm, rn, rd);\n } else {\n ASIMD3Same(0, size, 0b11111, rd, rn, rm);\n }\n }\n template\n void orr(T rd, T rn, T rm) {\n ASIMD3Same(0, SubRegSize::i32Bit, 0b00011, rd, rn, rm);\n }\n template\n void mov(T rd, T rn) {\n orr(rd, rn, rn);\n }\n template\n void fmlsl(T rd, T rn, T rm) {\n ASIMD3Same(0, SubRegSize::i32Bit, 0b11101, rd, rn, rm);\n }\n template\n void fmlsl2(T rd, T rn, T rm) {\n ASIMD3Same(1, SubRegSize::i32Bit, 0b11001, rd, rn, rm);\n }\n template\n void orn(T rd, T rn, T rm) {\n ASIMD3Same(0, SubRegSize::i64Bit, 0b00011, rd, rn, rm);\n }\n template\n void uhadd(SubRegSize size, T rd, T rn, T rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"64-bit subregsize not supported\");\n ASIMD3Same(1, size, 0b00000, rd, rn, rm);\n }\n template\n void uqadd(SubRegSize size, T rd, T rn, T rm) {\n ASIMD3Same(1, size, 0b00001, rd, rn, rm);\n }\n template\n void urhadd(SubRegSize size, T rd, T rn, T rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"64-bit subregsize not supported\");\n ASIMD3Same(1, size, 0b00010, rd, rn, rm);\n }\n template\n void uhsub(SubRegSize size, T rd, T rn, T rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"64-bit subregsize not supported\");\n ASIMD3Same(1, size, 0b00100, rd, rn, rm);\n }\n template\n void uqsub(SubRegSize size, T rd, T rn, T rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"64-bit subregsize not supported\");\n ASIMD3Same(1, size, 0b00101, rd, rn, rm);\n }\n template\n void cmhi(SubRegSize size, T rd, T rn, T rm) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid element size with 64-bit {}\", __func__);\n }\n ASIMD3Same(1, size, 0b00110, rd, rn, rm);\n }\n template\n void cmhs(SubRegSize size, T rd, T rn, T rm) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid element size with 64-bit {}\", __func__);\n }\n ASIMD3Same(1, size, 0b00111, rd, rn, rm);\n }\n template\n void ushl(SubRegSize size, T rd, T rn, T rm) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid element size with 64-bit {}\", __func__);\n }\n ASIMD3Same(1, size, 0b01000, rd, rn, rm);\n }\n template\n void uqshl(SubRegSize size, T rd, T rn, T rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"64-bit subregsize not supported\");\n ASIMD3Same(1, size, 0b01001, rd, rn, rm);\n }\n template\n void urshl(SubRegSize size, T rd, T rn, T rm) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid element size with 64-bit {}\", __func__);\n }\n ASIMD3Same(1, size, 0b01010, rd, rn, rm);\n }\n template\n void uqrshl(SubRegSize size, T rd, T rn, T rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"64-bit subregsize not supported\");\n ASIMD3Same(1, size, 0b01011, rd, rn, rm);\n }\n template\n void umax(SubRegSize size, T rd, T rn, T rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"64-bit subregsize not supported\");\n ASIMD3Same(1, size, 0b01100, rd, rn, rm);\n }\n template\n void umin(SubRegSize size, T rd, T rn, T rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"64-bit subregsize not supported\");\n ASIMD3Same(1, size, 0b01101, rd, rn, rm);\n }\n template\n void uabd(SubRegSize size, T rd, T rn, T rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"64-bit subregsize not supported\");\n ASIMD3Same(1, size, 0b01110, rd, rn, rm);\n }\n template\n void uaba(SubRegSize size, T rd, T rn, T rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"64-bit subregsize not supported\");\n ASIMD3Same(1, size, 0b01111, rd, rn, rm);\n }\n template\n void sub(SubRegSize size, T rd, T rn, T rm) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid element size with 64-bit {}\", __func__);\n }\n ASIMD3Same(1, size, 0b10000, rd, rn, rm);\n }\n template\n void cmeq(SubRegSize size, T rd, T rn, T rm) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid element size with 64-bit {}\", __func__);\n }\n ASIMD3Same(1, size, 0b10001, rd, rn, rm);\n }\n template\n void mls(SubRegSize size, T rd, T rn, T rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"64-bit subregsize not supported\");\n ASIMD3Same(1, size, 0b10010, rd, rn, rm);\n }\n template\n void pmul(T rd, T rn, T rm) {\n ASIMD3Same(1, SubRegSize::i8Bit, 0b10011, rd, rn, rm);\n }\n template\n void umaxp(SubRegSize size, T rd, T rn, T rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"64-bit subregsize not supported\");\n ASIMD3Same(1, size, 0b10100, rd, rn, rm);\n }\n template\n void uminp(SubRegSize size, T rd, T rn, T rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"64-bit subregsize not supported\");\n ASIMD3Same(1, size, 0b10101, rd, rn, rm);\n }\n template\n void sqrdmulh(SubRegSize size, T rd, T rn, T rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit && size != SubRegSize::i8Bit, \"8/64-bit subregsize not supported\");\n ASIMD3Same(1, size, 0b10110, rd, rn, rm);\n }\n template\n void fmaxnmp(SubRegSize size, T rd, T rn, T rm) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid element size with 64-bit {}\", __func__);\n }\n LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), \"Only 16/32/64-bit subregsize supported\");\n\n if (size == SubRegSize::i16Bit) {\n ASIMDThreeSameFP16(1, 0, 0b000, rm, rn, rd);\n } else {\n const auto ConvertedSize = size == SubRegSize::i64Bit ? SubRegSize::i16Bit : SubRegSize::i8Bit;\n ASIMD3Same(1, ConvertedSize, 0b11000, rd, rn, rm);\n }\n }\n template\n void faddp(SubRegSize size, T rd, T rn, T rm) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid element size with 64-bit {}\", __func__);\n }\n LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), \"Only 16/32/64-bit subregsize supported\");\n\n if (size == SubRegSize::i16Bit) {\n ASIMDThreeSameFP16(1, 0, 0b010, rm, rn, rd);\n } else {\n const auto ConvertedSize = size == SubRegSize::i64Bit ? SubRegSize::i16Bit : SubRegSize::i8Bit;\n ASIMD3Same(1, ConvertedSize, 0b11010, rd, rn, rm);\n }\n }\n template\n void fmul(SubRegSize size, T rd, T rn, T rm) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid element size with 64-bit {}\", __func__);\n }\n LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), \"Only 16/32/64-bit subregsize supported\");\n\n if (size == SubRegSize::i16Bit) {\n ASIMDThreeSameFP16(1, 0, 0b011, rm, rn, rd);\n } else {\n const auto ConvertedSize = size == SubRegSize::i64Bit ? SubRegSize::i16Bit : SubRegSize::i8Bit;\n ASIMD3Same(1, ConvertedSize, 0b11011, rd, rn, rm);\n }\n }\n template\n void fcmge(SubRegSize size, T rd, T rn, T rm) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid element size with 64-bit {}\", __func__);\n }\n LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), \"Only 16/32/64-bit subregsize supported\");\n\n if (size == SubRegSize::i16Bit) {\n ASIMDThreeSameFP16(1, 0, 0b100, rm, rn, rd);\n } else {\n const auto ConvertedSize = size == SubRegSize::i64Bit ? SubRegSize::i16Bit : SubRegSize::i8Bit;\n ASIMD3Same(1, ConvertedSize, 0b11100, rd, rn, rm);\n }\n }\n template\n void facge(SubRegSize size, T rd, T rn, T rm) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid element size with 64-bit {}\", __func__);\n }\n LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), \"Only 16/32/64-bit subregsize supported\");\n\n if (size == SubRegSize::i16Bit) {\n ASIMDThreeSameFP16(1, 0, 0b101, rm, rn, rd);\n } else {\n const auto ConvertedSize = size == SubRegSize::i64Bit ? SubRegSize::i16Bit : SubRegSize::i8Bit;\n ASIMD3Same(1, ConvertedSize, 0b11101, rd, rn, rm);\n }\n }\n template\n void fmaxp(SubRegSize size, T rd, T rn, T rm) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid element size with 64-bit {}\", __func__);\n }\n LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), \"Only 16/32/64-bit subregsize supported\");\n\n if (size == SubRegSize::i16Bit) {\n ASIMDThreeSameFP16(1, 0, 0b110, rm, rn, rd);\n } else {\n const auto ConvertedSize = size == SubRegSize::i64Bit ? SubRegSize::i16Bit : SubRegSize::i8Bit;\n ASIMD3Same(1, ConvertedSize, 0b11110, rd, rn, rm);\n }\n }\n template\n void fdiv(SubRegSize size, T rd, T rn, T rm) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid element size with 64-bit {}\", __func__);\n }\n LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), \"Only 16/32/64-bit subregsize supported\");\n\n if (size == SubRegSize::i16Bit) {\n ASIMDThreeSameFP16(1, 0, 0b111, rm, rn, rd);\n } else {\n const auto ConvertedSize = size == SubRegSize::i64Bit ? SubRegSize::i16Bit : SubRegSize::i8Bit;\n ASIMD3Same(1, ConvertedSize, 0b11111, rd, rn, rm);\n }\n }\n template\n void eor(T rd, T rn, T rm) {\n ASIMD3Same(1, SubRegSize::i8Bit, 0b00011, rd, rn, rm);\n }\n template\n void bsl(T rd, T rn, T rm) {\n ASIMD3Same(1, SubRegSize::i16Bit, 0b00011, rd, rn, rm);\n }\n template\n void fminnmp(SubRegSize size, T rd, T rn, T rm) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid element size with 64-bit {}\", __func__);\n }\n LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), \"Only 16/32/64-bit subregsize supported\");\n\n if (size == SubRegSize::i16Bit) {\n ASIMDThreeSameFP16(1, 1, 0b000, rm, rn, rd);\n } else {\n ASIMD3Same(1, size, 0b11000, rd, rn, rm);\n }\n }\n template\n void fabd(SubRegSize size, T rd, T rn, T rm) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid element size with 64-bit {}\", __func__);\n }\n LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), \"Only 16/32/64-bit subregsize supported\");\n\n if (size == SubRegSize::i16Bit) {\n ASIMDThreeSameFP16(1, 1, 0b010, rm, rn, rd);\n } else {\n ASIMD3Same(1, size, 0b11010, rd, rn, rm);\n }\n }\n template\n void fcmgt(SubRegSize size, T rd, T rn, T rm) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid element size with 64-bit {}\", __func__);\n }\n LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), \"Only 16/32/64-bit subregsize supported\");\n\n if (size == SubRegSize::i16Bit) {\n ASIMDThreeSameFP16(1, 1, 0b100, rm, rn, rd);\n } else {\n ASIMD3Same(1, size, 0b11100, rd, rn, rm);\n }\n }\n template\n void facgt(SubRegSize size, T rd, T rn, T rm) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid element size with 64-bit {}\", __func__);\n }\n LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), \"Only 16/32/64-bit subregsize supported\");\n\n if (size == SubRegSize::i16Bit) {\n ASIMDThreeSameFP16(1, 1, 0b101, rm, rn, rd);\n } else {\n ASIMD3Same(1, size, 0b11101, rd, rn, rm);\n }\n }\n template\n void fminp(SubRegSize size, T rd, T rn, T rm) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid element size with 64-bit {}\", __func__);\n }\n LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), \"Only 16/32/64-bit subregsize supported\");\n\n if (size == SubRegSize::i16Bit) {\n ASIMDThreeSameFP16(1, 1, 0b110, rm, rn, rd);\n } else {\n ASIMD3Same(1, size, 0b11110, rd, rn, rm);\n }\n }\n template\n void bit(T rd, T rn, T rm) {\n ASIMD3Same(1, SubRegSize::i32Bit, 0b00011, rd, rn, rm);\n }\n template\n void bif(T rd, T rn, T rm) {\n ASIMD3Same(1, SubRegSize::i64Bit, 0b00011, rd, rn, rm);\n }\n\n // Advanced SIMD modified immediate\n // XXX: ORR - 32-bit/16-bit\n // XXX: MOVI - Shifting ones\n template\n void fmov(SubRegSize size, T rd, float Value) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid element size with 64-bit {}\", __func__);\n }\n LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), \"Unsupported fmov size\");\n\n uint32_t op;\n uint32_t cmode = 0b1111;\n uint32_t o2;\n uint32_t Imm;\n if (size == SubRegSize::i16Bit) {\n LOGMAN_MSG_A_FMT(\"Unsupported\");\n FEX_UNREACHABLE;\n } else if (size == SubRegSize::i32Bit) {\n op = 0;\n o2 = 0;\n Imm = FP32ToImm8(Value);\n } else if (size == SubRegSize::i64Bit) {\n op = 1;\n o2 = 0;\n Imm = FP64ToImm8(Value);\n } else {\n LOGMAN_MSG_A_FMT(\"Invalid subregsize\");\n FEX_UNREACHABLE;\n }\n\n ASIMDModifiedImm(op, cmode, o2, Imm, rd);\n }\n // XXX: MVNI - Shifted immediate\n // XXX: BIC\n // void ASIMDModifiedImm(uint32_t Op, uint32_t op, uint32_t cmode, uint32_t o2, uint32_t imm, T rd) {\n\n template\n void movi(SubRegSize size, T rd, uint64_t Imm, uint16_t Shift = 0) {\n LOGMAN_THROW_A_FMT(size == SubRegSize::i8Bit || size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit,\n \"Unsupported movi size\");\n\n uint32_t cmode;\n uint32_t op;\n if (size == SubRegSize::i8Bit) {\n LOGMAN_THROW_A_FMT(Shift == 0, \"8-bit can't have shift\");\n LOGMAN_THROW_A_FMT((Imm & ~0xFF) == 0, \"Larger than 8-bit Imm not supported\");\n cmode = 0b1110;\n op = 0;\n } else if (size == SubRegSize::i16Bit) {\n LOGMAN_THROW_A_FMT(Shift == 0 || Shift == 8, \"Shift by invalid amount\");\n LOGMAN_THROW_A_FMT((Imm & ~0xFF) == 0, \"Larger than 8-bit Imm not supported\");\n cmode = 0b1000 | (Shift ? 0b10 : 0b00);\n op = 0;\n } else if (size == SubRegSize::i32Bit) {\n LOGMAN_THROW_A_FMT(Shift == 0 || Shift == 8 || Shift == 16 || Shift == 24, \"Shift by invalid amount\");\n LOGMAN_THROW_A_FMT((Imm & ~0xFF) == 0, \"Larger than 8-bit Imm not supported\");\n cmode = 0b0000 | ((Shift >> 3) << 1);\n op = 0;\n } else if (size == SubRegSize::i64Bit) {\n LOGMAN_THROW_A_FMT(Shift == 0, \"64-bit can't have shift\");\n cmode = 0b1110;\n op = 1;\n\n // 64-bit movi doesn't behave like the smaller types\n // Each bit of the 8-bit imm encoding is expanded to a full 8-bits.\n // This gives us a full 64-bits for the final result but needs special handling.\n uint8_t NewImm {};\n for (size_t i = 0; i < 8; ++i) {\n const size_t BitOffset = i * 8;\n uint8_t Section = (Imm >> BitOffset) & 0xFF;\n LOGMAN_THROW_A_FMT(Section == 0 || Section == 0xFF, \"Invalid 64-bit constant encoding\");\n if (Section == 0xFF) {\n NewImm |= (1 << i);\n }\n }\n Imm = NewImm;\n } else {\n LOGMAN_MSG_A_FMT(\"Invalid subregsize\");\n FEX_UNREACHABLE;\n }\n\n ASIMDModifiedImm(op, cmode, 0, Imm, rd);\n }\n\n // Advanced SIMD shift by immediate\n template\n void sshr(SubRegSize size, T rd, T rn, uint32_t Shift) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid element size with 64-bit {}\", __func__);\n }\n const size_t SubregSizeInBits = SubRegSizeInBits(size);\n\n // Shift encoded a bit weirdly.\n // shift = (esize * 2) - immh:immb but immh is /also/ used for element size.\n const uint32_t InvertedShift = (SubregSizeInBits * 2) - (Shift);\n const uint32_t immh = InvertedShift >> 3;\n const uint32_t immb = InvertedShift & 0b111;\n\n ASIMDShiftByImm(0, immh, immb, 0b00000, rn, rd);\n }\n template\n void ssra(SubRegSize size, T rd, T rn, uint32_t Shift) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid element size with 64-bit {}\", __func__);\n }\n const size_t SubregSizeInBits = SubRegSizeInBits(size);\n\n // Shift encoded a bit weirdly.\n // shift = (esize * 2) - immh:immb but immh is /also/ used for element size.\n const uint32_t InvertedShift = (SubregSizeInBits * 2) - (Shift);\n const uint32_t immh = InvertedShift >> 3;\n const uint32_t immb = InvertedShift & 0b111;\n\n ASIMDShiftByImm(0, immh, immb, 0b00010, rn, rd);\n }\n template\n void srshr(SubRegSize size, T rd, T rn, uint32_t Shift) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid element size with 64-bit {}\", __func__);\n }\n const size_t SubregSizeInBits = SubRegSizeInBits(size);\n\n // Shift encoded a bit weirdly.\n // shift = (esize * 2) - immh:immb but immh is /also/ used for element size.\n const uint32_t InvertedShift = (SubregSizeInBits * 2) - (Shift);\n const uint32_t immh = InvertedShift >> 3;\n const uint32_t immb = InvertedShift & 0b111;\n\n ASIMDShiftByImm(0, immh, immb, 0b00100, rn, rd);\n }\n template\n void srsra(SubRegSize size, T rd, T rn, uint32_t Shift) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid element size with 64-bit {}\", __func__);\n }\n const size_t SubregSizeInBits = SubRegSizeInBits(size);\n\n // Shift encoded a bit weirdly.\n // shift = (esize * 2) - immh:immb but immh is /also/ used for element size.\n const uint32_t InvertedShift = (SubregSizeInBits * 2) - (Shift);\n const uint32_t immh = InvertedShift >> 3;\n const uint32_t immb = InvertedShift & 0b111;\n\n ASIMDShiftByImm(0, immh, immb, 0b00110, rn, rd);\n }\n template\n void shl(SubRegSize size, T rd, T rn, uint32_t Shift) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid element size with 64-bit {}\", __func__);\n }\n const size_t SubregSizeInBits = SubRegSizeInBits(size);\n\n // Shift encoded a bit weirdly.\n // shift = immh:immb - esize but immh is /also/ used for element size.\n const uint32_t InvertedShift = SubregSizeInBits + Shift;\n const uint32_t immh = InvertedShift >> 3;\n const uint32_t immb = InvertedShift & 0b111;\n\n ASIMDShiftByImm(0, immh, immb, 0b01010, rn, rd);\n }\n template\n void sqshl(SubRegSize size, T rd, T rn, uint32_t Shift) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid element size with 64-bit {}\", __func__);\n }\n const size_t SubregSizeInBits = SubRegSizeInBits(size);\n\n // Shift encoded a bit weirdly.\n // shift = immh:immb - esize but immh is /also/ used for element size.\n const uint32_t InvertedShift = SubregSizeInBits + Shift;\n const uint32_t immh = InvertedShift >> 3;\n const uint32_t immb = InvertedShift & 0b111;\n\n ASIMDShiftByImm(0, immh, immb, 0b01110, rn, rd);\n }\n ///< size is destination size\n void shrn(SubRegSize size, DRegister rd, DRegister rn, uint32_t Shift) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid element size with 64-bit {}\", __func__);\n const size_t SubregSizeInBits = SubRegSizeInBits(size);\n\n // Shift encoded a bit weirdly.\n // shift = (esize * 2) - immh:immb but immh is /also/ used for element size.\n const uint32_t InvertedShift = (SubregSizeInBits * 2) - (Shift);\n const uint32_t immh = InvertedShift >> 3;\n const uint32_t immb = InvertedShift & 0b111;\n\n ASIMDShiftByImm(0, immh, immb, 0b10000, rn, rd);\n }\n ///< size is destination size\n void shrn2(SubRegSize size, QRegister rd, QRegister rn, uint32_t Shift) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid element size with 64-bit {}\", __func__);\n const size_t SubregSizeInBits = SubRegSizeInBits(size);\n\n // Shift encoded a bit weirdly.\n // shift = (esize * 2) - immh:immb but immh is /also/ used for element size.\n const uint32_t InvertedShift = (SubregSizeInBits * 2) - (Shift);\n const uint32_t immh = InvertedShift >> 3;\n const uint32_t immb = InvertedShift & 0b111;\n\n ASIMDShiftByImm(0, immh, immb, 0b10000, rn, rd);\n }\n ///< size is destination size\n void rshrn(SubRegSize size, DRegister rd, DRegister rn, uint32_t Shift) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid element size with 64-bit {}\", __func__);\n const size_t SubregSizeInBits = SubRegSizeInBits(size);\n\n // Shift encoded a bit weirdly.\n // shift = (esize * 2) - immh:immb but immh is /also/ used for element size.\n const uint32_t InvertedShift = (SubregSizeInBits * 2) - (Shift);\n const uint32_t immh = InvertedShift >> 3;\n const uint32_t immb = InvertedShift & 0b111;\n\n ASIMDShiftByImm(0, immh, immb, 0b10001, rn, rd);\n }\n ///< size is destination size\n void rshrn2(SubRegSize size, QRegister rd, QRegister rn, uint32_t Shift) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid element size with 64-bit {}\", __func__);\n const size_t SubregSizeInBits = SubRegSizeInBits(size);\n\n // Shift encoded a bit weirdly.\n // shift = (esize * 2) - immh:immb but immh is /also/ used for element size.\n const uint32_t InvertedShift = (SubregSizeInBits * 2) - (Shift);\n const uint32_t immh = InvertedShift >> 3;\n const uint32_t immb = InvertedShift & 0b111;\n\n ASIMDShiftByImm(0, immh, immb, 0b10001, rn, rd);\n }\n ///< size is destination size\n void sqshrn(SubRegSize size, DRegister rd, DRegister rn, uint32_t Shift) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid element size with 64-bit {}\", __func__);\n const size_t SubregSizeInBits = SubRegSizeInBits(size);\n\n // Shift encoded a bit weirdly.\n // shift = (esize * 2) - immh:immb but immh is /also/ used for element size.\n const uint32_t InvertedShift = (SubregSizeInBits * 2) - (Shift);\n const uint32_t immh = InvertedShift >> 3;\n const uint32_t immb = InvertedShift & 0b111;\n\n ASIMDShiftByImm(0, immh, immb, 0b10010, rn, rd);\n }\n ///< size is destination size\n void sqshrn2(SubRegSize size, QRegister rd, QRegister rn, uint32_t Shift) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid element size with 64-bit {}\", __func__);\n const size_t SubregSizeInBits = SubRegSizeInBits(size);\n\n // Shift encoded a bit weirdly.\n // shift = (esize * 2) - immh:immb but immh is /also/ used for element size.\n const uint32_t InvertedShift = (SubregSizeInBits * 2) - (Shift);\n const uint32_t immh = InvertedShift >> 3;\n const uint32_t immb = InvertedShift & 0b111;\n\n ASIMDShiftByImm(0, immh, immb, 0b10010, rn, rd);\n }\n ///< size is destination size\n void sqrshrn(SubRegSize size, DRegister rd, DRegister rn, uint32_t Shift) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid element size with 64-bit {}\", __func__);\n const size_t SubregSizeInBits = SubRegSizeInBits(size);\n\n // Shift encoded a bit weirdly.\n // shift = (esize * 2) - immh:immb but immh is /also/ used for element size.\n const uint32_t InvertedShift = (SubregSizeInBits * 2) - (Shift);\n const uint32_t immh = InvertedShift >> 3;\n const uint32_t immb = InvertedShift & 0b111;\n\n ASIMDShiftByImm(0, immh, immb, 0b10011, rn, rd);\n }\n ///< size is destination size\n void sqrshrn2(SubRegSize size, QRegister rd, QRegister rn, uint32_t Shift) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid element size with 64-bit {}\", __func__);\n const size_t SubregSizeInBits = SubRegSizeInBits(size);\n\n // Shift encoded a bit weirdly.\n // shift = (esize * 2) - immh:immb but immh is /also/ used for element size.\n const uint32_t InvertedShift = (SubregSizeInBits * 2) - (Shift);\n const uint32_t immh = InvertedShift >> 3;\n const uint32_t immb = InvertedShift & 0b111;\n\n ASIMDShiftByImm(0, immh, immb, 0b10011, rn, rd);\n }\n ///< size is destination size\n void sshll(SubRegSize size, DRegister rd, DRegister rn, uint32_t Shift) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i8Bit, \"Invalid size\");\n size = SubRegSize(FEXCore::ToUnderlying(size) - 1);\n\n const size_t SubregSizeInBits = SubRegSizeInBits(size);\n LOGMAN_THROW_A_FMT(Shift < SubregSizeInBits, \"Shift must not be larger than incoming element size\");\n\n // Shift encoded a bit weirdly.\n // shift = immh:immb - esize but immh is /also/ used for element size.\n const uint32_t InvertedShift = SubregSizeInBits + Shift;\n const uint32_t immh = InvertedShift >> 3;\n const uint32_t immb = InvertedShift & 0b111;\n\n ASIMDShiftByImm(0, immh, immb, 0b10100, rn, rd);\n }\n\n ///< size is destination size\n void sshll2(SubRegSize size, QRegister rd, QRegister rn, uint32_t Shift) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i8Bit, \"Invalid size\");\n size = SubRegSize(FEXCore::ToUnderlying(size) - 1);\n\n const size_t SubregSizeInBits = SubRegSizeInBits(size);\n LOGMAN_THROW_A_FMT(Shift < SubregSizeInBits, \"Shift must not be larger than incoming element size\");\n\n // Shift encoded a bit weirdly.\n // shift = immh:immb - esize but immh is /also/ used for element size.\n const uint32_t InvertedShift = SubregSizeInBits + Shift;\n const uint32_t immh = InvertedShift >> 3;\n const uint32_t immb = InvertedShift & 0b111;\n\n ASIMDShiftByImm(0, immh, immb, 0b10100, rn, rd);\n }\n ///< size is destination size\n void sxtl(SubRegSize size, VRegister rd, VRegister rn) {\n sshll(size, rd.D(), rn.D(), 0);\n }\n ///< size is destination size\n void sxtl2(SubRegSize size, VRegister rd, VRegister rn) {\n sshll2(size, rd.Q(), rn.Q(), 0);\n }\n\n template\n void scvtf(SubRegSize size, T rd, T rn, uint32_t FractionalBits) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i8Bit, \"Invalid size\");\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid element size with 64-bit {}\", __func__);\n }\n\n const size_t SubregSizeInBits = SubRegSizeInBits(size);\n LOGMAN_THROW_A_FMT(FractionalBits < SubregSizeInBits, \"FractionalBits must not be larger than incoming element size\");\n\n // fbits encoded a bit weirdly.\n // fbits = (esize * 2) - immh:immb but immh is /also/ used for element size.\n const uint32_t InvertedFractionalBits = (SubregSizeInBits * 2) - FractionalBits;\n const uint32_t immh = InvertedFractionalBits >> 3;\n const uint32_t immb = InvertedFractionalBits & 0b111;\n\n ASIMDShiftByImm(0, immh, immb, 0b11100, rn, rd);\n }\n\n template\n void fcvtzs(SubRegSize size, T rd, T rn, uint32_t FractionalBits) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i8Bit, \"Invalid size\");\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid element size with 64-bit {}\", __func__);\n }\n\n const size_t SubregSizeInBits = SubRegSizeInBits(size);\n LOGMAN_THROW_A_FMT(FractionalBits < SubregSizeInBits, \"FractionalBits must not be larger than incoming element size\");\n\n // fbits encoded a bit weirdly.\n // fbits = (esize * 2) - immh:immb but immh is /also/ used for element size.\n const uint32_t InvertedFractionalBits = (SubregSizeInBits * 2) - FractionalBits;\n const uint32_t immh = InvertedFractionalBits >> 3;\n const uint32_t immb = InvertedFractionalBits & 0b111;\n\n ASIMDShiftByImm(0, immh, immb, 0b11111, rn, rd);\n }\n\n template\n void ushr(SubRegSize size, T rd, T rn, uint32_t Shift) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid element size with 64-bit {}\", __func__);\n }\n const size_t SubregSizeInBits = SubRegSizeInBits(size);\n\n // Shift encoded a bit weirdly.\n // shift = (esize * 2) - immh:immb but immh is /also/ used for element size.\n const uint32_t InvertedShift = (SubregSizeInBits * 2) - Shift;\n const uint32_t immh = InvertedShift >> 3;\n const uint32_t immb = InvertedShift & 0b111;\n\n ASIMDShiftByImm(1, immh, immb, 0b00000, rn, rd);\n }\n template\n void usra(SubRegSize size, T rd, T rn, uint32_t Shift) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid element size with 64-bit {}\", __func__);\n }\n const size_t SubregSizeInBits = SubRegSizeInBits(size);\n\n // Shift encoded a bit weirdly.\n // shift = (esize * 2) - immh:immb but immh is /also/ used for element size.\n const uint32_t InvertedShift = (SubregSizeInBits * 2) - Shift;\n const uint32_t immh = InvertedShift >> 3;\n const uint32_t immb = InvertedShift & 0b111;\n\n ASIMDShiftByImm(1, immh, immb, 0b00010, rn, rd);\n }\n template\n void urshr(SubRegSize size, T rd, T rn, uint32_t Shift) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid element size with 64-bit {}\", __func__);\n }\n const size_t SubregSizeInBits = SubRegSizeInBits(size);\n\n // Shift encoded a bit weirdly.\n // shift = (esize * 2) - immh:immb but immh is /also/ used for element size.\n const uint32_t InvertedShift = (SubregSizeInBits * 2) - Shift;\n const uint32_t immh = InvertedShift >> 3;\n const uint32_t immb = InvertedShift & 0b111;\n\n ASIMDShiftByImm(1, immh, immb, 0b00100, rn, rd);\n }\n template\n void ursra(SubRegSize size, T rd, T rn, uint32_t Shift) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid element size with 64-bit {}\", __func__);\n }\n const size_t SubregSizeInBits = SubRegSizeInBits(size);\n\n // Shift encoded a bit weirdly.\n // shift = (esize * 2) - immh:immb but immh is /also/ used for element size.\n const uint32_t InvertedShift = (SubregSizeInBits * 2) - Shift;\n const uint32_t immh = InvertedShift >> 3;\n const uint32_t immb = InvertedShift & 0b111;\n\n ASIMDShiftByImm(1, immh, immb, 0b00110, rn, rd);\n }\n template\n void sri(SubRegSize size, T rd, T rn, uint32_t Shift) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid element size with 64-bit {}\", __func__);\n }\n const size_t SubregSizeInBits = SubRegSizeInBits(size);\n\n // Shift encoded a bit weirdly.\n // shift = (esize * 2) - immh:immb but immh is /also/ used for element size.\n const uint32_t InvertedShift = (SubregSizeInBits * 2) - Shift;\n const uint32_t immh = InvertedShift >> 3;\n const uint32_t immb = InvertedShift & 0b111;\n\n ASIMDShiftByImm(1, immh, immb, 0b01000, rn, rd);\n }\n template\n void sli(SubRegSize size, T rd, T rn, uint32_t Shift) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid element size with 64-bit {}\", __func__);\n }\n const size_t SubregSizeInBits = SubRegSizeInBits(size);\n\n // Shift encoded a bit weirdly.\n // shift = immh:immb - esize but immh is /also/ used for element size.\n const uint32_t InvertedShift = SubregSizeInBits + Shift;\n const uint32_t immh = InvertedShift >> 3;\n const uint32_t immb = InvertedShift & 0b111;\n\n ASIMDShiftByImm(1, immh, immb, 0b01010, rn, rd);\n }\n template\n void sqshlu(SubRegSize size, T rd, T rn, uint32_t Shift) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid element size with 64-bit {}\", __func__);\n }\n const size_t SubregSizeInBits = SubRegSizeInBits(size);\n\n // Shift encoded a bit weirdly.\n // shift = immh:immb - esize but immh is /also/ used for element size.\n const uint32_t InvertedShift = SubregSizeInBits + Shift;\n const uint32_t immh = InvertedShift >> 3;\n const uint32_t immb = InvertedShift & 0b111;\n\n ASIMDShiftByImm(1, immh, immb, 0b01100, rn, rd);\n }\n ///< size is destination size\n template\n void uqshl(SubRegSize size, T rd, T rn, uint32_t Shift) {\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid element size with 64-bit {}\", __func__);\n }\n const size_t SubregSizeInBits = SubRegSizeInBits(size);\n\n // Shift encoded a bit weirdly.\n // shift = immh:immb - esize but immh is /also/ used for element size.\n const uint32_t InvertedShift = SubregSizeInBits + Shift;\n const uint32_t immh = InvertedShift >> 3;\n const uint32_t immb = InvertedShift & 0b111;\n\n ASIMDShiftByImm(1, immh, immb, 0b01110, rn, rd);\n }\n ///< size is destination size\n void sqshrun(SubRegSize size, DRegister rd, DRegister rn, uint32_t Shift) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid element size with 64-bit {}\", __func__);\n const size_t SubregSizeInBits = SubRegSizeInBits(size);\n\n // Shift encoded a bit weirdly.\n // shift = (esize * 2) - immh:immb but immh is /also/ used for element size.\n const uint32_t InvertedShift = (SubregSizeInBits * 2) - Shift;\n const uint32_t immh = InvertedShift >> 3;\n const uint32_t immb = InvertedShift & 0b111;\n\n ASIMDShiftByImm(1, immh, immb, 0b10000, rn, rd);\n }\n ///< size is destination size\n void sqshrun2(SubRegSize size, QRegister rd, QRegister rn, uint32_t Shift) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid element size with 64-bit {}\", __func__);\n const size_t SubregSizeInBits = SubRegSizeInBits(size);\n\n // Shift encoded a bit weirdly.\n // shift = (esize * 2) - immh:immb but immh is /also/ used for element size.\n const uint32_t InvertedShift = (SubregSizeInBits * 2) - Shift;\n const uint32_t immh = InvertedShift >> 3;\n const uint32_t immb = InvertedShift & 0b111;\n\n ASIMDShiftByImm(1, immh, immb, 0b10000, rn, rd);\n }\n ///< size is destination size\n void sqrshrun(SubRegSize size, DRegister rd, DRegister rn, uint32_t Shift) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid element size with 64-bit {}\", __func__);\n const size_t SubregSizeInBits = SubRegSizeInBits(size);\n\n // Shift encoded a bit weirdly.\n // shift = (esize * 2) - immh:immb but immh is /also/ used for element size.\n const uint32_t InvertedShift = (SubregSizeInBits * 2) - Shift;\n const uint32_t immh = InvertedShift >> 3;\n const uint32_t immb = InvertedShift & 0b111;\n\n ASIMDShiftByImm(1, immh, immb, 0b10001, rn, rd);\n }\n ///< size is destination size\n void sqrshrun2(SubRegSize size, QRegister rd, QRegister rn, uint32_t Shift) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid element size with 64-bit {}\", __func__);\n const size_t SubregSizeInBits = SubRegSizeInBits(size);\n\n // Shift encoded a bit weirdly.\n // shift = (esize * 2) - immh:immb but immh is /also/ used for element size.\n const uint32_t InvertedShift = (SubregSizeInBits * 2) - Shift;\n const uint32_t immh = InvertedShift >> 3;\n const uint32_t immb = InvertedShift & 0b111;\n\n ASIMDShiftByImm(1, immh, immb, 0b10001, rn, rd);\n }\n ///< size is destination size\n void uqshrn(SubRegSize size, DRegister rd, DRegister rn, uint32_t Shift) {\n const size_t SubregSizeInBits = SubRegSizeInBits(size);\n\n // Shift encoded a bit weirdly.\n // shift = (esize * 2) - immh:immb but immh is /also/ used for element size.\n const uint32_t InvertedShift = (SubregSizeInBits * 2) - Shift;\n const uint32_t immh = InvertedShift >> 3;\n const uint32_t immb = InvertedShift & 0b111;\n\n ASIMDShiftByImm(1, immh, immb, 0b10010, rn, rd);\n }\n ///< size is destination size\n void uqshrn2(SubRegSize size, QRegister rd, QRegister rn, uint32_t Shift) {\n const size_t SubregSizeInBits = SubRegSizeInBits(size);\n\n // Shift encoded a bit weirdly.\n // shift = (esize * 2) - immh:immb but immh is /also/ used for element size.\n const uint32_t InvertedShift = (SubregSizeInBits * 2) - Shift;\n const uint32_t immh = InvertedShift >> 3;\n const uint32_t immb = InvertedShift & 0b111;\n\n ASIMDShiftByImm(1, immh, immb, 0b10010, rn, rd);\n }\n ///< size is destination size\n void uqrshrn(SubRegSize size, DRegister rd, DRegister rn, uint32_t Shift) {\n const size_t SubregSizeInBits = SubRegSizeInBits(size);\n\n // Shift encoded a bit weirdly.\n // shift = (esize * 2) - immh:immb but immh is /also/ used for element size.\n const uint32_t InvertedShift = (SubregSizeInBits * 2) - Shift;\n const uint32_t immh = InvertedShift >> 3;\n const uint32_t immb = InvertedShift & 0b111;\n\n ASIMDShiftByImm(1, immh, immb, 0b10011, rn, rd);\n }\n ///< size is destination size\n void uqrshrn2(SubRegSize size, QRegister rd, QRegister rn, uint32_t Shift) {\n const size_t SubregSizeInBits = SubRegSizeInBits(size);\n\n // Shift encoded a bit weirdly.\n // shift = (esize * 2) - immh:immb but immh is /also/ used for element size.\n const uint32_t InvertedShift = (SubregSizeInBits * 2) - Shift;\n const uint32_t immh = InvertedShift >> 3;\n const uint32_t immb = InvertedShift & 0b111;\n\n ASIMDShiftByImm(1, immh, immb, 0b10011, rn, rd);\n }\n ///< size is destination size\n void ushll(SubRegSize size, DRegister rd, DRegister rn, uint32_t Shift) {\n size = SubRegSize(FEXCore::ToUnderlying(size) - 1);\n const size_t SubregSizeInBits = SubRegSizeInBits(size);\n\n // Shift encoded a bit weirdly.\n // shift = immh:immb - esize but immh is /also/ used for element size.\n const uint32_t InvertedShift = SubregSizeInBits + Shift;\n const uint32_t immh = InvertedShift >> 3;\n const uint32_t immb = InvertedShift & 0b111;\n\n ASIMDShiftByImm(1, immh, immb, 0b10100, rn, rd);\n }\n ///< size is destination size\n void ushll2(SubRegSize size, QRegister rd, QRegister rn, uint32_t Shift) {\n size = SubRegSize(FEXCore::ToUnderlying(size) - 1);\n const size_t SubregSizeInBits = SubRegSizeInBits(size);\n\n // Shift encoded a bit weirdly.\n // shift = immh:immb - esize but immh is /also/ used for element size.\n const uint32_t InvertedShift = SubregSizeInBits + Shift;\n const uint32_t immh = InvertedShift >> 3;\n const uint32_t immb = InvertedShift & 0b111;\n\n\n ASIMDShiftByImm(1, immh, immb, 0b10100, rn, rd);\n }\n void uxtl(SubRegSize size, DRegister rd, DRegister rn) {\n ushll(size, rd, rn, 0);\n }\n void uxtl2(SubRegSize size, QRegister rd, QRegister rn) {\n ushll2(size, rd, rn, 0);\n }\n template\n void ucvtf(SubRegSize size, T rd, T rn, uint32_t FractionalBits) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i8Bit, \"Invalid size\");\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid element size with 64-bit {}\", __func__);\n }\n\n const size_t SubregSizeInBits = SubRegSizeInBits(size);\n LOGMAN_THROW_A_FMT(FractionalBits < SubregSizeInBits, \"FractionalBits must not be larger than incoming element size\");\n\n // fbits encoded a bit weirdly.\n // fbits = (esize * 2) - immh:immb but immh is /also/ used for element size.\n const uint32_t InvertedFractionalBits = (SubregSizeInBits * 2) - FractionalBits;\n const uint32_t immh = InvertedFractionalBits >> 3;\n const uint32_t immb = InvertedFractionalBits & 0b111;\n\n ASIMDShiftByImm(1, immh, immb, 0b11100, rn, rd);\n }\n\n template\n void fcvtzu(SubRegSize size, T rd, T rn, uint32_t FractionalBits) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i8Bit, \"Invalid size\");\n if constexpr (std::is_same_v) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Invalid element size with 64-bit {}\", __func__);\n }\n\n const size_t SubregSizeInBits = SubRegSizeInBits(size);\n LOGMAN_THROW_A_FMT(FractionalBits < SubregSizeInBits, \"FractionalBits must not be larger than incoming element size\");\n\n // fbits encoded a bit weirdly.\n // fbits = (esize * 2) - immh:immb but immh is /also/ used for element size.\n const uint32_t InvertedFractionalBits = (SubregSizeInBits * 2) - FractionalBits;\n const uint32_t immh = InvertedFractionalBits >> 3;\n const uint32_t immb = InvertedFractionalBits & 0b111;\n\n ASIMDShiftByImm(1, immh, immb, 0b11111, rn, rd);\n }\n\n // Advanced SIMD vector x indexed element\n ///< size is destination size\n void smlal(ARMEmitter::SubRegSize size, ARMEmitter::VRegister rd, ARMEmitter::VRegister rn, ARMEmitter::VRegister rm, uint32_t Index) {\n LOGMAN_THROW_A_FMT(size == ARMEmitter::SubRegSize::i32Bit || size == ARMEmitter::SubRegSize::i64Bit, \"Invalid destination size\");\n\n if (size == ARMEmitter::SubRegSize::i32Bit) {\n LOGMAN_THROW_A_FMT(rm.Idx() < 16, \"Rm can't be v16-v31 with half source size\");\n }\n const auto EncodedSubRegSize = ARMEmitter::SubRegSize(FEXCore::ToUnderlying(size) - 1);\n LOGMAN_THROW_A_FMT(Index < SubRegSizeInBits(EncodedSubRegSize), \"Index must be less than the source register size\");\n\n uint32_t H, L, M;\n if (size == ARMEmitter::SubRegSize::i32Bit) {\n // Index encoded in H:L:M\n H = (Index >> 2) & 1;\n L = (Index >> 1) & 1;\n M = (Index >> 0) & 1;\n } else {\n // Index encoded in H:L\n // M overlaps rm register.\n H = (Index >> 1) & 1;\n L = (Index >> 0) & 1;\n M = 0;\n }\n ASIMDVectorXIndexedElement(0b0, L, M, 0b0010, H, EncodedSubRegSize, rm.D(), rn.D(), rd.D());\n }\n ///< size is destination size\n void smlal2(ARMEmitter::SubRegSize size, ARMEmitter::VRegister rd, ARMEmitter::VRegister rn, ARMEmitter::VRegister rm, uint32_t Index) {\n LOGMAN_THROW_A_FMT(size == ARMEmitter::SubRegSize::i32Bit || size == ARMEmitter::SubRegSize::i64Bit, \"Invalid destination size\");\n\n if (size == ARMEmitter::SubRegSize::i32Bit) {\n LOGMAN_THROW_A_FMT(rm.Idx() < 16, \"Rm can't be v16-v31 with half source size\");\n }\n const auto EncodedSubRegSize = ARMEmitter::SubRegSize(FEXCore::ToUnderlying(size) - 1);\n LOGMAN_THROW_A_FMT(Index < SubRegSizeInBits(EncodedSubRegSize), \"Index must be less than the source register size\");\n\n uint32_t H, L, M;\n if (size == ARMEmitter::SubRegSize::i32Bit) {\n // Index encoded in H:L:M\n H = (Index >> 2) & 1;\n L = (Index >> 1) & 1;\n M = (Index >> 0) & 1;\n } else {\n // Index encoded in H:L\n // M overlaps rm register.\n H = (Index >> 1) & 1;\n L = (Index >> 0) & 1;\n M = 0;\n }\n ASIMDVectorXIndexedElement(0b0, L, M, 0b0010, H, EncodedSubRegSize, rm.Q(), rn.Q(), rd.Q());\n }\n ///< size is destination size\n void sqdmlal(ARMEmitter::SubRegSize size, ARMEmitter::VRegister rd, ARMEmitter::VRegister rn, ARMEmitter::VRegister rm, uint32_t Index) {\n LOGMAN_THROW_A_FMT(size == ARMEmitter::SubRegSize::i32Bit || size == ARMEmitter::SubRegSize::i64Bit, \"Invalid destination size\");\n\n if (size == ARMEmitter::SubRegSize::i32Bit) {\n LOGMAN_THROW_A_FMT(rm.Idx() < 16, \"Rm can't be v16-v31 with half source size\");\n }\n const auto EncodedSubRegSize = ARMEmitter::SubRegSize(FEXCore::ToUnderlying(size) - 1);\n LOGMAN_THROW_A_FMT(Index < SubRegSizeInBits(EncodedSubRegSize), \"Index must be less than the source register size\");\n\n uint32_t H, L, M;\n if (size == ARMEmitter::SubRegSize::i32Bit) {\n // Index encoded in H:L:M\n H = (Index >> 2) & 1;\n L = (Index >> 1) & 1;\n M = (Index >> 0) & 1;\n } else {\n // Index encoded in H:L\n // M overlaps rm register.\n H = (Index >> 1) & 1;\n L = (Index >> 0) & 1;\n M = 0;\n }\n ASIMDVectorXIndexedElement(0b0, L, M, 0b0011, H, EncodedSubRegSize, rm.D(), rn.D(), rd.D());\n }\n ///< size is destination size\n void sqdmlal2(ARMEmitter::SubRegSize size, ARMEmitter::VRegister rd, ARMEmitter::VRegister rn, ARMEmitter::VRegister rm, uint32_t Index) {\n LOGMAN_THROW_A_FMT(size == ARMEmitter::SubRegSize::i32Bit || size == ARMEmitter::SubRegSize::i64Bit, \"Invalid destination size\");\n\n if (size == ARMEmitter::SubRegSize::i32Bit) {\n LOGMAN_THROW_A_FMT(rm.Idx() < 16, \"Rm can't be v16-v31 with half source size\");\n }\n const auto EncodedSubRegSize = ARMEmitter::SubRegSize(FEXCore::ToUnderlying(size) - 1);\n LOGMAN_THROW_A_FMT(Index < SubRegSizeInBits(EncodedSubRegSize), \"Index must be less than the source register size\");\n\n uint32_t H, L, M;\n if (size == ARMEmitter::SubRegSize::i32Bit) {\n // Index encoded in H:L:M\n H = (Index >> 2) & 1;\n L = (Index >> 1) & 1;\n M = (Index >> 0) & 1;\n } else {\n // Index encoded in H:L\n // M overlaps rm register.\n H = (Index >> 1) & 1;\n L = (Index >> 0) & 1;\n M = 0;\n }\n ASIMDVectorXIndexedElement(0b0, L, M, 0b0011, H, EncodedSubRegSize, rm.Q(), rn.Q(), rd.Q());\n }\n ///< size is destination size\n void smlsl(ARMEmitter::SubRegSize size, ARMEmitter::VRegister rd, ARMEmitter::VRegister rn, ARMEmitter::VRegister rm, uint32_t Index) {\n LOGMAN_THROW_A_FMT(size == ARMEmitter::SubRegSize::i32Bit || size == ARMEmitter::SubRegSize::i64Bit, \"Invalid destination size\");\n\n if (size == ARMEmitter::SubRegSize::i32Bit) {\n LOGMAN_THROW_A_FMT(rm.Idx() < 16, \"Rm can't be v16-v31 with half source size\");\n }\n const auto EncodedSubRegSize = ARMEmitter::SubRegSize(FEXCore::ToUnderlying(size) - 1);\n LOGMAN_THROW_A_FMT(Index < SubRegSizeInBits(EncodedSubRegSize), \"Index must be less than the source register size\");\n\n uint32_t H, L, M;\n if (size == ARMEmitter::SubRegSize::i32Bit) {\n // Index encoded in H:L:M\n H = (Index >> 2) & 1;\n L = (Index >> 1) & 1;\n M = (Index >> 0) & 1;\n } else {\n // Index encoded in H:L\n // M overlaps rm register.\n H = (Index >> 1) & 1;\n L = (Index >> 0) & 1;\n M = 0;\n }\n ASIMDVectorXIndexedElement(0b0, L, M, 0b0110, H, EncodedSubRegSize, rm.D(), rn.D(), rd.D());\n }\n ///< size is destination size\n void smlsl2(ARMEmitter::SubRegSize size, ARMEmitter::VRegister rd, ARMEmitter::VRegister rn, ARMEmitter::VRegister rm, uint32_t Index) {\n LOGMAN_THROW_A_FMT(size == ARMEmitter::SubRegSize::i32Bit || size == ARMEmitter::SubRegSize::i64Bit, \"Invalid destination size\");\n\n if (size == ARMEmitter::SubRegSize::i32Bit) {\n LOGMAN_THROW_A_FMT(rm.Idx() < 16, \"Rm can't be v16-v31 with half source size\");\n }\n const auto EncodedSubRegSize = ARMEmitter::SubRegSize(FEXCore::ToUnderlying(size) - 1);\n LOGMAN_THROW_A_FMT(Index < SubRegSizeInBits(EncodedSubRegSize), \"Index must be less than the source register size\");\n\n uint32_t H, L, M;\n if (size == ARMEmitter::SubRegSize::i32Bit) {\n // Index encoded in H:L:M\n H = (Index >> 2) & 1;\n L = (Index >> 1) & 1;\n M = (Index >> 0) & 1;\n } else {\n // Index encoded in H:L\n // M overlaps rm register.\n H = (Index >> 1) & 1;\n L = (Index >> 0) & 1;\n M = 0;\n }\n ASIMDVectorXIndexedElement(0b0, L, M, 0b0110, H, EncodedSubRegSize, rm.Q(), rn.Q(), rd.Q());\n }\n ///< size is destination size\n void sqdmlsl(ARMEmitter::SubRegSize size, ARMEmitter::VRegister rd, ARMEmitter::VRegister rn, ARMEmitter::VRegister rm, uint32_t Index) {\n LOGMAN_THROW_A_FMT(size == ARMEmitter::SubRegSize::i32Bit || size == ARMEmitter::SubRegSize::i64Bit, \"Invalid destination size\");\n\n if (size == ARMEmitter::SubRegSize::i32Bit) {\n LOGMAN_THROW_A_FMT(rm.Idx() < 16, \"Rm can't be v16-v31 with half source size\");\n }\n const auto EncodedSubRegSize = ARMEmitter::SubRegSize(FEXCore::ToUnderlying(size) - 1);\n LOGMAN_THROW_A_FMT(Index < SubRegSizeInBits(EncodedSubRegSize), \"Index must be less than the source register size\");\n\n uint32_t H, L, M;\n if (size == ARMEmitter::SubRegSize::i32Bit) {\n // Index encoded in H:L:M\n H = (Index >> 2) & 1;\n L = (Index >> 1) & 1;\n M = (Index >> 0) & 1;\n } else {\n // Index encoded in H:L\n // M overlaps rm register.\n H = (Index >> 1) & 1;\n L = (Index >> 0) & 1;\n M = 0;\n }\n ASIMDVectorXIndexedElement(0b0, L, M, 0b0111, H, EncodedSubRegSize, rm.D(), rn.D(), rd.D());\n }\n ///< size is destination size\n void sqdmlsl2(ARMEmitter::SubRegSize size, ARMEmitter::VRegister rd, ARMEmitter::VRegister rn, ARMEmitter::VRegister rm, uint32_t Index) {\n LOGMAN_THROW_A_FMT(size == ARMEmitter::SubRegSize::i32Bit || size == ARMEmitter::SubRegSize::i64Bit, \"Invalid destination size\");\n\n if (size == ARMEmitter::SubRegSize::i32Bit) {\n LOGMAN_THROW_A_FMT(rm.Idx() < 16, \"Rm can't be v16-v31 with half source size\");\n }\n const auto EncodedSubRegSize = ARMEmitter::SubRegSize(FEXCore::ToUnderlying(size) - 1);\n LOGMAN_THROW_A_FMT(Index < SubRegSizeInBits(EncodedSubRegSize), \"Index must be less than the source register size\");\n\n uint32_t H, L, M;\n if (size == ARMEmitter::SubRegSize::i32Bit) {\n // Index encoded in H:L:M\n H = (Index >> 2) & 1;\n L = (Index >> 1) & 1;\n M = (Index >> 0) & 1;\n } else {\n // Index encoded in H:L\n // M overlaps rm register.\n H = (Index >> 1) & 1;\n L = (Index >> 0) & 1;\n M = 0;\n }\n ASIMDVectorXIndexedElement(0b0, L, M, 0b0111, H, EncodedSubRegSize, rm.Q(), rn.Q(), rd.Q());\n }\n template\n void mul(ARMEmitter::SubRegSize size, T rd, T rn, T rm, uint32_t Index) {\n LOGMAN_THROW_A_FMT(size == ARMEmitter::SubRegSize::i16Bit || size == ARMEmitter::SubRegSize::i32Bit, \"Invalid destination size\");\n\n if (size == ARMEmitter::SubRegSize::i16Bit) {\n LOGMAN_THROW_A_FMT(rm.Idx() < 16, \"Rm can't be v16-v31 with half source size\");\n }\n LOGMAN_THROW_A_FMT(Index < SubRegSizeInBits(size), \"Index must be less than the source register size\");\n\n uint32_t H, L, M;\n if (size == ARMEmitter::SubRegSize::i16Bit) {\n // Index encoded in H:L:M\n H = (Index >> 2) & 1;\n L = (Index >> 1) & 1;\n M = (Index >> 0) & 1;\n } else {\n // Index encoded in H:L\n // M overlaps rm register.\n H = (Index >> 1) & 1;\n L = (Index >> 0) & 1;\n M = 0;\n }\n ASIMDVectorXIndexedElement(0b0, L, M, 0b1000, H, size, rm, rn, rd);\n }\n ///< size is destination size\n void smull(ARMEmitter::SubRegSize size, ARMEmitter::VRegister rd, ARMEmitter::VRegister rn, ARMEmitter::VRegister rm, uint32_t Index) {\n LOGMAN_THROW_A_FMT(size == ARMEmitter::SubRegSize::i32Bit || size == ARMEmitter::SubRegSize::i64Bit, \"Invalid destination size\");\n\n if (size == ARMEmitter::SubRegSize::i32Bit) {\n LOGMAN_THROW_A_FMT(rm.Idx() < 16, \"Rm can't be v16-v31 with half source size\");\n }\n const auto EncodedSubRegSize = ARMEmitter::SubRegSize(FEXCore::ToUnderlying(size) - 1);\n LOGMAN_THROW_A_FMT(Index < SubRegSizeInBits(EncodedSubRegSize), \"Index must be less than the source register size\");\n\n uint32_t H, L, M;\n if (size == ARMEmitter::SubRegSize::i32Bit) {\n // Index encoded in H:L:M\n H = (Index >> 2) & 1;\n L = (Index >> 1) & 1;\n M = (Index >> 0) & 1;\n } else {\n // Index encoded in H:L\n // M overlaps rm register.\n H = (Index >> 1) & 1;\n L = (Index >> 0) & 1;\n M = 0;\n }\n ASIMDVectorXIndexedElement(0b0, L, M, 0b1010, H, EncodedSubRegSize, rm.D(), rn.D(), rd.D());\n }\n ///< size is destination size\n void smull2(ARMEmitter::SubRegSize size, ARMEmitter::VRegister rd, ARMEmitter::VRegister rn, ARMEmitter::VRegister rm, uint32_t Index) {\n LOGMAN_THROW_A_FMT(size == ARMEmitter::SubRegSize::i32Bit || size == ARMEmitter::SubRegSize::i64Bit, \"Invalid destination size\");\n\n if (size == ARMEmitter::SubRegSize::i32Bit) {\n LOGMAN_THROW_A_FMT(rm.Idx() < 16, \"Rm can't be v16-v31 with half source size\");\n }\n const auto EncodedSubRegSize = ARMEmitter::SubRegSize(FEXCore::ToUnderlying(size) - 1);\n LOGMAN_THROW_A_FMT(Index < SubRegSizeInBits(EncodedSubRegSize), \"Index must be less than the source register size\");\n\n uint32_t H, L, M;\n if (size == ARMEmitter::SubRegSize::i32Bit) {\n // Index encoded in H:L:M\n H = (Index >> 2) & 1;\n L = (Index >> 1) & 1;\n M = (Index >> 0) & 1;\n } else {\n // Index encoded in H:L\n // M overlaps rm register.\n H = (Index >> 1) & 1;\n L = (Index >> 0) & 1;\n M = 0;\n }\n ASIMDVectorXIndexedElement(0b0, L, M, 0b1010, H, EncodedSubRegSize, rm.Q(), rn.Q(), rd.Q());\n }\n ///< size is destination size\n void sqdmull(ARMEmitter::SubRegSize size, ARMEmitter::VRegister rd, ARMEmitter::VRegister rn, ARMEmitter::VRegister rm, uint32_t Index) {\n LOGMAN_THROW_A_FMT(size == ARMEmitter::SubRegSize::i32Bit || size == ARMEmitter::SubRegSize::i64Bit, \"Invalid destination size\");\n\n if (size == ARMEmitter::SubRegSize::i32Bit) {\n LOGMAN_THROW_A_FMT(rm.Idx() < 16, \"Rm can't be v16-v31 with half source size\");\n }\n const auto EncodedSubRegSize = ARMEmitter::SubRegSize(FEXCore::ToUnderlying(size) - 1);\n LOGMAN_THROW_A_FMT(Index < SubRegSizeInBits(EncodedSubRegSize), \"Index must be less than the source register size\");\n\n uint32_t H, L, M;\n if (size == ARMEmitter::SubRegSize::i32Bit) {\n // Index encoded in H:L:M\n H = (Index >> 2) & 1;\n L = (Index >> 1) & 1;\n M = (Index >> 0) & 1;\n } else {\n // Index encoded in H:L\n // M overlaps rm register.\n H = (Index >> 1) & 1;\n L = (Index >> 0) & 1;\n M = 0;\n }\n ASIMDVectorXIndexedElement(0b0, L, M, 0b1011, H, EncodedSubRegSize, rm.D(), rn.D(), rd.D());\n }\n ///< size is destination size\n void sqdmull2(ARMEmitter::SubRegSize size, ARMEmitter::VRegister rd, ARMEmitter::VRegister rn, ARMEmitter::VRegister rm, uint32_t Index) {\n LOGMAN_THROW_A_FMT(size == ARMEmitter::SubRegSize::i32Bit || size == ARMEmitter::SubRegSize::i64Bit, \"Invalid destination size\");\n\n if (size == ARMEmitter::SubRegSize::i32Bit) {\n LOGMAN_THROW_A_FMT(rm.Idx() < 16, \"Rm can't be v16-v31 with half source size\");\n }\n const auto EncodedSubRegSize = ARMEmitter::SubRegSize(FEXCore::ToUnderlying(size) - 1);\n LOGMAN_THROW_A_FMT(Index < SubRegSizeInBits(EncodedSubRegSize), \"Index must be less than the source register size\");\n\n uint32_t H, L, M;\n if (size == ARMEmitter::SubRegSize::i32Bit) {\n // Index encoded in H:L:M\n H = (Index >> 2) & 1;\n L = (Index >> 1) & 1;\n M = (Index >> 0) & 1;\n } else {\n // Index encoded in H:L\n // M overlaps rm register.\n H = (Index >> 1) & 1;\n L = (Index >> 0) & 1;\n M = 0;\n }\n ASIMDVectorXIndexedElement(0b0, L, M, 0b1011, H, EncodedSubRegSize, rm.Q(), rn.Q(), rd.Q());\n }\n template\n void sqdmulh(ARMEmitter::SubRegSize size, T rd, T rn, T rm, uint32_t Index) {\n LOGMAN_THROW_A_FMT(size == ARMEmitter::SubRegSize::i16Bit || size == ARMEmitter::SubRegSize::i32Bit, \"Invalid destination size\");\n\n if (size == ARMEmitter::SubRegSize::i16Bit) {\n LOGMAN_THROW_A_FMT(rm.Idx() < 16, \"Rm can't be v16-v31 with half source size\");\n }\n LOGMAN_THROW_A_FMT(Index < SubRegSizeInBits(size), \"Index must be less than the source register size\");\n\n uint32_t H, L, M;\n if (size == ARMEmitter::SubRegSize::i16Bit) {\n // Index encoded in H:L:M\n H = (Index >> 2) & 1;\n L = (Index >> 1) & 1;\n M = (Index >> 0) & 1;\n } else {\n // Index encoded in H:L\n // M overlaps rm register.\n H = (Index >> 1) & 1;\n L = (Index >> 0) & 1;\n M = 0;\n }\n ASIMDVectorXIndexedElement(0b0, L, M, 0b1100, H, size, rm, rn, rd);\n }\n template\n void sqrdmulh(ARMEmitter::SubRegSize size, T rd, T rn, T rm, uint32_t Index) {\n LOGMAN_THROW_A_FMT(size == ARMEmitter::SubRegSize::i16Bit || size == ARMEmitter::SubRegSize::i32Bit, \"Invalid destination size\");\n\n if (size == ARMEmitter::SubRegSize::i16Bit) {\n LOGMAN_THROW_A_FMT(rm.Idx() < 16, \"Rm can't be v16-v31 with half source size\");\n }\n LOGMAN_THROW_A_FMT(Index < SubRegSizeInBits(size), \"Index must be less than the source register size\");\n\n uint32_t H, L, M;\n if (size == ARMEmitter::SubRegSize::i16Bit) {\n // Index encoded in H:L:M\n H = (Index >> 2) & 1;\n L = (Index >> 1) & 1;\n M = (Index >> 0) & 1;\n } else {\n // Index encoded in H:L\n // M overlaps rm register.\n H = (Index >> 1) & 1;\n L = (Index >> 0) & 1;\n M = 0;\n }\n ASIMDVectorXIndexedElement(0b0, L, M, 0b1101, H, size, rm, rn, rd);\n }\n\n template\n void sdot(T rd, T rn, T rm, uint32_t Index) {\n LOGMAN_THROW_A_FMT(Index < 4, \"Index must be less than the source register size\");\n\n uint32_t H, L, M;\n // Index encoded in H:L\n // M overlaps rm register.\n H = (Index >> 1) & 1;\n L = (Index >> 0) & 1;\n M = 0;\n ASIMDVectorXIndexedElement(0b0, L, M, 0b1110, H, ARMEmitter::SubRegSize::i32Bit, rm, rn, rd);\n }\n\n template\n void sudot(T rd, T rn, T rm, uint32_t Index) {\n LOGMAN_THROW_A_FMT(Index < 4, \"Index must be less than the source register size\");\n\n uint32_t H, L, M;\n // Index encoded in H:L\n // M overlaps rm register.\n H = (Index >> 1) & 1;\n L = (Index >> 0) & 1;\n M = 0;\n ASIMDVectorXIndexedElement(0b0, L, M, 0b1111, H, ARMEmitter::SubRegSize::i8Bit, rm, rn, rd);\n }\n\n template\n void bfdot(T rd, T rn, T rm, uint32_t Index) {\n LOGMAN_THROW_A_FMT(Index < 4, \"Index must be less than the source register size\");\n\n uint32_t H, L, M;\n // Index encoded in H:L\n // M overlaps rm register.\n H = (Index >> 1) & 1;\n L = (Index >> 0) & 1;\n M = 0;\n ASIMDVectorXIndexedElement(0b0, L, M, 0b1111, H, ARMEmitter::SubRegSize::i16Bit, rm, rn, rd);\n }\n\n template\n void fmla(SubRegSize size, T rd, T rn, T rm, uint32_t Index) {\n LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), \"Invalid destination size\");\n LOGMAN_THROW_A_FMT(Index < SubRegSizeInBits(size), \"Index must be less than the source register size\");\n\n uint32_t H, L, M;\n auto EncodedSubRegSize = size;\n\n if (size == SubRegSize::i16Bit) {\n // Index encoded in H:L:M\n H = (Index >> 2) & 1;\n L = (Index >> 1) & 1;\n M = (Index >> 0) & 1;\n // ARM in their infinite wisdom decided to encode 16-bit as an 8-bit operation even though 16-bit was unallocated.\n EncodedSubRegSize = SubRegSize::i8Bit;\n } else if (size == SubRegSize::i32Bit) {\n // Index encoded in H:L\n H = (Index >> 1) & 1;\n L = (Index >> 0) & 1;\n M = 0;\n } else {\n LOGMAN_THROW_A_FMT((std::is_same_v), \"Can't encode DRegister with i64Bit\");\n // Index encoded in H\n H = Index;\n L = 0;\n M = 0;\n }\n ASIMDVectorXIndexedElement(0b0, L, M, 0b0001, H, EncodedSubRegSize, rm, rn, rd);\n }\n\n template\n void fmls(SubRegSize size, T rd, T rn, T rm, uint32_t Index) {\n LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), \"Invalid destination size\");\n LOGMAN_THROW_A_FMT(Index < SubRegSizeInBits(size), \"Index must be less than the source register size\");\n\n uint32_t H, L, M;\n auto EncodedSubRegSize = size;\n\n if (size == SubRegSize::i16Bit) {\n // Index encoded in H:L:M\n H = (Index >> 2) & 1;\n L = (Index >> 1) & 1;\n M = (Index >> 0) & 1;\n // ARM in their infinite wisdom decided to encode 16-bit as an 8-bit operation even though 16-bit was unallocated.\n EncodedSubRegSize = SubRegSize::i8Bit;\n } else if (size == SubRegSize::i32Bit) {\n // Index encoded in H:L\n H = (Index >> 1) & 1;\n L = (Index >> 0) & 1;\n M = 0;\n } else {\n LOGMAN_THROW_A_FMT((std::is_same_v), \"Can't encode DRegister with i64Bit\");\n // Index encoded in H\n H = Index;\n L = 0;\n M = 0;\n }\n ASIMDVectorXIndexedElement(0b0, L, M, 0b0101, H, EncodedSubRegSize, rm, rn, rd);\n }\n\n template\n void fmul(SubRegSize size, T rd, T rn, T rm, uint32_t Index) {\n LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), \"Invalid destination size\");\n LOGMAN_THROW_A_FMT(Index < SubRegSizeInBits(size), \"Index must be less than the source register size\");\n\n uint32_t H, L, M;\n auto EncodedSubRegSize = size;\n\n if (size == SubRegSize::i16Bit) {\n // Index encoded in H:L:M\n H = (Index >> 2) & 1;\n L = (Index >> 1) & 1;\n M = (Index >> 0) & 1;\n // ARM in their infinite wisdom decided to encode 16-bit as an 8-bit operation even though 16-bit was unallocated.\n EncodedSubRegSize = SubRegSize::i8Bit;\n } else if (size == SubRegSize::i32Bit) {\n // Index encoded in H:L\n H = (Index >> 1) & 1;\n L = (Index >> 0) & 1;\n M = 0;\n } else {\n LOGMAN_THROW_A_FMT((std::is_same_v), \"Can't encode DRegister with i64Bit\");\n // Index encoded in H\n H = Index;\n L = 0;\n M = 0;\n }\n ASIMDVectorXIndexedElement(0b0, L, M, 0b1001, H, EncodedSubRegSize, rm, rn, rd);\n }\n\n template\n void fmlal(T rd, T rn, T rm, uint32_t Index) {\n LOGMAN_THROW_A_FMT(Index < 8, \"Index must be less than the source register size\");\n\n uint32_t H, L, M;\n // Index encoded in H:L\n // M overlaps rm register.\n H = (Index >> 2) & 1;\n L = (Index >> 1) & 1;\n M = (Index >> 0) & 1;\n ASIMDVectorXIndexedElement(0b0, L, M, 0b0000, H, ARMEmitter::SubRegSize::i32Bit, rm, rn, rd);\n }\n\n template\n void fmlal2(T rd, T rn, T rm, uint32_t Index) {\n LOGMAN_THROW_A_FMT(Index < 8, \"Index must be less than the source register size\");\n\n uint32_t H, L, M;\n // Index encoded in H:L\n // M overlaps rm register.\n H = (Index >> 2) & 1;\n L = (Index >> 1) & 1;\n M = (Index >> 0) & 1;\n ASIMDVectorXIndexedElement(0b1, L, M, 0b1000, H, ARMEmitter::SubRegSize::i32Bit, rm, rn, rd);\n }\n\n template\n void fmlsl(T rd, T rn, T rm, uint32_t Index) {\n LOGMAN_THROW_A_FMT(Index < 8, \"Index must be less than the source register size\");\n\n uint32_t H, L, M;\n // Index encoded in H:L\n // M overlaps rm register.\n H = (Index >> 2) & 1;\n L = (Index >> 1) & 1;\n M = (Index >> 0) & 1;\n ASIMDVectorXIndexedElement(0b0, L, M, 0b0100, H, ARMEmitter::SubRegSize::i32Bit, rm, rn, rd);\n }\n\n template\n void fmlsl2(T rd, T rn, T rm, uint32_t Index) {\n LOGMAN_THROW_A_FMT(Index < 8, \"Index must be less than the source register size\");\n\n uint32_t H, L, M;\n // Index encoded in H:L\n // M overlaps rm register.\n H = (Index >> 2) & 1;\n L = (Index >> 1) & 1;\n M = (Index >> 0) & 1;\n ASIMDVectorXIndexedElement(0b1, L, M, 0b1100, H, ARMEmitter::SubRegSize::i32Bit, rm, rn, rd);\n }\n\n template\n void usdot(T rd, T rn, T rm, uint32_t Index) {\n LOGMAN_THROW_A_FMT(Index < 4, \"Index must be less than the source register size\");\n\n uint32_t H, L, M;\n // Index encoded in H:L\n // M overlaps rm register.\n H = (Index >> 1) & 1;\n L = (Index >> 0) & 1;\n M = 0;\n ASIMDVectorXIndexedElement(0b0, L, M, 0b1111, H, ARMEmitter::SubRegSize::i32Bit, rm, rn, rd);\n }\n\n void bfmlalb(ARMEmitter::VRegister rd, ARMEmitter::VRegister rn, ARMEmitter::VRegister rm, uint32_t Index) {\n LOGMAN_THROW_A_FMT(rm.Idx() < 16, \"Rm can't be v16-v31 with half source size\");\n LOGMAN_THROW_A_FMT(Index < 8, \"Index must be less than the source register size\");\n\n uint32_t H, L, M;\n // Index encoded in H:L\n // M overlaps rm register.\n H = (Index >> 2) & 1;\n L = (Index >> 1) & 1;\n M = (Index >> 0) & 1;\n ASIMDVectorXIndexedElement(0b0, L, M, 0b1111, H, ARMEmitter::SubRegSize::i64Bit, rm.D(), rn.D(), rd.D());\n }\n void bfmlalt(ARMEmitter::VRegister rd, ARMEmitter::VRegister rn, ARMEmitter::VRegister rm, uint32_t Index) {\n LOGMAN_THROW_A_FMT(rm.Idx() < 16, \"Rm can't be v16-v31 with half source size\");\n LOGMAN_THROW_A_FMT(Index < 8, \"Index must be less than the source register size\");\n\n uint32_t H, L, M;\n // Index encoded in H:L\n // M overlaps rm register.\n H = (Index >> 2) & 1;\n L = (Index >> 1) & 1;\n M = (Index >> 0) & 1;\n ASIMDVectorXIndexedElement(0b0, L, M, 0b1111, H, ARMEmitter::SubRegSize::i64Bit, rm.Q(), rn.Q(), rd.Q());\n }\n\n template\n void mla(ARMEmitter::SubRegSize size, T rd, T rn, T rm, uint32_t Index) {\n LOGMAN_THROW_A_FMT(size == ARMEmitter::SubRegSize::i16Bit || size == ARMEmitter::SubRegSize::i32Bit, \"Invalid destination size\");\n\n if (size == ARMEmitter::SubRegSize::i16Bit) {\n LOGMAN_THROW_A_FMT(rm.Idx() < 16, \"Rm can't be v16-v31 with half source size\");\n }\n LOGMAN_THROW_A_FMT(Index < SubRegSizeInBits(size), \"Index must be less than the source register size\");\n\n uint32_t H, L, M;\n if (size == ARMEmitter::SubRegSize::i16Bit) {\n // Index encoded in H:L:M\n H = (Index >> 2) & 1;\n L = (Index >> 1) & 1;\n M = (Index >> 0) & 1;\n } else {\n // Index encoded in H:L\n // M overlaps rm register.\n H = (Index >> 1) & 1;\n L = (Index >> 0) & 1;\n M = 0;\n }\n ASIMDVectorXIndexedElement(0b1, L, M, 0b0000, H, size, rm, rn, rd);\n }\n\n ///< size is destination size\n void umlal(ARMEmitter::SubRegSize size, ARMEmitter::VRegister rd, ARMEmitter::VRegister rn, ARMEmitter::VRegister rm, uint32_t Index) {\n LOGMAN_THROW_A_FMT(size == ARMEmitter::SubRegSize::i32Bit || size == ARMEmitter::SubRegSize::i64Bit, \"Invalid destination size\");\n\n if (size == ARMEmitter::SubRegSize::i32Bit) {\n LOGMAN_THROW_A_FMT(rm.Idx() < 16, \"Rm can't be v16-v31 with half source size\");\n }\n const auto EncodedSubRegSize = ARMEmitter::SubRegSize(FEXCore::ToUnderlying(size) - 1);\n LOGMAN_THROW_A_FMT(Index < SubRegSizeInBits(EncodedSubRegSize), \"Index must be less than the source register size\");\n\n uint32_t H, L, M;\n if (size == ARMEmitter::SubRegSize::i32Bit) {\n // Index encoded in H:L:M\n H = (Index >> 2) & 1;\n L = (Index >> 1) & 1;\n M = (Index >> 0) & 1;\n } else {\n // Index encoded in H:L\n // M overlaps rm register.\n H = (Index >> 1) & 1;\n L = (Index >> 0) & 1;\n M = 0;\n }\n ASIMDVectorXIndexedElement(0b1, L, M, 0b0010, H, EncodedSubRegSize, rm.D(), rn.D(), rd.D());\n }\n ///< size is destination size\n void umlal2(ARMEmitter::SubRegSize size, ARMEmitter::VRegister rd, ARMEmitter::VRegister rn, ARMEmitter::VRegister rm, uint32_t Index) {\n LOGMAN_THROW_A_FMT(size == ARMEmitter::SubRegSize::i32Bit || size == ARMEmitter::SubRegSize::i64Bit, \"Invalid destination size\");\n\n if (size == ARMEmitter::SubRegSize::i32Bit) {\n LOGMAN_THROW_A_FMT(rm.Idx() < 16, \"Rm can't be v16-v31 with half source size\");\n }\n const auto EncodedSubRegSize = ARMEmitter::SubRegSize(FEXCore::ToUnderlying(size) - 1);\n LOGMAN_THROW_A_FMT(Index < SubRegSizeInBits(EncodedSubRegSize), \"Index must be less than the source register size\");\n\n uint32_t H, L, M;\n if (size == ARMEmitter::SubRegSize::i32Bit) {\n // Index encoded in H:L:M\n H = (Index >> 2) & 1;\n L = (Index >> 1) & 1;\n M = (Index >> 0) & 1;\n } else {\n // Index encoded in H:L\n // M overlaps rm register.\n H = (Index >> 1) & 1;\n L = (Index >> 0) & 1;\n M = 0;\n }\n ASIMDVectorXIndexedElement(0b1, L, M, 0b0010, H, EncodedSubRegSize, rm.Q(), rn.Q(), rd.Q());\n }\n\n template\n void mls(ARMEmitter::SubRegSize size, T rd, T rn, T rm, uint32_t Index) {\n LOGMAN_THROW_A_FMT(size == ARMEmitter::SubRegSize::i16Bit || size == ARMEmitter::SubRegSize::i32Bit, \"Invalid destination size\");\n\n if (size == ARMEmitter::SubRegSize::i16Bit) {\n LOGMAN_THROW_A_FMT(rm.Idx() < 16, \"Rm can't be v16-v31 with half source size\");\n }\n LOGMAN_THROW_A_FMT(Index < SubRegSizeInBits(size), \"Index must be less than the source register size\");\n\n uint32_t H, L, M;\n if (size == ARMEmitter::SubRegSize::i16Bit) {\n // Index encoded in H:L:M\n H = (Index >> 2) & 1;\n L = (Index >> 1) & 1;\n M = (Index >> 0) & 1;\n } else {\n // Index encoded in H:L\n // M overlaps rm register.\n H = (Index >> 1) & 1;\n L = (Index >> 0) & 1;\n M = 0;\n }\n ASIMDVectorXIndexedElement(0b1, L, M, 0b0100, H, size, rm, rn, rd);\n }\n\n ///< size is destination size\n void umlsl(ARMEmitter::SubRegSize size, ARMEmitter::VRegister rd, ARMEmitter::VRegister rn, ARMEmitter::VRegister rm, uint32_t Index) {\n LOGMAN_THROW_A_FMT(size == ARMEmitter::SubRegSize::i32Bit || size == ARMEmitter::SubRegSize::i64Bit, \"Invalid destination size\");\n\n if (size == ARMEmitter::SubRegSize::i32Bit) {\n LOGMAN_THROW_A_FMT(rm.Idx() < 16, \"Rm can't be v16-v31 with half source size\");\n }\n const auto EncodedSubRegSize = ARMEmitter::SubRegSize(FEXCore::ToUnderlying(size) - 1);\n LOGMAN_THROW_A_FMT(Index < SubRegSizeInBits(EncodedSubRegSize), \"Index must be less than the source register size\");\n\n uint32_t H, L, M;\n if (size == ARMEmitter::SubRegSize::i32Bit) {\n // Index encoded in H:L:M\n H = (Index >> 2) & 1;\n L = (Index >> 1) & 1;\n M = (Index >> 0) & 1;\n } else {\n // Index encoded in H:L\n // M overlaps rm register.\n H = (Index >> 1) & 1;\n L = (Index >> 0) & 1;\n M = 0;\n }\n ASIMDVectorXIndexedElement(0b1, L, M, 0b0110, H, EncodedSubRegSize, rm.D(), rn.D(), rd.D());\n }\n ///< size is destination size\n void umlsl2(ARMEmitter::SubRegSize size, ARMEmitter::VRegister rd, ARMEmitter::VRegister rn, ARMEmitter::VRegister rm, uint32_t Index) {\n LOGMAN_THROW_A_FMT(size == ARMEmitter::SubRegSize::i32Bit || size == ARMEmitter::SubRegSize::i64Bit, \"Invalid destination size\");\n\n if (size == ARMEmitter::SubRegSize::i32Bit) {\n LOGMAN_THROW_A_FMT(rm.Idx() < 16, \"Rm can't be v16-v31 with half source size\");\n }\n const auto EncodedSubRegSize = ARMEmitter::SubRegSize(FEXCore::ToUnderlying(size) - 1);\n LOGMAN_THROW_A_FMT(Index < SubRegSizeInBits(EncodedSubRegSize), \"Index must be less than the source register size\");\n\n uint32_t H, L, M;\n if (size == ARMEmitter::SubRegSize::i32Bit) {\n // Index encoded in H:L:M\n H = (Index >> 2) & 1;\n L = (Index >> 1) & 1;\n M = (Index >> 0) & 1;\n } else {\n // Index encoded in H:L\n // M overlaps rm register.\n H = (Index >> 1) & 1;\n L = (Index >> 0) & 1;\n M = 0;\n }\n ASIMDVectorXIndexedElement(0b1, L, M, 0b0110, H, EncodedSubRegSize, rm.Q(), rn.Q(), rd.Q());\n }\n\n ///< size is destination size\n void umull(ARMEmitter::SubRegSize size, ARMEmitter::VRegister rd, ARMEmitter::VRegister rn, ARMEmitter::VRegister rm, uint32_t Index) {\n LOGMAN_THROW_A_FMT(size == ARMEmitter::SubRegSize::i32Bit || size == ARMEmitter::SubRegSize::i64Bit, \"Invalid destination size\");\n\n if (size == ARMEmitter::SubRegSize::i32Bit) {\n LOGMAN_THROW_A_FMT(rm.Idx() < 16, \"Rm can't be v16-v31 with half source size\");\n }\n const auto EncodedSubRegSize = ARMEmitter::SubRegSize(FEXCore::ToUnderlying(size) - 1);\n LOGMAN_THROW_A_FMT(Index < SubRegSizeInBits(EncodedSubRegSize), \"Index must be less than the source register size\");\n\n uint32_t H, L, M;\n if (size == ARMEmitter::SubRegSize::i32Bit) {\n // Index encoded in H:L:M\n H = (Index >> 2) & 1;\n L = (Index >> 1) & 1;\n M = (Index >> 0) & 1;\n } else {\n // Index encoded in H:L\n // M overlaps rm register.\n H = (Index >> 1) & 1;\n L = (Index >> 0) & 1;\n M = 0;\n }\n ASIMDVectorXIndexedElement(0b1, L, M, 0b1010, H, EncodedSubRegSize, rm.D(), rn.D(), rd.D());\n }\n ///< size is destination size\n void umull2(ARMEmitter::SubRegSize size, ARMEmitter::VRegister rd, ARMEmitter::VRegister rn, ARMEmitter::VRegister rm, uint32_t Index) {\n LOGMAN_THROW_A_FMT(size == ARMEmitter::SubRegSize::i32Bit || size == ARMEmitter::SubRegSize::i64Bit, \"Invalid destination size\");\n\n if (size == ARMEmitter::SubRegSize::i32Bit) {\n LOGMAN_THROW_A_FMT(rm.Idx() < 16, \"Rm can't be v16-v31 with half source size\");\n }\n const auto EncodedSubRegSize = ARMEmitter::SubRegSize(FEXCore::ToUnderlying(size) - 1);\n LOGMAN_THROW_A_FMT(Index < SubRegSizeInBits(EncodedSubRegSize), \"Index must be less than the source register size\");\n\n uint32_t H, L, M;\n if (size == ARMEmitter::SubRegSize::i32Bit) {\n // Index encoded in H:L:M\n H = (Index >> 2) & 1;\n L = (Index >> 1) & 1;\n M = (Index >> 0) & 1;\n } else {\n // Index encoded in H:L\n // M overlaps rm register.\n H = (Index >> 1) & 1;\n L = (Index >> 0) & 1;\n M = 0;\n }\n ASIMDVectorXIndexedElement(0b1, L, M, 0b1010, H, EncodedSubRegSize, rm.Q(), rn.Q(), rd.Q());\n }\n\n template\n void sqrdmlah(ARMEmitter::SubRegSize size, T rd, T rn, T rm, uint32_t Index) {\n LOGMAN_THROW_A_FMT(size == ARMEmitter::SubRegSize::i16Bit || size == ARMEmitter::SubRegSize::i32Bit, \"Invalid destination size\");\n\n if (size == ARMEmitter::SubRegSize::i16Bit) {\n LOGMAN_THROW_A_FMT(rm.Idx() < 16, \"Rm can't be v16-v31 with half source size\");\n }\n LOGMAN_THROW_A_FMT(Index < SubRegSizeInBits(size), \"Index must be less than the source register size\");\n\n uint32_t H, L, M;\n if (size == ARMEmitter::SubRegSize::i16Bit) {\n // Index encoded in H:L:M\n H = (Index >> 2) & 1;\n L = (Index >> 1) & 1;\n M = (Index >> 0) & 1;\n } else {\n // Index encoded in H:L\n // M overlaps rm register.\n H = (Index >> 1) & 1;\n L = (Index >> 0) & 1;\n M = 0;\n }\n ASIMDVectorXIndexedElement(0b1, L, M, 0b1101, H, size, rm, rn, rd);\n }\n template\n void udot(T rd, T rn, T rm, uint32_t Index) {\n LOGMAN_THROW_A_FMT(Index < 4, \"Index must be less than the source register size\");\n\n uint32_t H, L, M;\n // Index encoded in H:L\n // M overlaps rm register.\n H = (Index >> 1) & 1;\n L = (Index >> 0) & 1;\n M = 0;\n ASIMDVectorXIndexedElement(0b1, L, M, 0b1110, H, ARMEmitter::SubRegSize::i32Bit, rm, rn, rd);\n }\n\n template\n void sqrdmlsh(ARMEmitter::SubRegSize size, T rd, T rn, T rm, uint32_t Index) {\n LOGMAN_THROW_A_FMT(size == ARMEmitter::SubRegSize::i16Bit || size == ARMEmitter::SubRegSize::i32Bit, \"Invalid destination size\");\n\n if (size == ARMEmitter::SubRegSize::i16Bit) {\n LOGMAN_THROW_A_FMT(rm.Idx() < 16, \"Rm can't be v16-v31 with half source size\");\n }\n LOGMAN_THROW_A_FMT(Index < SubRegSizeInBits(size), \"Index must be less than the source register size\");\n\n uint32_t H, L, M;\n if (size == ARMEmitter::SubRegSize::i16Bit) {\n // Index encoded in H:L:M\n H = (Index >> 2) & 1;\n L = (Index >> 1) & 1;\n M = (Index >> 0) & 1;\n } else {\n // Index encoded in H:L\n // M overlaps rm register.\n H = (Index >> 1) & 1;\n L = (Index >> 0) & 1;\n M = 0;\n }\n ASIMDVectorXIndexedElement(0b1, L, M, 0b1111, H, size, rm, rn, rd);\n }\n\n // Cryptographic three-register, imm2\n void sm3tt1a(VRegister rd, VRegister rn, VRegister rm, uint32_t index) {\n Crypto3RegImm(index, 0b00, rm, rn, rd);\n }\n void sm3tt1b(VRegister rd, VRegister rn, VRegister rm, uint32_t index) {\n Crypto3RegImm(index, 0b01, rm, rn, rd);\n }\n void sm3tt2a(VRegister rd, VRegister rn, VRegister rm, uint32_t index) {\n Crypto3RegImm(index, 0b10, rm, rn, rd);\n }\n void sm3tt2b(VRegister rd, VRegister rn, VRegister rm, uint32_t index) {\n Crypto3RegImm(index, 0b11, rm, rn, rd);\n }\n\n // Cryptographic three-register SHA 512\n void sha512h(VRegister rd, VRegister rn, VRegister rm) {\n Crypto3RegSHA512(0, 0b00, rm, rn, rd);\n }\n void sha512h2(VRegister rd, VRegister rn, VRegister rm) {\n Crypto3RegSHA512(0, 0b01, rm, rn, rd);\n }\n void sha512su1(VRegister rd, VRegister rn, VRegister rm) {\n Crypto3RegSHA512(0, 0b10, rm, rn, rd);\n }\n void rax1(VRegister rd, VRegister rn, VRegister rm) {\n Crypto3RegSHA512(0, 0b11, rm, rn, rd);\n }\n void sm3partw1(VRegister rd, VRegister rn, VRegister rm) {\n Crypto3RegSHA512(1, 0b00, rm, rn, rd);\n }\n void sm3partw2(VRegister rd, VRegister rn, VRegister rm) {\n Crypto3RegSHA512(1, 0b01, rm, rn, rd);\n }\n void sm4ekey(VRegister rd, VRegister rn, VRegister rm) {\n Crypto3RegSHA512(1, 0b10, rm, rn, rd);\n }\n\n // Cryptographic four-register\n void eor3(VRegister rd, VRegister rn, VRegister rm, VRegister ra) {\n Crypto4Register(0b00, rm, ra, rn, rd);\n }\n void bcax(VRegister rd, VRegister rn, VRegister rm, VRegister ra) {\n Crypto4Register(0b01, rm, ra, rn, rd);\n }\n void sm3ss1(VRegister rd, VRegister rn, VRegister rm, VRegister ra) {\n Crypto4Register(0b10, rm, ra, rn, rd);\n }\n\n // Cryptographic two-register SHA 512\n void sha512su0(VRegister rd, VRegister rn) {\n Crypto2RegSHA512(0b00, rn, rd);\n }\n void sm4e(VRegister rd, VRegister rn) {\n Crypto2RegSHA512(0b01, rn, rd);\n }\n\n // Conversion between floating-point and fixed-point\n void scvtf(ARMEmitter::ScalarRegSize ScalarSize, ARMEmitter::VRegister rd, ARMEmitter::Size GPRSize, ARMEmitter::Register rn,\n uint32_t FractionalBits) {\n LOGMAN_THROW_A_FMT(FractionalBits >= 1 && FractionalBits <= ARMEmitter::RegSizeInBits(GPRSize), \"Fractional bits out of range\");\n\n uint32_t Scale = 64 - FractionalBits;\n const auto ConvertedSize = ScalarSize == ARMEmitter::ScalarRegSize::i64Bit ? 0b01 :\n ScalarSize == ARMEmitter::ScalarRegSize::i32Bit ? 0b00 :\n ScalarSize == ARMEmitter::ScalarRegSize::i16Bit ? 0b11 :\n 0;\n\n ScalarConvertBetweenFPAndFixed(0, 0b00, 0b010, Scale, GPRSize, ConvertedSize, rn, rd);\n }\n\n void ucvtf(ARMEmitter::ScalarRegSize ScalarSize, ARMEmitter::VRegister rd, ARMEmitter::Size GPRSize, ARMEmitter::Register rn,\n uint32_t FractionalBits) {\n LOGMAN_THROW_A_FMT(FractionalBits >= 1 && FractionalBits <= ARMEmitter::RegSizeInBits(GPRSize), \"Fractional bits out of range\");\n\n uint32_t Scale = 64 - FractionalBits;\n const auto ConvertedSize = ScalarSize == ARMEmitter::ScalarRegSize::i64Bit ? 0b01 :\n ScalarSize == ARMEmitter::ScalarRegSize::i32Bit ? 0b00 :\n ScalarSize == ARMEmitter::ScalarRegSize::i16Bit ? 0b11 :\n 0;\n\n ScalarConvertBetweenFPAndFixed(0, 0b00, 0b011, Scale, GPRSize, ConvertedSize, rn, rd);\n }\n\n void fcvtzs(ARMEmitter::Size GPRSize, ARMEmitter::Register rd, ARMEmitter::ScalarRegSize ScalarSize, ARMEmitter::VRegister rn,\n uint32_t FractionalBits) {\n LOGMAN_THROW_A_FMT(FractionalBits >= 1 && FractionalBits <= ARMEmitter::RegSizeInBits(GPRSize), \"Fractional bits out of range\");\n\n uint32_t Scale = 64 - FractionalBits;\n const auto ConvertedSize = ScalarSize == ARMEmitter::ScalarRegSize::i64Bit ? 0b01 :\n ScalarSize == ARMEmitter::ScalarRegSize::i32Bit ? 0b00 :\n ScalarSize == ARMEmitter::ScalarRegSize::i16Bit ? 0b11 :\n 0;\n\n ScalarConvertBetweenFPAndFixed(0, 0b11, 0b000, Scale, GPRSize, ConvertedSize, rn, rd);\n }\n\n void fcvtzu(ARMEmitter::Size GPRSize, ARMEmitter::Register rd, ARMEmitter::ScalarRegSize ScalarSize, ARMEmitter::VRegister rn,\n uint32_t FractionalBits) {\n LOGMAN_THROW_A_FMT(FractionalBits >= 1 && FractionalBits <= ARMEmitter::RegSizeInBits(GPRSize), \"Fractional bits out of range\");\n\n uint32_t Scale = 64 - FractionalBits;\n const auto ConvertedSize = ScalarSize == ARMEmitter::ScalarRegSize::i64Bit ? 0b01 :\n ScalarSize == ARMEmitter::ScalarRegSize::i32Bit ? 0b00 :\n ScalarSize == ARMEmitter::ScalarRegSize::i16Bit ? 0b11 :\n 0;\n\n ScalarConvertBetweenFPAndFixed(0, 0b11, 0b001, Scale, GPRSize, ConvertedSize, rn, rd);\n }\n\n // Conversion between floating-point and integer\n void fcvtns(ARMEmitter::Size size, Register rd, HRegister rn) {\n ASIMDFloatConvBetweenInt(size, 0, 0b11, 0b00, 0b000, rd, ToReg(rn));\n }\n void fcvtns(ARMEmitter::Size size, Register rd, SRegister rn) {\n ASIMDFloatConvBetweenInt(size, 0, 0b00, 0b00, 0b000, rd, ToReg(rn));\n }\n void fcvtns(ARMEmitter::Size size, Register rd, DRegister rn) {\n ASIMDFloatConvBetweenInt(size, 0, 0b01, 0b00, 0b000, rd, ToReg(rn));\n }\n void fcvtnu(ARMEmitter::Size size, Register rd, HRegister rn) {\n ASIMDFloatConvBetweenInt(size, 0, 0b11, 0b00, 0b001, rd, ToReg(rn));\n }\n void fcvtnu(ARMEmitter::Size size, Register rd, SRegister rn) {\n ASIMDFloatConvBetweenInt(size, 0, 0b00, 0b00, 0b001, rd, ToReg(rn));\n }\n void fcvtnu(ARMEmitter::Size size, Register rd, DRegister rn) {\n ASIMDFloatConvBetweenInt(size, 0, 0b01, 0b00, 0b001, rd, ToReg(rn));\n }\n void scvtf(ARMEmitter::Size size, HRegister rd, Register rn) {\n ASIMDFloatConvBetweenInt(size, 0, 0b11, 0b00, 0b010, ToReg(rd), rn);\n }\n void scvtf(ARMEmitter::Size size, SRegister rd, Register rn) {\n ASIMDFloatConvBetweenInt(size, 0, 0b00, 0b00, 0b010, ToReg(rd), rn);\n }\n void scvtf(ARMEmitter::Size size, DRegister rd, Register rn) {\n ASIMDFloatConvBetweenInt(size, 0, 0b01, 0b00, 0b010, ToReg(rd), rn);\n }\n void ucvtf(ARMEmitter::Size size, HRegister rd, Register rn) {\n ASIMDFloatConvBetweenInt(size, 0, 0b11, 0b00, 0b011, ToReg(rd), rn);\n }\n void ucvtf(ARMEmitter::Size size, SRegister rd, Register rn) {\n ASIMDFloatConvBetweenInt(size, 0, 0b00, 0b00, 0b011, ToReg(rd), rn);\n }\n void ucvtf(ARMEmitter::Size size, DRegister rd, Register rn) {\n ASIMDFloatConvBetweenInt(size, 0, 0b01, 0b00, 0b011, ToReg(rd), rn);\n }\n void fcvtas(ARMEmitter::Size size, Register rd, HRegister rn) {\n ASIMDFloatConvBetweenInt(size, 0, 0b11, 0b00, 0b100, rd, ToReg(rn));\n }\n void fcvtas(ARMEmitter::Size size, Register rd, SRegister rn) {\n ASIMDFloatConvBetweenInt(size, 0, 0b00, 0b00, 0b100, rd, ToReg(rn));\n }\n void fcvtas(ARMEmitter::Size size, Register rd, DRegister rn) {\n ASIMDFloatConvBetweenInt(size, 0, 0b01, 0b00, 0b100, rd, ToReg(rn));\n }\n void fcvtau(ARMEmitter::Size size, Register rd, HRegister rn) {\n ASIMDFloatConvBetweenInt(size, 0, 0b11, 0b00, 0b101, rd, ToReg(rn));\n }\n void fcvtau(ARMEmitter::Size size, Register rd, SRegister rn) {\n ASIMDFloatConvBetweenInt(size, 0, 0b00, 0b00, 0b101, rd, ToReg(rn));\n }\n void fcvtau(ARMEmitter::Size size, Register rd, DRegister rn) {\n ASIMDFloatConvBetweenInt(size, 0, 0b01, 0b00, 0b101, rd, ToReg(rn));\n }\n void fmov(ARMEmitter::Size size, Register rd, HRegister rn) {\n ASIMDFloatConvBetweenInt(size, 0, 0b11, 0b00, 0b110, rd, ToReg(rn));\n }\n void fmov(ARMEmitter::Size size, Register rd, SRegister rn) {\n LOGMAN_THROW_A_FMT(size != ARMEmitter::Size::i64Bit, \"Can't move SReg to 64-bit\");\n ASIMDFloatConvBetweenInt(size, 0, 0b00, 0b00, 0b110, rd, ToReg(rn));\n }\n void fmov(ARMEmitter::Size size, Register rd, DRegister rn) {\n LOGMAN_THROW_A_FMT(size != ARMEmitter::Size::i32Bit, \"Can't move DReg to 32-bit\");\n ASIMDFloatConvBetweenInt(size, 0, 0b01, 0b00, 0b110, rd, ToReg(rn));\n }\n void fmov(ARMEmitter::Size size, Register rd, VRegister rn, bool Upper) {\n if (Upper) {\n LOGMAN_THROW_A_FMT(size == ARMEmitter::Size::i64Bit, \"Can only move upper with 64-bit elements\");\n }\n ASIMDFloatConvBetweenInt(size, 0, Upper ? 0b10 : 0b01, Upper ? 0b01 : 0b00, 0b110, rd, ToReg(rn));\n }\n void fmov(ARMEmitter::Size size, HRegister rd, Register rn) {\n ASIMDFloatConvBetweenInt(size, 0, 0b11, 0b00, 0b111, ToReg(rd), rn);\n }\n void fmov(ARMEmitter::Size size, SRegister rd, Register rn) {\n LOGMAN_THROW_A_FMT(size != ARMEmitter::Size::i64Bit, \"Can't move SReg to 64-bit\");\n ASIMDFloatConvBetweenInt(size, 0, 0b00, 0b00, 0b111, ToReg(rd), rn);\n }\n void fmov(ARMEmitter::Size size, DRegister rd, Register rn) {\n LOGMAN_THROW_A_FMT(size != ARMEmitter::Size::i32Bit, \"Can't move DReg to 32-bit\");\n ASIMDFloatConvBetweenInt(size, 0, 0b01, 0b00, 0b111, ToReg(rd), rn);\n }\n void fmov(ARMEmitter::Size size, VRegister rd, Register rn, bool Upper) {\n if (Upper) {\n LOGMAN_THROW_A_FMT(size == ARMEmitter::Size::i64Bit, \"Can only move upper with 64-bit elements\");\n }\n ASIMDFloatConvBetweenInt(size, 0, Upper ? 0b10 : 0b01, Upper ? 0b01 : 0b00, 0b111, ToReg(rd), rn);\n }\n void fcvtps(ARMEmitter::Size size, Register rd, HRegister rn) {\n ASIMDFloatConvBetweenInt(size, 0, 0b11, 0b01, 0b000, rd, ToReg(rn));\n }\n void fcvtps(ARMEmitter::Size size, Register rd, SRegister rn) {\n ASIMDFloatConvBetweenInt(size, 0, 0b00, 0b01, 0b000, rd, ToReg(rn));\n }\n void fcvtps(ARMEmitter::Size size, Register rd, DRegister rn) {\n ASIMDFloatConvBetweenInt(size, 0, 0b01, 0b01, 0b000, rd, ToReg(rn));\n }\n void fcvtpu(ARMEmitter::Size size, Register rd, HRegister rn) {\n ASIMDFloatConvBetweenInt(size, 0, 0b11, 0b01, 0b001, rd, ToReg(rn));\n }\n void fcvtpu(ARMEmitter::Size size, Register rd, SRegister rn) {\n ASIMDFloatConvBetweenInt(size, 0, 0b00, 0b01, 0b001, rd, ToReg(rn));\n }\n void fcvtpu(ARMEmitter::Size size, Register rd, DRegister rn) {\n ASIMDFloatConvBetweenInt(size, 0, 0b01, 0b01, 0b001, rd, ToReg(rn));\n }\n void fcvtms(ARMEmitter::Size size, Register rd, HRegister rn) {\n ASIMDFloatConvBetweenInt(size, 0, 0b11, 0b10, 0b000, rd, ToReg(rn));\n }\n void fcvtms(ARMEmitter::Size size, Register rd, SRegister rn) {\n ASIMDFloatConvBetweenInt(size, 0, 0b00, 0b10, 0b000, rd, ToReg(rn));\n }\n void fcvtms(ARMEmitter::Size size, Register rd, DRegister rn) {\n ASIMDFloatConvBetweenInt(size, 0, 0b01, 0b10, 0b000, rd, ToReg(rn));\n }\n void fcvtmu(ARMEmitter::Size size, Register rd, HRegister rn) {\n ASIMDFloatConvBetweenInt(size, 0, 0b11, 0b10, 0b001, rd, ToReg(rn));\n }\n void fcvtmu(ARMEmitter::Size size, Register rd, SRegister rn) {\n ASIMDFloatConvBetweenInt(size, 0, 0b00, 0b10, 0b001, rd, ToReg(rn));\n }\n void fcvtmu(ARMEmitter::Size size, Register rd, DRegister rn) {\n ASIMDFloatConvBetweenInt(size, 0, 0b01, 0b10, 0b001, rd, ToReg(rn));\n }\n void fcvtzs(ARMEmitter::Size size, Register rd, HRegister rn) {\n ASIMDFloatConvBetweenInt(size, 0, 0b11, 0b11, 0b000, rd, ToReg(rn));\n }\n void fcvtzs(ARMEmitter::Size size, Register rd, SRegister rn) {\n ASIMDFloatConvBetweenInt(size, 0, 0b00, 0b11, 0b000, rd, ToReg(rn));\n }\n void fcvtzs(ARMEmitter::Size size, Register rd, DRegister rn) {\n ASIMDFloatConvBetweenInt(size, 0, 0b01, 0b11, 0b000, rd, ToReg(rn));\n }\n void fcvtzs(ARMEmitter::Size size, Register rd, VRegister rn) {\n ASIMDFloatConvBetweenInt(size, 0, 0b01, 0b11, 0b000, rd, ToReg(rn));\n }\n void fcvtzu(ARMEmitter::Size size, Register rd, HRegister rn) {\n ASIMDFloatConvBetweenInt(size, 0, 0b11, 0b11, 0b001, rd, ToReg(rn));\n }\n void fcvtzu(ARMEmitter::Size size, Register rd, SRegister rn) {\n ASIMDFloatConvBetweenInt(size, 0, 0b00, 0b11, 0b001, rd, ToReg(rn));\n }\n void fcvtzu(ARMEmitter::Size size, Register rd, DRegister rn) {\n ASIMDFloatConvBetweenInt(size, 0, 0b01, 0b11, 0b001, rd, ToReg(rn));\n }\n\nprivate:\n // Advanced SIMD three same (FP16)\n template\n void ASIMDThreeSameFP16(uint32_t U, uint32_t a, uint32_t opcode, T rm, T rn, T rd) {\n constexpr uint32_t Q = std::is_same_v ? 1U << 30 : 0;\n constexpr uint32_t Op = 0b0000'1110'0100'0000'0000'01 << 10;\n\n uint32_t Instr = Op;\n Instr |= Q;\n Instr |= U << 29;\n Instr |= a << 23;\n Instr |= rm.Idx() << 16;\n Instr |= opcode << 11;\n Instr |= rn.Idx() << 5;\n Instr |= rd.Idx();\n dc32(Instr);\n }\n\n // Advanced SIMD two-register miscellaneous (FP16)\n template\n void ASIMDTwoRegMiscFP16(uint32_t U, uint32_t a, uint32_t opcode, T rn, T rd) {\n constexpr uint32_t Q = std::is_same_v ? 1U << 30 : 0;\n constexpr uint32_t Op = 0b0000'1110'0111'1000'0000'10 << 10;\n\n uint32_t Instr = Op;\n Instr |= Q;\n Instr |= U << 29;\n Instr |= a << 23;\n Instr |= opcode << 12;\n Instr |= rn.Idx() << 5;\n Instr |= rd.Idx();\n dc32(Instr);\n }\n\n // Advanced SIMD three-register extension\n template\n void ASIMDThreeRegisterExt(uint32_t U, uint32_t opcode, ARMEmitter::SubRegSize size, T rm, T rn, T rd) {\n constexpr uint32_t Q = std::is_same_v ? 1U << 30 : 0;\n constexpr uint32_t Op = 0b0000'1110'0000'0000'1000'01 << 10;\n\n uint32_t Instr = Op;\n Instr |= Q;\n Instr |= U << 29;\n Instr |= FEXCore::ToUnderlying(size) << 22;\n Instr |= rm.Idx() << 16;\n Instr |= opcode << 11;\n Instr |= rn.Idx() << 5;\n Instr |= rd.Idx();\n dc32(Instr);\n }\n\n // Cryptographic AES\n void CryptoAES(uint32_t opcode, VRegister rd, VRegister rn) {\n uint32_t Instr = 0b0100'1110'0010'1000'0000'10U << 10;\n Instr |= opcode << 12;\n Instr |= Encode_rn(rn);\n Instr |= Encode_rd(rd);\n dc32(Instr);\n }\n\n // Cryptographic three-register SHA\n void Crypto3RegSHA(uint32_t opcode, VRegister rd, VRegister rn, VRegister rm) {\n uint32_t Instr = 0b0101'1110'0000'0000'0000'00U << 10;\n Instr |= Encode_rm(rm);\n Instr |= opcode << 12;\n Instr |= Encode_rn(rn);\n Instr |= Encode_rd(rd);\n dc32(Instr);\n }\n\n // Cryptographic two-register SHA\n void Crypto2RegSHA(uint32_t opcode, VRegister rd, VRegister rn) {\n uint32_t Instr = 0b0101'1110'0010'1000'0000'10U << 10;\n Instr |= opcode << 12;\n Instr |= Encode_rn(rn);\n Instr |= Encode_rd(rd);\n dc32(Instr);\n }\n\n // Advanced SIMD table lookup\n void ASIMDTable(uint32_t Q, uint32_t op2, uint32_t len, uint32_t op, VRegister rd, VRegister rn, VRegister rm) {\n uint32_t Instr = 0b0000'1110'000U << 21;\n Instr |= Q << 30;\n Instr |= op2 << 22;\n Instr |= Encode_rm(rm);\n Instr |= len << 13;\n Instr |= op << 12;\n Instr |= Encode_rn(rn);\n Instr |= Encode_rd(rd);\n dc32(Instr);\n }\n\n // Advanced SIMD permute\n void ASIMDPermute(uint32_t Q, SubRegSize size, uint32_t opcode, VRegister rd, VRegister rn, VRegister rm) {\n uint32_t Instr = 0b0000'1110'0000'0000'0000'10U << 10;\n Instr |= Q << 30;\n Instr |= FEXCore::ToUnderlying(size) << 22;\n Instr |= Encode_rm(rm);\n Instr |= opcode << 12;\n Instr |= Encode_rn(rn);\n Instr |= Encode_rd(rd);\n dc32(Instr);\n }\n\n // Advanced SIMD extract\n void ASIMDExtract(uint32_t Q, uint32_t op2, uint32_t imm4, VRegister rd, VRegister rn, VRegister rm) {\n uint32_t Instr = 0b0010'1110'000U << 21;\n Instr |= Q << 30;\n Instr |= op2 << 22;\n Instr |= Encode_rm(rm);\n Instr |= imm4 << 11;\n Instr |= Encode_rn(rn);\n Instr |= Encode_rd(rd);\n dc32(Instr);\n }\n\n // Advanced SIMD two-register miscellaneous\n template\n void ASIMD2RegMisc(uint32_t U, SubRegSize size, uint32_t opcode, T rd, T rn) {\n constexpr uint32_t Q = std::is_same_v ? 1U << 30 : 0;\n\n uint32_t Instr = 0b0000'1110'0010'0000'0000'10U << 10;\n Instr |= Q;\n Instr |= U << 29;\n Instr |= FEXCore::ToUnderlying(size) << 22;\n Instr |= opcode << 12;\n Instr |= Encode_rn(rn);\n Instr |= Encode_rd(rd);\n dc32(Instr);\n }\n\n // Advanced SIMD across lanes\n template\n void ASIMDAcrossLanes(uint32_t U, SubRegSize size, uint32_t opcode, T rd, T rn) {\n constexpr uint32_t Q = std::is_same_v ? 1U << 30 : 0;\n\n uint32_t Instr = 0b0000'1110'0011'0000'0000'10U << 10;\n Instr |= Q;\n Instr |= U << 29;\n Instr |= FEXCore::ToUnderlying(size) << 22;\n Instr |= opcode << 12;\n Instr |= Encode_rn(rn);\n Instr |= Encode_rd(rd);\n dc32(Instr);\n }\n\n // Advanced SIMD three different\n template\n void ASIMD3Different(uint32_t U, uint32_t opcode, SubRegSize size, T rd, T rn, T rm) {\n constexpr uint32_t Q = std::is_same_v ? 1U << 30 : 0;\n\n uint32_t Instr = 0b0000'1110'0010'0000'0000'00U << 10;\n Instr |= Q;\n Instr |= U << 29;\n Instr |= FEXCore::ToUnderlying(size) << 22;\n Instr |= Encode_rm(rm);\n Instr |= opcode << 12;\n Instr |= Encode_rn(rn);\n Instr |= Encode_rd(rd);\n dc32(Instr);\n }\n\n // Advanced SIMD three same\n template\n void ASIMD3Same(uint32_t U, SubRegSize size, uint32_t opcode, T rd, T rn, T rm) {\n constexpr uint32_t Q = std::is_same_v ? 1U << 30 : 0;\n\n uint32_t Instr = 0b0000'1110'0010'0000'0000'01U << 10;\n Instr |= Q;\n Instr |= U << 29;\n Instr |= FEXCore::ToUnderlying(size) << 22;\n Instr |= Encode_rm(rm);\n Instr |= opcode << 11;\n Instr |= Encode_rn(rn);\n Instr |= Encode_rd(rd);\n dc32(Instr);\n }\n\n // Advanced SIMD modified immediate\n template\n void ASIMDModifiedImm(uint32_t op, uint32_t cmode, uint32_t o2, uint32_t imm, T rd) {\n constexpr uint32_t Q = std::is_same_v ? 1U << 30 : 0;\n\n uint32_t Instr = 0b0000'1111'0000'0000'0000'01U << 10;\n Instr |= Q;\n Instr |= op << 29;\n Instr |= ((imm >> 7) & 1) << 18;\n Instr |= ((imm >> 6) & 1) << 17;\n Instr |= ((imm >> 5) & 1) << 16;\n Instr |= cmode << 12;\n Instr |= o2 << 11;\n Instr |= ((imm >> 4) & 1) << 9;\n Instr |= ((imm >> 3) & 1) << 8;\n Instr |= ((imm >> 2) & 1) << 7;\n Instr |= ((imm >> 1) & 1) << 6;\n Instr |= ((imm >> 0) & 1) << 5;\n\n Instr |= Encode_rd(rd);\n dc32(Instr);\n }\n\n // Advanced SIMD shift by immediate\n template\n void ASIMDShiftByImm(uint32_t U, uint32_t immh, uint32_t immb, uint32_t opcode, T rn, T rd) {\n constexpr uint32_t Q = std::is_same_v ? 1U << 30 : 0;\n LOGMAN_THROW_A_FMT(immh != 0, \"ImmH needs to not be zero\");\n\n uint32_t Instr = 0b0000'1111'0000'0000'0000'01U << 10;\n\n Instr |= Q;\n Instr |= U << 29;\n Instr |= immh << 19;\n Instr |= immb << 16;\n Instr |= opcode << 11;\n Instr |= Encode_rn(rn);\n Instr |= Encode_rd(rd);\n dc32(Instr);\n }\n\n // Advanced SIMD vector x indexed element\n template\n void ASIMDVectorXIndexedElement(uint32_t U, uint32_t L, uint32_t M, uint32_t opcode, uint32_t H, ARMEmitter::SubRegSize size, T rm, T rn, T rd) {\n constexpr uint32_t Op = 0b0000'1111'0000'0000'0000'00 << 10;\n constexpr uint32_t Q = std::is_same_v ? 1U << 30 : 0;\n\n uint32_t Instr = Op;\n\n Instr |= Q;\n Instr |= U << 29;\n Instr |= FEXCore::ToUnderlying(size) << 22;\n Instr |= L << 21;\n\n // M and Rm might overlap. It's up to the instruction emitter itself to ensure there is no conflict.\n Instr |= M << 20;\n Instr |= rm.Idx() << 16;\n Instr |= opcode << 12;\n Instr |= H << 11;\n Instr |= rn.Idx() << 5;\n Instr |= rd.Idx();\n dc32(Instr);\n }\n\n void Crypto3RegImm(uint32_t index, uint32_t opcode, VRegister rm, VRegister rn, VRegister rd) {\n LOGMAN_THROW_A_FMT(index <= 3, \"index ({}) must be within [0-3]\", index);\n\n uint32_t Instr = 0b1100'1110'0100'0000'1000'0000'0000'0000;\n Instr |= rm.Idx() << 16;\n Instr |= index << 12;\n Instr |= opcode << 10;\n Instr |= rn.Idx() << 5;\n Instr |= rd.Idx();\n dc32(Instr);\n }\n\n void Crypto3RegSHA512(uint32_t o, uint32_t opcode, VRegister rm, VRegister rn, VRegister rd) {\n uint32_t Instr = 0b1100'1110'0110'0000'1000'0000'0000'0000;\n Instr |= rm.Idx() << 16;\n Instr |= o << 14;\n Instr |= opcode << 10;\n Instr |= rn.Idx() << 5;\n Instr |= rd.Idx();\n dc32(Instr);\n }\n\n void Crypto4Register(uint32_t opcode, VRegister rm, VRegister ra, VRegister rn, VRegister rd) {\n uint32_t Instr = 0b1100'1110'0000'0000'0000'0000'0000'0000;\n Instr |= opcode << 21;\n Instr |= rm.Idx() << 16;\n Instr |= ra.Idx() << 10;\n Instr |= rn.Idx() << 5;\n Instr |= rd.Idx();\n dc32(Instr);\n }\n\n void Crypto2RegSHA512(uint32_t opcode, VRegister rn, VRegister rd) {\n uint32_t Instr = 0b1100'1110'1100'0000'1000'0000'0000'0000;\n Instr |= opcode << 10;\n Instr |= rn.Idx() << 5;\n Instr |= rd.Idx();\n dc32(Instr);\n }\n\n // Conversion between floating-point and fixed-point\n template\n void ScalarConvertBetweenFPAndFixed(uint32_t S, uint32_t rmode, uint32_t opcode, uint32_t scale, ARMEmitter::Size GPRSize,\n uint32_t ScalarSize, T rn, T2 rd) {\n constexpr uint32_t Op = 0b0001'1110'000 << 21;\n const uint32_t SF = GPRSize == ARMEmitter::Size::i64Bit ? (1U << 31) : 0;\n\n uint32_t Instr = Op;\n Instr |= SF;\n Instr |= S << 29;\n Instr |= ScalarSize << 22;\n Instr |= rmode << 19;\n Instr |= opcode << 16;\n Instr |= scale << 10;\n Instr |= rn.Idx() << 5;\n Instr |= rd.Idx();\n dc32(Instr);\n }\n\n // Conversion between floating-point and integer\n void ASIMDFloatConvBetweenInt(ARMEmitter::Size s, uint32_t S, uint32_t ptype, uint32_t rmode, uint32_t opcode, Register rd, Register rn) {\n const uint32_t SF = s == ARMEmitter::Size::i64Bit ? (1U << 31) : 0;\n\n uint32_t Instr = 0b0001'1110'001U << 21;\n Instr |= SF;\n Instr |= S << 29;\n Instr |= ptype << 22;\n Instr |= rmode << 19;\n Instr |= opcode << 16;\n Instr |= Encode_rn(rn);\n Instr |= Encode_rd(rd);\n dc32(Instr);\n }\n\n template\n void ASIMDLoadStoreMultipleStructure(uint32_t Op, uint32_t opcode, T rt, ARMEmitter::Register rn, ARMEmitter::Register rm) {\n constexpr uint32_t Q = std::is_same_v ? 1U << 30 : 0;\n\n uint32_t Instr = Op;\n\n Instr |= Q;\n Instr |= Load ? 1 << 22 : 0;\n Instr |= Encode_rm(rm);\n Instr |= opcode;\n Instr |= FEXCore::ToUnderlying(size) << 10;\n Instr |= Encode_rn(rn);\n Instr |= Encode_rt(rt);\n dc32(Instr);\n }\n template\n void ASIMDSTLD(uint32_t Op, uint32_t Opcode, ARMEmitter::VRegister rt, uint32_t Index, ARMEmitter::Register rn, ARMEmitter::Register rm) {\n LOGMAN_THROW_A_FMT((size == SubRegSize::i8Bit && Index < 16) || (size == SubRegSize::i16Bit && Index < 8) ||\n (size == SubRegSize::i32Bit && Index < 4) || (size == SubRegSize::i64Bit && Index < 2),\n \"Invalid Index selected\");\n\n uint32_t Q {};\n uint32_t S {};\n uint32_t Size {};\n\n // selem is for determining if we are doing 1-3 loadstore single structure operations\n // eg: ST1/2/3/4 or LD1/2/3/4\n constexpr uint32_t selem = Count - 1;\n const uint32_t opcode = Opcode | (selem >> 1);\n\n // Index is encoded as:\n // 8-bit: Q:S:size\n // 16-bit Q:S:size<1>\n // 32-bit: Q:S\n // 64-bit: Q\n if constexpr (size == SubRegSize::i8Bit) {\n Q = ((Index & 0b1000) >> 3) << 30;\n S = ((Index & 0b0100) >> 2);\n Size = Index & 0b11;\n } else if constexpr (size == SubRegSize::i16Bit) {\n Q = ((Index & 0b0100) >> 2) << 30;\n S = ((Index & 0b0010) >> 1);\n Size = (Index & 0b1) << 1;\n } else if constexpr (size == SubRegSize::i32Bit) {\n Q = ((Index & 0b0010) >> 1) << 30;\n S = Index & 0b0001;\n } else if constexpr (size == SubRegSize::i64Bit) {\n Q = (Index & 0b0001) << 30;\n Size = 1;\n }\n\n // scale = opcode<2:1>\n // selem = opcode<0>:R + 1\n //\n // scale:\n // - 0\n // - Index = Q:S:size - aka B[0-15]\n // - 1\n // - Index = Q:S:size<1> - aka H[0-7]\n // - 2\n // if (size == i32)\n // - Index = Q:S - aka S[0-3]\n // if (size == i64)\n // - Index = Q - aka D[0-1]\n // if (size == i128) undefined\n // - 3\n // Load+Replicate\n // scale = size\n\n ASIMDLoadStore(Op | Q, Load, selem & 1, opcode, S, Size, rt, rn, rm);\n }\n\n template\n void ASIMDSTLD(uint32_t Op, uint32_t Opcode, T rt, ARMEmitter::Register rn, ARMEmitter::Register rm) {\n constexpr uint32_t Q = std::is_same_v ? 1U << 30 : 0;\n constexpr uint32_t S = 0;\n\n // selem is for determining if we are doing 1-3 loadstore single structure operations\n // eg: ST1/2/3/4 or LD1/2/3/4\n constexpr uint32_t selem = Count - 1;\n const uint32_t opcode = Opcode | (selem >> 1);\n\n // scale = opcode<2:1>\n // selem = opcode<0>:R + 1\n //\n // scale:\n // - 0\n // - Index = Q:S:size - aka B[0-15]\n // - 1\n // - Index = Q:S:size<1> - aka H[0-7]\n // - 2\n // if (size == i32)\n // - Index = Q:S - aka S[0-3]\n // if (size == i64)\n // - Index = Q - aka D[0-1]\n // if (size == i128) undefined\n // - 3\n // Load+Replicate\n // scale = size\n\n ASIMDLoadStore(Op | Q, Load, selem & 1, opcode, S, FEXCore::ToUnderlying(size), rt, rn, rm);\n }\n void ASIMDLoadStore(uint32_t Op, uint32_t L, uint32_t R, uint32_t opcode, uint32_t S, uint32_t size, ARMEmitter::VRegister rt,\n ARMEmitter::Register rn, ARMEmitter::Register rm) {\n uint32_t Instr = Op;\n\n Instr |= L << 22;\n Instr |= R << 21;\n Instr |= Encode_rm(rm);\n Instr |= opcode << 13;\n Instr |= S << 12;\n Instr |= size << 10;\n Instr |= Encode_rn(rn);\n Instr |= Encode_rt(rt);\n\n dc32(Instr);\n }\n\n#ifndef INCLUDED_BY_EMITTER\n}; // struct LoadstoreEmitterOps\n} // namespace ARMEmitter\n#endif\n" }, { "path": "CodeEmitter/CodeEmitter/BranchOps.inl", "content": "// SPDX-License-Identifier: MIT\n/* Branch instruction emitters.\n *\n * Most of these instructions will use `BackwardLabel`, `ForwardLabel`, or `BiDirectionLabel` to determine where a branch targets.\n */\n\n#pragma once\n#ifndef INCLUDED_BY_EMITTER\n#include \nnamespace ARMEmitter {\nstruct EmitterOps : Emitter {\n#endif\n\npublic:\n // Branches, Exception Generating and System instructions\npublic:\n // Conditional branch immediate\n ///< Branch conditional\n void b(ARMEmitter::Condition Cond, uint32_t Imm) {\n constexpr uint32_t Op = 0b0101'010 << 25;\n Branch_Conditional(Op, 0, 0, Cond, Imm);\n }\n [[nodiscard]] BranchEncodeSucceeded b(ARMEmitter::Condition Cond, const BackwardLabel* Label) {\n int32_t Imm = static_cast(Label->Location - GetCursorAddress());\n if (Imm >= -1048576 && Imm <= 1048575 && ((Imm & 0b11) == 0)) {\n constexpr uint32_t Op = 0b0101'010 << 25;\n Branch_Conditional(Op, 0, 0, Cond, Imm >> 2);\n return BranchEncodeSucceeded::Success;\n }\n\n // Can't encode.\n return BranchEncodeSucceeded::Failure;\n }\n [[nodiscard]] BranchEncodeSucceeded b(ARMEmitter::Condition Cond, ForwardLabel* Label) {\n AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress(), .Type = ForwardLabel::InstType::BC});\n constexpr uint32_t Op = 0b0101'010 << 25;\n Branch_Conditional(Op, 0, 0, Cond, 0);\n\n // Forward label doesn't know if it can encode until Bind.\n return BranchEncodeSucceeded::Success;\n }\n\n [[nodiscard]] BranchEncodeSucceeded b(ARMEmitter::Condition Cond, BiDirectionalLabel* Label) {\n if (Label->Backward.Location) {\n return b(Cond, &Label->Backward);\n } else {\n return b(Cond, &Label->Forward);\n }\n }\n\n ///< Branch consistent conditional\n void bc(ARMEmitter::Condition Cond, uint32_t Imm) {\n constexpr uint32_t Op = 0b0101'010 << 25;\n Branch_Conditional(Op, 0, 1, Cond, Imm);\n }\n [[nodiscard]] BranchEncodeSucceeded bc(ARMEmitter::Condition Cond, const BackwardLabel* Label) {\n int32_t Imm = static_cast(Label->Location - GetCursorAddress());\n if (Imm >= -1048576 && Imm <= 1048575 && ((Imm & 0b11) == 0)) {\n constexpr uint32_t Op = 0b0101'010 << 25;\n Branch_Conditional(Op, 0, 1, Cond, Imm >> 2);\n return BranchEncodeSucceeded::Success;\n }\n\n // Can't encode.\n return BranchEncodeSucceeded::Failure;\n }\n\n [[nodiscard]] BranchEncodeSucceeded bc(ARMEmitter::Condition Cond, ForwardLabel* Label) {\n AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress(), .Type = ForwardLabel::InstType::BC});\n constexpr uint32_t Op = 0b0101'010 << 25;\n Branch_Conditional(Op, 0, 1, Cond, 0);\n\n // Forward label doesn't know if it can encode until Bind.\n return BranchEncodeSucceeded::Success;\n }\n\n [[nodiscard]] BranchEncodeSucceeded bc(ARMEmitter::Condition Cond, BiDirectionalLabel* Label) {\n if (Label->Backward.Location) {\n return bc(Cond, &Label->Backward);\n } else {\n return bc(Cond, &Label->Forward);\n }\n }\n\n // Unconditional branch register\n void br(ARMEmitter::Register rn) {\n constexpr uint32_t Op = 0b1101011 << 25 | 0b0'000 << 21 | // opc\n 0b1'1111 << 16 | // op2\n 0b0000'00 << 10 | // op3\n 0b0'0000; // op4\n\n UnconditionalBranch(Op, rn);\n }\n void blr(ARMEmitter::Register rn) {\n constexpr uint32_t Op = 0b1101011 << 25 | 0b0'001 << 21 | // opc\n 0b1'1111 << 16 | // op2\n 0b0000'00 << 10 | // op3\n 0b0'0000; // op4\n\n UnconditionalBranch(Op, rn);\n }\n void ret(ARMEmitter::Register rn = ARMEmitter::Reg::r30) {\n constexpr uint32_t Op = 0b1101011 << 25 | 0b0'010 << 21 | // opc\n 0b1'1111 << 16 | // op2\n 0b0000'00 << 10 | // op3\n 0b0'0000; // op4\n\n UnconditionalBranch(Op, rn);\n }\n\n // Unconditional branch immediate\n void b(uint32_t Imm) {\n constexpr uint32_t Op = 0b0001'01 << 26;\n\n UnconditionalBranch(Op, Imm);\n }\n [[nodiscard]] BranchEncodeSucceeded b(const BackwardLabel* Label) {\n int32_t Imm = static_cast(Label->Location - GetCursorAddress());\n if (Imm >= -134217728 && Imm <= 134217724 && ((Imm & 0b11) == 0)) {\n constexpr uint32_t Op = 0b0001'01 << 26;\n UnconditionalBranch(Op, Imm >> 2);\n return BranchEncodeSucceeded::Success;\n }\n\n // Can't encode.\n return BranchEncodeSucceeded::Failure;\n }\n [[nodiscard]] BranchEncodeSucceeded b(ForwardLabel* Label) {\n AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress(), .Type = ForwardLabel::InstType::B});\n constexpr uint32_t Op = 0b0001'01 << 26;\n\n UnconditionalBranch(Op, 0);\n\n // Forward label doesn't know if it can encode until Bind.\n return BranchEncodeSucceeded::Success;\n }\n\n [[nodiscard]] BranchEncodeSucceeded b(BiDirectionalLabel* Label) {\n if (Label->Backward.Location) {\n return b(&Label->Backward);\n } else {\n return b(&Label->Forward);\n }\n }\n\n void bl(uint32_t Imm) {\n constexpr uint32_t Op = 0b1001'01 << 26;\n\n UnconditionalBranch(Op, Imm);\n }\n\n [[nodiscard]] BranchEncodeSucceeded bl(const BackwardLabel* Label) {\n int32_t Imm = static_cast(Label->Location - GetCursorAddress());\n if (Imm >= -134217728 && Imm <= 134217724 && ((Imm & 0b11) == 0)) {\n constexpr uint32_t Op = 0b1001'01 << 26;\n UnconditionalBranch(Op, Imm >> 2);\n\n return BranchEncodeSucceeded::Success;\n }\n\n // Can't encode.\n return BranchEncodeSucceeded::Failure;\n }\n [[nodiscard]] BranchEncodeSucceeded bl(ForwardLabel* Label) {\n AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress(), .Type = ForwardLabel::InstType::B});\n constexpr uint32_t Op = 0b1001'01 << 26;\n\n UnconditionalBranch(Op, 0);\n\n // Forward label doesn't know if it can encode until Bind.\n return BranchEncodeSucceeded::Success;\n }\n\n [[nodiscard]] BranchEncodeSucceeded bl(BiDirectionalLabel* Label) {\n if (Label->Backward.Location) {\n return bl(&Label->Backward);\n } else {\n return bl(&Label->Forward);\n }\n }\n\n // Compare and branch\n void cbz(ARMEmitter::Size s, ARMEmitter::Register rt, uint32_t Imm) {\n constexpr uint32_t Op = 0b0011'0100 << 24;\n\n CompareAndBranch(Op, s, rt, Imm);\n }\n\n [[nodiscard]] BranchEncodeSucceeded cbz(ARMEmitter::Size s, ARMEmitter::Register rt, const BackwardLabel* Label) {\n int32_t Imm = static_cast(Label->Location - GetCursorAddress());\n\n if (Imm >= -1048576 && Imm <= 1048575 && ((Imm & 0b11) == 0)) {\n constexpr uint32_t Op = 0b0011'0100 << 24;\n CompareAndBranch(Op, s, rt, Imm >> 2);\n return BranchEncodeSucceeded::Success;\n }\n\n // Can't encode.\n return BranchEncodeSucceeded::Failure;\n }\n\n [[nodiscard]] BranchEncodeSucceeded cbz(ARMEmitter::Size s, ARMEmitter::Register rt, ForwardLabel* Label) {\n AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress(), .Type = ForwardLabel::InstType::BC});\n\n constexpr uint32_t Op = 0b0011'0100 << 24;\n\n CompareAndBranch(Op, s, rt, 0);\n\n // Forward label doesn't know if it can encode until Bind.\n return BranchEncodeSucceeded::Success;\n }\n\n [[nodiscard]] BranchEncodeSucceeded cbz(ARMEmitter::Size s, ARMEmitter::Register rt, BiDirectionalLabel* Label) {\n if (Label->Backward.Location) {\n return cbz(s, rt, &Label->Backward);\n } else {\n return cbz(s, rt, &Label->Forward);\n }\n }\n\n void cbnz(ARMEmitter::Size s, ARMEmitter::Register rt, uint32_t Imm) {\n constexpr uint32_t Op = 0b0011'0101 << 24;\n\n CompareAndBranch(Op, s, rt, Imm);\n }\n\n [[nodiscard]] BranchEncodeSucceeded cbnz(ARMEmitter::Size s, ARMEmitter::Register rt, const BackwardLabel* Label) {\n int32_t Imm = static_cast(Label->Location - GetCursorAddress());\n\n if (Imm >= -1048576 && Imm <= 1048575 && ((Imm & 0b11) == 0)) {\n constexpr uint32_t Op = 0b0011'0101 << 24;\n CompareAndBranch(Op, s, rt, Imm >> 2);\n return BranchEncodeSucceeded::Success;\n }\n\n // Can't encode.\n return BranchEncodeSucceeded::Failure;\n }\n\n [[nodiscard]] BranchEncodeSucceeded cbnz(ARMEmitter::Size s, ARMEmitter::Register rt, ForwardLabel* Label) {\n AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress(), .Type = ForwardLabel::InstType::BC});\n\n constexpr uint32_t Op = 0b0011'0101 << 24;\n\n CompareAndBranch(Op, s, rt, 0);\n\n // Forward label doesn't know if it can encode until Bind.\n return BranchEncodeSucceeded::Success;\n }\n\n [[nodiscard]] BranchEncodeSucceeded cbnz(ARMEmitter::Size s, ARMEmitter::Register rt, BiDirectionalLabel* Label) {\n if (Label->Backward.Location) {\n return cbnz(s, rt, &Label->Backward);\n } else {\n return cbnz(s, rt, &Label->Forward);\n }\n }\n\n // Test and branch immediate\n void tbz(ARMEmitter::Register rt, uint32_t Bit, uint32_t Imm) {\n constexpr uint32_t Op = 0b0011'0110 << 24;\n\n TestAndBranch(Op, rt, Bit, Imm);\n }\n [[nodiscard]] BranchEncodeSucceeded tbz(ARMEmitter::Register rt, uint32_t Bit, const BackwardLabel* Label) {\n int32_t Imm = static_cast(Label->Location - GetCursorAddress());\n\n if (Imm >= -32768 && Imm <= 32764 && ((Imm & 0b11) == 0)) {\n constexpr uint32_t Op = 0b0011'0110 << 24;\n TestAndBranch(Op, rt, Bit, Imm >> 2);\n return BranchEncodeSucceeded::Success;\n }\n\n // Can't encode.\n return BranchEncodeSucceeded::Failure;\n }\n\n [[nodiscard]] BranchEncodeSucceeded tbz(ARMEmitter::Register rt, uint32_t Bit, ForwardLabel* Label) {\n AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress(), .Type = ForwardLabel::InstType::TEST_BRANCH});\n\n constexpr uint32_t Op = 0b0011'0110 << 24;\n\n TestAndBranch(Op, rt, Bit, 0);\n\n // Forward label doesn't know if it can encode until Bind.\n return BranchEncodeSucceeded::Success;\n }\n\n [[nodiscard]] BranchEncodeSucceeded tbz(ARMEmitter::Register rt, uint32_t Bit, BiDirectionalLabel* Label) {\n if (Label->Backward.Location) {\n return tbz(rt, Bit, &Label->Backward);\n } else {\n return tbz(rt, Bit, &Label->Forward);\n }\n }\n\n void tbnz(ARMEmitter::Register rt, uint32_t Bit, uint32_t Imm) {\n constexpr uint32_t Op = 0b0011'0111 << 24;\n\n TestAndBranch(Op, rt, Bit, Imm);\n }\n [[nodiscard]] BranchEncodeSucceeded tbnz(ARMEmitter::Register rt, uint32_t Bit, const BackwardLabel* Label) {\n int32_t Imm = static_cast(Label->Location - GetCursorAddress());\n\n if (Imm >= -32768 && Imm <= 32764 && ((Imm & 0b11) == 0)) {\n constexpr uint32_t Op = 0b0011'0111 << 24;\n TestAndBranch(Op, rt, Bit, Imm >> 2);\n return BranchEncodeSucceeded::Success;\n }\n\n // Can't encode.\n return BranchEncodeSucceeded::Failure;\n }\n\n [[nodiscard]] BranchEncodeSucceeded tbnz(ARMEmitter::Register rt, uint32_t Bit, ForwardLabel* Label) {\n AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress(), .Type = ForwardLabel::InstType::TEST_BRANCH});\n constexpr uint32_t Op = 0b0011'0111 << 24;\n\n TestAndBranch(Op, rt, Bit, 0);\n\n // Forward label doesn't know if it can encode until Bind.\n return BranchEncodeSucceeded::Success;\n }\n\n [[nodiscard]] BranchEncodeSucceeded tbnz(ARMEmitter::Register rt, uint32_t Bit, BiDirectionalLabel* Label) {\n if (Label->Backward.Location) {\n return tbnz(rt, Bit, &Label->Backward);\n } else {\n return tbnz(rt, Bit, &Label->Forward);\n }\n }\n\nprivate:\n // Conditional branch immediate\n void Branch_Conditional(uint32_t Op, uint32_t Op1, uint32_t Op0, ARMEmitter::Condition Cond, uint32_t Imm) {\n uint32_t Instr = Op;\n\n Instr |= Op1 << 24;\n Instr |= (Imm & 0x7'FFFF) << 5;\n Instr |= Op0 << 4;\n Instr |= FEXCore::ToUnderlying(Cond);\n\n dc32(Instr);\n }\n\n // Unconditional branch register\n void UnconditionalBranch(uint32_t Op, ARMEmitter::Register rn) {\n uint32_t Instr = Op;\n Instr |= Encode_rn(rn);\n dc32(Instr);\n }\n\n // Unconditional branch - immediate\n void UnconditionalBranch(uint32_t Op, uint32_t Imm) {\n uint32_t Instr = Op;\n Instr |= Imm & 0x3FF'FFFF;\n dc32(Instr);\n }\n\n // Compare and branch\n void CompareAndBranch(uint32_t Op, ARMEmitter::Size s, ARMEmitter::Register rt, uint32_t Imm) {\n const uint32_t SF = s == ARMEmitter::Size::i64Bit ? (1U << 31) : 0;\n\n uint32_t Instr = Op;\n\n Instr |= SF;\n Instr |= (Imm & 0x7'FFFF) << 5;\n Instr |= Encode_rt(rt);\n dc32(Instr);\n }\n\n // Test and branch - immediate\n void TestAndBranch(uint32_t Op, ARMEmitter::Register rt, uint32_t Bit, uint32_t Imm) {\n uint32_t Instr = Op;\n\n Instr |= (Bit >> 5) << 31;\n Instr |= (Bit & 0b1'1111) << 19;\n Instr |= (Imm & 0x3FFF) << 5;\n Instr |= Encode_rt(rt);\n dc32(Instr);\n }\n\n#ifndef INCLUDED_BY_EMITTER\n}; // struct LoadstoreEmitterOps\n} // namespace ARMEmitter\n#endif\n" }, { "path": "CodeEmitter/CodeEmitter/Buffer.h", "content": "// SPDX-License-Identifier: MIT\n#pragma once\n#include \n#include \n#include \n#include \n\nnamespace ARMEmitter {\nclass Buffer {\npublic:\n Buffer() {\n SetBuffer(nullptr, 0);\n }\n\n Buffer(uint8_t* Base, uint64_t BaseSize) {\n SetBuffer(Base, BaseSize);\n }\n\n void SetBuffer(uint8_t* Base, uint64_t BaseSize) {\n BufferBase = Base;\n CurrentOffset = BufferBase;\n Size = BaseSize;\n }\n\n template\n requires (std::is_trivially_copyable_v)\n void dcn(const T& Data) {\n std::memcpy(CurrentOffset, &Data, sizeof(Data));\n CurrentOffset += sizeof(Data);\n }\n void dc8(uint8_t Data) {\n dcn(Data);\n }\n void dc16(uint16_t Data) {\n dcn(Data);\n }\n void dc32(uint32_t Data) {\n dcn(Data);\n }\n void dc64(uint64_t Data) {\n dcn(Data);\n }\n\n void EmitString(const char* String) {\n const auto StringLength = strlen(String);\n memcpy(CurrentOffset, String, StringLength);\n CurrentOffset += StringLength;\n }\n\n void Align(size_t Size = 4) {\n // Align the buffer to provided size.\n auto CurrentAlignment = reinterpret_cast(CurrentOffset) & (Size - 1);\n if (!CurrentAlignment) {\n return;\n }\n std::memset(CurrentOffset, 0, Size - CurrentAlignment);\n CurrentOffset += Size - CurrentAlignment;\n }\n\n template\n T GetCursorAddress() const {\n return reinterpret_cast(CurrentOffset);\n }\n\n static void ClearICache(void* Begin, std::size_t Length) {\n __builtin___clear_cache(static_cast(Begin), static_cast(Begin) + Length);\n }\n\n size_t GetCursorOffset() const {\n return static_cast(CurrentOffset - BufferBase);\n }\n\n uint8_t* GetBufferBase() const {\n return BufferBase;\n }\n\n void CursorIncrement(size_t Size) {\n CurrentOffset += Size;\n }\n\n void SetCursorOffset(size_t Offset) {\n CurrentOffset = BufferBase + Offset;\n }\n\n uint64_t GetBufferSize() const {\n return Size;\n }\n\n template\n size_t GetCursorOffsetFromAddress(const T* Address) const {\n return static_cast(reinterpret_cast(Address) - BufferBase);\n }\n\nprotected:\n\n uint8_t* BufferBase;\n uint8_t* CurrentOffset;\n uint64_t Size;\n};\n} // namespace ARMEmitter\n" }, { "path": "CodeEmitter/CodeEmitter/Emitter.h", "content": "// SPDX-License-Identifier: MIT\n#pragma once\n\n#include \n#include \n#include \n#include \n#include \n\n#include \n#include \n#include \n\n#include \n#include \n#include \n#include \n#include \n\n/*\n * Welcome to FEX-Emu's custom AArch64 emitter.\n * This was written specifically to avoid the performance cost of the vixl emitter.\n *\n * There are some specific design constraints in this design to target a couple features:\n * - High performance\n * - Low CPU cache performance hit\n * - Significantly reduced code footprint\n * - Low number of branches\n *\n * These requirements are mostly achieved by removing a bunch of developer conveniences\n * that vixl provides. The developer needs to take a lot of care to not shoot themselves in the foot.\n *\n * Misc design decisions:\n * - Registers are encoded as basic uint32_t enums.\n * - Converting between different registers is zero-cost.\n * - Passing around as arguments are as cheap as registers\n * - Contrast to vixl where every register requires living on the stack.\n * - Registers can get encoded in to instructions with a simple `BFM` instruction.\n *\n * - Instructions are very simply emitted, allowing direct inlining most of the time.\n * - These are simple enough that multiple back-to-back instructions get optimized to 128-bit load-store operations.\n * - Contrast to vixl where pretty much no instruction emitter gets inlined.\n *\n * - Instruction emitters are /mostly/ unsized. Most instructions take a size argument first, which gets encoded\n * directly in to the instruction.\n * - Contrast to vixl where the register arguments are how the instructions determine operating size.\n * - Size argument allows FEX to use `CSEL` to select a size at runtime, instead of branching.\n * - Some instructions are explicitly sized based on register type. Read comments in the respective `inl` files to\n * see why.\n * Some scalar/vector operations are an example of this.\n *\n * - Almost zero helper functions.\n * - Primary exception to this rule is load-store operations. These will use a helper to make\n * it easier to select the correct load-store instruction. Mostly because these are a nightmare selecting\n * the right instruction.\n */\nnamespace ARMEmitter {\n/*\n * This `Size` enum is used for most ALU operations.\n * These follow the AArch64 encoding style in most cases.\n */\nenum class Size : uint32_t {\n i32Bit = 0,\n i64Bit,\n};\n\n// This allows us to get the `Size` enum in bits.\n[[nodiscard]]\nconstexpr size_t RegSizeInBits(Size size) {\n return size_t {32} << FEXCore::ToUnderlying(size);\n}\n\n/* This `SubRegSize` enum is used for most ASIMD operations.\n * These follow the AArch64 encoding style in most cases.\n */\nenum class SubRegSize : uint32_t {\n i8Bit = 0b00,\n i16Bit = 0b01,\n i32Bit = 0b10,\n i64Bit = 0b11,\n i128Bit = 0b100,\n};\n\n// This allows us to get the `SubRegSize` in bits.\n[[nodiscard]]\nconstexpr size_t SubRegSizeInBits(SubRegSize size) {\n return size_t {8} << FEXCore::ToUnderlying(size);\n}\n\n// Many floating point operations constrain their element sizes to the\n// main three float sizes half, single, and double precision. This just\n// combines all the checks together for brevity.\n[[nodiscard]]\nconstexpr bool IsStandardFloatSize(SubRegSize size) {\n return size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit;\n}\n\n/* This `ScalarRegSize` enum is used for most scalar float\n * operations.\n *\n * This is specifically duplicated from `SubRegSize` to have strongly\n * typed functions.\n *\n * `ScalarRegSize` specifically doesn't have `i128Bit` because scalar operations\n * can't operate at 128-bit.\n */\nenum class ScalarRegSize : uint32_t {\n i8Bit = 0b00,\n i16Bit = 0b01,\n i32Bit = 0b10,\n i64Bit = 0b11,\n};\n\n// This allows us to get the `ScalarRegSize` in bits.\n[[nodiscard]]\nconstexpr size_t ScalarRegSizeInBits(ScalarRegSize size) {\n return size_t {8} << FEXCore::ToUnderlying(size);\n}\n\n/* This `VectorRegSizePair` union allows us to have an overlapping type\n * to select a scalar operation or a vector depending on which operation\n * we pass in.\n * Useful in FEX's vector operations that behave as scalar or vector\n * depending on various factors. But since the operation will have the sa,e\n * element size, we want to choose the operation more easily\n */\nunion VectorRegSizePair {\n ScalarRegSize Scalar;\n SubRegSize Vector;\n};\n\n// This allows us to create a `VectorRegSizePair` union.\n[[nodiscard]]\nconstexpr VectorRegSizePair ToVectorSizePair(SubRegSize size) {\n return VectorRegSizePair {.Vector = size};\n}\n[[nodiscard]]\nconstexpr VectorRegSizePair ToVectorSizePair(ScalarRegSize size) {\n return VectorRegSizePair {.Scalar = size};\n}\n\n// This `ShiftType` enum is used for ALU shift-register encoded instructions.\nenum class ShiftType : uint32_t {\n LSL = 0,\n LSR,\n ASR,\n ROR,\n};\n\n// This `ExtendedType` enum is used for ALU extended-register encoded instructions.\nenum class ExtendedType : uint32_t {\n UXTB = 0b000,\n UXTH = 0b001,\n UXTW = 0b010,\n UXTX = 0b011,\n SXTB = 0b100,\n SXTH = 0b101,\n SXTW = 0b110,\n SXTX = 0b111,\n LSL_32 = UXTW,\n LSL_64 = UXTX,\n};\n\n// This `Condition` enum is used for various conditional instructions.\nenum class Condition : uint32_t {\n // Meaning: Int - Float\n CC_EQ = 0, // Equal - Equal\n CC_NE, // Not Eq - Not Eq or unordered\n CC_CS, // Carry set - Greater than, equal, or unordered\n CC_CC, // Carry clear - Less than\n CC_MI, // Minus/Negative - Less than\n CC_PL, // Plus, positive or zero - GT, equal, or unordered\n CC_VS, // Overflow - Unordered\n CC_VC, // No Overflow - Ordered\n CC_HI, // Unsigned higher - GT, or unordered\n CC_LS, // Unsigned lower or same - LT or EQ\n CC_GE, // Signed GT or EQ - GT or EQ\n CC_LT, // Signed LT - LT or Unordered\n CC_GT, // Signed GT - GT\n CC_LE, // Signed LT or EQ - LT, EQ, or Unordered\n CC_AL, // Always - Always\n CC_NV, // Always - Always\n\n // Aliases\n CC_HS = CC_CS,\n CC_LO = CC_CC,\n};\n\n/*\n * This `StatusFlags` enum is used for conditional compare encoded instructions.\n * These directly encode to the `nzcv` flags.\n */\nenum class StatusFlags : uint32_t {\n None = 0,\n Flag_V = 0b0001,\n Flag_C = 0b0010,\n Flag_Z = 0b0100,\n Flag_N = 0b1000,\n\n Flag_NZCV = Flag_N | Flag_Z | Flag_C | Flag_V,\n};\n\n\n/*\n * This `IndexType` enum is used for load-store instructions.\n * Not all load-store instructions use this, so the user needs to be careful.\n */\nenum class IndexType {\n POST,\n OFFSET,\n PRE,\n\n UNPRIVILEGED,\n};\n\n// Used with adr and scalar + vector load/store variants to denote\n// a modifier operation.\nenum class SVEModType : uint8_t {\n MOD_UXTW,\n MOD_SXTW,\n MOD_LSL,\n MOD_NONE,\n};\n\n/* This `SVEMemOperand` class is used for the helper SVE load-store instructions.\n * Load-store instructions are quite expressive, so having a helper that handles these differences is worth it.\n */\nclass SVEMemOperand final {\npublic:\n enum class Type {\n ScalarPlusScalar,\n ScalarPlusImm,\n ScalarPlusVector,\n VectorPlusImm,\n };\n\n SVEMemOperand(XRegister rn, XRegister rm = XReg::zr)\n : rn {rn}\n , MemType {Type::ScalarPlusScalar}\n , MetaType {.ScalarScalarType {\n .rm = rm,\n }} {}\n SVEMemOperand(XRegister rn, int32_t imm = 0)\n : rn {rn}\n , MemType {Type::ScalarPlusImm}\n , MetaType {.ScalarImmType {\n .Imm = imm,\n }} {}\n SVEMemOperand(XRegister rn, ZRegister zm, SVEModType mod = SVEModType::MOD_NONE, uint8_t scale = 0)\n : rn {rn}\n , MemType {Type::ScalarPlusVector}\n , MetaType {.ScalarVectorType {\n .zm = zm,\n .mod = mod,\n .scale = scale,\n }} {}\n SVEMemOperand(ZRegister zn, uint32_t imm)\n : rn {Register {zn.Idx()}}\n , MemType {Type::VectorPlusImm}\n , MetaType {.VectorImmType {\n .Imm = imm,\n }} {}\n\n [[nodiscard]]\n bool IsScalarPlusScalar() const {\n return MemType == Type::ScalarPlusScalar;\n }\n [[nodiscard]]\n bool IsScalarPlusImm() const {\n return MemType == Type::ScalarPlusImm;\n }\n [[nodiscard]]\n bool IsScalarPlusVector() const {\n return MemType == Type::ScalarPlusVector;\n }\n [[nodiscard]]\n bool IsVectorPlusImm() const {\n return MemType == Type::VectorPlusImm;\n }\n\n union Data {\n struct {\n Register rm;\n } ScalarScalarType;\n\n struct {\n int32_t Imm;\n } ScalarImmType;\n\n struct {\n ZRegister zm;\n SVEModType mod;\n uint8_t scale;\n } ScalarVectorType;\n\n struct {\n // rn will be a ZRegister\n uint32_t Imm;\n } VectorImmType;\n };\n\n Register rn;\n Type MemType;\n Data MetaType;\n};\n\n/* This `ExtendedMemOperand` class is used for the helper load-store instructions.\n * Load-store instructions are quite expressive, so having a helper that handles these differences is worth it.\n */\nclass ExtendedMemOperand final {\npublic:\n ExtendedMemOperand(XRegister rn, XRegister rm = XReg::zr, ExtendedType Option = ExtendedType::LSL_64, uint32_t Shift = 0)\n : rn {rn}\n , MetaType {.Extended {\n .Header = {.MemType = TYPE_EXTENDED},\n .rm = rm,\n .Option = Option,\n .Shift = Shift,\n }} {}\n ExtendedMemOperand(XRegister rn, IndexType Index = IndexType::OFFSET, int32_t Imm = 0)\n : rn {rn}\n , MetaType {.ImmType {\n .Header = {.MemType = TYPE_IMM},\n .Index = Index,\n .Imm = Imm,\n }} {}\n\n Register rn;\n enum Type {\n TYPE_EXTENDED,\n TYPE_IMM,\n };\n struct HeaderStruct {\n Type MemType;\n };\n union {\n HeaderStruct Header;\n struct {\n HeaderStruct Header;\n Register rm;\n ExtendedType Option;\n uint32_t Shift;\n } Extended;\n struct {\n HeaderStruct Header;\n IndexType Index;\n int32_t Imm;\n } ImmType;\n } MetaType;\n};\n\ntemplate\ninline constexpr uint32_t GenSystemReg = op0 << 19 | op1 << 16 | CRn << 12 | CRm << 8 | op2 << 5;\n\n// This `SystemRegister` enum is used for the mrs/msr instructions.\nenum class SystemRegister : uint32_t {\n CTR_EL0 = GenSystemReg<0b11, 0b011, 0b0000, 0b0000, 0b001>,\n DCZID_EL0 = GenSystemReg<0b11, 0b011, 0b0000, 0b0000, 0b111>,\n TPIDR_EL0 = GenSystemReg<0b11, 0b011, 0b1101, 0b0000, 0b010>,\n RNDR = GenSystemReg<0b11, 0b011, 0b0010, 0b0100, 0b000>,\n RNDRRS = GenSystemReg<0b11, 0b011, 0b0010, 0b0100, 0b001>,\n NZCV = GenSystemReg<0b11, 0b011, 0b0100, 0b0010, 0b000>,\n FPCR = GenSystemReg<0b11, 0b011, 0b0100, 0b0100, 0b000>,\n TPIDRRO_EL0 = GenSystemReg<0b11, 0b011, 0b1101, 0b0000, 0b011>,\n CNTFRQ_EL0 = GenSystemReg<0b11, 0b011, 0b1110, 0b0000, 0b000>,\n CNTVCT_EL0 = GenSystemReg<0b11, 0b011, 0b1110, 0b0000, 0b010>,\n CNTVCTSS_EL0 = GenSystemReg<0b11, 0b011, 0b1110, 0b0000, 0b110>,\n};\n\ntemplate\ninline constexpr uint32_t GenDCReg = op1 << 16 | CRm << 8 | op2 << 5;\n\n// This `DataCacheOperation` enum is used for the dc instruction.\nenum class DataCacheOperation : uint32_t {\n IVAC = GenDCReg<0b000, 0b0110, 0b001>,\n ISW = GenDCReg<0b000, 0b0110, 0b010>,\n CSW = GenDCReg<0b000, 0b1010, 0b010>,\n CISW = GenDCReg<0b000, 0b1110, 0b010>,\n ZVA = GenDCReg<0b011, 0b0100, 0b001>,\n CVAC = GenDCReg<0b011, 0b1010, 0b001>,\n CVAU = GenDCReg<0b011, 0b1011, 0b001>,\n CIVAC = GenDCReg<0b011, 0b1110, 0b001>,\n\n // MTE2\n IGVAC = GenDCReg<0b000, 0b0110, 0b011>,\n IGSW = GenDCReg<0b000, 0b0110, 0b100>,\n IGDVAC = GenDCReg<0b000, 0b0110, 0b101>,\n IGDSW = GenDCReg<0b000, 0b0110, 0b110>,\n CGSW = GenDCReg<0b000, 0b1010, 0b100>,\n CGDSW = GenDCReg<0b000, 0b1010, 0b110>,\n CIGSW = GenDCReg<0b000, 0b1110, 0b100>,\n CIGDSW = GenDCReg<0b000, 0b1110, 0b110>,\n\n // MTE\n GVA = GenDCReg<0b011, 0b0100, 0b011>,\n GZVA = GenDCReg<0b011, 0b0100, 0b100>,\n CGVAC = GenDCReg<0b011, 0b1010, 0b011>,\n CGDVAC = GenDCReg<0b011, 0b1010, 0b101>,\n CGVAP = GenDCReg<0b011, 0b1100, 0b011>,\n CGDVAP = GenDCReg<0b011, 0b1100, 0b101>,\n CGVADP = GenDCReg<0b011, 0b1101, 0b011>,\n CGDVADP = GenDCReg<0b011, 0b1101, 0b101>,\n CIGVAC = GenDCReg<0b011, 0b1110, 0b011>,\n CIGDVAC = GenDCReg<0b011, 0b1110, 0b101>,\n\n // DPB\n CVAP = GenDCReg<0b011, 0b1100, 0b001>,\n\n // DPB2\n CVADP = GenDCReg<0b011, 0b1101, 0b001>,\n};\n\ntemplate\ninline constexpr uint32_t GenHintBarrierReg = CRm << 8 | op2 << 5;\n\n// This `HintRegister` enum is used for the hint instruction.\nenum class HintRegister : uint32_t {\n NOP = GenHintBarrierReg<0b0000, 0b000>,\n YIELD = GenHintBarrierReg<0b0000, 0b001>,\n WFE = GenHintBarrierReg<0b0000, 0b010>,\n WFI = GenHintBarrierReg<0b0000, 0b011>,\n SEV = GenHintBarrierReg<0b0000, 0b100>,\n SEVL = GenHintBarrierReg<0b0000, 0b101>,\n DGH = GenHintBarrierReg<0b0000, 0b110>,\n CSDB = GenHintBarrierReg<0b0010, 0b100>,\n};\n\n// This `BarrierRegister` enum is used for the various barrier instructions.\nenum class BarrierRegister : uint32_t {\n CLREX = GenHintBarrierReg<0b0000, 0b010>,\n TCOMMIT = GenHintBarrierReg<0b0000, 0b011>,\n DSB = GenHintBarrierReg<0b0000, 0b100>,\n DMB = GenHintBarrierReg<0b0000, 0b101>,\n ISB = GenHintBarrierReg<0b0000, 0b110>,\n SB = GenHintBarrierReg<0b0000, 0b111>,\n};\n\n// This `BarrierScope` enum is used for the dsb/dmb instructions.\nenum class BarrierScope : uint32_t {\n // Outer shareable\n OSHLD = 0b0001,\n OSHST = 0b0010,\n OSH = 0b0011,\n // Non shareable\n NSHLD = 0b0101,\n NSHST = 0b0110,\n NSH = 0b0111,\n // Inner shareable\n ISHLD = 0b1001,\n ISHST = 0b1010,\n ISH = 0b1011,\n // Full System visibility\n LD = 0b1101,\n ST = 0b1110,\n SY = 0b1111,\n};\n\n// This `Prefetch` enum is used for prefetch instructions.\nenum class Prefetch : uint32_t {\n // Prefetch for load\n PLDL1KEEP = 0b00000,\n PLDL1STRM = 0b00001,\n PLDL2KEEP = 0b00010,\n PLDL2STRM = 0b00011,\n PLDL3KEEP = 0b00100,\n PLDL3STRM = 0b00101,\n\n // Preload instructions\n PLIL1KEEP = 0b01000,\n PLIL1STRM = 0b01001,\n PLIL2KEEP = 0b01010,\n PLIL2STRM = 0b01011,\n PLIL3KEEP = 0b01100,\n PLIL3STRM = 0b01101,\n\n // Preload for store\n PSTL1KEEP = 0b10000,\n PSTL1STRM = 0b10001,\n PSTL2KEEP = 0b10010,\n PSTL2STRM = 0b10011,\n PSTL3KEEP = 0b10100,\n PSTL3STRM = 0b10101,\n};\n\n// This `PredicatePattern` enun is used for some SVE instructions.\nenum class PredicatePattern : uint32_t {\n SVE_POW2 = 0b00000,\n SVE_VL1 = 0b00001,\n SVE_VL2 = 0b00010,\n SVE_VL3 = 0b00011,\n SVE_VL4 = 0b00100,\n SVE_VL5 = 0b00101,\n SVE_VL6 = 0b00110,\n SVE_VL7 = 0b00111,\n SVE_VL8 = 0b01000,\n SVE_VL16 = 0b01001,\n SVE_VL32 = 0b01010,\n SVE_VL64 = 0b01011,\n SVE_VL128 = 0b01100,\n SVE_VL256 = 0b01101,\n SVE_MUL4 = 0b11101,\n SVE_MUL3 = 0b11110,\n SVE_ALL = 0b11111,\n};\n\n// Used with SVE FP immediate arithmetic instructions\nenum class SVEFAddSubImm : uint32_t {\n _0_5,\n _1_0,\n};\nenum class SVEFMulImm : uint32_t {\n _0_5,\n _2_0,\n};\nenum class SVEFMaxMinImm : uint32_t {\n _0_0,\n _1_0,\n};\n\n/* This `BackwardLabel` struct is used for retaining a location for PC-Relative instructions.\n * This is specifically a label for a target that is logically `below` an instruction that uses it.\n * Which means that a branch would jump backwards.\n */\nstruct BackwardLabel {\n uint8_t* Location {};\n};\n\n/* This `ForwardLabel` struct is used for retaining a location for PC-Relative instructions.\n * This is specifically a label for a target that is logically `above` an instruction that uses it.\n * Which means that a branch would jump forwards.\n */\nstruct ForwardLabel {\n enum class InstType {\n UNKNOWN,\n ADR,\n ADRP,\n B,\n BC,\n TEST_BRANCH,\n RELATIVE_LOAD,\n LONG_ADDRESS_GEN,\n };\n\n struct Reference {\n uint8_t* Location {};\n InstType Type = InstType::UNKNOWN;\n };\n\n // The first element is stored separately to avoid allocations for simple cases\n Reference FirstInst;\n\n fextl::vector Insts;\n};\n\n/* This `BiDirectionalLabel` struct used for retaining a location for PC-Relative instructions.\n * This is specifically a label for a target that is in either direction of an instruction that uses it.\n * Which means a branch could jump backwards or forwards depending on situation.\n */\nstruct BiDirectionalLabel {\n BackwardLabel Backward;\n ForwardLabel Forward;\n};\n\nstatic inline void AddLocationToLabel(ForwardLabel* Label, ForwardLabel::Reference&& Location) {\n if (Label->FirstInst.Location == nullptr) {\n Label->FirstInst = Location;\n } else {\n Label->Insts.push_back(Location);\n }\n}\n\n// Some FCMA ASIMD instructions support a rotation argument.\nenum class Rotation : uint32_t {\n ROTATE_0 = 0b00,\n ROTATE_90 = 0b01,\n ROTATE_180 = 0b10,\n ROTATE_270 = 0b11,\n};\n\n// Concept for contraining some instructions to accept only an XRegister or WRegister.\n// Particularly for operations that differ encodings depending on which one is used.\ntemplate\nconcept IsXOrWRegister = std::is_same_v || std::is_same_v;\n\n// Concept for contraining some instructions to accept only a QRegister or DRegister.\ntemplate\nconcept IsQOrDRegister = std::is_same_v || std::is_same_v;\n\ntemplate\nconcept IsLabel = std::is_same_v || std::is_same_v ||\n std::is_same_v || std::is_same_v;\n\nenum class BranchEncodeSucceeded {\n Success,\n Failure,\n};\n\n// Whether or not a given set of vector registers are sequential\n// in increasing order as far as the register file is concerned (modulo its size)\n//\n// For example, a set of registers like:\n//\n// v1, v2, v3 and\n// v31, v0, v1\n//\n// would both be considered sequential sequences, and some instructions in particular\n// limit register lists to these kind of sequences.\n//\ntemplate\nconstexpr bool AreVectorsSequential(T first, const Args&... args) {\n // Ensure we always have a pair of registers to compare against.\n static_assert(sizeof...(args) >= 1, \"Number of arguments must be greater than 1\");\n\n const auto fn = [](auto& lhs, const auto& rhs) {\n const auto result = ((lhs.Idx() + 1) % 32) == rhs.Idx();\n lhs = rhs;\n return result;\n };\n\n return (fn(first, args) && ...);\n}\n\n// Returns if the immediate can fit in to add/sub immediate instruction encodings.\nconstexpr bool IsImmAddSub(uint64_t imm) {\n constexpr uint64_t U12Mask = 0xFFF;\n auto FitsWithin12Bits = [](uint64_t imm) {\n return (imm & ~U12Mask) == 0;\n };\n // Can fit in to the instruction encoding:\n // - if only bits [11:0] are set.\n // - if only bits [23:12] are set.\n return FitsWithin12Bits(imm) || (FitsWithin12Bits(imm >> 12) && (imm & U12Mask) == 0);\n}\n\n// This is an emitter that is designed around the smallest code bloat as possible.\n// Eschewing most developer convenience in order to keep code as small as possible.\n\n// Choices:\n// - Size of ops passed as an argument rather than template to let the compiler use csel instead of branching.\n// - Registers are unsized so they can be passed in a GPR and not need conversion operations\nclass Emitter : public ARMEmitter::Buffer {\npublic:\n Emitter() = default;\n\n Emitter(uint8_t* Base, uint64_t BaseSize)\n : Buffer(Base, BaseSize) {}\n\n // Bind a backward label to an address.\n // Address that is bound is the current emitter location.\n [[nodiscard]] bool Bind(BackwardLabel* Label) {\n LOGMAN_THROW_A_FMT(Label->Location == nullptr, \"Trying to bind a label twice\");\n Label->Location = GetCursorAddress();\n\n // Always binds because it is only storing a location.\n return true;\n }\n\n [[nodiscard]] bool Bind(const ForwardLabel::Reference* Label) {\n uint8_t* CurrentAddress = GetCursorAddress();\n // Patch up the instructions\n switch (Label->Type) {\n case ForwardLabel::InstType::ADR: {\n uint32_t* Instruction = reinterpret_cast(Label->Location);\n int64_t Imm = reinterpret_cast(CurrentAddress) - reinterpret_cast(Instruction);\n if (!IsADRRange(Imm)) {\n // Can't bind.\n return false;\n }\n uint32_t InstMask = 0b11 << 29 | 0b1111'1111'1111'1111'111 << 5;\n uint32_t Offset = static_cast(Imm) & 0x3F'FFFF;\n uint32_t Inst = *Instruction & ~InstMask;\n Inst |= (Offset & 0b11) << 29;\n Inst |= (Offset >> 2) << 5;\n *Instruction = Inst;\n break;\n }\n case ForwardLabel::InstType::ADRP: {\n uint32_t* Instruction = reinterpret_cast(Label->Location);\n int64_t Imm = reinterpret_cast(CurrentAddress) - reinterpret_cast(Instruction);\n\n if (!(IsADRPRange(Imm) && IsADRPAligned(Imm))) {\n // Can't bind.\n return false;\n }\n\n Imm >>= 12;\n uint32_t InstMask = 0b11 << 29 | 0b1111'1111'1111'1111'111 << 5;\n uint32_t Offset = static_cast(Imm) & 0x3F'FFFF;\n uint32_t Inst = *Instruction & ~InstMask;\n Inst |= (Offset & 0b11) << 29;\n Inst |= (Offset >> 2) << 5;\n *Instruction = Inst;\n break;\n }\n case ForwardLabel::InstType::B: {\n uint32_t* Instruction = reinterpret_cast(Label->Location);\n int64_t Imm = reinterpret_cast(CurrentAddress) - reinterpret_cast(Instruction);\n if (!(Imm >= -134217728 && Imm <= 134217724 && ((Imm & 0b11) == 0))) {\n // Can't bind.\n return false;\n }\n Imm >>= 2;\n uint32_t InstMask = 0x3FF'FFFF;\n uint32_t Offset = static_cast(Imm) & InstMask;\n uint32_t Inst = *Instruction & ~InstMask;\n Inst |= Offset;\n *Instruction = Inst;\n\n break;\n }\n case ForwardLabel::InstType::TEST_BRANCH: {\n uint32_t* Instruction = reinterpret_cast(Label->Location);\n int64_t Imm = reinterpret_cast(CurrentAddress) - reinterpret_cast(Instruction);\n if (!(Imm >= -32768 && Imm <= 32764 && ((Imm & 0b11) == 0))) {\n // Can't bind.\n return false;\n }\n Imm >>= 2;\n uint32_t InstMask = 0x3FFF;\n uint32_t Offset = static_cast(Imm) & InstMask;\n uint32_t Inst = *Instruction & ~(InstMask << 5);\n Inst |= Offset << 5;\n *Instruction = Inst;\n\n break;\n }\n case ForwardLabel::InstType::BC:\n case ForwardLabel::InstType::RELATIVE_LOAD: {\n uint32_t* Instruction = reinterpret_cast(Label->Location);\n int64_t Imm = reinterpret_cast(CurrentAddress) - reinterpret_cast(Instruction);\n if (!(Imm >= -1048576 && Imm <= 1048575 && ((Imm & 0b11) == 0))) {\n // Can't bind.\n return false;\n }\n Imm >>= 2;\n uint32_t InstMask = 0x7'FFFF;\n uint32_t Offset = static_cast(Imm) & InstMask;\n uint32_t Inst = *Instruction & ~(InstMask << 5);\n Inst |= Offset << 5;\n *Instruction = Inst;\n break;\n }\n case ForwardLabel::InstType::LONG_ADDRESS_GEN: {\n const auto* Instructions = reinterpret_cast(Label->Location);\n const auto ImmInstOne = reinterpret_cast(CurrentAddress) - reinterpret_cast(&Instructions[0]);\n const auto ImmInstTwo = reinterpret_cast(CurrentAddress) - reinterpret_cast(&Instructions[1]);\n const auto ImmInstThree = reinterpret_cast(CurrentAddress) - reinterpret_cast(&Instructions[2]);\n const auto OriginalOffset = GetCursorOffset();\n\n const auto InstOffset = GetCursorOffsetFromAddress(Instructions);\n SetCursorOffset(InstOffset);\n\n // We encoded the destination register in to the first instruction space.\n // Read it back.\n ARMEmitter::Register DestReg(Instructions[0]);\n\n if (IsADRRange(ImmInstThree)) {\n // If within ADR range from the third instruction, then we can emit NOP+NOP+ADR\n nop();\n nop();\n adr(DestReg, static_cast(ImmInstThree) & 0x7FFF);\n } else if (IsADRPRange(ImmInstTwo)) {\n\n // If within ADRP range from the first instruction, then we are /definitely/ in range for the second instruction.\n // First check if we are in non-offset range for second instruction.\n if (IsADRPAligned(reinterpret_cast(CurrentAddress))) {\n // We can emit nop + nop + adrp\n nop();\n nop();\n adrp(DestReg, static_cast(ImmInstThree >> 12) & 0x7FFF);\n } else {\n // Not aligned, need nop + adrp + add\n nop();\n adrp(DestReg, static_cast(ImmInstTwo >> 12) & 0x7FFF);\n add(ARMEmitter::Size::i64Bit, DestReg, DestReg, ImmInstTwo & 0xFFF);\n }\n } else {\n // Stinky path, we need to emit a movz+movk+movk sequence.\n movz(ARMEmitter::Size::i64Bit, DestReg, uint32_t(ImmInstOne >> 32) & 0x7FFF, 32);\n movk(ARMEmitter::Size::i64Bit, DestReg, uint32_t(ImmInstOne >> 16) & 0xFFFF, 16);\n movk(ARMEmitter::Size::i64Bit, DestReg, uint32_t(ImmInstOne) & 0xFFFF);\n }\n\n SetCursorOffset(OriginalOffset);\n break;\n }\n default: LOGMAN_MSG_A_FMT(\"Unexpected inst type in label fixup\");\n }\n\n return true;\n }\n\n // Bind a forward label to a location.\n // This walks all the instructions in the label's vector.\n // Then backpatching all instructions that have used the label.\n [[nodiscard]] bool Bind(ForwardLabel* Label) {\n bool Bound = true;\n if (Label->FirstInst.Location) {\n Bound &= Bind(&Label->FirstInst);\n }\n for (auto& Inst : Label->Insts) {\n Bound &= Bind(&Inst);\n }\n\n return Bound;\n }\n\n // Bind a bidirectional location to a location.\n // Binds both forwards and backwards depending on how the label was used.\n [[nodiscard]] bool Bind(BiDirectionalLabel* Label) {\n bool Bound = true;\n if (!Label->Backward.Location) {\n Bound &= Bind(&Label->Backward);\n }\n Bound &= Bind(&Label->Forward);\n\n return Bound;\n }\n\n static constexpr Condition InvertCondition(Condition cond) {\n // These behave as always, so it makes no sense to allow inverting these.\n LOGMAN_THROW_A_FMT(cond != Condition::CC_AL && cond != Condition::CC_NV, \"Cannot invert CC_AL or CC_NV\");\n return static_cast(FEXCore::ToUnderlying(cond) ^ 1);\n }\n\n#include \n\npublic:\n\n// This symbol is used to allow external tooling (IDEs, clang-format, ...) to process the included files individually:\n// If defined, the files will inject member functions into this class.\n// If not, the files will wrap the member functions in a class so that tooling will process them properly.\n#define INCLUDED_BY_EMITTER\n\n // TODO: Implement SME when it matters.\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n#undef INCLUDED_BY_EMITTER\n\nprotected:\n template\n uint32_t Encode_ra(T Reg) const {\n return Reg.Idx() << 10;\n }\n uint32_t Encode_ra(uint32_t Reg) const {\n return Reg << 10;\n }\n template\n uint32_t Encode_rt2(T Reg) const {\n return Reg.Idx() << 10;\n }\n uint32_t Encode_rt2(uint32_t Reg) const {\n return Reg << 10;\n }\n template\n uint32_t Encode_rm(T Reg) const {\n return Reg.Idx() << 16;\n }\n uint32_t Encode_rm(uint32_t Reg) const {\n return Reg << 16;\n }\n template\n uint32_t Encode_rs(T Reg) const {\n return Reg.Idx() << 16;\n }\n uint32_t Encode_rs(uint32_t Reg) const {\n return Reg << 16;\n }\n template\n uint32_t Encode_rn(T Reg) const {\n return Reg.Idx() << 5;\n }\n uint32_t Encode_rn(uint32_t Reg) const {\n return Reg << 5;\n }\n template\n uint32_t Encode_rd(T Reg) const {\n return Reg.Idx();\n }\n uint32_t Encode_rd(uint32_t Reg) const {\n return Reg;\n }\n template\n uint32_t Encode_rt(T Reg) const {\n return Reg.Idx();\n }\n uint32_t Encode_rt(Prefetch Reg) const {\n return FEXCore::ToUnderlying(Reg);\n }\n uint32_t Encode_rt(uint32_t Reg) const {\n return Reg;\n }\n template\n uint32_t Encode_pd(T Reg) const {\n return FEXCore::ToUnderlying(Reg);\n }\n};\n} // namespace ARMEmitter\n" }, { "path": "CodeEmitter/CodeEmitter/LoadstoreOps.inl", "content": "// SPDX-License-Identifier: MIT\n/* Load-store instruction emitters\n *\n * For GPR load-stores that take a `Size` argument as their first argument can be 32-bit or 64-bit.\n * For GPR load-stores that don't take a `Size` argument, then their operating size is determined by the name of the instruction.\n *\n * For Vector load-stores, most take a `SubRegSize` to determine the size of the elements getting loaded or stored.\n * Depending on the instruction it can be an single element or the full instruction, it depends on the instruction.\n *\n * There are some load-store helper functions which take a `ExtendedMemOperand` argument.\n * This helper will select the viable load-store that can work with the provided encapsulated arguments.\n */\n\n#pragma once\n#ifndef INCLUDED_BY_EMITTER\n#include \nnamespace ARMEmitter {\nstruct EmitterOps : Emitter {\n#endif\n\npublic:\n // Compare and swap pair\n void casp(ARMEmitter::Size s, ARMEmitter::Register rs, ARMEmitter::Register rs2, ARMEmitter::Register rt, ARMEmitter::Register rt2,\n ARMEmitter::Register rn) {\n LOGMAN_THROW_A_FMT((rs.Idx() + 1) == rs2.Idx(), \"These must be sequential\");\n LOGMAN_THROW_A_FMT((rt.Idx() + 1) == rt2.Idx(), \"These must be sequential\");\n constexpr uint32_t Op = 0b0000'1000'001 << 21;\n AtomicOp(Op, s, 0, 0, rs, rt, ARMEmitter::Reg::r31, rn);\n }\n void caspa(ARMEmitter::Size s, ARMEmitter::Register rs, ARMEmitter::Register rs2, ARMEmitter::Register rt, ARMEmitter::Register rt2,\n ARMEmitter::Register rn) {\n LOGMAN_THROW_A_FMT((rs.Idx() + 1) == rs2.Idx(), \"These must be sequential\");\n LOGMAN_THROW_A_FMT((rt.Idx() + 1) == rt2.Idx(), \"These must be sequential\");\n constexpr uint32_t Op = 0b0000'1000'001 << 21;\n AtomicOp(Op, s, 1, 0, rs, rt, ARMEmitter::Reg::r31, rn);\n }\n void caspl(ARMEmitter::Size s, ARMEmitter::Register rs, ARMEmitter::Register rs2, ARMEmitter::Register rt, ARMEmitter::Register rt2,\n ARMEmitter::Register rn) {\n LOGMAN_THROW_A_FMT((rs.Idx() + 1) == rs2.Idx(), \"These must be sequential\");\n LOGMAN_THROW_A_FMT((rt.Idx() + 1) == rt2.Idx(), \"These must be sequential\");\n constexpr uint32_t Op = 0b0000'1000'001 << 21;\n AtomicOp(Op, s, 0, 1, rs, rt, ARMEmitter::Reg::r31, rn);\n }\n void caspal(ARMEmitter::Size s, ARMEmitter::Register rs, ARMEmitter::Register rs2, ARMEmitter::Register rt, ARMEmitter::Register rt2,\n ARMEmitter::Register rn) {\n LOGMAN_THROW_A_FMT((rs.Idx() + 1) == rs2.Idx(), \"These must be sequential\");\n LOGMAN_THROW_A_FMT((rt.Idx() + 1) == rt2.Idx(), \"These must be sequential\");\n constexpr uint32_t Op = 0b0000'1000'001 << 21;\n AtomicOp(Op, s, 1, 1, rs, rt, ARMEmitter::Reg::r31, rn);\n }\n\n // Advanced SIMD load/store multiple structures\n template\n void ld1(T rt, Register rn) {\n constexpr uint32_t Op = 0b0000'1100'000 << 21;\n constexpr uint32_t Opcode = 0b0111 << 12;\n ASIMDLoadStoreMultipleStructure(Op, Opcode, rt, rn, Reg::r0);\n }\n template\n void ld1(T rt, T rt2, Register rn) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rt, rt2), \"rt and rt2 must be sequential\");\n constexpr uint32_t Op = 0b0000'1100'000 << 21;\n constexpr uint32_t Opcode = 0b1010 << 12;\n ASIMDLoadStoreMultipleStructure(Op, Opcode, rt, rn, Reg::r0);\n }\n template\n void ld1(T rt, T rt2, T rt3, Register rn) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rt, rt2, rt3), \"rt, rt2, and rt3 must be sequential\");\n constexpr uint32_t Op = 0b0000'1100'000 << 21;\n constexpr uint32_t Opcode = 0b0110 << 12;\n ASIMDLoadStoreMultipleStructure(Op, Opcode, rt, rn, Reg::r0);\n }\n template\n void ld1(T rt, T rt2, T rt3, T rt4, Register rn) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rt, rt2, rt3, rt4), \"rt, rt2, rt3, and rt4 must be sequential\");\n constexpr uint32_t Op = 0b0000'1100'000 << 21;\n constexpr uint32_t Opcode = 0b0010 << 12;\n ASIMDLoadStoreMultipleStructure(Op, Opcode, rt, rn, Reg::r0);\n }\n template\n void st1(T rt, Register rn) {\n constexpr uint32_t Op = 0b0000'1100'000 << 21;\n constexpr uint32_t Opcode = 0b0111 << 12;\n ASIMDLoadStoreMultipleStructure(Op, Opcode, rt, rn, Reg::r0);\n }\n template\n void st1(T rt, T rt2, Register rn) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rt, rt2), \"rt and rt2 must be sequential\");\n constexpr uint32_t Op = 0b0000'1100'000 << 21;\n constexpr uint32_t Opcode = 0b1010 << 12;\n ASIMDLoadStoreMultipleStructure(Op, Opcode, rt, rn, Reg::r0);\n }\n template\n void st1(T rt, T rt2, T rt3, Register rn) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rt, rt2, rt3), \"rt, rt2, and rt3 must be sequential\");\n constexpr uint32_t Op = 0b0000'1100'000 << 21;\n constexpr uint32_t Opcode = 0b0110 << 12;\n ASIMDLoadStoreMultipleStructure(Op, Opcode, rt, rn, Reg::r0);\n }\n template\n void st1(T rt, T rt2, T rt3, T rt4, Register rn) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rt, rt2, rt3, rt4), \"rt, rt2, rt3, and rt4 must be sequential\");\n constexpr uint32_t Op = 0b0000'1100'000 << 21;\n constexpr uint32_t Opcode = 0b0010 << 12;\n ASIMDLoadStoreMultipleStructure(Op, Opcode, rt, rn, Reg::r0);\n }\n template\n void ld2(T rt, T rt2, Register rn) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rt, rt2), \"rt and rt2 must be sequential\");\n constexpr uint32_t Op = 0b0000'1100'000 << 21;\n constexpr uint32_t Opcode = 0b1000 << 12;\n ASIMDLoadStoreMultipleStructure(Op, Opcode, rt, rn, Reg::r0);\n }\n template\n void st2(T rt, T rt2, Register rn) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rt, rt2), \"rt and rt2 must be sequential\");\n constexpr uint32_t Op = 0b0000'1100'000 << 21;\n constexpr uint32_t Opcode = 0b1000 << 12;\n ASIMDLoadStoreMultipleStructure(Op, Opcode, rt, rn, Reg::r0);\n }\n template\n void ld3(T rt, T rt2, T rt3, Register rn) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rt, rt2, rt3), \"rt, rt2, and rt3 must be sequential\");\n constexpr uint32_t Op = 0b0000'1100'000 << 21;\n constexpr uint32_t Opcode = 0b0100 << 12;\n ASIMDLoadStoreMultipleStructure(Op, Opcode, rt, rn, Reg::r0);\n }\n template\n void st3(T rt, T rt2, T rt3, Register rn) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rt, rt2, rt3), \"rt, rt2, and rt3 must be sequential\");\n constexpr uint32_t Op = 0b0000'1100'000 << 21;\n constexpr uint32_t Opcode = 0b0100 << 12;\n ASIMDLoadStoreMultipleStructure(Op, Opcode, rt, rn, Reg::r0);\n }\n template\n void ld4(T rt, T rt2, T rt3, T rt4, Register rn) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rt, rt2, rt3, rt4), \"rt, rt2, rt3, and rt4 must be sequential\");\n constexpr uint32_t Op = 0b0000'1100'000 << 21;\n constexpr uint32_t Opcode = 0b0000 << 12;\n ASIMDLoadStoreMultipleStructure(Op, Opcode, rt, rn, Reg::r0);\n }\n template\n void st4(T rt, T rt2, T rt3, T rt4, Register rn) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rt, rt2, rt3, rt4), \"rt, rt2, rt3, and rt4 must be sequential\");\n constexpr uint32_t Op = 0b0000'1100'000 << 21;\n constexpr uint32_t Opcode = 0b0000 << 12;\n ASIMDLoadStoreMultipleStructure(Op, Opcode, rt, rn, Reg::r0);\n }\n // Advanced SIMD load/store multiple structures (post-indexed)\n static constexpr uint32_t ASIMDLoadstoreMultiplePost_Op = 0b0000'1100'100 << 21;\n template\n void ld1(T rt, Register rn, Register rm) {\n constexpr uint32_t Opcode = 0b0111 << 12;\n ASIMDLoadStoreMultipleStructure(ASIMDLoadstoreMultiplePost_Op, Opcode, rt, rn, rm);\n }\n template\n void ld1(T rt, Register rn, uint32_t PostOffset) {\n LOGMAN_THROW_A_FMT((std::is_same_v && (PostOffset == 16)) || (std::is_same_v && (PostOffset == 8)),\n \"Post-index offset needs to match number of elements times their size\");\n\n constexpr uint32_t Opcode = 0b0111 << 12;\n ASIMDLoadStoreMultipleStructure(ASIMDLoadstoreMultiplePost_Op, Opcode, rt, rn, Reg::r31);\n }\n template\n void ld1(T rt, T rt2, Register rn, Register rm) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rt, rt2), \"rt and rt2 must be sequential\");\n constexpr uint32_t Opcode = 0b1010 << 12;\n ASIMDLoadStoreMultipleStructure(ASIMDLoadstoreMultiplePost_Op, Opcode, rt, rn, rm);\n }\n template\n void ld1(T rt, T rt2, Register rn, uint32_t PostOffset) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rt, rt2), \"rt and rt2 must be sequential\");\n LOGMAN_THROW_A_FMT((std::is_same_v && (PostOffset == 32)) || (std::is_same_v && (PostOffset == 16)),\n \"Post-index offset needs to match number of elements times their size\");\n\n constexpr uint32_t Opcode = 0b1010 << 12;\n ASIMDLoadStoreMultipleStructure(ASIMDLoadstoreMultiplePost_Op, Opcode, rt, rn, Reg::r31);\n }\n template\n void ld1(T rt, T rt2, T rt3, Register rn, Register rm) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rt, rt2, rt3), \"rt, rt2, and rt3 must be sequential\");\n constexpr uint32_t Opcode = 0b0110 << 12;\n ASIMDLoadStoreMultipleStructure(ASIMDLoadstoreMultiplePost_Op, Opcode, rt, rn, rm);\n }\n template\n void ld1(T rt, T rt2, T rt3, Register rn, uint32_t PostOffset) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rt, rt2, rt3), \"rt, rt2, and rt3 must be sequential\");\n LOGMAN_THROW_A_FMT((std::is_same_v && (PostOffset == 48)) || (std::is_same_v && (PostOffset == 24)),\n \"Post-index offset needs to match number of elements times their size\");\n\n constexpr uint32_t Opcode = 0b0110 << 12;\n ASIMDLoadStoreMultipleStructure(ASIMDLoadstoreMultiplePost_Op, Opcode, rt, rn, Reg::r31);\n }\n template\n void ld1(T rt, T rt2, T rt3, T rt4, Register rn, Register rm) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rt, rt2, rt3, rt4), \"rt, rt2, rt3, and rt4 must be sequential\");\n constexpr uint32_t Opcode = 0b0010 << 12;\n ASIMDLoadStoreMultipleStructure(ASIMDLoadstoreMultiplePost_Op, Opcode, rt, rn, rm);\n }\n template\n void ld1(T rt, T rt2, T rt3, T rt4, Register rn, uint32_t PostOffset) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rt, rt2, rt3, rt4), \"rt, rt2, rt3, and rt4 must be sequential\");\n LOGMAN_THROW_A_FMT((std::is_same_v && (PostOffset == 64)) || (std::is_same_v && (PostOffset == 32)),\n \"Post-index offset needs to match number of elements times their size\");\n\n constexpr uint32_t Opcode = 0b0010 << 12;\n ASIMDLoadStoreMultipleStructure(ASIMDLoadstoreMultiplePost_Op, Opcode, rt, rn, Reg::r31);\n }\n\n template\n void st1(T rt, Register rn, Register rm) {\n constexpr uint32_t Opcode = 0b0111 << 12;\n ASIMDLoadStoreMultipleStructure(ASIMDLoadstoreMultiplePost_Op, Opcode, rt, rn, rm);\n }\n template\n void st1(T rt, Register rn, uint32_t PostOffset) {\n LOGMAN_THROW_A_FMT((std::is_same_v && (PostOffset == 16)) || (std::is_same_v && (PostOffset == 8)),\n \"Post-index offset needs to match number of elements times their size\");\n\n constexpr uint32_t Opcode = 0b0111 << 12;\n ASIMDLoadStoreMultipleStructure(ASIMDLoadstoreMultiplePost_Op, Opcode, rt, rn, Reg::r31);\n }\n template\n void st1(T rt, T rt2, Register rn, Register rm) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rt, rt2), \"rt and rt2 must be sequential\");\n constexpr uint32_t Opcode = 0b1010 << 12;\n ASIMDLoadStoreMultipleStructure(ASIMDLoadstoreMultiplePost_Op, Opcode, rt, rn, rm);\n }\n template\n void st1(T rt, T rt2, Register rn, uint32_t PostOffset) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rt, rt2), \"rt and rt2 must be sequential\");\n LOGMAN_THROW_A_FMT((std::is_same_v && (PostOffset == 32)) || (std::is_same_v && (PostOffset == 16)),\n \"Post-index offset needs to match number of elements times their size\");\n\n constexpr uint32_t Opcode = 0b1010 << 12;\n ASIMDLoadStoreMultipleStructure(ASIMDLoadstoreMultiplePost_Op, Opcode, rt, rn, Reg::r31);\n }\n template\n void st1(T rt, T rt2, T rt3, Register rn, Register rm) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rt, rt2, rt3), \"rt, rt2, and rt3 must be sequential\");\n constexpr uint32_t Opcode = 0b0110 << 12;\n ASIMDLoadStoreMultipleStructure(ASIMDLoadstoreMultiplePost_Op, Opcode, rt, rn, rm);\n }\n template\n void st1(T rt, T rt2, T rt3, Register rn, uint32_t PostOffset) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rt, rt2, rt3), \"rt, rt2, and rt3 must be sequential\");\n LOGMAN_THROW_A_FMT((std::is_same_v && (PostOffset == 48)) || (std::is_same_v && (PostOffset == 24)),\n \"Post-index offset needs to match number of elements times their size\");\n\n constexpr uint32_t Opcode = 0b0110 << 12;\n ASIMDLoadStoreMultipleStructure(ASIMDLoadstoreMultiplePost_Op, Opcode, rt, rn, Reg::r31);\n }\n template\n void st1(T rt, T rt2, T rt3, T rt4, Register rn, Register rm) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rt, rt2, rt3, rt4), \"rt, rt2, rt3, and rt4 must be sequential\");\n constexpr uint32_t Opcode = 0b0010 << 12;\n ASIMDLoadStoreMultipleStructure(ASIMDLoadstoreMultiplePost_Op, Opcode, rt, rn, rm);\n }\n template\n void st1(T rt, T rt2, T rt3, T rt4, Register rn, uint32_t PostOffset) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rt, rt2, rt3, rt4), \"rt, rt2, rt3, and rt4 must be sequential\");\n LOGMAN_THROW_A_FMT((std::is_same_v && (PostOffset == 64)) || (std::is_same_v && (PostOffset == 32)),\n \"Post-index offset needs to match number of elements times their size\");\n\n constexpr uint32_t Opcode = 0b0010 << 12;\n ASIMDLoadStoreMultipleStructure(ASIMDLoadstoreMultiplePost_Op, Opcode, rt, rn, Reg::r31);\n }\n\n template\n void ld2(T rt, T rt2, Register rn, Register rm) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rt, rt2), \"rt and rt2 must be sequential\");\n constexpr uint32_t Opcode = 0b1000 << 12;\n ASIMDLoadStoreMultipleStructure(ASIMDLoadstoreMultiplePost_Op, Opcode, rt, rn, rm);\n }\n template\n void ld2(T rt, T rt2, Register rn, uint32_t PostOffset) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rt, rt2), \"rt and rt2 must be sequential\");\n LOGMAN_THROW_A_FMT((std::is_same_v && (PostOffset == 32)) || (std::is_same_v && (PostOffset == 16)),\n \"Post-index offset needs to match number of elements times their size\");\n\n constexpr uint32_t Opcode = 0b1000 << 12;\n ASIMDLoadStoreMultipleStructure(ASIMDLoadstoreMultiplePost_Op, Opcode, rt, rn, Reg::r31);\n }\n template\n void st2(T rt, T rt2, Register rn, Register rm) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rt, rt2), \"rt and rt2 must be sequential\");\n constexpr uint32_t Opcode = 0b1000 << 12;\n ASIMDLoadStoreMultipleStructure(ASIMDLoadstoreMultiplePost_Op, Opcode, rt, rn, rm);\n }\n template\n void st2(T rt, T rt2, Register rn, uint32_t PostOffset) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rt, rt2), \"rt and rt2 must be sequential\");\n LOGMAN_THROW_A_FMT((std::is_same_v && (PostOffset == 32)) || (std::is_same_v && (PostOffset == 16)),\n \"Post-index offset needs to match number of elements times their size\");\n\n constexpr uint32_t Opcode = 0b1000 << 12;\n ASIMDLoadStoreMultipleStructure(ASIMDLoadstoreMultiplePost_Op, Opcode, rt, rn, Reg::r31);\n }\n template\n void ld3(T rt, T rt2, T rt3, Register rn, Register rm) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rt, rt2, rt3), \"rt, rt2, and rt3 must be sequential\");\n constexpr uint32_t Opcode = 0b0100 << 12;\n ASIMDLoadStoreMultipleStructure(ASIMDLoadstoreMultiplePost_Op, Opcode, rt, rn, rm);\n }\n template\n void ld3(T rt, T rt2, T rt3, Register rn, uint32_t PostOffset) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rt, rt2, rt3), \"rt, rt2, and rt3 must be sequential\");\n LOGMAN_THROW_A_FMT((std::is_same_v && (PostOffset == 48)) || (std::is_same_v && (PostOffset == 24)),\n \"Post-index offset needs to match number of elements times their size\");\n\n constexpr uint32_t Opcode = 0b0100 << 12;\n ASIMDLoadStoreMultipleStructure(ASIMDLoadstoreMultiplePost_Op, Opcode, rt, rn, Reg::r31);\n }\n template\n void st3(T rt, T rt2, T rt3, Register rn, Register rm) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rt, rt2, rt3), \"rt, rt2, and rt3 must be sequential\");\n constexpr uint32_t Opcode = 0b0100 << 12;\n ASIMDLoadStoreMultipleStructure(ASIMDLoadstoreMultiplePost_Op, Opcode, rt, rn, rm);\n }\n template\n void st3(T rt, T rt2, T rt3, Register rn, uint32_t PostOffset) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rt, rt2, rt3), \"rt, rt2, and rt3 must be sequential\");\n LOGMAN_THROW_A_FMT((std::is_same_v && (PostOffset == 48)) || (std::is_same_v && (PostOffset == 24)),\n \"Post-index offset needs to match number of elements times their size\");\n\n constexpr uint32_t Opcode = 0b0100 << 12;\n ASIMDLoadStoreMultipleStructure(ASIMDLoadstoreMultiplePost_Op, Opcode, rt, rn, Reg::r31);\n }\n template\n void ld4(T rt, T rt2, T rt3, T rt4, Register rn, Register rm) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rt, rt2, rt3, rt4), \"rt, rt2, rt3, and rt4 must be sequential\");\n constexpr uint32_t Opcode = 0b0000 << 12;\n ASIMDLoadStoreMultipleStructure(ASIMDLoadstoreMultiplePost_Op, Opcode, rt, rn, rm);\n }\n template\n void ld4(T rt, T rt2, T rt3, T rt4, Register rn, uint32_t PostOffset) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rt, rt2, rt3, rt4), \"rt, rt2, rt3, and rt4 must be sequential\");\n LOGMAN_THROW_A_FMT((std::is_same_v && (PostOffset == 64)) || (std::is_same_v && (PostOffset == 32)),\n \"Post-index offset needs to match number of elements times their size\");\n\n constexpr uint32_t Opcode = 0b0000 << 12;\n ASIMDLoadStoreMultipleStructure(ASIMDLoadstoreMultiplePost_Op, Opcode, rt, rn, Reg::r31);\n }\n template\n void st4(T rt, T rt2, T rt3, T rt4, Register rn, Register rm) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rt, rt2, rt3, rt4), \"rt, rt2, rt3, and rt4 must be sequential\");\n constexpr uint32_t Opcode = 0b0000 << 12;\n ASIMDLoadStoreMultipleStructure(ASIMDLoadstoreMultiplePost_Op, Opcode, rt, rn, rm);\n }\n template\n void st4(T rt, T rt2, T rt3, T rt4, Register rn, uint32_t PostOffset) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rt, rt2, rt3, rt4), \"rt, rt2, rt3, and rt4 must be sequential\");\n LOGMAN_THROW_A_FMT((std::is_same_v && (PostOffset == 64)) || (std::is_same_v && (PostOffset == 32)),\n \"Post-index offset needs to match number of elements times their size\");\n\n constexpr uint32_t Opcode = 0b0000 << 12;\n ASIMDLoadStoreMultipleStructure(ASIMDLoadstoreMultiplePost_Op, Opcode, rt, rn, Reg::r31);\n }\n\n // ASIMD loadstore single\n template\n void st1(VRegister rt, uint32_t Index, Register rn) {\n constexpr uint32_t Op = 0b0000'1101'000 << 21;\n constexpr uint32_t Opcode =\n size == SubRegSize::i8Bit ? 0b000 : // Scale = 0\n size == SubRegSize::i16Bit ? 0b010 : // Scale = 1\n size == SubRegSize::i32Bit ? 0b100 : // Scale = 2\n size == SubRegSize::i64Bit ? 0b100 : // Scale = 2 (Uses size to determine difference between 32-bit).\n 0;\n ASIMDSTLD(Op, Opcode, rt, Index, rn, Reg::r0);\n }\n template\n void st2(VRegister rt, VRegister rt2, uint32_t Index, Register rn) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rt, rt2), \"rt and rt2 must be sequential\");\n constexpr uint32_t Op = 0b0000'1101'000 << 21;\n constexpr uint32_t Opcode =\n size == SubRegSize::i8Bit ? 0b000 : // Scale = 0\n size == SubRegSize::i16Bit ? 0b010 : // Scale = 1\n size == SubRegSize::i32Bit ? 0b100 : // Scale = 2\n size == SubRegSize::i64Bit ? 0b100 : // Scale = 2 (Uses size to determine difference between 32-bit).\n 0;\n ASIMDSTLD(Op, Opcode, rt, Index, rn, Reg::r0);\n }\n template\n void st3(VRegister rt, VRegister rt2, VRegister rt3, uint32_t Index, Register rn) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rt, rt2, rt3), \"rt, rt2, and rt3 must be sequential\");\n constexpr uint32_t Op = 0b0000'1101'000 << 21;\n constexpr uint32_t Opcode =\n size == SubRegSize::i8Bit ? 0b000 : // Scale = 0\n size == SubRegSize::i16Bit ? 0b010 : // Scale = 1\n size == SubRegSize::i32Bit ? 0b100 : // Scale = 2\n size == SubRegSize::i64Bit ? 0b100 : // Scale = 2 (Uses size to determine difference between 32-bit).\n 0;\n ASIMDSTLD(Op, Opcode, rt, Index, rn, Reg::r0);\n }\n template\n void st4(VRegister rt, VRegister rt2, VRegister rt3, VRegister rt4, uint32_t Index, Register rn) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rt, rt2, rt3, rt4), \"rt, rt2, rt3, and rt4 must be sequential\");\n constexpr uint32_t Op = 0b0000'1101'000 << 21;\n constexpr uint32_t Opcode =\n size == SubRegSize::i8Bit ? 0b000 : // Scale = 0\n size == SubRegSize::i16Bit ? 0b010 : // Scale = 1\n size == SubRegSize::i32Bit ? 0b100 : // Scale = 2\n size == SubRegSize::i64Bit ? 0b100 : // Scale = 2 (Uses size to determine difference between 32-bit).\n 0;\n ASIMDSTLD(Op, Opcode, rt, Index, rn, Reg::r0);\n }\n template\n void ld1(VRegister rt, uint32_t Index, Register rn) {\n constexpr uint32_t Op = 0b0000'1101'000 << 21;\n constexpr uint32_t Opcode =\n size == SubRegSize::i8Bit ? 0b000 : // Scale = 0\n size == SubRegSize::i16Bit ? 0b010 : // Scale = 1\n size == SubRegSize::i32Bit ? 0b100 : // Scale = 2\n size == SubRegSize::i64Bit ? 0b100 : // Scale = 2 (Uses size to determine difference between 32-bit).\n 0;\n ASIMDSTLD(Op, Opcode, rt, Index, rn, Reg::r0);\n }\n template\n void ld1r(T rt, Register rn) {\n constexpr uint32_t Op = 0b0000'1101'000 << 21;\n constexpr uint32_t Opcode = 0b110;\n ASIMDSTLD(Op, Opcode, rt, rn, Reg::r0);\n }\n template\n void ld2(VRegister rt, VRegister rt2, uint32_t Index, Register rn) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rt, rt2), \"rt and rt2 must be sequential\");\n constexpr uint32_t Op = 0b0000'1101'000 << 21;\n constexpr uint32_t Opcode =\n size == SubRegSize::i8Bit ? 0b000 : // Scale = 0\n size == SubRegSize::i16Bit ? 0b010 : // Scale = 1\n size == SubRegSize::i32Bit ? 0b100 : // Scale = 2\n size == SubRegSize::i64Bit ? 0b100 : // Scale = 2 (Uses size to determine difference between 32-bit).\n 0;\n ASIMDSTLD(Op, Opcode, rt, Index, rn, Reg::r0);\n }\n template\n void ld2r(T rt, T rt2, Register rn) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rt, rt2), \"rt and rt2 must be sequential\");\n constexpr uint32_t Op = 0b0000'1101'000 << 21;\n constexpr uint32_t Opcode = 0b110;\n ASIMDSTLD(Op, Opcode, rt, rn, Reg::r0);\n }\n template\n void ld3(VRegister rt, VRegister rt2, VRegister rt3, uint32_t Index, Register rn) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rt, rt2, rt3), \"rt, rt2, and rt3 must be sequential\");\n constexpr uint32_t Op = 0b0000'1101'000 << 21;\n constexpr uint32_t Opcode =\n size == SubRegSize::i8Bit ? 0b000 : // Scale = 0\n size == SubRegSize::i16Bit ? 0b010 : // Scale = 1\n size == SubRegSize::i32Bit ? 0b100 : // Scale = 2\n size == SubRegSize::i64Bit ? 0b100 : // Scale = 2 (Uses size to determine difference between 32-bit).\n 0;\n ASIMDSTLD(Op, Opcode, rt, Index, rn, Reg::r0);\n }\n template\n void ld3r(T rt, T rt2, T rt3, Register rn) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rt, rt2, rt3), \"rt, rt2, and rt3 must be sequential\");\n constexpr uint32_t Op = 0b0000'1101'000 << 21;\n constexpr uint32_t Opcode = 0b110;\n ASIMDSTLD(Op, Opcode, rt, rn, Reg::r0);\n }\n template\n void ld4(VRegister rt, VRegister rt2, VRegister rt3, VRegister rt4, uint32_t Index, Register rn) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rt, rt2, rt3, rt4), \"rt, rt2, rt3, and rt4 must be sequential\");\n constexpr uint32_t Op = 0b0000'1101'000 << 21;\n constexpr uint32_t Opcode =\n size == SubRegSize::i8Bit ? 0b000 : // Scale = 0\n size == SubRegSize::i16Bit ? 0b010 : // Scale = 1\n size == SubRegSize::i32Bit ? 0b100 : // Scale = 2\n size == SubRegSize::i64Bit ? 0b100 : // Scale = 2 (Uses size to determine difference between 32-bit).\n 0;\n ASIMDSTLD(Op, Opcode, rt, Index, rn, Reg::r0);\n }\n template\n void ld4r(T rt, T rt2, T rt3, T rt4, Register rn) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rt, rt2, rt3, rt4), \"rt, rt2, rt3, and rt4 must be sequential\");\n constexpr uint32_t Op = 0b0000'1101'000 << 21;\n constexpr uint32_t Opcode = 0b110;\n ASIMDSTLD(Op, Opcode, rt, rn, Reg::r0);\n }\n\n // ASIMD loadstore single post-indexed\n template\n void st1(VRegister rt, uint32_t Index, Register rn, Register rm) {\n constexpr uint32_t Op = 0b0000'1101'100 << 21;\n constexpr uint32_t Opcode =\n size == SubRegSize::i8Bit ? 0b000 : // Scale = 0\n size == SubRegSize::i16Bit ? 0b010 : // Scale = 1\n size == SubRegSize::i32Bit ? 0b100 : // Scale = 2\n size == SubRegSize::i64Bit ? 0b100 : // Scale = 2 (Uses size to determine difference between 32-bit).\n 0;\n ASIMDSTLD(Op, Opcode, rt, Index, rn, rm);\n }\n template\n void st1(VRegister rt, uint32_t Index, Register rn, uint32_t PostOffset) {\n LOGMAN_THROW_A_FMT((size == SubRegSize::i8Bit && (PostOffset == 1)) || (size == SubRegSize::i16Bit && (PostOffset == 2)) ||\n (size == SubRegSize::i32Bit && (PostOffset == 4)) || (size == SubRegSize::i64Bit && (PostOffset == 8)),\n \"Post-index offset needs to match number of elements times their size\");\n\n constexpr uint32_t Op = 0b0000'1101'100 << 21;\n constexpr uint32_t Opcode =\n size == SubRegSize::i8Bit ? 0b000 : // Scale = 0\n size == SubRegSize::i16Bit ? 0b010 : // Scale = 1\n size == SubRegSize::i32Bit ? 0b100 : // Scale = 2\n size == SubRegSize::i64Bit ? 0b100 : // Scale = 2 (Uses size to determine difference between 32-bit).\n 0;\n ASIMDSTLD(Op, Opcode, rt, Index, rn, Reg::r31);\n }\n template\n void st2(VRegister rt, VRegister rt2, uint32_t Index, Register rn, Register rm) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rt, rt2), \"rt and rt2 must be sequential\");\n constexpr uint32_t Op = 0b0000'1101'100 << 21;\n constexpr uint32_t Opcode =\n size == SubRegSize::i8Bit ? 0b000 : // Scale = 0\n size == SubRegSize::i16Bit ? 0b010 : // Scale = 1\n size == SubRegSize::i32Bit ? 0b100 : // Scale = 2\n size == SubRegSize::i64Bit ? 0b100 : // Scale = 2 (Uses size to determine difference between 32-bit).\n 0;\n ASIMDSTLD(Op, Opcode, rt, Index, rn, rm);\n }\n template\n void st2(VRegister rt, uint32_t Index, Register rn, uint32_t PostOffset) {\n LOGMAN_THROW_A_FMT((size == SubRegSize::i8Bit && (PostOffset == 2)) || (size == SubRegSize::i16Bit && (PostOffset == 4)) ||\n (size == SubRegSize::i32Bit && (PostOffset == 8)) || (size == SubRegSize::i64Bit && (PostOffset == 16)),\n \"Post-index offset needs to match number of elements times their size\");\n\n constexpr uint32_t Op = 0b0000'1101'100 << 21;\n constexpr uint32_t Opcode =\n size == SubRegSize::i8Bit ? 0b000 : // Scale = 0\n size == SubRegSize::i16Bit ? 0b010 : // Scale = 1\n size == SubRegSize::i32Bit ? 0b100 : // Scale = 2\n size == SubRegSize::i64Bit ? 0b100 : // Scale = 2 (Uses size to determine difference between 32-bit).\n 0;\n ASIMDSTLD(Op, Opcode, rt, Index, rn, Reg::r31);\n }\n template\n void st3(VRegister rt, VRegister rt2, VRegister rt3, uint32_t Index, Register rn, Register rm) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rt, rt2, rt3), \"rt, rt2, and rt3 must be sequential\");\n constexpr uint32_t Op = 0b0000'1101'100 << 21;\n constexpr uint32_t Opcode =\n size == SubRegSize::i8Bit ? 0b000 : // Scale = 0\n size == SubRegSize::i16Bit ? 0b010 : // Scale = 1\n size == SubRegSize::i32Bit ? 0b100 : // Scale = 2\n size == SubRegSize::i64Bit ? 0b100 : // Scale = 2 (Uses size to determine difference between 32-bit).\n 0;\n ASIMDSTLD(Op, Opcode, rt, Index, rn, rm);\n }\n template\n void st3(VRegister rt, uint32_t Index, Register rn, uint32_t PostOffset) {\n LOGMAN_THROW_A_FMT((size == SubRegSize::i8Bit && (PostOffset == 3)) || (size == SubRegSize::i16Bit && (PostOffset == 6)) ||\n (size == SubRegSize::i32Bit && (PostOffset == 8)) || (size == SubRegSize::i64Bit && (PostOffset == 24)),\n \"Post-index offset needs to match number of elements times their size\");\n\n constexpr uint32_t Op = 0b0000'1101'100 << 21;\n constexpr uint32_t Opcode =\n size == SubRegSize::i8Bit ? 0b000 : // Scale = 0\n size == SubRegSize::i16Bit ? 0b010 : // Scale = 1\n size == SubRegSize::i32Bit ? 0b100 : // Scale = 2\n size == SubRegSize::i64Bit ? 0b100 : // Scale = 2 (Uses size to determine difference between 32-bit).\n 0;\n ASIMDSTLD(Op, Opcode, rt, Index, rn, Reg::r31);\n }\n template\n void st4(VRegister rt, VRegister rt2, VRegister rt3, VRegister rt4, uint32_t Index, Register rn, Register rm) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rt, rt2, rt3, rt4), \"rt, rt2, rt3, and rt4 must be sequential\");\n constexpr uint32_t Op = 0b0000'1101'100 << 21;\n constexpr uint32_t Opcode =\n size == SubRegSize::i8Bit ? 0b000 : // Scale = 0\n size == SubRegSize::i16Bit ? 0b010 : // Scale = 1\n size == SubRegSize::i32Bit ? 0b100 : // Scale = 2\n size == SubRegSize::i64Bit ? 0b100 : // Scale = 2 (Uses size to determine difference between 32-bit).\n 0;\n ASIMDSTLD(Op, Opcode, rt, Index, rn, rm);\n }\n template\n void st4(VRegister rt, uint32_t Index, Register rn, uint32_t PostOffset) {\n LOGMAN_THROW_A_FMT((size == SubRegSize::i8Bit && (PostOffset == 4)) || (size == SubRegSize::i16Bit && (PostOffset == 8)) ||\n (size == SubRegSize::i32Bit && (PostOffset == 16)) || (size == SubRegSize::i64Bit && (PostOffset == 32)),\n \"Post-index offset needs to match number of elements times their size\");\n\n constexpr uint32_t Op = 0b0000'1101'100 << 21;\n constexpr uint32_t Opcode =\n size == SubRegSize::i8Bit ? 0b000 : // Scale = 0\n size == SubRegSize::i16Bit ? 0b010 : // Scale = 1\n size == SubRegSize::i32Bit ? 0b100 : // Scale = 2\n size == SubRegSize::i64Bit ? 0b100 : // Scale = 2 (Uses size to determine difference between 32-bit).\n 0;\n ASIMDSTLD(Op, Opcode, rt, Index, rn, Reg::r31);\n }\n template\n void ld1(VRegister rt, uint32_t Index, Register rn, Register rm) {\n constexpr uint32_t Op = 0b0000'1101'100 << 21;\n constexpr uint32_t Opcode =\n size == SubRegSize::i8Bit ? 0b000 : // Scale = 0\n size == SubRegSize::i16Bit ? 0b010 : // Scale = 1\n size == SubRegSize::i32Bit ? 0b100 : // Scale = 2\n size == SubRegSize::i64Bit ? 0b100 : // Scale = 2 (Uses size to determine difference between 32-bit).\n 0;\n ASIMDSTLD(Op, Opcode, rt, Index, rn, rm);\n }\n template\n void ld1(VRegister rt, uint32_t Index, Register rn, uint32_t PostOffset) {\n LOGMAN_THROW_A_FMT((size == SubRegSize::i8Bit && (PostOffset == 1)) || (size == SubRegSize::i16Bit && (PostOffset == 2)) ||\n (size == SubRegSize::i32Bit && (PostOffset == 4)) || (size == SubRegSize::i64Bit && (PostOffset == 8)),\n \"Post-index offset needs to match number of elements times their size\");\n constexpr uint32_t Op = 0b0000'1101'100 << 21;\n constexpr uint32_t Opcode =\n size == SubRegSize::i8Bit ? 0b000 : // Scale = 0\n size == SubRegSize::i16Bit ? 0b010 : // Scale = 1\n size == SubRegSize::i32Bit ? 0b100 : // Scale = 2\n size == SubRegSize::i64Bit ? 0b100 : // Scale = 2 (Uses size to determine difference between 32-bit).\n 0;\n ASIMDSTLD(Op, Opcode, rt, Index, rn, Reg::r31);\n }\n template\n void ld1r(VRegister rt, Register rn, Register rm) {\n constexpr uint32_t Op = 0b0000'1101'100 << 21;\n constexpr uint32_t Opcode = 0b110;\n ASIMDSTLD(Op, Opcode, rt, 0, rn, rm);\n }\n template\n void ld1r(VRegister rt, Register rn, uint32_t PostOffset) {\n LOGMAN_THROW_A_FMT((size == SubRegSize::i8Bit && (PostOffset == 1)) || (size == SubRegSize::i16Bit && (PostOffset == 2)) ||\n (size == SubRegSize::i32Bit && (PostOffset == 4)) || (size == SubRegSize::i64Bit && (PostOffset == 8)),\n \"Post-index offset needs to match number of elements times their size\");\n constexpr uint32_t Op = 0b0000'1101'100 << 21;\n constexpr uint32_t Opcode = 0b110;\n ASIMDSTLD(Op, Opcode, rt, 0, rn, Reg::r31);\n }\n template\n void ld2(VRegister rt, VRegister rt2, uint32_t Index, Register rn, Register rm) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rt, rt2), \"rt and rt2 must be sequential\");\n constexpr uint32_t Op = 0b0000'1101'100 << 21;\n constexpr uint32_t Opcode =\n size == SubRegSize::i8Bit ? 0b000 : // Scale = 0\n size == SubRegSize::i16Bit ? 0b010 : // Scale = 1\n size == SubRegSize::i32Bit ? 0b100 : // Scale = 2\n size == SubRegSize::i64Bit ? 0b100 : // Scale = 2 (Uses size to determine difference between 32-bit).\n 0;\n ASIMDSTLD(Op, Opcode, rt, Index, rn, rm);\n }\n template\n void ld2(VRegister rt, uint32_t Index, Register rn, uint32_t PostOffset) {\n LOGMAN_THROW_A_FMT((size == SubRegSize::i8Bit && (PostOffset == 2)) || (size == SubRegSize::i16Bit && (PostOffset == 4)) ||\n (size == SubRegSize::i32Bit && (PostOffset == 8)) || (size == SubRegSize::i64Bit && (PostOffset == 16)),\n \"Post-index offset needs to match number of elements times their size\");\n constexpr uint32_t Op = 0b0000'1101'100 << 21;\n constexpr uint32_t Opcode =\n size == SubRegSize::i8Bit ? 0b000 : // Scale = 0\n size == SubRegSize::i16Bit ? 0b010 : // Scale = 1\n size == SubRegSize::i32Bit ? 0b100 : // Scale = 2\n size == SubRegSize::i64Bit ? 0b100 : // Scale = 2 (Uses size to determine difference between 32-bit).\n 0;\n ASIMDSTLD(Op, Opcode, rt, Index, rn, Reg::r31);\n }\n template\n void ld2r(VRegister rt, VRegister rt2, Register rn, Register rm) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rt, rt2), \"rt and rt2 must be sequential\");\n constexpr uint32_t Op = 0b0000'1101'100 << 21;\n constexpr uint32_t Opcode = 0b110;\n ASIMDSTLD(Op, Opcode, rt, 0, rn, rm);\n }\n template\n void ld2r(VRegister rt, VRegister rt2, Register rn, uint32_t PostOffset) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rt, rt2), \"rt and rt2 must be sequential\");\n LOGMAN_THROW_A_FMT((size == SubRegSize::i8Bit && (PostOffset == 2)) || (size == SubRegSize::i16Bit && (PostOffset == 4)) ||\n (size == SubRegSize::i32Bit && (PostOffset == 8)) || (size == SubRegSize::i64Bit && (PostOffset == 16)),\n \"Post-index offset needs to match number of elements times their size\");\n constexpr uint32_t Op = 0b0000'1101'100 << 21;\n constexpr uint32_t Opcode = 0b110;\n ASIMDSTLD(Op, Opcode, rt, 0, rn, Reg::r31);\n }\n template\n void ld3(VRegister rt, VRegister rt2, VRegister rt3, uint32_t Index, Register rn, Register rm) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rt, rt2, rt3), \"rt, rt2, and rt3 must be sequential\");\n constexpr uint32_t Op = 0b0000'1101'100 << 21;\n constexpr uint32_t Opcode =\n size == SubRegSize::i8Bit ? 0b000 : // Scale = 0\n size == SubRegSize::i16Bit ? 0b010 : // Scale = 1\n size == SubRegSize::i32Bit ? 0b100 : // Scale = 2\n size == SubRegSize::i64Bit ? 0b100 : // Scale = 2 (Uses size to determine difference between 32-bit).\n 0;\n ASIMDSTLD(Op, Opcode, rt, Index, rn, rm);\n }\n template\n void ld3(VRegister rt, uint32_t Index, Register rn, uint32_t PostOffset) {\n LOGMAN_THROW_A_FMT((size == SubRegSize::i8Bit && (PostOffset == 3)) || (size == SubRegSize::i16Bit && (PostOffset == 6)) ||\n (size == SubRegSize::i32Bit && (PostOffset == 12)) || (size == SubRegSize::i64Bit && (PostOffset == 16)),\n \"Post-index offset needs to match number of elements times their size\");\n constexpr uint32_t Op = 0b0000'1101'100 << 21;\n constexpr uint32_t Opcode =\n size == SubRegSize::i8Bit ? 0b000 : // Scale = 0\n size == SubRegSize::i16Bit ? 0b010 : // Scale = 1\n size == SubRegSize::i32Bit ? 0b100 : // Scale = 2\n size == SubRegSize::i64Bit ? 0b100 : // Scale = 2 (Uses size to determine difference between 32-bit).\n 0;\n ASIMDSTLD(Op, Opcode, rt, Index, rn, Reg::r31);\n }\n template\n void ld3r(VRegister rt, VRegister rt2, VRegister rt3, Register rn, Register rm) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rt, rt2, rt3), \"rt, rt2, and rt3 must be sequential\");\n constexpr uint32_t Op = 0b0000'1101'100 << 21;\n constexpr uint32_t Opcode = 0b110;\n ASIMDSTLD(Op, Opcode, rt, 0, rn, rm);\n }\n template\n void ld3r(VRegister rt, VRegister rt2, VRegister rt3, Register rn, uint32_t PostOffset) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rt, rt2, rt3), \"rt, rt2, and rt3 must be sequential\");\n LOGMAN_THROW_A_FMT((size == SubRegSize::i8Bit && (PostOffset == 3)) || (size == SubRegSize::i16Bit && (PostOffset == 6)) ||\n (size == SubRegSize::i32Bit && (PostOffset == 12)) || (size == SubRegSize::i64Bit && (PostOffset == 16)),\n \"Post-index offset needs to match number of elements times their size\");\n constexpr uint32_t Op = 0b0000'1101'100 << 21;\n constexpr uint32_t Opcode = 0b110;\n ASIMDSTLD(Op, Opcode, rt, 0, rn, Reg::r31);\n }\n template\n void ld4(VRegister rt, VRegister rt2, VRegister rt3, VRegister rt4, uint32_t Index, Register rn, Register rm) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rt, rt2, rt3, rt4), \"rt, rt2, rt3, and rt4 must be sequential\");\n constexpr uint32_t Op = 0b0000'1101'100 << 21;\n constexpr uint32_t Opcode =\n size == SubRegSize::i8Bit ? 0b000 : // Scale = 0\n size == SubRegSize::i16Bit ? 0b010 : // Scale = 1\n size == SubRegSize::i32Bit ? 0b100 : // Scale = 2\n size == SubRegSize::i64Bit ? 0b100 : // Scale = 2 (Uses size to determine difference between 32-bit).\n 0;\n ASIMDSTLD(Op, Opcode, rt, Index, rn, rm);\n }\n template\n void ld4(VRegister rt, uint32_t Index, Register rn, uint32_t PostOffset) {\n LOGMAN_THROW_A_FMT((size == SubRegSize::i8Bit && (PostOffset == 4)) || (size == SubRegSize::i16Bit && (PostOffset == 8)) ||\n (size == SubRegSize::i32Bit && (PostOffset == 16)) || (size == SubRegSize::i64Bit && (PostOffset == 32)),\n \"Post-index offset needs to match number of elements times their size\");\n constexpr uint32_t Op = 0b0000'1101'100 << 21;\n constexpr uint32_t Opcode =\n size == SubRegSize::i8Bit ? 0b000 : // Scale = 0\n size == SubRegSize::i16Bit ? 0b010 : // Scale = 1\n size == SubRegSize::i32Bit ? 0b100 : // Scale = 2\n size == SubRegSize::i64Bit ? 0b100 : // Scale = 2 (Uses size to determine difference between 32-bit).\n 0;\n ASIMDSTLD(Op, Opcode, rt, Index, rn, Reg::r31);\n }\n template\n void ld4r(VRegister rt, VRegister rt2, VRegister rt3, VRegister rt4, Register rn, Register rm) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rt, rt2, rt3, rt4), \"rt, rt2, rt3, and rt4 must be sequential\");\n constexpr uint32_t Op = 0b0000'1101'100 << 21;\n constexpr uint32_t Opcode = 0b110;\n ASIMDSTLD(Op, Opcode, rt, 0, rn, rm);\n }\n template\n void ld4r(VRegister rt, VRegister rt2, VRegister rt3, VRegister rt4, Register rn, uint32_t PostOffset) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rt, rt2, rt3, rt4), \"rt, rt2, rt3, and rt4 must be sequential\");\n LOGMAN_THROW_A_FMT((size == SubRegSize::i8Bit && (PostOffset == 4)) || (size == SubRegSize::i16Bit && (PostOffset == 8)) ||\n (size == SubRegSize::i32Bit && (PostOffset == 16)) || (size == SubRegSize::i64Bit && (PostOffset == 32)),\n \"Post-index offset needs to match number of elements times their size\");\n constexpr uint32_t Op = 0b0000'1101'100 << 21;\n constexpr uint32_t Opcode = 0b110;\n ASIMDSTLD(Op, Opcode, rt, 0, rn, Reg::r31);\n }\n\n // Advanced SIMD load/store single structure (post-indexed)\n template\n void st1(ARMEmitter::SubRegSize size, T rt, uint32_t Index, ARMEmitter::Register rn, uint32_t PostOffset) {\n LOGMAN_THROW_A_FMT(size == SubRegSize::i8Bit || size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit,\n \"Incorrect size\");\n LOGMAN_THROW_A_FMT((PostOffset * 8) == SubRegSizeInBits(size), \"Post-Index size must match element size\");\n constexpr uint32_t Op = 0b0000'1101'1 << 23;\n uint32_t Q;\n uint32_t R = 0;\n uint32_t opcode;\n uint32_t S;\n uint32_t Size;\n if (size == SubRegSize::i8Bit) {\n LOGMAN_THROW_A_FMT(Index < 16, \"Index too large\");\n Q = Index >> 3;\n S = (Index >> 2) & 1;\n opcode = 0b000;\n Size = Index & 0b11;\n } else if (size == SubRegSize::i16Bit) {\n LOGMAN_THROW_A_FMT(Index < 8, \"Index too large\");\n Q = Index >> 2;\n S = (Index >> 1) & 1;\n opcode = 0b010;\n Size = (Index & 0b1) << 1;\n } else if (size == SubRegSize::i32Bit) {\n LOGMAN_THROW_A_FMT(Index < 4, \"Index too large\");\n Q = Index >> 1;\n S = Index & 1;\n opcode = 0b100;\n Size = 0b00;\n } else if (size == SubRegSize::i64Bit) {\n LOGMAN_THROW_A_FMT(Index < 2, \"Index too large\");\n Q = Index;\n S = 0;\n opcode = 0b100;\n Size = 0b01;\n } else {\n LOGMAN_MSG_A_FMT(\"Unknown size\");\n FEX_UNREACHABLE;\n }\n\n ASIMDLoadStoreSinglePost(Op, Q, 0, R, opcode, S, Size, ARMEmitter::Reg::r31, rn, rt.Q());\n }\n template\n void ld1(ARMEmitter::SubRegSize size, T rt, uint32_t Index, ARMEmitter::Register rn, uint32_t PostOffset) {\n LOGMAN_THROW_A_FMT(size == SubRegSize::i8Bit || size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit,\n \"Incorrect size\");\n LOGMAN_THROW_A_FMT((PostOffset * 8) == SubRegSizeInBits(size), \"Post-Index size must match element size\");\n constexpr uint32_t Op = 0b0000'1101'1 << 23;\n uint32_t Q;\n uint32_t R = 0;\n uint32_t opcode;\n uint32_t S;\n uint32_t Size;\n if (size == SubRegSize::i8Bit) {\n LOGMAN_THROW_A_FMT(Index < 16, \"Index too large\");\n Q = Index >> 3;\n S = (Index >> 2) & 1;\n opcode = 0b001;\n Size = Index & 0b11;\n } else if (size == SubRegSize::i16Bit) {\n LOGMAN_THROW_A_FMT(Index < 8, \"Index too large\");\n Q = Index >> 2;\n S = (Index >> 1) & 1;\n opcode = 0b011;\n Size = (Index & 0b1) << 1;\n } else if (size == SubRegSize::i32Bit) {\n LOGMAN_THROW_A_FMT(Index < 4, \"Index too large\");\n Q = Index >> 1;\n S = Index & 1;\n opcode = 0b100;\n Size = 0b00;\n } else if (size == SubRegSize::i64Bit) {\n LOGMAN_THROW_A_FMT(Index < 2, \"Index too large\");\n Q = Index;\n S = 0;\n opcode = 0b101;\n Size = 0b01;\n } else {\n LOGMAN_MSG_A_FMT(\"Unknown size\");\n FEX_UNREACHABLE;\n }\n\n ASIMDLoadStoreSinglePost(Op, Q, 1, R, opcode, S, Size, ARMEmitter::Reg::r31, rn, rt.Q());\n }\n template\n void ld1r(ARMEmitter::SubRegSize size, T rt, ARMEmitter::Register rn, uint32_t PostOffset) {\n LOGMAN_THROW_A_FMT(PostOffset == 1 || PostOffset == 2 || PostOffset == 4 || PostOffset == 8, \"Index too large\");\n constexpr uint32_t Op = 0b0000'1101'1 << 23;\n constexpr uint32_t Q = std::is_same_v ? 1 : 0;\n uint32_t R = 0;\n uint32_t opcode = 0b110;\n uint32_t S = 0;\n uint32_t Size = FEXCore::ToUnderlying(size);\n ASIMDLoadStoreSinglePost(Op, Q, 1, R, opcode, S, Size, ARMEmitter::Reg::r31, rn, rt);\n }\n\n template\n void ld2r(SubRegSize size, T rt, T rt2, Register rn, uint32_t PostOffset) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rt, rt2), \"rt and rt2 must be sequential\");\n LOGMAN_THROW_A_FMT(PostOffset == 2 || PostOffset == 4 || PostOffset == 8 || PostOffset == 16, \"Index too large\");\n constexpr uint32_t Op = 0b0000'1101'1 << 23;\n constexpr uint32_t Q = std::is_same_v ? 1 : 0;\n uint32_t R = 1;\n uint32_t opcode = 0b110;\n uint32_t S = 0;\n uint32_t Size = FEXCore::ToUnderlying(size);\n ASIMDLoadStoreSinglePost(Op, Q, 1, R, opcode, S, Size, Reg::r31, rn, rt);\n }\n\n template\n void ld3r(SubRegSize size, T rt, T rt2, T rt3, Register rn, uint32_t PostOffset) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rt, rt2, rt3), \"rt, rt2, and rt3 must be sequential\");\n LOGMAN_THROW_A_FMT(PostOffset == 3 || PostOffset == 6 || PostOffset == 12 || PostOffset == 24, \"Index too large\");\n constexpr uint32_t Op = 0b0000'1101'1 << 23;\n constexpr uint32_t Q = std::is_same_v ? 1 : 0;\n uint32_t R = 0;\n uint32_t opcode = 0b111;\n uint32_t S = 0;\n uint32_t Size = FEXCore::ToUnderlying(size);\n ASIMDLoadStoreSinglePost(Op, Q, 1, R, opcode, S, Size, Reg::r31, rn, rt);\n }\n template\n void ld4r(SubRegSize size, T rt, T rt2, T rt3, T rt4, Register rn, uint32_t PostOffset) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rt, rt2, rt3, rt4), \"rt, rt2, rt3, and rt4 must be sequential\");\n LOGMAN_THROW_A_FMT(PostOffset == 4 || PostOffset == 8 || PostOffset == 16 || PostOffset == 32, \"Index too large\");\n constexpr uint32_t Op = 0b0000'1101'1 << 23;\n constexpr uint32_t Q = std::is_same_v ? 1 : 0;\n uint32_t R = 1;\n uint32_t opcode = 0b111;\n uint32_t S = 0;\n uint32_t Size = FEXCore::ToUnderlying(size);\n ASIMDLoadStoreSinglePost(Op, Q, 1, R, opcode, S, Size, Reg::r31, rn, rt);\n }\n\n template\n void st2(SubRegSize size, T rt, T rt2, uint32_t Index, Register rn, uint32_t PostOffset) {\n LOGMAN_THROW_A_FMT(size == SubRegSize::i8Bit || size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit,\n \"Incorrect size\");\n LOGMAN_THROW_A_FMT((PostOffset * 8) == (SubRegSizeInBits(size) * 2), \"Post-Index size must match element size\");\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rt, rt2), \"rt and rt2 must be sequential\");\n\n constexpr uint32_t Op = 0b0000'1101'1 << 23;\n uint32_t Q;\n uint32_t R = 1;\n uint32_t opcode;\n uint32_t S;\n uint32_t Size;\n if (size == SubRegSize::i8Bit) {\n LOGMAN_THROW_A_FMT(Index < 16, \"Index too large\");\n Q = Index >> 3;\n S = (Index >> 2) & 1;\n opcode = 0b000;\n Size = Index & 0b11;\n } else if (size == SubRegSize::i16Bit) {\n LOGMAN_THROW_A_FMT(Index < 8, \"Index too large\");\n Q = Index >> 2;\n S = (Index >> 1) & 1;\n opcode = 0b010;\n Size = (Index & 0b1) << 1;\n } else if (size == SubRegSize::i32Bit) {\n LOGMAN_THROW_A_FMT(Index < 4, \"Index too large\");\n Q = Index >> 1;\n S = Index & 1;\n opcode = 0b100;\n Size = 0b00;\n } else if (size == SubRegSize::i64Bit) {\n LOGMAN_THROW_A_FMT(Index < 2, \"Index too large\");\n Q = Index;\n S = 0;\n opcode = 0b100;\n Size = 0b01;\n } else {\n LOGMAN_MSG_A_FMT(\"Unknown size\");\n FEX_UNREACHABLE;\n }\n\n ASIMDLoadStoreSinglePost(Op, Q, 0, R, opcode, S, Size, Reg::r31, rn, rt.Q());\n }\n template\n void ld2(SubRegSize size, T rt, T rt2, uint32_t Index, Register rn, uint32_t PostOffset) {\n LOGMAN_THROW_A_FMT(size == SubRegSize::i8Bit || size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit,\n \"Incorrect size\");\n LOGMAN_THROW_A_FMT((PostOffset * 8) == (SubRegSizeInBits(size) * 2), \"Post-Index size must match element size\");\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rt, rt2), \"rt and rt2 must be sequential\");\n\n constexpr uint32_t Op = 0b0000'1101'1 << 23;\n uint32_t Q;\n uint32_t R = 1;\n uint32_t opcode;\n uint32_t S;\n uint32_t Size;\n if (size == SubRegSize::i8Bit) {\n LOGMAN_THROW_A_FMT(Index < 16, \"Index too large\");\n Q = Index >> 3;\n S = (Index >> 2) & 1;\n opcode = 0b000;\n Size = Index & 0b11;\n } else if (size == SubRegSize::i16Bit) {\n LOGMAN_THROW_A_FMT(Index < 8, \"Index too large\");\n Q = Index >> 2;\n S = (Index >> 1) & 1;\n opcode = 0b010;\n Size = (Index & 0b1) << 1;\n } else if (size == SubRegSize::i32Bit) {\n LOGMAN_THROW_A_FMT(Index < 4, \"Index too large\");\n Q = Index >> 1;\n S = Index & 1;\n opcode = 0b100;\n Size = 0b00;\n } else if (size == SubRegSize::i64Bit) {\n LOGMAN_THROW_A_FMT(Index < 2, \"Index too large\");\n Q = Index;\n S = 0;\n opcode = 0b100;\n Size = 0b01;\n } else {\n LOGMAN_MSG_A_FMT(\"Unknown size\");\n FEX_UNREACHABLE;\n }\n\n ASIMDLoadStoreSinglePost(Op, Q, 1, R, opcode, S, Size, Reg::r31, rn, rt.Q());\n }\n template\n void st3(SubRegSize size, T rt, T rt2, T rt3, uint32_t Index, Register rn, uint32_t PostOffset) {\n LOGMAN_THROW_A_FMT(size == SubRegSize::i8Bit || size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit,\n \"Incorrect size\");\n LOGMAN_THROW_A_FMT((PostOffset * 8) == (SubRegSizeInBits(size) * 3), \"Post-Index size must match element size\");\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rt, rt2, rt3), \"rt, rt2, and rt3 must be sequential\");\n\n constexpr uint32_t Op = 0b0000'1101'1 << 23;\n uint32_t Q;\n uint32_t R = 0;\n uint32_t opcode;\n uint32_t S;\n uint32_t Size;\n if (size == SubRegSize::i8Bit) {\n LOGMAN_THROW_A_FMT(Index < 16, \"Index too large\");\n Q = Index >> 3;\n S = (Index >> 2) & 1;\n opcode = 0b001;\n Size = Index & 0b11;\n } else if (size == SubRegSize::i16Bit) {\n LOGMAN_THROW_A_FMT(Index < 8, \"Index too large\");\n Q = Index >> 2;\n S = (Index >> 1) & 1;\n opcode = 0b011;\n Size = (Index & 0b1) << 1;\n } else if (size == SubRegSize::i32Bit) {\n LOGMAN_THROW_A_FMT(Index < 4, \"Index too large\");\n Q = Index >> 1;\n S = Index & 1;\n opcode = 0b101;\n Size = 0b00;\n } else if (size == SubRegSize::i64Bit) {\n LOGMAN_THROW_A_FMT(Index < 2, \"Index too large\");\n Q = Index;\n S = 0;\n opcode = 0b101;\n Size = 0b01;\n } else {\n LOGMAN_MSG_A_FMT(\"Unknown size\");\n FEX_UNREACHABLE;\n }\n\n ASIMDLoadStoreSinglePost(Op, Q, 0, R, opcode, S, Size, Reg::r31, rn, rt.Q());\n }\n template\n void ld3(SubRegSize size, T rt, T rt2, T rt3, uint32_t Index, Register rn, uint32_t PostOffset) {\n LOGMAN_THROW_A_FMT(size == SubRegSize::i8Bit || size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit,\n \"Incorrect size\");\n LOGMAN_THROW_A_FMT((PostOffset * 8) == (SubRegSizeInBits(size) * 3), \"Post-Index size must match element size\");\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rt, rt2, rt3), \"rt, rt2, and rt3 must be sequential\");\n\n constexpr uint32_t Op = 0b0000'1101'1 << 23;\n uint32_t Q;\n uint32_t R = 0;\n uint32_t opcode;\n uint32_t S;\n uint32_t Size;\n if (size == SubRegSize::i8Bit) {\n LOGMAN_THROW_A_FMT(Index < 16, \"Index too large\");\n Q = Index >> 3;\n S = (Index >> 2) & 1;\n opcode = 0b001;\n Size = Index & 0b11;\n } else if (size == SubRegSize::i16Bit) {\n LOGMAN_THROW_A_FMT(Index < 8, \"Index too large\");\n Q = Index >> 2;\n S = (Index >> 1) & 1;\n opcode = 0b011;\n Size = (Index & 0b1) << 1;\n } else if (size == SubRegSize::i32Bit) {\n LOGMAN_THROW_A_FMT(Index < 4, \"Index too large\");\n Q = Index >> 1;\n S = Index & 1;\n opcode = 0b101;\n Size = 0b00;\n } else if (size == SubRegSize::i64Bit) {\n LOGMAN_THROW_A_FMT(Index < 2, \"Index too large\");\n Q = Index;\n S = 0;\n opcode = 0b101;\n Size = 0b01;\n } else {\n LOGMAN_MSG_A_FMT(\"Unknown size\");\n FEX_UNREACHABLE;\n }\n\n ASIMDLoadStoreSinglePost(Op, Q, 1, R, opcode, S, Size, Reg::r31, rn, rt.Q());\n }\n template\n void st4(SubRegSize size, T rt, T rt2, T rt3, T rt4, uint32_t Index, Register rn, uint32_t PostOffset) {\n LOGMAN_THROW_A_FMT(size == SubRegSize::i8Bit || size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit,\n \"Incorrect size\");\n LOGMAN_THROW_A_FMT((PostOffset * 8) == (SubRegSizeInBits(size) * 4), \"Post-Index size must match element size\");\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rt, rt2, rt3, rt4), \"rt, rt2, rt3, and rt4 must be sequential\");\n\n constexpr uint32_t Op = 0b0000'1101'1 << 23;\n uint32_t Q;\n uint32_t R = 1;\n uint32_t opcode;\n uint32_t S;\n uint32_t Size;\n if (size == SubRegSize::i8Bit) {\n LOGMAN_THROW_A_FMT(Index < 16, \"Index too large\");\n Q = Index >> 3;\n S = (Index >> 2) & 1;\n opcode = 0b001;\n Size = Index & 0b11;\n } else if (size == SubRegSize::i16Bit) {\n LOGMAN_THROW_A_FMT(Index < 8, \"Index too large\");\n Q = Index >> 2;\n S = (Index >> 1) & 1;\n opcode = 0b011;\n Size = (Index & 0b1) << 1;\n } else if (size == SubRegSize::i32Bit) {\n LOGMAN_THROW_A_FMT(Index < 4, \"Index too large\");\n Q = Index >> 1;\n S = Index & 1;\n opcode = 0b101;\n Size = 0b00;\n } else if (size == SubRegSize::i64Bit) {\n LOGMAN_THROW_A_FMT(Index < 2, \"Index too large\");\n Q = Index;\n S = 0;\n opcode = 0b101;\n Size = 0b01;\n } else {\n LOGMAN_MSG_A_FMT(\"Unknown size\");\n FEX_UNREACHABLE;\n }\n\n ASIMDLoadStoreSinglePost(Op, Q, 0, R, opcode, S, Size, Reg::r31, rn, rt.Q());\n }\n template\n void ld4(SubRegSize size, T rt, T rt2, T rt3, T rt4, uint32_t Index, Register rn, uint32_t PostOffset) {\n LOGMAN_THROW_A_FMT(size == SubRegSize::i8Bit || size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit,\n \"Incorrect size\");\n LOGMAN_THROW_A_FMT((PostOffset * 8) == (SubRegSizeInBits(size) * 4), \"Post-Index size must match element size\");\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rt, rt2, rt3, rt4), \"rt, rt2, rt3, and rt4 must be sequential\");\n\n constexpr uint32_t Op = 0b0000'1101'1 << 23;\n uint32_t Q;\n uint32_t R = 1;\n uint32_t opcode;\n uint32_t S;\n uint32_t Size;\n if (size == SubRegSize::i8Bit) {\n LOGMAN_THROW_A_FMT(Index < 16, \"Index too large\");\n Q = Index >> 3;\n S = (Index >> 2) & 1;\n opcode = 0b001;\n Size = Index & 0b11;\n } else if (size == SubRegSize::i16Bit) {\n LOGMAN_THROW_A_FMT(Index < 8, \"Index too large\");\n Q = Index >> 2;\n S = (Index >> 1) & 1;\n opcode = 0b011;\n Size = (Index & 0b1) << 1;\n } else if (size == SubRegSize::i32Bit) {\n LOGMAN_THROW_A_FMT(Index < 4, \"Index too large\");\n Q = Index >> 1;\n S = Index & 1;\n opcode = 0b101;\n Size = 0b00;\n } else if (size == SubRegSize::i64Bit) {\n LOGMAN_THROW_A_FMT(Index < 2, \"Index too large\");\n Q = Index;\n S = 0;\n opcode = 0b101;\n Size = 0b01;\n } else {\n LOGMAN_MSG_A_FMT(\"Unknown size\");\n FEX_UNREACHABLE;\n }\n\n ASIMDLoadStoreSinglePost(Op, Q, 1, R, opcode, S, Size, Reg::r31, rn, rt.Q());\n }\n\n template\n void st1(ARMEmitter::SubRegSize size, T rt, uint32_t Index, ARMEmitter::Register rn, ARMEmitter::Register rm) {\n LOGMAN_THROW_A_FMT(size == SubRegSize::i8Bit || size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit,\n \"Incorrect size\");\n constexpr uint32_t Op = 0b0000'1101'1 << 23;\n uint32_t Q;\n uint32_t R = 0;\n uint32_t opcode;\n uint32_t S;\n uint32_t Size;\n if (size == SubRegSize::i8Bit) {\n LOGMAN_THROW_A_FMT(Index < 16, \"Index too large\");\n Q = Index >> 3;\n S = (Index >> 2) & 1;\n opcode = 0b000;\n Size = Index & 0b11;\n } else if (size == SubRegSize::i16Bit) {\n LOGMAN_THROW_A_FMT(Index < 8, \"Index too large\");\n Q = Index >> 2;\n S = (Index >> 1) & 1;\n opcode = 0b010;\n Size = (Index & 0b1) << 1;\n } else if (size == SubRegSize::i32Bit) {\n LOGMAN_THROW_A_FMT(Index < 4, \"Index too large\");\n Q = Index >> 1;\n S = Index & 1;\n opcode = 0b100;\n Size = 0b00;\n } else if (size == SubRegSize::i64Bit) {\n LOGMAN_THROW_A_FMT(Index < 2, \"Index too large\");\n Q = Index;\n S = 0;\n opcode = 0b100;\n Size = 0b01;\n } else {\n LOGMAN_MSG_A_FMT(\"Unknown size\");\n FEX_UNREACHABLE;\n }\n\n ASIMDLoadStoreSinglePost(Op, Q, 0, R, opcode, S, Size, rm, rn, rt.Q());\n }\n template\n void ld1(ARMEmitter::SubRegSize size, T rt, uint32_t Index, ARMEmitter::Register rn, ARMEmitter::Register rm) {\n LOGMAN_THROW_A_FMT(size == SubRegSize::i8Bit || size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit,\n \"Incorrect size\");\n constexpr uint32_t Op = 0b0000'1101'1 << 23;\n uint32_t Q;\n uint32_t R = 0;\n uint32_t opcode;\n uint32_t S;\n uint32_t Size;\n if (size == SubRegSize::i8Bit) {\n LOGMAN_THROW_A_FMT(Index < 16, \"Index too large\");\n Q = Index >> 3;\n S = (Index >> 2) & 1;\n opcode = 0b001;\n Size = Index & 0b11;\n } else if (size == SubRegSize::i16Bit) {\n LOGMAN_THROW_A_FMT(Index < 8, \"Index too large\");\n Q = Index >> 2;\n S = (Index >> 1) & 1;\n opcode = 0b011;\n Size = (Index & 0b1) << 1;\n } else if (size == SubRegSize::i32Bit) {\n LOGMAN_THROW_A_FMT(Index < 4, \"Index too large\");\n Q = Index >> 1;\n S = Index & 1;\n opcode = 0b100;\n Size = 0b00;\n } else if (size == SubRegSize::i64Bit) {\n LOGMAN_THROW_A_FMT(Index < 2, \"Index too large\");\n Q = Index;\n S = 0;\n opcode = 0b101;\n Size = 0b01;\n } else {\n LOGMAN_MSG_A_FMT(\"Unknown size\");\n FEX_UNREACHABLE;\n }\n\n ASIMDLoadStoreSinglePost(Op, Q, 1, R, opcode, S, Size, rm, rn, rt.Q());\n }\n template\n void ld1r(SubRegSize size, T rt, Register rn, Register rm) {\n constexpr uint32_t Op = 0b0000'1101'1 << 23;\n constexpr uint32_t Q = std::is_same_v ? 1 : 0;\n uint32_t R = 0;\n uint32_t opcode = 0b110;\n uint32_t S = 0;\n uint32_t Size = FEXCore::ToUnderlying(size);\n ASIMDLoadStoreSinglePost(Op, Q, 1, R, opcode, S, Size, rm, rn, rt);\n }\n\n template\n void ld2r(SubRegSize size, T rt, T rt2, Register rn, Register rm) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rt, rt2), \"rt and rt2 must be sequential\");\n constexpr uint32_t Op = 0b0000'1101'1 << 23;\n constexpr uint32_t Q = std::is_same_v ? 1 : 0;\n uint32_t R = 1;\n uint32_t opcode = 0b110;\n uint32_t S = 0;\n uint32_t Size = FEXCore::ToUnderlying(size);\n ASIMDLoadStoreSinglePost(Op, Q, 1, R, opcode, S, Size, rm, rn, rt);\n }\n\n template\n void ld3r(SubRegSize size, T rt, T rt2, T rt3, Register rn, Register rm) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rt, rt2, rt3), \"rt, rt2, and rt3 must be sequential\");\n constexpr uint32_t Op = 0b0000'1101'1 << 23;\n constexpr uint32_t Q = std::is_same_v ? 1 : 0;\n uint32_t R = 0;\n uint32_t opcode = 0b111;\n uint32_t S = 0;\n uint32_t Size = FEXCore::ToUnderlying(size);\n ASIMDLoadStoreSinglePost(Op, Q, 1, R, opcode, S, Size, rm, rn, rt);\n }\n template\n void ld4r(SubRegSize size, T rt, T rt2, T rt3, T rt4, Register rn, Register rm) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rt, rt2, rt3, rt4), \"rt, rt2, rt3, and rt4 must be sequential\");\n constexpr uint32_t Op = 0b0000'1101'1 << 23;\n constexpr uint32_t Q = std::is_same_v ? 1 : 0;\n uint32_t R = 1;\n uint32_t opcode = 0b111;\n uint32_t S = 0;\n uint32_t Size = FEXCore::ToUnderlying(size);\n ASIMDLoadStoreSinglePost(Op, Q, 1, R, opcode, S, Size, rm, rn, rt);\n }\n\n template\n void st2(SubRegSize size, T rt, T rt2, uint32_t Index, Register rn, Register rm) {\n LOGMAN_THROW_A_FMT(size == SubRegSize::i8Bit || size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit,\n \"Incorrect size\");\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rt, rt2), \"rt and rt2 must be sequential\");\n\n constexpr uint32_t Op = 0b0000'1101'1 << 23;\n uint32_t Q;\n uint32_t R = 1;\n uint32_t opcode;\n uint32_t S;\n uint32_t Size;\n if (size == SubRegSize::i8Bit) {\n LOGMAN_THROW_A_FMT(Index < 16, \"Index too large\");\n Q = Index >> 3;\n S = (Index >> 2) & 1;\n opcode = 0b000;\n Size = Index & 0b11;\n } else if (size == SubRegSize::i16Bit) {\n LOGMAN_THROW_A_FMT(Index < 8, \"Index too large\");\n Q = Index >> 2;\n S = (Index >> 1) & 1;\n opcode = 0b010;\n Size = (Index & 0b1) << 1;\n } else if (size == SubRegSize::i32Bit) {\n LOGMAN_THROW_A_FMT(Index < 4, \"Index too large\");\n Q = Index >> 1;\n S = Index & 1;\n opcode = 0b100;\n Size = 0b00;\n } else if (size == SubRegSize::i64Bit) {\n LOGMAN_THROW_A_FMT(Index < 2, \"Index too large\");\n Q = Index;\n S = 0;\n opcode = 0b100;\n Size = 0b01;\n } else {\n LOGMAN_MSG_A_FMT(\"Unknown size\");\n FEX_UNREACHABLE;\n }\n\n ASIMDLoadStoreSinglePost(Op, Q, 0, R, opcode, S, Size, rm, rn, rt.Q());\n }\n template\n void ld2(SubRegSize size, T rt, T rt2, uint32_t Index, Register rn, Register rm) {\n LOGMAN_THROW_A_FMT(size == SubRegSize::i8Bit || size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit,\n \"Incorrect size\");\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rt, rt2), \"rt and rt2 must be sequential\");\n\n constexpr uint32_t Op = 0b0000'1101'1 << 23;\n uint32_t Q;\n uint32_t R = 1;\n uint32_t opcode;\n uint32_t S;\n uint32_t Size;\n if (size == SubRegSize::i8Bit) {\n LOGMAN_THROW_A_FMT(Index < 16, \"Index too large\");\n Q = Index >> 3;\n S = (Index >> 2) & 1;\n opcode = 0b000;\n Size = Index & 0b11;\n } else if (size == SubRegSize::i16Bit) {\n LOGMAN_THROW_A_FMT(Index < 8, \"Index too large\");\n Q = Index >> 2;\n S = (Index >> 1) & 1;\n opcode = 0b010;\n Size = (Index & 0b1) << 1;\n } else if (size == SubRegSize::i32Bit) {\n LOGMAN_THROW_A_FMT(Index < 4, \"Index too large\");\n Q = Index >> 1;\n S = Index & 1;\n opcode = 0b100;\n Size = 0b00;\n } else if (size == SubRegSize::i64Bit) {\n LOGMAN_THROW_A_FMT(Index < 2, \"Index too large\");\n Q = Index;\n S = 0;\n opcode = 0b100;\n Size = 0b01;\n } else {\n LOGMAN_MSG_A_FMT(\"Unknown size\");\n FEX_UNREACHABLE;\n }\n\n ASIMDLoadStoreSinglePost(Op, Q, 1, R, opcode, S, Size, rm, rn, rt.Q());\n }\n template\n void st3(SubRegSize size, T rt, T rt2, T rt3, uint32_t Index, Register rn, Register rm) {\n LOGMAN_THROW_A_FMT(size == SubRegSize::i8Bit || size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit,\n \"Incorrect size\");\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rt, rt2, rt3), \"rt, rt2, and rt3 must be sequential\");\n\n constexpr uint32_t Op = 0b0000'1101'1 << 23;\n uint32_t Q;\n uint32_t R = 0;\n uint32_t opcode;\n uint32_t S;\n uint32_t Size;\n if (size == SubRegSize::i8Bit) {\n LOGMAN_THROW_A_FMT(Index < 16, \"Index too large\");\n Q = Index >> 3;\n S = (Index >> 2) & 1;\n opcode = 0b001;\n Size = Index & 0b11;\n } else if (size == SubRegSize::i16Bit) {\n LOGMAN_THROW_A_FMT(Index < 8, \"Index too large\");\n Q = Index >> 2;\n S = (Index >> 1) & 1;\n opcode = 0b011;\n Size = (Index & 0b1) << 1;\n } else if (size == SubRegSize::i32Bit) {\n LOGMAN_THROW_A_FMT(Index < 4, \"Index too large\");\n Q = Index >> 1;\n S = Index & 1;\n opcode = 0b101;\n Size = 0b00;\n } else if (size == SubRegSize::i64Bit) {\n LOGMAN_THROW_A_FMT(Index < 2, \"Index too large\");\n Q = Index;\n S = 0;\n opcode = 0b101;\n Size = 0b01;\n } else {\n LOGMAN_MSG_A_FMT(\"Unknown size\");\n FEX_UNREACHABLE;\n }\n\n ASIMDLoadStoreSinglePost(Op, Q, 0, R, opcode, S, Size, rm, rn, rt.Q());\n }\n template\n void ld3(SubRegSize size, T rt, T rt2, T rt3, uint32_t Index, Register rn, Register rm) {\n LOGMAN_THROW_A_FMT(size == SubRegSize::i8Bit || size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit,\n \"Incorrect size\");\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rt, rt2, rt3), \"rt, rt2, and rt3 must be sequential\");\n\n constexpr uint32_t Op = 0b0000'1101'1 << 23;\n uint32_t Q;\n uint32_t R = 0;\n uint32_t opcode;\n uint32_t S;\n uint32_t Size;\n if (size == SubRegSize::i8Bit) {\n LOGMAN_THROW_A_FMT(Index < 16, \"Index too large\");\n Q = Index >> 3;\n S = (Index >> 2) & 1;\n opcode = 0b001;\n Size = Index & 0b11;\n } else if (size == SubRegSize::i16Bit) {\n LOGMAN_THROW_A_FMT(Index < 8, \"Index too large\");\n Q = Index >> 2;\n S = (Index >> 1) & 1;\n opcode = 0b011;\n Size = (Index & 0b1) << 1;\n } else if (size == SubRegSize::i32Bit) {\n LOGMAN_THROW_A_FMT(Index < 4, \"Index too large\");\n Q = Index >> 1;\n S = Index & 1;\n opcode = 0b101;\n Size = 0b00;\n } else if (size == SubRegSize::i64Bit) {\n LOGMAN_THROW_A_FMT(Index < 2, \"Index too large\");\n Q = Index;\n S = 0;\n opcode = 0b101;\n Size = 0b01;\n } else {\n LOGMAN_MSG_A_FMT(\"Unknown size\");\n FEX_UNREACHABLE;\n }\n\n ASIMDLoadStoreSinglePost(Op, Q, 1, R, opcode, S, Size, rm, rn, rt.Q());\n }\n template\n void st4(SubRegSize size, T rt, T rt2, T rt3, T rt4, uint32_t Index, Register rn, Register rm) {\n LOGMAN_THROW_A_FMT(size == SubRegSize::i8Bit || size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit,\n \"Incorrect size\");\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rt, rt2, rt3, rt4), \"rt, rt2, rt3, and rt4 must be sequential\");\n\n constexpr uint32_t Op = 0b0000'1101'1 << 23;\n uint32_t Q;\n uint32_t R = 1;\n uint32_t opcode;\n uint32_t S;\n uint32_t Size;\n if (size == SubRegSize::i8Bit) {\n LOGMAN_THROW_A_FMT(Index < 16, \"Index too large\");\n Q = Index >> 3;\n S = (Index >> 2) & 1;\n opcode = 0b001;\n Size = Index & 0b11;\n } else if (size == SubRegSize::i16Bit) {\n LOGMAN_THROW_A_FMT(Index < 8, \"Index too large\");\n Q = Index >> 2;\n S = (Index >> 1) & 1;\n opcode = 0b011;\n Size = (Index & 0b1) << 1;\n } else if (size == SubRegSize::i32Bit) {\n LOGMAN_THROW_A_FMT(Index < 4, \"Index too large\");\n Q = Index >> 1;\n S = Index & 1;\n opcode = 0b101;\n Size = 0b00;\n } else if (size == SubRegSize::i64Bit) {\n LOGMAN_THROW_A_FMT(Index < 2, \"Index too large\");\n Q = Index;\n S = 0;\n opcode = 0b101;\n Size = 0b01;\n } else {\n LOGMAN_MSG_A_FMT(\"Unknown size\");\n FEX_UNREACHABLE;\n }\n\n ASIMDLoadStoreSinglePost(Op, Q, 0, R, opcode, S, Size, rm, rn, rt.Q());\n }\n template\n void ld4(SubRegSize size, T rt, T rt2, T rt3, T rt4, uint32_t Index, Register rn, Register rm) {\n LOGMAN_THROW_A_FMT(size == SubRegSize::i8Bit || size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit,\n \"Incorrect size\");\n LOGMAN_THROW_A_FMT(AreVectorsSequential(rt, rt2, rt3, rt4), \"rt, rt2, rt3, and rt4 must be sequential\");\n\n constexpr uint32_t Op = 0b0000'1101'1 << 23;\n uint32_t Q;\n uint32_t R = 1;\n uint32_t opcode;\n uint32_t S;\n uint32_t Size;\n if (size == SubRegSize::i8Bit) {\n LOGMAN_THROW_A_FMT(Index < 16, \"Index too large\");\n Q = Index >> 3;\n S = (Index >> 2) & 1;\n opcode = 0b001;\n Size = Index & 0b11;\n } else if (size == SubRegSize::i16Bit) {\n LOGMAN_THROW_A_FMT(Index < 8, \"Index too large\");\n Q = Index >> 2;\n S = (Index >> 1) & 1;\n opcode = 0b011;\n Size = (Index & 0b1) << 1;\n } else if (size == SubRegSize::i32Bit) {\n LOGMAN_THROW_A_FMT(Index < 4, \"Index too large\");\n Q = Index >> 1;\n S = Index & 1;\n opcode = 0b101;\n Size = 0b00;\n } else if (size == SubRegSize::i64Bit) {\n LOGMAN_THROW_A_FMT(Index < 2, \"Index too large\");\n Q = Index;\n S = 0;\n opcode = 0b101;\n Size = 0b01;\n } else {\n LOGMAN_MSG_A_FMT(\"Unknown size\");\n FEX_UNREACHABLE;\n }\n\n ASIMDLoadStoreSinglePost(Op, Q, 1, R, opcode, S, Size, rm, rn, rt.Q());\n }\n\n template\n void st1(T rt, uint32_t Index, ARMEmitter::Register rn, uint32_t PostOffset) {\n st1(size, rt, Index, rn, PostOffset);\n }\n template\n void ld1(T rt, uint32_t Index, ARMEmitter::Register rn, uint32_t PostOffset) {\n ld1(size, rt, Index, rn, PostOffset);\n }\n template\n void ld1r(T rt, ARMEmitter::Register rn, uint32_t PostOffset) {\n ld1r(size, rt, rn, PostOffset);\n }\n template\n void ld2r(T rt, T rt2, ARMEmitter::Register rn, uint32_t PostOffset) {\n ld2r(size, rt, rt2, rn, PostOffset);\n }\n template\n void ld3r(T rt, T rt2, T rt3, ARMEmitter::Register rn, uint32_t PostOffset) {\n ld3r(size, rt, rt2, rt3, rn, PostOffset);\n }\n template\n void ld4r(T rt, T rt2, T rt3, T rt4, ARMEmitter::Register rn, uint32_t PostOffset) {\n ld4r(size, rt, rt2, rt3, rt4, rn, PostOffset);\n }\n\n template\n void st2(T rt, T rt2, uint32_t Index, ARMEmitter::Register rn, uint32_t PostOffset) {\n st2(size, rt, rt2, Index, rn, PostOffset);\n }\n template\n void ld2(T rt, T rt2, uint32_t Index, ARMEmitter::Register rn, uint32_t PostOffset) {\n ld2(size, rt, rt2, Index, rn, PostOffset);\n }\n template\n void st3(T rt, T rt2, T rt3, uint32_t Index, ARMEmitter::Register rn, uint32_t PostOffset) {\n st3(size, rt, rt2, rt3, Index, rn, PostOffset);\n }\n template\n void ld3(T rt, T rt2, T rt3, uint32_t Index, ARMEmitter::Register rn, uint32_t PostOffset) {\n ld3(size, rt, rt2, rt3, Index, rn, PostOffset);\n }\n template\n void st4(T rt, T rt2, T rt3, T rt4, uint32_t Index, ARMEmitter::Register rn, uint32_t PostOffset) {\n st4(size, rt, rt2, rt3, rt4, Index, rn, PostOffset);\n }\n template\n void ld4(T rt, T rt2, T rt3, T rt4, uint32_t Index, ARMEmitter::Register rn, uint32_t PostOffset) {\n ld4(size, rt, rt2, rt3, rt4, Index, rn, PostOffset);\n }\n\n template\n void st1(T rt, uint32_t Index, ARMEmitter::Register rn, ARMEmitter::Register rm) {\n st1(size, rt, Index, rn, rm);\n }\n template\n void ld1(T rt, uint32_t Index, ARMEmitter::Register rn, ARMEmitter::Register rm) {\n ld1(size, rt, Index, rn, rm);\n }\n template\n void ld1r(T rt, ARMEmitter::Register rn, ARMEmitter::Register rm) {\n ld1r(size, rt, rn, rm);\n }\n template\n void ld2r(T rt, T rt2, ARMEmitter::Register rn, ARMEmitter::Register rm) {\n ld2r(size, rt, rt2, rn, rm);\n }\n template\n void ld3r(T rt, T rt2, T rt3, ARMEmitter::Register rn, ARMEmitter::Register rm) {\n ld3r(size, rt, rt2, rt3, rn, rm);\n }\n template\n void ld4r(T rt, T rt2, T rt3, T rt4, ARMEmitter::Register rn, ARMEmitter::Register rm) {\n ld4r(size, rt, rt2, rt3, rt4, rn, rm);\n }\n\n template\n void st2(T rt, T rt2, uint32_t Index, ARMEmitter::Register rn, ARMEmitter::Register rm) {\n st2(size, rt, rt2, Index, rn, rm);\n }\n template\n void ld2(T rt, T rt2, uint32_t Index, ARMEmitter::Register rn, ARMEmitter::Register rm) {\n ld2(size, rt, rt2, Index, rn, rm);\n }\n template\n void st3(T rt, T rt2, T rt3, uint32_t Index, ARMEmitter::Register rn, ARMEmitter::Register rm) {\n st3(size, rt, rt2, rt3, Index, rn, rm);\n }\n template\n void ld3(T rt, T rt2, T rt3, uint32_t Index, ARMEmitter::Register rn, ARMEmitter::Register rm) {\n ld3(size, rt, rt2, rt3, Index, rn, rm);\n }\n template\n void st4(T rt, T rt2, T rt3, T rt4, uint32_t Index, ARMEmitter::Register rn, ARMEmitter::Register rm) {\n st4(size, rt, rt2, rt3, rt4, Index, rn, rm);\n }\n template\n void ld4(T rt, T rt2, T rt3, T rt4, uint32_t Index, ARMEmitter::Register rn, ARMEmitter::Register rm) {\n ld4(size, rt, rt2, rt3, rt4, Index, rn, rm);\n }\n\n template\n void ASIMDLoadStoreSinglePost(uint32_t Op, uint32_t Q, uint32_t L, uint32_t R, uint32_t opcode, uint32_t S, uint32_t size,\n ARMEmitter::Register rm, ARMEmitter::Register rn, T rt) {\n LOGMAN_THROW_A_FMT((std::is_same_v || std::is_same_v), \"Only supports 128-bit and \"\n \"64-bit vector registers.\");\n uint32_t Instr = Op;\n\n Instr |= Q << 30;\n Instr |= L << 22;\n Instr |= R << 21;\n Instr |= Encode_rm(rm);\n Instr |= opcode << 13;\n Instr |= S << 12;\n Instr |= size << 10;\n Instr |= Encode_rn(rn);\n Instr |= Encode_rt(rt);\n dc32(Instr);\n }\n // Loadstore exclusive pair\n void stxp(ARMEmitter::Size s, ARMEmitter::Register rs, ARMEmitter::Register rt, ARMEmitter::Register rt2, ARMEmitter::Register rn) {\n constexpr uint32_t Op = 0b1000'1000'001 << 21;\n AtomicOp(Op, s, 0, 0, rs, rt, rt2, rn);\n }\n void stlxp(ARMEmitter::Size s, ARMEmitter::Register rs, ARMEmitter::Register rt, ARMEmitter::Register rt2, ARMEmitter::Register rn) {\n constexpr uint32_t Op = 0b1000'1000'001 << 21;\n AtomicOp(Op, s, 0, 1, rs, rt, rt2, rn);\n }\n void ldxp(ARMEmitter::Size s, ARMEmitter::Register rt, ARMEmitter::Register rt2, ARMEmitter::Register rn) {\n constexpr uint32_t Op = 0b1000'1000'001 << 21;\n AtomicOp(Op, s, 1, 0, ARMEmitter::Reg::r31, rt, rt2, rn);\n }\n void ldaxp(ARMEmitter::Size s, ARMEmitter::Register rt, ARMEmitter::Register rt2, ARMEmitter::Register rn) {\n constexpr uint32_t Op = 0b1000'1000'001 << 21;\n AtomicOp(Op, s, 1, 1, ARMEmitter::Reg::r31, rt, rt2, rn);\n }\n // Loadstore exclusive register\n void stxrb(ARMEmitter::Register rs, ARMEmitter::Register rt, ARMEmitter::Register rn) {\n constexpr uint32_t Op = 0b0000'1000'000 << 21;\n SubAtomicOp(Op, ARMEmitter::SubRegSize::i8Bit, 0, 0, rs, rt, ARMEmitter::Reg::r31, rn);\n }\n void stlxrb(ARMEmitter::Register rs, ARMEmitter::Register rt, ARMEmitter::Register rn) {\n constexpr uint32_t Op = 0b0000'1000'000 << 21;\n SubAtomicOp(Op, ARMEmitter::SubRegSize::i8Bit, 0, 1, rs, rt, ARMEmitter::Reg::r31, rn);\n }\n void ldxrb(ARMEmitter::Register rt, ARMEmitter::Register rn) {\n constexpr uint32_t Op = 0b0000'1000'000 << 21;\n SubAtomicOp(Op, ARMEmitter::SubRegSize::i8Bit, 1, 0, ARMEmitter::Reg::r31, rt, ARMEmitter::Reg::r31, rn);\n }\n void ldaxrb(ARMEmitter::Register rt, ARMEmitter::Register rn) {\n constexpr uint32_t Op = 0b0000'1000'000 << 21;\n SubAtomicOp(Op, ARMEmitter::SubRegSize::i8Bit, 1, 1, ARMEmitter::Reg::r31, rt, ARMEmitter::Reg::r31, rn);\n }\n void stxrh(ARMEmitter::Register rs, ARMEmitter::Register rt, ARMEmitter::Register rn) {\n constexpr uint32_t Op = 0b0000'1000'000 << 21;\n SubAtomicOp(Op, ARMEmitter::SubRegSize::i16Bit, 0, 0, rs, rt, ARMEmitter::Reg::r31, rn);\n }\n void stlxrh(ARMEmitter::Register rs, ARMEmitter::Register rt, ARMEmitter::Register rn) {\n constexpr uint32_t Op = 0b0000'1000'000 << 21;\n SubAtomicOp(Op, ARMEmitter::SubRegSize::i16Bit, 0, 1, rs, rt, ARMEmitter::Reg::r31, rn);\n }\n void ldxrh(ARMEmitter::Register rt, ARMEmitter::Register rn) {\n constexpr uint32_t Op = 0b0000'1000'000 << 21;\n SubAtomicOp(Op, ARMEmitter::SubRegSize::i16Bit, 1, 0, ARMEmitter::Reg::r31, rt, ARMEmitter::Reg::r31, rn);\n }\n void ldaxrh(ARMEmitter::Register rt, ARMEmitter::Register rn) {\n constexpr uint32_t Op = 0b0000'1000'000 << 21;\n SubAtomicOp(Op, ARMEmitter::SubRegSize::i16Bit, 1, 1, ARMEmitter::Reg::r31, rt, ARMEmitter::Reg::r31, rn);\n }\n void stxr(ARMEmitter::WRegister rs, ARMEmitter::WRegister rt, ARMEmitter::Register rn) {\n constexpr uint32_t Op = 0b0000'1000'000 << 21;\n SubAtomicOp(Op, ARMEmitter::SubRegSize::i32Bit, 0, 0, rs, rt, ARMEmitter::WReg::w31, rn);\n }\n void stlxr(ARMEmitter::WRegister rs, ARMEmitter::WRegister rt, ARMEmitter::Register rn) {\n constexpr uint32_t Op = 0b0000'1000'000 << 21;\n SubAtomicOp(Op, ARMEmitter::SubRegSize::i32Bit, 0, 1, rs, rt, ARMEmitter::WReg::w31, rn);\n }\n void ldxr(ARMEmitter::WRegister rt, ARMEmitter::Register rn) {\n constexpr uint32_t Op = 0b0000'1000'000 << 21;\n SubAtomicOp(Op, ARMEmitter::SubRegSize::i32Bit, 1, 0, ARMEmitter::WReg::w31, rt, ARMEmitter::WReg::w31, rn);\n }\n void ldaxr(ARMEmitter::WRegister rt, ARMEmitter::Register rn) {\n constexpr uint32_t Op = 0b0000'1000'000 << 21;\n SubAtomicOp(Op, ARMEmitter::SubRegSize::i32Bit, 1, 1, ARMEmitter::WReg::w31, rt, ARMEmitter::WReg::w31, rn);\n }\n void stxr(ARMEmitter::XRegister rs, ARMEmitter::XRegister rt, ARMEmitter::Register rn) {\n constexpr uint32_t Op = 0b0000'1000'000 << 21;\n SubAtomicOp(Op, ARMEmitter::SubRegSize::i64Bit, 0, 0, rs, rt, ARMEmitter::XReg::x31, rn);\n }\n void stlxr(ARMEmitter::WRegister rs, ARMEmitter::XRegister rt, ARMEmitter::Register rn) {\n constexpr uint32_t Op = 0b0000'1000'000 << 21;\n SubAtomicOp(Op, ARMEmitter::SubRegSize::i64Bit, 0, 1, rs.R(), rt.R(), ARMEmitter::Reg::r31, rn);\n }\n void ldxr(ARMEmitter::XRegister rt, ARMEmitter::Register rn) {\n constexpr uint32_t Op = 0b0000'1000'000 << 21;\n SubAtomicOp(Op, ARMEmitter::SubRegSize::i64Bit, 1, 0, ARMEmitter::XReg::x31, rt, ARMEmitter::XReg::x31, rn);\n }\n void ldaxr(ARMEmitter::XRegister rt, ARMEmitter::Register rn) {\n constexpr uint32_t Op = 0b0000'1000'000 << 21;\n SubAtomicOp(Op, ARMEmitter::SubRegSize::i64Bit, 1, 1, ARMEmitter::XReg::x31, rt, ARMEmitter::XReg::x31, rn);\n }\n void stxr(ARMEmitter::SubRegSize size, ARMEmitter::Register rs, ARMEmitter::Register rt, ARMEmitter::Register rn) {\n constexpr uint32_t Op = 0b0000'1000'000 << 21;\n SubAtomicOp(Op, size, 0, 0, rs, rt, ARMEmitter::Reg::r31, rn);\n }\n void stlxr(ARMEmitter::SubRegSize size, ARMEmitter::Register rs, ARMEmitter::Register rt, ARMEmitter::Register rn) {\n constexpr uint32_t Op = 0b0000'1000'000 << 21;\n SubAtomicOp(Op, size, 0, 1, rs, rt, ARMEmitter::Reg::r31, rn);\n }\n void ldxr(ARMEmitter::SubRegSize size, ARMEmitter::Register rt, ARMEmitter::Register rn) {\n constexpr uint32_t Op = 0b0000'1000'000 << 21;\n SubAtomicOp(Op, size, 1, 0, ARMEmitter::Reg::r31, rt, ARMEmitter::Reg::r31, rn);\n }\n void ldaxr(ARMEmitter::SubRegSize size, ARMEmitter::Register rt, ARMEmitter::Register rn) {\n constexpr uint32_t Op = 0b0000'1000'000 << 21;\n SubAtomicOp(Op, size, 1, 1, ARMEmitter::Reg::r31, rt, ARMEmitter::Reg::r31, rn);\n }\n\n // Load/store ordered\n static constexpr uint32_t LoadStoreOrdered_Op = 0b0000'1000'100 << 21;\n void stllrb(ARMEmitter::Register rt, ARMEmitter::Register rn) {\n SubAtomicOp(LoadStoreOrdered_Op, ARMEmitter::SubRegSize::i8Bit, 0, 0, ARMEmitter::Reg::r31, rt, ARMEmitter::Reg::r31, rn);\n }\n void stlrb(ARMEmitter::Register rt, ARMEmitter::Register rn) {\n SubAtomicOp(LoadStoreOrdered_Op, ARMEmitter::SubRegSize::i8Bit, 0, 1, ARMEmitter::Reg::r31, rt, ARMEmitter::Reg::r31, rn);\n }\n void ldlarb(ARMEmitter::Register rt, ARMEmitter::Register rn) {\n SubAtomicOp(LoadStoreOrdered_Op, ARMEmitter::SubRegSize::i8Bit, 1, 0, ARMEmitter::Reg::r31, rt, ARMEmitter::Reg::r31, rn);\n }\n void ldarb(ARMEmitter::Register rt, ARMEmitter::Register rn) {\n SubAtomicOp(LoadStoreOrdered_Op, ARMEmitter::SubRegSize::i8Bit, 1, 1, ARMEmitter::Reg::r31, rt, ARMEmitter::Reg::r31, rn);\n }\n void stllrh(ARMEmitter::Register rt, ARMEmitter::Register rn) {\n SubAtomicOp(LoadStoreOrdered_Op, ARMEmitter::SubRegSize::i16Bit, 0, 0, ARMEmitter::Reg::r31, rt, ARMEmitter::Reg::r31, rn);\n }\n void stlrh(ARMEmitter::Register rt, ARMEmitter::Register rn) {\n SubAtomicOp(LoadStoreOrdered_Op, ARMEmitter::SubRegSize::i16Bit, 0, 1, ARMEmitter::Reg::r31, rt, ARMEmitter::Reg::r31, rn);\n }\n void ldlarh(ARMEmitter::Register rt, ARMEmitter::Register rn) {\n SubAtomicOp(LoadStoreOrdered_Op, ARMEmitter::SubRegSize::i16Bit, 1, 0, ARMEmitter::Reg::r31, rt, ARMEmitter::Reg::r31, rn);\n }\n void ldarh(ARMEmitter::Register rt, ARMEmitter::Register rn) {\n SubAtomicOp(LoadStoreOrdered_Op, ARMEmitter::SubRegSize::i16Bit, 1, 1, ARMEmitter::Reg::r31, rt, ARMEmitter::Reg::r31, rn);\n }\n void stllr(ARMEmitter::WRegister rt, ARMEmitter::Register rn) {\n SubAtomicOp(LoadStoreOrdered_Op, ARMEmitter::SubRegSize::i32Bit, 0, 0, ARMEmitter::WReg::w31, rt, ARMEmitter::WReg::w31, rn);\n }\n void stlr(ARMEmitter::WRegister rt, ARMEmitter::Register rn) {\n SubAtomicOp(LoadStoreOrdered_Op, ARMEmitter::SubRegSize::i32Bit, 0, 1, ARMEmitter::WReg::w31, rt, ARMEmitter::WReg::w31, rn);\n }\n void ldlar(ARMEmitter::WRegister rt, ARMEmitter::Register rn) {\n SubAtomicOp(LoadStoreOrdered_Op, ARMEmitter::SubRegSize::i32Bit, 1, 0, ARMEmitter::WReg::w31, rt, ARMEmitter::WReg::w31, rn);\n }\n void ldar(ARMEmitter::WRegister rt, ARMEmitter::Register rn) {\n SubAtomicOp(LoadStoreOrdered_Op, ARMEmitter::SubRegSize::i32Bit, 1, 1, ARMEmitter::WReg::w31, rt, ARMEmitter::WReg::w31, rn);\n }\n void stllr(ARMEmitter::XRegister rt, ARMEmitter::Register rn) {\n SubAtomicOp(LoadStoreOrdered_Op, ARMEmitter::SubRegSize::i64Bit, 0, 0, ARMEmitter::XReg::x31, rt, ARMEmitter::XReg::x31, rn);\n }\n void stlr(ARMEmitter::XRegister rt, ARMEmitter::Register rn) {\n SubAtomicOp(LoadStoreOrdered_Op, ARMEmitter::SubRegSize::i64Bit, 0, 1, ARMEmitter::XReg::x31, rt, ARMEmitter::XReg::x31, rn);\n }\n void ldlar(ARMEmitter::XRegister rt, ARMEmitter::Register rn) {\n SubAtomicOp(LoadStoreOrdered_Op, ARMEmitter::SubRegSize::i64Bit, 1, 0, ARMEmitter::XReg::x31, rt, ARMEmitter::XReg::x31, rn);\n }\n void ldar(ARMEmitter::XRegister rt, ARMEmitter::Register rn) {\n SubAtomicOp(LoadStoreOrdered_Op, ARMEmitter::SubRegSize::i64Bit, 1, 1, ARMEmitter::XReg::x31, rt, ARMEmitter::XReg::x31, rn);\n }\n // Compare and swap\n void casb(ARMEmitter::Register rs, ARMEmitter::Register rt, ARMEmitter::Register rn) {\n constexpr uint32_t Op = 0b0000'1000'101 << 21;\n SubAtomicOp(Op, ARMEmitter::SubRegSize::i8Bit, 0, 0, rs, rt, ARMEmitter::Reg::r31, rn);\n }\n void caslb(ARMEmitter::Register rs, ARMEmitter::Register rt, ARMEmitter::Register rn) {\n constexpr uint32_t Op = 0b0000'1000'101 << 21;\n SubAtomicOp(Op, ARMEmitter::SubRegSize::i8Bit, 0, 1, rs, rt, ARMEmitter::Reg::r31, rn);\n }\n void casab(ARMEmitter::Register rs, ARMEmitter::Register rt, ARMEmitter::Register rn) {\n constexpr uint32_t Op = 0b0000'1000'101 << 21;\n SubAtomicOp(Op, ARMEmitter::SubRegSize::i8Bit, 1, 0, rs, rt, ARMEmitter::Reg::r31, rn);\n }\n void casalb(ARMEmitter::Register rs, ARMEmitter::Register rt, ARMEmitter::Register rn) {\n constexpr uint32_t Op = 0b0000'1000'101 << 21;\n SubAtomicOp(Op, ARMEmitter::SubRegSize::i8Bit, 1, 1, rs, rt, ARMEmitter::Reg::r31, rn);\n }\n void cash(ARMEmitter::Register rs, ARMEmitter::Register rt, ARMEmitter::Register rn) {\n constexpr uint32_t Op = 0b0000'1000'101 << 21;\n SubAtomicOp(Op, ARMEmitter::SubRegSize::i16Bit, 0, 0, rs, rt, ARMEmitter::Reg::r31, rn);\n }\n void caslh(ARMEmitter::Register rs, ARMEmitter::Register rt, ARMEmitter::Register rn) {\n constexpr uint32_t Op = 0b0000'1000'101 << 21;\n SubAtomicOp(Op, ARMEmitter::SubRegSize::i16Bit, 0, 1, rs, rt, ARMEmitter::Reg::r31, rn);\n }\n void casah(ARMEmitter::Register rs, ARMEmitter::Register rt, ARMEmitter::Register rn) {\n constexpr uint32_t Op = 0b0000'1000'101 << 21;\n SubAtomicOp(Op, ARMEmitter::SubRegSize::i16Bit, 1, 0, rs, rt, ARMEmitter::Reg::r31, rn);\n }\n void casalh(ARMEmitter::Register rs, ARMEmitter::Register rt, ARMEmitter::Register rn) {\n constexpr uint32_t Op = 0b0000'1000'101 << 21;\n SubAtomicOp(Op, ARMEmitter::SubRegSize::i16Bit, 1, 1, rs, rt, ARMEmitter::Reg::r31, rn);\n }\n void cas(ARMEmitter::WRegister rs, ARMEmitter::WRegister rt, ARMEmitter::Register rn) {\n constexpr uint32_t Op = 0b0000'1000'101 << 21;\n SubAtomicOp(Op, ARMEmitter::SubRegSize::i32Bit, 0, 0, rs.R(), rt.R(), ARMEmitter::Reg::r31, rn);\n }\n void casl(ARMEmitter::WRegister rs, ARMEmitter::WRegister rt, ARMEmitter::Register rn) {\n constexpr uint32_t Op = 0b0000'1000'101 << 21;\n SubAtomicOp(Op, ARMEmitter::SubRegSize::i32Bit, 0, 1, rs.R(), rt.R(), ARMEmitter::Reg::r31, rn);\n }\n void casa(ARMEmitter::WRegister rs, ARMEmitter::WRegister rt, ARMEmitter::Register rn) {\n constexpr uint32_t Op = 0b0000'1000'101 << 21;\n SubAtomicOp(Op, ARMEmitter::SubRegSize::i32Bit, 1, 0, rs.R(), rt.R(), ARMEmitter::Reg::r31, rn);\n }\n void casal(ARMEmitter::WRegister rs, ARMEmitter::WRegister rt, ARMEmitter::Register rn) {\n constexpr uint32_t Op = 0b0000'1000'101 << 21;\n SubAtomicOp(Op, ARMEmitter::SubRegSize::i32Bit, 1, 1, rs.R(), rt.R(), ARMEmitter::Reg::r31, rn);\n }\n void cas(ARMEmitter::XRegister rs, ARMEmitter::XRegister rt, ARMEmitter::Register rn) {\n constexpr uint32_t Op = 0b0000'1000'101 << 21;\n SubAtomicOp(Op, ARMEmitter::SubRegSize::i64Bit, 0, 0, rs.R(), rt.R(), ARMEmitter::Reg::r31, rn);\n }\n void casl(ARMEmitter::XRegister rs, ARMEmitter::XRegister rt, ARMEmitter::Register rn) {\n constexpr uint32_t Op = 0b0000'1000'101 << 21;\n SubAtomicOp(Op, ARMEmitter::SubRegSize::i64Bit, 0, 1, rs.R(), rt.R(), ARMEmitter::Reg::r31, rn);\n }\n void casa(ARMEmitter::XRegister rs, ARMEmitter::XRegister rt, ARMEmitter::Register rn) {\n constexpr uint32_t Op = 0b0000'1000'101 << 21;\n SubAtomicOp(Op, ARMEmitter::SubRegSize::i64Bit, 1, 0, rs.R(), rt.R(), ARMEmitter::Reg::r31, rn);\n }\n void casal(ARMEmitter::XRegister rs, ARMEmitter::XRegister rt, ARMEmitter::Register rn) {\n constexpr uint32_t Op = 0b0000'1000'101 << 21;\n SubAtomicOp(Op, ARMEmitter::SubRegSize::i64Bit, 1, 1, rs.R(), rt.R(), ARMEmitter::Reg::r31, rn);\n }\n\n void cas(ARMEmitter::SubRegSize size, ARMEmitter::Register rs, ARMEmitter::Register rt, ARMEmitter::Register rn) {\n constexpr uint32_t Op = 0b0000'1000'101 << 21;\n SubAtomicOp(Op, size, 0, 0, rs, rt, ARMEmitter::Reg::r31, rn);\n }\n void casl(ARMEmitter::SubRegSize size, ARMEmitter::Register rs, ARMEmitter::Register rt, ARMEmitter::Register rn) {\n constexpr uint32_t Op = 0b0000'1000'101 << 21;\n SubAtomicOp(Op, size, 0, 1, rs, rt, ARMEmitter::Reg::r31, rn);\n }\n void casa(ARMEmitter::SubRegSize size, ARMEmitter::Register rs, ARMEmitter::Register rt, ARMEmitter::Register rn) {\n constexpr uint32_t Op = 0b0000'1000'101 << 21;\n SubAtomicOp(Op, size, 1, 0, rs, rt, ARMEmitter::Reg::r31, rn);\n }\n void casal(ARMEmitter::SubRegSize size, ARMEmitter::Register rs, ARMEmitter::Register rt, ARMEmitter::Register rn) {\n constexpr uint32_t Op = 0b0000'1000'101 << 21;\n SubAtomicOp(Op, size, 1, 1, rs, rt, ARMEmitter::Reg::r31, rn);\n }\n // LDAPR/STLR unscaled immediate\n void stlurb(ARMEmitter::Register rt, ARMEmitter::Register rn, int32_t Imm) {\n LOGMAN_THROW_A_FMT(Imm >= -256 && Imm <= 255, \"Unscaled offset too large\");\n constexpr uint32_t Op = 0b0001'1001'000 << 21;\n SubAtomicImm(Op, ARMEmitter::SubRegSize::i8Bit, 0b00, rt, rn, static_cast(Imm) & 0x1'FF);\n }\n void ldapurb(ARMEmitter::Register rt, ARMEmitter::Register rn, int32_t Imm) {\n LOGMAN_THROW_A_FMT(Imm >= -256 && Imm <= 255, \"Unscaled offset too large\");\n constexpr uint32_t Op = 0b0001'1001'000 << 21;\n SubAtomicImm(Op, ARMEmitter::SubRegSize::i8Bit, 0b01, rt, rn, static_cast(Imm) & 0x1'FF);\n }\n void ldapursb(ARMEmitter::WRegister rt, ARMEmitter::Register rn, int32_t Imm) {\n LOGMAN_THROW_A_FMT(Imm >= -256 && Imm <= 255, \"Unscaled offset too large\");\n constexpr uint32_t Op = 0b0001'1001'000 << 21;\n SubAtomicImm(Op, ARMEmitter::SubRegSize::i8Bit, 0b11, rt, rn, static_cast(Imm) & 0x1'FF);\n }\n void ldapursb(ARMEmitter::XRegister rt, ARMEmitter::Register rn, int32_t Imm) {\n LOGMAN_THROW_A_FMT(Imm >= -256 && Imm <= 255, \"Unscaled offset too large\");\n constexpr uint32_t Op = 0b0001'1001'000 << 21;\n SubAtomicImm(Op, ARMEmitter::SubRegSize::i8Bit, 0b10, rt, rn, static_cast(Imm) & 0x1'FF);\n }\n void stlurh(ARMEmitter::Register rt, ARMEmitter::Register rn, int32_t Imm) {\n LOGMAN_THROW_A_FMT(Imm >= -256 && Imm <= 255, \"Unscaled offset too large\");\n constexpr uint32_t Op = 0b0001'1001'000 << 21;\n SubAtomicImm(Op, ARMEmitter::SubRegSize::i16Bit, 0b00, rt, rn, static_cast(Imm) & 0x1'FF);\n }\n void ldapurh(ARMEmitter::Register rt, ARMEmitter::Register rn, int32_t Imm) {\n LOGMAN_THROW_A_FMT(Imm >= -256 && Imm <= 255, \"Unscaled offset too large\");\n constexpr uint32_t Op = 0b0001'1001'000 << 21;\n SubAtomicImm(Op, ARMEmitter::SubRegSize::i16Bit, 0b01, rt, rn, static_cast(Imm) & 0x1'FF);\n }\n void ldapursh(ARMEmitter::WRegister rt, ARMEmitter::Register rn, int32_t Imm) {\n LOGMAN_THROW_A_FMT(Imm >= -256 && Imm <= 255, \"Unscaled offset too large\");\n constexpr uint32_t Op = 0b0001'1001'000 << 21;\n SubAtomicImm(Op, ARMEmitter::SubRegSize::i16Bit, 0b11, rt, rn, static_cast(Imm) & 0x1'FF);\n }\n void ldapursh(ARMEmitter::XRegister rt, ARMEmitter::Register rn, int32_t Imm) {\n LOGMAN_THROW_A_FMT(Imm >= -256 && Imm <= 255, \"Unscaled offset too large\");\n constexpr uint32_t Op = 0b0001'1001'000 << 21;\n SubAtomicImm(Op, ARMEmitter::SubRegSize::i16Bit, 0b10, rt, rn, static_cast(Imm) & 0x1'FF);\n }\n void stlur(ARMEmitter::WRegister rt, ARMEmitter::Register rn, int32_t Imm) {\n LOGMAN_THROW_A_FMT(Imm >= -256 && Imm <= 255, \"Unscaled offset too large\");\n constexpr uint32_t Op = 0b0001'1001'000 << 21;\n SubAtomicImm(Op, ARMEmitter::SubRegSize::i32Bit, 0b00, rt, rn, static_cast(Imm) & 0x1'FF);\n }\n void ldapur(ARMEmitter::WRegister rt, ARMEmitter::Register rn, int32_t Imm) {\n LOGMAN_THROW_A_FMT(Imm >= -256 && Imm <= 255, \"Unscaled offset too large\");\n constexpr uint32_t Op = 0b0001'1001'000 << 21;\n SubAtomicImm(Op, ARMEmitter::SubRegSize::i32Bit, 0b01, rt, rn, static_cast(Imm) & 0x1'FF);\n }\n void ldapursw(ARMEmitter::XRegister rt, ARMEmitter::Register rn, int32_t Imm) {\n LOGMAN_THROW_A_FMT(Imm >= -256 && Imm <= 255, \"Unscaled offset too large\");\n constexpr uint32_t Op = 0b0001'1001'000 << 21;\n SubAtomicImm(Op, ARMEmitter::SubRegSize::i32Bit, 0b10, rt, rn, static_cast(Imm) & 0x1'FF);\n }\n void stlur(ARMEmitter::XRegister rt, ARMEmitter::Register rn, int32_t Imm) {\n LOGMAN_THROW_A_FMT(Imm >= -256 && Imm <= 255, \"Unscaled offset too large\");\n constexpr uint32_t Op = 0b0001'1001'000 << 21;\n SubAtomicImm(Op, ARMEmitter::SubRegSize::i64Bit, 0b00, rt, rn, static_cast(Imm) & 0x1'FF);\n }\n void ldapur(ARMEmitter::XRegister rt, ARMEmitter::Register rn, int32_t Imm) {\n LOGMAN_THROW_A_FMT(Imm >= -256 && Imm <= 255, \"Unscaled offset too large\");\n constexpr uint32_t Op = 0b0001'1001'000 << 21;\n SubAtomicImm(Op, ARMEmitter::SubRegSize::i64Bit, 0b01, rt, rn, static_cast(Imm) & 0x1'FF);\n }\n // Load register literal\n void ldr(ARMEmitter::WRegister rt, int32_t Imm) {\n LOGMAN_THROW_A_FMT(Imm >= -1048576 && Imm <= 1048575 && ((Imm & 0b11) == 0), \"Unscaled offset too large\");\n constexpr uint32_t Op = 0b0001'1000 << 24;\n LoadStoreLiteral(Op, rt, static_cast(Imm >> 2) & 0x7'FFFF);\n }\n void ldr(ARMEmitter::SRegister rt, int32_t Imm) {\n LOGMAN_THROW_A_FMT(Imm >= -1048576 && Imm <= 1048575 && ((Imm & 0b11) == 0), \"Unscaled offset too large\");\n constexpr uint32_t Op = 0b0001'1100 << 24;\n LoadStoreLiteral(Op, rt, static_cast(Imm >> 2) & 0x7'FFFF);\n }\n void ldr(ARMEmitter::XRegister rt, int32_t Imm) {\n LOGMAN_THROW_A_FMT(Imm >= -1048576 && Imm <= 1048575 && ((Imm & 0b11) == 0), \"Unscaled offset too large\");\n constexpr uint32_t Op = 0b0101'1000 << 24;\n LoadStoreLiteral(Op, rt, static_cast(Imm >> 2) & 0x7'FFFF);\n }\n void ldr(ARMEmitter::DRegister rt, int32_t Imm) {\n LOGMAN_THROW_A_FMT(Imm >= -1048576 && Imm <= 1048575 && ((Imm & 0b11) == 0), \"Unscaled offset too large\");\n constexpr uint32_t Op = 0b0101'1100 << 24;\n LoadStoreLiteral(Op, rt, static_cast(Imm >> 2) & 0x7'FFFF);\n }\n void ldrs(ARMEmitter::WRegister rt, int32_t Imm) {\n LOGMAN_THROW_A_FMT(Imm >= -1048576 && Imm <= 1048575 && ((Imm & 0b11) == 0), \"Unscaled offset too large\");\n constexpr uint32_t Op = 0b1001'1000 << 24;\n LoadStoreLiteral(Op, rt, static_cast(Imm >> 2) & 0x7'FFFF);\n }\n void ldr(ARMEmitter::QRegister rt, int32_t Imm) {\n LOGMAN_THROW_A_FMT(Imm >= -1048576 && Imm <= 1048575 && ((Imm & 0b11) == 0), \"Unscaled offset too large\");\n constexpr uint32_t Op = 0b1001'1100 << 24;\n LoadStoreLiteral(Op, rt, static_cast(Imm >> 2) & 0x7'FFFF);\n }\n void prfm(ARMEmitter::Prefetch prfop, int32_t Imm) {\n LOGMAN_THROW_A_FMT(Imm >= -1048576 && Imm <= 1048575 && ((Imm & 0b11) == 0), \"Unscaled offset too large\");\n constexpr uint32_t Op = 0b1101'1000 << 24;\n LoadStoreLiteral(Op, prfop, static_cast(Imm >> 2) & 0x7'FFFF);\n }\n void ldr(ARMEmitter::WRegister rt, const BackwardLabel* Label) {\n int32_t Imm = static_cast(Label->Location - GetCursorAddress());\n LOGMAN_THROW_A_FMT(Imm >= -1048576 && Imm <= 1048575 && ((Imm & 0b11) == 0), \"Unscaled offset too large\");\n constexpr uint32_t Op = 0b0001'1000 << 24;\n LoadStoreLiteral(Op, rt, static_cast(Imm >> 2) & 0x7'FFFF);\n }\n void ldr(ARMEmitter::SRegister rt, const BackwardLabel* Label) {\n int32_t Imm = static_cast(Label->Location - GetCursorAddress());\n LOGMAN_THROW_A_FMT(Imm >= -1048576 && Imm <= 1048575 && ((Imm & 0b11) == 0), \"Unscaled offset too large\");\n constexpr uint32_t Op = 0b0001'1100 << 24;\n LoadStoreLiteral(Op, rt, static_cast(Imm >> 2) & 0x7'FFFF);\n }\n void ldr(ARMEmitter::XRegister rt, const BackwardLabel* Label) {\n int32_t Imm = static_cast(Label->Location - GetCursorAddress());\n LOGMAN_THROW_A_FMT(Imm >= -1048576 && Imm <= 1048575 && ((Imm & 0b11) == 0), \"Unscaled offset too large\");\n constexpr uint32_t Op = 0b0101'1000 << 24;\n LoadStoreLiteral(Op, rt, static_cast(Imm >> 2) & 0x7'FFFF);\n }\n void ldr(ARMEmitter::DRegister rt, const BackwardLabel* Label) {\n int32_t Imm = static_cast(Label->Location - GetCursorAddress());\n LOGMAN_THROW_A_FMT(Imm >= -1048576 && Imm <= 1048575 && ((Imm & 0b11) == 0), \"Unscaled offset too large\");\n constexpr uint32_t Op = 0b0101'1100 << 24;\n LoadStoreLiteral(Op, rt, static_cast(Imm >> 2) & 0x7'FFFF);\n }\n void ldrsw(ARMEmitter::XRegister rt, const BackwardLabel* Label) {\n int32_t Imm = static_cast(Label->Location - GetCursorAddress());\n LOGMAN_THROW_A_FMT(Imm >= -1048576 && Imm <= 1048575 && ((Imm & 0b11) == 0), \"Unscaled offset too large\");\n constexpr uint32_t Op = 0b1001'1000 << 24;\n LoadStoreLiteral(Op, rt, static_cast(Imm >> 2) & 0x7'FFFF);\n }\n void ldr(ARMEmitter::QRegister rt, const BackwardLabel* Label) {\n int32_t Imm = static_cast(Label->Location - GetCursorAddress());\n LOGMAN_THROW_A_FMT(Imm >= -1048576 && Imm <= 1048575 && ((Imm & 0b11) == 0), \"Unscaled offset too large\");\n constexpr uint32_t Op = 0b1001'1100 << 24;\n LoadStoreLiteral(Op, rt, static_cast(Imm >> 2) & 0x7'FFFF);\n }\n void prfm(ARMEmitter::Prefetch prfop, const BackwardLabel* Label) {\n int32_t Imm = static_cast(Label->Location - GetCursorAddress());\n LOGMAN_THROW_A_FMT(Imm >= -1048576 && Imm <= 1048575 && ((Imm & 0b11) == 0), \"Unscaled offset too large\");\n constexpr uint32_t Op = 0b1101'1000 << 24;\n LoadStoreLiteral(Op, prfop, static_cast(Imm >> 2) & 0x7'FFFF);\n }\n\n void ldr(ARMEmitter::WRegister rt, ForwardLabel* Label) {\n AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress(), .Type = ForwardLabel::InstType::RELATIVE_LOAD});\n constexpr uint32_t Op = 0b0001'1000 << 24;\n LoadStoreLiteral(Op, rt, 0);\n }\n\n void ldr(ARMEmitter::SRegister rt, ForwardLabel* Label) {\n AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress(), .Type = ForwardLabel::InstType::RELATIVE_LOAD});\n constexpr uint32_t Op = 0b0001'1100 << 24;\n LoadStoreLiteral(Op, rt, 0);\n }\n\n void ldr(ARMEmitter::XRegister rt, ForwardLabel* Label) {\n AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress(), .Type = ForwardLabel::InstType::RELATIVE_LOAD});\n constexpr uint32_t Op = 0b0101'1000 << 24;\n LoadStoreLiteral(Op, rt, 0);\n }\n\n void ldr(ARMEmitter::DRegister rt, ForwardLabel* Label) {\n AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress(), .Type = ForwardLabel::InstType::RELATIVE_LOAD});\n constexpr uint32_t Op = 0b0101'1100 << 24;\n LoadStoreLiteral(Op, rt, 0);\n }\n\n void ldrsw(ARMEmitter::XRegister rt, ForwardLabel* Label) {\n AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress(), .Type = ForwardLabel::InstType::RELATIVE_LOAD});\n constexpr uint32_t Op = 0b1001'1000 << 24;\n LoadStoreLiteral(Op, rt, 0);\n }\n\n void ldr(ARMEmitter::QRegister rt, ForwardLabel* Label) {\n AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress(), .Type = ForwardLabel::InstType::RELATIVE_LOAD});\n constexpr uint32_t Op = 0b1001'1100 << 24;\n LoadStoreLiteral(Op, rt, 0);\n }\n\n void prfm(ARMEmitter::Prefetch prfop, ForwardLabel* Label) {\n AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress(), .Type = ForwardLabel::InstType::RELATIVE_LOAD});\n constexpr uint32_t Op = 0b1101'1000 << 24;\n LoadStoreLiteral(Op, prfop, 0);\n }\n\n void ldr(ARMEmitter::WRegister rt, BiDirectionalLabel* Label) {\n if (Label->Backward.Location) {\n ldr(rt, &Label->Backward);\n } else {\n ldr(rt, &Label->Forward);\n }\n }\n void ldr(ARMEmitter::SRegister rt, BiDirectionalLabel* Label) {\n if (Label->Backward.Location) {\n ldr(rt, &Label->Backward);\n } else {\n ldr(rt, &Label->Forward);\n }\n }\n void ldr(ARMEmitter::XRegister rt, BiDirectionalLabel* Label) {\n if (Label->Backward.Location) {\n ldr(rt, &Label->Backward);\n } else {\n ldr(rt, &Label->Forward);\n }\n }\n void ldr(ARMEmitter::DRegister rt, BiDirectionalLabel* Label) {\n if (Label->Backward.Location) {\n ldr(rt, &Label->Backward);\n } else {\n ldr(rt, &Label->Forward);\n }\n }\n void ldrs(ARMEmitter::WRegister rt, BiDirectionalLabel* Label) {\n if (Label->Backward.Location) {\n ldr(rt, &Label->Backward);\n } else {\n ldr(rt, &Label->Forward);\n }\n }\n void ldr(ARMEmitter::QRegister rt, BiDirectionalLabel* Label) {\n if (Label->Backward.Location) {\n ldr(rt, &Label->Backward);\n } else {\n ldr(rt, &Label->Forward);\n }\n }\n void prfm(ARMEmitter::Prefetch prfop, BiDirectionalLabel* Label) {\n if (Label->Backward.Location) {\n prfm(prfop, &Label->Backward);\n } else {\n prfm(prfop, &Label->Forward);\n }\n }\n\n // Memory copy/set\n void cpyfp(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 0, 0b00, 0b0000, rs, rn, rd);\n }\n void cpyfm(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 0, 0b01, 0b0000, rs, rn, rd);\n }\n void cpyfe(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 0, 0b10, 0b0000, rs, rn, rd);\n }\n void cpyfpwt(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 0, 0b00, 0b0001, rs, rn, rd);\n }\n void cpyfmwt(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 0, 0b01, 0b0001, rs, rn, rd);\n }\n void cpyfewt(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 0, 0b10, 0b0001, rs, rn, rd);\n }\n void cpyfprt(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 0, 0b00, 0b0010, rs, rn, rd);\n }\n void cpyfmrt(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 0, 0b01, 0b0010, rs, rn, rd);\n }\n void cpyfert(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 0, 0b10, 0b0010, rs, rn, rd);\n }\n void cpyfpt(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 0, 0b00, 0b0011, rs, rn, rd);\n }\n void cpyfmt(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 0, 0b01, 0b0011, rs, rn, rd);\n }\n void cpyfet(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 0, 0b10, 0b0011, rs, rn, rd);\n }\n void cpyfpwn(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 0, 0b00, 0b0100, rs, rn, rd);\n }\n void cpyfmwn(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 0, 0b01, 0b0100, rs, rn, rd);\n }\n void cpyfewn(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 0, 0b10, 0b0100, rs, rn, rd);\n }\n void cpyfpwtwn(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 0, 0b00, 0b0101, rs, rn, rd);\n }\n void cpyfmwtwn(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 0, 0b01, 0b0101, rs, rn, rd);\n }\n void cpyfewtwn(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 0, 0b10, 0b0101, rs, rn, rd);\n }\n void cpyfprtwn(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 0, 0b00, 0b0110, rs, rn, rd);\n }\n void cpyfmrtwn(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 0, 0b01, 0b0110, rs, rn, rd);\n }\n void cpyfertwn(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 0, 0b10, 0b0110, rs, rn, rd);\n }\n void cpyfptwn(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 0, 0b00, 0b0111, rs, rn, rd);\n }\n void cpyfmtwn(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 0, 0b01, 0b0111, rs, rn, rd);\n }\n void cpyfetwn(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 0, 0b10, 0b0111, rs, rn, rd);\n }\n void cpyfprn(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 0, 0b00, 0b1000, rs, rn, rd);\n }\n void cpyfmrn(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 0, 0b01, 0b1000, rs, rn, rd);\n }\n void cpyfern(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 0, 0b10, 0b1000, rs, rn, rd);\n }\n void cpyfpwtrn(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 0, 0b00, 0b1001, rs, rn, rd);\n }\n void cpyfmwtrn(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 0, 0b01, 0b1001, rs, rn, rd);\n }\n void cpyfewtrn(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 0, 0b10, 0b1001, rs, rn, rd);\n }\n void cpyfprtrn(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 0, 0b00, 0b1010, rs, rn, rd);\n }\n void cpyfmrtrn(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 0, 0b01, 0b1010, rs, rn, rd);\n }\n void cpyfertrn(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 0, 0b10, 0b1010, rs, rn, rd);\n }\n void cpyfptrn(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 0, 0b00, 0b1011, rs, rn, rd);\n }\n void cpyfmtrn(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 0, 0b01, 0b1011, rs, rn, rd);\n }\n void cpyfetrn(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 0, 0b10, 0b1011, rs, rn, rd);\n }\n void cpyfpn(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 0, 0b00, 0b1100, rs, rn, rd);\n }\n void cpyfmn(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 0, 0b01, 0b1100, rs, rn, rd);\n }\n void cpyfen(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 0, 0b10, 0b1100, rs, rn, rd);\n }\n void cpyfpwtn(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 0, 0b00, 0b1101, rs, rn, rd);\n }\n void cpyfmwtn(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 0, 0b01, 0b1101, rs, rn, rd);\n }\n void cpyfewtn(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 0, 0b10, 0b1101, rs, rn, rd);\n }\n void cpyfprtn(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 0, 0b00, 0b1110, rs, rn, rd);\n }\n void cpyfmrtn(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 0, 0b01, 0b1110, rs, rn, rd);\n }\n void cpyfertn(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 0, 0b10, 0b1110, rs, rn, rd);\n }\n void cpyfptn(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 0, 0b00, 0b1111, rs, rn, rd);\n }\n void cpyfmtn(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 0, 0b01, 0b1111, rs, rn, rd);\n }\n void cpyfetn(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 0, 0b10, 0b1111, rs, rn, rd);\n }\n\n void setp(Register rd, Register rn, Register rs) {\n MemoryCopyAndMemorySet(0, 0, 0b11, 0b0000, rs, rn, rd);\n }\n void setm(Register rd, Register rn, Register rs) {\n MemoryCopyAndMemorySet(0, 0, 0b11, 0b0100, rs, rn, rd);\n }\n void sete(Register rd, Register rn, Register rs) {\n MemoryCopyAndMemorySet(0, 0, 0b11, 0b1000, rs, rn, rd);\n }\n void setpt(Register rd, Register rn, Register rs) {\n MemoryCopyAndMemorySet(0, 0, 0b11, 0b0001, rs, rn, rd);\n }\n void setmt(Register rd, Register rn, Register rs) {\n MemoryCopyAndMemorySet(0, 0, 0b11, 0b0101, rs, rn, rd);\n }\n void setet(Register rd, Register rn, Register rs) {\n MemoryCopyAndMemorySet(0, 0, 0b11, 0b1001, rs, rn, rd);\n }\n void setpn(Register rd, Register rn, Register rs) {\n MemoryCopyAndMemorySet(0, 0, 0b11, 0b0010, rs, rn, rd);\n }\n void setmn(Register rd, Register rn, Register rs) {\n MemoryCopyAndMemorySet(0, 0, 0b11, 0b0110, rs, rn, rd);\n }\n void seten(Register rd, Register rn, Register rs) {\n MemoryCopyAndMemorySet(0, 0, 0b11, 0b1010, rs, rn, rd);\n }\n void setptn(Register rd, Register rn, Register rs) {\n MemoryCopyAndMemorySet(0, 0, 0b11, 0b0011, rs, rn, rd);\n }\n void setmtn(Register rd, Register rn, Register rs) {\n MemoryCopyAndMemorySet(0, 0, 0b11, 0b0111, rs, rn, rd);\n }\n void setetn(Register rd, Register rn, Register rs) {\n MemoryCopyAndMemorySet(0, 0, 0b11, 0b1011, rs, rn, rd);\n }\n\n void cpyp(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 1, 0b00, 0b0000, rs, rn, rd);\n }\n void cpym(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 1, 0b01, 0b0000, rs, rn, rd);\n }\n void cpye(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 1, 0b10, 0b0000, rs, rn, rd);\n }\n void cpypwt(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 1, 0b00, 0b0001, rs, rn, rd);\n }\n void cpymwt(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 1, 0b01, 0b0001, rs, rn, rd);\n }\n void cpyewt(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 1, 0b10, 0b0001, rs, rn, rd);\n }\n void cpyprt(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 1, 0b00, 0b0010, rs, rn, rd);\n }\n void cpymrt(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 1, 0b01, 0b0010, rs, rn, rd);\n }\n void cpyert(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 1, 0b10, 0b0010, rs, rn, rd);\n }\n void cpypt(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 1, 0b00, 0b0011, rs, rn, rd);\n }\n void cpymt(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 1, 0b01, 0b0011, rs, rn, rd);\n }\n void cpyet(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 1, 0b10, 0b0011, rs, rn, rd);\n }\n void cpypwn(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 1, 0b00, 0b0100, rs, rn, rd);\n }\n void cpymwn(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 1, 0b01, 0b0100, rs, rn, rd);\n }\n void cpyewn(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 1, 0b10, 0b0100, rs, rn, rd);\n }\n void cpypwtwn(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 1, 0b00, 0b0101, rs, rn, rd);\n }\n void cpymwtwn(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 1, 0b01, 0b0101, rs, rn, rd);\n }\n void cpyewtwn(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 1, 0b10, 0b0101, rs, rn, rd);\n }\n void cpyprtwn(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 1, 0b00, 0b0110, rs, rn, rd);\n }\n void cpymrtwn(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 1, 0b01, 0b0110, rs, rn, rd);\n }\n void cpyertwn(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 1, 0b10, 0b0110, rs, rn, rd);\n }\n void cpyptwn(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 1, 0b00, 0b0111, rs, rn, rd);\n }\n void cpymtwn(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 1, 0b01, 0b0111, rs, rn, rd);\n }\n void cpyetwn(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 1, 0b10, 0b0111, rs, rn, rd);\n }\n void cpyprn(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 1, 0b00, 0b1000, rs, rn, rd);\n }\n void cpymrn(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 1, 0b01, 0b1000, rs, rn, rd);\n }\n void cpyern(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 1, 0b10, 0b1000, rs, rn, rd);\n }\n void cpypwtrn(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 1, 0b00, 0b1001, rs, rn, rd);\n }\n void cpymwtrn(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 1, 0b01, 0b1001, rs, rn, rd);\n }\n void cpyewtrn(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 1, 0b10, 0b1001, rs, rn, rd);\n }\n void cpyprtrn(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 1, 0b00, 0b1010, rs, rn, rd);\n }\n void cpymrtrn(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 1, 0b01, 0b1010, rs, rn, rd);\n }\n void cpyertrn(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 1, 0b10, 0b1010, rs, rn, rd);\n }\n void cpyptrn(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 1, 0b00, 0b1011, rs, rn, rd);\n }\n void cpymtrn(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 1, 0b01, 0b1011, rs, rn, rd);\n }\n void cpyetrn(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 1, 0b10, 0b1011, rs, rn, rd);\n }\n void cpypn(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 1, 0b00, 0b1100, rs, rn, rd);\n }\n void cpymn(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 1, 0b01, 0b1100, rs, rn, rd);\n }\n void cpyen(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 1, 0b10, 0b1100, rs, rn, rd);\n }\n void cpypwtn(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 1, 0b00, 0b1101, rs, rn, rd);\n }\n void cpymwtn(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 1, 0b01, 0b1101, rs, rn, rd);\n }\n void cpyewtn(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 1, 0b10, 0b1101, rs, rn, rd);\n }\n void cpyprtn(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 1, 0b00, 0b1110, rs, rn, rd);\n }\n void cpymrtn(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 1, 0b01, 0b1110, rs, rn, rd);\n }\n void cpyertn(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 1, 0b10, 0b1110, rs, rn, rd);\n }\n void cpyptn(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 1, 0b00, 0b1111, rs, rn, rd);\n }\n void cpymtn(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 1, 0b01, 0b1111, rs, rn, rd);\n }\n void cpyetn(Register rd, Register rs, Register rn) {\n MemoryCopyAndMemorySet(0, 1, 0b10, 0b1111, rs, rn, rd);\n }\n\n void setgp(Register rd, Register rn, Register rs) {\n MemoryCopyAndMemorySet(0, 1, 0b11, 0b0000, rs, rn, rd);\n }\n void setgm(Register rd, Register rn, Register rs) {\n MemoryCopyAndMemorySet(0, 1, 0b11, 0b0100, rs, rn, rd);\n }\n void setge(Register rd, Register rn, Register rs) {\n MemoryCopyAndMemorySet(0, 1, 0b11, 0b1000, rs, rn, rd);\n }\n void setgpt(Register rd, Register rn, Register rs) {\n MemoryCopyAndMemorySet(0, 1, 0b11, 0b0001, rs, rn, rd);\n }\n void setgmt(Register rd, Register rn, Register rs) {\n MemoryCopyAndMemorySet(0, 1, 0b11, 0b0101, rs, rn, rd);\n }\n void setget(Register rd, Register rn, Register rs) {\n MemoryCopyAndMemorySet(0, 1, 0b11, 0b1001, rs, rn, rd);\n }\n void setgpn(Register rd, Register rn, Register rs) {\n MemoryCopyAndMemorySet(0, 1, 0b11, 0b0010, rs, rn, rd);\n }\n void setgmn(Register rd, Register rn, Register rs) {\n MemoryCopyAndMemorySet(0, 1, 0b11, 0b0110, rs, rn, rd);\n }\n void setgen(Register rd, Register rn, Register rs) {\n MemoryCopyAndMemorySet(0, 1, 0b11, 0b1010, rs, rn, rd);\n }\n void setgptn(Register rd, Register rn, Register rs) {\n MemoryCopyAndMemorySet(0, 1, 0b11, 0b0011, rs, rn, rd);\n }\n void setgmtn(Register rd, Register rn, Register rs) {\n MemoryCopyAndMemorySet(0, 1, 0b11, 0b0111, rs, rn, rd);\n }\n void setgetn(Register rd, Register rn, Register rs) {\n MemoryCopyAndMemorySet(0, 1, 0b11, 0b1011, rs, rn, rd);\n }\n\n // Loadstore no-allocate pair\n void stnp(ARMEmitter::WRegister rt, ARMEmitter::WRegister rt2, ARMEmitter::Register rn, int32_t Imm) {\n LOGMAN_THROW_A_FMT(Imm >= -256 && Imm <= 252 && ((Imm & 0b11) == 0), \"Unscaled offset too large\");\n constexpr uint32_t Op = 0b0010'1000'00 << 22;\n LoadStoreNoAllocate(Op, rt, rt2, rn, static_cast(Imm >> 2) & 0b111'1111);\n }\n void ldnp(ARMEmitter::WRegister rt, ARMEmitter::WRegister rt2, ARMEmitter::Register rn, int32_t Imm) {\n LOGMAN_THROW_A_FMT(Imm >= -256 && Imm <= 252 && ((Imm & 0b11) == 0), \"Unscaled offset too large\");\n constexpr uint32_t Op = 0b0010'1000'01 << 22;\n LoadStoreNoAllocate(Op, rt, rt2, rn, static_cast(Imm >> 2) & 0b111'1111);\n }\n void stnp(ARMEmitter::SRegister rt, ARMEmitter::SRegister rt2, ARMEmitter::Register rn, int32_t Imm) {\n LOGMAN_THROW_A_FMT(Imm >= -256 && Imm <= 252 && ((Imm & 0b11) == 0), \"Unscaled offset too large\");\n constexpr uint32_t Op = 0b0010'1100'00 << 22;\n LoadStoreNoAllocate(Op, rt, rt2, rn, static_cast(Imm >> 2) & 0b111'1111);\n }\n void ldnp(ARMEmitter::SRegister rt, ARMEmitter::SRegister rt2, ARMEmitter::Register rn, int32_t Imm) {\n LOGMAN_THROW_A_FMT(Imm >= -256 && Imm <= 252 && ((Imm & 0b11) == 0), \"Unscaled offset too large\");\n constexpr uint32_t Op = 0b0010'1100'01 << 22;\n LoadStoreNoAllocate(Op, rt, rt2, rn, static_cast(Imm >> 2) & 0b111'1111);\n }\n void stnp(ARMEmitter::XRegister rt, ARMEmitter::XRegister rt2, ARMEmitter::Register rn, int32_t Imm) {\n LOGMAN_THROW_A_FMT(Imm >= -512 && Imm <= 504 && ((Imm & 0b111) == 0), \"Unscaled offset too large\");\n constexpr uint32_t Op = 0b1010'1000'00 << 22;\n LoadStoreNoAllocate(Op, rt, rt2, rn, static_cast(Imm >> 3) & 0b111'1111);\n }\n void ldnp(ARMEmitter::XRegister rt, ARMEmitter::XRegister rt2, ARMEmitter::Register rn, int32_t Imm) {\n LOGMAN_THROW_A_FMT(Imm >= -512 && Imm <= 504 && ((Imm & 0b111) == 0), \"Unscaled offset too large\");\n constexpr uint32_t Op = 0b1010'1000'01 << 22;\n LoadStoreNoAllocate(Op, rt, rt2, rn, static_cast(Imm >> 3) & 0b111'1111);\n }\n void stnp(ARMEmitter::DRegister rt, ARMEmitter::DRegister rt2, ARMEmitter::Register rn, int32_t Imm) {\n LOGMAN_THROW_A_FMT(Imm >= -512 && Imm <= 504 && ((Imm & 0b111) == 0), \"Unscaled offset too large\");\n constexpr uint32_t Op = 0b0110'1100'00 << 22;\n LoadStoreNoAllocate(Op, rt, rt2, rn, static_cast(Imm >> 3) & 0b111'1111);\n }\n void ldnp(ARMEmitter::DRegister rt, ARMEmitter::DRegister rt2, ARMEmitter::Register rn, int32_t Imm) {\n LOGMAN_THROW_A_FMT(Imm >= -512 && Imm <= 504 && ((Imm & 0b111) == 0), \"Unscaled offset too large\");\n constexpr uint32_t Op = 0b0110'1100'01 << 22;\n LoadStoreNoAllocate(Op, rt, rt2, rn, static_cast(Imm >> 3) & 0b111'1111);\n }\n void stnp(ARMEmitter::QRegister rt, ARMEmitter::QRegister rt2, ARMEmitter::Register rn, int32_t Imm) {\n LOGMAN_THROW_A_FMT(Imm >= -1024 && Imm <= 1008 && ((Imm & 0b1111) == 0), \"Unscaled offset too large\");\n constexpr uint32_t Op = 0b1010'1100'00 << 22;\n LoadStoreNoAllocate(Op, rt, rt2, rn, static_cast(Imm >> 4) & 0b111'1111);\n }\n void ldnp(ARMEmitter::QRegister rt, ARMEmitter::QRegister rt2, ARMEmitter::Register rn, int32_t Imm) {\n LOGMAN_THROW_A_FMT(Imm >= -1024 && Imm <= 1008 && ((Imm & 0b1111) == 0), \"Unscaled offset too large\");\n constexpr uint32_t Op = 0b1010'1100'01 << 22;\n LoadStoreNoAllocate(Op, rt, rt2, rn, static_cast(Imm >> 4) & 0b111'1111);\n }\n // Loadstore register pair post-indexed\n // Loadstore register pair offset\n // Loadstore register pair pre-indexed\n template\n void stp(ARMEmitter::WRegister rt, ARMEmitter::WRegister rt2, ARMEmitter::Register rn, int32_t Imm = 0) {\n LOGMAN_THROW_A_FMT(Imm >= -256 && Imm <= 252 && ((Imm & 0b11) == 0), \"Unscaled offset too large\");\n constexpr uint32_t Op = (0b0010'1000'00 << 22) | (Index == IndexType::POST ? (0b01 << 23) :\n Index == IndexType::PRE ? (0b11 << 23) :\n Index == IndexType::OFFSET ? (0b10 << 23) :\n -1);\n\n LoadStorePair(Op, rt, rt2, rn, (Imm >> 2) & 0b111'1111);\n }\n template\n void ldp(ARMEmitter::WRegister rt, ARMEmitter::WRegister rt2, ARMEmitter::Register rn, int32_t Imm = 0) {\n LOGMAN_THROW_A_FMT(Imm >= -256 && Imm <= 252 && ((Imm & 0b11) == 0), \"Unscaled offset too large\");\n constexpr uint32_t Op = (0b0010'1000'01 << 22) | (Index == IndexType::POST ? (0b01 << 23) :\n Index == IndexType::PRE ? (0b11 << 23) :\n Index == IndexType::OFFSET ? (0b10 << 23) :\n -1);\n\n LoadStorePair(Op, rt, rt2, rn, (Imm >> 2) & 0b111'1111);\n }\n template\n void ldpsw(ARMEmitter::XRegister rt, ARMEmitter::XRegister rt2, ARMEmitter::Register rn, int32_t Imm = 0) {\n LOGMAN_THROW_A_FMT(Imm >= -256 && Imm <= 252 && ((Imm & 0b11) == 0), \"Unscaled offset too large\");\n constexpr uint32_t Op = (0b0110'1000'01 << 22) | (Index == IndexType::POST ? (0b01 << 23) :\n Index == IndexType::PRE ? (0b11 << 23) :\n Index == IndexType::OFFSET ? (0b10 << 23) :\n -1);\n LoadStorePair(Op, rt, rt2, rn, (Imm >> 2) & 0b111'1111);\n }\n template\n void stp(ARMEmitter::XRegister rt, ARMEmitter::XRegister rt2, ARMEmitter::Register rn, int32_t Imm = 0) {\n LOGMAN_THROW_A_FMT(Imm >= -512 && Imm <= 504 && ((Imm & 0b111) == 0), \"Unscaled offset too large\");\n constexpr uint32_t Op = (0b1010'1000'00 << 22) | (Index == IndexType::POST ? (0b01 << 23) :\n Index == IndexType::PRE ? (0b11 << 23) :\n Index == IndexType::OFFSET ? (0b10 << 23) :\n -1);\n\n LoadStorePair(Op, rt, rt2, rn, (Imm >> 3) & 0b111'1111);\n }\n template\n void ldp(ARMEmitter::XRegister rt, ARMEmitter::XRegister rt2, ARMEmitter::Register rn, int32_t Imm = 0) {\n LOGMAN_THROW_A_FMT(Imm >= -512 && Imm <= 504 && ((Imm & 0b111) == 0), \"Unscaled offset too large\");\n constexpr uint32_t Op = (0b1010'1000'01 << 22) | (Index == IndexType::POST ? (0b01 << 23) :\n Index == IndexType::PRE ? (0b11 << 23) :\n Index == IndexType::OFFSET ? (0b10 << 23) :\n -1);\n\n LoadStorePair(Op, rt, rt2, rn, (Imm >> 3) & 0b111'1111);\n }\n template\n void stp(ARMEmitter::SRegister rt, ARMEmitter::SRegister rt2, ARMEmitter::Register rn, int32_t Imm = 0) {\n stp_w(rt.V(), rt2.V(), rn, Imm);\n }\n template\n void ldp(ARMEmitter::SRegister rt, ARMEmitter::SRegister rt2, ARMEmitter::Register rn, int32_t Imm = 0) {\n ldp_w(rt.V(), rt2.V(), rn, Imm);\n }\n template\n void stp(ARMEmitter::DRegister rt, ARMEmitter::DRegister rt2, ARMEmitter::Register rn, int32_t Imm = 0) {\n stp_x(rt.V(), rt2.V(), rn, Imm);\n }\n template\n void ldp(ARMEmitter::DRegister rt, ARMEmitter::DRegister rt2, ARMEmitter::Register rn, int32_t Imm = 0) {\n ldp_x(rt.V(), rt2.V(), rn, Imm);\n }\n template\n void stp(ARMEmitter::QRegister rt, ARMEmitter::QRegister rt2, ARMEmitter::Register rn, int32_t Imm = 0) {\n stp_q(rt.V(), rt2.V(), rn, Imm);\n }\n template\n void ldp(ARMEmitter::QRegister rt, ARMEmitter::QRegister rt2, ARMEmitter::Register rn, int32_t Imm = 0) {\n ldp_q(rt.V(), rt2.V(), rn, Imm);\n }\n\n // Loadstore register unscaled immediate\n void sturb(ARMEmitter::Register rt, ARMEmitter::Register rn, int32_t Imm = 0) {\n stXrb(rt, rn, Imm);\n }\n void ldurb(ARMEmitter::Register rt, ARMEmitter::Register rn, int32_t Imm = 0) {\n ldXrb(rt, rn, Imm);\n }\n void sturb(ARMEmitter::VRegister rt, ARMEmitter::Register rn, int32_t Imm = 0) {\n stXrb(rt, rn, Imm);\n }\n void ldurb(ARMEmitter::VRegister rt, ARMEmitter::Register rn, int32_t Imm = 0) {\n ldXrb(rt, rn, Imm);\n }\n void ldursb(ARMEmitter::XRegister rt, ARMEmitter::Register rn, int32_t Imm = 0) {\n ldXrsb(rt, rn, Imm);\n }\n void ldursb(ARMEmitter::WRegister rt, ARMEmitter::Register rn, int32_t Imm = 0) {\n ldXrsb(rt, rn, Imm);\n }\n void sturh(ARMEmitter::Register rt, ARMEmitter::Register rn, int32_t Imm = 0) {\n stXrh(rt, rn, Imm);\n }\n void ldurh(ARMEmitter::Register rt, ARMEmitter::Register rn, int32_t Imm = 0) {\n ldXrh(rt, rn, Imm);\n }\n void sturh(ARMEmitter::VRegister rt, ARMEmitter::Register rn, int32_t Imm = 0) {\n stXrh(rt, rn, Imm);\n }\n void ldurh(ARMEmitter::VRegister rt, ARMEmitter::Register rn, int32_t Imm = 0) {\n ldXrh(rt, rn, Imm);\n }\n void ldursh(ARMEmitter::XRegister rt, ARMEmitter::Register rn, int32_t Imm = 0) {\n ldXrsh(rt, rn, Imm);\n }\n void ldursh(ARMEmitter::WRegister rt, ARMEmitter::Register rn, int32_t Imm = 0) {\n ldXrsh(rt, rn, Imm);\n }\n void stur(ARMEmitter::WRegister rt, ARMEmitter::Register rn, int32_t Imm = 0) {\n stXr(rt, rn, Imm);\n }\n void ldur(ARMEmitter::WRegister rt, ARMEmitter::Register rn, int32_t Imm = 0) {\n ldXr(rt, rn, Imm);\n }\n void stur(ARMEmitter::SRegister rt, ARMEmitter::Register rn, int32_t Imm = 0) {\n stXr(rt, rn, Imm);\n }\n void ldur(ARMEmitter::SRegister rt, ARMEmitter::Register rn, int32_t Imm = 0) {\n ldXr(rt, rn, Imm);\n }\n void ldursw(ARMEmitter::XRegister rt, ARMEmitter::Register rn, int32_t Imm = 0) {\n ldXrsw(rt, rn, Imm);\n }\n void stur(ARMEmitter::XRegister rt, ARMEmitter::Register rn, int32_t Imm = 0) {\n stXr(rt, rn, Imm);\n }\n void ldur(ARMEmitter::XRegister rt, ARMEmitter::Register rn, int32_t Imm = 0) {\n ldXr(rt, rn, Imm);\n }\n void stur(ARMEmitter::DRegister rt, ARMEmitter::Register rn, int32_t Imm = 0) {\n stXr(rt, rn, Imm);\n }\n void ldur(ARMEmitter::DRegister rt, ARMEmitter::Register rn, int32_t Imm = 0) {\n ldXr(rt, rn, Imm);\n }\n void stur(ARMEmitter::QRegister rt, ARMEmitter::Register rn, int32_t Imm = 0) {\n stXr(rt, rn, Imm);\n }\n void ldur(ARMEmitter::QRegister rt, ARMEmitter::Register rn, int32_t Imm = 0) {\n ldXr(rt, rn, Imm);\n }\n template\n void prfum(ARMEmitter::Prefetch prfop, ARMEmitter::Register rn, int32_t Imm = 0) {\n LOGMAN_THROW_A_FMT(Imm >= -256 && Imm <= 255, \"Unscaled offset too large\");\n static_assert(Index == IndexType::OFFSET, \"Doesn't support another index type\");\n\n constexpr uint32_t Op = 0b1111'1000'10 << 22;\n constexpr uint32_t o2 = 0b00;\n\n LoadStoreImm(Op, o2, prfop, rn, Imm);\n }\n\n // Loadstore register immediate post-indexed\n // Loadstore register immediate pre-indexed\n template\n requires (Index == IndexType::POST || Index == IndexType::PRE)\n void strb(ARMEmitter::Register rt, ARMEmitter::Register rn, int32_t Imm = 0) {\n stXrb(rt, rn, Imm);\n }\n template\n requires (Index == IndexType::POST || Index == IndexType::PRE)\n void ldrb(ARMEmitter::Register rt, ARMEmitter::Register rn, int32_t Imm = 0) {\n ldXrb(rt, rn, Imm);\n }\n template\n requires (Index == IndexType::POST || Index == IndexType::PRE)\n void strb(ARMEmitter::VRegister rt, ARMEmitter::Register rn, int32_t Imm = 0) {\n stXrb(rt, rn, Imm);\n }\n template\n requires (Index == IndexType::POST || Index == IndexType::PRE)\n void ldrb(ARMEmitter::VRegister rt, ARMEmitter::Register rn, int32_t Imm = 0) {\n ldXrb(rt, rn, Imm);\n }\n template\n requires (Index == IndexType::POST || Index == IndexType::PRE)\n void ldrsb(ARMEmitter::XRegister rt, ARMEmitter::Register rn, int32_t Imm = 0) {\n ldXrsb(rt, rn, Imm);\n }\n template\n requires (Index == IndexType::POST || Index == IndexType::PRE)\n void ldrsb(ARMEmitter::WRegister rt, ARMEmitter::Register rn, int32_t Imm = 0) {\n ldXrsb(rt, rn, Imm);\n }\n template\n requires (Index == IndexType::POST || Index == IndexType::PRE)\n void strh(ARMEmitter::Register rt, ARMEmitter::Register rn, int32_t Imm = 0) {\n stXrh(rt, rn, Imm);\n }\n template\n requires (Index == IndexType::POST || Index == IndexType::PRE)\n void ldrh(ARMEmitter::Register rt, ARMEmitter::Register rn, int32_t Imm = 0) {\n ldXrh(rt, rn, Imm);\n }\n template\n requires (Index == IndexType::POST || Index == IndexType::PRE)\n void strh(ARMEmitter::VRegister rt, ARMEmitter::Register rn, int32_t Imm = 0) {\n stXrh(rt, rn, Imm);\n }\n template\n requires (Index == IndexType::POST || Index == IndexType::PRE)\n void ldrh(ARMEmitter::VRegister rt, ARMEmitter::Register rn, int32_t Imm = 0) {\n ldXrh(rt, rn, Imm);\n }\n template\n requires (Index == IndexType::POST || Index == IndexType::PRE)\n void ldrsh(ARMEmitter::XRegister rt, ARMEmitter::Register rn, int32_t Imm = 0) {\n ldXrsh(rt, rn, Imm);\n }\n template\n requires (Index == IndexType::POST || Index == IndexType::PRE)\n void ldrsh(ARMEmitter::WRegister rt, ARMEmitter::Register rn, int32_t Imm = 0) {\n ldXrsh(rt, rn, Imm);\n }\n template\n requires (Index == IndexType::POST || Index == IndexType::PRE)\n void str(ARMEmitter::WRegister rt, ARMEmitter::Register rn, int32_t Imm = 0) {\n stXr(rt, rn, Imm);\n }\n template\n requires (Index == IndexType::POST || Index == IndexType::PRE)\n void ldr(ARMEmitter::WRegister rt, ARMEmitter::Register rn, int32_t Imm = 0) {\n ldXr(rt, rn, Imm);\n }\n template\n requires (Index == IndexType::POST || Index == IndexType::PRE)\n void str(ARMEmitter::SRegister rt, ARMEmitter::Register rn, int32_t Imm = 0) {\n stXr(rt, rn, Imm);\n }\n template\n requires (Index == IndexType::POST || Index == IndexType::PRE)\n void ldr(ARMEmitter::SRegister rt, ARMEmitter::Register rn, int32_t Imm = 0) {\n ldXr(rt, rn, Imm);\n }\n template\n requires (Index == IndexType::POST || Index == IndexType::PRE)\n void ldrsw(ARMEmitter::XRegister rt, ARMEmitter::Register rn, int32_t Imm = 0) {\n ldXrsw(rt, rn, Imm);\n }\n template\n requires (Index == IndexType::POST || Index == IndexType::PRE)\n void str(ARMEmitter::XRegister rt, ARMEmitter::Register rn, int32_t Imm = 0) {\n stXr(rt, rn, Imm);\n }\n template\n requires (Index == IndexType::POST || Index == IndexType::PRE)\n void ldr(ARMEmitter::XRegister rt, ARMEmitter::Register rn, int32_t Imm = 0) {\n ldXr(rt, rn, Imm);\n }\n template\n requires (Index == IndexType::POST || Index == IndexType::PRE)\n void str(ARMEmitter::DRegister rt, ARMEmitter::Register rn, int32_t Imm = 0) {\n stXr(rt, rn, Imm);\n }\n template\n requires (Index == IndexType::POST || Index == IndexType::PRE)\n void ldr(ARMEmitter::DRegister rt, ARMEmitter::Register rn, int32_t Imm = 0) {\n ldXr(rt, rn, Imm);\n }\n template\n requires (Index == IndexType::POST || Index == IndexType::PRE)\n void str(ARMEmitter::QRegister rt, ARMEmitter::Register rn, int32_t Imm = 0) {\n stXr(rt, rn, Imm);\n }\n template\n requires (Index == IndexType::POST || Index == IndexType::PRE)\n void ldr(ARMEmitter::QRegister rt, ARMEmitter::Register rn, int32_t Imm = 0) {\n ldXr(rt, rn, Imm);\n }\n\n // Loadstore register unprivileged\n void sttrb(ARMEmitter::Register rt, ARMEmitter::Register rn, int32_t Imm = 0) {\n stXrb(rt, rn, Imm);\n }\n void ldtrb(ARMEmitter::Register rt, ARMEmitter::Register rn, int32_t Imm = 0) {\n ldXrb(rt, rn, Imm);\n }\n void ldtrsb(ARMEmitter::XRegister rt, ARMEmitter::Register rn, int32_t Imm = 0) {\n ldXrsb(rt, rn, Imm);\n }\n void ldtrsb(ARMEmitter::WRegister rt, ARMEmitter::Register rn, int32_t Imm = 0) {\n ldXrsb(rt, rn, Imm);\n }\n void sttrh(ARMEmitter::Register rt, ARMEmitter::Register rn, int32_t Imm = 0) {\n stXrh(rt, rn, Imm);\n }\n void ldtrh(ARMEmitter::Register rt, ARMEmitter::Register rn, int32_t Imm = 0) {\n ldXrh(rt, rn, Imm);\n }\n void ldtrsh(ARMEmitter::XRegister rt, ARMEmitter::Register rn, int32_t Imm = 0) {\n ldXrsh(rt, rn, Imm);\n }\n void ldtrsh(ARMEmitter::WRegister rt, ARMEmitter::Register rn, int32_t Imm = 0) {\n ldXrsh(rt, rn, Imm);\n }\n void sttr(ARMEmitter::WRegister rt, ARMEmitter::Register rn, int32_t Imm = 0) {\n stXr(rt, rn, Imm);\n }\n void ldtr(ARMEmitter::WRegister rt, ARMEmitter::Register rn, int32_t Imm = 0) {\n ldXr(rt, rn, Imm);\n }\n void ldtrsw(ARMEmitter::XRegister rt, ARMEmitter::Register rn, int32_t Imm = 0) {\n ldXrsw(rt, rn, Imm);\n }\n void sttr(ARMEmitter::XRegister rt, ARMEmitter::Register rn, int32_t Imm = 0) {\n stXr(rt, rn, Imm);\n }\n void ldtr(ARMEmitter::XRegister rt, ARMEmitter::Register rn, int32_t Imm = 0) {\n ldXr(rt, rn, Imm);\n }\n // Atomic memory operations\n void stadd(SubRegSize size, Register rs, Register rn) {\n LoadStoreAtomicLSE(size, 0, 0, 0, 0b000, rs, Reg::zr, rn);\n }\n void staddl(SubRegSize size, Register rs, Register rn) {\n LoadStoreAtomicLSE(size, 0, 1, 0, 0b000, rs, Reg::zr, rn);\n }\n void stadda(SubRegSize size, Register rs, Register rn) {\n LoadStoreAtomicLSE(size, 1, 0, 0, 0b000, rs, Reg::zr, rn);\n }\n void staddal(SubRegSize size, Register rs, Register rn) {\n LoadStoreAtomicLSE(size, 1, 1, 0, 0b000, rs, Reg::zr, rn);\n }\n void stclr(SubRegSize size, Register rs, Register rn) {\n LoadStoreAtomicLSE(size, 0, 0, 0, 0b001, rs, Reg::zr, rn);\n }\n void stclrl(SubRegSize size, Register rs, Register rn) {\n LoadStoreAtomicLSE(size, 0, 1, 0, 0b001, rs, Reg::zr, rn);\n }\n void stclra(SubRegSize size, Register rs, Register rn) {\n LoadStoreAtomicLSE(size, 1, 0, 0, 0b001, rs, Reg::zr, rn);\n }\n void stclral(SubRegSize size, Register rs, Register rn) {\n LoadStoreAtomicLSE(size, 1, 1, 0, 0b001, rs, Reg::zr, rn);\n }\n void stset(SubRegSize size, Register rs, Register rn) {\n LoadStoreAtomicLSE(size, 0, 0, 0, 0b011, rs, Reg::zr, rn);\n }\n void stsetl(SubRegSize size, Register rs, Register rn) {\n LoadStoreAtomicLSE(size, 0, 1, 0, 0b011, rs, Reg::zr, rn);\n }\n void stseta(SubRegSize size, Register rs, Register rn) {\n LoadStoreAtomicLSE(size, 1, 0, 0, 0b011, rs, Reg::zr, rn);\n }\n void stsetal(SubRegSize size, Register rs, Register rn) {\n LoadStoreAtomicLSE(size, 1, 1, 0, 0b011, rs, Reg::zr, rn);\n }\n void steor(SubRegSize size, Register rs, Register rn) {\n LoadStoreAtomicLSE(size, 0, 0, 0, 0b010, rs, Reg::zr, rn);\n }\n void steorl(SubRegSize size, Register rs, Register rn) {\n LoadStoreAtomicLSE(size, 0, 1, 0, 0b010, rs, Reg::zr, rn);\n }\n void steora(SubRegSize size, Register rs, Register rn) {\n LoadStoreAtomicLSE(size, 1, 0, 0, 0b010, rs, Reg::zr, rn);\n }\n void steoral(SubRegSize size, Register rs, Register rn) {\n LoadStoreAtomicLSE(size, 1, 1, 0, 0b010, rs, Reg::zr, rn);\n }\n void stsmax(SubRegSize size, Register rs, Register rn) {\n LoadStoreAtomicLSE(size, 0, 0, 0, 0b100, rs, Reg::zr, rn);\n }\n void stsmaxl(SubRegSize size, Register rs, Register rn) {\n LoadStoreAtomicLSE(size, 0, 1, 0, 0b100, rs, Reg::zr, rn);\n }\n void stsmaxa(SubRegSize size, Register rs, Register rn) {\n LoadStoreAtomicLSE(size, 1, 0, 0, 0b100, rs, Reg::zr, rn);\n }\n void stsmaxal(SubRegSize size, Register rs, Register rn) {\n LoadStoreAtomicLSE(size, 1, 1, 0, 0b100, rs, Reg::zr, rn);\n }\n void stsmin(SubRegSize size, Register rs, Register rn) {\n LoadStoreAtomicLSE(size, 0, 0, 0, 0b101, rs, Reg::zr, rn);\n }\n void stsminl(SubRegSize size, Register rs, Register rn) {\n LoadStoreAtomicLSE(size, 0, 1, 0, 0b101, rs, Reg::zr, rn);\n }\n void stsmina(SubRegSize size, Register rs, Register rn) {\n LoadStoreAtomicLSE(size, 1, 0, 0, 0b101, rs, Reg::zr, rn);\n }\n void stsminal(SubRegSize size, Register rs, Register rn) {\n LoadStoreAtomicLSE(size, 1, 1, 0, 0b101, rs, Reg::zr, rn);\n }\n void stumax(SubRegSize size, Register rs, Register rn) {\n LoadStoreAtomicLSE(size, 0, 0, 0, 0b110, rs, Reg::zr, rn);\n }\n void stumaxl(SubRegSize size, Register rs, Register rn) {\n LoadStoreAtomicLSE(size, 0, 1, 0, 0b110, rs, Reg::zr, rn);\n }\n void stumaxa(SubRegSize size, Register rs, Register rn) {\n LoadStoreAtomicLSE(size, 1, 0, 0, 0b110, rs, Reg::zr, rn);\n }\n void stumaxal(SubRegSize size, Register rs, Register rn) {\n LoadStoreAtomicLSE(size, 1, 1, 0, 0b110, rs, Reg::zr, rn);\n }\n void stumin(SubRegSize size, Register rs, Register rn) {\n LoadStoreAtomicLSE(size, 0, 0, 0, 0b111, rs, Reg::zr, rn);\n }\n void stuminl(SubRegSize size, Register rs, Register rn) {\n LoadStoreAtomicLSE(size, 0, 1, 0, 0b111, rs, Reg::zr, rn);\n }\n void stumina(SubRegSize size, Register rs, Register rn) {\n LoadStoreAtomicLSE(size, 1, 0, 0, 0b111, rs, Reg::zr, rn);\n }\n void stuminal(SubRegSize size, Register rs, Register rn) {\n LoadStoreAtomicLSE(size, 1, 1, 0, 0b111, rs, Reg::zr, rn);\n }\n void ldswp(SubRegSize size, Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(size, 0, 0, 1, 0b000, rs, rt, rn);\n }\n void ldswpl(SubRegSize size, Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(size, 0, 1, 1, 0b000, rs, rt, rn);\n }\n void ldswpa(SubRegSize size, Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(size, 1, 0, 1, 0b000, rs, rt, rn);\n }\n void ldswpal(SubRegSize size, Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(size, 1, 1, 1, 0b000, rs, rt, rn);\n }\n\n void ldadd(SubRegSize size, Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(size, 0, 0, 0, 0b000, rs, rt, rn);\n }\n void ldadda(SubRegSize size, Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(size, 1, 0, 0, 0b000, rs, rt, rn);\n }\n void ldaddl(SubRegSize size, Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(size, 0, 1, 0, 0b000, rs, rt, rn);\n }\n void ldaddal(SubRegSize size, Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(size, 1, 1, 0, 0b000, rs, rt, rn);\n }\n void ldclr(SubRegSize size, Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(size, 0, 0, 0, 0b001, rs, rt, rn);\n }\n void ldclra(SubRegSize size, Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(size, 1, 0, 0, 0b001, rs, rt, rn);\n }\n void ldclrl(SubRegSize size, Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(size, 0, 1, 0, 0b001, rs, rt, rn);\n }\n void ldclral(SubRegSize size, Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(size, 1, 1, 0, 0b001, rs, rt, rn);\n }\n\n void ldset(SubRegSize size, Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(size, 0, 0, 0, 0b011, rs, rt, rn);\n }\n void ldseta(SubRegSize size, Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(size, 1, 0, 0, 0b011, rs, rt, rn);\n }\n void ldsetl(SubRegSize size, Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(size, 0, 1, 0, 0b011, rs, rt, rn);\n }\n void ldsetal(SubRegSize size, Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(size, 1, 1, 0, 0b011, rs, rt, rn);\n }\n void ldeor(SubRegSize size, Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(size, 0, 0, 0, 0b010, rs, rt, rn);\n }\n void ldeora(SubRegSize size, Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(size, 1, 0, 0, 0b010, rs, rt, rn);\n }\n void ldeorl(SubRegSize size, Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(size, 0, 1, 0, 0b010, rs, rt, rn);\n }\n void ldeoral(SubRegSize size, Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(size, 1, 1, 0, 0b010, rs, rt, rn);\n }\n\n\n // 8-bit\n void ldaddb(Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i8Bit, 0, 0, 0, 0b000, rs, rt, rn);\n }\n void ldclrb(Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i8Bit, 0, 0, 0, 0b001, rs, rt, rn);\n }\n void ldeorb(Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i8Bit, 0, 0, 0, 0b010, rs, rt, rn);\n }\n void ldsetb(Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i8Bit, 0, 0, 0, 0b011, rs, rt, rn);\n }\n void ldsmaxb(Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i8Bit, 0, 0, 0, 0b100, rs, rt, rn);\n }\n void ldsminb(Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i8Bit, 0, 0, 0, 0b101, rs, rt, rn);\n }\n void ldumaxb(Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i8Bit, 0, 0, 0, 0b110, rs, rt, rn);\n }\n void lduminb(Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i8Bit, 0, 0, 0, 0b111, rs, rt, rn);\n }\n void ldswpb(Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i8Bit, 0, 0, 1, 0b000, rs, rt, rn);\n }\n void ldaddlb(Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i8Bit, 0, 1, 0, 0b000, rs, rt, rn);\n }\n void ldclrlb(Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i8Bit, 0, 1, 0, 0b001, rs, rt, rn);\n }\n void ldeorlb(Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i8Bit, 0, 1, 0, 0b010, rs, rt, rn);\n }\n void ldsetlb(Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i8Bit, 0, 1, 0, 0b011, rs, rt, rn);\n }\n void ldsmaxlb(Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i8Bit, 0, 1, 0, 0b100, rs, rt, rn);\n }\n void ldsminlb(Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i8Bit, 0, 1, 0, 0b101, rs, rt, rn);\n }\n void ldumaxlb(Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i8Bit, 0, 1, 0, 0b110, rs, rt, rn);\n }\n void lduminlb(Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i8Bit, 0, 1, 0, 0b111, rs, rt, rn);\n }\n void ldswplb(Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i8Bit, 0, 1, 1, 0b000, rs, rt, rn);\n }\n void ldaddab(Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i8Bit, 1, 0, 0, 0b000, rs, rt, rn);\n }\n void ldclrab(Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i8Bit, 1, 0, 0, 0b001, rs, rt, rn);\n }\n void ldeorab(Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i8Bit, 1, 0, 0, 0b010, rs, rt, rn);\n }\n void ldsetab(Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i8Bit, 1, 0, 0, 0b011, rs, rt, rn);\n }\n void ldsmaxab(Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i8Bit, 1, 0, 0, 0b100, rs, rt, rn);\n }\n void ldsminab(Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i8Bit, 1, 0, 0, 0b101, rs, rt, rn);\n }\n void ldumaxab(Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i8Bit, 1, 0, 0, 0b110, rs, rt, rn);\n }\n void lduminab(Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i8Bit, 1, 0, 0, 0b111, rs, rt, rn);\n }\n void ldswpab(Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i8Bit, 1, 0, 1, 0b000, rs, rt, rn);\n }\n void ldaddalb(Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i8Bit, 1, 1, 0, 0b000, rs, rt, rn);\n }\n void ldclralb(Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i8Bit, 1, 1, 0, 0b001, rs, rt, rn);\n }\n void ldeoralb(Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i8Bit, 1, 1, 0, 0b010, rs, rt, rn);\n }\n void ldsetalb(Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i8Bit, 1, 1, 0, 0b011, rs, rt, rn);\n }\n void ldsmaxalb(Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i8Bit, 1, 1, 0, 0b100, rs, rt, rn);\n }\n void ldsminalb(Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i8Bit, 1, 1, 0, 0b101, rs, rt, rn);\n }\n void ldumaxalb(Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i8Bit, 1, 1, 0, 0b110, rs, rt, rn);\n }\n void lduminalb(Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i8Bit, 1, 1, 0, 0b111, rs, rt, rn);\n }\n void ldswpalb(Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i8Bit, 1, 1, 1, 0b000, rs, rt, rn);\n }\n // 16-bit\n void ldaddh(Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i16Bit, 0, 0, 0, 0b000, rs, rt, rn);\n }\n void ldclrh(Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i16Bit, 0, 0, 0, 0b001, rs, rt, rn);\n }\n void ldeorh(Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i16Bit, 0, 0, 0, 0b010, rs, rt, rn);\n }\n void ldseth(Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i16Bit, 0, 0, 0, 0b011, rs, rt, rn);\n }\n void ldsmaxh(Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i16Bit, 0, 0, 0, 0b100, rs, rt, rn);\n }\n void ldsminh(Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i16Bit, 0, 0, 0, 0b101, rs, rt, rn);\n }\n void ldumaxh(Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i16Bit, 0, 0, 0, 0b110, rs, rt, rn);\n }\n void lduminh(Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i16Bit, 0, 0, 0, 0b111, rs, rt, rn);\n }\n void ldswph(Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i16Bit, 0, 0, 1, 0b000, rs, rt, rn);\n }\n void ldaddlh(Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i16Bit, 0, 1, 0, 0b000, rs, rt, rn);\n }\n void ldclrlh(Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i16Bit, 0, 1, 0, 0b001, rs, rt, rn);\n }\n void ldeorlh(Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i16Bit, 0, 1, 0, 0b010, rs, rt, rn);\n }\n void ldsetlh(Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i16Bit, 0, 1, 0, 0b011, rs, rt, rn);\n }\n void ldsmaxlh(Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i16Bit, 0, 1, 0, 0b100, rs, rt, rn);\n }\n void ldsminlh(Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i16Bit, 0, 1, 0, 0b101, rs, rt, rn);\n }\n void ldumaxlh(Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i16Bit, 0, 1, 0, 0b110, rs, rt, rn);\n }\n void lduminlh(Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i16Bit, 0, 1, 0, 0b111, rs, rt, rn);\n }\n void ldswplh(Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i16Bit, 0, 1, 1, 0b000, rs, rt, rn);\n }\n void ldaddah(Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i16Bit, 1, 0, 0, 0b000, rs, rt, rn);\n }\n void ldclrah(Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i16Bit, 1, 0, 0, 0b001, rs, rt, rn);\n }\n void ldeorah(Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i16Bit, 1, 0, 0, 0b010, rs, rt, rn);\n }\n void ldsetah(Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i16Bit, 1, 0, 0, 0b011, rs, rt, rn);\n }\n void ldsmaxah(Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i16Bit, 1, 0, 0, 0b100, rs, rt, rn);\n }\n void ldsminah(Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i16Bit, 1, 0, 0, 0b101, rs, rt, rn);\n }\n void ldumaxah(Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i16Bit, 1, 0, 0, 0b110, rs, rt, rn);\n }\n void lduminah(Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i16Bit, 1, 0, 0, 0b111, rs, rt, rn);\n }\n void ldswpah(Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i16Bit, 1, 0, 1, 0b000, rs, rt, rn);\n }\n void ldaddalh(Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i16Bit, 1, 1, 0, 0b000, rs, rt, rn);\n }\n void ldclralh(Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i16Bit, 1, 1, 0, 0b001, rs, rt, rn);\n }\n void ldeoralh(Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i16Bit, 1, 1, 0, 0b010, rs, rt, rn);\n }\n void ldsetalh(Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i16Bit, 1, 1, 0, 0b011, rs, rt, rn);\n }\n void ldsmaxalh(Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i16Bit, 1, 1, 0, 0b100, rs, rt, rn);\n }\n void ldsminalh(Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i16Bit, 1, 1, 0, 0b101, rs, rt, rn);\n }\n void ldumaxalh(Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i16Bit, 1, 1, 0, 0b110, rs, rt, rn);\n }\n void lduminalh(Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i16Bit, 1, 1, 0, 0b111, rs, rt, rn);\n }\n void ldswpalh(Register rs, Register rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i16Bit, 1, 1, 1, 0b000, rs, rt, rn);\n }\n // 32-bit\n void ldadd(WRegister rs, WRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i32Bit, 0, 0, 0, 0b000, rs, rt, rn);\n }\n void ldclr(WRegister rs, WRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i32Bit, 0, 0, 0, 0b001, rs, rt, rn);\n }\n void ldeor(WRegister rs, WRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i32Bit, 0, 0, 0, 0b010, rs, rt, rn);\n }\n void ldset(WRegister rs, WRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i32Bit, 0, 0, 0, 0b011, rs, rt, rn);\n }\n void ldsmax(WRegister rs, WRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i32Bit, 0, 0, 0, 0b100, rs, rt, rn);\n }\n void ldsmin(WRegister rs, WRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i32Bit, 0, 0, 0, 0b101, rs, rt, rn);\n }\n void ldumax(WRegister rs, WRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i32Bit, 0, 0, 0, 0b110, rs, rt, rn);\n }\n void ldumin(WRegister rs, WRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i32Bit, 0, 0, 0, 0b111, rs, rt, rn);\n }\n void ldswp(WRegister rs, WRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i32Bit, 0, 0, 1, 0b000, rs, rt, rn);\n }\n void ldaddl(WRegister rs, WRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i32Bit, 0, 1, 0, 0b000, rs, rt, rn);\n }\n void ldclrl(WRegister rs, WRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i32Bit, 0, 1, 0, 0b001, rs, rt, rn);\n }\n void ldeorl(WRegister rs, WRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i32Bit, 0, 1, 0, 0b010, rs, rt, rn);\n }\n void ldsetl(WRegister rs, WRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i32Bit, 0, 1, 0, 0b011, rs, rt, rn);\n }\n void ldsmaxl(WRegister rs, WRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i32Bit, 0, 1, 0, 0b100, rs, rt, rn);\n }\n void ldsminl(WRegister rs, WRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i32Bit, 0, 1, 0, 0b101, rs, rt, rn);\n }\n void ldumaxl(WRegister rs, WRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i32Bit, 0, 1, 0, 0b110, rs, rt, rn);\n }\n void lduminl(WRegister rs, WRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i32Bit, 0, 1, 0, 0b111, rs, rt, rn);\n }\n void ldswpl(WRegister rs, WRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i32Bit, 0, 1, 1, 0b000, rs, rt, rn);\n }\n void ldadda(WRegister rs, WRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i32Bit, 1, 0, 0, 0b000, rs, rt, rn);\n }\n void ldclra(WRegister rs, WRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i32Bit, 1, 0, 0, 0b001, rs, rt, rn);\n }\n void ldeora(WRegister rs, WRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i32Bit, 1, 0, 0, 0b010, rs, rt, rn);\n }\n void ldseta(WRegister rs, WRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i32Bit, 1, 0, 0, 0b011, rs, rt, rn);\n }\n void ldsmaxa(WRegister rs, WRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i32Bit, 1, 0, 0, 0b100, rs, rt, rn);\n }\n void ldsmina(WRegister rs, WRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i32Bit, 1, 0, 0, 0b101, rs, rt, rn);\n }\n void ldumaxa(WRegister rs, WRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i32Bit, 1, 0, 0, 0b110, rs, rt, rn);\n }\n void ldumina(WRegister rs, WRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i32Bit, 1, 0, 0, 0b111, rs, rt, rn);\n }\n void ldswpa(WRegister rs, WRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i32Bit, 1, 0, 1, 0b000, rs, rt, rn);\n }\n void ldaddal(WRegister rs, WRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i32Bit, 1, 1, 0, 0b000, rs, rt, rn);\n }\n void ldclral(WRegister rs, WRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i32Bit, 1, 1, 0, 0b001, rs, rt, rn);\n }\n void ldeoral(WRegister rs, WRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i32Bit, 1, 1, 0, 0b010, rs, rt, rn);\n }\n void ldsetal(WRegister rs, WRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i32Bit, 1, 1, 0, 0b011, rs, rt, rn);\n }\n void ldsmaxal(WRegister rs, WRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i32Bit, 1, 1, 0, 0b100, rs, rt, rn);\n }\n void ldsminal(WRegister rs, WRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i32Bit, 1, 1, 0, 0b101, rs, rt, rn);\n }\n void ldumaxal(WRegister rs, WRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i32Bit, 1, 1, 0, 0b110, rs, rt, rn);\n }\n void lduminal(WRegister rs, WRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i32Bit, 1, 1, 0, 0b111, rs, rt, rn);\n }\n void ldswpal(WRegister rs, WRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i32Bit, 1, 1, 1, 0b000, rs, rt, rn);\n }\n // 64-bit\n void ldadd(XRegister rs, XRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i64Bit, 0, 0, 0, 0b000, rs, rt, rn);\n }\n void ldclr(XRegister rs, XRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i64Bit, 0, 0, 0, 0b001, rs, rt, rn);\n }\n void ldeor(XRegister rs, XRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i64Bit, 0, 0, 0, 0b010, rs, rt, rn);\n }\n void ldset(XRegister rs, XRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i64Bit, 0, 0, 0, 0b011, rs, rt, rn);\n }\n void ldsmax(XRegister rs, XRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i64Bit, 0, 0, 0, 0b100, rs, rt, rn);\n }\n void ldsmin(XRegister rs, XRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i64Bit, 0, 0, 0, 0b101, rs, rt, rn);\n }\n void ldumax(XRegister rs, XRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i64Bit, 0, 0, 0, 0b110, rs, rt, rn);\n }\n void ldumin(XRegister rs, XRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i64Bit, 0, 0, 0, 0b111, rs, rt, rn);\n }\n void ldswp(XRegister rs, XRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i64Bit, 0, 0, 1, 0b000, rs, rt, rn);\n }\n void ldaddl(XRegister rs, XRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i64Bit, 0, 1, 0, 0b000, rs, rt, rn);\n }\n void ldclrl(XRegister rs, XRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i64Bit, 0, 1, 0, 0b001, rs, rt, rn);\n }\n void ldeorl(XRegister rs, XRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i64Bit, 0, 1, 0, 0b010, rs, rt, rn);\n }\n void ldsetl(XRegister rs, XRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i64Bit, 0, 1, 0, 0b011, rs, rt, rn);\n }\n void ldsmaxl(XRegister rs, XRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i64Bit, 0, 1, 0, 0b100, rs, rt, rn);\n }\n void ldsminl(XRegister rs, XRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i64Bit, 0, 1, 0, 0b101, rs, rt, rn);\n }\n void ldumaxl(XRegister rs, XRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i64Bit, 0, 1, 0, 0b110, rs, rt, rn);\n }\n void lduminl(XRegister rs, XRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i64Bit, 0, 1, 0, 0b111, rs, rt, rn);\n }\n void ldswpl(XRegister rs, XRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i64Bit, 0, 1, 1, 0b000, rs, rt, rn);\n }\n void ldadda(XRegister rs, XRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i64Bit, 1, 0, 0, 0b000, rs, rt, rn);\n }\n void ldclra(XRegister rs, XRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i64Bit, 1, 0, 0, 0b001, rs, rt, rn);\n }\n void ldeora(XRegister rs, XRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i64Bit, 1, 0, 0, 0b010, rs, rt, rn);\n }\n void ldseta(XRegister rs, XRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i64Bit, 1, 0, 0, 0b011, rs, rt, rn);\n }\n void ldsmaxa(XRegister rs, XRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i64Bit, 1, 0, 0, 0b100, rs, rt, rn);\n }\n void ldsmina(XRegister rs, XRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i64Bit, 1, 0, 0, 0b101, rs, rt, rn);\n }\n void ldumaxa(XRegister rs, XRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i64Bit, 1, 0, 0, 0b110, rs, rt, rn);\n }\n void ldumina(XRegister rs, XRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i64Bit, 1, 0, 0, 0b111, rs, rt, rn);\n }\n void ldswpa(XRegister rs, XRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i64Bit, 1, 0, 1, 0b000, rs, rt, rn);\n }\n void ldaddal(XRegister rs, XRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i64Bit, 1, 1, 0, 0b000, rs, rt, rn);\n }\n void ldclral(XRegister rs, XRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i64Bit, 1, 1, 0, 0b001, rs, rt, rn);\n }\n void ldeoral(XRegister rs, XRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i64Bit, 1, 1, 0, 0b010, rs, rt, rn);\n }\n void ldsetal(XRegister rs, XRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i64Bit, 1, 1, 0, 0b011, rs, rt, rn);\n }\n void ldsmaxal(XRegister rs, XRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i64Bit, 1, 1, 0, 0b100, rs, rt, rn);\n }\n void ldsminal(XRegister rs, XRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i64Bit, 1, 1, 0, 0b101, rs, rt, rn);\n }\n void ldumaxal(XRegister rs, XRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i64Bit, 1, 1, 0, 0b110, rs, rt, rn);\n }\n void lduminal(XRegister rs, XRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i64Bit, 1, 1, 0, 0b111, rs, rt, rn);\n }\n void ldswpal(XRegister rs, XRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i64Bit, 1, 1, 1, 0b000, rs, rt, rn);\n }\n void ldaprb(WRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i8Bit, 1, 0, 1, 0b100, WReg::w31, rt, rn);\n }\n void ldaprh(WRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i16Bit, 1, 0, 1, 0b100, WReg::w31, rt, rn);\n }\n void ldapr(WRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i32Bit, 1, 0, 1, 0b100, WReg::w31, rt, rn);\n }\n void ldapr(XRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i64Bit, 1, 0, 1, 0b100, XReg::x31, rt, rn);\n }\n void st64bv0(XRegister rs, XRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i64Bit, 0, 0, 1, 0b010, rs, rt, rn);\n }\n void st64bv(XRegister rs, XRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i64Bit, 0, 0, 1, 0b011, rs, rt, rn);\n }\n void st64b(XRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i64Bit, 0, 0, 1, 0b001, XReg::x31, rt, rn);\n }\n void ld64b(XRegister rt, Register rn) {\n LoadStoreAtomicLSE(SubRegSize::i64Bit, 0, 0, 1, 0b101, XReg::x31, rt, rn);\n }\n\n // Loadstore register-register offset\n void strb(ARMEmitter::Register rt, ARMEmitter::Register rn, ARMEmitter::Register rm, ARMEmitter::ExtendedType Option, bool Shift = false) {\n LOGMAN_THROW_A_FMT((FEXCore::ToUnderlying(Option) & 0b010) == 0b010, \"Unsupported Extendtype\");\n constexpr uint32_t Op = 0b0011'1000'001 << 21 | (0b10 << 10);\n LoadStoreRegisterOffset(Op, 0b00, rt, rn, rm, Option, Shift ? 1 : 0);\n }\n void ldrb(ARMEmitter::Register rt, ARMEmitter::Register rn, ARMEmitter::Register rm, ARMEmitter::ExtendedType Option, bool Shift = false) {\n LOGMAN_THROW_A_FMT((FEXCore::ToUnderlying(Option) & 0b010) == 0b010, \"Unsupported Extendtype\");\n constexpr uint32_t Op = 0b0011'1000'001 << 21 | (0b10 << 10);\n LoadStoreRegisterOffset(Op, 0b01, rt, rn, rm, Option, Shift ? 1 : 0);\n }\n void ldrsb(ARMEmitter::XRegister rt, ARMEmitter::Register rn, ARMEmitter::Register rm, ARMEmitter::ExtendedType Option, bool Shift = false) {\n LOGMAN_THROW_A_FMT((FEXCore::ToUnderlying(Option) & 0b010) == 0b010, \"Unsupported Extendtype\");\n constexpr uint32_t Op = 0b0011'1000'001 << 21 | (0b10 << 10);\n LoadStoreRegisterOffset(Op, 0b10, rt, rn, rm, Option, Shift ? 1 : 0);\n }\n void ldrsb(ARMEmitter::WRegister rt, ARMEmitter::Register rn, ARMEmitter::Register rm, ARMEmitter::ExtendedType Option, bool Shift = false) {\n LOGMAN_THROW_A_FMT((FEXCore::ToUnderlying(Option) & 0b010) == 0b010, \"Unsupported Extendtype\");\n constexpr uint32_t Op = 0b0011'1000'001 << 21 | (0b10 << 10);\n LoadStoreRegisterOffset(Op, 0b11, rt, rn, rm, Option, Shift ? 1 : 0);\n }\n void strh(ARMEmitter::Register rt, ARMEmitter::Register rn, ARMEmitter::Register rm, ARMEmitter::ExtendedType Option, uint32_t Shift) {\n LOGMAN_THROW_A_FMT((FEXCore::ToUnderlying(Option) & 0b010) == 0b010, \"Unsupported Extendtype\");\n LOGMAN_THROW_A_FMT(Shift == 0 || Shift == 1, \"Unsupported shift amount\");\n constexpr uint32_t Op = 0b0111'1000'001 << 21 | (0b10 << 10);\n LoadStoreRegisterOffset(Op, 0b00, rt, rn, rm, Option, Shift ? 1 : 0);\n }\n void ldrh(ARMEmitter::Register rt, ARMEmitter::Register rn, ARMEmitter::Register rm, ARMEmitter::ExtendedType Option, uint32_t Shift) {\n LOGMAN_THROW_A_FMT((FEXCore::ToUnderlying(Option) & 0b010) == 0b010, \"Unsupported Extendtype\");\n LOGMAN_THROW_A_FMT(Shift == 0 || Shift == 1, \"Unsupported shift amount\");\n constexpr uint32_t Op = 0b0111'1000'001 << 21 | (0b10 << 10);\n LoadStoreRegisterOffset(Op, 0b01, rt, rn, rm, Option, Shift ? 1 : 0);\n }\n void ldrsh(ARMEmitter::XRegister rt, ARMEmitter::Register rn, ARMEmitter::Register rm, ARMEmitter::ExtendedType Option, uint32_t Shift) {\n LOGMAN_THROW_A_FMT((FEXCore::ToUnderlying(Option) & 0b010) == 0b010, \"Unsupported Extendtype\");\n LOGMAN_THROW_A_FMT(Shift == 0 || Shift == 1, \"Unsupported shift amount\");\n constexpr uint32_t Op = 0b0111'1000'001 << 21 | (0b10 << 10);\n LoadStoreRegisterOffset(Op, 0b10, rt, rn, rm, Option, Shift ? 1 : 0);\n }\n void ldrsh(ARMEmitter::WRegister rt, ARMEmitter::Register rn, ARMEmitter::Register rm, ARMEmitter::ExtendedType Option, uint32_t Shift) {\n LOGMAN_THROW_A_FMT((FEXCore::ToUnderlying(Option) & 0b010) == 0b010, \"Unsupported Extendtype\");\n LOGMAN_THROW_A_FMT(Shift == 0 || Shift == 1, \"Unsupported shift amount\");\n constexpr uint32_t Op = 0b0111'1000'001 << 21 | (0b10 << 10);\n LoadStoreRegisterOffset(Op, 0b11, rt, rn, rm, Option, Shift ? 1 : 0);\n }\n void str(ARMEmitter::WRegister rt, ARMEmitter::Register rn, ARMEmitter::Register rm, ARMEmitter::ExtendedType Option, uint32_t Shift) {\n LOGMAN_THROW_A_FMT((FEXCore::ToUnderlying(Option) & 0b010) == 0b010, \"Unsupported Extendtype\");\n LOGMAN_THROW_A_FMT(Shift == 0 || Shift == 2, \"Unsupported shift amount\");\n constexpr uint32_t Op = 0b1011'1000'001 << 21 | (0b10 << 10);\n LoadStoreRegisterOffset(Op, 0b00, rt, rn, rm, Option, Shift ? 1 : 0);\n }\n void ldr(ARMEmitter::WRegister rt, ARMEmitter::Register rn, ARMEmitter::Register rm, ARMEmitter::ExtendedType Option, uint32_t Shift) {\n LOGMAN_THROW_A_FMT((FEXCore::ToUnderlying(Option) & 0b010) == 0b010, \"Unsupported Extendtype\");\n LOGMAN_THROW_A_FMT(Shift == 0 || Shift == 2, \"Unsupported shift amount: {}\", Shift);\n constexpr uint32_t Op = 0b1011'1000'001 << 21 | (0b10 << 10);\n LoadStoreRegisterOffset(Op, 0b01, rt, rn, rm, Option, Shift ? 1 : 0);\n }\n void ldrsw(ARMEmitter::XRegister rt, ARMEmitter::Register rn, ARMEmitter::Register rm, ARMEmitter::ExtendedType Option, uint32_t Shift) {\n LOGMAN_THROW_A_FMT((FEXCore::ToUnderlying(Option) & 0b010) == 0b010, \"Unsupported Extendtype\");\n LOGMAN_THROW_A_FMT(Shift == 0 || Shift == 2, \"Unsupported shift amount\");\n constexpr uint32_t Op = 0b1011'1000'001 << 21 | (0b10 << 10);\n LoadStoreRegisterOffset(Op, 0b10, rt, rn, rm, Option, Shift ? 1 : 0);\n }\n void str(ARMEmitter::XRegister rt, ARMEmitter::Register rn, ARMEmitter::Register rm, ARMEmitter::ExtendedType Option, uint32_t Shift) {\n LOGMAN_THROW_A_FMT((FEXCore::ToUnderlying(Option) & 0b010) == 0b010, \"Unsupported Extendtype\");\n LOGMAN_THROW_A_FMT(Shift == 0 || Shift == 3, \"Unsupported shift amount\");\n constexpr uint32_t Op = 0b1111'1000'001 << 21 | (0b10 << 10);\n LoadStoreRegisterOffset(Op, 0b00, rt, rn, rm, Option, Shift ? 1 : 0);\n }\n void ldr(ARMEmitter::XRegister rt, ARMEmitter::Register rn, ARMEmitter::Register rm, ARMEmitter::ExtendedType Option, uint32_t Shift) {\n LOGMAN_THROW_A_FMT((FEXCore::ToUnderlying(Option) & 0b010) == 0b010, \"Unsupported Extendtype\");\n LOGMAN_THROW_A_FMT(Shift == 0 || Shift == 3, \"Unsupported shift amount\");\n constexpr uint32_t Op = 0b1111'1000'001 << 21 | (0b10 << 10);\n LoadStoreRegisterOffset(Op, 0b01, rt, rn, rm, Option, Shift ? 1 : 0);\n }\n void prfm(ARMEmitter::Prefetch prfop, ARMEmitter::Register rn, ARMEmitter::Register rm, ARMEmitter::ExtendedType Option, uint32_t Shift) {\n LOGMAN_THROW_A_FMT((FEXCore::ToUnderlying(Option) & 0b010) == 0b010, \"Unsupported Extendtype\");\n LOGMAN_THROW_A_FMT(Shift == 0 || Shift == 3, \"Unsupported shift amount\");\n constexpr uint32_t Op = 0b1111'1000'001 << 21 | (0b10 << 10);\n LoadStoreRegisterOffset(Op, 0b10, prfop, rn, rm, Option, Shift ? 1 : 0);\n }\n void strb(ARMEmitter::VRegister rt, ARMEmitter::Register rn, ARMEmitter::Register rm, ARMEmitter::ExtendedType Option) {\n LOGMAN_THROW_A_FMT((FEXCore::ToUnderlying(Option) & 0b010) == 0b010, \"Unsupported Extendtype\");\n constexpr uint32_t Op = 0b0011'1100'001 << 21 | (0b10 << 10);\n LoadStoreRegisterOffset(Op, 0b00, rt, rn, rm, Option, 0);\n }\n void ldrb(ARMEmitter::VRegister rt, ARMEmitter::Register rn, ARMEmitter::Register rm, ARMEmitter::ExtendedType Option) {\n LOGMAN_THROW_A_FMT((FEXCore::ToUnderlying(Option) & 0b010) == 0b010, \"Unsupported Extendtype\");\n constexpr uint32_t Op = 0b0011'1100'001 << 21 | (0b10 << 10);\n LoadStoreRegisterOffset(Op, 0b01, rt, rn, rm, Option, 0);\n }\n void strh(ARMEmitter::VRegister rt, ARMEmitter::Register rn, ARMEmitter::Register rm, ARMEmitter::ExtendedType Option, uint32_t Shift) {\n LOGMAN_THROW_A_FMT((FEXCore::ToUnderlying(Option) & 0b010) == 0b010, \"Unsupported Extendtype\");\n LOGMAN_THROW_A_FMT(Shift == 0 || Shift == 1, \"Unsupported shift amount\");\n constexpr uint32_t Op = 0b0111'1100'001 << 21 | (0b10 << 10);\n LoadStoreRegisterOffset(Op, 0b00, rt, rn, rm, Option, Shift ? 1 : 0);\n }\n void ldrh(ARMEmitter::VRegister rt, ARMEmitter::Register rn, ARMEmitter::Register rm, ARMEmitter::ExtendedType Option, uint32_t Shift) {\n LOGMAN_THROW_A_FMT((FEXCore::ToUnderlying(Option) & 0b010) == 0b010, \"Unsupported Extendtype\");\n LOGMAN_THROW_A_FMT(Shift == 0 || Shift == 1, \"Unsupported shift amount\");\n constexpr uint32_t Op = 0b0111'1100'001 << 21 | (0b10 << 10);\n LoadStoreRegisterOffset(Op, 0b01, rt, rn, rm, Option, Shift ? 1 : 0);\n }\n void str(ARMEmitter::SRegister rt, ARMEmitter::Register rn, ARMEmitter::Register rm, ARMEmitter::ExtendedType Option, uint32_t Shift) {\n LOGMAN_THROW_A_FMT((FEXCore::ToUnderlying(Option) & 0b010) == 0b010, \"Unsupported Extendtype\");\n LOGMAN_THROW_A_FMT(Shift == 0 || Shift == 2, \"Unsupported shift amount\");\n constexpr uint32_t Op = 0b1011'1100'001 << 21 | (0b10 << 10);\n LoadStoreRegisterOffset(Op, 0b00, rt, rn, rm, Option, Shift ? 1 : 0);\n }\n void ldr(ARMEmitter::SRegister rt, ARMEmitter::Register rn, ARMEmitter::Register rm, ARMEmitter::ExtendedType Option, uint32_t Shift) {\n LOGMAN_THROW_A_FMT((FEXCore::ToUnderlying(Option) & 0b010) == 0b010, \"Unsupported Extendtype\");\n LOGMAN_THROW_A_FMT(Shift == 0 || Shift == 2, \"Unsupported shift amount\");\n constexpr uint32_t Op = 0b1011'1100'001 << 21 | (0b10 << 10);\n LoadStoreRegisterOffset(Op, 0b01, rt, rn, rm, Option, Shift ? 1 : 0);\n }\n void str(ARMEmitter::DRegister rt, ARMEmitter::Register rn, ARMEmitter::Register rm, ARMEmitter::ExtendedType Option, uint32_t Shift) {\n LOGMAN_THROW_A_FMT((FEXCore::ToUnderlying(Option) & 0b010) == 0b010, \"Unsupported Extendtype\");\n LOGMAN_THROW_A_FMT(Shift == 0 || Shift == 3, \"Unsupported shift amount\");\n constexpr uint32_t Op = 0b1111'1100'001 << 21 | (0b10 << 10);\n LoadStoreRegisterOffset(Op, 0b00, rt, rn, rm, Option, Shift ? 1 : 0);\n }\n void ldr(ARMEmitter::DRegister rt, ARMEmitter::Register rn, ARMEmitter::Register rm, ARMEmitter::ExtendedType Option, uint32_t Shift) {\n LOGMAN_THROW_A_FMT((FEXCore::ToUnderlying(Option) & 0b010) == 0b010, \"Unsupported Extendtype\");\n LOGMAN_THROW_A_FMT(Shift == 0 || Shift == 3, \"Unsupported shift amount\");\n constexpr uint32_t Op = 0b1111'1100'001 << 21 | (0b10 << 10);\n LoadStoreRegisterOffset(Op, 0b01, rt, rn, rm, Option, Shift ? 1 : 0);\n }\n void str(ARMEmitter::QRegister rt, ARMEmitter::Register rn, ARMEmitter::Register rm, ARMEmitter::ExtendedType Option, uint32_t Shift) {\n LOGMAN_THROW_A_FMT((FEXCore::ToUnderlying(Option) & 0b010) == 0b010, \"Unsupported Extendtype\");\n LOGMAN_THROW_A_FMT(Shift == 0 || Shift == 4, \"Unsupported shift amount\");\n constexpr uint32_t Op = 0b0011'1100'001 << 21 | (0b10 << 10);\n LoadStoreRegisterOffset(Op, 0b10, rt, rn, rm, Option, Shift ? 1 : 0);\n }\n void ldr(ARMEmitter::QRegister rt, ARMEmitter::Register rn, ARMEmitter::Register rm, ARMEmitter::ExtendedType Option, uint32_t Shift) {\n LOGMAN_THROW_A_FMT((FEXCore::ToUnderlying(Option) & 0b010) == 0b010, \"Unsupported Extendtype\");\n LOGMAN_THROW_A_FMT(Shift == 0 || Shift == 4, \"Unsupported shift amount\");\n constexpr uint32_t Op = 0b0011'1100'001 << 21 | (0b10 << 10);\n LoadStoreRegisterOffset(Op, 0b11, rt, rn, rm, Option, Shift ? 1 : 0);\n }\n\n void strb(ARMEmitter::Register rt, ARMEmitter::ExtendedMemOperand MemSrc) {\n if (MemSrc.MetaType.Header.MemType == ARMEmitter::ExtendedMemOperand::Type::TYPE_EXTENDED &&\n MemSrc.MetaType.Extended.rm.Idx() != ARMEmitter::Reg::r31.Idx()) {\n strb(rt, MemSrc.rn, MemSrc.MetaType.Extended.rm, MemSrc.MetaType.Extended.Option, MemSrc.MetaType.Extended.Shift);\n } else if (MemSrc.MetaType.Header.MemType == ARMEmitter::ExtendedMemOperand::Type::TYPE_EXTENDED) {\n strb(rt, MemSrc.rn);\n } else {\n if (MemSrc.MetaType.ImmType.Index == ARMEmitter::IndexType::OFFSET) {\n if (MemSrc.MetaType.ImmType.Imm < 0) {\n sturb(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n } else {\n strb(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n }\n } else if (MemSrc.MetaType.ImmType.Index == ARMEmitter::IndexType::POST) {\n strb(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n } else if (MemSrc.MetaType.ImmType.Index == ARMEmitter::IndexType::PRE) {\n strb(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n } else {\n LOGMAN_MSG_A_FMT(\"Unexpected loadstore index type\");\n FEX_UNREACHABLE;\n }\n }\n }\n void ldrb(ARMEmitter::Register rt, ARMEmitter::ExtendedMemOperand MemSrc) {\n if (MemSrc.MetaType.Header.MemType == ARMEmitter::ExtendedMemOperand::Type::TYPE_EXTENDED &&\n MemSrc.MetaType.Extended.rm.Idx() != ARMEmitter::Reg::r31.Idx()) {\n ldrb(rt, MemSrc.rn, MemSrc.MetaType.Extended.rm, MemSrc.MetaType.Extended.Option, MemSrc.MetaType.Extended.Shift);\n } else if (MemSrc.MetaType.Header.MemType == ARMEmitter::ExtendedMemOperand::Type::TYPE_EXTENDED) {\n ldrb(rt, MemSrc.rn);\n } else {\n if (MemSrc.MetaType.ImmType.Index == ARMEmitter::IndexType::OFFSET) {\n if (MemSrc.MetaType.ImmType.Imm < 0) {\n ldurb(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n } else {\n ldrb(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n }\n } else if (MemSrc.MetaType.ImmType.Index == ARMEmitter::IndexType::POST) {\n ldrb(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n } else if (MemSrc.MetaType.ImmType.Index == ARMEmitter::IndexType::PRE) {\n ldrb(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n } else {\n LOGMAN_MSG_A_FMT(\"Unexpected loadstore index type\");\n FEX_UNREACHABLE;\n }\n }\n }\n void ldrsb(ARMEmitter::XRegister rt, ARMEmitter::ExtendedMemOperand MemSrc) {\n if (MemSrc.MetaType.Header.MemType == ARMEmitter::ExtendedMemOperand::Type::TYPE_EXTENDED &&\n MemSrc.MetaType.Extended.rm.Idx() != ARMEmitter::Reg::r31.Idx()) {\n ldrsb(rt, MemSrc.rn, MemSrc.MetaType.Extended.rm, MemSrc.MetaType.Extended.Option, MemSrc.MetaType.Extended.Shift);\n } else if (MemSrc.MetaType.Header.MemType == ARMEmitter::ExtendedMemOperand::Type::TYPE_EXTENDED) {\n ldrsb(rt, MemSrc.rn);\n } else {\n if (MemSrc.MetaType.ImmType.Index == ARMEmitter::IndexType::OFFSET) {\n if (MemSrc.MetaType.ImmType.Imm < 0) {\n ldursb(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n } else {\n ldrsb(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n }\n } else if (MemSrc.MetaType.ImmType.Index == ARMEmitter::IndexType::POST) {\n ldrsb(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n } else if (MemSrc.MetaType.ImmType.Index == ARMEmitter::IndexType::PRE) {\n ldrsb(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n } else {\n LOGMAN_MSG_A_FMT(\"Unexpected loadstore index type\");\n FEX_UNREACHABLE;\n }\n }\n }\n void ldrsb(ARMEmitter::WRegister rt, ARMEmitter::ExtendedMemOperand MemSrc) {\n if (MemSrc.MetaType.Header.MemType == ARMEmitter::ExtendedMemOperand::Type::TYPE_EXTENDED &&\n MemSrc.MetaType.Extended.rm.Idx() != ARMEmitter::Reg::r31.Idx()) {\n ldrsb(rt, MemSrc.rn, MemSrc.MetaType.Extended.rm, MemSrc.MetaType.Extended.Option, MemSrc.MetaType.Extended.Shift);\n } else if (MemSrc.MetaType.Header.MemType == ARMEmitter::ExtendedMemOperand::Type::TYPE_EXTENDED) {\n ldrsb(rt, MemSrc.rn);\n } else {\n if (MemSrc.MetaType.ImmType.Index == ARMEmitter::IndexType::OFFSET) {\n if (MemSrc.MetaType.ImmType.Imm < 0) {\n ldursb(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n } else {\n ldrsb(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n }\n } else if (MemSrc.MetaType.ImmType.Index == ARMEmitter::IndexType::POST) {\n ldrsb(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n } else if (MemSrc.MetaType.ImmType.Index == ARMEmitter::IndexType::PRE) {\n ldrsb(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n } else {\n LOGMAN_MSG_A_FMT(\"Unexpected loadstore index type\");\n FEX_UNREACHABLE;\n }\n }\n }\n void strh(ARMEmitter::Register rt, ARMEmitter::ExtendedMemOperand MemSrc) {\n if (MemSrc.MetaType.Header.MemType == ARMEmitter::ExtendedMemOperand::Type::TYPE_EXTENDED &&\n MemSrc.MetaType.Extended.rm.Idx() != ARMEmitter::Reg::r31.Idx()) {\n strh(rt, MemSrc.rn, MemSrc.MetaType.Extended.rm, MemSrc.MetaType.Extended.Option, MemSrc.MetaType.Extended.Shift);\n } else if (MemSrc.MetaType.Header.MemType == ARMEmitter::ExtendedMemOperand::Type::TYPE_EXTENDED) {\n strh(rt, MemSrc.rn);\n } else {\n if (MemSrc.MetaType.ImmType.Index == ARMEmitter::IndexType::OFFSET) {\n if ((MemSrc.MetaType.ImmType.Imm & 0b1) || MemSrc.MetaType.ImmType.Imm < 0) {\n sturh(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n } else {\n strh(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n }\n } else if (MemSrc.MetaType.ImmType.Index == ARMEmitter::IndexType::POST) {\n strh(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n } else if (MemSrc.MetaType.ImmType.Index == ARMEmitter::IndexType::PRE) {\n strh(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n } else {\n LOGMAN_MSG_A_FMT(\"Unexpected loadstore index type\");\n FEX_UNREACHABLE;\n }\n }\n }\n void ldrh(ARMEmitter::Register rt, ARMEmitter::ExtendedMemOperand MemSrc) {\n if (MemSrc.MetaType.Header.MemType == ARMEmitter::ExtendedMemOperand::Type::TYPE_EXTENDED &&\n MemSrc.MetaType.Extended.rm.Idx() != ARMEmitter::Reg::r31.Idx()) {\n ldrh(rt, MemSrc.rn, MemSrc.MetaType.Extended.rm, MemSrc.MetaType.Extended.Option, MemSrc.MetaType.Extended.Shift);\n } else if (MemSrc.MetaType.Header.MemType == ARMEmitter::ExtendedMemOperand::Type::TYPE_EXTENDED) {\n ldrh(rt, MemSrc.rn);\n } else {\n if (MemSrc.MetaType.ImmType.Index == ARMEmitter::IndexType::OFFSET) {\n if ((MemSrc.MetaType.ImmType.Imm & 0b1) || MemSrc.MetaType.ImmType.Imm < 0) {\n ldurh(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n } else {\n ldrh(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n }\n } else if (MemSrc.MetaType.ImmType.Index == ARMEmitter::IndexType::POST) {\n ldrh(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n } else if (MemSrc.MetaType.ImmType.Index == ARMEmitter::IndexType::PRE) {\n ldrh(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n } else {\n LOGMAN_MSG_A_FMT(\"Unexpected loadstore index type\");\n FEX_UNREACHABLE;\n }\n }\n }\n void ldrsh(ARMEmitter::XRegister rt, ARMEmitter::ExtendedMemOperand MemSrc) {\n if (MemSrc.MetaType.Header.MemType == ARMEmitter::ExtendedMemOperand::Type::TYPE_EXTENDED &&\n MemSrc.MetaType.Extended.rm.Idx() != ARMEmitter::Reg::r31.Idx()) {\n ldrsh(rt, MemSrc.rn, MemSrc.MetaType.Extended.rm, MemSrc.MetaType.Extended.Option, MemSrc.MetaType.Extended.Shift);\n } else if (MemSrc.MetaType.Header.MemType == ARMEmitter::ExtendedMemOperand::Type::TYPE_EXTENDED) {\n ldrsh(rt, MemSrc.rn);\n } else {\n if (MemSrc.MetaType.ImmType.Index == ARMEmitter::IndexType::OFFSET) {\n if ((MemSrc.MetaType.ImmType.Imm & 0b1) || MemSrc.MetaType.ImmType.Imm < 0) {\n ldursh(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n } else {\n ldrsh(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n }\n } else if (MemSrc.MetaType.ImmType.Index == ARMEmitter::IndexType::POST) {\n ldrsh(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n } else if (MemSrc.MetaType.ImmType.Index == ARMEmitter::IndexType::PRE) {\n ldrsh(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n } else {\n LOGMAN_MSG_A_FMT(\"Unexpected loadstore index type\");\n FEX_UNREACHABLE;\n }\n }\n }\n void ldrsh(ARMEmitter::WRegister rt, ARMEmitter::ExtendedMemOperand MemSrc) {\n if (MemSrc.MetaType.Header.MemType == ARMEmitter::ExtendedMemOperand::Type::TYPE_EXTENDED &&\n MemSrc.MetaType.Extended.rm.Idx() != ARMEmitter::Reg::r31.Idx()) {\n ldrsh(rt, MemSrc.rn, MemSrc.MetaType.Extended.rm, MemSrc.MetaType.Extended.Option, MemSrc.MetaType.Extended.Shift);\n } else if (MemSrc.MetaType.Header.MemType == ARMEmitter::ExtendedMemOperand::Type::TYPE_EXTENDED) {\n ldrsh(rt, MemSrc.rn);\n } else {\n if (MemSrc.MetaType.ImmType.Index == ARMEmitter::IndexType::OFFSET) {\n if ((MemSrc.MetaType.ImmType.Imm & 0b1) || MemSrc.MetaType.ImmType.Imm < 0) {\n ldursh(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n } else {\n ldrsh(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n }\n } else if (MemSrc.MetaType.ImmType.Index == ARMEmitter::IndexType::POST) {\n ldrsh(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n } else if (MemSrc.MetaType.ImmType.Index == ARMEmitter::IndexType::PRE) {\n ldrsh(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n } else {\n LOGMAN_MSG_A_FMT(\"Unexpected loadstore index type\");\n FEX_UNREACHABLE;\n }\n }\n }\n void str(ARMEmitter::WRegister rt, ARMEmitter::ExtendedMemOperand MemSrc) {\n if (MemSrc.MetaType.Header.MemType == ARMEmitter::ExtendedMemOperand::Type::TYPE_EXTENDED &&\n MemSrc.MetaType.Extended.rm.Idx() != ARMEmitter::Reg::r31.Idx()) {\n str(rt, MemSrc.rn, MemSrc.MetaType.Extended.rm, MemSrc.MetaType.Extended.Option, MemSrc.MetaType.Extended.Shift);\n } else if (MemSrc.MetaType.Header.MemType == ARMEmitter::ExtendedMemOperand::Type::TYPE_EXTENDED) {\n str(rt, MemSrc.rn);\n } else {\n if (MemSrc.MetaType.ImmType.Index == ARMEmitter::IndexType::OFFSET) {\n if ((MemSrc.MetaType.ImmType.Imm & 0b11) || MemSrc.MetaType.ImmType.Imm < 0) {\n stur(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n } else {\n str(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n }\n } else if (MemSrc.MetaType.ImmType.Index == ARMEmitter::IndexType::POST) {\n str(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n } else if (MemSrc.MetaType.ImmType.Index == ARMEmitter::IndexType::PRE) {\n str(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n } else {\n LOGMAN_MSG_A_FMT(\"Unexpected loadstore index type\");\n FEX_UNREACHABLE;\n }\n }\n }\n void ldr(ARMEmitter::WRegister rt, ARMEmitter::ExtendedMemOperand MemSrc) {\n if (MemSrc.MetaType.Header.MemType == ARMEmitter::ExtendedMemOperand::Type::TYPE_EXTENDED &&\n MemSrc.MetaType.Extended.rm.Idx() != ARMEmitter::Reg::r31.Idx()) {\n ldr(rt, MemSrc.rn, MemSrc.MetaType.Extended.rm, MemSrc.MetaType.Extended.Option, MemSrc.MetaType.Extended.Shift);\n } else if (MemSrc.MetaType.Header.MemType == ARMEmitter::ExtendedMemOperand::Type::TYPE_EXTENDED) {\n ldr(rt, MemSrc.rn);\n } else {\n if (MemSrc.MetaType.ImmType.Index == ARMEmitter::IndexType::OFFSET) {\n if ((MemSrc.MetaType.ImmType.Imm & 0b11) || MemSrc.MetaType.ImmType.Imm < 0) {\n ldur(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n } else {\n ldr(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n }\n } else if (MemSrc.MetaType.ImmType.Index == ARMEmitter::IndexType::POST) {\n ldr(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n } else if (MemSrc.MetaType.ImmType.Index == ARMEmitter::IndexType::PRE) {\n ldr(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n } else {\n LOGMAN_MSG_A_FMT(\"Unexpected loadstore index type\");\n FEX_UNREACHABLE;\n }\n }\n }\n void ldrsw(ARMEmitter::XRegister rt, ARMEmitter::ExtendedMemOperand MemSrc) {\n if (MemSrc.MetaType.Header.MemType == ARMEmitter::ExtendedMemOperand::Type::TYPE_EXTENDED &&\n MemSrc.MetaType.Extended.rm.Idx() != ARMEmitter::Reg::r31.Idx()) {\n ldrsw(rt, MemSrc.rn, MemSrc.MetaType.Extended.rm, MemSrc.MetaType.Extended.Option, MemSrc.MetaType.Extended.Shift);\n } else if (MemSrc.MetaType.Header.MemType == ARMEmitter::ExtendedMemOperand::Type::TYPE_EXTENDED) {\n ldrsw(rt, MemSrc.rn);\n } else {\n if (MemSrc.MetaType.ImmType.Index == ARMEmitter::IndexType::OFFSET) {\n if ((MemSrc.MetaType.ImmType.Imm & 0b11) || MemSrc.MetaType.ImmType.Imm < 0) {\n ldursw(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n } else {\n ldrsw(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n }\n } else if (MemSrc.MetaType.ImmType.Index == ARMEmitter::IndexType::POST) {\n ldrsw(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n } else if (MemSrc.MetaType.ImmType.Index == ARMEmitter::IndexType::PRE) {\n ldrsw(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n } else {\n LOGMAN_MSG_A_FMT(\"Unexpected loadstore index type\");\n FEX_UNREACHABLE;\n }\n }\n }\n void str(ARMEmitter::XRegister rt, ARMEmitter::ExtendedMemOperand MemSrc) {\n if (MemSrc.MetaType.Header.MemType == ARMEmitter::ExtendedMemOperand::Type::TYPE_EXTENDED &&\n MemSrc.MetaType.Extended.rm.Idx() != ARMEmitter::Reg::r31.Idx()) {\n str(rt, MemSrc.rn, MemSrc.MetaType.Extended.rm, MemSrc.MetaType.Extended.Option, MemSrc.MetaType.Extended.Shift);\n } else if (MemSrc.MetaType.Header.MemType == ARMEmitter::ExtendedMemOperand::Type::TYPE_EXTENDED) {\n str(rt, MemSrc.rn);\n } else {\n if (MemSrc.MetaType.ImmType.Index == ARMEmitter::IndexType::OFFSET) {\n if ((MemSrc.MetaType.ImmType.Imm & 0b111) || MemSrc.MetaType.ImmType.Imm < 0) {\n stur(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n } else {\n str(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n }\n } else if (MemSrc.MetaType.ImmType.Index == ARMEmitter::IndexType::POST) {\n str(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n } else if (MemSrc.MetaType.ImmType.Index == ARMEmitter::IndexType::PRE) {\n str(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n } else {\n LOGMAN_MSG_A_FMT(\"Unexpected loadstore index type\");\n FEX_UNREACHABLE;\n }\n }\n }\n void ldr(ARMEmitter::XRegister rt, ARMEmitter::ExtendedMemOperand MemSrc) {\n if (MemSrc.MetaType.Header.MemType == ARMEmitter::ExtendedMemOperand::Type::TYPE_EXTENDED &&\n MemSrc.MetaType.Extended.rm.Idx() != ARMEmitter::Reg::r31.Idx()) {\n ldr(rt, MemSrc.rn, MemSrc.MetaType.Extended.rm, MemSrc.MetaType.Extended.Option, MemSrc.MetaType.Extended.Shift);\n } else if (MemSrc.MetaType.Header.MemType == ARMEmitter::ExtendedMemOperand::Type::TYPE_EXTENDED) {\n ldr(rt, MemSrc.rn);\n } else {\n if (MemSrc.MetaType.ImmType.Index == ARMEmitter::IndexType::OFFSET) {\n if ((MemSrc.MetaType.ImmType.Imm & 0b111) || MemSrc.MetaType.ImmType.Imm < 0) {\n ldur(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n } else {\n ldr(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n }\n } else if (MemSrc.MetaType.ImmType.Index == ARMEmitter::IndexType::POST) {\n ldr(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n } else if (MemSrc.MetaType.ImmType.Index == ARMEmitter::IndexType::PRE) {\n ldr(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n } else {\n LOGMAN_MSG_A_FMT(\"Unexpected loadstore index type\");\n FEX_UNREACHABLE;\n }\n }\n }\n void prfm(ARMEmitter::Prefetch prfop, ARMEmitter::ExtendedMemOperand MemSrc) {\n if (MemSrc.MetaType.Header.MemType == ARMEmitter::ExtendedMemOperand::Type::TYPE_EXTENDED &&\n MemSrc.MetaType.Extended.rm.Idx() != ARMEmitter::Reg::r31.Idx()) {\n prfm(prfop, MemSrc.rn, MemSrc.MetaType.Extended.rm, MemSrc.MetaType.Extended.Option, MemSrc.MetaType.Extended.Shift);\n } else if (MemSrc.MetaType.Header.MemType == ARMEmitter::ExtendedMemOperand::Type::TYPE_EXTENDED) {\n prfm(prfop, MemSrc.rn);\n } else {\n if (MemSrc.MetaType.ImmType.Index == ARMEmitter::IndexType::OFFSET) {\n if ((MemSrc.MetaType.ImmType.Imm & 0b111) || MemSrc.MetaType.ImmType.Imm < 0) {\n prfum(prfop, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n } else {\n prfm(prfop, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n }\n } else {\n LOGMAN_MSG_A_FMT(\"Unexpected loadstore index type\");\n FEX_UNREACHABLE;\n }\n }\n }\n\n void strb(ARMEmitter::VRegister rt, ARMEmitter::ExtendedMemOperand MemSrc) {\n if (MemSrc.MetaType.Header.MemType == ARMEmitter::ExtendedMemOperand::Type::TYPE_EXTENDED &&\n MemSrc.MetaType.Extended.rm.Idx() != ARMEmitter::Reg::r31.Idx()) {\n LOGMAN_THROW_A_FMT(MemSrc.MetaType.Extended.Shift == false, \"Can't shift byte\");\n strb(rt, MemSrc.rn, MemSrc.MetaType.Extended.rm, MemSrc.MetaType.Extended.Option);\n } else if (MemSrc.MetaType.Header.MemType == ARMEmitter::ExtendedMemOperand::Type::TYPE_EXTENDED) {\n strb(rt, MemSrc.rn);\n } else {\n if (MemSrc.MetaType.ImmType.Index == ARMEmitter::IndexType::OFFSET) {\n if (MemSrc.MetaType.ImmType.Imm < 0) {\n sturb(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n } else {\n strb(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n }\n } else if (MemSrc.MetaType.ImmType.Index == ARMEmitter::IndexType::POST) {\n strb(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n } else if (MemSrc.MetaType.ImmType.Index == ARMEmitter::IndexType::PRE) {\n strb(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n } else {\n LOGMAN_MSG_A_FMT(\"Unexpected loadstore index type\");\n FEX_UNREACHABLE;\n }\n }\n }\n void ldrb(ARMEmitter::VRegister rt, ARMEmitter::ExtendedMemOperand MemSrc) {\n if (MemSrc.MetaType.Header.MemType == ARMEmitter::ExtendedMemOperand::Type::TYPE_EXTENDED &&\n MemSrc.MetaType.Extended.rm.Idx() != ARMEmitter::Reg::r31.Idx()) {\n LOGMAN_THROW_A_FMT(MemSrc.MetaType.Extended.Shift == false, \"Can't shift byte\");\n ldrb(rt, MemSrc.rn, MemSrc.MetaType.Extended.rm, MemSrc.MetaType.Extended.Option);\n } else if (MemSrc.MetaType.Header.MemType == ARMEmitter::ExtendedMemOperand::Type::TYPE_EXTENDED) {\n ldrb(rt, MemSrc.rn);\n } else {\n if (MemSrc.MetaType.ImmType.Index == ARMEmitter::IndexType::OFFSET) {\n if (MemSrc.MetaType.ImmType.Imm < 0) {\n ldurb(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n } else {\n ldrb(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n }\n } else if (MemSrc.MetaType.ImmType.Index == ARMEmitter::IndexType::POST) {\n ldrb(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n } else if (MemSrc.MetaType.ImmType.Index == ARMEmitter::IndexType::PRE) {\n ldrb(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n } else {\n LOGMAN_MSG_A_FMT(\"Unexpected loadstore index type\");\n FEX_UNREACHABLE;\n }\n }\n }\n void strh(ARMEmitter::VRegister rt, ARMEmitter::ExtendedMemOperand MemSrc) {\n if (MemSrc.MetaType.Header.MemType == ARMEmitter::ExtendedMemOperand::Type::TYPE_EXTENDED &&\n MemSrc.MetaType.Extended.rm.Idx() != ARMEmitter::Reg::r31.Idx()) {\n strh(rt, MemSrc.rn, MemSrc.MetaType.Extended.rm, MemSrc.MetaType.Extended.Option, MemSrc.MetaType.Extended.Shift);\n } else if (MemSrc.MetaType.Header.MemType == ARMEmitter::ExtendedMemOperand::Type::TYPE_EXTENDED) {\n strh(rt, MemSrc.rn);\n } else {\n if (MemSrc.MetaType.ImmType.Index == ARMEmitter::IndexType::OFFSET) {\n if ((MemSrc.MetaType.ImmType.Imm & 0b1) || MemSrc.MetaType.ImmType.Imm < 0) {\n sturh(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n } else {\n strh(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n }\n } else if (MemSrc.MetaType.ImmType.Index == ARMEmitter::IndexType::POST) {\n strh(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n } else if (MemSrc.MetaType.ImmType.Index == ARMEmitter::IndexType::PRE) {\n strh(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n } else {\n LOGMAN_MSG_A_FMT(\"Unexpected loadstore index type\");\n FEX_UNREACHABLE;\n }\n }\n }\n void ldrh(ARMEmitter::VRegister rt, ARMEmitter::ExtendedMemOperand MemSrc) {\n if (MemSrc.MetaType.Header.MemType == ARMEmitter::ExtendedMemOperand::Type::TYPE_EXTENDED &&\n MemSrc.MetaType.Extended.rm.Idx() != ARMEmitter::Reg::r31.Idx()) {\n ldrh(rt, MemSrc.rn, MemSrc.MetaType.Extended.rm, MemSrc.MetaType.Extended.Option, MemSrc.MetaType.Extended.Shift);\n } else if (MemSrc.MetaType.Header.MemType == ARMEmitter::ExtendedMemOperand::Type::TYPE_EXTENDED) {\n ldrh(rt, MemSrc.rn);\n } else {\n if (MemSrc.MetaType.ImmType.Index == ARMEmitter::IndexType::OFFSET) {\n if ((MemSrc.MetaType.ImmType.Imm & 0b1) || MemSrc.MetaType.ImmType.Imm < 0) {\n ldurh(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n } else {\n ldrh(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n }\n } else if (MemSrc.MetaType.ImmType.Index == ARMEmitter::IndexType::POST) {\n ldrh(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n } else if (MemSrc.MetaType.ImmType.Index == ARMEmitter::IndexType::PRE) {\n ldrh(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n } else {\n LOGMAN_MSG_A_FMT(\"Unexpected loadstore index type\");\n FEX_UNREACHABLE;\n }\n }\n }\n void str(ARMEmitter::SRegister rt, ARMEmitter::ExtendedMemOperand MemSrc) {\n if (MemSrc.MetaType.Header.MemType == ARMEmitter::ExtendedMemOperand::Type::TYPE_EXTENDED &&\n MemSrc.MetaType.Extended.rm.Idx() != ARMEmitter::Reg::r31.Idx()) {\n str(rt, MemSrc.rn, MemSrc.MetaType.Extended.rm, MemSrc.MetaType.Extended.Option, MemSrc.MetaType.Extended.Shift);\n } else if (MemSrc.MetaType.Header.MemType == ARMEmitter::ExtendedMemOperand::Type::TYPE_EXTENDED) {\n str(rt, MemSrc.rn);\n } else {\n if (MemSrc.MetaType.ImmType.Index == ARMEmitter::IndexType::OFFSET) {\n if ((MemSrc.MetaType.ImmType.Imm & 0b11) || MemSrc.MetaType.ImmType.Imm < 0) {\n stur(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n } else {\n str(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n }\n } else if (MemSrc.MetaType.ImmType.Index == ARMEmitter::IndexType::POST) {\n str(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n } else if (MemSrc.MetaType.ImmType.Index == ARMEmitter::IndexType::PRE) {\n str(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n } else {\n LOGMAN_MSG_A_FMT(\"Unexpected loadstore index type\");\n FEX_UNREACHABLE;\n }\n }\n }\n void ldr(ARMEmitter::SRegister rt, ARMEmitter::ExtendedMemOperand MemSrc) {\n if (MemSrc.MetaType.Header.MemType == ARMEmitter::ExtendedMemOperand::Type::TYPE_EXTENDED &&\n MemSrc.MetaType.Extended.rm.Idx() != ARMEmitter::Reg::r31.Idx()) {\n ldr(rt, MemSrc.rn, MemSrc.MetaType.Extended.rm, MemSrc.MetaType.Extended.Option, MemSrc.MetaType.Extended.Shift);\n } else if (MemSrc.MetaType.Header.MemType == ARMEmitter::ExtendedMemOperand::Type::TYPE_EXTENDED) {\n ldr(rt, MemSrc.rn);\n } else {\n if (MemSrc.MetaType.ImmType.Index == ARMEmitter::IndexType::OFFSET) {\n if ((MemSrc.MetaType.ImmType.Imm & 0b11) || MemSrc.MetaType.ImmType.Imm < 0) {\n ldur(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n } else {\n ldr(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n }\n } else if (MemSrc.MetaType.ImmType.Index == ARMEmitter::IndexType::POST) {\n ldr(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n } else if (MemSrc.MetaType.ImmType.Index == ARMEmitter::IndexType::PRE) {\n ldr(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n } else {\n LOGMAN_MSG_A_FMT(\"Unexpected loadstore index type\");\n FEX_UNREACHABLE;\n }\n }\n }\n void str(ARMEmitter::DRegister rt, ARMEmitter::ExtendedMemOperand MemSrc) {\n if (MemSrc.MetaType.Header.MemType == ARMEmitter::ExtendedMemOperand::Type::TYPE_EXTENDED &&\n MemSrc.MetaType.Extended.rm.Idx() != ARMEmitter::Reg::r31.Idx()) {\n str(rt, MemSrc.rn, MemSrc.MetaType.Extended.rm, MemSrc.MetaType.Extended.Option, MemSrc.MetaType.Extended.Shift);\n } else if (MemSrc.MetaType.Header.MemType == ARMEmitter::ExtendedMemOperand::Type::TYPE_EXTENDED) {\n str(rt, MemSrc.rn);\n } else {\n if (MemSrc.MetaType.ImmType.Index == ARMEmitter::IndexType::OFFSET) {\n if ((MemSrc.MetaType.ImmType.Imm & 0b111) || MemSrc.MetaType.ImmType.Imm < 0) {\n stur(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n } else {\n str(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n }\n } else if (MemSrc.MetaType.ImmType.Index == ARMEmitter::IndexType::POST) {\n str(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n } else if (MemSrc.MetaType.ImmType.Index == ARMEmitter::IndexType::PRE) {\n str(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n } else {\n LOGMAN_MSG_A_FMT(\"Unexpected loadstore index type\");\n FEX_UNREACHABLE;\n }\n }\n }\n void ldr(ARMEmitter::DRegister rt, ARMEmitter::ExtendedMemOperand MemSrc) {\n if (MemSrc.MetaType.Header.MemType == ARMEmitter::ExtendedMemOperand::Type::TYPE_EXTENDED &&\n MemSrc.MetaType.Extended.rm.Idx() != ARMEmitter::Reg::r31.Idx()) {\n ldr(rt, MemSrc.rn, MemSrc.MetaType.Extended.rm, MemSrc.MetaType.Extended.Option, MemSrc.MetaType.Extended.Shift);\n } else if (MemSrc.MetaType.Header.MemType == ARMEmitter::ExtendedMemOperand::Type::TYPE_EXTENDED) {\n ldr(rt, MemSrc.rn);\n } else {\n if (MemSrc.MetaType.ImmType.Index == ARMEmitter::IndexType::OFFSET) {\n if ((MemSrc.MetaType.ImmType.Imm & 0b111) || MemSrc.MetaType.ImmType.Imm < 0) {\n ldur(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n } else {\n ldr(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n }\n } else if (MemSrc.MetaType.ImmType.Index == ARMEmitter::IndexType::POST) {\n ldr(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n } else if (MemSrc.MetaType.ImmType.Index == ARMEmitter::IndexType::PRE) {\n ldr(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n } else {\n LOGMAN_MSG_A_FMT(\"Unexpected loadstore index type\");\n FEX_UNREACHABLE;\n }\n }\n }\n void str(ARMEmitter::QRegister rt, ARMEmitter::ExtendedMemOperand MemSrc) {\n if (MemSrc.MetaType.Header.MemType == ARMEmitter::ExtendedMemOperand::Type::TYPE_EXTENDED &&\n MemSrc.MetaType.Extended.rm.Idx() != ARMEmitter::Reg::r31.Idx()) {\n str(rt, MemSrc.rn, MemSrc.MetaType.Extended.rm, MemSrc.MetaType.Extended.Option, MemSrc.MetaType.Extended.Shift);\n } else if (MemSrc.MetaType.Header.MemType == ARMEmitter::ExtendedMemOperand::Type::TYPE_EXTENDED) {\n str(rt, MemSrc.rn);\n } else {\n if (MemSrc.MetaType.ImmType.Index == ARMEmitter::IndexType::OFFSET) {\n if ((MemSrc.MetaType.ImmType.Imm & 0b1111) || MemSrc.MetaType.ImmType.Imm < 0) {\n stur(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n } else {\n str(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n }\n } else if (MemSrc.MetaType.ImmType.Index == ARMEmitter::IndexType::POST) {\n str(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n } else if (MemSrc.MetaType.ImmType.Index == ARMEmitter::IndexType::PRE) {\n str(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n } else {\n LOGMAN_MSG_A_FMT(\"Unexpected loadstore index type\");\n FEX_UNREACHABLE;\n }\n }\n }\n void ldr(ARMEmitter::QRegister rt, ARMEmitter::ExtendedMemOperand MemSrc) {\n if (MemSrc.MetaType.Header.MemType == ARMEmitter::ExtendedMemOperand::Type::TYPE_EXTENDED &&\n MemSrc.MetaType.Extended.rm.Idx() != ARMEmitter::Reg::r31.Idx()) {\n ldr(rt, MemSrc.rn, MemSrc.MetaType.Extended.rm, MemSrc.MetaType.Extended.Option, MemSrc.MetaType.Extended.Shift);\n } else if (MemSrc.MetaType.Header.MemType == ARMEmitter::ExtendedMemOperand::Type::TYPE_EXTENDED) {\n ldr(rt, MemSrc.rn);\n } else {\n if (MemSrc.MetaType.ImmType.Index == ARMEmitter::IndexType::OFFSET) {\n if ((MemSrc.MetaType.ImmType.Imm & 0b1111) || MemSrc.MetaType.ImmType.Imm < 0) {\n ldur(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n } else {\n ldr(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n }\n } else if (MemSrc.MetaType.ImmType.Index == ARMEmitter::IndexType::POST) {\n ldr(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n } else if (MemSrc.MetaType.ImmType.Index == ARMEmitter::IndexType::PRE) {\n ldr(rt, MemSrc.rn, MemSrc.MetaType.ImmType.Imm);\n } else {\n LOGMAN_MSG_A_FMT(\"Unexpected loadstore index type\");\n FEX_UNREACHABLE;\n }\n }\n }\n\n // Loadstore PAC\n void ldraa(XRegister rt, XRegister rn, IndexType type, int32_t offset = 0) {\n LoadStorePAC(0b11, 0, 0, offset, type, rn, rt);\n }\n void ldrab(XRegister rt, XRegister rn, IndexType type, int32_t offset = 0) {\n LoadStorePAC(0b11, 0, 1, offset, type, rn, rt);\n }\n\n // Loadstore unsigned immediate\n // Maximum values of unsigned immediate offsets for particular data sizes.\n static constexpr uint32_t LSByteMaxUnsignedOffset = 4095;\n static constexpr uint32_t LSHalfMaxUnsignedOffset = 8190;\n static constexpr uint32_t LSWordMaxUnsignedOffset = 16380;\n static constexpr uint32_t LSDWordMaxUnsignedOffset = 32760;\n static constexpr uint32_t LSQWordMaxUnsignedOffset = 65520;\n\n void strb(Register rt, Register rn, uint32_t Imm = 0) {\n LoadStoreUnsigned(0b00, 0, 0b00, rt, rn, Imm);\n }\n void ldrb(Register rt, Register rn, uint32_t Imm = 0) {\n LoadStoreUnsigned(0b00, 0, 0b01, rt, rn, Imm);\n }\n void ldrsb(XRegister rt, Register rn, uint32_t Imm = 0) {\n LoadStoreUnsigned(0b00, 0, 0b10, rt, rn, Imm);\n }\n void ldrsb(WRegister rt, Register rn, uint32_t Imm = 0) {\n LoadStoreUnsigned(0b00, 0, 0b11, rt, rn, Imm);\n }\n void strb(VRegister rt, Register rn, uint32_t Imm = 0) {\n LoadStoreUnsigned(0b00, 1, 0b00, rt, rn, Imm);\n }\n void ldrb(VRegister rt, Register rn, uint32_t Imm = 0) {\n LoadStoreUnsigned(0b00, 1, 0b01, rt, rn, Imm);\n }\n void strh(Register rt, Register rn, uint32_t Imm = 0) {\n LoadStoreUnsigned(0b01, 0, 0b00, rt, rn, Imm);\n }\n void ldrh(Register rt, Register rn, uint32_t Imm = 0) {\n LoadStoreUnsigned(0b01, 0, 0b01, rt, rn, Imm);\n }\n void ldrsh(XRegister rt, Register rn, uint32_t Imm = 0) {\n LoadStoreUnsigned(0b01, 0, 0b10, rt, rn, Imm);\n }\n void ldrsh(WRegister rt, Register rn, uint32_t Imm = 0) {\n LoadStoreUnsigned(0b01, 0, 0b11, rt, rn, Imm);\n }\n void strh(VRegister rt, Register rn, uint32_t Imm = 0) {\n LoadStoreUnsigned(0b01, 1, 0b00, rt, rn, Imm);\n }\n void ldrh(VRegister rt, Register rn, uint32_t Imm = 0) {\n LoadStoreUnsigned(0b01, 1, 0b01, rt, rn, Imm);\n }\n void str(WRegister rt, Register rn, uint32_t Imm = 0) {\n LoadStoreUnsigned(0b10, 0, 0b00, rt, rn, Imm);\n }\n void ldr(WRegister rt, Register rn, uint32_t Imm = 0) {\n LoadStoreUnsigned(0b10, 0, 0b01, rt, rn, Imm);\n }\n void ldrsw(XRegister rt, Register rn, uint32_t Imm = 0) {\n LoadStoreUnsigned(0b10, 0, 0b10, rt, rn, Imm);\n }\n void str(SRegister rt, Register rn, uint32_t Imm = 0) {\n LoadStoreUnsigned(0b10, 1, 0b00, rt, rn, Imm);\n }\n void ldr(SRegister rt, Register rn, uint32_t Imm = 0) {\n LoadStoreUnsigned(0b10, 1, 0b01, rt, rn, Imm);\n }\n void str(XRegister rt, Register rn, uint32_t Imm = 0) {\n LoadStoreUnsigned(0b11, 0, 0b00, rt, rn, Imm);\n }\n void ldr(XRegister rt, Register rn, uint32_t Imm = 0) {\n LoadStoreUnsigned(0b11, 0, 0b01, rt, rn, Imm);\n }\n\n void ldr(SubRegSize size, Register rt, Register rn, uint32_t Imm = 0) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i128Bit, \"Can't use 128-bit size\");\n LoadStoreUnsigned(FEXCore::ToUnderlying(size), 0, 0b01, rt, rn, Imm);\n }\n void str(SubRegSize size, Register rt, Register rn, uint32_t Imm = 0) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i128Bit, \"Can't use 128-bit size\");\n LoadStoreUnsigned(FEXCore::ToUnderlying(size), 0, 0b00, rt, rn, Imm);\n }\n\n void prfm(Prefetch prfop, Register rn, uint32_t Imm = 0) {\n LoadStoreUnsigned(0b11, 0, 0b10, prfop, rn, Imm);\n }\n void str(DRegister rt, Register rn, uint32_t Imm = 0) {\n LoadStoreUnsigned(0b11, 1, 0b00, rt, rn, Imm);\n }\n void ldr(DRegister rt, Register rn, uint32_t Imm = 0) {\n LoadStoreUnsigned(0b11, 1, 0b01, rt, rn, Imm);\n }\n void str(QRegister rt, Register rn, uint32_t Imm = 0) {\n LoadStoreUnsigned(0b00, 1, 0b10, rt, rn, Imm);\n }\n void ldr(QRegister rt, Register rn, uint32_t Imm = 0) {\n LoadStoreUnsigned(0b00, 1, 0b11, rt, rn, Imm);\n }\n\nprivate:\n void AtomicOp(uint32_t Op, ARMEmitter::Size s, uint32_t L, uint32_t o0, ARMEmitter::Register rs, ARMEmitter::Register rt,\n ARMEmitter::Register rt2, ARMEmitter::Register rn) {\n const uint32_t sz = s == ARMEmitter::Size::i64Bit ? (1U << 30) : 0;\n uint32_t Instr = Op;\n\n Instr |= sz;\n Instr |= L << 22;\n Instr |= Encode_rs(rs);\n Instr |= o0 << 15;\n Instr |= Encode_rt2(rt2);\n Instr |= Encode_rn(rn);\n Instr |= Encode_rt(rt);\n\n dc32(Instr);\n }\n\n template\n void SubAtomicOp(uint32_t Op, ARMEmitter::SubRegSize s, uint32_t L, uint32_t o0, T rs, T rt, T rt2, ARMEmitter::Register rn) {\n const uint32_t sz = FEXCore::ToUnderlying(s) << 30;\n uint32_t Instr = Op;\n\n Instr |= sz;\n Instr |= L << 22;\n Instr |= Encode_rs(rs);\n Instr |= o0 << 15;\n Instr |= Encode_rt2(rt2);\n Instr |= Encode_rn(rn);\n Instr |= Encode_rt(rt);\n\n dc32(Instr);\n }\n\n template\n void SubAtomicImm(uint32_t Op, ARMEmitter::SubRegSize s, uint32_t opc, T rt, ARMEmitter::Register rn, uint32_t Imm) {\n const uint32_t sz = FEXCore::ToUnderlying(s) << 30;\n uint32_t Instr = Op;\n\n Instr |= sz;\n Instr |= opc << 22;\n Instr |= Imm << 12;\n Instr |= Encode_rn(rn);\n Instr |= Encode_rt(rt);\n\n dc32(Instr);\n }\n // Load register literal\n template\n void LoadStoreLiteral(uint32_t Op, T rt, uint32_t Imm) {\n uint32_t Instr = Op;\n\n Instr |= Imm << 5;\n Instr |= Encode_rt(rt);\n dc32(Instr);\n }\n\n void MemoryCopyAndMemorySet(uint32_t sz, uint32_t o0, uint32_t op1, uint32_t op2, Register rs, Register rn, Register rd) {\n uint32_t Instr = 0b0001'1001'0000'0000'0000'0100'0000'0000;\n\n Instr |= sz << 30;\n Instr |= o0 << 26;\n Instr |= op1 << 22;\n Instr |= rs.Idx() << 16;\n Instr |= op2 << 12;\n Instr |= rn.Idx() << 5;\n Instr |= rd.Idx();\n\n dc32(Instr);\n }\n\n // Loadstore no-allocate pair\n template\n void LoadStoreNoAllocate(uint32_t Op, T rt, T rt2, ARMEmitter::Register rn, uint32_t Imm) {\n uint32_t Instr = Op;\n\n Instr |= Imm << 15;\n Instr |= Encode_rt2(rt2);\n Instr |= Encode_rn(rn);\n Instr |= Encode_rt(rt);\n dc32(Instr);\n }\n // Loadstore register pair post-indexed\n template\n void LoadStorePair(uint32_t Op, T rt, T rt2, ARMEmitter::Register rn, uint32_t Imm) {\n uint32_t Instr = Op;\n Instr |= Imm << 15;\n Instr |= Encode_rt2(rt2);\n Instr |= Encode_rn(rn);\n Instr |= Encode_rt(rt);\n dc32(Instr);\n }\n\n // Loadstore register unscaled immediate\n // Loadstore register immediate post-indexed\n // Loadstore register unprivileged\n // Loadstore register immediate pre-indexed\n template\n void LoadStoreImm(uint32_t Op, uint32_t o2, T rt, ARMEmitter::Register rn, uint32_t Imm) {\n uint32_t Instr = Op;\n\n Instr |= Imm << 12;\n Instr |= o2 << 10;\n Instr |= Encode_rn(rn);\n Instr |= Encode_rt(rt);\n dc32(Instr);\n }\n\n // Atomic memory operations\n void LoadStoreAtomicLSE(SubRegSize s, uint32_t A, uint32_t R, uint32_t o3, uint32_t opc, Register rs, Register rt, Register rn) {\n uint32_t Instr = 0b0011'1000'0010'0000'0000'0000'0000'0000;\n Instr |= FEXCore::ToUnderlying(s) << 30;\n Instr |= A << 23;\n Instr |= R << 22;\n Instr |= Encode_rs(rs);\n Instr |= o3 << 15;\n Instr |= opc << 12;\n Instr |= Encode_rn(rn);\n Instr |= Encode_rt(rt);\n dc32(Instr);\n }\n\n // Loadstore register-register offset\n template\n void LoadStoreRegisterOffset(uint32_t Op, uint32_t opc, T rt, ARMEmitter::Register rn, ARMEmitter::Register rm,\n ARMEmitter::ExtendedType Option, uint32_t Shift) {\n uint32_t Instr = Op;\n\n Instr |= opc << 22;\n Instr |= Encode_rt(rt);\n Instr |= FEXCore::ToUnderlying(Option) << 13;\n Instr |= Shift << 12;\n Instr |= Encode_rn(rn);\n Instr |= Encode_rm(rm);\n dc32(Instr);\n }\n\n void LoadStorePAC(uint32_t size, uint32_t VR, uint32_t M, int32_t imm, IndexType type, Register rn, Register rt) {\n LOGMAN_THROW_A_FMT((imm % 8) == 0, \"Immediate ({}) must be divisible by 8\", imm);\n LOGMAN_THROW_A_FMT(imm >= -4096 && imm <= 4088, \"Immediate ({}) must be within [-4096, 4088]\", imm);\n LOGMAN_THROW_A_FMT(type == IndexType::OFFSET || type == IndexType::PRE, \"PAC may only use offset or pre-indexed values\");\n\n // The immediate is scaled down in order to fit within the available 10 immediate bits.\n const auto scaled_imm = static_cast(imm / 8);\n const auto imm9 = scaled_imm & 0b1'1111'1111;\n const auto S = (scaled_imm >> 9) & 1;\n\n const auto W = type == IndexType::OFFSET ? 0U : 1U;\n\n uint32_t Instr = 0b0011'1000'0010'0000'0000'0100'0000'0000;\n Instr |= size << 30;\n Instr |= VR << 26;\n Instr |= M << 23;\n Instr |= S << 22;\n Instr |= imm9 << 12;\n Instr |= W << 11;\n Instr |= rn.Idx() << 5;\n Instr |= rt.Idx();\n dc32(Instr);\n }\n\n // Loadstore unsigned immediate\n template\n void LoadStoreUnsigned(uint32_t size, uint32_t V, uint32_t opc, T rt, Register rn, uint32_t Imm) {\n uint32_t SizeShift = size;\n if constexpr (std::is_same_v) {\n // 128-bit variant is specified via size=0b00, V=1, opc=0b1x\n // so we need to special case this one based on whether or not\n // rt indicates a 128-bit vector. Nice thing is this can be\n // checked at compile-time.\n SizeShift = 4;\n }\n\n [[maybe_unused]] const uint32_t MaxImm = LSByteMaxUnsignedOffset << SizeShift;\n [[maybe_unused]] const uint32_t ElementSize = 1U << SizeShift;\n\n LOGMAN_THROW_A_FMT(Imm <= MaxImm, \"{}: Offset not valid: Imm: 0x{:x} Max: 0x{:x}\", __func__, Imm, MaxImm);\n LOGMAN_THROW_A_FMT((Imm % ElementSize) == 0, \"{}: Offset must be a multiple of {}. Offset: 0x{:x}\", __func__, ElementSize, Imm);\n\n const uint32_t ShiftedImm = Imm >> SizeShift;\n\n uint32_t Instr = 0b0011'1001'0000'0000'0000'0000'0000'0000;\n Instr |= size << 30;\n Instr |= V << 26;\n Instr |= opc << 22;\n Instr |= ShiftedImm << 10;\n Instr |= Encode_rn(rn);\n Instr |= Encode_rt(rt);\n dc32(Instr);\n }\n\n template\n void ldp_w(ARMEmitter::VRegister rt, ARMEmitter::VRegister rt2, ARMEmitter::Register rn, int32_t Imm) {\n LOGMAN_THROW_A_FMT(Imm >= -256 && Imm <= 252 && ((Imm & 0b11) == 0), \"Unscaled offset too large\");\n constexpr uint32_t Op = (0b0010'1100'01 << 22) | (Index == IndexType::POST ? (0b01 << 23) :\n Index == IndexType::PRE ? (0b11 << 23) :\n Index == IndexType::OFFSET ? (0b10 << 23) :\n -1);\n\n LoadStorePair(Op, rt, rt2, rn, (Imm >> 2) & 0b111'1111);\n }\n template\n void ldp_x(ARMEmitter::VRegister rt, ARMEmitter::VRegister rt2, ARMEmitter::Register rn, int32_t Imm) {\n LOGMAN_THROW_A_FMT(Imm >= -512 && Imm <= 504 && ((Imm & 0b111) == 0), \"Unscaled offset too large\");\n constexpr uint32_t Op = (0b0110'1100'01 << 22) | (Index == IndexType::POST ? (0b01 << 23) :\n Index == IndexType::PRE ? (0b11 << 23) :\n Index == IndexType::OFFSET ? (0b10 << 23) :\n -1);\n\n LoadStorePair(Op, rt, rt2, rn, (Imm >> 3) & 0b111'1111);\n }\n template\n void stp_w(ARMEmitter::VRegister rt, ARMEmitter::VRegister rt2, ARMEmitter::Register rn, int32_t Imm) {\n LOGMAN_THROW_A_FMT(Imm >= -256 && Imm <= 252 && ((Imm & 0b11) == 0), \"Unscaled offset too large\");\n constexpr uint32_t Op = (0b0010'1100'00 << 22) | (Index == IndexType::POST ? (0b01 << 23) :\n Index == IndexType::PRE ? (0b11 << 23) :\n Index == IndexType::OFFSET ? (0b10 << 23) :\n -1);\n\n LoadStorePair(Op, rt, rt2, rn, (Imm >> 2) & 0b111'1111);\n }\n template\n void stp_x(ARMEmitter::VRegister rt, ARMEmitter::VRegister rt2, ARMEmitter::Register rn, int32_t Imm) {\n LOGMAN_THROW_A_FMT(Imm >= -512 && Imm <= 504 && ((Imm & 0b111) == 0), \"Unscaled offset too large\");\n constexpr uint32_t Op = (0b0110'1100'00 << 22) | (Index == IndexType::POST ? (0b01 << 23) :\n Index == IndexType::PRE ? (0b11 << 23) :\n Index == IndexType::OFFSET ? (0b10 << 23) :\n -1);\n\n LoadStorePair(Op, rt, rt2, rn, (Imm >> 3) & 0b111'1111);\n }\n template\n void ldp_q(ARMEmitter::VRegister rt, ARMEmitter::VRegister rt2, ARMEmitter::Register rn, int32_t Imm) {\n LOGMAN_THROW_A_FMT(Imm >= -1024 && Imm <= 1008 && ((Imm & 0b1111) == 0), \"Unscaled offset too large\");\n constexpr uint32_t Op = (0b1010'1100'01 << 22) | (Index == IndexType::POST ? (0b01 << 23) :\n Index == IndexType::PRE ? (0b11 << 23) :\n Index == IndexType::OFFSET ? (0b10 << 23) :\n -1);\n\n LoadStorePair(Op, rt, rt2, rn, (Imm >> 4) & 0b111'1111);\n }\n template\n void stp_q(ARMEmitter::VRegister rt, ARMEmitter::VRegister rt2, ARMEmitter::Register rn, int32_t Imm) {\n LOGMAN_THROW_A_FMT(Imm >= -1024 && Imm <= 1008 && ((Imm & 0b1111) == 0), \"Unscaled offset too large\");\n constexpr uint32_t Op = (0b1010'1100'00 << 22) | (Index == IndexType::POST ? (0b01 << 23) :\n Index == IndexType::PRE ? (0b11 << 23) :\n Index == IndexType::OFFSET ? (0b10 << 23) :\n -1);\n\n LoadStorePair(Op, rt, rt2, rn, (Imm >> 4) & 0b111'1111);\n }\n\n template\n void stXrb(ARMEmitter::Register rt, ARMEmitter::Register rn, int32_t Imm) {\n LOGMAN_THROW_A_FMT(Imm >= -256 && Imm <= 255, \"Unscaled offset too large\");\n\n constexpr uint32_t Op = 0b0011'1000'00 << 22;\n constexpr uint32_t o2 = Index == IndexType::POST ? 0b01 :\n Index == IndexType::PRE ? 0b11 :\n Index == IndexType::OFFSET ? 0b00 :\n Index == IndexType::UNPRIVILEGED ? 0b10 :\n -1;\n\n LoadStoreImm(Op, o2, rt, rn, Imm & 0b1'1111'1111);\n }\n template\n void ldXrb(ARMEmitter::Register rt, ARMEmitter::Register rn, int32_t Imm) {\n LOGMAN_THROW_A_FMT(Imm >= -256 && Imm <= 255, \"Unscaled offset too large\");\n\n constexpr uint32_t Op = 0b0011'1000'01 << 22;\n constexpr uint32_t o2 = Index == IndexType::POST ? 0b01 :\n Index == IndexType::PRE ? 0b11 :\n Index == IndexType::OFFSET ? 0b00 :\n Index == IndexType::UNPRIVILEGED ? 0b10 :\n -1;\n\n LoadStoreImm(Op, o2, rt, rn, Imm & 0b1'1111'1111);\n }\n template\n void stXrb(ARMEmitter::VRegister rt, ARMEmitter::Register rn, int32_t Imm) {\n LOGMAN_THROW_A_FMT(Imm >= -256 && Imm <= 255, \"Unscaled offset too large\");\n\n constexpr uint32_t Op = 0b0011'1100'00 << 22;\n constexpr uint32_t o2 = Index == IndexType::POST ? 0b01 :\n Index == IndexType::PRE ? 0b11 :\n Index == IndexType::OFFSET ? 0b00 :\n Index == IndexType::UNPRIVILEGED ? 0b10 :\n -1;\n\n LoadStoreImm(Op, o2, rt, rn, Imm & 0b1'1111'1111);\n }\n template\n void ldXrb(ARMEmitter::VRegister rt, ARMEmitter::Register rn, int32_t Imm) {\n LOGMAN_THROW_A_FMT(Imm >= -256 && Imm <= 255, \"Unscaled offset too large\");\n\n constexpr uint32_t Op = 0b0011'1100'01 << 22;\n constexpr uint32_t o2 = Index == IndexType::POST ? 0b01 :\n Index == IndexType::PRE ? 0b11 :\n Index == IndexType::OFFSET ? 0b00 :\n Index == IndexType::UNPRIVILEGED ? 0b10 :\n -1;\n\n LoadStoreImm(Op, o2, rt, rn, Imm & 0b1'1111'1111);\n }\n template\n void ldXrsb(ARMEmitter::XRegister rt, ARMEmitter::Register rn, int32_t Imm) {\n LOGMAN_THROW_A_FMT(Imm >= -256 && Imm <= 255, \"Unscaled offset too large\");\n\n constexpr uint32_t Op = 0b0011'1000'10 << 22;\n constexpr uint32_t o2 = Index == IndexType::POST ? 0b01 :\n Index == IndexType::PRE ? 0b11 :\n Index == IndexType::OFFSET ? 0b00 :\n Index == IndexType::UNPRIVILEGED ? 0b10 :\n -1;\n\n LoadStoreImm(Op, o2, rt, rn, Imm & 0b1'1111'1111);\n }\n template\n void ldXrsb(ARMEmitter::WRegister rt, ARMEmitter::Register rn, int32_t Imm) {\n LOGMAN_THROW_A_FMT(Imm >= -256 && Imm <= 255, \"Unscaled offset too large\");\n\n constexpr uint32_t Op = 0b0011'1000'11 << 22;\n constexpr uint32_t o2 = Index == IndexType::POST ? 0b01 :\n Index == IndexType::PRE ? 0b11 :\n Index == IndexType::OFFSET ? 0b00 :\n Index == IndexType::UNPRIVILEGED ? 0b10 :\n -1;\n\n LoadStoreImm(Op, o2, rt, rn, Imm & 0b1'1111'1111);\n }\n template\n void stXrh(ARMEmitter::Register rt, ARMEmitter::Register rn, int32_t Imm) {\n LOGMAN_THROW_A_FMT(Imm >= -256 && Imm <= 255, \"Unscaled offset too large\");\n\n constexpr uint32_t Op = 0b0111'1000'00 << 22;\n constexpr uint32_t o2 = Index == IndexType::POST ? 0b01 :\n Index == IndexType::PRE ? 0b11 :\n Index == IndexType::OFFSET ? 0b00 :\n Index == IndexType::UNPRIVILEGED ? 0b10 :\n -1;\n\n LoadStoreImm(Op, o2, rt, rn, Imm & 0b1'1111'1111);\n }\n template\n void ldXrh(ARMEmitter::Register rt, ARMEmitter::Register rn, int32_t Imm) {\n LOGMAN_THROW_A_FMT(Imm >= -256 && Imm <= 255, \"Unscaled offset too large\");\n\n constexpr uint32_t Op = 0b0111'1000'01 << 22;\n constexpr uint32_t o2 = Index == IndexType::POST ? 0b01 :\n Index == IndexType::PRE ? 0b11 :\n Index == IndexType::OFFSET ? 0b00 :\n Index == IndexType::UNPRIVILEGED ? 0b10 :\n -1;\n\n LoadStoreImm(Op, o2, rt, rn, Imm & 0b1'1111'1111);\n }\n template\n void stXrh(ARMEmitter::VRegister rt, ARMEmitter::Register rn, int32_t Imm) {\n LOGMAN_THROW_A_FMT(Imm >= -256 && Imm <= 255, \"Unscaled offset too large\");\n\n constexpr uint32_t Op = 0b0111'1100'00 << 22;\n constexpr uint32_t o2 = Index == IndexType::POST ? 0b01 :\n Index == IndexType::PRE ? 0b11 :\n Index == IndexType::OFFSET ? 0b00 :\n Index == IndexType::UNPRIVILEGED ? 0b10 :\n -1;\n\n LoadStoreImm(Op, o2, rt, rn, Imm & 0b1'1111'1111);\n }\n template\n void ldXrh(ARMEmitter::VRegister rt, ARMEmitter::Register rn, int32_t Imm) {\n LOGMAN_THROW_A_FMT(Imm >= -256 && Imm <= 255, \"Unscaled offset too large\");\n\n constexpr uint32_t Op = 0b0111'1100'01 << 22;\n constexpr uint32_t o2 = Index == IndexType::POST ? 0b01 :\n Index == IndexType::PRE ? 0b11 :\n Index == IndexType::OFFSET ? 0b00 :\n Index == IndexType::UNPRIVILEGED ? 0b10 :\n -1;\n\n LoadStoreImm(Op, o2, rt, rn, Imm & 0b1'1111'1111);\n }\n template\n void ldXrsh(ARMEmitter::XRegister rt, ARMEmitter::Register rn, int32_t Imm) {\n LOGMAN_THROW_A_FMT(Imm >= -256 && Imm <= 255, \"Unscaled offset too large\");\n\n constexpr uint32_t Op = 0b0111'1000'10 << 22;\n constexpr uint32_t o2 = Index == IndexType::POST ? 0b01 :\n Index == IndexType::PRE ? 0b11 :\n Index == IndexType::OFFSET ? 0b00 :\n Index == IndexType::UNPRIVILEGED ? 0b10 :\n -1;\n\n LoadStoreImm(Op, o2, rt, rn, Imm & 0b1'1111'1111);\n }\n template\n void ldXrsh(ARMEmitter::WRegister rt, ARMEmitter::Register rn, int32_t Imm) {\n LOGMAN_THROW_A_FMT(Imm >= -256 && Imm <= 255, \"Unscaled offset too large\");\n\n constexpr uint32_t Op = 0b0111'1000'11 << 22;\n constexpr uint32_t o2 = Index == IndexType::POST ? 0b01 :\n Index == IndexType::PRE ? 0b11 :\n Index == IndexType::OFFSET ? 0b00 :\n Index == IndexType::UNPRIVILEGED ? 0b10 :\n -1;\n\n LoadStoreImm(Op, o2, rt, rn, Imm & 0b1'1111'1111);\n }\n template\n void stXr(ARMEmitter::WRegister rt, ARMEmitter::Register rn, int32_t Imm) {\n LOGMAN_THROW_A_FMT(Imm >= -256 && Imm <= 255, \"Unscaled offset too large\");\n\n constexpr uint32_t Op = 0b1011'1000'00 << 22;\n constexpr uint32_t o2 = Index == IndexType::POST ? 0b01 :\n Index == IndexType::PRE ? 0b11 :\n Index == IndexType::OFFSET ? 0b00 :\n Index == IndexType::UNPRIVILEGED ? 0b10 :\n -1;\n\n LoadStoreImm(Op, o2, rt, rn, Imm & 0b1'1111'1111);\n }\n template\n void ldXr(ARMEmitter::WRegister rt, ARMEmitter::Register rn, int32_t Imm) {\n LOGMAN_THROW_A_FMT(Imm >= -256 && Imm <= 255, \"Unscaled offset too large\");\n\n constexpr uint32_t Op = 0b1011'1000'01 << 22;\n constexpr uint32_t o2 = Index == IndexType::POST ? 0b01 :\n Index == IndexType::PRE ? 0b11 :\n Index == IndexType::OFFSET ? 0b00 :\n Index == IndexType::UNPRIVILEGED ? 0b10 :\n -1;\n\n LoadStoreImm(Op, o2, rt, rn, Imm & 0b1'1111'1111);\n }\n template\n void stXr(ARMEmitter::SRegister rt, ARMEmitter::Register rn, int32_t Imm) {\n LOGMAN_THROW_A_FMT(Imm >= -256 && Imm <= 255, \"Unscaled offset too large\");\n\n constexpr uint32_t Op = 0b1011'1100'00 << 22;\n constexpr uint32_t o2 = Index == IndexType::POST ? 0b01 :\n Index == IndexType::PRE ? 0b11 :\n Index == IndexType::OFFSET ? 0b00 :\n Index == IndexType::UNPRIVILEGED ? 0b10 :\n -1;\n\n LoadStoreImm(Op, o2, rt, rn, Imm & 0b1'1111'1111);\n }\n template\n void ldXr(ARMEmitter::SRegister rt, ARMEmitter::Register rn, int32_t Imm) {\n LOGMAN_THROW_A_FMT(Imm >= -256 && Imm <= 255, \"Unscaled offset too large\");\n\n constexpr uint32_t Op = 0b1011'1100'01 << 22;\n constexpr uint32_t o2 = Index == IndexType::POST ? 0b01 :\n Index == IndexType::PRE ? 0b11 :\n Index == IndexType::OFFSET ? 0b00 :\n Index == IndexType::UNPRIVILEGED ? 0b10 :\n -1;\n\n LoadStoreImm(Op, o2, rt, rn, Imm & 0b1'1111'1111);\n }\n template\n void ldXrsw(ARMEmitter::XRegister rt, ARMEmitter::Register rn, int32_t Imm) {\n LOGMAN_THROW_A_FMT(Imm >= -256 && Imm <= 255, \"Unscaled offset too large\");\n\n constexpr uint32_t Op = 0b1011'1000'10 << 22;\n constexpr uint32_t o2 = Index == IndexType::POST ? 0b01 :\n Index == IndexType::PRE ? 0b11 :\n Index == IndexType::OFFSET ? 0b00 :\n Index == IndexType::UNPRIVILEGED ? 0b10 :\n -1;\n\n LoadStoreImm(Op, o2, rt, rn, Imm & 0b1'1111'1111);\n }\n template\n void stXr(ARMEmitter::XRegister rt, ARMEmitter::Register rn, int32_t Imm) {\n LOGMAN_THROW_A_FMT(Imm >= -256 && Imm <= 255, \"Unscaled offset too large\");\n\n constexpr uint32_t Op = 0b1111'1000'00 << 22;\n constexpr uint32_t o2 = Index == IndexType::POST ? 0b01 :\n Index == IndexType::PRE ? 0b11 :\n Index == IndexType::OFFSET ? 0b00 :\n Index == IndexType::UNPRIVILEGED ? 0b10 :\n -1;\n\n LoadStoreImm(Op, o2, rt, rn, Imm & 0b1'1111'1111);\n }\n template\n void ldXr(ARMEmitter::XRegister rt, ARMEmitter::Register rn, int32_t Imm) {\n LOGMAN_THROW_A_FMT(Imm >= -256 && Imm <= 255, \"Unscaled offset too large\");\n\n constexpr uint32_t Op = 0b1111'1000'01 << 22;\n constexpr uint32_t o2 = Index == IndexType::POST ? 0b01 :\n Index == IndexType::PRE ? 0b11 :\n Index == IndexType::OFFSET ? 0b00 :\n Index == IndexType::UNPRIVILEGED ? 0b10 :\n -1;\n\n LoadStoreImm(Op, o2, rt, rn, Imm & 0b1'1111'1111);\n }\n template\n void stXr(ARMEmitter::DRegister rt, ARMEmitter::Register rn, int32_t Imm) {\n LOGMAN_THROW_A_FMT(Imm >= -256 && Imm <= 255, \"Unscaled offset too large\");\n\n constexpr uint32_t Op = 0b1111'1100'00 << 22;\n constexpr uint32_t o2 = Index == IndexType::POST ? 0b01 :\n Index == IndexType::PRE ? 0b11 :\n Index == IndexType::OFFSET ? 0b00 :\n Index == IndexType::UNPRIVILEGED ? 0b10 :\n -1;\n\n LoadStoreImm(Op, o2, rt, rn, Imm & 0b1'1111'1111);\n }\n template\n void ldXr(ARMEmitter::DRegister rt, ARMEmitter::Register rn, int32_t Imm) {\n LOGMAN_THROW_A_FMT(Imm >= -256 && Imm <= 255, \"Unscaled offset too large\");\n\n constexpr uint32_t Op = 0b1111'1100'01 << 22;\n constexpr uint32_t o2 = Index == IndexType::POST ? 0b01 :\n Index == IndexType::PRE ? 0b11 :\n Index == IndexType::OFFSET ? 0b00 :\n Index == IndexType::UNPRIVILEGED ? 0b10 :\n -1;\n\n LoadStoreImm(Op, o2, rt, rn, Imm & 0b1'1111'1111);\n }\n template\n void stXr(ARMEmitter::QRegister rt, ARMEmitter::Register rn, int32_t Imm) {\n LOGMAN_THROW_A_FMT(Imm >= -256 && Imm <= 255, \"Unscaled offset too large\");\n\n constexpr uint32_t Op = 0b0011'1100'10 << 22;\n constexpr uint32_t o2 = Index == IndexType::POST ? 0b01 :\n Index == IndexType::PRE ? 0b11 :\n Index == IndexType::OFFSET ? 0b00 :\n Index == IndexType::UNPRIVILEGED ? 0b10 :\n -1;\n\n LoadStoreImm(Op, o2, rt, rn, Imm & 0b1'1111'1111);\n }\n template\n void ldXr(ARMEmitter::QRegister rt, ARMEmitter::Register rn, int32_t Imm) {\n LOGMAN_THROW_A_FMT(Imm >= -256 && Imm <= 255, \"Unscaled offset too large\");\n\n constexpr uint32_t Op = 0b0011'1100'11 << 22;\n constexpr uint32_t o2 = Index == IndexType::POST ? 0b01 :\n Index == IndexType::PRE ? 0b11 :\n Index == IndexType::OFFSET ? 0b00 :\n Index == IndexType::UNPRIVILEGED ? 0b10 :\n -1;\n\n LoadStoreImm(Op, o2, rt, rn, Imm & 0b1'1111'1111);\n }\n\n#ifndef INCLUDED_BY_EMITTER\n}; // struct LoadstoreEmitterOps\n} // namespace ARMEmitter\n#endif\n" }, { "path": "CodeEmitter/CodeEmitter/Registers.h", "content": "// SPDX-License-Identifier: MIT\n#pragma once\n\n#include \n\n#include \n#include \n\nnamespace ARMEmitter {\nclass WRegister;\nclass XRegister;\n\n/* Unsized GPR register class\n * This class doesn't imply a size when used\n */\nclass Register {\npublic:\n Register() = delete;\n constexpr explicit Register(uint32_t Idx)\n : Index {Idx} {}\n\n friend constexpr auto operator<=>(const Register&, const Register&) = default;\n\n constexpr uint32_t Idx() const {\n return Index;\n }\n constexpr WRegister W() const;\n constexpr XRegister X() const;\n\nprivate:\n uint32_t Index;\n};\nstatic_assert(sizeof(Register) == sizeof(uint32_t));\nstatic_assert(std::is_trivially_copyable_v);\nstatic_assert(std::is_standard_layout_v);\n\n/* 32-bit GPR register class.\n * This class will imply a 32-bit register size being used.\n */\nclass WRegister {\npublic:\n WRegister() = delete;\n constexpr explicit WRegister(uint32_t Idx)\n : Index {Idx} {}\n\n friend constexpr auto operator<=>(const WRegister&, const WRegister&) = default;\n\n constexpr uint32_t Idx() const {\n return Index;\n }\n\n constexpr operator Register() const {\n return Register(Index);\n }\n constexpr XRegister X() const;\n constexpr Register R() const;\n\nprivate:\n uint32_t Index;\n};\nstatic_assert(sizeof(WRegister) == sizeof(uint32_t));\nstatic_assert(std::is_trivially_copyable_v);\nstatic_assert(std::is_standard_layout_v);\n\n/* 64-bit GPR register class.\n * This class will imply a 64-bit register size being used.\n */\nclass XRegister {\npublic:\n XRegister() = delete;\n constexpr explicit XRegister(uint32_t Idx)\n : Index {Idx} {}\n\n friend constexpr auto operator<=>(const XRegister&, const XRegister&) = default;\n\n constexpr uint32_t Idx() const {\n return Index;\n }\n\n constexpr operator Register() const {\n return Register(Index);\n }\n constexpr WRegister W() const;\n constexpr Register R() const;\n\nprivate:\n uint32_t Index;\n};\nstatic_assert(sizeof(XRegister) == sizeof(uint32_t));\nstatic_assert(std::is_trivially_copyable_v);\nstatic_assert(std::is_standard_layout_v);\n\ninline constexpr WRegister Register::W() const {\n return WRegister {Index};\n}\n\ninline constexpr XRegister Register::X() const {\n return XRegister {Index};\n}\n\ninline constexpr XRegister WRegister::X() const {\n return XRegister {Index};\n}\n\ninline constexpr Register WRegister::R() const {\n return *this;\n}\n\ninline constexpr WRegister XRegister::W() const {\n return WRegister {Index};\n}\n\ninline constexpr Register XRegister::R() const {\n return *this;\n}\n\n// Namespace containing all unsized GPR register objects.\nnamespace Reg {\n constexpr static Register r0(0);\n constexpr static Register r1(1);\n constexpr static Register r2(2);\n constexpr static Register r3(3);\n constexpr static Register r4(4);\n constexpr static Register r5(5);\n constexpr static Register r6(6);\n constexpr static Register r7(7);\n constexpr static Register r8(8);\n constexpr static Register r9(9);\n constexpr static Register r10(10);\n constexpr static Register r11(11);\n constexpr static Register r12(12);\n constexpr static Register r13(13);\n constexpr static Register r14(14);\n constexpr static Register r15(15);\n constexpr static Register r16(16);\n constexpr static Register r17(17);\n constexpr static Register r18(18);\n constexpr static Register r19(19);\n constexpr static Register r20(20);\n constexpr static Register r21(21);\n constexpr static Register r22(22);\n constexpr static Register r23(23);\n constexpr static Register r24(24);\n constexpr static Register r25(25);\n constexpr static Register r26(26);\n constexpr static Register r27(27);\n constexpr static Register r28(28);\n constexpr static Register r29(29);\n constexpr static Register r30(30);\n constexpr static Register r31(31);\n\n // Named registers\n constexpr static Register ip0(16);\n constexpr static Register ip1(17);\n\n constexpr static Register fp(29);\n constexpr static Register lr(30);\n constexpr static Register rsp(31);\n constexpr static Register zr(31);\n} // namespace Reg\n\n// Namespace containing all 64-bit GPR register objects.\nnamespace XReg {\n constexpr static XRegister x0(0);\n constexpr static XRegister x1(1);\n constexpr static XRegister x2(2);\n constexpr static XRegister x3(3);\n constexpr static XRegister x4(4);\n constexpr static XRegister x5(5);\n constexpr static XRegister x6(6);\n constexpr static XRegister x7(7);\n constexpr static XRegister x8(8);\n constexpr static XRegister x9(9);\n constexpr static XRegister x10(10);\n constexpr static XRegister x11(11);\n constexpr static XRegister x12(12);\n constexpr static XRegister x13(13);\n constexpr static XRegister x14(14);\n constexpr static XRegister x15(15);\n constexpr static XRegister x16(16);\n constexpr static XRegister x17(17);\n constexpr static XRegister x18(18);\n constexpr static XRegister x19(19);\n constexpr static XRegister x20(20);\n constexpr static XRegister x21(21);\n constexpr static XRegister x22(22);\n constexpr static XRegister x23(23);\n constexpr static XRegister x24(24);\n constexpr static XRegister x25(25);\n constexpr static XRegister x26(26);\n constexpr static XRegister x27(27);\n constexpr static XRegister x28(28);\n constexpr static XRegister x29(29);\n constexpr static XRegister x30(30);\n constexpr static XRegister x31(31);\n\n // Named registers\n constexpr static XRegister ip0(16);\n constexpr static XRegister ip1(17);\n\n constexpr static XRegister fp(29);\n constexpr static XRegister lr(30);\n constexpr static XRegister rsp(31);\n constexpr static XRegister zr(31);\n} // namespace XReg\n\n// Namespace containing all 32-bit GPR register objects.\nnamespace WReg {\n constexpr static WRegister w0(0);\n constexpr static WRegister w1(1);\n constexpr static WRegister w2(2);\n constexpr static WRegister w3(3);\n constexpr static WRegister w4(4);\n constexpr static WRegister w5(5);\n constexpr static WRegister w6(6);\n constexpr static WRegister w7(7);\n constexpr static WRegister w8(8);\n constexpr static WRegister w9(9);\n constexpr static WRegister w10(10);\n constexpr static WRegister w11(11);\n constexpr static WRegister w12(12);\n constexpr static WRegister w13(13);\n constexpr static WRegister w14(14);\n constexpr static WRegister w15(15);\n constexpr static WRegister w16(16);\n constexpr static WRegister w17(17);\n constexpr static WRegister w18(18);\n constexpr static WRegister w19(19);\n constexpr static WRegister w20(20);\n constexpr static WRegister w21(21);\n constexpr static WRegister w22(22);\n constexpr static WRegister w23(23);\n constexpr static WRegister w24(24);\n constexpr static WRegister w25(25);\n constexpr static WRegister w26(26);\n constexpr static WRegister w27(27);\n constexpr static WRegister w28(28);\n constexpr static WRegister w29(29);\n constexpr static WRegister w30(30);\n constexpr static WRegister w31(31);\n\n // Named registers\n constexpr static WRegister ip0(16);\n constexpr static WRegister ip1(17);\n\n constexpr static WRegister fp(29);\n constexpr static WRegister lr(30);\n constexpr static WRegister rsp(31);\n constexpr static WRegister zr(31);\n} // namespace WReg\n\nclass VRegister;\nclass BRegister;\nclass HRegister;\nclass SRegister;\nclass DRegister;\nclass QRegister;\nclass ZRegister;\n\n/* Unsized ASIMD register class\n * This class doesn't imply a size when used, nor implies Vector or Scalar.\n * It does imply that this instruction isn't using the register for SVE.\n */\nclass VRegister {\npublic:\n VRegister() = delete;\n constexpr explicit VRegister(uint32_t Idx)\n : Index {Idx} {}\n\n friend constexpr auto operator<=>(const VRegister&, const VRegister&) = default;\n\n constexpr uint32_t Idx() const {\n return Index;\n }\n\n constexpr BRegister B() const;\n constexpr HRegister H() const;\n constexpr SRegister S() const;\n constexpr DRegister D() const;\n constexpr QRegister Q() const;\n constexpr ZRegister Z() const;\n\nprivate:\n uint32_t Index;\n};\nstatic_assert(sizeof(VRegister) == sizeof(uint32_t));\nstatic_assert(std::is_trivially_copyable_v);\nstatic_assert(std::is_standard_layout_v);\n\n/* 8-bit ASIMD register class\n * This class implies 8-bit scalar register.\n */\nclass BRegister {\npublic:\n BRegister() = delete;\n constexpr explicit BRegister(uint32_t Idx)\n : Index {Idx} {}\n\n friend constexpr auto operator<=>(const BRegister&, const BRegister&) = default;\n\n constexpr uint32_t Idx() const {\n return Index;\n }\n\n constexpr operator VRegister() const {\n return VRegister(Index);\n }\n constexpr BRegister V() const;\n constexpr HRegister H() const;\n constexpr SRegister S() const;\n constexpr DRegister D() const;\n constexpr QRegister Q() const;\n constexpr ZRegister Z() const;\n\nprivate:\n uint32_t Index;\n};\nstatic_assert(sizeof(BRegister) == sizeof(uint32_t));\nstatic_assert(std::is_trivially_copyable_v);\nstatic_assert(std::is_standard_layout_v);\n\n/* 16-bit ASIMD register class\n * This class implies 16-bit scalar register.\n */\nclass HRegister {\npublic:\n HRegister() = delete;\n constexpr explicit HRegister(uint32_t Idx)\n : Index {Idx} {}\n\n friend constexpr auto operator<=>(const HRegister&, const HRegister&) = default;\n\n constexpr uint32_t Idx() const {\n return Index;\n }\n\n constexpr operator VRegister() const {\n return VRegister(Index);\n }\n constexpr HRegister V() const;\n constexpr BRegister B() const;\n constexpr SRegister S() const;\n constexpr DRegister D() const;\n constexpr QRegister Q() const;\n constexpr ZRegister Z() const;\n\nprivate:\n uint32_t Index;\n};\nstatic_assert(sizeof(HRegister) == sizeof(uint32_t));\nstatic_assert(std::is_trivially_copyable_v);\nstatic_assert(std::is_standard_layout_v);\n\n/* 32-bit ASIMD register class\n * This class implies 32-bit scalar register.\n */\nclass SRegister {\npublic:\n SRegister() = delete;\n constexpr explicit SRegister(uint32_t Idx)\n : Index {Idx} {}\n\n friend constexpr auto operator<=>(const SRegister&, const SRegister&) = default;\n\n constexpr uint32_t Idx() const {\n return Index;\n }\n\n constexpr operator VRegister() const {\n return VRegister(Index);\n }\n constexpr SRegister V() const;\n constexpr BRegister B() const;\n constexpr HRegister H() const;\n constexpr DRegister D() const;\n constexpr QRegister Q() const;\n constexpr ZRegister Z() const;\n\nprivate:\n uint32_t Index;\n};\nstatic_assert(sizeof(SRegister) == sizeof(uint32_t));\nstatic_assert(std::is_trivially_copyable_v);\nstatic_assert(std::is_standard_layout_v);\n\n/* 64-bit ASIMD register class\n * This class doesn't imply Vector or Scalar.\n * Associated with operating the instruction at 64-bit.\n */\nclass DRegister {\npublic:\n DRegister() = delete;\n constexpr explicit DRegister(uint32_t Idx)\n : Index {Idx} {}\n\n friend constexpr auto operator<=>(const DRegister&, const DRegister&) = default;\n\n constexpr uint32_t Idx() const {\n return Index;\n }\n\n constexpr operator VRegister() const {\n return VRegister(Index);\n }\n constexpr DRegister V() const;\n constexpr BRegister B() const;\n constexpr HRegister H() const;\n constexpr SRegister S() const;\n constexpr QRegister Q() const;\n constexpr ZRegister Z() const;\n\nprivate:\n uint32_t Index;\n};\nstatic_assert(sizeof(DRegister) == sizeof(uint32_t));\nstatic_assert(std::is_trivially_copyable_v);\nstatic_assert(std::is_standard_layout_v);\n\n/* 128-bit ASIMD register class\n * This class doesn't imply Vector or Scalar.\n * Associated with operating the instruction at 128-bit.\n */\nclass QRegister {\npublic:\n QRegister() = delete;\n constexpr explicit QRegister(uint32_t Idx)\n : Index {Idx} {}\n\n friend constexpr auto operator<=>(const QRegister&, const QRegister&) = default;\n\n constexpr uint32_t Idx() const {\n return Index;\n }\n\n constexpr operator VRegister() const {\n return VRegister(Index);\n }\n constexpr QRegister V() const;\n constexpr BRegister B() const;\n constexpr HRegister H() const;\n constexpr SRegister S() const;\n constexpr DRegister D() const;\n constexpr ZRegister Z() const;\n\nprivate:\n uint32_t Index;\n};\nstatic_assert(sizeof(QRegister) == sizeof(uint32_t));\nstatic_assert(std::is_trivially_copyable_v);\nstatic_assert(std::is_standard_layout_v);\n\n/* Unsized SVE register class.\n * This class explicitly implies the instruction will operate using SVE.\n */\nclass ZRegister {\npublic:\n ZRegister() = delete;\n constexpr explicit ZRegister(uint32_t Idx)\n : Index {Idx} {}\n\n friend constexpr auto operator<=>(const ZRegister&, const ZRegister&) = default;\n\n constexpr uint32_t Idx() const {\n return Index;\n }\n\n constexpr VRegister V() const;\n constexpr BRegister B() const;\n constexpr HRegister H() const;\n constexpr SRegister S() const;\n constexpr DRegister D() const;\n constexpr QRegister Q() const;\n\nprivate:\n uint32_t Index;\n};\nstatic_assert(sizeof(ZRegister) == sizeof(uint32_t));\nstatic_assert(std::is_trivially_copyable_v);\nstatic_assert(std::is_standard_layout_v);\n\n// VRegister\ninline constexpr BRegister VRegister::B() const {\n return BRegister {Index};\n}\ninline constexpr HRegister VRegister::H() const {\n return HRegister {Index};\n}\ninline constexpr SRegister VRegister::S() const {\n return SRegister {Index};\n}\ninline constexpr DRegister VRegister::D() const {\n return DRegister {Index};\n}\ninline constexpr QRegister VRegister::Q() const {\n return QRegister {Index};\n}\ninline constexpr ZRegister VRegister::Z() const {\n return ZRegister {Index};\n}\n\n// BRegister\ninline constexpr BRegister BRegister::V() const {\n return *this;\n}\ninline constexpr HRegister BRegister::H() const {\n return HRegister {Index};\n}\ninline constexpr SRegister BRegister::S() const {\n return SRegister {Index};\n}\ninline constexpr DRegister BRegister::D() const {\n return DRegister {Index};\n}\ninline constexpr QRegister BRegister::Q() const {\n return QRegister {Index};\n}\ninline constexpr ZRegister BRegister::Z() const {\n return ZRegister {Index};\n}\n\n// HRegister\ninline constexpr HRegister HRegister::V() const {\n return *this;\n}\ninline constexpr BRegister HRegister::B() const {\n return BRegister {Index};\n}\ninline constexpr SRegister HRegister::S() const {\n return SRegister {Index};\n}\ninline constexpr DRegister HRegister::D() const {\n return DRegister {Index};\n}\ninline constexpr QRegister HRegister::Q() const {\n return QRegister {Index};\n}\ninline constexpr ZRegister HRegister::Z() const {\n return ZRegister {Index};\n}\n\n// SRegister\ninline constexpr SRegister SRegister::V() const {\n return *this;\n}\ninline constexpr BRegister SRegister::B() const {\n return BRegister {Index};\n}\ninline constexpr HRegister SRegister::H() const {\n return HRegister {Index};\n}\ninline constexpr DRegister SRegister::D() const {\n return DRegister {Index};\n}\ninline constexpr QRegister SRegister::Q() const {\n return QRegister {Index};\n}\ninline constexpr ZRegister SRegister::Z() const {\n return ZRegister {Index};\n}\n\n// DRegister\ninline constexpr DRegister DRegister::V() const {\n return DRegister {Index};\n}\ninline constexpr BRegister DRegister::B() const {\n return BRegister {Index};\n}\ninline constexpr HRegister DRegister::H() const {\n return HRegister {Index};\n}\ninline constexpr SRegister DRegister::S() const {\n return SRegister {Index};\n}\ninline constexpr QRegister DRegister::Q() const {\n return QRegister {Index};\n}\ninline constexpr ZRegister DRegister::Z() const {\n return ZRegister {Index};\n}\n\n// QRegister\ninline constexpr QRegister QRegister::V() const {\n return *this;\n}\ninline constexpr BRegister QRegister::B() const {\n return BRegister {Index};\n}\ninline constexpr HRegister QRegister::H() const {\n return HRegister {Index};\n}\ninline constexpr SRegister QRegister::S() const {\n return SRegister {Index};\n}\ninline constexpr DRegister QRegister::D() const {\n return DRegister {Index};\n}\ninline constexpr ZRegister QRegister::Z() const {\n return ZRegister {Index};\n}\n\n// ZRegister\ninline constexpr VRegister ZRegister::V() const {\n return VRegister(Index);\n}\ninline constexpr BRegister ZRegister::B() const {\n return BRegister(Index);\n}\ninline constexpr HRegister ZRegister::H() const {\n return HRegister(Index);\n}\ninline constexpr SRegister ZRegister::S() const {\n return SRegister(Index);\n}\ninline constexpr DRegister ZRegister::D() const {\n return DRegister(Index);\n}\ninline constexpr QRegister ZRegister::Q() const {\n return QRegister(Index);\n}\n\n// Namespace containing all unsized ASIMD register objects.\nnamespace VReg {\n constexpr static VRegister v0(0);\n constexpr static VRegister v1(1);\n constexpr static VRegister v2(2);\n constexpr static VRegister v3(3);\n constexpr static VRegister v4(4);\n constexpr static VRegister v5(5);\n constexpr static VRegister v6(6);\n constexpr static VRegister v7(7);\n constexpr static VRegister v8(8);\n constexpr static VRegister v9(9);\n constexpr static VRegister v10(10);\n constexpr static VRegister v11(11);\n constexpr static VRegister v12(12);\n constexpr static VRegister v13(13);\n constexpr static VRegister v14(14);\n constexpr static VRegister v15(15);\n constexpr static VRegister v16(16);\n constexpr static VRegister v17(17);\n constexpr static VRegister v18(18);\n constexpr static VRegister v19(19);\n constexpr static VRegister v20(20);\n constexpr static VRegister v21(21);\n constexpr static VRegister v22(22);\n constexpr static VRegister v23(23);\n constexpr static VRegister v24(24);\n constexpr static VRegister v25(25);\n constexpr static VRegister v26(26);\n constexpr static VRegister v27(27);\n constexpr static VRegister v28(28);\n constexpr static VRegister v29(29);\n constexpr static VRegister v30(30);\n constexpr static VRegister v31(31);\n} // namespace VReg\n\n// Namespace containing all 8-bit ASIMD register objects.\nnamespace BReg {\n constexpr static BRegister b0(0);\n constexpr static BRegister b1(1);\n constexpr static BRegister b2(2);\n constexpr static BRegister b3(3);\n constexpr static BRegister b4(4);\n constexpr static BRegister b5(5);\n constexpr static BRegister b6(6);\n constexpr static BRegister b7(7);\n constexpr static BRegister b8(8);\n constexpr static BRegister b9(9);\n constexpr static BRegister b10(10);\n constexpr static BRegister b11(11);\n constexpr static BRegister b12(12);\n constexpr static BRegister b13(13);\n constexpr static BRegister b14(14);\n constexpr static BRegister b15(15);\n constexpr static BRegister b16(16);\n constexpr static BRegister b17(17);\n constexpr static BRegister b18(18);\n constexpr static BRegister b19(19);\n constexpr static BRegister b20(20);\n constexpr static BRegister b21(21);\n constexpr static BRegister b22(22);\n constexpr static BRegister b23(23);\n constexpr static BRegister b24(24);\n constexpr static BRegister b25(25);\n constexpr static BRegister b26(26);\n constexpr static BRegister b27(27);\n constexpr static BRegister b28(28);\n constexpr static BRegister b29(29);\n constexpr static BRegister b30(30);\n constexpr static BRegister b31(31);\n} // namespace BReg\n\n// Namespace containing all 16-bit ASIMD register objects.\nnamespace HReg {\n constexpr static HRegister h0(0);\n constexpr static HRegister h1(1);\n constexpr static HRegister h2(2);\n constexpr static HRegister h3(3);\n constexpr static HRegister h4(4);\n constexpr static HRegister h5(5);\n constexpr static HRegister h6(6);\n constexpr static HRegister h7(7);\n constexpr static HRegister h8(8);\n constexpr static HRegister h9(9);\n constexpr static HRegister h10(10);\n constexpr static HRegister h11(11);\n constexpr static HRegister h12(12);\n constexpr static HRegister h13(13);\n constexpr static HRegister h14(14);\n constexpr static HRegister h15(15);\n constexpr static HRegister h16(16);\n constexpr static HRegister h17(17);\n constexpr static HRegister h18(18);\n constexpr static HRegister h19(19);\n constexpr static HRegister h20(20);\n constexpr static HRegister h21(21);\n constexpr static HRegister h22(22);\n constexpr static HRegister h23(23);\n constexpr static HRegister h24(24);\n constexpr static HRegister h25(25);\n constexpr static HRegister h26(26);\n constexpr static HRegister h27(27);\n constexpr static HRegister h28(28);\n constexpr static HRegister h29(29);\n constexpr static HRegister h30(30);\n constexpr static HRegister h31(31);\n} // namespace HReg\n\n// Namespace containing all 32-bit ASIMD register objects.\nnamespace SReg {\n constexpr static SRegister s0(0);\n constexpr static SRegister s1(1);\n constexpr static SRegister s2(2);\n constexpr static SRegister s3(3);\n constexpr static SRegister s4(4);\n constexpr static SRegister s5(5);\n constexpr static SRegister s6(6);\n constexpr static SRegister s7(7);\n constexpr static SRegister s8(8);\n constexpr static SRegister s9(9);\n constexpr static SRegister s10(10);\n constexpr static SRegister s11(11);\n constexpr static SRegister s12(12);\n constexpr static SRegister s13(13);\n constexpr static SRegister s14(14);\n constexpr static SRegister s15(15);\n constexpr static SRegister s16(16);\n constexpr static SRegister s17(17);\n constexpr static SRegister s18(18);\n constexpr static SRegister s19(19);\n constexpr static SRegister s20(20);\n constexpr static SRegister s21(21);\n constexpr static SRegister s22(22);\n constexpr static SRegister s23(23);\n constexpr static SRegister s24(24);\n constexpr static SRegister s25(25);\n constexpr static SRegister s26(26);\n constexpr static SRegister s27(27);\n constexpr static SRegister s28(28);\n constexpr static SRegister s29(29);\n constexpr static SRegister s30(30);\n constexpr static SRegister s31(31);\n} // namespace SReg\n\n// Namespace containing all 64-bit ASIMD register objects.\nnamespace DReg {\n constexpr static DRegister d0(0);\n constexpr static DRegister d1(1);\n constexpr static DRegister d2(2);\n constexpr static DRegister d3(3);\n constexpr static DRegister d4(4);\n constexpr static DRegister d5(5);\n constexpr static DRegister d6(6);\n constexpr static DRegister d7(7);\n constexpr static DRegister d8(8);\n constexpr static DRegister d9(9);\n constexpr static DRegister d10(10);\n constexpr static DRegister d11(11);\n constexpr static DRegister d12(12);\n constexpr static DRegister d13(13);\n constexpr static DRegister d14(14);\n constexpr static DRegister d15(15);\n constexpr static DRegister d16(16);\n constexpr static DRegister d17(17);\n constexpr static DRegister d18(18);\n constexpr static DRegister d19(19);\n constexpr static DRegister d20(20);\n constexpr static DRegister d21(21);\n constexpr static DRegister d22(22);\n constexpr static DRegister d23(23);\n constexpr static DRegister d24(24);\n constexpr static DRegister d25(25);\n constexpr static DRegister d26(26);\n constexpr static DRegister d27(27);\n constexpr static DRegister d28(28);\n constexpr static DRegister d29(29);\n constexpr static DRegister d30(30);\n constexpr static DRegister d31(31);\n} // namespace DReg\n\n// Namespace containing all 128-bit ASIMD register objects.\nnamespace QReg {\n constexpr static QRegister q0(0);\n constexpr static QRegister q1(1);\n constexpr static QRegister q2(2);\n constexpr static QRegister q3(3);\n constexpr static QRegister q4(4);\n constexpr static QRegister q5(5);\n constexpr static QRegister q6(6);\n constexpr static QRegister q7(7);\n constexpr static QRegister q8(8);\n constexpr static QRegister q9(9);\n constexpr static QRegister q10(10);\n constexpr static QRegister q11(11);\n constexpr static QRegister q12(12);\n constexpr static QRegister q13(13);\n constexpr static QRegister q14(14);\n constexpr static QRegister q15(15);\n constexpr static QRegister q16(16);\n constexpr static QRegister q17(17);\n constexpr static QRegister q18(18);\n constexpr static QRegister q19(19);\n constexpr static QRegister q20(20);\n constexpr static QRegister q21(21);\n constexpr static QRegister q22(22);\n constexpr static QRegister q23(23);\n constexpr static QRegister q24(24);\n constexpr static QRegister q25(25);\n constexpr static QRegister q26(26);\n constexpr static QRegister q27(27);\n constexpr static QRegister q28(28);\n constexpr static QRegister q29(29);\n constexpr static QRegister q30(30);\n constexpr static QRegister q31(31);\n} // namespace QReg\n\n// Namespace containing all unsigned SVE register objects.\nnamespace ZReg {\n constexpr static ZRegister z0(0);\n constexpr static ZRegister z1(1);\n constexpr static ZRegister z2(2);\n constexpr static ZRegister z3(3);\n constexpr static ZRegister z4(4);\n constexpr static ZRegister z5(5);\n constexpr static ZRegister z6(6);\n constexpr static ZRegister z7(7);\n constexpr static ZRegister z8(8);\n constexpr static ZRegister z9(9);\n constexpr static ZRegister z10(10);\n constexpr static ZRegister z11(11);\n constexpr static ZRegister z12(12);\n constexpr static ZRegister z13(13);\n constexpr static ZRegister z14(14);\n constexpr static ZRegister z15(15);\n constexpr static ZRegister z16(16);\n constexpr static ZRegister z17(17);\n constexpr static ZRegister z18(18);\n constexpr static ZRegister z19(19);\n constexpr static ZRegister z20(20);\n constexpr static ZRegister z21(21);\n constexpr static ZRegister z22(22);\n constexpr static ZRegister z23(23);\n constexpr static ZRegister z24(24);\n constexpr static ZRegister z25(25);\n constexpr static ZRegister z26(26);\n constexpr static ZRegister z27(27);\n constexpr static ZRegister z28(28);\n constexpr static ZRegister z29(29);\n constexpr static ZRegister z30(30);\n constexpr static ZRegister z31(31);\n} // namespace ZReg\n\n// Zero-cost FPR->GPR\ninline constexpr Register ToReg(HRegister Reg) {\n return Register(Reg.Idx());\n}\ninline constexpr Register ToReg(SRegister Reg) {\n return Register(Reg.Idx());\n}\ninline constexpr Register ToReg(DRegister Reg) {\n return Register(Reg.Idx());\n}\ninline constexpr Register ToReg(VRegister Reg) {\n return Register(Reg.Idx());\n}\n\n// Zero-cost GPR->FPR\ninline constexpr VRegister ToVReg(Register Reg) {\n return VRegister(Reg.Idx());\n}\ninline constexpr VRegister ToVReg(XRegister Reg) {\n return VRegister(Reg.Idx());\n}\ninline constexpr VRegister ToVReg(WRegister Reg) {\n return VRegister(Reg.Idx());\n}\n\nclass PRegisterZero;\nclass PRegisterMerge;\n\n/* Unsized predicate register for SVE.\n * This is unsized because of how SVE operates.\n */\nclass PRegister {\npublic:\n PRegister() = delete;\n constexpr PRegister(uint32_t Idx)\n : Index {Idx} {}\n\n friend constexpr auto operator<=>(const PRegister&, const PRegister&) = default;\n\n constexpr uint32_t Idx() const {\n return Index;\n }\n\n constexpr PRegisterZero Zeroing() const;\n constexpr PRegisterMerge Merging() const;\n\nprivate:\n uint32_t Index;\n};\nstatic_assert(sizeof(PRegister) == sizeof(uint32_t));\nstatic_assert(std::is_trivially_copyable_v);\nstatic_assert(std::is_standard_layout_v);\n\n// Unsized predicate register for SVE with zeroing semantics.\nclass PRegisterZero {\npublic:\n PRegisterZero() = delete;\n constexpr PRegisterZero(uint32_t Idx)\n : Index {Idx} {}\n\n friend constexpr auto operator<=>(const PRegisterZero&, const PRegisterZero&) = default;\n\n constexpr uint32_t Idx() const {\n return Index;\n }\n\n constexpr operator PRegister() const {\n return PRegister(Index);\n }\n constexpr PRegister P() const {\n return PRegister(Index);\n }\n constexpr PRegisterMerge Merging() const;\n\nprivate:\n uint32_t Index;\n};\nstatic_assert(sizeof(PRegisterZero) == sizeof(uint32_t));\nstatic_assert(std::is_trivially_copyable_v);\nstatic_assert(std::is_standard_layout_v);\n\n// Unsized predicate register for SVE with merging semantics.\nclass PRegisterMerge {\npublic:\n PRegisterMerge() = delete;\n constexpr PRegisterMerge(uint32_t Idx)\n : Index {Idx} {}\n\n friend constexpr auto operator<=>(const PRegisterMerge&, const PRegisterMerge&) = default;\n\n constexpr uint32_t Idx() const {\n return Index;\n }\n\n constexpr operator PRegister() const {\n return PRegister(Index);\n }\n constexpr PRegister P() const {\n return PRegister(Index);\n }\n constexpr PRegisterZero Zeroing() const;\n\nprivate:\n uint32_t Index;\n};\nstatic_assert(sizeof(PRegisterMerge) == sizeof(uint32_t));\nstatic_assert(std::is_trivially_copyable_v);\nstatic_assert(std::is_standard_layout_v);\n\n// PRegister\ninline constexpr PRegisterZero PRegister::Zeroing() const {\n return PRegisterZero(Idx());\n}\ninline constexpr PRegisterMerge PRegister::Merging() const {\n return PRegisterMerge(Idx());\n}\n\n// PRegisterZero\ninline constexpr PRegisterMerge PRegisterZero::Merging() const {\n return PRegisterMerge(Idx());\n}\n\n// PRegisterMerge\ninline constexpr PRegisterZero PRegisterMerge::Zeroing() const {\n return PRegisterZero(Idx());\n}\n\n// Namespace containing all unsigned SVE predicate register objects.\nnamespace PReg {\n constexpr static PRegister p0(0);\n constexpr static PRegister p1(1);\n constexpr static PRegister p2(2);\n constexpr static PRegister p3(3);\n constexpr static PRegister p4(4);\n constexpr static PRegister p5(5);\n constexpr static PRegister p6(6);\n constexpr static PRegister p7(7);\n constexpr static PRegister p8(8);\n constexpr static PRegister p9(9);\n constexpr static PRegister p10(10);\n constexpr static PRegister p11(11);\n constexpr static PRegister p12(12);\n constexpr static PRegister p13(13);\n constexpr static PRegister p14(14);\n constexpr static PRegister p15(15);\n} // namespace PReg\n\n/* `OpType` enum describes how some SVE instructions operate if they support both forms.\n * Not all SVE instructions support this.\n */\nenum class OpType : uint32_t {\n Destructive = 0,\n Constructive,\n};\n} // namespace ARMEmitter\n" }, { "path": "CodeEmitter/CodeEmitter/SVEOps.inl", "content": "// SPDX-License-Identifier: MIT\n/* SVE instruction emitters\n * These contain instruction emitters for AArch64 SVE and SVE2 operations.\n *\n * All of these SVE emitters have a `SubRegSize` as their first argument to set the element size on the instruction.\n * Since nearly every SVE instruction is unsized they don't need more than `ZRegister` and `PRegister` arguments.\n *\n * Most predicated instructions take a `PRegister` argument, not explicitly stating if it is merging or zeroing behaviour.\n * This is because the instruction only supports one style.\n * For instructions that take an explicit `PRegisterMerge` or `PRegisterZero`, then this instruction likely\n * supports both so we support both implementations depending on predicate register type.\n *\n * Some instructions take a templated `OpType` to choose between a destructive or constructive version of the instruction.\n *\n * Some instructions support the `i128Bit` SubRegSize, mostly around data movement.\n *\n * There are some SVE load-store helper functions which take a `SVEMemOperand` argument.\n * This helper will select the viable SVE load-store that can work with the provided encapsulated arguments.\n */\n\n#pragma once\n#ifndef INCLUDED_BY_EMITTER\n#include \nnamespace ARMEmitter {\nstruct EmitterOps : Emitter {\n#endif\n\npublic:\n // SVE encodings\n void dup(SubRegSize size, ZRegister zd, ZRegister zn, uint32_t Index) {\n SVEDupIndexed(size, zn, zd, Index);\n }\n\n void sel(SubRegSize size, ZRegister zd, PRegister pv, ZRegister zn, ZRegister zm) {\n SVESel(size, zm, pv, zn, zd);\n }\n void mov(SubRegSize size, ZRegister zd, PRegisterMerge pv, ZRegister zn) {\n sel(size, zd, pv, zn, zd);\n }\n\n void histcnt(SubRegSize size, ZRegister zd, PRegisterZero pv, ZRegister zn, ZRegister zm) {\n LOGMAN_THROW_A_FMT(size == SubRegSize::i32Bit || size == SubRegSize::i64Bit, \"SubRegSize must be 32-bit or 64-bit\");\n LOGMAN_THROW_A_FMT(pv <= PReg::p7.Zeroing(), \"histcnt can only use p0 to p7\");\n\n uint32_t Op = 0b0100'0101'0010'0000'1100'0000'0000'0000;\n Op |= FEXCore::ToUnderlying(size) << 22;\n Op |= zm.Idx() << 16;\n Op |= pv.Idx() << 10;\n Op |= zn.Idx() << 5;\n Op |= zd.Idx();\n dc32(Op);\n }\n\n void histseg(ZRegister zd, ZRegister zn, ZRegister zm) {\n uint32_t Op = 0b0100'0101'0010'0000'1010'0000'0000'0000;\n Op |= zm.Idx() << 16;\n Op |= zn.Idx() << 5;\n Op |= zd.Idx();\n dc32(Op);\n }\n\n void fcmla(SubRegSize size, ZRegister zda, PRegisterMerge pv, ZRegister zn, ZRegister zm, Rotation rot) {\n LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), \"SubRegSize must be 16-bit, 32-bit, or 64-bit\");\n LOGMAN_THROW_A_FMT(pv <= PReg::p7.Merging(), \"fcmla can only use p0 to p7\");\n\n uint32_t Op = 0b0110'0100'0000'0000'0000'0000'0000'0000;\n Op |= FEXCore::ToUnderlying(size) << 22;\n Op |= zm.Idx() << 16;\n Op |= FEXCore::ToUnderlying(rot) << 13;\n Op |= pv.Idx() << 10;\n Op |= zn.Idx() << 5;\n Op |= zda.Idx();\n\n dc32(Op);\n }\n\n void fcadd(SubRegSize size, ZRegister zd, PRegisterMerge pv, ZRegister zn, ZRegister zm, Rotation rot) {\n LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), \"SubRegSize must be 16-bit, 32-bit, or 64-bit\");\n LOGMAN_THROW_A_FMT(pv <= PReg::p7.Merging(), \"fcadd can only use p0 to p7\");\n LOGMAN_THROW_A_FMT(rot == Rotation::ROTATE_90 || rot == Rotation::ROTATE_270, \"fcadd rotation may only be 90 or 270 degrees\");\n LOGMAN_THROW_A_FMT(zd == zn, \"fcadd zd and zn must be the same register\");\n\n const uint32_t ConvertedRotation = rot == Rotation::ROTATE_90 ? 0 : 1;\n\n uint32_t Op = 0b0110'0100'0000'0000'1000'0000'0000'0000;\n Op |= FEXCore::ToUnderlying(size) << 22;\n Op |= ConvertedRotation << 16;\n Op |= pv.Idx() << 10;\n Op |= zm.Idx() << 5;\n Op |= zd.Idx();\n\n dc32(Op);\n }\n\n // SVE integer add/subtract vectors (unpredicated)\n void add(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n SVEIntegerAddSubUnpredicated(0b000, size, zm, zn, zd);\n }\n void sub(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n SVEIntegerAddSubUnpredicated(0b001, size, zm, zn, zd);\n }\n void sqadd(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n SVEIntegerAddSubUnpredicated(0b100, size, zm, zn, zd);\n }\n void uqadd(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n SVEIntegerAddSubUnpredicated(0b101, size, zm, zn, zd);\n }\n void sqsub(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n SVEIntegerAddSubUnpredicated(0b110, size, zm, zn, zd);\n }\n void uqsub(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n SVEIntegerAddSubUnpredicated(0b111, size, zm, zn, zd);\n }\n\n // SVE address generation\n void adr(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm, SVEModType mod = SVEModType::MOD_NONE, uint32_t scale = 0) {\n SVEAddressGeneration(size, zd, zn, zm, mod, scale);\n }\n\n // SVE table lookup (three sources)\n void tbl(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n SVETableLookup(0b100, size, zm, zn, zd);\n }\n void tbl(SubRegSize size, ZRegister zd, ZRegister zn1, ZRegister zn2, ZRegister zm) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(zn1, zn2), \"TBL zn1 and zn2 must be sequential\");\n SVETableLookup(0b010, size, zm, zn1, zd);\n }\n void tbx(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n SVETableLookup(0b011, size, zm, zn, zd);\n }\n\n // SVE permute vector elements\n void zip1(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n SVEPermute(0b000, size, zm, zn, zd);\n }\n void zip2(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n SVEPermute(0b001, size, zm, zn, zd);\n }\n void uzp1(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n SVEPermute(0b010, size, zm, zn, zd);\n }\n void uzp2(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n SVEPermute(0b011, size, zm, zn, zd);\n }\n void trn1(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n SVEPermute(0b100, size, zm, zn, zd);\n }\n void trn2(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n SVEPermute(0b101, size, zm, zn, zd);\n }\n\n // SVE integer compare with unsigned immediate\n void cmphi(SubRegSize size, PRegister pd, PRegisterZero pg, ZRegister zn, uint32_t imm) {\n SVEIntegerCompareImm(0, 1, imm, size, pg, zn, pd);\n }\n void cmphs(SubRegSize size, PRegister pd, PRegisterZero pg, ZRegister zn, uint32_t imm) {\n SVEIntegerCompareImm(0, 0, imm, size, pg, zn, pd);\n }\n void cmplo(SubRegSize size, PRegister pd, PRegisterZero pg, ZRegister zn, uint32_t imm) {\n SVEIntegerCompareImm(1, 0, imm, size, pg, zn, pd);\n }\n void cmpls(SubRegSize size, PRegister pd, PRegisterZero pg, ZRegister zn, uint32_t imm) {\n SVEIntegerCompareImm(1, 1, imm, size, pg, zn, pd);\n }\n\n // SVE integer compare with signed immediate\n void cmpeq(SubRegSize size, PRegister pd, PRegisterZero pg, ZRegister zn, int32_t imm) {\n SVEIntegerCompareSignedImm(1, 0, 0, imm, size, pg, zn, pd);\n }\n void cmpgt(SubRegSize size, PRegister pd, PRegisterZero pg, ZRegister zn, int32_t imm) {\n SVEIntegerCompareSignedImm(0, 0, 1, imm, size, pg, zn, pd);\n }\n void cmpge(SubRegSize size, PRegister pd, PRegisterZero pg, ZRegister zn, int32_t imm) {\n SVEIntegerCompareSignedImm(0, 0, 0, imm, size, pg, zn, pd);\n }\n void cmplt(SubRegSize size, PRegister pd, PRegisterZero pg, ZRegister zn, int32_t imm) {\n SVEIntegerCompareSignedImm(0, 1, 0, imm, size, pg, zn, pd);\n }\n void cmple(SubRegSize size, PRegister pd, PRegisterZero pg, ZRegister zn, int32_t imm) {\n SVEIntegerCompareSignedImm(0, 1, 1, imm, size, pg, zn, pd);\n }\n void cmpne(SubRegSize size, PRegister pd, PRegisterZero pg, ZRegister zn, int32_t imm) {\n SVEIntegerCompareSignedImm(1, 0, 1, imm, size, pg, zn, pd);\n }\n\n // SVE predicate logical operations\n void and_(PRegister pd, PRegisterZero pg, PRegister pn, PRegister pm) {\n SVEPredicateLogical(0, 0, 0, 0, pm, pg, pn, pd);\n }\n void ands(PRegister pd, PRegisterZero pg, PRegister pn, PRegister pm) {\n SVEPredicateLogical(0, 1, 0, 0, pm, pg, pn, pd);\n }\n\n void mov(PRegister pd, PRegisterMerge pg, PRegister pn) {\n SVEPredicateLogical(0, 0, 1, 1, pd, pg, pn, pd);\n }\n void mov(PRegister pd, PRegisterZero pg, PRegister pn) {\n SVEPredicateLogical(0, 0, 0, 0, pn, pg, pn, pd);\n }\n\n void movs(PRegister pd, PRegisterZero pg, PRegister pn) {\n SVEPredicateLogical(0, 1, 0, 0, pn, pg, pn, pd);\n }\n void bic(PRegister pd, PRegisterZero pg, PRegister pn, PRegister pm) {\n SVEPredicateLogical(0, 0, 0, 1, pm, pg, pn, pd);\n }\n void bics(PRegister pd, PRegisterZero pg, PRegister pn, PRegister pm) {\n SVEPredicateLogical(0, 1, 0, 1, pm, pg, pn, pd);\n }\n\n void eor(PRegister pd, PRegisterZero pg, PRegister pn, PRegister pm) {\n SVEPredicateLogical(0, 0, 1, 0, pm, pg, pn, pd);\n }\n void eors(PRegister pd, PRegisterZero pg, PRegister pn, PRegister pm) {\n SVEPredicateLogical(0, 1, 1, 0, pm, pg, pn, pd);\n }\n\n void not_(PRegister pd, PRegisterZero pg, PRegister pn) {\n SVEPredicateLogical(0, 0, 1, 0, pg, pg, pn, pd);\n }\n void sel(PRegister pd, PRegister pg, PRegister pn, PRegister pm) {\n SVEPredicateLogical(0, 0, 1, 1, pm, pg, pn, pd);\n }\n void orr(PRegister pd, PRegisterZero pg, PRegister pn, PRegister pm) {\n SVEPredicateLogical(1, 0, 0, 0, pm, pg, pn, pd);\n }\n void mov(PRegister pd, PRegister pn) {\n SVEPredicateLogical(1, 0, 0, 0, pn, pn, pn, pd);\n }\n void orn(PRegister pd, PRegisterZero pg, PRegister pn, PRegister pm) {\n SVEPredicateLogical(1, 0, 0, 1, pm, pg, pn, pd);\n }\n void nor(PRegister pd, PRegisterZero pg, PRegister pn, PRegister pm) {\n SVEPredicateLogical(1, 0, 1, 0, pm, pg, pn, pd);\n }\n void nand(PRegister pd, PRegisterZero pg, PRegister pn, PRegister pm) {\n SVEPredicateLogical(1, 0, 1, 1, pm, pg, pn, pd);\n }\n void orrs(PRegister pd, PRegisterZero pg, PRegister pn, PRegister pm) {\n SVEPredicateLogical(1, 1, 0, 0, pm, pg, pn, pd);\n }\n void movs(PRegister pd, PRegister pn) {\n SVEPredicateLogical(1, 1, 0, 0, pn, pn, pn, pd);\n }\n void orns(PRegister pd, PRegisterZero pg, PRegister pn, PRegister pm) {\n SVEPredicateLogical(1, 1, 0, 1, pm, pg, pn, pd);\n }\n void nors(PRegister pd, PRegisterZero pg, PRegister pn, PRegister pm) {\n SVEPredicateLogical(1, 1, 1, 0, pm, pg, pn, pd);\n }\n void nands(PRegister pd, PRegisterZero pg, PRegister pn, PRegister pm) {\n SVEPredicateLogical(1, 1, 1, 1, pm, pg, pn, pd);\n }\n\n // SVE broadcast predicate element\n // XXX:\n\n // SVE integer clamp\n // XXX:\n\n // SVE2 character match\n void match(SubRegSize size, PRegister pd, PRegisterZero pg, ZRegister zn, ZRegister zm) {\n SVECharacterMatch(0, size, pd, pg, zn, zm);\n }\n void nmatch(SubRegSize size, PRegister pd, PRegisterZero pg, ZRegister zn, ZRegister zm) {\n SVECharacterMatch(1, size, pd, pg, zn, zm);\n }\n\n // SVE floating-point convert precision odd elements\n void fcvtxnt(ZRegister zd, PRegisterMerge pg, ZRegister zn) {\n SVEFloatConvertOdd(0b00, 0b10, pg, zn, zd);\n }\n ///< Size is destination size\n void fcvtnt(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn) {\n LOGMAN_THROW_A_FMT(size == SubRegSize::i32Bit || size == SubRegSize::i16Bit, \"Unsupported size in {}\", __func__);\n\n const auto ConvertedDestSize = size == SubRegSize::i16Bit ? 0b00 : size == SubRegSize::i32Bit ? 0b10 : 0b00;\n\n const auto ConvertedSrcSize = size == SubRegSize::i16Bit ? 0b10 : size == SubRegSize::i32Bit ? 0b11 : 0b00;\n\n SVEFloatConvertOdd(ConvertedSrcSize, ConvertedDestSize, pg, zn, zd);\n }\n\n ///< Size is destination size\n void fcvtlt(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn) {\n LOGMAN_THROW_A_FMT(size == SubRegSize::i64Bit || size == SubRegSize::i32Bit, \"Unsupported size in {}\", __func__);\n\n const auto ConvertedDestSize = size == SubRegSize::i32Bit ? 0b01 : size == SubRegSize::i64Bit ? 0b11 : 0b00;\n\n const auto ConvertedSrcSize = size == SubRegSize::i32Bit ? 0b10 : size == SubRegSize::i64Bit ? 0b11 : 0b00;\n\n SVEFloatConvertOdd(ConvertedSrcSize, ConvertedDestSize, pg, zn, zd);\n }\n\n // XXX: BFCVTNT\n\n // SVE2 floating-point pairwise operations\n void faddp(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n SVEFloatPairwiseArithmetic(0b000, size, pg, zd, zn, zm);\n }\n void fmaxnmp(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n SVEFloatPairwiseArithmetic(0b100, size, pg, zd, zn, zm);\n }\n void fminnmp(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n SVEFloatPairwiseArithmetic(0b101, size, pg, zd, zn, zm);\n }\n void fmaxp(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n SVEFloatPairwiseArithmetic(0b110, size, pg, zd, zn, zm);\n }\n void fminp(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n SVEFloatPairwiseArithmetic(0b111, size, pg, zd, zn, zm);\n }\n\n // SVE floating-point multiply-add (indexed)\n void fmla(SubRegSize size, ZRegister zda, ZRegister zn, ZRegister zm, uint32_t index) {\n SVEFPMultiplyAddIndexed(0, size, zda, zn, zm, index);\n }\n void fmls(SubRegSize size, ZRegister zda, ZRegister zn, ZRegister zm, uint32_t index) {\n SVEFPMultiplyAddIndexed(1, size, zda, zn, zm, index);\n }\n\n // SVE floating-point complex multiply-add (indexed)\n void fcmla(SubRegSize size, ZRegister zda, ZRegister zn, ZRegister zm, uint32_t index, Rotation rot) {\n LOGMAN_THROW_A_FMT(size == SubRegSize::i16Bit || size == SubRegSize::i32Bit, \"SubRegSize must be 16-bit or 32-bit\");\n\n // 16 -> 32, 32 -> 64, since fcmla (indexed)'s restrictions and encodings\n // are essentially as if 16-bit were 32-bit and 32-bit were 64-bit.\n const auto DoubledSize = static_cast(FEXCore::ToUnderlying(size) + 1);\n\n SVEFPMultiplyAddIndexed(0b100 | FEXCore::ToUnderlying(rot), DoubledSize, zda, zn, zm, index);\n }\n\n // SVE floating-point multiply (indexed)\n void fmul(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm, uint32_t index) {\n SVEFPMultiplyAddIndexed(0b1000, size, zd, zn, zm, index);\n }\n\n // SVE floating point matrix multiply accumulate\n // XXX: BFMMLA\n void fmmla(SubRegSize size, ZRegister zda, ZRegister zn, ZRegister zm) {\n SVEFPMatrixMultiplyAccumulate(size, zda, zn, zm);\n }\n\n // SVE floating-point compare vectors\n void fcmeq(SubRegSize size, PRegister pd, PRegisterZero pg, ZRegister zn, ZRegister zm) {\n SVEFloatCompareVector(0, 1, 0, size, zm, pg, zn, pd);\n }\n void fcmgt(SubRegSize size, PRegister pd, PRegisterZero pg, ZRegister zn, ZRegister zm) {\n SVEFloatCompareVector(0, 0, 1, size, zm, pg, zn, pd);\n }\n void fcmge(SubRegSize size, PRegister pd, PRegisterZero pg, ZRegister zn, ZRegister zm) {\n SVEFloatCompareVector(0, 0, 0, size, zm, pg, zn, pd);\n }\n void fcmne(SubRegSize size, PRegister pd, PRegisterZero pg, ZRegister zn, ZRegister zm) {\n SVEFloatCompareVector(0, 1, 1, size, zm, pg, zn, pd);\n }\n void fcmuo(SubRegSize size, PRegister pd, PRegisterZero pg, ZRegister zn, ZRegister zm) {\n SVEFloatCompareVector(1, 0, 0, size, zm, pg, zn, pd);\n }\n void facge(SubRegSize size, PRegister pd, PRegisterZero pg, ZRegister zn, ZRegister zm) {\n SVEFloatCompareVector(1, 0, 1, size, zm, pg, zn, pd);\n }\n void facgt(SubRegSize size, PRegister pd, PRegisterZero pg, ZRegister zn, ZRegister zm) {\n SVEFloatCompareVector(1, 1, 1, size, zm, pg, zn, pd);\n }\n void facle(SubRegSize size, PRegister pd, PRegisterZero pg, ZRegister zm, ZRegister zn) {\n facge(size, pd, pg, zn, zm);\n }\n void faclt(SubRegSize size, PRegister pd, PRegisterZero pg, ZRegister zm, ZRegister zn) {\n facgt(size, pd, pg, zn, zm);\n }\n\n // SVE floating-point arithmetic (unpredicated)\n void fadd(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n SVEFloatArithmeticUnpredicated(0b000, size, zm, zn, zd);\n }\n void fsub(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n SVEFloatArithmeticUnpredicated(0b001, size, zm, zn, zd);\n }\n void fmul(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n SVEFloatArithmeticUnpredicated(0b010, size, zm, zn, zd);\n }\n void ftsmul(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n SVEFloatArithmeticUnpredicated(0b011, size, zm, zn, zd);\n }\n void frecps(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n SVEFloatArithmeticUnpredicated(0b110, size, zm, zn, zd);\n }\n void frsqrts(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n SVEFloatArithmeticUnpredicated(0b111, size, zm, zn, zd);\n }\n\n // SVE floating-point recursive reduction\n void faddv(SubRegSize size, VRegister vd, PRegister pg, ZRegister zn) {\n SVEFPRecursiveReduction(0b000, size, vd, pg, zn);\n }\n void fmaxnmv(SubRegSize size, VRegister vd, PRegister pg, ZRegister zn) {\n SVEFPRecursiveReduction(0b100, size, vd, pg, zn);\n }\n void fminnmv(SubRegSize size, VRegister vd, PRegister pg, ZRegister zn) {\n SVEFPRecursiveReduction(0b101, size, vd, pg, zn);\n }\n void fmaxv(SubRegSize size, VRegister vd, PRegister pg, ZRegister zn) {\n SVEFPRecursiveReduction(0b110, size, vd, pg, zn);\n }\n void fminv(SubRegSize size, VRegister vd, PRegister pg, ZRegister zn) {\n SVEFPRecursiveReduction(0b111, size, vd, pg, zn);\n }\n\n // SVE integer Multiply-Add - Predicated\n // SVE integer multiply-accumulate writing addend (predicated)\n void mla(SubRegSize size, ZRegister zda, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n SVEIntegerMultiplyAddSubPredicated(0b0, 0b0, size, zda, pg, zn, zm);\n }\n void mls(SubRegSize size, ZRegister zda, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n SVEIntegerMultiplyAddSubPredicated(0b0, 0b1, size, zda, pg, zn, zm);\n }\n\n // SVE integer multiply-add writing multiplicand (predicated)\n void mad(SubRegSize size, ZRegister zdn, PRegisterMerge pg, ZRegister zm, ZRegister za) {\n SVEIntegerMultiplyAddSubPredicated(0b1, 0b0, size, zdn, pg, za, zm);\n }\n void msb(SubRegSize size, ZRegister zdn, PRegisterMerge pg, ZRegister zm, ZRegister za) {\n SVEIntegerMultiplyAddSubPredicated(0b1, 0b1, size, zdn, pg, za, zm);\n }\n\n // SVE Integer Binary Arithmetic - Predicated\n // SVE integer add/subtract vectors (predicated)\n void add(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n SVEAddSubVectorsPredicated(0b000, size, zd, pg, zn, zm);\n }\n void sub(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n SVEAddSubVectorsPredicated(0b001, size, zd, pg, zn, zm);\n }\n void subr(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n SVEAddSubVectorsPredicated(0b011, size, zd, pg, zn, zm);\n }\n\n // SVE integer min/max/difference (predicated)\n void smax(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zdn, ZRegister zm) {\n SVEIntegerMinMaxDifferencePredicated(0b00, 0, size, pg, zdn, zm, zd);\n }\n void umax(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zdn, ZRegister zm) {\n SVEIntegerMinMaxDifferencePredicated(0b00, 1, size, pg, zdn, zm, zd);\n }\n void smin(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zdn, ZRegister zm) {\n SVEIntegerMinMaxDifferencePredicated(0b01, 0, size, pg, zdn, zm, zd);\n }\n void umin(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zdn, ZRegister zm) {\n SVEIntegerMinMaxDifferencePredicated(0b01, 1, size, pg, zdn, zm, zd);\n }\n void sabd(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zdn, ZRegister zm) {\n SVEIntegerMinMaxDifferencePredicated(0b10, 0, size, pg, zdn, zm, zd);\n }\n void uabd(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zdn, ZRegister zm) {\n SVEIntegerMinMaxDifferencePredicated(0b10, 1, size, pg, zdn, zm, zd);\n }\n\n // SVE integer multiply vectors (predicated)\n void mul(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n SVEIntegerMulDivVectorsPredicated(0b0, 0b00, size, zd, pg, zn, zm);\n }\n void smulh(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n SVEIntegerMulDivVectorsPredicated(0b0, 0b10, size, zd, pg, zn, zm);\n }\n void umulh(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n SVEIntegerMulDivVectorsPredicated(0b0, 0b11, size, zd, pg, zn, zm);\n }\n\n // SVE integer divide vectors (predicated)\n void sdiv(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n SVEIntegerMulDivVectorsPredicated(0b1, 0b00, size, zd, pg, zn, zm);\n }\n void udiv(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n SVEIntegerMulDivVectorsPredicated(0b1, 0b01, size, zd, pg, zn, zm);\n }\n void sdivr(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n SVEIntegerMulDivVectorsPredicated(0b1, 0b10, size, zd, pg, zn, zm);\n }\n void udivr(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n SVEIntegerMulDivVectorsPredicated(0b1, 0b11, size, zd, pg, zn, zm);\n }\n\n // SVE bitwise logical operations (predicated)\n void orr(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zdn, ZRegister zm) {\n SVEBitwiseLogicalPredicated(0b000, size, pg, zdn, zm, zd);\n }\n void eor(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zdn, ZRegister zm) {\n SVEBitwiseLogicalPredicated(0b001, size, pg, zdn, zm, zd);\n }\n void and_(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zdn, ZRegister zm) {\n SVEBitwiseLogicalPredicated(0b010, size, pg, zdn, zm, zd);\n }\n void bic(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zdn, ZRegister zm) {\n SVEBitwiseLogicalPredicated(0b011, size, pg, zdn, zm, zd);\n }\n\n // SVE Integer Reduction\n // SVE integer add reduction (predicated)\n void saddv(SubRegSize size, DRegister vd, PRegister pg, ZRegister zn) {\n LOGMAN_THROW_A_FMT(size == SubRegSize::i8Bit || size == SubRegSize::i16Bit || size == SubRegSize::i32Bit, \"saddv may only use 8-bit, \"\n \"16-bit, or 32-bit \"\n \"elements.\");\n constexpr uint32_t Op = 0b0000'0100'0000'0000'0010'0000'0000'0000;\n SVEIntegerReductionOperation(Op, 0b00, size, vd, pg, zn);\n }\n void uaddv(SubRegSize size, DRegister vd, PRegister pg, ZRegister zn) {\n LOGMAN_THROW_A_FMT(size == SubRegSize::i8Bit || size == SubRegSize::i16Bit || size == SubRegSize::i32Bit, \"uaddv may only use 8-bit, \"\n \"16-bit, or 32-bit \"\n \"elements.\");\n constexpr uint32_t Op = 0b0000'0100'0000'0000'0010'0000'0000'0000;\n SVEIntegerReductionOperation(Op, 0b01, size, vd, pg, zn);\n }\n\n // SVE integer min/max reduction (predicated)\n void smaxv(SubRegSize size, VRegister vd, PRegister pg, ZRegister zn) {\n constexpr uint32_t Op = 0b0000'0100'0000'1000'001 << 13;\n SVEIntegerReductionOperation(Op, 0b00, size, vd, pg, zn);\n }\n void umaxv(SubRegSize size, VRegister vd, PRegister pg, ZRegister zn) {\n constexpr uint32_t Op = 0b0000'0100'0000'1000'001 << 13;\n SVEIntegerReductionOperation(Op, 0b01, size, vd, pg, zn);\n }\n void sminv(SubRegSize size, VRegister vd, PRegister pg, ZRegister zn) {\n constexpr uint32_t Op = 0b0000'0100'0000'1000'001 << 13;\n SVEIntegerReductionOperation(Op, 0b10, size, vd, pg, zn);\n }\n void uminv(SubRegSize size, VRegister vd, PRegister pg, ZRegister zn) {\n constexpr uint32_t Op = 0b0000'0100'0000'1000'001 << 13;\n SVEIntegerReductionOperation(Op, 0b11, size, vd, pg, zn);\n }\n\n // SVE constructive prefix (predicated)\n template\n requires (std::is_same_v || std::is_same_v)\n void movprfx(SubRegSize size, ZRegister zd, T pg, ZRegister zn) {\n constexpr uint32_t M = std::is_same_v ? 1 : 0;\n SVEConstructivePrefixPredicated(0b00, M, size, pg, zn, zd);\n }\n\n // SVE bitwise logical reduction (predicated)\n void orv(SubRegSize size, VRegister vd, PRegister pg, ZRegister zn) {\n constexpr uint32_t Op = 0b0000'0100'0001'1000'0010'0000'0000'0000;\n SVEIntegerReductionOperation(Op, 0b00, size, vd, pg, zn);\n }\n void eorv(SubRegSize size, VRegister vd, PRegister pg, ZRegister zn) {\n constexpr uint32_t Op = 0b0000'0100'0001'1000'0010'0000'0000'0000;\n SVEIntegerReductionOperation(Op, 0b01, size, vd, pg, zn);\n }\n void andv(SubRegSize size, VRegister vd, PRegister pg, ZRegister zn) {\n constexpr uint32_t Op = 0b0000'0100'0001'1000'0010'0000'0000'0000;\n SVEIntegerReductionOperation(Op, 0b10, size, vd, pg, zn);\n }\n\n // SVE Bitwise Shift - Predicated\n // SVE bitwise shift by immediate (predicated)\n void asr(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zdn, uint32_t Shift) {\n SVEBitWiseShiftImmediatePred(size, 0b00, 0, 0, pg, zd, zdn, Shift);\n }\n void lsr(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zdn, uint32_t Shift) {\n SVEBitWiseShiftImmediatePred(size, 0b00, 0, 1, pg, zd, zdn, Shift);\n }\n void lsl(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zdn, uint32_t Shift) {\n SVEBitWiseShiftImmediatePred(size, 0b00, 1, 1, pg, zd, zdn, Shift);\n }\n void asrd(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zdn, uint32_t Shift) {\n SVEBitWiseShiftImmediatePred(size, 0b01, 0, 0, pg, zd, zdn, Shift);\n }\n void sqshl(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zdn, uint32_t Shift) {\n SVEBitWiseShiftImmediatePred(size, 0b01, 1, 0, pg, zd, zdn, Shift);\n }\n void uqshl(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zdn, uint32_t Shift) {\n SVEBitWiseShiftImmediatePred(size, 0b01, 1, 1, pg, zd, zdn, Shift);\n }\n void srshr(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zdn, uint32_t Shift) {\n SVEBitWiseShiftImmediatePred(size, 0b11, 0, 0, pg, zd, zdn, Shift);\n }\n void urshr(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zdn, uint32_t Shift) {\n SVEBitWiseShiftImmediatePred(size, 0b11, 0, 1, pg, zd, zdn, Shift);\n }\n void sqshlu(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zdn, uint32_t Shift) {\n SVEBitWiseShiftImmediatePred(size, 0b11, 1, 1, pg, zd, zdn, Shift);\n }\n\n // SVE bitwise shift by vector (predicated)\n void asr(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n SVEBitwiseShiftbyVector(0, 0, 0, size, pg, zd, zn, zm);\n }\n void lsr(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n SVEBitwiseShiftbyVector(0, 0, 1, size, pg, zd, zn, zm);\n }\n void lsl(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n SVEBitwiseShiftbyVector(0, 1, 1, size, pg, zd, zn, zm);\n }\n void asrr(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n SVEBitwiseShiftbyVector(1, 0, 0, size, pg, zd, zn, zm);\n }\n void lsrr(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n SVEBitwiseShiftbyVector(1, 0, 1, size, pg, zd, zn, zm);\n }\n void lslr(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n SVEBitwiseShiftbyVector(1, 1, 1, size, pg, zd, zn, zm);\n }\n\n // SVE bitwise shift by wide elements (predicated)\n void asr_wide(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n SVEBitwiseShiftByWideElementPredicated(size, 0b000, zd, pg, zn, zm);\n }\n void lsr_wide(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n SVEBitwiseShiftByWideElementPredicated(size, 0b001, zd, pg, zn, zm);\n }\n void lsl_wide(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n SVEBitwiseShiftByWideElementPredicated(size, 0b011, zd, pg, zn, zm);\n }\n\n // SVE Integer Unary Arithmetic - Predicated\n // SVE integer unary operations (predicated)\n void sxtb(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn) {\n LOGMAN_THROW_A_FMT(size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit, \"Invalid subregsize size\");\n SVEIntegerUnaryPredicated(0b10, 0b000, size, pg, zn, zd);\n }\n void uxtb(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn) {\n LOGMAN_THROW_A_FMT(size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit, \"Invalid subregsize size\");\n SVEIntegerUnaryPredicated(0b10, 0b001, size, pg, zn, zd);\n }\n void sxth(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn) {\n LOGMAN_THROW_A_FMT(size == SubRegSize::i32Bit || size == SubRegSize::i64Bit, \"Invalid subregsize size\");\n SVEIntegerUnaryPredicated(0b10, 0b010, size, pg, zn, zd);\n }\n void uxth(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn) {\n LOGMAN_THROW_A_FMT(size == SubRegSize::i32Bit || size == SubRegSize::i64Bit, \"Invalid subregsize size\");\n SVEIntegerUnaryPredicated(0b10, 0b011, size, pg, zn, zd);\n }\n void sxtw(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn) {\n LOGMAN_THROW_A_FMT(size == SubRegSize::i64Bit, \"Invalid subregsize size\");\n SVEIntegerUnaryPredicated(0b10, 0b100, size, pg, zn, zd);\n }\n void uxtw(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn) {\n LOGMAN_THROW_A_FMT(size == SubRegSize::i64Bit, \"Invalid subregsize size\");\n SVEIntegerUnaryPredicated(0b10, 0b101, size, pg, zn, zd);\n }\n void abs(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn) {\n SVEIntegerUnaryPredicated(0b10, 0b110, size, pg, zn, zd);\n }\n void neg(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn) {\n SVEIntegerUnaryPredicated(0b10, 0b111, size, pg, zn, zd);\n }\n\n // SVE bitwise unary operations (predicated)\n void cls(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn) {\n SVEIntegerUnaryPredicated(0b11, 0b000, size, pg, zn, zd);\n }\n void clz(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn) {\n SVEIntegerUnaryPredicated(0b11, 0b001, size, pg, zn, zd);\n }\n void cnt(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn) {\n SVEIntegerUnaryPredicated(0b11, 0b010, size, pg, zn, zd);\n }\n void cnot(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn) {\n SVEIntegerUnaryPredicated(0b11, 0b011, size, pg, zn, zd);\n }\n void fabs(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i8Bit, \"Invalid size\");\n SVEIntegerUnaryPredicated(0b11, 0b100, size, pg, zn, zd);\n }\n void fneg(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i8Bit, \"Invalid size\");\n SVEIntegerUnaryPredicated(0b11, 0b101, size, pg, zn, zd);\n }\n void not_(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn) {\n SVEIntegerUnaryPredicated(0b11, 0b110, size, pg, zn, zd);\n }\n\n // SVE Bitwise Logical - Unpredicated\n // SVE bitwise logical operations (unpredicated)\n void and_(ZRegister zd, ZRegister zn, ZRegister zm) {\n SVEBitwiseLogicalUnpredicated(0b00, zm, zn, zd);\n }\n void orr(ZRegister zd, ZRegister zn, ZRegister zm) {\n SVEBitwiseLogicalUnpredicated(0b01, zm, zn, zd);\n }\n void mov(ZRegister zd, ZRegister zn) {\n SVEBitwiseLogicalUnpredicated(0b01, zn, zn, zd);\n }\n void eor(ZRegister zd, ZRegister zn, ZRegister zm) {\n SVEBitwiseLogicalUnpredicated(0b10, zm, zn, zd);\n }\n void bic(ZRegister zd, ZRegister zn, ZRegister zm) {\n SVEBitwiseLogicalUnpredicated(0b11, zm, zn, zd);\n }\n\n void xar(SubRegSize size, ZRegister zd, ZRegister zm, uint32_t rotate) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i128Bit, \"Element size cannot be 128-bit.\");\n\n const auto [tszh, tszl_imm3] = EncodeSVEShiftImmediate(size, rotate);\n\n uint32_t Inst = 0b0000'0100'0010'0000'0011'0100'0000'0000;\n Inst |= tszh << 22;\n Inst |= tszl_imm3 << 16;\n Inst |= zm.Idx() << 5;\n Inst |= zd.Idx();\n dc32(Inst);\n }\n\n // SVE2 bitwise ternary operations\n void eor3(ZRegister zd, ZRegister zdn, ZRegister zm, ZRegister zk) {\n SVE2BitwiseTernary(0b00, 0, zm, zk, zd, zdn);\n }\n void bsl(ZRegister zd, ZRegister zdn, ZRegister zm, ZRegister zk) {\n SVE2BitwiseTernary(0b00, 1, zm, zk, zd, zdn);\n }\n void bcax(ZRegister zd, ZRegister zdn, ZRegister zm, ZRegister zk) {\n SVE2BitwiseTernary(0b01, 0, zm, zk, zd, zdn);\n }\n void bsl1n(ZRegister zd, ZRegister zdn, ZRegister zm, ZRegister zk) {\n SVE2BitwiseTernary(0b01, 1, zm, zk, zd, zdn);\n }\n void bsl2n(ZRegister zd, ZRegister zdn, ZRegister zm, ZRegister zk) {\n SVE2BitwiseTernary(0b10, 1, zm, zk, zd, zdn);\n }\n void nbsl(ZRegister zd, ZRegister zdn, ZRegister zm, ZRegister zk) {\n SVE2BitwiseTernary(0b11, 1, zm, zk, zd, zdn);\n }\n\n // SVE Index Generation\n void index(SubRegSize size, ZRegister zd, int32_t initial, int32_t increment) {\n LOGMAN_THROW_A_FMT(initial >= -16 && initial <= 15, \"initial value must be within -16-15. initial: {}\", initial);\n LOGMAN_THROW_A_FMT(increment >= -16 && increment <= 15, \"increment value must be within -16-15. increment: {}\", increment);\n SVEIndexGeneration(0b00, size, zd, initial, increment);\n }\n void index(SubRegSize size, ZRegister zd, Register initial, int32_t increment) {\n LOGMAN_THROW_A_FMT(increment >= -16 && increment <= 15, \"increment value must be within -16-15. increment: {}\", increment);\n SVEIndexGeneration(0b01, size, zd, static_cast(initial.Idx()), increment);\n }\n void index(SubRegSize size, ZRegister zd, int32_t initial, Register increment) {\n LOGMAN_THROW_A_FMT(initial >= -16 && initial <= 15, \"initial value must be within -16-15. initial: {}\", initial);\n SVEIndexGeneration(0b10, size, zd, initial, static_cast(increment.Idx()));\n }\n void index(SubRegSize size, ZRegister zd, Register initial, Register increment) {\n SVEIndexGeneration(0b11, size, zd, static_cast(initial.Idx()), static_cast(increment.Idx()));\n }\n\n // SVE Stack Allocation\n // SVE stack frame adjustment\n void addvl(XRegister rd, XRegister rn, int32_t imm) {\n SVEStackFrameOperation(0b00, rd, rn, imm);\n }\n void addpl(XRegister rd, XRegister rn, int32_t imm) {\n SVEStackFrameOperation(0b01, rd, rn, imm);\n }\n\n // Streaming SVE stack frame adjustment (SME)\n // XXX:\n\n // SVE stack frame size\n void rdvl(XRegister rd, int32_t imm) {\n // Would-be Rn field is just set to all 1's, which is the same\n // as writing the encoding for the SP into it.\n SVEStackFrameOperation(0b10, rd, XReg::rsp, imm);\n }\n\n // Streaming SVE stack frame size (SME)\n // XXX:\n\n // SVE2 Integer Multiply - Unpredicated\n // SVE2 integer multiply vectors (unpredicated)\n void mul(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n SVE2IntegerMultiplyVectors(0b00, size, zm, zn, zd);\n }\n void smulh(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n SVE2IntegerMultiplyVectors(0b10, size, zm, zn, zd);\n }\n\n void umulh(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n SVE2IntegerMultiplyVectors(0b11, size, zm, zn, zd);\n }\n\n void pmul(ZRegister zd, ZRegister zn, ZRegister zm) {\n SVE2IntegerMultiplyVectors(0b01, SubRegSize::i8Bit, zm, zn, zd);\n }\n\n // SVE2 signed saturating doubling multiply high (unpredicated)\n void sqdmulh(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n SVE2IntegerMultiplyVectors(0b100, size, zm, zn, zd);\n }\n void sqrdmulh(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n SVE2IntegerMultiplyVectors(0b101, size, zm, zn, zd);\n }\n\n // SVE Bitwise Shift - Unpredicated\n // SVE bitwise shift by wide elements (unpredicated)\n void asr_wide(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n SVEBitwiseShiftByWideElementsUnpredicated(size, 0b00, zd, zn, zm);\n }\n void lsr_wide(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n SVEBitwiseShiftByWideElementsUnpredicated(size, 0b01, zd, zn, zm);\n }\n void lsl_wide(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n SVEBitwiseShiftByWideElementsUnpredicated(size, 0b11, zd, zn, zm);\n }\n\n // SVE bitwise shift by immediate (unpredicated)\n void asr(SubRegSize size, ZRegister zd, ZRegister zn, uint32_t shift) {\n SVEBitWiseShiftImmediateUnpred(size, 0b00, zd, zn, shift);\n }\n void lsr(SubRegSize size, ZRegister zd, ZRegister zn, uint32_t shift) {\n SVEBitWiseShiftImmediateUnpred(size, 0b01, zd, zn, shift);\n }\n void lsl(SubRegSize size, ZRegister zd, ZRegister zn, uint32_t shift) {\n SVEBitWiseShiftImmediateUnpred(size, 0b11, zd, zn, shift);\n }\n\n // SVE Integer Misc - Unpredicated\n // SVE floating-point trig select coefficient\n void ftssel(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), \"ftssel may only use 16/32/64-bit element sizes\");\n SVEIntegerMiscUnpredicated(0b00, zm.Idx(), FEXCore::ToUnderlying(size), zd, zn);\n }\n // SVE floating-point exponential accelerator\n void fexpa(SubRegSize size, ZRegister zd, ZRegister zn) {\n LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), \"fexpa may only use 16/32/64-bit element sizes\");\n SVEIntegerMiscUnpredicated(0b10, 0b00000, FEXCore::ToUnderlying(size), zd, zn);\n }\n // SVE constructive prefix (unpredicated)\n void movprfx(ZRegister zd, ZRegister zn) {\n SVEIntegerMiscUnpredicated(0b11, 0b00000, 0b00, zd, zn);\n }\n\n // SVE Element Count\n // SVE saturating inc/dec vector by element count\n void sqinch(ZRegister zdn, PredicatePattern pattern, uint32_t imm4) {\n SVEElementCount(0, 0b0000, SubRegSize::i16Bit, zdn, pattern, imm4);\n }\n void uqinch(ZRegister zdn, PredicatePattern pattern, uint32_t imm4) {\n SVEElementCount(0, 0b0001, SubRegSize::i16Bit, zdn, pattern, imm4);\n }\n void sqdech(ZRegister zdn, PredicatePattern pattern, uint32_t imm4) {\n SVEElementCount(0, 0b0010, SubRegSize::i16Bit, zdn, pattern, imm4);\n }\n void uqdech(ZRegister zdn, PredicatePattern pattern, uint32_t imm4) {\n SVEElementCount(0, 0b0011, SubRegSize::i16Bit, zdn, pattern, imm4);\n }\n void sqincw(ZRegister zdn, PredicatePattern pattern, uint32_t imm4) {\n SVEElementCount(0, 0b0000, SubRegSize::i32Bit, zdn, pattern, imm4);\n }\n void uqincw(ZRegister zdn, PredicatePattern pattern, uint32_t imm4) {\n SVEElementCount(0, 0b0001, SubRegSize::i32Bit, zdn, pattern, imm4);\n }\n void sqdecw(ZRegister zdn, PredicatePattern pattern, uint32_t imm4) {\n SVEElementCount(0, 0b0010, SubRegSize::i32Bit, zdn, pattern, imm4);\n }\n void uqdecw(ZRegister zdn, PredicatePattern pattern, uint32_t imm4) {\n SVEElementCount(0, 0b0011, SubRegSize::i32Bit, zdn, pattern, imm4);\n }\n void sqincd(ZRegister zdn, PredicatePattern pattern, uint32_t imm4) {\n SVEElementCount(0, 0b0000, SubRegSize::i64Bit, zdn, pattern, imm4);\n }\n void uqincd(ZRegister zdn, PredicatePattern pattern, uint32_t imm4) {\n SVEElementCount(0, 0b0001, SubRegSize::i64Bit, zdn, pattern, imm4);\n }\n void sqdecd(ZRegister zdn, PredicatePattern pattern, uint32_t imm4) {\n SVEElementCount(0, 0b0010, SubRegSize::i64Bit, zdn, pattern, imm4);\n }\n void uqdecd(ZRegister zdn, PredicatePattern pattern, uint32_t imm4) {\n SVEElementCount(0, 0b0011, SubRegSize::i64Bit, zdn, pattern, imm4);\n }\n\n // SVE element count\n void cntb(XRegister rd, PredicatePattern pattern, uint32_t imm) {\n SVEElementCount(0, 0b1000, SubRegSize::i8Bit, ZRegister {rd.Idx()}, pattern, imm);\n }\n void cnth(XRegister rd, PredicatePattern pattern, uint32_t imm) {\n SVEElementCount(0, 0b1000, SubRegSize::i16Bit, ZRegister {rd.Idx()}, pattern, imm);\n }\n void cntw(XRegister rd, PredicatePattern pattern, uint32_t imm) {\n SVEElementCount(0, 0b1000, SubRegSize::i32Bit, ZRegister {rd.Idx()}, pattern, imm);\n }\n void cntd(XRegister rd, PredicatePattern pattern, uint32_t imm) {\n SVEElementCount(0, 0b1000, SubRegSize::i64Bit, ZRegister {rd.Idx()}, pattern, imm);\n }\n\n // SVE inc/dec vector by element count\n void inch(ZRegister zdn, PredicatePattern pattern, uint32_t imm) {\n SVEElementCount(1, 0b0000, SubRegSize::i16Bit, zdn, pattern, imm);\n }\n void dech(ZRegister zdn, PredicatePattern pattern, uint32_t imm) {\n SVEElementCount(1, 0b0001, SubRegSize::i16Bit, zdn, pattern, imm);\n }\n void incw(ZRegister zdn, PredicatePattern pattern, uint32_t imm) {\n SVEElementCount(1, 0b0000, SubRegSize::i32Bit, zdn, pattern, imm);\n }\n void decw(ZRegister zdn, PredicatePattern pattern, uint32_t imm) {\n SVEElementCount(1, 0b0001, SubRegSize::i32Bit, zdn, pattern, imm);\n }\n void incd(ZRegister zdn, PredicatePattern pattern, uint32_t imm) {\n SVEElementCount(1, 0b0000, SubRegSize::i64Bit, zdn, pattern, imm);\n }\n void decd(ZRegister zdn, PredicatePattern pattern, uint32_t imm) {\n SVEElementCount(1, 0b0001, SubRegSize::i64Bit, zdn, pattern, imm);\n }\n\n // SVE inc/dec register by element count\n void incb(XRegister rdn, PredicatePattern pattern, uint32_t imm) {\n SVEElementCount(1, 0b1000, SubRegSize::i8Bit, ZRegister {rdn.Idx()}, pattern, imm);\n }\n void decb(XRegister rdn, PredicatePattern pattern, uint32_t imm) {\n SVEElementCount(1, 0b1001, SubRegSize::i8Bit, ZRegister {rdn.Idx()}, pattern, imm);\n }\n void inch(XRegister rdn, PredicatePattern pattern, uint32_t imm) {\n SVEElementCount(1, 0b1000, SubRegSize::i16Bit, ZRegister {rdn.Idx()}, pattern, imm);\n }\n void dech(XRegister rdn, PredicatePattern pattern, uint32_t imm) {\n SVEElementCount(1, 0b1001, SubRegSize::i16Bit, ZRegister {rdn.Idx()}, pattern, imm);\n }\n void incw(XRegister rdn, PredicatePattern pattern, uint32_t imm) {\n SVEElementCount(1, 0b1000, SubRegSize::i32Bit, ZRegister {rdn.Idx()}, pattern, imm);\n }\n void decw(XRegister rdn, PredicatePattern pattern, uint32_t imm) {\n SVEElementCount(1, 0b1001, SubRegSize::i32Bit, ZRegister {rdn.Idx()}, pattern, imm);\n }\n void incd(XRegister rdn, PredicatePattern pattern, uint32_t imm) {\n SVEElementCount(1, 0b1000, SubRegSize::i64Bit, ZRegister {rdn.Idx()}, pattern, imm);\n }\n void decd(XRegister rdn, PredicatePattern pattern, uint32_t imm) {\n SVEElementCount(1, 0b1001, SubRegSize::i64Bit, ZRegister {rdn.Idx()}, pattern, imm);\n }\n\n // SVE saturating inc/dec register by element count\n void sqincb(XRegister rdn, PredicatePattern pattern, uint32_t imm) {\n SVEElementCount(1, 0b1100, SubRegSize::i8Bit, ZRegister {rdn.Idx()}, pattern, imm);\n }\n void sqincb(WRegister rdn, PredicatePattern pattern, uint32_t imm) {\n SVEElementCount(0, 0b1100, SubRegSize::i8Bit, ZRegister {rdn.Idx()}, pattern, imm);\n }\n void uqincb(XRegister rdn, PredicatePattern pattern, uint32_t imm) {\n SVEElementCount(1, 0b1101, SubRegSize::i8Bit, ZRegister {rdn.Idx()}, pattern, imm);\n }\n void uqincb(WRegister rdn, PredicatePattern pattern, uint32_t imm) {\n SVEElementCount(0, 0b1101, SubRegSize::i8Bit, ZRegister {rdn.Idx()}, pattern, imm);\n }\n void sqdecb(XRegister rdn, PredicatePattern pattern, uint32_t imm) {\n SVEElementCount(1, 0b1110, SubRegSize::i8Bit, ZRegister {rdn.Idx()}, pattern, imm);\n }\n void sqdecb(WRegister rdn, PredicatePattern pattern, uint32_t imm) {\n SVEElementCount(0, 0b1110, SubRegSize::i8Bit, ZRegister {rdn.Idx()}, pattern, imm);\n }\n void uqdecb(XRegister rdn, PredicatePattern pattern, uint32_t imm) {\n SVEElementCount(1, 0b1111, SubRegSize::i8Bit, ZRegister {rdn.Idx()}, pattern, imm);\n }\n void uqdecb(WRegister rdn, PredicatePattern pattern, uint32_t imm) {\n SVEElementCount(0, 0b1111, SubRegSize::i8Bit, ZRegister {rdn.Idx()}, pattern, imm);\n }\n\n void sqinch(XRegister rdn, PredicatePattern pattern, uint32_t imm) {\n SVEElementCount(1, 0b1100, SubRegSize::i16Bit, ZRegister {rdn.Idx()}, pattern, imm);\n }\n void sqinch(WRegister rdn, PredicatePattern pattern, uint32_t imm) {\n SVEElementCount(0, 0b1100, SubRegSize::i16Bit, ZRegister {rdn.Idx()}, pattern, imm);\n }\n void uqinch(XRegister rdn, PredicatePattern pattern, uint32_t imm) {\n SVEElementCount(1, 0b1101, SubRegSize::i16Bit, ZRegister {rdn.Idx()}, pattern, imm);\n }\n void uqinch(WRegister rdn, PredicatePattern pattern, uint32_t imm) {\n SVEElementCount(0, 0b1101, SubRegSize::i16Bit, ZRegister {rdn.Idx()}, pattern, imm);\n }\n void sqdech(XRegister rdn, PredicatePattern pattern, uint32_t imm) {\n SVEElementCount(1, 0b1110, SubRegSize::i16Bit, ZRegister {rdn.Idx()}, pattern, imm);\n }\n void sqdech(WRegister rdn, PredicatePattern pattern, uint32_t imm) {\n SVEElementCount(0, 0b1110, SubRegSize::i16Bit, ZRegister {rdn.Idx()}, pattern, imm);\n }\n void uqdech(XRegister rdn, PredicatePattern pattern, uint32_t imm) {\n SVEElementCount(1, 0b1111, SubRegSize::i16Bit, ZRegister {rdn.Idx()}, pattern, imm);\n }\n void uqdech(WRegister rdn, PredicatePattern pattern, uint32_t imm) {\n SVEElementCount(0, 0b1111, SubRegSize::i16Bit, ZRegister {rdn.Idx()}, pattern, imm);\n }\n\n void sqincw(XRegister rdn, PredicatePattern pattern, uint32_t imm) {\n SVEElementCount(1, 0b1100, SubRegSize::i32Bit, ZRegister {rdn.Idx()}, pattern, imm);\n }\n void sqincw(WRegister rdn, PredicatePattern pattern, uint32_t imm) {\n SVEElementCount(0, 0b1100, SubRegSize::i32Bit, ZRegister {rdn.Idx()}, pattern, imm);\n }\n void uqincw(XRegister rdn, PredicatePattern pattern, uint32_t imm) {\n SVEElementCount(1, 0b1101, SubRegSize::i32Bit, ZRegister {rdn.Idx()}, pattern, imm);\n }\n void uqincw(WRegister rdn, PredicatePattern pattern, uint32_t imm) {\n SVEElementCount(0, 0b1101, SubRegSize::i32Bit, ZRegister {rdn.Idx()}, pattern, imm);\n }\n void sqdecw(XRegister rdn, PredicatePattern pattern, uint32_t imm) {\n SVEElementCount(1, 0b1110, SubRegSize::i32Bit, ZRegister {rdn.Idx()}, pattern, imm);\n }\n void sqdecw(WRegister rdn, PredicatePattern pattern, uint32_t imm) {\n SVEElementCount(0, 0b1110, SubRegSize::i32Bit, ZRegister {rdn.Idx()}, pattern, imm);\n }\n void uqdecw(XRegister rdn, PredicatePattern pattern, uint32_t imm) {\n SVEElementCount(1, 0b1111, SubRegSize::i32Bit, ZRegister {rdn.Idx()}, pattern, imm);\n }\n void uqdecw(WRegister rdn, PredicatePattern pattern, uint32_t imm) {\n SVEElementCount(0, 0b1111, SubRegSize::i32Bit, ZRegister {rdn.Idx()}, pattern, imm);\n }\n\n void sqincd(XRegister rdn, PredicatePattern pattern, uint32_t imm) {\n SVEElementCount(1, 0b1100, SubRegSize::i64Bit, ZRegister {rdn.Idx()}, pattern, imm);\n }\n void sqincd(WRegister rdn, PredicatePattern pattern, uint32_t imm) {\n SVEElementCount(0, 0b1100, SubRegSize::i64Bit, ZRegister {rdn.Idx()}, pattern, imm);\n }\n void uqincd(XRegister rdn, PredicatePattern pattern, uint32_t imm) {\n SVEElementCount(1, 0b1101, SubRegSize::i64Bit, ZRegister {rdn.Idx()}, pattern, imm);\n }\n void uqincd(WRegister rdn, PredicatePattern pattern, uint32_t imm) {\n SVEElementCount(0, 0b1101, SubRegSize::i64Bit, ZRegister {rdn.Idx()}, pattern, imm);\n }\n void sqdecd(XRegister rdn, PredicatePattern pattern, uint32_t imm) {\n SVEElementCount(1, 0b1110, SubRegSize::i64Bit, ZRegister {rdn.Idx()}, pattern, imm);\n }\n void sqdecd(WRegister rdn, PredicatePattern pattern, uint32_t imm) {\n SVEElementCount(0, 0b1110, SubRegSize::i64Bit, ZRegister {rdn.Idx()}, pattern, imm);\n }\n void uqdecd(XRegister rdn, PredicatePattern pattern, uint32_t imm) {\n SVEElementCount(1, 0b1111, SubRegSize::i64Bit, ZRegister {rdn.Idx()}, pattern, imm);\n }\n void uqdecd(WRegister rdn, PredicatePattern pattern, uint32_t imm) {\n SVEElementCount(0, 0b1111, SubRegSize::i64Bit, ZRegister {rdn.Idx()}, pattern, imm);\n }\n\n // SVE Bitwise Immediate\n // XXX: DUPM\n // SVE bitwise logical with immediate (unpredicated)\n // XXX:\n\n // SVE Integer Wide Immediate - Predicated\n void fcpy(SubRegSize size, ZRegister zd, PRegisterMerge pg, float value) {\n SVEBroadcastFloatImmPredicated(size, zd, pg, value);\n }\n void fmov(SubRegSize size, ZRegister zd, PRegisterMerge pg, float value) {\n fcpy(size, zd, pg, value);\n }\n\n // SVE copy integer immediate (predicated)\n void cpy(SubRegSize size, ZRegister zd, PRegisterZero pg, int32_t imm) {\n SVEBroadcastIntegerImmPredicated(0, size, zd, pg, imm);\n }\n void cpy(SubRegSize size, ZRegister zd, PRegisterMerge pg, int32_t imm) {\n SVEBroadcastIntegerImmPredicated(1, size, zd, pg, imm);\n }\n void mov_imm(SubRegSize size, ZRegister zd, PRegisterZero pg, int32_t imm) {\n cpy(size, zd, pg, imm);\n }\n void mov_imm(SubRegSize size, ZRegister zd, PRegisterMerge pg, int32_t imm) {\n cpy(size, zd, pg, imm);\n }\n\n // SVE Permute Vector - Unpredicated\n void dup(SubRegSize size, ZRegister zd, Register rn) {\n SVEPermuteUnpredicated(size, 0b00000, zd, ZRegister {rn.Idx()});\n }\n void mov(SubRegSize size, ZRegister zd, Register rn) {\n dup(size, zd, rn);\n }\n void insr(SubRegSize size, ZRegister zdn, Register rm) {\n SVEPermuteUnpredicated(size, 0b00100, zdn, ZRegister {rm.Idx()});\n }\n void insr(SubRegSize size, ZRegister zdn, VRegister vm) {\n SVEPermuteUnpredicated(size, 0b10100, zdn, vm.Z());\n }\n void rev(SubRegSize size, ZRegister zd, ZRegister zn) {\n SVEPermuteUnpredicated(size, 0b11000, zd, zn);\n }\n\n // SVE unpack vector elements\n void sunpklo(SubRegSize size, ZRegister zd, ZRegister zn) {\n LOGMAN_THROW_A_FMT(size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit, \"Invalid subregsize size\");\n SVEPermuteUnpredicated(size, 0b10000, zd, zn);\n }\n void sunpkhi(SubRegSize size, ZRegister zd, ZRegister zn) {\n LOGMAN_THROW_A_FMT(size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit, \"Invalid subregsize size\");\n SVEPermuteUnpredicated(size, 0b10001, zd, zn);\n }\n void uunpklo(SubRegSize size, ZRegister zd, ZRegister zn) {\n LOGMAN_THROW_A_FMT(size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit, \"Invalid subregsize size\");\n SVEPermuteUnpredicated(size, 0b10010, zd, zn);\n }\n void uunpkhi(SubRegSize size, ZRegister zd, ZRegister zn) {\n LOGMAN_THROW_A_FMT(size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit, \"Invalid subregsize size\");\n SVEPermuteUnpredicated(size, 0b10011, zd, zn);\n }\n\n // SVE Permute Predicate\n void rev(SubRegSize size, PRegister pd, PRegister pn) {\n SVEPermutePredicate(size, 0b10100, 0b0000, 0b0, pd, pn);\n }\n\n // SVE unpack predicate elements\n void punpklo(PRegister pd, PRegister pn) {\n SVEPermutePredicate(SubRegSize::i8Bit, 0b10000, 0b0000, 0b0, pd, pn);\n }\n void punpkhi(PRegister pd, PRegister pn) {\n SVEPermutePredicate(SubRegSize::i8Bit, 0b10001, 0b0000, 0b0, pd, pn);\n }\n\n // SVE permute predicate elements\n void zip1(SubRegSize size, PRegister pd, PRegister pn, PRegister pm) {\n SVEPermutePredicate(size, pm.Idx(), 0b0000, 0b0, pd, pn);\n }\n void zip2(SubRegSize size, PRegister pd, PRegister pn, PRegister pm) {\n SVEPermutePredicate(size, pm.Idx(), 0b0010, 0b0, pd, pn);\n }\n void uzp1(SubRegSize size, PRegister pd, PRegister pn, PRegister pm) {\n SVEPermutePredicate(size, pm.Idx(), 0b0100, 0b0, pd, pn);\n }\n void uzp2(SubRegSize size, PRegister pd, PRegister pn, PRegister pm) {\n SVEPermutePredicate(size, pm.Idx(), 0b0110, 0b0, pd, pn);\n }\n void trn1(SubRegSize size, PRegister pd, PRegister pn, PRegister pm) {\n SVEPermutePredicate(size, pm.Idx(), 0b1000, 0b0, pd, pn);\n }\n void trn2(SubRegSize size, PRegister pd, PRegister pn, PRegister pm) {\n SVEPermutePredicate(size, pm.Idx(), 0b1010, 0b0, pd, pn);\n }\n\n // SVE Permute Vector - Predicated - Base\n // CPY (SIMD&FP scalar)\n void cpy(SubRegSize size, ZRegister zd, PRegisterMerge pg, VRegister vn) {\n SVEPermuteVectorPredicated(0b00000, 0b0, size, zd, pg, ZRegister {vn.Idx()});\n }\n\n void compact(SubRegSize size, ZRegister zd, PRegister pg, ZRegister zn) {\n LOGMAN_THROW_A_FMT(size == SubRegSize::i64Bit || size == SubRegSize::i32Bit, \"Invalid element size\");\n SVEPermuteVectorPredicated(0b00001, 0b0, size, zd, pg, zn);\n }\n\n // CPY (scalar)\n void cpy(SubRegSize size, ZRegister zd, PRegisterMerge pg, Register rn) {\n SVEPermuteVectorPredicated(0b01000, 0b1, size, zd, pg, ZRegister {rn.Idx()});\n }\n\n template\n requires (optype == OpType::Constructive)\n void splice(SubRegSize size, ZRegister zd, PRegister pv, ZRegister zn, ZRegister zn2) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(zn, zn2), \"zn and zn2 must be sequential registers\");\n SVEPermuteVectorPredicated(0b01101, 0b0, size, zd, pv, zn);\n }\n\n template\n requires (optype == OpType::Destructive)\n void splice(SubRegSize size, ZRegister zd, PRegister pv, ZRegister zn, ZRegister zm) {\n LOGMAN_THROW_A_FMT(zd == zn, \"zd needs to equal zn\");\n SVEPermuteVectorPredicated(0b01100, 0b0, size, zd, pv, zm);\n }\n\n // SVE Permute Vector - Predicated\n // SVE extract element to general register\n void lasta(SubRegSize size, Register rd, PRegister pg, ZRegister zn) {\n SVEPermuteVectorPredicated(0b00000, 0b1, size, ZRegister {rd.Idx()}, pg, zn);\n }\n void lastb(SubRegSize size, Register rd, PRegister pg, ZRegister zn) {\n SVEPermuteVectorPredicated(0b00001, 0b1, size, ZRegister {rd.Idx()}, pg, zn);\n }\n\n // SVE extract element to SIMD&FP scalar register\n void lasta(SubRegSize size, VRegister vd, PRegister pg, ZRegister zn) {\n SVEPermuteVectorPredicated(0b00010, 0b0, size, ZRegister {vd.Idx()}, pg, zn);\n }\n void lastb(SubRegSize size, VRegister vd, PRegister pg, ZRegister zn) {\n SVEPermuteVectorPredicated(0b00011, 0b0, size, ZRegister {vd.Idx()}, pg, zn);\n }\n\n // SVE reverse within elements\n void revb(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i8Bit, \"Can't use 8-bit element size\");\n SVEPermuteVectorPredicated(0b00100, 0b0, size, zd, pg, zn);\n }\n void revh(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i8Bit && size != SubRegSize::i16Bit, \"Can't use 8/16-bit element sizes\");\n SVEPermuteVectorPredicated(0b00101, 0b0, size, zd, pg, zn);\n }\n void revw(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn) {\n LOGMAN_THROW_A_FMT(size == SubRegSize::i64Bit, \"Can't use 8/16/32-bit element sizes\");\n SVEPermuteVectorPredicated(0b00110, 0b0, size, zd, pg, zn);\n }\n void rbit(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn) {\n SVEPermuteVectorPredicated(0b00111, 0b0, size, zd, pg, zn);\n }\n\n // SVE conditionally broadcast element to vector\n void clasta(SubRegSize size, ZRegister zd, PRegister pg, ZRegister zn, ZRegister zm) {\n LOGMAN_THROW_A_FMT(zd == zn, \"zd must be the same as zn\");\n SVEPermuteVectorPredicated(0b01000, 0b0, size, zd, pg, zm);\n }\n void clastb(SubRegSize size, ZRegister zd, PRegister pg, ZRegister zn, ZRegister zm) {\n LOGMAN_THROW_A_FMT(zd == zn, \"zd must be the same as zn\");\n SVEPermuteVectorPredicated(0b01001, 0b0, size, zd, pg, zm);\n }\n\n // SVE conditionally extract element to SIMD&FP scalar\n void clasta(SubRegSize size, VRegister vd, PRegister pg, VRegister vn, ZRegister zm) {\n LOGMAN_THROW_A_FMT(vd == vn, \"vd must be the same as vn\");\n SVEPermuteVectorPredicated(0b01010, 0b0, size, ZRegister {vd.Idx()}, pg, zm);\n }\n void clastb(SubRegSize size, VRegister vd, PRegister pg, VRegister vn, ZRegister zm) {\n LOGMAN_THROW_A_FMT(vd == vn, \"vd must be the same as vn\");\n SVEPermuteVectorPredicated(0b01011, 0b0, size, ZRegister {vd.Idx()}, pg, zm);\n }\n\n // SVE reverse doublewords (SME)\n // XXX:\n\n // SVE conditionally extract element to general register\n void clasta(SubRegSize size, Register rd, PRegister pg, Register rn, ZRegister zm) {\n LOGMAN_THROW_A_FMT(rd == rn, \"rd must be the same as rn\");\n SVEPermuteVectorPredicated(0b10000, 0b1, size, ZRegister {rd.Idx()}, pg, zm);\n }\n void clastb(SubRegSize size, Register rd, PRegister pg, Register rn, ZRegister zm) {\n LOGMAN_THROW_A_FMT(rd == rn, \"rd must be the same as rn\");\n SVEPermuteVectorPredicated(0b10001, 0b1, size, ZRegister {rd.Idx()}, pg, zm);\n }\n\n // SVE Permute Vector - Extract\n // Constructive\n template\n requires (optype == OpType::Constructive)\n void ext(ZRegister zd, ZRegister zn, ZRegister zn2, uint8_t Imm) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(zn, zn2), \"zn and zn2 must be sequential registers\");\n SVEPermuteVector(1, zd, zn, Imm);\n }\n\n // Destructive\n template\n requires (optype == OpType::Destructive)\n void ext(ZRegister zd, ZRegister zdn, ZRegister zm, uint8_t Imm) {\n LOGMAN_THROW_A_FMT(zd == zdn, \"Dest needs to equal zdn\");\n SVEPermuteVector(0, zd, zm, Imm);\n }\n\n // SVE Permute Vector - Segments\n // SVE permute vector segments\n // XXX:\n\n // SVE Integer Compare - Vectors\n // SVE integer compare vectors\n void cmpeq(SubRegSize size, PRegister pd, PRegisterZero pg, ZRegister zn, ZRegister zm) {\n SVEIntegerCompareVector(1, 1, 0, size, zm, pg, zn, pd);\n }\n void cmpge(SubRegSize size, PRegister pd, PRegisterZero pg, ZRegister zn, ZRegister zm) {\n SVEIntegerCompareVector(1, 0, 0, size, zm, pg, zn, pd);\n }\n void cmpgt(SubRegSize size, PRegister pd, PRegisterZero pg, ZRegister zn, ZRegister zm) {\n SVEIntegerCompareVector(1, 0, 1, size, zm, pg, zn, pd);\n }\n void cmphi(SubRegSize size, PRegister pd, PRegisterZero pg, ZRegister zn, ZRegister zm) {\n SVEIntegerCompareVector(0, 0, 1, size, zm, pg, zn, pd);\n }\n void cmphs(SubRegSize size, PRegister pd, PRegisterZero pg, ZRegister zn, ZRegister zm) {\n SVEIntegerCompareVector(0, 0, 0, size, zm, pg, zn, pd);\n }\n void cmpne(SubRegSize size, PRegister pd, PRegisterZero pg, ZRegister zn, ZRegister zm) {\n SVEIntegerCompareVector(1, 1, 1, size, zm, pg, zn, pd);\n }\n\n // SVE integer compare with wide elements\n void cmpeq_wide(SubRegSize size, PRegister pd, PRegisterZero pg, ZRegister zn, ZRegister zm) {\n SVEIntegerCompareVectorWide(0, 0b01, 0, size, pd, pg, zn, zm);\n }\n void cmpgt_wide(SubRegSize size, PRegister pd, PRegisterZero pg, ZRegister zn, ZRegister zm) {\n SVEIntegerCompareVectorWide(0, 0b10, 1, size, pd, pg, zn, zm);\n }\n void cmpge_wide(SubRegSize size, PRegister pd, PRegisterZero pg, ZRegister zn, ZRegister zm) {\n SVEIntegerCompareVectorWide(0, 0b10, 0, size, pd, pg, zn, zm);\n }\n void cmphi_wide(SubRegSize size, PRegister pd, PRegisterZero pg, ZRegister zn, ZRegister zm) {\n SVEIntegerCompareVectorWide(1, 0b10, 1, size, pd, pg, zn, zm);\n }\n void cmphs_wide(SubRegSize size, PRegister pd, PRegisterZero pg, ZRegister zn, ZRegister zm) {\n SVEIntegerCompareVectorWide(1, 0b10, 0, size, pd, pg, zn, zm);\n }\n void cmplt_wide(SubRegSize size, PRegister pd, PRegisterZero pg, ZRegister zn, ZRegister zm) {\n SVEIntegerCompareVectorWide(0, 0b11, 0, size, pd, pg, zn, zm);\n }\n void cmple_wide(SubRegSize size, PRegister pd, PRegisterZero pg, ZRegister zn, ZRegister zm) {\n SVEIntegerCompareVectorWide(0, 0b11, 1, size, pd, pg, zn, zm);\n }\n void cmplo_wide(SubRegSize size, PRegister pd, PRegisterZero pg, ZRegister zn, ZRegister zm) {\n SVEIntegerCompareVectorWide(1, 0b11, 0, size, pd, pg, zn, zm);\n }\n void cmpls_wide(SubRegSize size, PRegister pd, PRegisterZero pg, ZRegister zn, ZRegister zm) {\n SVEIntegerCompareVectorWide(1, 0b11, 1, size, pd, pg, zn, zm);\n }\n void cmpne_wide(SubRegSize size, PRegister pd, PRegisterZero pg, ZRegister zn, ZRegister zm) {\n SVEIntegerCompareVectorWide(0, 0b01, 1, size, pd, pg, zn, zm);\n }\n\n // SVE Propagate Break\n // SVE propagate break from previous partition\n void brkpa(PRegister pd, PRegisterZero pg, PRegister pn, PRegister pm) {\n SVEPropagateBreak(0b0000, 0b11, 0, pd, pg, pn, pm);\n }\n void brkpb(PRegister pd, PRegisterZero pg, PRegister pn, PRegister pm) {\n SVEPropagateBreak(0b0000, 0b11, 1, pd, pg, pn, pm);\n }\n void brkpas(PRegister pd, PRegisterZero pg, PRegister pn, PRegister pm) {\n SVEPropagateBreak(0b0100, 0b11, 0, pd, pg, pn, pm);\n }\n void brkpbs(PRegister pd, PRegisterZero pg, PRegister pn, PRegister pm) {\n SVEPropagateBreak(0b0100, 0b11, 1, pd, pg, pn, pm);\n }\n\n // SVE Partition Break\n // SVE propagate break to next partition\n void brkn(PRegister pd, PRegisterZero pg, PRegister pn, PRegister pm) {\n LOGMAN_THROW_A_FMT(pd == pm, \"pd and pm need to be the same\");\n SVEPropagateBreak(0b0001, 0b01, 0, pd, pg, pn, PReg::p8);\n }\n void brkns(PRegister pd, PRegisterZero pg, PRegister pn, PRegister pm) {\n LOGMAN_THROW_A_FMT(pd == pm, \"pd and pm need to be the same\");\n SVEPropagateBreak(0b0101, 0b01, 0, pd, pg, pn, PReg::p8);\n }\n\n // SVE partition break condition\n void brka(PRegister pd, PRegisterZero pg, PRegister pn) {\n SVEPropagateBreak(0b0001, 0b01, 0, pd, pg, pn, PReg::p0);\n }\n void brka(PRegister pd, PRegisterMerge pg, PRegister pn) {\n SVEPropagateBreak(0b0001, 0b01, 1, pd, pg, pn, PReg::p0);\n }\n void brkas(PRegister pd, PRegisterZero pg, PRegister pn) {\n SVEPropagateBreak(0b0101, 0b01, 0, pd, pg, pn, PReg::p0);\n }\n void brkb(PRegister pd, PRegisterZero pg, PRegister pn) {\n SVEPropagateBreak(0b1001, 0b01, 0, pd, pg, pn, PReg::p0);\n }\n void brkb(PRegister pd, PRegisterMerge pg, PRegister pn) {\n SVEPropagateBreak(0b1001, 0b01, 1, pd, pg, pn, PReg::p0);\n }\n void brkbs(PRegister pd, PRegisterZero pg, PRegister pn) {\n SVEPropagateBreak(0b1101, 0b01, 0, pd, pg, pn, PReg::p0);\n }\n\n // SVE Predicate Misc\n void pnext(SubRegSize size, PRegister pd, PRegister pv, PRegister pn) {\n LOGMAN_THROW_A_FMT(pd == pn, \"pd and pn need to be the same\");\n SVEPredicateMisc(0b1001, 0b00010, pv.Idx(), size, pd);\n }\n\n // SVE predicate test\n void ptest(PRegister pg, PRegister pn) {\n SVEPredicateMisc(0b0000, pg.Idx() << 1, pn.Idx(), SubRegSize::i16Bit, PReg::p0);\n }\n\n // SVE predicate first active\n void pfirst(PRegister pd, PRegister pg, PRegister pn) {\n LOGMAN_THROW_A_FMT(pd == pn, \"pd and pn need to be the same\");\n SVEPredicateMisc(0b1000, 0b00000, pg.Idx(), SubRegSize::i16Bit, pd);\n }\n\n // SVE predicate zero\n void pfalse(PRegister pd) {\n SVEPredicateMisc(0b1000, 0b10010, 0b0000, SubRegSize::i8Bit, pd);\n }\n\n // SVE predicate read from FFR (predicated)\n void rdffr(PRegister pd, PRegisterZero pg) {\n SVEPredicateMisc(0b1000, 0b11000, pg.Idx(), SubRegSize::i8Bit, pd);\n }\n\n void rdffrs(PRegister pd, PRegisterZero pg) {\n SVEPredicateMisc(0b1000, 0b11000, pg.Idx(), SubRegSize::i16Bit, pd);\n }\n\n // SVE predicate read from FFR (unpredicated)\n void rdffr(PRegister pd) {\n SVEPredicateMisc(0b1001, 0b11000, 0b0000, SubRegSize::i8Bit, pd);\n }\n\n // SVE predicate initialize\n void ptrue(SubRegSize size, PRegister pd, PredicatePattern pattern) {\n SVEPredicateMisc(0b1000, 0b10000, FEXCore::ToUnderlying(pattern), size, pd);\n }\n void ptrues(SubRegSize size, PRegister pd, PredicatePattern pattern) {\n SVEPredicateMisc(0b1001, 0b10000, FEXCore::ToUnderlying(pattern), size, pd);\n }\n\n // SVE Integer Compare - Scalars\n // SVE integer compare scalar count and limit\n template\n void whilege(SubRegSize size, PRegister pd, T rn, T rm) {\n constexpr auto IsXRegister = static_cast(std::is_same_v);\n SVEIntCompareScalar(IsXRegister << 2, 0, pd.Idx(), size, rn, rm);\n }\n template\n void whilegt(SubRegSize size, PRegister pd, T rn, T rm) {\n constexpr auto IsXRegister = static_cast(std::is_same_v);\n SVEIntCompareScalar(IsXRegister << 2, 1, pd.Idx(), size, rn, rm);\n }\n template\n void whilelt(SubRegSize size, PRegister pd, T rn, T rm) {\n constexpr auto IsXRegister = static_cast(std::is_same_v);\n SVEIntCompareScalar((IsXRegister << 2) | 0b001, 0, pd.Idx(), size, rn, rm);\n }\n template\n void whilele(SubRegSize size, PRegister pd, T rn, T rm) {\n constexpr auto IsXRegister = static_cast(std::is_same_v);\n SVEIntCompareScalar((IsXRegister << 2) | 0b001, 1, pd.Idx(), size, rn, rm);\n }\n template\n void whilehs(SubRegSize size, PRegister pd, T rn, T rm) {\n constexpr auto IsXRegister = static_cast(std::is_same_v);\n SVEIntCompareScalar((IsXRegister << 2) | 0b010, 0, pd.Idx(), size, rn, rm);\n }\n template\n void whilehi(SubRegSize size, PRegister pd, T rn, T rm) {\n constexpr auto IsXRegister = static_cast(std::is_same_v);\n SVEIntCompareScalar((IsXRegister << 2) | 0b010, 1, pd.Idx(), size, rn, rm);\n }\n template\n void whilelo(SubRegSize size, PRegister pd, T rn, T rm) {\n constexpr auto IsXRegister = static_cast(std::is_same_v);\n SVEIntCompareScalar((IsXRegister << 2) | 0b011, 0, pd.Idx(), size, rn, rm);\n }\n template\n void whilels(SubRegSize size, PRegister pd, T rn, T rm) {\n constexpr auto IsXRegister = static_cast(std::is_same_v);\n SVEIntCompareScalar((IsXRegister << 2) | 0b011, 1, pd.Idx(), size, rn, rm);\n }\n\n // SVE conditionally terminate scalars\n template\n void ctermeq(T rn, T rm) {\n constexpr auto size = std::is_same_v ? SubRegSize::i64Bit : SubRegSize::i32Bit;\n SVEIntCompareScalar(0b1000, 0, 0b0000, size, rn, rm);\n }\n template\n void ctermne(T rn, T rm) {\n constexpr auto size = std::is_same_v ? SubRegSize::i64Bit : SubRegSize::i32Bit;\n SVEIntCompareScalar(0b1000, 1, 0b0000, size, rn, rm);\n }\n\n // SVE pointer conflict compare\n void whilewr(SubRegSize size, PRegister pd, XRegister rn, XRegister rm) {\n SVEIntCompareScalar(0b1100, 0, pd.Idx(), size, rn, rm);\n }\n void whilerw(SubRegSize size, PRegister pd, XRegister rn, XRegister rm) {\n SVEIntCompareScalar(0b1100, 1, pd.Idx(), size, rn, rm);\n }\n\n // SVE Integer Wide Immediate - Unpredicated\n // SVE integer add/subtract immediate (unpredicated)\n void add(SubRegSize size, ZRegister zd, ZRegister zn, uint32_t imm) {\n SVEAddSubImmediateUnpred(0b000, size, zd, zn, imm);\n }\n void sub(SubRegSize size, ZRegister zd, ZRegister zn, uint32_t imm) {\n SVEAddSubImmediateUnpred(0b001, size, zd, zn, imm);\n }\n void subr(SubRegSize size, ZRegister zd, ZRegister zn, uint32_t imm) {\n SVEAddSubImmediateUnpred(0b011, size, zd, zn, imm);\n }\n void sqadd(SubRegSize size, ZRegister zd, ZRegister zn, uint32_t imm) {\n SVEAddSubImmediateUnpred(0b100, size, zd, zn, imm);\n }\n void uqadd(SubRegSize size, ZRegister zd, ZRegister zn, uint32_t imm) {\n SVEAddSubImmediateUnpred(0b101, size, zd, zn, imm);\n }\n void sqsub(SubRegSize size, ZRegister zd, ZRegister zn, uint32_t imm) {\n SVEAddSubImmediateUnpred(0b110, size, zd, zn, imm);\n }\n void uqsub(SubRegSize size, ZRegister zd, ZRegister zn, uint32_t imm) {\n SVEAddSubImmediateUnpred(0b111, size, zd, zn, imm);\n }\n\n // SVE integer min/max immediate (unpredicated)\n void smax(SubRegSize size, ZRegister zd, ZRegister zn, int32_t imm) {\n SVEMinMaxImmediateUnpred(0b000, size, zd, zn, imm);\n }\n void umax(SubRegSize size, ZRegister zd, ZRegister zn, int32_t imm) {\n SVEMinMaxImmediateUnpred(0b001, size, zd, zn, imm);\n }\n void smin(SubRegSize size, ZRegister zd, ZRegister zn, int32_t imm) {\n SVEMinMaxImmediateUnpred(0b010, size, zd, zn, imm);\n }\n void umin(SubRegSize size, ZRegister zd, ZRegister zn, int32_t imm) {\n SVEMinMaxImmediateUnpred(0b011, size, zd, zn, imm);\n }\n\n // SVE integer multiply immediate (unpredicated)\n void mul(SubRegSize size, ZRegister zd, ZRegister zn, int32_t imm) {\n SVEMultiplyImmediateUnpred(0b000, size, zd, zn, imm);\n }\n\n // SVE broadcast integer immediate (unpredicated)\n void dup_imm(SubRegSize size, ZRegister zd, int32_t Value) {\n SVEBroadcastImm(0b00, Value, size, zd);\n }\n void mov_imm(SubRegSize size, ZRegister zd, int32_t Value) {\n dup_imm(size, zd, Value);\n }\n\n // SVE broadcast floating-point immediate (unpredicated)\n void fdup(SubRegSize size, ZRegister zd, float Value) {\n LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), \"Unsupported fmov size\");\n\n uint32_t Imm {};\n if (size == SubRegSize::i16Bit) {\n LOGMAN_MSG_A_FMT(\"Unsupported\");\n FEX_UNREACHABLE;\n } else if (size == SubRegSize::i32Bit) {\n Imm = FP32ToImm8(Value);\n } else if (size == SubRegSize::i64Bit) {\n Imm = FP64ToImm8(Value);\n }\n\n SVEBroadcastFloatImmUnpredicated(0b00, 0, Imm, size, zd);\n }\n void fmov(SubRegSize size, ZRegister zd, float Value) {\n fdup(size, zd, Value);\n }\n\n // SVE Predicate Count\n // SVE predicate count\n void cntp(SubRegSize size, XRegister rd, PRegister pg, PRegister pn) {\n SVEPredicateCount(0b000, size, rd, pg, pn);\n }\n\n // SVE Inc/Dec by Predicate Count\n // SVE saturating inc/dec vector by predicate count\n void sqincp(SubRegSize size, ZRegister zdn, PRegister pm) {\n SVEIncDecPredicateCountVector(0, 0, 0b00, 0b00, size, zdn, pm);\n }\n void uqincp(SubRegSize size, ZRegister zdn, PRegister pm) {\n SVEIncDecPredicateCountVector(0, 0, 0b00, 0b01, size, zdn, pm);\n }\n void sqdecp(SubRegSize size, ZRegister zdn, PRegister pm) {\n SVEIncDecPredicateCountVector(0, 0, 0b00, 0b10, size, zdn, pm);\n }\n void uqdecp(SubRegSize size, ZRegister zdn, PRegister pm) {\n SVEIncDecPredicateCountVector(0, 0, 0b00, 0b11, size, zdn, pm);\n }\n\n // SVE saturating inc/dec register by predicate count\n void sqincp(SubRegSize size, XRegister rdn, PRegister pm) {\n SVEIncDecPredicateCountScalar(0, 1, 0b10, 0b00, size, rdn, pm);\n }\n void sqincp(SubRegSize size, XRegister rdn, PRegister pm, WRegister wn) {\n LOGMAN_THROW_A_FMT(rdn.Idx() == wn.Idx(), \"rdn and wn must be the same\");\n SVEIncDecPredicateCountScalar(0, 1, 0b00, 0b00, size, rdn, pm);\n }\n void uqincp(SubRegSize size, XRegister rdn, PRegister pm) {\n SVEIncDecPredicateCountScalar(0, 1, 0b10, 0b01, size, rdn, pm);\n }\n void uqincp(SubRegSize size, WRegister rdn, PRegister pm) {\n SVEIncDecPredicateCountScalar(0, 1, 0b00, 0b01, size, rdn, pm);\n }\n void sqdecp(SubRegSize size, XRegister rdn, PRegister pm) {\n SVEIncDecPredicateCountScalar(0, 1, 0b10, 0b10, size, rdn, pm);\n }\n void sqdecp(SubRegSize size, XRegister rdn, PRegister pm, WRegister wn) {\n LOGMAN_THROW_A_FMT(rdn.Idx() == wn.Idx(), \"rdn and wn must be the same\");\n SVEIncDecPredicateCountScalar(0, 1, 0b00, 0b10, size, rdn, pm);\n }\n void uqdecp(SubRegSize size, XRegister rdn, PRegister pm) {\n SVEIncDecPredicateCountScalar(0, 1, 0b10, 0b11, size, rdn, pm);\n }\n void uqdecp(SubRegSize size, WRegister rdn, PRegister pm) {\n SVEIncDecPredicateCountScalar(0, 1, 0b00, 0b11, size, rdn, pm);\n }\n\n // SVE inc/dec vector by predicate count\n void incp(SubRegSize size, ZRegister zdn, PRegister pm) {\n SVEIncDecPredicateCountVector(1, 0, 0b00, 0b00, size, zdn, pm);\n }\n void decp(SubRegSize size, ZRegister zdn, PRegister pm) {\n SVEIncDecPredicateCountVector(1, 0, 0b00, 0b01, size, zdn, pm);\n }\n\n // SVE inc/dec register by predicate count\n void incp(SubRegSize size, XRegister rdn, PRegister pm) {\n SVEIncDecPredicateCountScalar(1, 1, 0b00, 0b00, size, rdn, pm);\n }\n void decp(SubRegSize size, XRegister rdn, PRegister pm) {\n SVEIncDecPredicateCountScalar(1, 1, 0b00, 0b01, size, rdn, pm);\n }\n\n // SVE Write FFR\n // SVE FFR write from predicate\n void wrffr(PRegister pn) {\n SVEWriteFFR(0, 0b00, 0b000, pn.Idx(), 0b00000);\n }\n // SVE FFR initialise\n void setffr() {\n SVEWriteFFR(1, 0b00, 0b000, 0b0000, 0b00000);\n }\n\n // SVE Integer Multiply-Add - Unpredicated\n void cdot(SubRegSize size, ZRegister zda, ZRegister zn, ZRegister zm, Rotation rot) {\n SVEIntegerDotProduct(0b0001, size, zda, zn, zm, rot);\n }\n\n // SVE integer dot product (unpredicated)\n void sdot(SubRegSize size, ZRegister zda, ZRegister zn, ZRegister zm) {\n SVEIntegerDotProduct(0b0000, size, zda, zn, zm, Rotation::ROTATE_0);\n }\n void udot(SubRegSize size, ZRegister zda, ZRegister zn, ZRegister zm) {\n SVEIntegerDotProduct(0b0000, size, zda, zn, zm, Rotation::ROTATE_90);\n }\n\n // SVE2 saturating multiply-add interleaved long\n void sqdmlalbt(SubRegSize size, ZRegister zda, ZRegister zn, ZRegister zm) {\n SVE2SaturatingMulAddInterleaved(0b000010, size, zda, zn, zm);\n }\n void sqdmlslbt(SubRegSize size, ZRegister zda, ZRegister zn, ZRegister zm) {\n SVE2SaturatingMulAddInterleaved(0b000011, size, zda, zn, zm);\n }\n\n // SVE2 complex integer multiply-add\n void cmla(SubRegSize size, ZRegister zda, ZRegister zn, ZRegister zm, Rotation rot) {\n SVEIntegerComplexMulAdd(0b0010, size, zda, zn, zm, rot);\n }\n void sqrdcmlah(SubRegSize size, ZRegister zda, ZRegister zn, ZRegister zm, Rotation rot) {\n SVEIntegerComplexMulAdd(0b0011, size, zda, zn, zm, rot);\n }\n\n // SVE2 integer multiply-add long\n void smlalb(SubRegSize size, ZRegister zda, ZRegister zn, ZRegister zm) {\n SVE2IntegerMulAddLong(0b010'000, size, zda, zn, zm);\n }\n void smlalt(SubRegSize size, ZRegister zda, ZRegister zn, ZRegister zm) {\n SVE2IntegerMulAddLong(0b010'001, size, zda, zn, zm);\n }\n void umlalb(SubRegSize size, ZRegister zda, ZRegister zn, ZRegister zm) {\n SVE2IntegerMulAddLong(0b010'010, size, zda, zn, zm);\n }\n void umlalt(SubRegSize size, ZRegister zda, ZRegister zn, ZRegister zm) {\n SVE2IntegerMulAddLong(0b010'011, size, zda, zn, zm);\n }\n void smlslb(SubRegSize size, ZRegister zda, ZRegister zn, ZRegister zm) {\n SVE2IntegerMulAddLong(0b010'100, size, zda, zn, zm);\n }\n void smlslt(SubRegSize size, ZRegister zda, ZRegister zn, ZRegister zm) {\n SVE2IntegerMulAddLong(0b010'101, size, zda, zn, zm);\n }\n void umlslb(SubRegSize size, ZRegister zda, ZRegister zn, ZRegister zm) {\n SVE2IntegerMulAddLong(0b010'110, size, zda, zn, zm);\n }\n void umlslt(SubRegSize size, ZRegister zda, ZRegister zn, ZRegister zm) {\n SVE2IntegerMulAddLong(0b010'111, size, zda, zn, zm);\n }\n\n // SVE2 saturating multiply-add long\n void sqdmlalb(SubRegSize size, ZRegister zda, ZRegister zn, ZRegister zm) {\n SVE2IntegerMulAddLong(0b0110'00, size, zda, zn, zm);\n }\n void sqdmlalt(SubRegSize size, ZRegister zda, ZRegister zn, ZRegister zm) {\n SVE2IntegerMulAddLong(0b0110'01, size, zda, zn, zm);\n }\n void sqdmlslb(SubRegSize size, ZRegister zda, ZRegister zn, ZRegister zm) {\n SVE2IntegerMulAddLong(0b0110'10, size, zda, zn, zm);\n }\n void sqdmlslt(SubRegSize size, ZRegister zda, ZRegister zn, ZRegister zm) {\n SVE2IntegerMulAddLong(0b0110'11, size, zda, zn, zm);\n }\n\n // SVE2 saturating multiply-add high\n void sqrdmlah(SubRegSize size, ZRegister zda, ZRegister zn, ZRegister zm) {\n SVEIntegerMultiplyAddUnpredicated(0b011'100, size, zda, zn, zm);\n }\n void sqrdmlsh(SubRegSize size, ZRegister zda, ZRegister zn, ZRegister zm) {\n SVEIntegerMultiplyAddUnpredicated(0b011'101, size, zda, zn, zm);\n }\n\n // SVE mixed sign dot product\n void usdot(ZRegister zda, ZRegister zn, ZRegister zm) {\n SVEIntegerDotProduct(0b0111, SubRegSize::i32Bit, zda, zn, zm, Rotation::ROTATE_180);\n }\n\n // SVE2 Integer - Predicated\n // SVE2 integer pairwise add and accumulate long\n void sadalp(SubRegSize size, ZRegister zda, PRegisterMerge pg, ZRegister zn) {\n SVE2IntegerPairwiseAddAccumulateLong(0, size, zda, pg, zn);\n }\n void uadalp(SubRegSize size, ZRegister zda, PRegisterMerge pg, ZRegister zn) {\n SVE2IntegerPairwiseAddAccumulateLong(1, size, zda, pg, zn);\n }\n\n // SVE2 integer unary operations (predicated)\n void urecpe(ZRegister zd, PRegisterMerge pg, ZRegister zn) {\n SVE2IntegerUnaryOpsPredicated(0b00000, SubRegSize::i32Bit, zd, pg, zn);\n }\n void ursqrte(ZRegister zd, PRegisterMerge pg, ZRegister zn) {\n SVE2IntegerUnaryOpsPredicated(0b00001, SubRegSize::i32Bit, zd, pg, zn);\n }\n void sqabs(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn) {\n SVE2IntegerUnaryOpsPredicated(0b01000, size, zd, pg, zn);\n }\n void sqneg(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn) {\n SVE2IntegerUnaryOpsPredicated(0b01001, size, zd, pg, zn);\n }\n\n // SVE2 saturating/rounding bitwise shift left (predicated)\n void srshl(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n SVE2SaturatingRoundingBitwiseShiftLeft(0b00010, size, zd, pg, zn, zm);\n }\n void urshl(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n SVE2SaturatingRoundingBitwiseShiftLeft(0b00011, size, zd, pg, zn, zm);\n }\n void srshlr(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n SVE2SaturatingRoundingBitwiseShiftLeft(0b00110, size, zd, pg, zn, zm);\n }\n void urshlr(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n SVE2SaturatingRoundingBitwiseShiftLeft(0b00111, size, zd, pg, zn, zm);\n }\n void sqshl(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n SVE2SaturatingRoundingBitwiseShiftLeft(0b01000, size, zd, pg, zn, zm);\n }\n void uqshl(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n SVE2SaturatingRoundingBitwiseShiftLeft(0b01001, size, zd, pg, zn, zm);\n }\n void sqrshl(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n SVE2SaturatingRoundingBitwiseShiftLeft(0b01010, size, zd, pg, zn, zm);\n }\n void uqrshl(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n SVE2SaturatingRoundingBitwiseShiftLeft(0b01011, size, zd, pg, zn, zm);\n }\n void sqshlr(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n SVE2SaturatingRoundingBitwiseShiftLeft(0b01100, size, zd, pg, zn, zm);\n }\n void uqshlr(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n SVE2SaturatingRoundingBitwiseShiftLeft(0b01101, size, zd, pg, zn, zm);\n }\n void sqrshlr(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n SVE2SaturatingRoundingBitwiseShiftLeft(0b01110, size, zd, pg, zn, zm);\n }\n void uqrshlr(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n SVE2SaturatingRoundingBitwiseShiftLeft(0b01111, size, zd, pg, zn, zm);\n }\n\n // SVE2 integer halving add/subtract (predicated)\n void shadd(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n SVE2IntegerHalvingPredicated(0b000, size, pg, zd, zn, zm);\n }\n void uhadd(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n SVE2IntegerHalvingPredicated(0b001, size, pg, zd, zn, zm);\n }\n void shsub(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n SVE2IntegerHalvingPredicated(0b010, size, pg, zd, zn, zm);\n }\n void uhsub(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n SVE2IntegerHalvingPredicated(0b011, size, pg, zd, zn, zm);\n }\n void srhadd(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n SVE2IntegerHalvingPredicated(0b100, size, pg, zd, zn, zm);\n }\n void urhadd(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n SVE2IntegerHalvingPredicated(0b101, size, pg, zd, zn, zm);\n }\n void shsubr(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n SVE2IntegerHalvingPredicated(0b110, size, pg, zd, zn, zm);\n }\n void uhsubr(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n SVE2IntegerHalvingPredicated(0b111, size, pg, zd, zn, zm);\n }\n\n // SVE2 integer pairwise arithmetic\n void addp(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n SVEIntegerPairwiseArithmetic(0b00, 1, size, pg, zd, zn, zm);\n }\n void smaxp(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n SVEIntegerPairwiseArithmetic(0b10, 0, size, pg, zd, zn, zm);\n }\n void umaxp(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n SVEIntegerPairwiseArithmetic(0b10, 1, size, pg, zd, zn, zm);\n }\n void sminp(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n SVEIntegerPairwiseArithmetic(0b11, 0, size, pg, zd, zn, zm);\n }\n void uminp(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n SVEIntegerPairwiseArithmetic(0b11, 1, size, pg, zd, zn, zm);\n }\n\n // SVE2 saturating add/subtract\n void sqadd(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n SVE2IntegerSaturatingAddSub(0b000, size, zd, pg, zn, zm);\n }\n void uqadd(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n SVE2IntegerSaturatingAddSub(0b001, size, zd, pg, zn, zm);\n }\n void sqsub(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n SVE2IntegerSaturatingAddSub(0b010, size, zd, pg, zn, zm);\n }\n void uqsub(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n SVE2IntegerSaturatingAddSub(0b011, size, zd, pg, zn, zm);\n }\n void suqadd(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n SVE2IntegerSaturatingAddSub(0b100, size, zd, pg, zn, zm);\n }\n void usqadd(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n SVE2IntegerSaturatingAddSub(0b101, size, zd, pg, zn, zm);\n }\n void sqsubr(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n SVE2IntegerSaturatingAddSub(0b110, size, zd, pg, zn, zm);\n }\n void uqsubr(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n SVE2IntegerSaturatingAddSub(0b111, size, zd, pg, zn, zm);\n }\n\n // SVE2 Widening Integer Arithmetic\n // SVE2 integer add/subtract long\n void saddlb(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n SVE2IntegerAddSubLong(0, 0b000, size, zd, zn, zm);\n }\n\n void saddlt(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n SVE2IntegerAddSubLong(0, 0b001, size, zd, zn, zm);\n }\n\n void uaddlb(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n SVE2IntegerAddSubLong(0, 0b010, size, zd, zn, zm);\n }\n\n void uaddlt(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n SVE2IntegerAddSubLong(0, 0b011, size, zd, zn, zm);\n }\n\n void ssublb(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n SVE2IntegerAddSubLong(0, 0b100, size, zd, zn, zm);\n }\n\n void ssublt(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n SVE2IntegerAddSubLong(0, 0b101, size, zd, zn, zm);\n }\n\n void usublb(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n SVE2IntegerAddSubLong(0, 0b110, size, zd, zn, zm);\n }\n\n void usublt(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n SVE2IntegerAddSubLong(0, 0b111, size, zd, zn, zm);\n }\n\n void sabdlb(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n SVE2IntegerAddSubLong(1, 0b100, size, zd, zn, zm);\n }\n\n void sabdlt(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n SVE2IntegerAddSubLong(1, 0b101, size, zd, zn, zm);\n }\n\n void uabdlb(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n SVE2IntegerAddSubLong(1, 0b110, size, zd, zn, zm);\n }\n\n void uabdlt(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n SVE2IntegerAddSubLong(1, 0b111, size, zd, zn, zm);\n }\n\n // SVE2 integer add/subtract wide\n void saddwb(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n SVE2IntegerAddSubWide(0b000, size, zd, zn, zm);\n }\n void saddwt(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n SVE2IntegerAddSubWide(0b001, size, zd, zn, zm);\n }\n void uaddwb(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n SVE2IntegerAddSubWide(0b010, size, zd, zn, zm);\n }\n void uaddwt(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n SVE2IntegerAddSubWide(0b011, size, zd, zn, zm);\n }\n void ssubwb(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n SVE2IntegerAddSubWide(0b100, size, zd, zn, zm);\n }\n void ssubwt(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n SVE2IntegerAddSubWide(0b101, size, zd, zn, zm);\n }\n void usubwb(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n SVE2IntegerAddSubWide(0b110, size, zd, zn, zm);\n }\n void usubwt(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n SVE2IntegerAddSubWide(0b111, size, zd, zn, zm);\n }\n\n // SVE2 integer multiply long\n void sqdmullb(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n SVE2IntegerMultiplyLong(0b000, size, zd, zn, zm);\n }\n void sqdmullt(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n SVE2IntegerMultiplyLong(0b001, size, zd, zn, zm);\n }\n void pmullb(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n SVE2IntegerMultiplyLong(0b010, size, zd, zn, zm);\n }\n void pmullt(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n SVE2IntegerMultiplyLong(0b011, size, zd, zn, zm);\n }\n void smullb(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n SVE2IntegerMultiplyLong(0b100, size, zd, zn, zm);\n }\n void smullt(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n SVE2IntegerMultiplyLong(0b101, size, zd, zn, zm);\n }\n void umullb(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n SVE2IntegerMultiplyLong(0b110, size, zd, zn, zm);\n }\n void umullt(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n SVE2IntegerMultiplyLong(0b111, size, zd, zn, zm);\n }\n\n //\n // SVE Misc\n // SVE2 bitwise shift left long\n void sshllb(SubRegSize size, ZRegister zd, ZRegister zn, uint32_t shift) {\n SVE2BitwiseShiftLeftLong(size, 0b00, zd, zn, shift);\n }\n void sshllt(SubRegSize size, ZRegister zd, ZRegister zn, uint32_t shift) {\n SVE2BitwiseShiftLeftLong(size, 0b01, zd, zn, shift);\n }\n void ushllb(SubRegSize size, ZRegister zd, ZRegister zn, uint32_t shift) {\n SVE2BitwiseShiftLeftLong(size, 0b10, zd, zn, shift);\n }\n void ushllt(SubRegSize size, ZRegister zd, ZRegister zn, uint32_t shift) {\n SVE2BitwiseShiftLeftLong(size, 0b11, zd, zn, shift);\n }\n\n // SVE2 integer add/subtract interleaved long\n void saddlbt(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n SVE2IntegerAddSubInterleavedLong(size, 0b00, zd, zn, zm);\n }\n void ssublbt(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n SVE2IntegerAddSubInterleavedLong(size, 0b10, zd, zn, zm);\n }\n void ssubltb(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n SVE2IntegerAddSubInterleavedLong(size, 0b11, zd, zn, zm);\n }\n\n // SVE2 bitwise exclusive-or interleaved\n void eorbt(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n SVE2BitwiseXorInterleaved(size, 0b0, zd, zn, zm);\n }\n void eortb(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n SVE2BitwiseXorInterleaved(size, 0b1, zd, zn, zm);\n }\n\n // SVE integer matrix multiply accumulate\n void smmla(ZRegister zda, ZRegister zn, ZRegister zm) {\n SVEIntegerMatrixMulAccumulate(0b00, zda, zn, zm);\n }\n void usmmla(ZRegister zda, ZRegister zn, ZRegister zm) {\n SVEIntegerMatrixMulAccumulate(0b10, zda, zn, zm);\n }\n void ummla(ZRegister zda, ZRegister zn, ZRegister zm) {\n SVEIntegerMatrixMulAccumulate(0b11, zda, zn, zm);\n }\n\n // SVE2 bitwise permute\n void bext(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n SVE2BitwisePermute(size, 0b00, zd, zn, zm);\n }\n void bdep(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n SVE2BitwisePermute(size, 0b01, zd, zn, zm);\n }\n void bgrp(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n SVE2BitwisePermute(size, 0b10, zd, zn, zm);\n }\n\n // SVE2 Accumulate\n // SVE2 complex integer add\n void cadd(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm, Rotation rot) {\n SVE2ComplexIntAdd(size, 0b0, rot, zd, zn, zm);\n }\n void sqcadd(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm, Rotation rot) {\n SVE2ComplexIntAdd(size, 0b1, rot, zd, zn, zm);\n }\n\n // SVE2 integer absolute difference and accumulate long\n void sabalb(SubRegSize size, ZRegister zda, ZRegister zn, ZRegister zm) {\n SVE2IntegerAddSubInterleavedLong(size, 0b10000, zda, zn, zm);\n }\n void sabalt(SubRegSize size, ZRegister zda, ZRegister zn, ZRegister zm) {\n SVE2IntegerAddSubInterleavedLong(size, 0b10001, zda, zn, zm);\n }\n void uabalb(SubRegSize size, ZRegister zda, ZRegister zn, ZRegister zm) {\n SVE2IntegerAddSubInterleavedLong(size, 0b10010, zda, zn, zm);\n }\n void uabalt(SubRegSize size, ZRegister zda, ZRegister zn, ZRegister zm) {\n SVE2IntegerAddSubInterleavedLong(size, 0b10011, zda, zn, zm);\n }\n\n // SVE2 integer add/subtract long with carry\n void adclb(SubRegSize size, ZRegister zda, ZRegister zn, ZRegister zm) {\n SVE2IntegerAddSubLongWithCarry(size, 0, 0, zda, zn, zm);\n }\n void adclt(SubRegSize size, ZRegister zda, ZRegister zn, ZRegister zm) {\n SVE2IntegerAddSubLongWithCarry(size, 0, 1, zda, zn, zm);\n }\n void sbclb(SubRegSize size, ZRegister zda, ZRegister zn, ZRegister zm) {\n SVE2IntegerAddSubLongWithCarry(size, 1, 0, zda, zn, zm);\n }\n void sbclt(SubRegSize size, ZRegister zda, ZRegister zn, ZRegister zm) {\n SVE2IntegerAddSubLongWithCarry(size, 1, 1, zda, zn, zm);\n }\n\n // SVE2 bitwise shift right and accumulate\n void ssra(SubRegSize size, ZRegister zda, ZRegister zn, uint32_t shift) {\n SVE2BitwiseShiftRightAndAccumulate(size, 0b00, zda, zn, shift);\n }\n void usra(SubRegSize size, ZRegister zda, ZRegister zn, uint32_t shift) {\n SVE2BitwiseShiftRightAndAccumulate(size, 0b01, zda, zn, shift);\n }\n void srsra(SubRegSize size, ZRegister zda, ZRegister zn, uint32_t shift) {\n SVE2BitwiseShiftRightAndAccumulate(size, 0b10, zda, zn, shift);\n }\n void ursra(SubRegSize size, ZRegister zda, ZRegister zn, uint32_t shift) {\n SVE2BitwiseShiftRightAndAccumulate(size, 0b11, zda, zn, shift);\n }\n\n // SVE2 bitwise shift and insert\n void sri(SubRegSize size, ZRegister zda, ZRegister zn, uint32_t shift) {\n SVE2BitwiseShiftAndInsert(size, 0b0, zda, zn, shift);\n }\n void sli(SubRegSize size, ZRegister zda, ZRegister zn, uint32_t shift) {\n SVE2BitwiseShiftAndInsert(size, 0b1, zda, zn, shift);\n }\n\n // SVE2 integer absolute difference and accumulate\n void saba(SubRegSize size, ZRegister zda, ZRegister zn, ZRegister zm) {\n SVE2IntegerAbsDiffAndAccumulate(size, 0b0, zda, zn, zm);\n }\n void uaba(SubRegSize size, ZRegister zda, ZRegister zn, ZRegister zm) {\n SVE2IntegerAbsDiffAndAccumulate(size, 0b1, zda, zn, zm);\n }\n\n // SVE2 Narrowing\n // SVE2 saturating extract narrow\n void sqxtnb(SubRegSize size, ZRegister zd, ZRegister zn) {\n SVE2SaturatingExtractNarrow(size, 0b00, 0, zn, zd);\n }\n void sqxtnt(SubRegSize size, ZRegister zd, ZRegister zn) {\n SVE2SaturatingExtractNarrow(size, 0b00, 1, zn, zd);\n }\n void uqxtnb(SubRegSize size, ZRegister zd, ZRegister zn) {\n SVE2SaturatingExtractNarrow(size, 0b01, 0, zn, zd);\n }\n void uqxtnt(SubRegSize size, ZRegister zd, ZRegister zn) {\n SVE2SaturatingExtractNarrow(size, 0b01, 1, zn, zd);\n }\n void sqxtunb(SubRegSize size, ZRegister zd, ZRegister zn) {\n SVE2SaturatingExtractNarrow(size, 0b10, 0, zn, zd);\n }\n void sqxtunt(SubRegSize size, ZRegister zd, ZRegister zn) {\n SVE2SaturatingExtractNarrow(size, 0b10, 1, zn, zd);\n }\n\n // SVE2 bitwise shift right narrow\n void sqshrunb(SubRegSize size, ZRegister zd, ZRegister zn, uint32_t Shift) {\n SVE2BitwiseShiftRightNarrow(size, Shift, 0, 0, 0, 0, zn, zd);\n }\n void sqshrunt(SubRegSize size, ZRegister zd, ZRegister zn, uint32_t Shift) {\n SVE2BitwiseShiftRightNarrow(size, Shift, 0, 0, 0, 1, zn, zd);\n }\n void sqrshrunb(SubRegSize size, ZRegister zd, ZRegister zn, uint32_t Shift) {\n SVE2BitwiseShiftRightNarrow(size, Shift, 0, 0, 1, 0, zn, zd);\n }\n void sqrshrunt(SubRegSize size, ZRegister zd, ZRegister zn, uint32_t Shift) {\n SVE2BitwiseShiftRightNarrow(size, Shift, 0, 0, 1, 1, zn, zd);\n }\n void shrnb(SubRegSize size, ZRegister zd, ZRegister zn, uint32_t Shift) {\n SVE2BitwiseShiftRightNarrow(size, Shift, 0, 1, 0, 0, zn, zd);\n }\n void shrnt(SubRegSize size, ZRegister zd, ZRegister zn, uint32_t Shift) {\n SVE2BitwiseShiftRightNarrow(size, Shift, 0, 1, 0, 1, zn, zd);\n }\n void rshrnb(SubRegSize size, ZRegister zd, ZRegister zn, uint32_t Shift) {\n SVE2BitwiseShiftRightNarrow(size, Shift, 0, 1, 1, 0, zn, zd);\n }\n void rshrnt(SubRegSize size, ZRegister zd, ZRegister zn, uint32_t Shift) {\n SVE2BitwiseShiftRightNarrow(size, Shift, 0, 1, 1, 1, zn, zd);\n }\n void sqshrnb(SubRegSize size, ZRegister zd, ZRegister zn, uint32_t Shift) {\n SVE2BitwiseShiftRightNarrow(size, Shift, 1, 0, 0, 0, zn, zd);\n }\n void sqshrnt(SubRegSize size, ZRegister zd, ZRegister zn, uint32_t Shift) {\n SVE2BitwiseShiftRightNarrow(size, Shift, 1, 0, 0, 1, zn, zd);\n }\n void sqrshrnb(SubRegSize size, ZRegister zd, ZRegister zn, uint32_t Shift) {\n SVE2BitwiseShiftRightNarrow(size, Shift, 1, 0, 1, 0, zn, zd);\n }\n void sqrshrnt(SubRegSize size, ZRegister zd, ZRegister zn, uint32_t Shift) {\n SVE2BitwiseShiftRightNarrow(size, Shift, 1, 0, 1, 1, zn, zd);\n }\n void uqshrnb(SubRegSize size, ZRegister zd, ZRegister zn, uint32_t Shift) {\n SVE2BitwiseShiftRightNarrow(size, Shift, 1, 1, 0, 0, zn, zd);\n }\n void uqshrnt(SubRegSize size, ZRegister zd, ZRegister zn, uint32_t Shift) {\n SVE2BitwiseShiftRightNarrow(size, Shift, 1, 1, 0, 1, zn, zd);\n }\n void uqrshrnb(SubRegSize size, ZRegister zd, ZRegister zn, uint32_t Shift) {\n SVE2BitwiseShiftRightNarrow(size, Shift, 1, 1, 1, 0, zn, zd);\n }\n void uqrshrnt(SubRegSize size, ZRegister zd, ZRegister zn, uint32_t Shift) {\n SVE2BitwiseShiftRightNarrow(size, Shift, 1, 1, 1, 1, zn, zd);\n }\n\n // SVE2 integer add/subtract narrow high part\n void addhnb(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n SVE2IntegerAddSubNarrowHighPart(size, 0b000, zd, zn, zm);\n }\n void addhnt(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n SVE2IntegerAddSubNarrowHighPart(size, 0b001, zd, zn, zm);\n }\n void raddhnb(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n SVE2IntegerAddSubNarrowHighPart(size, 0b010, zd, zn, zm);\n }\n void raddhnt(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n SVE2IntegerAddSubNarrowHighPart(size, 0b011, zd, zn, zm);\n }\n void subhnb(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n SVE2IntegerAddSubNarrowHighPart(size, 0b100, zd, zn, zm);\n }\n void subhnt(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n SVE2IntegerAddSubNarrowHighPart(size, 0b101, zd, zn, zm);\n }\n void rsubhnb(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n SVE2IntegerAddSubNarrowHighPart(size, 0b110, zd, zn, zm);\n }\n void rsubhnt(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n SVE2IntegerAddSubNarrowHighPart(size, 0b111, zd, zn, zm);\n }\n\n // SVE2 Crypto Extensions\n // SVE2 crypto unary operations\n void aesimc(ZRegister zdn, ZRegister zn) {\n SVE2CryptoUnaryOperation(1, zdn, zn);\n }\n void aesmc(ZRegister zdn, ZRegister zn) {\n SVE2CryptoUnaryOperation(0, zdn, zn);\n }\n\n // SVE2 crypto destructive binary operations\n void aese(ZRegister zdn, ZRegister zn, ZRegister zm) {\n SVE2CryptoDestructiveBinaryOperation(0, 0, zdn, zn, zm);\n }\n void aesd(ZRegister zdn, ZRegister zn, ZRegister zm) {\n SVE2CryptoDestructiveBinaryOperation(0, 1, zdn, zn, zm);\n }\n void sm4e(ZRegister zdn, ZRegister zn, ZRegister zm) {\n SVE2CryptoDestructiveBinaryOperation(1, 0, zdn, zn, zm);\n }\n\n // SVE2 crypto constructive binary operations\n void sm4ekey(ZRegister zd, ZRegister zn, ZRegister zm) {\n SVE2CryptoConstructiveBinaryOperation(0, zd, zn, zm);\n }\n void rax1(ZRegister zd, ZRegister zn, ZRegister zm) {\n SVE2CryptoConstructiveBinaryOperation(1, zd, zn, zm);\n }\n\n // SVE Floating Point Widening Multiply-Add - Indexed\n // SVE BFloat16 floating-point dot product (indexed)\n // XXX:\n\n // SVE floating-point multiply-add long (indexed)\n void fmlalb(SubRegSize dstsize, ZRegister zda, ZRegister zn, ZRegister zm, uint32_t index) {\n SVEFPMultiplyAddLongIndexed(0, 0, 0, dstsize, zda, zn, zm, index);\n }\n void fmlalt(SubRegSize dstsize, ZRegister zda, ZRegister zn, ZRegister zm, uint32_t index) {\n SVEFPMultiplyAddLongIndexed(0, 0, 1, dstsize, zda, zn, zm, index);\n }\n void fmlslb(SubRegSize dstsize, ZRegister zda, ZRegister zn, ZRegister zm, uint32_t index) {\n SVEFPMultiplyAddLongIndexed(0, 1, 0, dstsize, zda, zn, zm, index);\n }\n void fmlslt(SubRegSize dstsize, ZRegister zda, ZRegister zn, ZRegister zm, uint32_t index) {\n SVEFPMultiplyAddLongIndexed(0, 1, 1, dstsize, zda, zn, zm, index);\n }\n void bfmlalb(SubRegSize dstsize, ZRegister zda, ZRegister zn, ZRegister zm, uint32_t index) {\n SVEFPMultiplyAddLongIndexed(1, 0, 0, dstsize, zda, zn, zm, index);\n }\n void bfmlalt(SubRegSize dstsize, ZRegister zda, ZRegister zn, ZRegister zm, uint32_t index) {\n SVEFPMultiplyAddLongIndexed(1, 0, 1, dstsize, zda, zn, zm, index);\n }\n void bfmlslb(SubRegSize dstsize, ZRegister zda, ZRegister zn, ZRegister zm, uint32_t index) {\n SVEFPMultiplyAddLongIndexed(1, 1, 0, dstsize, zda, zn, zm, index);\n }\n void bfmlslt(SubRegSize dstsize, ZRegister zda, ZRegister zn, ZRegister zm, uint32_t index) {\n SVEFPMultiplyAddLongIndexed(1, 1, 1, dstsize, zda, zn, zm, index);\n }\n\n // SVE Floating Point Widening Multiply-Add\n // SVE BFloat16 floating-point dot product\n // XXX:\n\n // SVE floating-point multiply-add long\n void fmlalb(SubRegSize dstsize, ZRegister zda, ZRegister zn, ZRegister zm) {\n SVEFPMultiplyAddLong(0, 0, 0, dstsize, zda, zn, zm);\n }\n void fmlalt(SubRegSize dstsize, ZRegister zda, ZRegister zn, ZRegister zm) {\n SVEFPMultiplyAddLong(0, 0, 1, dstsize, zda, zn, zm);\n }\n void fmlslb(SubRegSize dstsize, ZRegister zda, ZRegister zn, ZRegister zm) {\n SVEFPMultiplyAddLong(0, 1, 0, dstsize, zda, zn, zm);\n }\n void fmlslt(SubRegSize dstsize, ZRegister zda, ZRegister zn, ZRegister zm) {\n SVEFPMultiplyAddLong(0, 1, 1, dstsize, zda, zn, zm);\n }\n void bfmlalb(SubRegSize dstsize, ZRegister zda, ZRegister zn, ZRegister zm) {\n SVEFPMultiplyAddLong(1, 0, 0, dstsize, zda, zn, zm);\n }\n void bfmlalt(SubRegSize dstsize, ZRegister zda, ZRegister zn, ZRegister zm) {\n SVEFPMultiplyAddLong(1, 0, 1, dstsize, zda, zn, zm);\n }\n void bfmlslb(SubRegSize dstsize, ZRegister zda, ZRegister zn, ZRegister zm) {\n SVEFPMultiplyAddLong(1, 1, 0, dstsize, zda, zn, zm);\n }\n void bfmlslt(SubRegSize dstsize, ZRegister zda, ZRegister zn, ZRegister zm) {\n SVEFPMultiplyAddLong(1, 1, 1, dstsize, zda, zn, zm);\n }\n\n // SVE Floating Point Arithmetic - Predicated\n void ftmad(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm, uint32_t imm) {\n LOGMAN_THROW_A_FMT(imm <= 7, \"ftmad immediate must be within 0-7\");\n SVEFloatArithmeticPredicated(0b10000 | imm, size, PReg::p0, zd, zn, zm);\n }\n // SVE floating-point arithmetic (predicated)\n void fadd(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n SVEFloatArithmeticPredicated(0b0000, size, pg, zd, zn, zm);\n }\n void fsub(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n SVEFloatArithmeticPredicated(0b0001, size, pg, zd, zn, zm);\n }\n void fmul(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n SVEFloatArithmeticPredicated(0b0010, size, pg, zd, zn, zm);\n }\n void fsubr(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n SVEFloatArithmeticPredicated(0b0011, size, pg, zd, zn, zm);\n }\n void fmaxnm(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n SVEFloatArithmeticPredicated(0b0100, size, pg, zd, zn, zm);\n }\n void fminnm(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n SVEFloatArithmeticPredicated(0b0101, size, pg, zd, zn, zm);\n }\n void fmax(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n SVEFloatArithmeticPredicated(0b0110, size, pg, zd, zn, zm);\n }\n void fmin(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n SVEFloatArithmeticPredicated(0b0111, size, pg, zd, zn, zm);\n }\n void fabd(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n SVEFloatArithmeticPredicated(0b1000, size, pg, zd, zn, zm);\n }\n void fscale(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n SVEFloatArithmeticPredicated(0b1001, size, pg, zd, zn, zm);\n }\n void fmulx(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n SVEFloatArithmeticPredicated(0b1010, size, pg, zd, zn, zm);\n }\n void fdivr(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n SVEFloatArithmeticPredicated(0b1100, size, pg, zd, zn, zm);\n }\n void fdiv(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n SVEFloatArithmeticPredicated(0b1101, size, pg, zd, zn, zm);\n }\n\n // SVE floating-point arithmetic with immediate (predicated)\n void fadd(SubRegSize size, ZRegister zd, PRegisterMerge pg, SVEFAddSubImm imm) {\n SVEFPArithWithImmediate(0b000, size, zd, pg, FEXCore::ToUnderlying(imm));\n }\n void fsub(SubRegSize size, ZRegister zd, PRegisterMerge pg, SVEFAddSubImm imm) {\n SVEFPArithWithImmediate(0b001, size, zd, pg, FEXCore::ToUnderlying(imm));\n }\n void fmul(SubRegSize size, ZRegister zd, PRegisterMerge pg, SVEFMulImm imm) {\n SVEFPArithWithImmediate(0b010, size, zd, pg, FEXCore::ToUnderlying(imm));\n }\n void fsubr(SubRegSize size, ZRegister zd, PRegisterMerge pg, SVEFAddSubImm imm) {\n SVEFPArithWithImmediate(0b011, size, zd, pg, FEXCore::ToUnderlying(imm));\n }\n void fmaxnm(SubRegSize size, ZRegister zd, PRegisterMerge pg, SVEFMaxMinImm imm) {\n SVEFPArithWithImmediate(0b100, size, zd, pg, FEXCore::ToUnderlying(imm));\n }\n void fminnm(SubRegSize size, ZRegister zd, PRegisterMerge pg, SVEFMaxMinImm imm) {\n SVEFPArithWithImmediate(0b101, size, zd, pg, FEXCore::ToUnderlying(imm));\n }\n void fmax(SubRegSize size, ZRegister zd, PRegisterMerge pg, SVEFMaxMinImm imm) {\n SVEFPArithWithImmediate(0b110, size, zd, pg, FEXCore::ToUnderlying(imm));\n }\n void fmin(SubRegSize size, ZRegister zd, PRegisterMerge pg, SVEFMaxMinImm imm) {\n SVEFPArithWithImmediate(0b111, size, zd, pg, FEXCore::ToUnderlying(imm));\n }\n\n // SVE Floating Point Unary Operations - Predicated\n // SVE floating-point round to integral value\n void frinti(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn) {\n SVEFloatRoundIntegral(0b111, size, zd, pg, zn);\n }\n void frintx(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn) {\n SVEFloatRoundIntegral(0b110, size, zd, pg, zn);\n }\n void frinta(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn) {\n SVEFloatRoundIntegral(0b100, size, zd, pg, zn);\n }\n void frintn(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn) {\n SVEFloatRoundIntegral(0b000, size, zd, pg, zn);\n }\n void frintz(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn) {\n SVEFloatRoundIntegral(0b011, size, zd, pg, zn);\n }\n void frintm(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn) {\n SVEFloatRoundIntegral(0b010, size, zd, pg, zn);\n }\n void frintp(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn) {\n SVEFloatRoundIntegral(0b001, size, zd, pg, zn);\n }\n\n // SVE floating-point convert precision\n void fcvt(SubRegSize to, SubRegSize from, ZRegister zd, PRegisterMerge pg, ZRegister zn) {\n SVEFPConvertPrecision(to, from, zd, pg, zn);\n }\n void fcvtx(ZRegister zd, PRegisterMerge pg, ZRegister zn) {\n LOGMAN_THROW_A_FMT(pg <= PReg::p7, \"Can only use p0-p7 as a governing predicate\");\n uint32_t Instr = 0b0110'0101'0000'1010'1010'0000'0000'0000;\n Instr |= pg.Idx() << 10;\n Instr |= zn.Idx() << 5;\n Instr |= zd.Idx();\n dc32(Instr);\n }\n\n // SVE floating-point unary operations\n void frecpx(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn) {\n SVEFloatUnary(0b00, size, pg, zn, zd);\n }\n void fsqrt(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn) {\n SVEFloatUnary(0b01, size, pg, zn, zd);\n }\n\n // SVE integer convert to floating-point\n void scvtf(ZRegister zd, SubRegSize dstsize, PRegisterMerge pg, ZRegister zn, SubRegSize srcsize) {\n uint32_t opc1, opc2;\n if (srcsize == SubRegSize::i16Bit) {\n // Srcsize = fp16, opc2 encodes dst size\n LOGMAN_THROW_A_FMT(dstsize == SubRegSize::i16Bit, \"Unsupported size in {}\", __func__);\n opc1 = 0b01;\n opc2 = 0b01;\n } else if (srcsize == SubRegSize::i32Bit) {\n // Srcsize = fp32, opc1 encodes dst size\n opc1 = dstsize == SubRegSize::i64Bit ? 0b11 : dstsize == SubRegSize::i32Bit ? 0b10 : dstsize == SubRegSize::i16Bit ? 0b01 : 0b00;\n\n opc2 = dstsize == SubRegSize::i64Bit ? 0b00 : dstsize == SubRegSize::i32Bit ? 0b10 : dstsize == SubRegSize::i16Bit ? 0b10 : 0b00;\n } else if (srcsize == SubRegSize::i64Bit) {\n // SrcSize = fp64, opc2 encodes dst size\n opc1 = dstsize == SubRegSize::i64Bit ? 0b11 : dstsize == SubRegSize::i32Bit ? 0b11 : dstsize == SubRegSize::i16Bit ? 0b01 : 0b00;\n opc2 = dstsize == SubRegSize::i64Bit ? 0b11 : dstsize == SubRegSize::i32Bit ? 0b10 : dstsize == SubRegSize::i16Bit ? 0b11 : 0b00;\n } else {\n FEX_UNREACHABLE;\n }\n SVEIntegerConvertToFloat(dstsize, srcsize, opc1, opc2, 0, pg, zn, zd);\n }\n void ucvtf(ZRegister zd, SubRegSize dstsize, PRegisterMerge pg, ZRegister zn, SubRegSize srcsize) {\n uint32_t opc1, opc2;\n if (srcsize == SubRegSize::i16Bit) {\n // Srcsize = fp16, opc2 encodes dst size\n LOGMAN_THROW_A_FMT(dstsize == SubRegSize::i16Bit, \"Unsupported size in {}\", __func__);\n opc1 = 0b01;\n opc2 = 0b01;\n } else if (srcsize == SubRegSize::i32Bit) {\n // Srcsize = fp32, opc1 encodes dst size\n opc1 = dstsize == SubRegSize::i64Bit ? 0b11 : dstsize == SubRegSize::i32Bit ? 0b10 : dstsize == SubRegSize::i16Bit ? 0b01 : 0b00;\n\n opc2 = dstsize == SubRegSize::i64Bit ? 0b00 : dstsize == SubRegSize::i32Bit ? 0b10 : dstsize == SubRegSize::i16Bit ? 0b10 : 0b00;\n } else if (srcsize == SubRegSize::i64Bit) {\n // SrcSize = fp64, opc2 encodes dst size\n opc1 = dstsize == SubRegSize::i64Bit ? 0b11 : dstsize == SubRegSize::i32Bit ? 0b11 : dstsize == SubRegSize::i16Bit ? 0b01 : 0b00;\n opc2 = dstsize == SubRegSize::i64Bit ? 0b11 : dstsize == SubRegSize::i32Bit ? 0b10 : dstsize == SubRegSize::i16Bit ? 0b11 : 0b00;\n } else {\n FEX_UNREACHABLE;\n }\n SVEIntegerConvertToFloat(dstsize, srcsize, opc1, opc2, 1, pg, zn, zd);\n }\n\n // SVE floating-point convert to integer\n void flogb(SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn) {\n const auto ConvertedSize = size == SubRegSize::i64Bit ? 0b11 :\n size == SubRegSize::i32Bit ? 0b10 :\n size == SubRegSize::i16Bit ? 0b01 :\n 0b00;\n\n SVEFloatConvertToInt(size, size, 1, 0b00, ConvertedSize, 0, pg, zn, zd);\n }\n void fcvtzs(ZRegister zd, SubRegSize dstsize, PRegisterMerge pg, ZRegister zn, SubRegSize srcsize) {\n uint32_t opc1, opc2;\n if (srcsize == SubRegSize::i16Bit) {\n // Srcsize = fp16, opc2 encodes dst size\n opc1 = 0b01;\n opc2 = dstsize == SubRegSize::i64Bit ? 0b11 : dstsize == SubRegSize::i32Bit ? 0b10 : dstsize == SubRegSize::i16Bit ? 0b01 : 0b00;\n } else if (srcsize == SubRegSize::i32Bit) {\n // Srcsize = fp32, opc1 encodes dst size\n LOGMAN_THROW_A_FMT(dstsize != SubRegSize::i16Bit, \"Unsupported size in {}\", __func__);\n opc1 = dstsize == SubRegSize::i64Bit ? 0b11 : 0b10;\n opc2 = 0b10;\n } else if (srcsize == SubRegSize::i64Bit) {\n LOGMAN_THROW_A_FMT(dstsize != SubRegSize::i16Bit, \"Unsupported size in {}\", __func__);\n // SrcSize = fp64, opc2 encodes dst size\n opc1 = 0b11;\n opc2 = dstsize == SubRegSize::i64Bit ? 0b11 : 0b00;\n } else {\n FEX_UNREACHABLE;\n }\n SVEFloatConvertToInt(dstsize, srcsize, 1, opc1, opc2, 0, pg, zn, zd);\n }\n void fcvtzu(ZRegister zd, SubRegSize dstsize, PRegisterMerge pg, ZRegister zn, SubRegSize srcsize) {\n uint32_t opc1, opc2;\n if (srcsize == SubRegSize::i16Bit) {\n // Srcsize = fp16, opc2 encodes dst size\n opc1 = 0b01;\n opc2 = dstsize == SubRegSize::i64Bit ? 0b11 : dstsize == SubRegSize::i32Bit ? 0b10 : dstsize == SubRegSize::i16Bit ? 0b01 : 0b00;\n } else if (srcsize == SubRegSize::i32Bit) {\n // Srcsize = fp32, opc1 encodes dst size\n LOGMAN_THROW_A_FMT(dstsize != SubRegSize::i16Bit, \"Unsupported size in {}\", __func__);\n opc1 = dstsize == SubRegSize::i64Bit ? 0b11 : 0b10;\n opc2 = 0b10;\n } else if (srcsize == SubRegSize::i64Bit) {\n LOGMAN_THROW_A_FMT(dstsize != SubRegSize::i16Bit, \"Unsupported size in {}\", __func__);\n // SrcSize = fp64, opc2 encodes dst size\n opc1 = 0b11;\n opc2 = dstsize == SubRegSize::i64Bit ? 0b11 : 0b00;\n } else {\n FEX_UNREACHABLE;\n }\n SVEFloatConvertToInt(dstsize, srcsize, 1, opc1, opc2, 1, pg, zn, zd);\n }\n\n // SVE Floating Point Unary Operations - Unpredicated\n // SVE floating-point reciprocal estimate (unpredicated)\n void frecpe(SubRegSize size, ZRegister zd, ZRegister zn) {\n SVEFPUnaryOpsUnpredicated(0b110, size, zd, zn);\n }\n void frsqrte(SubRegSize size, ZRegister zd, ZRegister zn) {\n SVEFPUnaryOpsUnpredicated(0b111, size, zd, zn);\n }\n\n // SVE Floating Point Compare - with Zero\n // SVE floating-point compare with zero\n void fcmge(SubRegSize size, PRegister pd, PRegisterZero pg, ZRegister zn) {\n SVEFPCompareWithZero(0b00, 0, size, pd, pg, zn);\n }\n void fcmgt(SubRegSize size, PRegister pd, PRegisterZero pg, ZRegister zn) {\n SVEFPCompareWithZero(0b00, 1, size, pd, pg, zn);\n }\n void fcmlt(SubRegSize size, PRegister pd, PRegisterZero pg, ZRegister zn) {\n SVEFPCompareWithZero(0b01, 0, size, pd, pg, zn);\n }\n void fcmle(SubRegSize size, PRegister pd, PRegisterZero pg, ZRegister zn) {\n SVEFPCompareWithZero(0b01, 1, size, pd, pg, zn);\n }\n void fcmeq(SubRegSize size, PRegister pd, PRegisterZero pg, ZRegister zn) {\n SVEFPCompareWithZero(0b10, 0, size, pd, pg, zn);\n }\n void fcmne(SubRegSize size, PRegister pd, PRegisterZero pg, ZRegister zn) {\n SVEFPCompareWithZero(0b11, 0, size, pd, pg, zn);\n }\n\n // SVE Floating Point Accumulating Reduction\n // SVE floating-point serial reduction (predicated)\n void fadda(SubRegSize size, VRegister vd, PRegister pg, VRegister vn, ZRegister zm) {\n SVEFPSerialReductionPredicated(0b00, size, vd, pg, vn, zm);\n }\n\n // SVE Floating Point Multiply-Add\n // SVE floating-point multiply-accumulate writing addend\n void fmla(SubRegSize size, ZRegister zda, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n SVEFPMultiplyAdd(0b000, size, zda, pg, zn, zm);\n }\n void fmls(SubRegSize size, ZRegister zda, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n SVEFPMultiplyAdd(0b001, size, zda, pg, zn, zm);\n }\n void fnmla(SubRegSize size, ZRegister zda, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n SVEFPMultiplyAdd(0b010, size, zda, pg, zn, zm);\n }\n void fnmls(SubRegSize size, ZRegister zda, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n SVEFPMultiplyAdd(0b011, size, zda, pg, zn, zm);\n }\n\n // SVE floating-point multiply-accumulate writing multiplicand\n void fmad(SubRegSize size, ZRegister zdn, PRegisterMerge pg, ZRegister zm, ZRegister za) {\n SVEFPMultiplyAdd(0b100, size, zdn, pg, zm, za);\n }\n void fmsb(SubRegSize size, ZRegister zdn, PRegisterMerge pg, ZRegister zm, ZRegister za) {\n SVEFPMultiplyAdd(0b101, size, zdn, pg, zm, za);\n }\n void fnmad(SubRegSize size, ZRegister zdn, PRegisterMerge pg, ZRegister zm, ZRegister za) {\n SVEFPMultiplyAdd(0b110, size, zdn, pg, zm, za);\n }\n void fnmsb(SubRegSize size, ZRegister zdn, PRegisterMerge pg, ZRegister zm, ZRegister za) {\n SVEFPMultiplyAdd(0b111, size, zdn, pg, zm, za);\n }\n\n // SVE Memory - 32-bit Gather and Unsized Contiguous\n void ldr(PRegister pt, XRegister rn, int32_t imm = 0) {\n SVEUnsizedLoadStoreContiguous(0b0, imm, ZRegister {pt.Idx()}, rn, false);\n }\n void ldr(ZRegister zt, XRegister rn, int32_t imm = 0) {\n SVEUnsizedLoadStoreContiguous(0b1, imm, zt, rn, false);\n }\n\n // SVE 32-bit gather prefetch (scalar plus 32-bit scaled offsets)\n // XXX:\n // SVE contiguous prefetch (scalar plus immediate)\n // XXX:\n // SVE2 32-bit gather non-temporal load (vector plus scalar)\n // XXX:\n // SVE contiguous prefetch (scalar plus scalar)\n // XXX:\n // SVE 32-bit gather prefetch (vector plus immediate)\n // XXX:\n\n // SVE load and broadcast element\n void ld1rb(SubRegSize esize, ZRegister zt, PRegisterZero pg, Register rn, uint32_t imm = 0) {\n SVELoadAndBroadcastElement(false, esize, SubRegSize::i8Bit, zt, pg, rn, imm);\n }\n void ld1rsb(SubRegSize esize, ZRegister zt, PRegisterZero pg, Register rn, uint32_t imm = 0) {\n SVELoadAndBroadcastElement(true, esize, SubRegSize::i8Bit, zt, pg, rn, imm);\n }\n void ld1rh(SubRegSize esize, ZRegister zt, PRegisterZero pg, Register rn, uint32_t imm = 0) {\n SVELoadAndBroadcastElement(false, esize, SubRegSize::i16Bit, zt, pg, rn, imm);\n }\n void ld1rsh(SubRegSize esize, ZRegister zt, PRegisterZero pg, Register rn, uint32_t imm = 0) {\n SVELoadAndBroadcastElement(true, esize, SubRegSize::i16Bit, zt, pg, rn, imm);\n }\n void ld1rw(SubRegSize esize, ZRegister zt, PRegisterZero pg, Register rn, uint32_t imm = 0) {\n SVELoadAndBroadcastElement(false, esize, SubRegSize::i32Bit, zt, pg, rn, imm);\n }\n void ld1rsw(ZRegister zt, PRegisterZero pg, Register rn, uint32_t imm = 0) {\n SVELoadAndBroadcastElement(true, SubRegSize::i64Bit, SubRegSize::i32Bit, zt, pg, rn, imm);\n }\n void ld1rd(ZRegister zt, PRegisterZero pg, Register rn, uint32_t imm = 0) {\n SVELoadAndBroadcastElement(false, SubRegSize::i64Bit, SubRegSize::i64Bit, zt, pg, rn, imm);\n }\n\n // SVE contiguous non-temporal load (scalar plus immediate)\n void ldnt1b(ZRegister zt, PRegister pg, Register rn, int32_t Imm = 0) {\n SVEContiguousNontemporalLoad(0b00, zt, pg, rn, Imm);\n }\n void ldnt1h(ZRegister zt, PRegister pg, Register rn, int32_t Imm = 0) {\n SVEContiguousNontemporalLoad(0b01, zt, pg, rn, Imm);\n }\n void ldnt1w(ZRegister zt, PRegister pg, Register rn, int32_t Imm = 0) {\n SVEContiguousNontemporalLoad(0b10, zt, pg, rn, Imm);\n }\n void ldnt1d(ZRegister zt, PRegister pg, Register rn, int32_t Imm = 0) {\n SVEContiguousNontemporalLoad(0b11, zt, pg, rn, Imm);\n }\n\n // SVE contiguous non-temporal load (scalar plus scalar)\n // XXX:\n // SVE load multiple structures (scalar plus immediate)\n void ld2b(ZRegister zt1, ZRegister zt2, PRegisterZero pg, Register rn, int32_t Imm = 0) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(zt1, zt2), \"Registers need to be contiguous\");\n SVEContiguousMultipleStructures(2, false, 0b00, Imm, zt1, pg, rn);\n }\n void ld3b(ZRegister zt1, ZRegister zt2, ZRegister zt3, PRegisterZero pg, Register rn, int32_t Imm = 0) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(zt1, zt2, zt3), \"Registers need to be contiguous\");\n SVEContiguousMultipleStructures(3, false, 0b00, Imm, zt1, pg, rn);\n }\n void ld4b(ZRegister zt1, ZRegister zt2, ZRegister zt3, ZRegister zt4, PRegisterZero pg, Register rn, int32_t Imm = 0) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(zt1, zt2, zt3, zt4), \"Registers need to be contiguous\");\n SVEContiguousMultipleStructures(4, false, 0b00, Imm, zt1, pg, rn);\n }\n void ld2h(ZRegister zt1, ZRegister zt2, PRegisterZero pg, Register rn, int32_t Imm = 0) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(zt1, zt2), \"Registers need to be contiguous\");\n SVEContiguousMultipleStructures(2, false, 0b01, Imm, zt1, pg, rn);\n }\n void ld3h(ZRegister zt1, ZRegister zt2, ZRegister zt3, PRegisterZero pg, Register rn, int32_t Imm = 0) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(zt1, zt2, zt3), \"Registers need to be contiguous\");\n SVEContiguousMultipleStructures(3, false, 0b01, Imm, zt1, pg, rn);\n }\n void ld4h(ZRegister zt1, ZRegister zt2, ZRegister zt3, ZRegister zt4, PRegisterZero pg, Register rn, int32_t Imm = 0) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(zt1, zt2, zt3, zt4), \"Registers need to be contiguous\");\n SVEContiguousMultipleStructures(4, false, 0b01, Imm, zt1, pg, rn);\n }\n void ld2w(ZRegister zt1, ZRegister zt2, PRegisterZero pg, Register rn, int32_t Imm = 0) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(zt1, zt2), \"Registers need to be contiguous\");\n SVEContiguousMultipleStructures(2, false, 0b10, Imm, zt1, pg, rn);\n }\n void ld3w(ZRegister zt1, ZRegister zt2, ZRegister zt3, PRegisterZero pg, Register rn, int32_t Imm = 0) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(zt1, zt2, zt3), \"Registers need to be contiguous\");\n SVEContiguousMultipleStructures(3, false, 0b10, Imm, zt1, pg, rn);\n }\n void ld4w(ZRegister zt1, ZRegister zt2, ZRegister zt3, ZRegister zt4, PRegisterZero pg, Register rn, int32_t Imm = 0) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(zt1, zt2, zt3, zt4), \"Registers need to be contiguous\");\n SVEContiguousMultipleStructures(4, false, 0b10, Imm, zt1, pg, rn);\n }\n void ld2d(ZRegister zt1, ZRegister zt2, PRegisterZero pg, Register rn, int32_t Imm = 0) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(zt1, zt2), \"Registers need to be contiguous\");\n SVEContiguousMultipleStructures(2, false, 0b11, Imm, zt1, pg, rn);\n }\n void ld3d(ZRegister zt1, ZRegister zt2, ZRegister zt3, PRegisterZero pg, Register rn, int32_t Imm = 0) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(zt1, zt2, zt3), \"Registers need to be contiguous\");\n SVEContiguousMultipleStructures(3, false, 0b11, Imm, zt1, pg, rn);\n }\n void ld4d(ZRegister zt1, ZRegister zt2, ZRegister zt3, ZRegister zt4, PRegisterZero pg, Register rn, int32_t Imm = 0) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(zt1, zt2, zt3, zt4), \"Registers need to be contiguous\");\n SVEContiguousMultipleStructures(4, false, 0b11, Imm, zt1, pg, rn);\n }\n\n // SVE helper implementations\n template\n void ld1b(ZRegister zt, PRegisterZero pg, SVEMemOperand Src) {\n if (Src.IsScalarPlusScalar()) {\n ld1b(zt, pg, Src.rn, Src.MetaType.ScalarScalarType.rm);\n } else if (Src.IsScalarPlusImm()) {\n ld1b(zt, pg, Src.rn, Src.MetaType.ScalarImmType.Imm);\n } else if (Src.IsScalarPlusVector()) {\n SVEGatherLoadScalarPlusVector(size, SubRegSize::i8Bit, zt, pg, Src, true, false);\n } else if (Src.IsVectorPlusImm()) {\n SVEGatherLoadVectorPlusImm(size, SubRegSize::i8Bit, zt, pg, Src, true, false);\n } else {\n FEX_UNREACHABLE;\n }\n }\n\n template\n void ldff1b(ZRegister zt, PRegisterZero pg, SVEMemOperand Src) {\n if (Src.IsScalarPlusScalar()) {\n ldff1b(zt, pg, Src.rn, Src.MetaType.ScalarScalarType.rm);\n } else if (Src.IsScalarPlusImm()) {\n LOGMAN_THROW_A_FMT(false, \"ldff1b doesn't have a scalar plus immediate variant\");\n } else if (Src.IsScalarPlusVector()) {\n SVEGatherLoadScalarPlusVector(size, SubRegSize::i8Bit, zt, pg, Src, true, true);\n } else if (Src.IsVectorPlusImm()) {\n SVEGatherLoadVectorPlusImm(size, SubRegSize::i8Bit, zt, pg, Src, true, true);\n } else {\n FEX_UNREACHABLE;\n }\n }\n\n void ld1sw(ZRegister zt, PRegisterZero pg, SVEMemOperand Src) {\n if (Src.IsScalarPlusScalar()) {\n ld1sw(zt, pg, Src.rn, Src.MetaType.ScalarScalarType.rm);\n } else if (Src.IsScalarPlusImm()) {\n ld1sw(zt, pg, Src.rn, Src.MetaType.ScalarImmType.Imm);\n } else if (Src.IsScalarPlusVector()) {\n SVEGatherLoadScalarPlusVector(SubRegSize::i64Bit, SubRegSize::i32Bit, zt, pg, Src, false, false);\n } else if (Src.IsVectorPlusImm()) {\n SVEGatherLoadVectorPlusImm(SubRegSize::i64Bit, SubRegSize::i32Bit, zt, pg, Src, false, false);\n } else {\n FEX_UNREACHABLE;\n }\n }\n\n template\n void ld1h(ZRegister zt, PRegisterZero pg, SVEMemOperand Src) {\n if (Src.IsScalarPlusScalar()) {\n ld1h(zt, pg, Src.rn, Src.MetaType.ScalarScalarType.rm);\n } else if (Src.IsScalarPlusImm()) {\n ld1h(zt, pg, Src.rn, Src.MetaType.ScalarImmType.Imm);\n } else if (Src.IsScalarPlusVector()) {\n SVEGatherLoadScalarPlusVector(size, SubRegSize::i16Bit, zt, pg, Src, true, false);\n } else if (Src.IsVectorPlusImm()) {\n SVEGatherLoadVectorPlusImm(size, SubRegSize::i16Bit, zt, pg, Src, true, false);\n } else {\n FEX_UNREACHABLE;\n }\n }\n\n template\n void ld1sh(ZRegister zt, PRegisterZero pg, SVEMemOperand Src) {\n if (Src.IsScalarPlusScalar()) {\n ld1sh(zt, pg, Src.rn, Src.MetaType.ScalarScalarType.rm);\n } else if (Src.IsScalarPlusImm()) {\n ld1sh(zt, pg, Src.rn, Src.MetaType.ScalarImmType.Imm);\n } else if (Src.IsScalarPlusVector()) {\n SVEGatherLoadScalarPlusVector(size, SubRegSize::i16Bit, zt, pg, Src, false, false);\n } else if (Src.IsVectorPlusImm()) {\n SVEGatherLoadVectorPlusImm(size, SubRegSize::i16Bit, zt, pg, Src, false, false);\n } else {\n FEX_UNREACHABLE;\n }\n }\n\n template\n void ldff1h(ZRegister zt, PRegisterZero pg, SVEMemOperand Src) {\n if (Src.IsScalarPlusScalar()) {\n ldff1h(zt, pg, Src.rn, Src.MetaType.ScalarScalarType.rm);\n } else if (Src.IsScalarPlusImm()) {\n LOGMAN_THROW_A_FMT(false, \"ldff1h doesn't have a scalar plus immediate variant\");\n } else if (Src.IsScalarPlusVector()) {\n SVEGatherLoadScalarPlusVector(size, SubRegSize::i16Bit, zt, pg, Src, true, true);\n } else if (Src.IsVectorPlusImm()) {\n SVEGatherLoadVectorPlusImm(size, SubRegSize::i16Bit, zt, pg, Src, true, true);\n } else {\n FEX_UNREACHABLE;\n }\n }\n\n template\n void ldff1sh(ZRegister zt, PRegisterZero pg, SVEMemOperand Src) {\n if (Src.IsScalarPlusScalar()) {\n ldff1sh(zt, pg, Src.rn, Src.MetaType.ScalarScalarType.rm);\n } else if (Src.IsScalarPlusImm()) {\n LOGMAN_THROW_A_FMT(false, \"ldff1sh doesn't have a scalar plus immediate variant\");\n } else if (Src.IsScalarPlusVector()) {\n SVEGatherLoadScalarPlusVector(size, SubRegSize::i16Bit, zt, pg, Src, false, true);\n } else if (Src.IsVectorPlusImm()) {\n SVEGatherLoadVectorPlusImm(size, SubRegSize::i16Bit, zt, pg, Src, false, true);\n } else {\n FEX_UNREACHABLE;\n }\n }\n\n template\n void ld1w(ZRegister zt, PRegisterZero pg, SVEMemOperand Src) {\n if (Src.IsScalarPlusScalar()) {\n ld1w(zt, pg, Src.rn, Src.MetaType.ScalarScalarType.rm);\n } else if (Src.IsScalarPlusImm()) {\n ld1w(zt, pg, Src.rn, Src.MetaType.ScalarImmType.Imm);\n } else if (Src.IsScalarPlusVector()) {\n SVEGatherLoadScalarPlusVector(size, SubRegSize::i32Bit, zt, pg, Src, true, false);\n } else if (Src.IsVectorPlusImm()) {\n SVEGatherLoadVectorPlusImm(size, SubRegSize::i32Bit, zt, pg, Src, true, false);\n } else {\n FEX_UNREACHABLE;\n }\n }\n\n template\n void ldff1w(ZRegister zt, PRegisterZero pg, SVEMemOperand Src) {\n if (Src.IsScalarPlusScalar()) {\n ldff1w(zt, pg, Src.rn, Src.MetaType.ScalarScalarType.rm);\n } else if (Src.IsScalarPlusImm()) {\n LOGMAN_THROW_A_FMT(false, \"ldff1w doesn't have a scalar plus immediate variant\");\n } else if (Src.IsScalarPlusVector()) {\n SVEGatherLoadScalarPlusVector(size, SubRegSize::i32Bit, zt, pg, Src, true, true);\n } else if (Src.IsVectorPlusImm()) {\n SVEGatherLoadVectorPlusImm(size, SubRegSize::i32Bit, zt, pg, Src, true, true);\n } else {\n FEX_UNREACHABLE;\n }\n }\n\n void ldff1sw(ZRegister zt, PRegisterZero pg, SVEMemOperand Src) {\n if (Src.IsScalarPlusScalar()) {\n ldff1sw(zt, pg, Src.rn, Src.MetaType.ScalarScalarType.rm);\n } else if (Src.IsScalarPlusImm()) {\n LOGMAN_THROW_A_FMT(false, \"ldff1sw doesn't have a scalar plus immediate variant\");\n } else if (Src.IsScalarPlusVector()) {\n SVEGatherLoadScalarPlusVector(SubRegSize::i64Bit, SubRegSize::i32Bit, zt, pg, Src, false, true);\n } else if (Src.IsVectorPlusImm()) {\n SVEGatherLoadVectorPlusImm(SubRegSize::i64Bit, SubRegSize::i32Bit, zt, pg, Src, false, true);\n } else {\n FEX_UNREACHABLE;\n }\n }\n\n template\n void ld1sb(ZRegister zt, PRegisterZero pg, SVEMemOperand Src) {\n if (Src.IsScalarPlusScalar()) {\n ld1sb(zt, pg, Src.rn, Src.MetaType.ScalarScalarType.rm);\n } else if (Src.IsScalarPlusImm()) {\n ld1sb(zt, pg, Src.rn, Src.MetaType.ScalarImmType.Imm);\n } else if (Src.IsScalarPlusVector()) {\n SVEGatherLoadScalarPlusVector(size, SubRegSize::i8Bit, zt, pg, Src, false, false);\n } else if (Src.IsVectorPlusImm()) {\n SVEGatherLoadVectorPlusImm(size, SubRegSize::i8Bit, zt, pg, Src, false, false);\n } else {\n FEX_UNREACHABLE;\n }\n }\n\n template\n void ldff1sb(ZRegister zt, PRegisterZero pg, SVEMemOperand Src) {\n if (Src.IsScalarPlusScalar()) {\n ldff1sb(zt, pg, Src.rn, Src.MetaType.ScalarScalarType.rm);\n } else if (Src.IsScalarPlusImm()) {\n LOGMAN_THROW_A_FMT(false, \"ldff1sb doesn't have a scalar plus immediate variant\");\n } else if (Src.IsScalarPlusVector()) {\n SVEGatherLoadScalarPlusVector(size, SubRegSize::i8Bit, zt, pg, Src, false, true);\n } else if (Src.IsVectorPlusImm()) {\n SVEGatherLoadVectorPlusImm(size, SubRegSize::i8Bit, zt, pg, Src, false, true);\n } else {\n FEX_UNREACHABLE;\n }\n }\n\n void ld1d(ZRegister zt, PRegisterZero pg, SVEMemOperand Src) {\n if (Src.IsScalarPlusScalar()) {\n ld1d(zt, pg, Src.rn, Src.MetaType.ScalarScalarType.rm);\n } else if (Src.IsScalarPlusImm()) {\n ld1d(zt, pg, Src.rn, Src.MetaType.ScalarImmType.Imm);\n } else if (Src.IsScalarPlusVector()) {\n SVEGatherLoadScalarPlusVector(SubRegSize::i64Bit, SubRegSize::i64Bit, zt, pg, Src, true, false);\n } else if (Src.IsVectorPlusImm()) {\n SVEGatherLoadVectorPlusImm(SubRegSize::i64Bit, SubRegSize::i64Bit, zt, pg, Src, true, false);\n } else {\n FEX_UNREACHABLE;\n }\n }\n\n void ldff1d(ZRegister zt, PRegisterZero pg, SVEMemOperand Src) {\n if (Src.IsScalarPlusScalar()) {\n ldff1d(zt, pg, Src.rn, Src.MetaType.ScalarScalarType.rm);\n } else if (Src.IsScalarPlusImm()) {\n LOGMAN_THROW_A_FMT(false, \"ldff1d doesn't have a scalar plus immediate variant\");\n } else if (Src.IsScalarPlusVector()) {\n SVEGatherLoadScalarPlusVector(SubRegSize::i64Bit, SubRegSize::i64Bit, zt, pg, Src, true, true);\n } else if (Src.IsVectorPlusImm()) {\n SVEGatherLoadVectorPlusImm(SubRegSize::i64Bit, SubRegSize::i64Bit, zt, pg, Src, true, true);\n } else {\n FEX_UNREACHABLE;\n }\n }\n\n template\n void st1b(ZRegister zt, PRegister pg, SVEMemOperand Src) {\n if (Src.IsScalarPlusScalar()) {\n st1b(zt, pg, Src.rn, Src.MetaType.ScalarScalarType.rm);\n } else if (Src.IsScalarPlusImm()) {\n st1b(zt, pg, Src.rn, Src.MetaType.ScalarImmType.Imm);\n } else if (Src.IsScalarPlusVector()) {\n SVEScatterStoreScalarPlusVector(size, SubRegSize::i8Bit, zt, pg, Src);\n } else if (Src.IsVectorPlusImm()) {\n SVEScatterStoreVectorPlusImm(size, SubRegSize::i8Bit, zt, pg, Src);\n } else {\n FEX_UNREACHABLE;\n }\n }\n\n template\n void st1h(ZRegister zt, PRegister pg, SVEMemOperand Src) {\n if (Src.IsScalarPlusScalar()) {\n st1h(zt, pg, Src.rn, Src.MetaType.ScalarScalarType.rm);\n } else if (Src.IsScalarPlusImm()) {\n st1h(zt, pg, Src.rn, Src.MetaType.ScalarImmType.Imm);\n } else if (Src.IsScalarPlusVector()) {\n SVEScatterStoreScalarPlusVector(size, SubRegSize::i16Bit, zt, pg, Src);\n } else if (Src.IsVectorPlusImm()) {\n SVEScatterStoreVectorPlusImm(size, SubRegSize::i16Bit, zt, pg, Src);\n } else {\n FEX_UNREACHABLE;\n }\n }\n\n template\n void st1w(ZRegister zt, PRegister pg, SVEMemOperand Src) {\n if (Src.IsScalarPlusScalar()) {\n st1w(zt, pg, Src.rn, Src.MetaType.ScalarScalarType.rm);\n } else if (Src.IsScalarPlusImm()) {\n st1w(zt, pg, Src.rn, Src.MetaType.ScalarImmType.Imm);\n } else if (Src.IsScalarPlusVector()) {\n SVEScatterStoreScalarPlusVector(size, SubRegSize::i32Bit, zt, pg, Src);\n } else if (Src.IsVectorPlusImm()) {\n SVEScatterStoreVectorPlusImm(size, SubRegSize::i32Bit, zt, pg, Src);\n } else {\n FEX_UNREACHABLE;\n }\n }\n\n void st1d(ZRegister zt, PRegister pg, SVEMemOperand Src) {\n if (Src.IsScalarPlusScalar()) {\n st1d(zt, pg, Src.rn, Src.MetaType.ScalarScalarType.rm);\n } else if (Src.IsScalarPlusImm()) {\n st1d(zt, pg, Src.rn, Src.MetaType.ScalarImmType.Imm);\n } else if (Src.IsScalarPlusVector()) {\n SVEScatterStoreScalarPlusVector(SubRegSize::i64Bit, SubRegSize::i64Bit, zt, pg, Src);\n } else if (Src.IsVectorPlusImm()) {\n SVEScatterStoreVectorPlusImm(SubRegSize::i64Bit, SubRegSize::i64Bit, zt, pg, Src);\n } else {\n FEX_UNREACHABLE;\n }\n }\n\n // SVE load multiple structures (scalar plus scalar)\n void ld2b(ZRegister zt1, ZRegister zt2, PRegisterZero pg, Register rn, Register rm) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(zt1, zt2), \"Registers need to be contiguous\");\n SVEContiguousLoadStoreMultipleScalar(false, SubRegSize::i8Bit, 0b01, zt1, pg, rn, rm);\n }\n void ld3b(ZRegister zt1, ZRegister zt2, ZRegister zt3, PRegisterZero pg, Register rn, Register rm) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(zt1, zt2, zt3), \"Registers need to be contiguous\");\n SVEContiguousLoadStoreMultipleScalar(false, SubRegSize::i8Bit, 0b10, zt1, pg, rn, rm);\n }\n void ld4b(ZRegister zt1, ZRegister zt2, ZRegister zt3, ZRegister zt4, PRegisterZero pg, Register rn, Register rm) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(zt1, zt2, zt3, zt4), \"Registers need to be contiguous\");\n SVEContiguousLoadStoreMultipleScalar(false, SubRegSize::i8Bit, 0b11, zt1, pg, rn, rm);\n }\n void ld2h(ZRegister zt1, ZRegister zt2, PRegisterZero pg, Register rn, Register rm) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(zt1, zt2), \"Registers need to be contiguous\");\n SVEContiguousLoadStoreMultipleScalar(false, SubRegSize::i16Bit, 0b01, zt1, pg, rn, rm);\n }\n void ld3h(ZRegister zt1, ZRegister zt2, ZRegister zt3, PRegisterZero pg, Register rn, Register rm) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(zt1, zt2, zt3), \"Registers need to be contiguous\");\n SVEContiguousLoadStoreMultipleScalar(false, SubRegSize::i16Bit, 0b10, zt1, pg, rn, rm);\n }\n void ld4h(ZRegister zt1, ZRegister zt2, ZRegister zt3, ZRegister zt4, PRegisterZero pg, Register rn, Register rm) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(zt1, zt2, zt3, zt4), \"Registers need to be contiguous\");\n SVEContiguousLoadStoreMultipleScalar(false, SubRegSize::i16Bit, 0b11, zt1, pg, rn, rm);\n }\n void ld2w(ZRegister zt1, ZRegister zt2, PRegisterZero pg, Register rn, Register rm) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(zt1, zt2), \"Registers need to be contiguous\");\n SVEContiguousLoadStoreMultipleScalar(false, SubRegSize::i32Bit, 0b01, zt1, pg, rn, rm);\n }\n void ld3w(ZRegister zt1, ZRegister zt2, ZRegister zt3, PRegisterZero pg, Register rn, Register rm) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(zt1, zt2, zt3), \"Registers need to be contiguous\");\n SVEContiguousLoadStoreMultipleScalar(false, SubRegSize::i32Bit, 0b10, zt1, pg, rn, rm);\n }\n void ld4w(ZRegister zt1, ZRegister zt2, ZRegister zt3, ZRegister zt4, PRegisterZero pg, Register rn, Register rm) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(zt1, zt2, zt3, zt4), \"Registers need to be contiguous\");\n SVEContiguousLoadStoreMultipleScalar(false, SubRegSize::i32Bit, 0b11, zt1, pg, rn, rm);\n }\n void ld2d(ZRegister zt1, ZRegister zt2, PRegisterZero pg, Register rn, Register rm) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(zt1, zt2), \"Registers need to be contiguous\");\n SVEContiguousLoadStoreMultipleScalar(false, SubRegSize::i64Bit, 0b01, zt1, pg, rn, rm);\n }\n void ld3d(ZRegister zt1, ZRegister zt2, ZRegister zt3, PRegisterZero pg, Register rn, Register rm) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(zt1, zt2, zt3), \"Registers need to be contiguous\");\n SVEContiguousLoadStoreMultipleScalar(false, SubRegSize::i64Bit, 0b10, zt1, pg, rn, rm);\n }\n void ld4d(ZRegister zt1, ZRegister zt2, ZRegister zt3, ZRegister zt4, PRegisterZero pg, Register rn, Register rm) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(zt1, zt2, zt3, zt4), \"Registers need to be contiguous\");\n SVEContiguousLoadStoreMultipleScalar(false, SubRegSize::i64Bit, 0b11, zt1, pg, rn, rm);\n }\n\n // SVE load and broadcast quadword (scalar plus immediate)\n void ld1rqb(ZRegister zt, PRegisterZero pg, Register rn, int imm = 0) {\n SVELoadBroadcastQuadScalarPlusImm(0b00, 0b00, zt, pg, rn, imm);\n }\n void ld1rob(ZRegister zt, PRegisterZero pg, Register rn, int imm = 0) {\n SVELoadBroadcastQuadScalarPlusImm(0b00, 0b01, zt, pg, rn, imm);\n }\n void ld1rqh(ZRegister zt, PRegisterZero pg, Register rn, int imm = 0) {\n SVELoadBroadcastQuadScalarPlusImm(0b01, 0b00, zt, pg, rn, imm);\n }\n void ld1roh(ZRegister zt, PRegisterZero pg, Register rn, int imm = 0) {\n SVELoadBroadcastQuadScalarPlusImm(0b01, 0b01, zt, pg, rn, imm);\n }\n void ld1rqw(ZRegister zt, PRegisterZero pg, Register rn, int imm = 0) {\n SVELoadBroadcastQuadScalarPlusImm(0b10, 0b00, zt, pg, rn, imm);\n }\n void ld1row(ZRegister zt, PRegisterZero pg, Register rn, int imm = 0) {\n SVELoadBroadcastQuadScalarPlusImm(0b10, 0b01, zt, pg, rn, imm);\n }\n void ld1rqd(ZRegister zt, PRegisterZero pg, Register rn, int imm = 0) {\n SVELoadBroadcastQuadScalarPlusImm(0b11, 0b00, zt, pg, rn, imm);\n }\n void ld1rod(ZRegister zt, PRegisterZero pg, Register rn, int imm = 0) {\n SVELoadBroadcastQuadScalarPlusImm(0b11, 0b01, zt, pg, rn, imm);\n }\n\n // SVE contiguous load (scalar plus immediate)\n template\n void ld1b(ZRegister zt, PRegisterZero pg, Register rn, int32_t Imm = 0) {\n SVEContiguousLoadImm(false, 0b0000 | FEXCore::ToUnderlying(size), Imm, pg, rn, zt);\n }\n\n void ld1sw(ZRegister zt, PRegisterZero pg, Register rn, int32_t Imm = 0) {\n SVEContiguousLoadImm(false, 0b0100, Imm, pg, rn, zt);\n }\n\n template\n void ld1h(ZRegister zt, PRegisterZero pg, Register rn, int32_t Imm = 0) {\n static_assert(size != SubRegSize::i8Bit, \"Invalid size\");\n SVEContiguousLoadImm(false, 0b0100 | FEXCore::ToUnderlying(size), Imm, pg, rn, zt);\n }\n\n template\n void ld1sh(ZRegister zt, PRegisterZero pg, Register rn, int32_t Imm = 0) {\n static_assert(size == SubRegSize::i32Bit || size == SubRegSize::i64Bit, \"Invalid size\");\n\n constexpr uint32_t ConvertedSize = size == SubRegSize::i32Bit ? 1 : size == SubRegSize::i64Bit ? 0 : -1;\n\n SVEContiguousLoadImm(false, 0b1000 | ConvertedSize, Imm, pg, rn, zt);\n }\n\n template\n void ld1w(ZRegister zt, PRegisterZero pg, Register rn, int32_t Imm = 0) {\n static_assert(size == SubRegSize::i32Bit || size == SubRegSize::i64Bit, \"Invalid size\");\n\n constexpr uint32_t ConvertedSize = size == SubRegSize::i32Bit ? 0 : size == SubRegSize::i64Bit ? 1 : -1;\n\n SVEContiguousLoadImm(false, 0b1010 | ConvertedSize, Imm, pg, rn, zt);\n }\n\n template\n void ld1sb(ZRegister zt, PRegisterZero pg, Register rn, int32_t Imm = 0) {\n static_assert(size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit, \"Invalid size\");\n\n constexpr uint32_t ConvertedSize = size == SubRegSize::i16Bit ? 0b10 :\n size == SubRegSize::i32Bit ? 0b01 :\n size == SubRegSize::i64Bit ? 0b00 :\n -1;\n\n SVEContiguousLoadImm(false, 0b1100 | ConvertedSize, Imm, pg, rn, zt);\n }\n void ld1d(ZRegister zt, PRegisterZero pg, Register rn, int32_t Imm = 0) {\n SVEContiguousLoadImm(false, 0b1111, Imm, pg, rn, zt);\n }\n\n // SVE contiguous non-fault load (scalar plus immediate)\n // XXX:\n\n // SVE load and broadcast quadword (scalar plus scalar)\n void ld1rqb(ZRegister zt, PRegisterZero pg, Register rn, Register rm) {\n SVELoadBroadcastQuadScalarPlusScalar(0b00, 0b00, zt, pg, rn, rm);\n }\n void ld1rob(ZRegister zt, PRegisterZero pg, Register rn, Register rm) {\n SVELoadBroadcastQuadScalarPlusScalar(0b00, 0b01, zt, pg, rn, rm);\n }\n void ld1rqh(ZRegister zt, PRegisterZero pg, Register rn, Register rm) {\n SVELoadBroadcastQuadScalarPlusScalar(0b01, 0b00, zt, pg, rn, rm);\n }\n void ld1roh(ZRegister zt, PRegisterZero pg, Register rn, Register rm) {\n SVELoadBroadcastQuadScalarPlusScalar(0b01, 0b01, zt, pg, rn, rm);\n }\n void ld1rqw(ZRegister zt, PRegisterZero pg, Register rn, Register rm) {\n SVELoadBroadcastQuadScalarPlusScalar(0b10, 0b00, zt, pg, rn, rm);\n }\n void ld1row(ZRegister zt, PRegisterZero pg, Register rn, Register rm) {\n SVELoadBroadcastQuadScalarPlusScalar(0b10, 0b01, zt, pg, rn, rm);\n }\n void ld1rqd(ZRegister zt, PRegisterZero pg, Register rn, Register rm) {\n SVELoadBroadcastQuadScalarPlusScalar(0b11, 0b00, zt, pg, rn, rm);\n }\n void ld1rod(ZRegister zt, PRegisterZero pg, Register rn, Register rm) {\n SVELoadBroadcastQuadScalarPlusScalar(0b11, 0b01, zt, pg, rn, rm);\n }\n\n // SVE contiguous load (scalar plus scalar)\n template\n void ld1b(ZRegister zt, PRegisterZero pg, Register rn, Register rm) {\n SVEContiguousLoadStore(0, 0, 0b0000 | FEXCore::ToUnderlying(size), rm, pg, rn, zt);\n }\n\n void ld1sw(ZRegister zt, PRegisterZero pg, Register rn, Register rm) {\n SVEContiguousLoadStore(0, 0, 0b0100, rm, pg, rn, zt);\n }\n\n template\n void ld1h(ZRegister zt, PRegisterZero pg, Register rn, Register rm) {\n static_assert(size != SubRegSize::i8Bit, \"Invalid size\");\n SVEContiguousLoadStore(0, 0, 0b0100 | FEXCore::ToUnderlying(size), rm, pg, rn, zt);\n }\n\n template\n void ld1sh(ZRegister zt, PRegisterZero pg, Register rn, Register rm) {\n static_assert(size == SubRegSize::i32Bit || size == SubRegSize::i64Bit, \"Invalid size\");\n constexpr uint32_t ConvertedSize = size == SubRegSize::i32Bit ? 1 : size == SubRegSize::i64Bit ? 0 : -1;\n SVEContiguousLoadStore(0, 0, 0b1000 | ConvertedSize, rm, pg, rn, zt);\n }\n\n template\n void ld1w(ZRegister zt, PRegisterZero pg, Register rn, Register rm) {\n static_assert(size == SubRegSize::i32Bit || size == SubRegSize::i64Bit, \"Invalid size\");\n constexpr uint32_t ConvertedSize = size == SubRegSize::i32Bit ? 0 : size == SubRegSize::i64Bit ? 1 : -1;\n SVEContiguousLoadStore(0, 0, 0b1010 | ConvertedSize, rm, pg, rn, zt);\n }\n template\n void ld1sb(ZRegister zt, PRegisterZero pg, Register rn, Register rm) {\n static_assert(size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit, \"Invalid size\");\n constexpr uint32_t ConvertedSize = size == SubRegSize::i16Bit ? 0b10 :\n size == SubRegSize::i32Bit ? 0b01 :\n size == SubRegSize::i64Bit ? 0b00 :\n -1;\n SVEContiguousLoadStore(0, 0, 0b1100 | ConvertedSize, rm, pg, rn, zt);\n }\n\n void ld1d(ZRegister zt, PRegisterZero pg, Register rn, Register rm) {\n SVEContiguousLoadStore(0, 0, 0b1111, rm, pg, rn, zt);\n }\n\n // SVE contiguous first-fault load (scalar plus scalar)\n template\n void ldff1b(ZRegister zt, PRegisterZero pg, Register rn, Register rm) {\n SVEContiguousLoadStore(0, 1, 0b0000 | FEXCore::ToUnderlying(size), rm, pg, rn, zt);\n }\n template\n void ldff1sb(ZRegister zt, PRegisterZero pg, Register rn, Register rm) {\n static_assert(size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit, \"Invalid size\");\n constexpr uint32_t ConvertedSize = size == SubRegSize::i16Bit ? 0b10 :\n size == SubRegSize::i32Bit ? 0b01 :\n size == SubRegSize::i64Bit ? 0b00 :\n -1;\n SVEContiguousLoadStore(0, 1, 0b1100 | ConvertedSize, rm, pg, rn, zt);\n }\n template\n void ldff1h(ZRegister zt, PRegisterZero pg, Register rn, Register rm) {\n static_assert(size != SubRegSize::i8Bit, \"Invalid size\");\n SVEContiguousLoadStore(0, 1, 0b0100 | FEXCore::ToUnderlying(size), rm, pg, rn, zt);\n }\n template\n void ldff1sh(ZRegister zt, PRegisterZero pg, Register rn, Register rm) {\n static_assert(size == SubRegSize::i32Bit || size == SubRegSize::i64Bit, \"Invalid size\");\n constexpr uint32_t ConvertedSize = size == SubRegSize::i32Bit ? 1 : size == SubRegSize::i64Bit ? 0 : -1;\n SVEContiguousLoadStore(0, 1, 0b1000 | ConvertedSize, rm, pg, rn, zt);\n }\n template\n void ldff1w(ZRegister zt, PRegisterZero pg, Register rn, Register rm) {\n static_assert(size == SubRegSize::i32Bit || size == SubRegSize::i64Bit, \"Invalid size\");\n constexpr uint32_t ConvertedSize = size == SubRegSize::i32Bit ? 0 : size == SubRegSize::i64Bit ? 1 : -1;\n SVEContiguousLoadStore(0, 1, 0b1010 | ConvertedSize, rm, pg, rn, zt);\n }\n void ldff1sw(ZRegister zt, PRegisterZero pg, Register rn, Register rm) {\n SVEContiguousLoadStore(0, 1, 0b0100, rm, pg, rn, zt);\n }\n void ldff1d(ZRegister zt, PRegisterZero pg, Register rn, Register rm) {\n SVEContiguousLoadStore(0, 1, 0b1111, rm, pg, rn, zt);\n }\n\n // SVE Memory - 64-bit Gather\n // SVE 64-bit gather prefetch (scalar plus 64-bit scaled offsets)\n // XXX:\n // SVE 64-bit gather prefetch (scalar plus unpacked 32-bit scaled offsets)\n // XXX:\n // SVE 64-bit gather prefetch (vector plus immediate)\n // XXX:\n // SVE2 64-bit gather non-temporal load (vector plus scalar)\n // XXX:\n\n // SVE Memory - Contiguous Store and Unsized Contiguous\n void str(PRegister pt, XRegister rn, int32_t imm = 0) {\n SVEUnsizedLoadStoreContiguous(0b0, imm, ZRegister {pt.Idx()}, rn, true);\n }\n void str(ZRegister zt, XRegister rn, int32_t imm = 0) {\n SVEUnsizedLoadStoreContiguous(0b1, imm, zt, rn, true);\n }\n\n // SVE contiguous store (scalar plus scalar)\n template\n void st1b(ZRegister zt, PRegister pg, Register rn, Register rm) {\n SVEContiguousLoadStore(1, 0, 0b0000 | FEXCore::ToUnderlying(size), rm, pg, rn, zt);\n }\n\n template\n void st1h(ZRegister zt, PRegister pg, Register rn, Register rm) {\n static_assert(size != SubRegSize::i8Bit, \"Invalid size\");\n SVEContiguousLoadStore(1, 0, 0b0100 | FEXCore::ToUnderlying(size), rm, pg, rn, zt);\n }\n\n template\n void st1w(ZRegister zt, PRegister pg, Register rn, Register rm) {\n static_assert(size == SubRegSize::i32Bit || size == SubRegSize::i64Bit, \"Invalid size\");\n constexpr uint32_t ConvertedSize = size == SubRegSize::i32Bit ? 0 : size == SubRegSize::i64Bit ? 1 : -1;\n\n SVEContiguousLoadStore(1, 0, 0b1010 | ConvertedSize, rm, pg, rn, zt);\n }\n void st1d(ZRegister zt, PRegister pg, Register rn, Register rm) {\n SVEContiguousLoadStore(1, 0, 0b1111, rm, pg, rn, zt);\n }\n\n // SVE Memory - Non-temporal and Multi-register Store\n // SVE2 64-bit scatter non-temporal store (vector plus scalar)\n // XXX:\n // SVE contiguous non-temporal store (scalar plus scalar)\n // XXX:\n // SVE2 32-bit scatter non-temporal store (vector plus scalar)\n // XXX:\n\n // SVE store multiple structures (scalar plus scalar)\n void st2b(ZRegister zt1, ZRegister zt2, PRegister pg, Register rn, Register rm) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(zt1, zt2), \"Registers need to be contiguous\");\n SVEContiguousLoadStoreMultipleScalar(true, SubRegSize::i8Bit, 0b01, zt1, pg, rn, rm);\n }\n void st3b(ZRegister zt1, ZRegister zt2, ZRegister zt3, PRegister pg, Register rn, Register rm) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(zt1, zt2, zt3), \"Registers need to be contiguous\");\n SVEContiguousLoadStoreMultipleScalar(true, SubRegSize::i8Bit, 0b10, zt1, pg, rn, rm);\n }\n void st4b(ZRegister zt1, ZRegister zt2, ZRegister zt3, ZRegister zt4, PRegister pg, Register rn, Register rm) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(zt1, zt2, zt3, zt4), \"Registers need to be contiguous\");\n SVEContiguousLoadStoreMultipleScalar(true, SubRegSize::i8Bit, 0b11, zt1, pg, rn, rm);\n }\n void st2h(ZRegister zt1, ZRegister zt2, PRegister pg, Register rn, Register rm) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(zt1, zt2), \"Registers need to be contiguous\");\n SVEContiguousLoadStoreMultipleScalar(true, SubRegSize::i16Bit, 0b01, zt1, pg, rn, rm);\n }\n void st3h(ZRegister zt1, ZRegister zt2, ZRegister zt3, PRegister pg, Register rn, Register rm) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(zt1, zt2, zt3), \"Registers need to be contiguous\");\n SVEContiguousLoadStoreMultipleScalar(true, SubRegSize::i16Bit, 0b10, zt1, pg, rn, rm);\n }\n void st4h(ZRegister zt1, ZRegister zt2, ZRegister zt3, ZRegister zt4, PRegister pg, Register rn, Register rm) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(zt1, zt2, zt3, zt4), \"Registers need to be contiguous\");\n SVEContiguousLoadStoreMultipleScalar(true, SubRegSize::i16Bit, 0b11, zt1, pg, rn, rm);\n }\n void st2w(ZRegister zt1, ZRegister zt2, PRegister pg, Register rn, Register rm) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(zt1, zt2), \"Registers need to be contiguous\");\n SVEContiguousLoadStoreMultipleScalar(true, SubRegSize::i32Bit, 0b01, zt1, pg, rn, rm);\n }\n void st3w(ZRegister zt1, ZRegister zt2, ZRegister zt3, PRegister pg, Register rn, Register rm) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(zt1, zt2, zt3), \"Registers need to be contiguous\");\n SVEContiguousLoadStoreMultipleScalar(true, SubRegSize::i32Bit, 0b10, zt1, pg, rn, rm);\n }\n void st4w(ZRegister zt1, ZRegister zt2, ZRegister zt3, ZRegister zt4, PRegister pg, Register rn, Register rm) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(zt1, zt2, zt3, zt4), \"Registers need to be contiguous\");\n SVEContiguousLoadStoreMultipleScalar(true, SubRegSize::i32Bit, 0b11, zt1, pg, rn, rm);\n }\n void st2d(ZRegister zt1, ZRegister zt2, PRegister pg, Register rn, Register rm) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(zt1, zt2), \"Registers need to be contiguous\");\n SVEContiguousLoadStoreMultipleScalar(true, SubRegSize::i64Bit, 0b01, zt1, pg, rn, rm);\n }\n void st3d(ZRegister zt1, ZRegister zt2, ZRegister zt3, PRegister pg, Register rn, Register rm) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(zt1, zt2, zt3), \"Registers need to be contiguous\");\n SVEContiguousLoadStoreMultipleScalar(true, SubRegSize::i64Bit, 0b10, zt1, pg, rn, rm);\n }\n void st4d(ZRegister zt1, ZRegister zt2, ZRegister zt3, ZRegister zt4, PRegister pg, Register rn, Register rm) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(zt1, zt2, zt3, zt4), \"Registers need to be contiguous\");\n SVEContiguousLoadStoreMultipleScalar(true, SubRegSize::i64Bit, 0b11, zt1, pg, rn, rm);\n }\n\n // SVE Memory - Contiguous Store with Immediate Offset\n // SVE contiguous non-temporal store (scalar plus immediate)\n void stnt1b(ZRegister zt, PRegister pg, Register rn, int32_t Imm = 0) {\n SVEContiguousNontemporalStore(0b00, zt, pg, rn, Imm);\n }\n void stnt1h(ZRegister zt, PRegister pg, Register rn, int32_t Imm = 0) {\n SVEContiguousNontemporalStore(0b01, zt, pg, rn, Imm);\n }\n void stnt1w(ZRegister zt, PRegister pg, Register rn, int32_t Imm = 0) {\n SVEContiguousNontemporalStore(0b10, zt, pg, rn, Imm);\n }\n void stnt1d(ZRegister zt, PRegister pg, Register rn, int32_t Imm = 0) {\n SVEContiguousNontemporalStore(0b11, zt, pg, rn, Imm);\n }\n\n // SVE store multiple structures (scalar plus immediate)\n void st2b(ZRegister zt1, ZRegister zt2, PRegister pg, Register rn, int32_t Imm = 0) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(zt1, zt2), \"Registers need to be contiguous\");\n SVEContiguousMultipleStructures(2, true, 0b00, Imm, zt1, pg, rn);\n }\n void st3b(ZRegister zt1, ZRegister zt2, ZRegister zt3, PRegister pg, Register rn, int32_t Imm = 0) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(zt1, zt2, zt3), \"Registers need to be contiguous\");\n SVEContiguousMultipleStructures(3, true, 0b00, Imm, zt1, pg, rn);\n }\n void st4b(ZRegister zt1, ZRegister zt2, ZRegister zt3, ZRegister zt4, PRegister pg, Register rn, int32_t Imm = 0) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(zt1, zt2, zt3, zt4), \"Registers need to be contiguous\");\n SVEContiguousMultipleStructures(4, true, 0b00, Imm, zt1, pg, rn);\n }\n void st2h(ZRegister zt1, ZRegister zt2, PRegister pg, Register rn, int32_t Imm = 0) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(zt1, zt2), \"Registers need to be contiguous\");\n SVEContiguousMultipleStructures(2, true, 0b01, Imm, zt1, pg, rn);\n }\n void st3h(ZRegister zt1, ZRegister zt2, ZRegister zt3, PRegister pg, Register rn, int32_t Imm = 0) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(zt1, zt2, zt3), \"Registers need to be contiguous\");\n SVEContiguousMultipleStructures(3, true, 0b01, Imm, zt1, pg, rn);\n }\n void st4h(ZRegister zt1, ZRegister zt2, ZRegister zt3, ZRegister zt4, PRegister pg, Register rn, int32_t Imm = 0) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(zt1, zt2, zt3, zt4), \"Registers need to be contiguous\");\n SVEContiguousMultipleStructures(4, true, 0b01, Imm, zt1, pg, rn);\n }\n void st2w(ZRegister zt1, ZRegister zt2, PRegister pg, Register rn, int32_t Imm = 0) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(zt1, zt2), \"Registers need to be contiguous\");\n SVEContiguousMultipleStructures(2, true, 0b10, Imm, zt1, pg, rn);\n }\n void st3w(ZRegister zt1, ZRegister zt2, ZRegister zt3, PRegister pg, Register rn, int32_t Imm = 0) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(zt1, zt2, zt3), \"Registers need to be contiguous\");\n SVEContiguousMultipleStructures(3, true, 0b10, Imm, zt1, pg, rn);\n }\n void st4w(ZRegister zt1, ZRegister zt2, ZRegister zt3, ZRegister zt4, PRegister pg, Register rn, int32_t Imm = 0) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(zt1, zt2, zt3, zt4), \"Registers need to be contiguous\");\n SVEContiguousMultipleStructures(4, true, 0b10, Imm, zt1, pg, rn);\n }\n void st2d(ZRegister zt1, ZRegister zt2, PRegister pg, Register rn, int32_t Imm = 0) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(zt1, zt2), \"Registers need to be contiguous\");\n SVEContiguousMultipleStructures(2, true, 0b11, Imm, zt1, pg, rn);\n }\n void st3d(ZRegister zt1, ZRegister zt2, ZRegister zt3, PRegister pg, Register rn, int32_t Imm = 0) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(zt1, zt2, zt3), \"Registers need to be contiguous\");\n SVEContiguousMultipleStructures(3, true, 0b11, Imm, zt1, pg, rn);\n }\n void st4d(ZRegister zt1, ZRegister zt2, ZRegister zt3, ZRegister zt4, PRegister pg, Register rn, int32_t Imm = 0) {\n LOGMAN_THROW_A_FMT(AreVectorsSequential(zt1, zt2, zt3, zt4), \"Registers need to be contiguous\");\n SVEContiguousMultipleStructures(4, true, 0b11, Imm, zt1, pg, rn);\n }\n\n // SVE contiguous store (scalar plus immediate)\n template\n void st1b(ZRegister zt, PRegister pg, Register rn, int32_t Imm = 0) {\n SVEContiguousLoadImm(true, 0b0000 | FEXCore::ToUnderlying(size), Imm, pg, rn, zt);\n }\n\n template\n void st1h(ZRegister zt, PRegister pg, Register rn, int32_t Imm = 0) {\n static_assert(size != SubRegSize::i8Bit, \"Invalid size\");\n SVEContiguousLoadImm(true, 0b0100 | FEXCore::ToUnderlying(size), Imm, pg, rn, zt);\n }\n\n template\n void st1w(ZRegister zt, PRegister pg, Register rn, int32_t Imm = 0) {\n static_assert(size == SubRegSize::i32Bit || size == SubRegSize::i64Bit, \"Invalid size\");\n\n constexpr uint32_t ConvertedSize = size == SubRegSize::i32Bit ? 0 : size == SubRegSize::i64Bit ? 1 : -1;\n\n SVEContiguousLoadImm(true, 0b1010 | ConvertedSize, Imm, pg, rn, zt);\n }\n\n void st1d(ZRegister zt, PRegister pg, Register rn, int32_t Imm = 0) {\n SVEContiguousLoadImm(true, 0b1111, Imm, pg, rn, zt);\n }\nprivate:\n // SVE encodings\n void SVEDupIndexed(SubRegSize size, ZRegister zn, ZRegister zd, uint32_t Index) {\n const auto size_bytes = 1U << FEXCore::ToUnderlying(size);\n const auto log2_size_bytes = FEXCore::ilog2(size_bytes);\n\n // We can index up to 512-bit registers with dup\n const auto max_index = (64U >> log2_size_bytes) - 1;\n LOGMAN_THROW_A_FMT(Index <= max_index, \"dup index ({}) too large. Must be within [0, {}].\", Index, max_index);\n\n // imm2:tsz make up a 7 bit wide field, with each increasing element size\n // restricting the range of those 7 bits (e.g. B: tsz=xxxx1, H: tsz=xxx10,\n // S: tsz=xx100. etc). So we can just use the log2 of the element size\n // to construct the overall immediate and form both imm2 and tsz.\n const auto imm7 = (Index << (log2_size_bytes + 1)) | (1U << log2_size_bytes);\n const auto imm2 = imm7 >> 5;\n const auto tsz = imm7 & 0b11111;\n\n uint32_t Instr = 0b0000'0101'0010'0000'0010'0000'0000'0000;\n Instr |= imm2 << 22;\n Instr |= tsz << 16;\n Instr |= Encode_rn(zn);\n Instr |= Encode_rd(zd);\n dc32(Instr);\n }\n\n void SVEAddSubImmediateUnpred(uint32_t opc, SubRegSize size, ZRegister zd, ZRegister zn, uint32_t imm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i128Bit, \"Can't use 128-bit element size\");\n LOGMAN_THROW_A_FMT(zd == zn, \"zd needs to equal zn\");\n\n const bool is_uint8_imm = (imm >> 8) == 0;\n if (size == SubRegSize::i8Bit) {\n LOGMAN_THROW_A_FMT(is_uint8_imm, \"Can't perform LSL #8 shift on 8-bit elements.\");\n }\n\n uint32_t shift = 0;\n if (!is_uint8_imm) {\n const bool is_uint16_imm = (imm >> 16) == 0;\n\n LOGMAN_THROW_A_FMT(is_uint16_imm, \"Immediate ({}) must be a 16-bit value within [256, 65280]\", imm);\n LOGMAN_THROW_A_FMT((imm % 256) == 0, \"Immediate ({}) must be a multiple of 256\", imm);\n\n imm /= 256;\n shift = 1;\n }\n\n uint32_t Instr = 0b0010'0101'0010'0000'1100'0000'0000'0000;\n Instr |= FEXCore::ToUnderlying(size) << 22;\n Instr |= opc << 16;\n Instr |= shift << 13;\n Instr |= imm << 5;\n Instr |= zd.Idx();\n dc32(Instr);\n }\n\n void SVEMinMaxImmediateUnpred(uint32_t opc, SubRegSize size, ZRegister zd, ZRegister zn, int32_t imm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i128Bit, \"Can't use 128-bit element size\");\n LOGMAN_THROW_A_FMT(zd == zn, \"zd needs to equal zn\");\n\n const bool is_signed = (opc & 1) == 0;\n if (is_signed) {\n LOGMAN_THROW_A_FMT(imm >= -128 && imm <= 127, \"Invalid immediate ({}). Must be within [-127, 128]\", imm);\n } else {\n LOGMAN_THROW_A_FMT(imm >= 0 && imm <= 255, \"Invalid immediate ({}). Must be within [0, 255]\", imm);\n }\n\n const auto imm8 = static_cast(imm) & 0xFF;\n\n uint32_t Instr = 0b0010'0101'0010'1000'1100'0000'0000'0000;\n Instr |= FEXCore::ToUnderlying(size) << 22;\n Instr |= opc << 16;\n Instr |= imm8 << 5;\n Instr |= zd.Idx();\n dc32(Instr);\n }\n\n void SVEMultiplyImmediateUnpred(uint32_t opc, SubRegSize size, ZRegister zd, ZRegister zn, int32_t imm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i128Bit, \"Can't use 128-bit element size\");\n LOGMAN_THROW_A_FMT(zd == zn, \"zd needs to equal zn\");\n LOGMAN_THROW_A_FMT(imm >= -128 && imm <= 127, \"Invalid immediate ({}). Must be within [-127, 128]\", imm);\n\n const auto imm8 = static_cast(imm) & 0xFF;\n\n uint32_t Instr = 0b0010'0101'0011'0000'1100'0000'0000'0000;\n Instr |= FEXCore::ToUnderlying(size) << 22;\n Instr |= opc << 16;\n Instr |= imm8 << 5;\n Instr |= zd.Idx();\n dc32(Instr);\n }\n\n void SVEBroadcastImm(uint32_t opc, int32_t imm, SubRegSize size, ZRegister zd) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i128Bit, \"Can't use 128-bit size\");\n\n const auto [new_imm, is_shift] = HandleSVESImm8Shift(size, imm);\n\n uint32_t Instr = 0b0010'0101'0011'1000'1100'0000'0000'0000;\n Instr |= FEXCore::ToUnderlying(size) << 22;\n Instr |= opc << 17;\n Instr |= is_shift << 13;\n Instr |= (static_cast(new_imm) & 0xFF) << 5;\n Instr |= zd.Idx();\n dc32(Instr);\n }\n\n void SVEBroadcastFloatImmPredicated(SubRegSize size, ZRegister zd, PRegister pg, float value) {\n LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), \"Unsupported fcpy/fmov size\");\n\n uint32_t imm {};\n if (size == SubRegSize::i16Bit) {\n LOGMAN_MSG_A_FMT(\"Unsupported\");\n FEX_UNREACHABLE;\n } else if (size == SubRegSize::i32Bit) {\n imm = FP32ToImm8(value);\n } else if (size == SubRegSize::i64Bit) {\n imm = FP64ToImm8(value);\n }\n\n uint32_t Instr = 0b0000'0101'0001'0000'1100'0000'0000'0000;\n Instr |= FEXCore::ToUnderlying(size) << 22;\n Instr |= pg.Idx() << 16;\n Instr |= imm << 5;\n Instr |= zd.Idx();\n dc32(Instr);\n }\n\n void SVEBroadcastFloatImmUnpredicated(uint32_t opc, uint32_t o2, uint32_t imm, SubRegSize size, ZRegister zd) {\n uint32_t Instr = 0b0010'0101'0011'1001'1100'0000'0000'0000;\n Instr |= FEXCore::ToUnderlying(size) << 22;\n Instr |= opc << 17;\n Instr |= o2 << 13;\n Instr |= imm << 5;\n Instr |= zd.Idx();\n dc32(Instr);\n }\n\n void SVEBroadcastIntegerImmPredicated(uint32_t m, SubRegSize size, ZRegister zd, PRegister pg, int32_t imm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i128Bit, \"Can't use 128-bit element size\");\n\n const auto [new_imm, is_shift] = HandleSVESImm8Shift(size, imm);\n\n uint32_t Instr = 0b0000'0101'0001'0000'0000'0000'0000'0000;\n Instr |= FEXCore::ToUnderlying(size) << 22;\n Instr |= pg.Idx() << 16;\n Instr |= m << 14;\n Instr |= is_shift << 13;\n Instr |= (static_cast(new_imm) & 0xFF) << 5;\n Instr |= zd.Idx();\n dc32(Instr);\n }\n\n void SVEAddressGeneration(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm, SVEModType mod, uint32_t scale) {\n LOGMAN_THROW_A_FMT(scale <= 3, \"Scale ({}) must be within [0, 3]\", scale);\n\n uint32_t Instr = 0b0000'0100'0010'0000'1010'0000'0000'0000;\n\n switch (mod) {\n case SVEModType::MOD_UXTW:\n case SVEModType::MOD_SXTW: {\n LOGMAN_THROW_A_FMT(size == SubRegSize::i64Bit, \"Unpacked ADR must be using 64-bit elements\");\n\n const auto is_unsigned = mod == SVEModType::MOD_UXTW;\n if (is_unsigned) {\n Instr |= 1U << 22;\n }\n break;\n }\n case SVEModType::MOD_NONE:\n case SVEModType::MOD_LSL: {\n if (mod == SVEModType::MOD_NONE) {\n LOGMAN_THROW_A_FMT(scale == 0, \"Cannot scale packed ADR without a modifier\");\n }\n LOGMAN_THROW_A_FMT(size == SubRegSize::i32Bit || size == SubRegSize::i64Bit, \"Packed ADR must be using 32-bit or 64-bit elements\");\n Instr |= FEXCore::ToUnderlying(size) << 22;\n break;\n }\n }\n\n Instr |= zm.Idx() << 16;\n Instr |= scale << 10;\n Instr |= zn.Idx() << 5;\n Instr |= zd.Idx();\n dc32(Instr);\n }\n\n void SVESel(SubRegSize size, ZRegister zm, PRegister pv, ZRegister zn, ZRegister zd) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i128Bit, \"Can't use 128-bit element size\");\n\n uint32_t Instr = 0b0000'0101'0010'0000'1100'0000'0000'0000;\n Instr |= FEXCore::ToUnderlying(size) << 22;\n Instr |= zm.Idx() << 16;\n Instr |= pv.Idx() << 10;\n Instr |= Encode_rn(zn);\n Instr |= Encode_rd(zd);\n dc32(Instr);\n }\n\n void SVEBitwiseShiftbyVector(uint32_t R, uint32_t L, uint32_t U, SubRegSize size, PRegister pg, ZRegister zd, ZRegister zn, ZRegister zm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i128Bit, \"Can't use 128-bit size\");\n LOGMAN_THROW_A_FMT(zd == zn, \"Dest needs to equal zn\");\n LOGMAN_THROW_A_FMT(pg <= PReg::p7, \"Can only use p0-p7 as a governing predicate\");\n\n uint32_t Instr = 0b0000'0100'0001'0000'1000'0000'0000'0000;\n\n Instr |= FEXCore::ToUnderlying(size) << 22;\n Instr |= R << 18;\n Instr |= L << 17;\n Instr |= U << 16;\n Instr |= pg.Idx() << 10;\n Instr |= zm.Idx() << 5;\n Instr |= zd.Idx();\n dc32(Instr);\n }\n\n // SVE integer add/subtract vectors (unpredicated)\n void SVEIntegerAddSubUnpredicated(uint32_t opc, SubRegSize size, ZRegister zm, ZRegister zn, ZRegister zd) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i128Bit, \"Can't use 128-bit size\");\n\n uint32_t Instr = 0b0000'0100'0010'0000'0000'0000'0000'0000;\n Instr |= FEXCore::ToUnderlying(size) << 22;\n Instr |= zm.Idx() << 16;\n Instr |= opc << 10;\n Instr |= zn.Idx() << 5;\n Instr |= zd.Idx();\n dc32(Instr);\n }\n\n // SVE table lookup (three sources)\n void SVETableLookup(uint32_t op, SubRegSize size, ZRegister zm, ZRegister zn, ZRegister zd) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i128Bit, \"Can't use 128-bit size\");\n\n uint32_t Instr = 0b0000'0101'0010'0000'0010'0000'0000'0000;\n Instr |= FEXCore::ToUnderlying(size) << 22;\n Instr |= zm.Idx() << 16;\n Instr |= op << 10;\n Instr |= zn.Idx() << 5;\n Instr |= zd.Idx();\n dc32(Instr);\n }\n\n // SVE permute vector elements\n void SVEPermute(uint32_t opc, SubRegSize size, ZRegister zm, ZRegister zn, ZRegister zd) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i128Bit, \"Can't use 128-bit size\");\n\n uint32_t Instr = 0b0000'0101'0010'0000'0110'0000'0000'0000;\n Instr |= FEXCore::ToUnderlying(size) << 22;\n Instr |= zm.Idx() << 16;\n Instr |= opc << 10;\n Instr |= zn.Idx() << 5;\n Instr |= zd.Idx();\n dc32(Instr);\n }\n\n // SVE predicate logical operations\n void SVEPredicateLogical(uint32_t op, uint32_t S, uint32_t o2, uint32_t o3, PRegister pm, PRegister pg, PRegister pn, PRegister pd) {\n uint32_t Instr = 0b0010'0101'0000'0000'0100'0000'0000'0000;\n Instr |= op << 23;\n Instr |= S << 22;\n Instr |= pm.Idx() << 16;\n Instr |= pg.Idx() << 10;\n Instr |= o2 << 9;\n Instr |= pn.Idx() << 5;\n Instr |= o3 << 4;\n Instr |= pd.Idx();\n dc32(Instr);\n }\n\n // SVE floating-point convert precision odd elements\n void SVEFloatConvertOdd(uint32_t opc, uint32_t opc2, PRegister pg, ZRegister zn, ZRegister zd) {\n LOGMAN_THROW_A_FMT(pg <= PReg::p7, \"Can only use p0-p7 as a governing predicate\");\n\n uint32_t Instr = 0b0110'0100'0000'1000'1010'0000'0000'0000;\n Instr |= opc << 22;\n Instr |= opc2 << 16;\n Instr |= pg.Idx() << 10;\n Instr |= zn.Idx() << 5;\n Instr |= zd.Idx();\n dc32(Instr);\n }\n\n // SVE2 floating-point pairwise operations\n void SVEFloatPairwiseArithmetic(uint32_t opc, SubRegSize size, PRegister pg, ZRegister zd, ZRegister zn, ZRegister zm) {\n LOGMAN_THROW_A_FMT(zd == zn, \"zd needs to equal zn\");\n LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), \"Invalid float size\");\n LOGMAN_THROW_A_FMT(pg <= PReg::p7, \"Can only use p0-p7 as a governing predicate\");\n\n uint32_t Instr = 0b0110'0100'0001'0000'1000'0000'0000'0000;\n Instr |= FEXCore::ToUnderlying(size) << 22;\n Instr |= opc << 16;\n Instr |= pg.Idx() << 10;\n Instr |= zm.Idx() << 5;\n Instr |= zd.Idx();\n dc32(Instr);\n }\n\n // SVE floating-point arithmetic (unpredicated)\n void SVEFloatArithmeticUnpredicated(uint32_t opc, SubRegSize size, ZRegister zm, ZRegister zn, ZRegister zd) {\n LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), \"Invalid float size\");\n\n uint32_t Instr = 0b0110'0101'0000'0000'0000'0000'0000'0000;\n Instr |= FEXCore::ToUnderlying(size) << 22;\n Instr |= zm.Idx() << 16;\n Instr |= opc << 10;\n Instr |= zn.Idx() << 5;\n Instr |= zd.Idx();\n dc32(Instr);\n }\n\n // SVE bitwise logical operations (predicated)\n void SVEBitwiseLogicalPredicated(uint32_t opc, SubRegSize size, PRegister pg, ZRegister zdn, ZRegister zm, ZRegister zd) {\n LOGMAN_THROW_A_FMT(size != ARMEmitter::SubRegSize::i128Bit, \"Can't use 128-bit size\");\n LOGMAN_THROW_A_FMT(zd == zdn, \"zd needs to equal zdn\");\n LOGMAN_THROW_A_FMT(pg <= PReg::p7, \"Can only use p0-p7 as a governing predicate\");\n\n uint32_t Instr = 0b0000'0100'0001'1000'0000'0000'0000'0000;\n Instr |= FEXCore::ToUnderlying(size) << 22;\n Instr |= opc << 16;\n Instr |= pg.Idx() << 10;\n Instr |= zm.Idx() << 5;\n Instr |= zd.Idx();\n dc32(Instr);\n }\n\n // SVE constructive prefix (predicated)\n void SVEConstructivePrefixPredicated(uint32_t opc, uint32_t M, SubRegSize size, PRegister pg, ZRegister zn, ZRegister zd) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i128Bit, \"Can't use 128-bit element size\");\n LOGMAN_THROW_A_FMT(pg <= PReg::p7, \"Can only use p0-p7 as a governing predicate\");\n\n uint32_t Instr = 0b0000'0100'0001'0000'0010'0000'0000'0000;\n Instr |= FEXCore::ToUnderlying(size) << 22;\n Instr |= opc << 17;\n Instr |= M << 16;\n Instr |= pg.Idx() << 10;\n Instr |= Encode_rn(zn);\n Instr |= Encode_rd(zd);\n dc32(Instr);\n }\n\n // SVE bitwise unary operations (predicated)\n void SVEIntegerUnaryPredicated(uint32_t op0, uint32_t opc, SubRegSize size, PRegister pg, ZRegister zn, ZRegister zd) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i128Bit, \"Can't use 128-bit size\");\n LOGMAN_THROW_A_FMT(pg <= PReg::p7, \"Can only use p0-p7 as a governing predicate\");\n\n uint32_t Instr = 0b0000'0100'0000'0000'1010'0000'0000'0000;\n\n Instr |= FEXCore::ToUnderlying(size) << 22;\n Instr |= op0 << 19;\n Instr |= opc << 16;\n Instr |= pg.Idx() << 10;\n Instr |= zn.Idx() << 5;\n Instr |= zd.Idx();\n dc32(Instr);\n }\n\n // SVE bitwise logical operations (unpredicated)\n void SVEBitwiseLogicalUnpredicated(uint32_t opc, ZRegister zm, ZRegister zn, ZRegister zd) {\n uint32_t Instr = 0b0000'0100'0010'0000'0011'0000'0000'0000;\n\n Instr |= opc << 22;\n Instr |= zm.Idx() << 16;\n Instr |= zn.Idx() << 5;\n Instr |= zd.Idx();\n dc32(Instr);\n }\n\n // SVE Permute Vector - Unpredicated\n void SVEPermuteUnpredicated(SubRegSize size, uint32_t opc, ZRegister zdn, ZRegister zm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i128Bit, \"Cannot use 128-bit element size\");\n\n uint32_t Instr = 0b0000'0101'0010'0000'0011'1000'0000'0000;\n Instr |= FEXCore::ToUnderlying(size) << 22;\n Instr |= opc << 16;\n Instr |= zm.Idx() << 5;\n Instr |= zdn.Idx();\n dc32(Instr);\n }\n\n // SVE Permute Predicate\n void SVEPermutePredicate(SubRegSize size, uint32_t op1, uint32_t op2, uint32_t op3, PRegister pd, PRegister pn) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i128Bit, \"Cannot use 128-bit element size\");\n\n uint32_t Instr = 0b0000'0101'0010'0000'0100'0000'0000'0000;\n Instr |= FEXCore::ToUnderlying(size) << 22;\n Instr |= op1 << 16;\n Instr |= op2 << 9;\n Instr |= op3 << 4;\n Instr |= pn.Idx() << 5;\n Instr |= pd.Idx();\n dc32(Instr);\n }\n\n // SVE Integer Misc - Unpredicated\n void SVEIntegerMiscUnpredicated(uint32_t op0, uint32_t opc, uint32_t opc2, ZRegister zd, ZRegister zn) {\n uint32_t Instr = 0b0000'0100'0010'0000'1011'0000'0000'0000;\n Instr |= opc2 << 22;\n Instr |= opc << 16;\n Instr |= op0 << 10;\n Instr |= zn.Idx() << 5;\n Instr |= zd.Idx();\n dc32(Instr);\n }\n\n // SVE floating-point arithmetic (predicated)\n void SVEFloatArithmeticPredicated(uint32_t opc, SubRegSize size, PRegister pg, ZRegister zd, ZRegister zn, ZRegister zm) {\n LOGMAN_THROW_A_FMT(zd == zn, \"zn needs to equal zd\");\n LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), \"Invalid float size\");\n LOGMAN_THROW_A_FMT(pg <= PReg::p7, \"Can only use p0-p7 as a governing predicate\");\n\n uint32_t Instr = 0b0110'0101'0000'0000'1000'0000'0000'0000;\n\n Instr |= FEXCore::ToUnderlying(size) << 22;\n Instr |= opc << 16;\n Instr |= pg.Idx() << 10;\n Instr |= zm.Idx() << 5;\n Instr |= zd.Idx();\n dc32(Instr);\n }\n\n void SVECharacterMatch(uint32_t opc, SubRegSize size, PRegister pd, PRegisterZero pg, ZRegister zn, ZRegister zm) {\n LOGMAN_THROW_A_FMT(size == SubRegSize::i8Bit || size == SubRegSize::i16Bit, \"match/nmatch can only use 8-bit or 16-bit element sizes\");\n LOGMAN_THROW_A_FMT(pg <= PReg::p7.Zeroing(), \"match/nmatch can only use p0-p7 as a governing predicate\");\n\n uint32_t Instr = 0b0100'0101'0010'0000'1000'0000'0000'0000;\n Instr |= FEXCore::ToUnderlying(size) << 22;\n Instr |= opc << 4;\n Instr |= zm.Idx() << 16;\n Instr |= pg.Idx() << 10;\n Instr |= zn.Idx() << 5;\n Instr |= pd.Idx();\n dc32(Instr);\n }\n\n void SVEFPRecursiveReduction(uint32_t opc, SubRegSize size, VRegister vd, PRegister pg, ZRegister zn) {\n LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), \"FP reduction operation can only use 16/32/64-bit element sizes\");\n LOGMAN_THROW_A_FMT(pg <= PReg::p7, \"FP reduction operation can only use p0-p7 as a governing predicate\");\n\n uint32_t Instr = 0b0110'0101'0000'0000'0010'0000'0000'0000;\n Instr |= FEXCore::ToUnderlying(size) << 22;\n Instr |= opc << 16;\n Instr |= pg.Idx() << 10;\n Instr |= zn.Idx() << 5;\n Instr |= vd.Idx();\n dc32(Instr);\n }\n\n void SVEAddSubVectorsPredicated(uint32_t opc, SubRegSize size, ZRegister zd, PRegister pg, ZRegister zn, ZRegister zm) {\n LOGMAN_THROW_A_FMT(zd == zn, \"zd and zn must be the same register\");\n LOGMAN_THROW_A_FMT(pg <= PReg::p7, \"Add/Sub operation can only use p0-p7 as a governing predicate\");\n\n uint32_t Instr = 0b0000'0100'0000'0000'0000'0000'0000'0000;\n Instr |= FEXCore::ToUnderlying(size) << 22;\n Instr |= opc << 16;\n Instr |= pg.Idx() << 10;\n Instr |= zm.Idx() << 5;\n Instr |= zd.Idx();\n dc32(Instr);\n }\n\n void SVEIntegerMulDivVectorsPredicated(uint32_t b18, uint32_t opc, SubRegSize size, ZRegister zd, PRegister pg, ZRegister zn, ZRegister zm) {\n LOGMAN_THROW_A_FMT(zd == zn, \"zd and zn must be the same register\");\n LOGMAN_THROW_A_FMT(pg <= PReg::p7, \"Mul/Div operation can only use p0-p7 as a governing predicate\");\n\n // Division instruction\n if (b18 != 0) {\n LOGMAN_THROW_A_FMT(size == SubRegSize::i32Bit || size == SubRegSize::i64Bit, \"Predicated divide only handles 32-bit or 64-bit \"\n \"elements\");\n }\n\n uint32_t Instr = 0b0000'0100'0001'0000'0000'0000'0000'0000;\n Instr |= FEXCore::ToUnderlying(size) << 22;\n Instr |= b18 << 18;\n Instr |= opc << 16;\n Instr |= pg.Idx() << 10;\n Instr |= zm.Idx() << 5;\n Instr |= zd.Idx();\n dc32(Instr);\n }\n\n void SVEIntegerReductionOperation(uint32_t op, uint32_t opc, SubRegSize size, VRegister vd, PRegister pg, ZRegister zn) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i128Bit, \"Can't use 128-bit element size for reduction operation\");\n LOGMAN_THROW_A_FMT(pg <= PReg::p7, \"Integer reduction operation can only use p0-p7 as a governing predicate\");\n\n uint32_t Instr = op;\n Instr |= FEXCore::ToUnderlying(size) << 22;\n Instr |= opc << 16;\n Instr |= pg.Idx() << 10;\n Instr |= zn.Idx() << 5;\n Instr |= vd.Idx();\n dc32(Instr);\n }\n\n void SVEIntegerMultiplyAddSubPredicated(uint32_t op0, uint32_t opc, SubRegSize size, ZRegister zd, PRegister pg, ZRegister zn, ZRegister zm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i128Bit, \"Can't use 128-bit element size\");\n LOGMAN_THROW_A_FMT(pg <= PReg::p7, \"Can only use p0-p7 as a governing predicate\");\n\n uint32_t Instr = 0b0000'0100'0000'0000'0100'0000'0000'0000;\n Instr |= FEXCore::ToUnderlying(size) << 22;\n Instr |= zm.Idx() << 16;\n Instr |= op0 << 15;\n Instr |= opc << 13;\n Instr |= pg.Idx() << 10;\n Instr |= zn.Idx() << 5;\n Instr |= zd.Idx();\n dc32(Instr);\n }\n\n void SVEStackFrameOperation(uint32_t opc, XRegister rd, XRegister rn, int32_t imm) {\n LOGMAN_THROW_A_FMT(imm >= -32 && imm <= 31, \"Stack frame operation immediate must be within -32 to 31\");\n\n uint32_t Instr = 0b0000'0100'0010'0000'0101'0000'0000'0000;\n Instr |= opc << 22;\n Instr |= rn.Idx() << 16;\n Instr |= (static_cast(imm) & 0b111111) << 5;\n Instr |= rd.Idx();\n dc32(Instr);\n }\n\n void SVEBitwiseShiftByWideElementPredicated(SubRegSize size, uint32_t opc, ZRegister zd, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit && size != SubRegSize::i128Bit, \"Can't use 64-bit or 128-bit element size\");\n LOGMAN_THROW_A_FMT(zd == zn, \"zd and zn must be the same register\");\n LOGMAN_THROW_A_FMT(pg <= PReg::p7.Merging(), \"Wide shift can only use p0-p7 as a governing predicate\");\n\n uint32_t Instr = 0b0000'0100'0001'1000'1000'0000'0000'0000;\n Instr |= FEXCore::ToUnderlying(size) << 22;\n Instr |= opc << 16;\n Instr |= pg.Idx() << 10;\n Instr |= zm.Idx() << 5;\n Instr |= zd.Idx();\n dc32(Instr);\n }\n\n void SVEBitwiseShiftByWideElementsUnpredicated(SubRegSize size, uint32_t opc, ZRegister zd, ZRegister zn, ZRegister zm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit && size != SubRegSize::i128Bit, \"Can't use 64-bit or 128-bit element size\");\n\n uint32_t Instr = 0b0000'0100'0010'0000'1000'0000'0000'0000;\n Instr |= FEXCore::ToUnderlying(size) << 22;\n Instr |= opc << 10;\n Instr |= zm.Idx() << 16;\n Instr |= zn.Idx() << 5;\n Instr |= zd.Idx();\n dc32(Instr);\n }\n\n void SVEFPArithWithImmediate(uint32_t opc, SubRegSize size, ZRegister zd, PRegister pg, uint32_t i1) {\n LOGMAN_THROW_A_FMT(pg <= PReg::p7, \"Can only use p0-p7 as a governing predicate\");\n LOGMAN_THROW_A_FMT(size != SubRegSize::i8Bit && size != SubRegSize::i128Bit, \"Can't use 8-bit or 128-bit element size\");\n\n uint32_t Instr = 0b0110'0101'0001'1000'1000'0000'0000'0000;\n Instr |= FEXCore::ToUnderlying(size) << 22;\n Instr |= opc << 16;\n Instr |= pg.Idx() << 10;\n Instr |= i1 << 5;\n Instr |= zd.Idx();\n dc32(Instr);\n }\n\n void SVEFPConvertPrecision(SubRegSize to, SubRegSize from, ZRegister zd, PRegister pg, ZRegister zn) {\n LOGMAN_THROW_A_FMT(pg <= PReg::p7, \"Can only use p0-p7 as a governing predicate\");\n LOGMAN_THROW_A_FMT(to != from, \"to and from sizes cannot be the same.\");\n LOGMAN_THROW_A_FMT(to != SubRegSize::i8Bit && to != SubRegSize::i128Bit && from != SubRegSize::i8Bit && from != SubRegSize::i128Bit,\n \"Can't use 8-bit or 128-bit element size\");\n\n // Encodings for the to and from sizes can get a little funky\n // depending on what is being converted to/from.\n const uint32_t op = [&] {\n switch (from) {\n case SubRegSize::i16Bit: {\n switch (to) {\n case SubRegSize::i32Bit: return 0x00810000U;\n case SubRegSize::i64Bit: return 0x00C10000U;\n default: return UINT32_MAX;\n }\n }\n\n case SubRegSize::i32Bit: {\n switch (to) {\n case SubRegSize::i16Bit: return 0x00800000U;\n case SubRegSize::i64Bit: return 0x00C30000U;\n default: return UINT32_MAX;\n }\n }\n\n case SubRegSize::i64Bit: {\n switch (to) {\n case SubRegSize::i16Bit: return 0x00C00000U;\n case SubRegSize::i32Bit: return 0x00C20000U;\n default: return UINT32_MAX;\n }\n }\n\n default: return UINT32_MAX;\n }\n }();\n LOGMAN_THROW_A_FMT(op != UINT32_MAX, \"Invalid conversion op value: {}\", op);\n\n uint32_t Instr = 0b0110'0101'0000'1000'1010'0000'0000'0000;\n Instr |= op;\n Instr |= pg.Idx() << 10;\n Instr |= zn.Idx() << 5;\n Instr |= zd.Idx();\n dc32(Instr);\n }\n\n void SVE2IntegerAddSubNarrowHighPart(SubRegSize size, uint32_t opc, ZRegister zd, ZRegister zn, ZRegister zm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit && size != SubRegSize::i128Bit, \"Can't use 64-bit or 128-bit element size\");\n\n uint32_t Instr = 0b0100'0101'0010'0000'0110'0000'0000'0000;\n Instr |= (FEXCore::ToUnderlying(size) + 1) << 22;\n Instr |= zm.Idx() << 16;\n Instr |= opc << 10;\n Instr |= zn.Idx() << 5;\n Instr |= zd.Idx();\n dc32(Instr);\n }\n\n void SVE2CryptoUnaryOperation(uint32_t op, ZRegister zdn, ZRegister zn) {\n LOGMAN_THROW_A_FMT(zdn == zn, \"zdn and zn must be the same register\");\n\n uint32_t Instr = 0b0100'0101'0010'0000'1110'0000'0000'0000;\n Instr |= op << 10;\n Instr |= zdn.Idx();\n dc32(Instr);\n }\n\n void SVE2CryptoDestructiveBinaryOperation(uint32_t op, uint32_t o2, ZRegister zdn, ZRegister zn, ZRegister zm) {\n LOGMAN_THROW_A_FMT(zdn == zn, \"zdn and zn must be the same register\");\n\n uint32_t Instr = 0b0100'0101'0010'0010'1110'0000'0000'0000;\n Instr |= op << 16;\n Instr |= o2 << 10;\n Instr |= zm.Idx() << 5;\n Instr |= zdn.Idx();\n dc32(Instr);\n }\n\n void SVE2CryptoConstructiveBinaryOperation(uint32_t op, ZRegister zd, ZRegister zn, ZRegister zm) {\n uint32_t Instr = 0b0100'0101'0010'0000'1111'0000'0000'0000;\n Instr |= zm.Idx() << 16;\n Instr |= op << 10;\n Instr |= zn.Idx() << 5;\n Instr |= zd.Idx();\n dc32(Instr);\n }\n\n void SVE2BitwisePermute(SubRegSize size, uint32_t opc, ZRegister zd, ZRegister zn, ZRegister zm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i128Bit, \"Can't use 128-bit element size\");\n\n uint32_t Instr = 0b0100'0101'0000'0000'1011'0000'0000'0000;\n Instr |= FEXCore::ToUnderlying(size) << 22;\n Instr |= zm.Idx() << 16;\n Instr |= opc << 10;\n Instr |= zn.Idx() << 5;\n Instr |= zd.Idx();\n dc32(Instr);\n }\n\n void SVE2BitwiseXorInterleaved(SubRegSize size, uint32_t opc, ZRegister zd, ZRegister zn, ZRegister zm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i128Bit, \"Can't use 128-bit element size\");\n\n uint32_t Instr = 0b0100'0101'0000'0000'1001'0000'0000'0000;\n Instr |= FEXCore::ToUnderlying(size) << 22;\n Instr |= zm.Idx() << 16;\n Instr |= opc << 10;\n Instr |= zn.Idx() << 5;\n Instr |= zd.Idx();\n dc32(Instr);\n }\n\n void SVEIntegerMatrixMulAccumulate(uint32_t opc, ZRegister zda, ZRegister zn, ZRegister zm) {\n uint32_t Instr = 0b0100'0101'0000'0000'1001'1000'0000'0000;\n Instr |= opc << 22;\n Instr |= zm.Idx() << 16;\n Instr |= zn.Idx() << 5;\n Instr |= zda.Idx();\n dc32(Instr);\n }\n\n void SVE2IntegerAddSubInterleavedLong(SubRegSize size, uint32_t opc, ZRegister zd, ZRegister zn, ZRegister zm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i8Bit && size != SubRegSize::i128Bit, \"Can't use 8-bit or 128-bit element size\");\n\n uint32_t Instr = 0b0100'0101'0000'0000'1000'0000'0000'0000;\n Instr |= FEXCore::ToUnderlying(size) << 22;\n Instr |= zm.Idx() << 16;\n Instr |= opc << 10;\n Instr |= zn.Idx() << 5;\n Instr |= zd.Idx();\n dc32(Instr);\n }\n\n void SVE2IntegerAbsDiffAndAccumulate(SubRegSize size, uint32_t opc, ZRegister zda, ZRegister zn, ZRegister zm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i128Bit, \"Can't use 128-bit element size\");\n\n uint32_t Instr = 0b0100'0101'0000'0000'1111'1000'0000'0000;\n Instr |= FEXCore::ToUnderlying(size) << 22;\n Instr |= zm.Idx() << 16;\n Instr |= opc << 10;\n Instr |= zn.Idx() << 5;\n Instr |= zda.Idx();\n dc32(Instr);\n }\n\n void SVE2IntegerAddSubLongWithCarry(SubRegSize size, uint32_t sizep1, uint32_t T, ZRegister zda, ZRegister zn, ZRegister zm) {\n LOGMAN_THROW_A_FMT(size == SubRegSize::i32Bit || size == SubRegSize::i64Bit, \"Element size must be 32-bit or 64-bit\");\n\n const uint32_t NewSize = size == SubRegSize::i32Bit ? 0 : 1;\n\n uint32_t Instr = 0b0100'0101'0000'0000'1101'0000'0000'0000;\n Instr |= sizep1 << 23;\n Instr |= NewSize << 22;\n Instr |= zm.Idx() << 16;\n Instr |= T << 10;\n Instr |= zn.Idx() << 5;\n Instr |= zda.Idx();\n dc32(Instr);\n }\n\n void SVE2BitwiseShiftRightAndAccumulate(SubRegSize size, uint32_t opc, ZRegister zda, ZRegister zn, uint32_t shift) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i128Bit, \"Element size cannot be 128-bit\");\n\n const auto [tszh, tszl_imm3] = EncodeSVEShiftImmediate(size, shift);\n\n uint32_t Instr = 0b0100'0101'0000'0000'1110'0000'0000'0000;\n Instr |= tszh << 22;\n Instr |= tszl_imm3 << 16;\n Instr |= opc << 10;\n Instr |= zn.Idx() << 5;\n Instr |= zda.Idx();\n dc32(Instr);\n }\n\n void SVE2BitwiseShiftAndInsert(SubRegSize size, uint32_t opc, ZRegister zd, ZRegister zn, uint32_t shift) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i128Bit, \"Element size cannot be 128-bit\");\n\n const bool IsLeftShift = opc != 0;\n const auto [tszh, tszl_imm3] = EncodeSVEShiftImmediate(size, shift, IsLeftShift);\n\n uint32_t Instr = 0b0100'0101'0000'0000'1111'0000'0000'0000;\n Instr |= tszh << 22;\n Instr |= tszl_imm3 << 16;\n Instr |= opc << 10;\n Instr |= zn.Idx() << 5;\n Instr |= zd.Idx();\n dc32(Instr);\n }\n\n void SVE2BitwiseShiftLeftLong(SubRegSize size, uint32_t opc, ZRegister zd, ZRegister zn, uint32_t shift) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i8Bit && size != SubRegSize::i128Bit, \"Can't use 8-bit or 128-bit element size\");\n\n // The size provided in is the size to expand to (e.g. 16-bit means a long shift\n // expanding from 8-bit) so we just need to subtract the size by 1 so that our\n // encoding helper will perform the proper encoding.\n const auto size_minus_1 = SubRegSize {FEXCore::ToUnderlying(size) - 1};\n const auto [tszh, tszl_imm3] = EncodeSVEShiftImmediate(size_minus_1, shift, true);\n\n uint32_t Instr = 0b0100'0101'0000'0000'1010'0000'0000'0000;\n Instr |= tszh << 22;\n Instr |= tszl_imm3 << 16;\n Instr |= opc << 10;\n Instr |= zn.Idx() << 5;\n Instr |= zd.Idx();\n dc32(Instr);\n }\n\n void SVE2ComplexIntAdd(SubRegSize size, uint32_t opc, Rotation rot, ZRegister zd, ZRegister zn, ZRegister zm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i128Bit, \"Complex add cannot use 128-bit element size\");\n LOGMAN_THROW_A_FMT(zd == zn, \"zd and zn must be the same register\");\n LOGMAN_THROW_A_FMT(rot == Rotation::ROTATE_90 || rot == Rotation::ROTATE_270, \"Rotation must be 90 or 270 degrees\");\n\n const uint32_t SanitizedRot = rot == Rotation::ROTATE_90 ? 0 : 1;\n\n uint32_t Instr = 0b0100'0101'0000'0000'1101'1000'0000'0000;\n Instr |= FEXCore::ToUnderlying(size) << 22;\n Instr |= opc << 16;\n Instr |= SanitizedRot << 10;\n Instr |= zm.Idx() << 5;\n Instr |= zd.Idx();\n dc32(Instr);\n }\n\n void SVE2AbsDiffAccLong(SubRegSize size, uint32_t opc, ZRegister zda, ZRegister zn, ZRegister zm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i8Bit && size != SubRegSize::i128Bit, \"Cannot use 8-bit or 128-bit element size\");\n\n uint32_t Instr = 0b0100'0101'0000'0000'1100'0000'0000'0000;\n Instr |= FEXCore::ToUnderlying(size) << 22;\n Instr |= zm.Idx() << 16;\n Instr |= opc << 10;\n Instr |= zn.Idx() << 5;\n Instr |= zda.Idx();\n dc32(Instr);\n }\n\n void SVEPermuteVectorUnpredicated(SubRegSize size, uint32_t opc, ZRegister zdn, VRegister vm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i128Bit, \"Cannot use 128-bit element size\");\n\n uint32_t Instr = 0b0000'0101'0010'0000'0011'1000'0000'0000;\n Instr |= FEXCore::ToUnderlying(size) << 22;\n Instr |= opc << 16;\n Instr |= vm.Idx() << 5;\n Instr |= zdn.Idx();\n dc32(Instr);\n }\n\n // SVE floating-point round to integral value\n void SVEFloatRoundIntegral(uint32_t opc, SubRegSize size, ZRegister zd, PRegister pg, ZRegister zn) {\n // opc = round mode\n // 0b000 - N - Neaest ties to even\n // 0b001 - P - Towards +inf\n // 0b010 - M - Towards -inf\n // 0b011 - Z - Towards zero\n // 0b100 - A - Nearest away from zero\n // 0b101 - Unallocated\n // 0b110 - X - Current signalling inexact\n // 0b111 - I - Current\n\n LOGMAN_THROW_A_FMT(pg <= PReg::p7, \"Can only use p0-p7 as a governing predicate\");\n LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), \"Unsupported size in {}\", __func__);\n\n uint32_t Instr = 0b0110'0101'0000'0000'1010'0000'0000'0000;\n Instr |= FEXCore::ToUnderlying(size) << 22;\n Instr |= opc << 16;\n Instr |= pg.Idx() << 10;\n Instr |= zn.Idx() << 5;\n Instr |= zd.Idx();\n dc32(Instr);\n }\n\n // SVE floating-point convert to integer\n void SVEFloatConvertToInt(SubRegSize dstsize, SubRegSize srcsize, uint32_t b19, uint32_t opc, uint32_t opc2, uint32_t U, PRegister pg,\n ZRegister zn, ZRegister zd) {\n LOGMAN_THROW_A_FMT(pg <= PReg::p7, \"Can only use p0-p7 as a governing predicate\");\n LOGMAN_THROW_A_FMT(srcsize == SubRegSize::i16Bit || srcsize == SubRegSize::i32Bit || srcsize == SubRegSize::i64Bit,\n \"Unsupported src size in {}\", __func__);\n LOGMAN_THROW_A_FMT(dstsize == SubRegSize::i16Bit || dstsize == SubRegSize::i32Bit || dstsize == SubRegSize::i64Bit,\n \"Unsupported dst size in {}\", __func__);\n\n uint32_t Instr = 0b0110'0101'0001'0000'1010'0000'0000'0000;\n Instr |= opc << 22;\n Instr |= b19 << 19;\n Instr |= opc2 << 17;\n Instr |= U << 16;\n Instr |= pg.Idx() << 10;\n Instr |= zn.Idx() << 5;\n Instr |= zd.Idx();\n dc32(Instr);\n }\n // SVE integer convert to floating-point\n // We can implement this in terms of the floating-point to int version above,\n // since the only difference in encoding is setting bit 19 to 0.\n void SVEIntegerConvertToFloat(SubRegSize dstsize, SubRegSize srcsize, uint32_t opc, uint32_t opc2, uint32_t U, PRegister pg, ZRegister zn,\n ZRegister zd) {\n SVEFloatConvertToInt(dstsize, srcsize, 0, opc, opc2, U, pg, zn, zd);\n }\n\n // SVE Memory - 32-bit Gather and Unsized Contiguous\n // Note: This also handles 64-bit variants to keep overall handling code\n // compact and in the same place.\n void SVEGatherLoadScalarPlusVector(SubRegSize esize, SubRegSize msize, ZRegister zt, PRegisterZero pg, SVEMemOperand mem_op,\n bool is_unsigned, bool is_fault_first) {\n LOGMAN_THROW_A_FMT(esize == SubRegSize::i32Bit || esize == SubRegSize::i64Bit, \"Gather load element size must be 32-bit or 64-bit\");\n LOGMAN_THROW_A_FMT(pg <= PReg::p7, \"Can only use p0-p7 as a governing predicate\");\n\n const auto& op_data = mem_op.MetaType.ScalarVectorType;\n const bool is_scaled = op_data.scale != 0;\n const auto msize_value = FEXCore::ToUnderlying(msize);\n\n LOGMAN_THROW_A_FMT(op_data.scale == 0 || op_data.scale == msize_value, \"scale may only be 0 or {}\", msize_value);\n\n uint32_t mod_value = FEXCore::ToUnderlying(op_data.mod);\n uint32_t Instr = 0b1000'0100'0000'0000'0000'0000'0000'0000;\n\n if (esize == SubRegSize::i64Bit) {\n Instr |= 1U << 30;\n\n const auto mod = op_data.mod;\n const bool is_lsl = mod == SVEModType::MOD_LSL;\n const bool is_none = mod == SVEModType::MOD_NONE;\n\n // LSL and no modifier encodings should be setting bit 22 to 1.\n if (is_lsl || is_none) {\n if (is_lsl) {\n LOGMAN_THROW_A_FMT(op_data.scale == msize_value, \"mod type of LSL must have a scale of {}\", msize_value);\n } else {\n LOGMAN_THROW_A_FMT(op_data.scale == 0, \"mod type of none must have a scale of 0\");\n }\n\n Instr |= 1U << 15;\n mod_value = 1;\n }\n } else {\n LOGMAN_THROW_A_FMT(op_data.mod == SVEModType::MOD_UXTW || op_data.mod == SVEModType::MOD_SXTW, \"mod type for 32-bit lane size may \"\n \"only be UXTW or SXTW\");\n }\n\n Instr |= FEXCore::ToUnderlying(msize) << 23;\n Instr |= static_cast(mod_value) << 22;\n Instr |= static_cast(is_scaled) << 21;\n Instr |= op_data.zm.Idx() << 16;\n Instr |= static_cast(is_unsigned) << 14;\n Instr |= static_cast(is_fault_first) << 13;\n Instr |= pg.Idx() << 10;\n Instr |= mem_op.rn.Idx() << 5;\n Instr |= zt.Idx();\n\n dc32(Instr);\n }\n\n void SVEScatterStoreScalarPlusVector(SubRegSize esize, SubRegSize msize, ZRegister zt, PRegister pg, SVEMemOperand mem_op) {\n LOGMAN_THROW_A_FMT(esize == SubRegSize::i32Bit || esize == SubRegSize::i64Bit, \"Gather load element size must be 32-bit or 64-bit\");\n LOGMAN_THROW_A_FMT(pg <= PReg::p7, \"Can only use p0-p7 as a governing predicate\");\n\n const auto& op_data = mem_op.MetaType.ScalarVectorType;\n const bool is_scaled = op_data.scale != 0;\n\n const auto msize_value = FEXCore::ToUnderlying(msize);\n uint32_t mod_value = FEXCore::ToUnderlying(op_data.mod);\n\n LOGMAN_THROW_A_FMT(op_data.scale == 0 || op_data.scale == msize_value, \"scale may only be 0 or {}\", msize_value);\n\n uint32_t Instr = 0b1110'0100'0000'0000'1000'0000'0000'0000;\n\n if (esize == SubRegSize::i64Bit) {\n const auto mod = op_data.mod;\n const bool is_lsl = mod == SVEModType::MOD_LSL;\n const bool is_none = mod == SVEModType::MOD_NONE;\n\n if (is_lsl || is_none) {\n if (is_lsl) {\n LOGMAN_THROW_A_FMT(op_data.scale == msize_value, \"mod type of LSL must have a scale of {}\", msize_value);\n } else {\n LOGMAN_THROW_A_FMT(op_data.scale == 0, \"mod type of none must have a scale of 0\");\n }\n if (is_lsl || is_scaled) {\n LOGMAN_THROW_A_FMT(msize != SubRegSize::i8Bit, \"Cannot use 8-bit store elements with unpacked 32-bit scaled offset and \"\n \"64-bit scaled offset variants. Instructions not allocated.\");\n }\n\n // 64-bit scaled/unscaled scatters need to set bit 13\n Instr |= 1U << 13;\n mod_value = 0;\n }\n } else {\n if (is_scaled) {\n LOGMAN_THROW_A_FMT(msize != SubRegSize::i8Bit && msize != SubRegSize::i64Bit, \"Cannot use 8-bit or 64-bit store elements with \"\n \"32-bit scaled offset variant. \"\n \"Instructions not allocated\");\n } else {\n LOGMAN_THROW_A_FMT(msize != SubRegSize::i64Bit, \"Cannot use 64-bit store elements with 32-bit unscaled offset variant. \"\n \"Instruction not allocated.\");\n }\n\n LOGMAN_THROW_A_FMT(op_data.mod == SVEModType::MOD_UXTW || op_data.mod == SVEModType::MOD_SXTW, \"mod type for 32-bit lane size may \"\n \"only be UXTW or SXTW\");\n\n // 32-bit scatters need to set bit 22.\n Instr |= 1U << 22;\n }\n\n Instr |= msize_value << 23;\n Instr |= static_cast(is_scaled) << 21;\n Instr |= op_data.zm.Idx() << 16;\n Instr |= static_cast(mod_value) << 14;\n Instr |= pg.Idx() << 10;\n Instr |= mem_op.rn.Idx() << 5;\n Instr |= zt.Idx();\n\n dc32(Instr);\n }\n\n void SVEGatherScatterVectorPlusImm(SubRegSize esize, SubRegSize msize, ZRegister zt, PRegister pg, SVEMemOperand mem_op, bool is_store,\n bool is_unsigned, bool is_fault_first) {\n LOGMAN_THROW_A_FMT(esize == SubRegSize::i32Bit || esize == SubRegSize::i64Bit, \"Gather load/store element size must be 32-bit or \"\n \"64-bit\");\n LOGMAN_THROW_A_FMT(pg <= PReg::p7, \"Can only use p0-p7 as a governing predicate\");\n\n const auto msize_value = FEXCore::ToUnderlying(msize);\n const auto msize_bytes = 1U << msize_value;\n\n const auto imm_limit = (32U << msize_value) - msize_bytes;\n const auto imm = mem_op.MetaType.VectorImmType.Imm;\n const auto imm_to_encode = imm >> msize_value;\n\n LOGMAN_THROW_A_FMT(imm <= imm_limit, \"Immediate must be within [0, {}]\", imm_limit);\n LOGMAN_THROW_A_FMT(imm == 0 || (imm % msize_bytes) == 0, \"Immediate must be cleanly divisible by {}\", msize_bytes);\n\n uint32_t Instr = 0b1000'0100'0000'0000'1000'0000'0000'0000;\n\n if (is_store) {\n Instr |= 0x60402000U;\n if (esize == SubRegSize::i32Bit) {\n Instr |= 1U << 21;\n }\n } else {\n Instr |= 0x00200000U;\n if (esize == SubRegSize::i64Bit) {\n Instr |= 1U << 30;\n }\n }\n\n Instr |= msize_value << 23;\n Instr |= imm_to_encode << 16;\n Instr |= static_cast(is_unsigned) << 14;\n Instr |= static_cast(is_fault_first) << 13;\n Instr |= pg.Idx() << 10;\n Instr |= mem_op.rn.Idx() << 5;\n Instr |= zt.Idx();\n\n dc32(Instr);\n }\n\n void SVEGatherLoadVectorPlusImm(SubRegSize esize, SubRegSize msize, ZRegister zt, PRegisterZero pg, SVEMemOperand mem_op,\n bool is_unsigned, bool is_fault_first) {\n SVEGatherScatterVectorPlusImm(esize, msize, zt, pg, mem_op, false, is_unsigned, is_fault_first);\n }\n\n void SVEScatterStoreVectorPlusImm(SubRegSize esize, SubRegSize msize, ZRegister zt, PRegister pg, SVEMemOperand mem_op) {\n SVEGatherScatterVectorPlusImm(esize, msize, zt, pg, mem_op, true, false, true);\n }\n\n void SVEUnsizedLoadStoreContiguous(uint32_t op2, int32_t imm, ZRegister zt, Register rn, bool is_store) {\n LOGMAN_THROW_A_FMT(imm >= -256 && imm <= 255, \"Immediate offset ({}) too large. Must be within [-256, 255].\", imm);\n\n const auto imm9 = static_cast(imm) & 0b1'1111'1111;\n\n uint32_t Instr = 0b1000'0101'1000'0000'0000'0000'0000'0000;\n\n if (is_store) {\n Instr |= 0x60000000U;\n }\n\n Instr |= (imm9 >> 3) << 16;\n Instr |= op2 << 14;\n Instr |= (imm9 & 0b111) << 10;\n Instr |= rn.Idx() << 5;\n Instr |= zt.Idx();\n\n dc32(Instr);\n }\n\n // SVE load/store multiple structures (scalar plus immediate)\n void SVEContiguousMultipleStructures(int32_t num_regs, bool is_store, uint32_t msz, int32_t imm, ZRegister zt, PRegister pg, Register rn) {\n LOGMAN_THROW_A_FMT(pg <= PReg::p7, \"Can only use p0-p7 as a governing predicate\");\n LOGMAN_THROW_A_FMT((imm % num_regs) == 0, \"Offset must be a multiple of {}\", num_regs);\n\n const auto min_offset = -8 * num_regs;\n const auto max_offset = 7 * num_regs;\n LOGMAN_THROW_A_FMT(imm >= min_offset && imm <= max_offset,\n \"Invalid load/store offset ({}). Offset must be a multiple of {} and be within [{}, {}]\", imm, num_regs, min_offset,\n max_offset);\n\n const auto imm4 = static_cast(imm / num_regs) & 0xF;\n const auto opc = static_cast(num_regs - 1);\n\n uint32_t Instr = 0b1010'0100'0000'0000'1110'0000'0000'0000;\n Instr |= msz << 23;\n Instr |= opc << 21;\n Instr |= imm4 << 16;\n Instr |= pg.Idx() << 10;\n Instr |= Encode_rn(rn);\n Instr |= zt.Idx();\n if (is_store) {\n Instr |= 0x40100000U;\n }\n dc32(Instr);\n }\n\n // SVE contiguous non-temporal load (scalar plus immediate)\n void SVEContiguousNontemporalLoad(uint32_t msz, ZRegister zt, PRegister pg, Register rn, int32_t imm) {\n LOGMAN_THROW_A_FMT(pg <= PReg::p7, \"Can only use p0-p7 as a governing predicate\");\n LOGMAN_THROW_A_FMT(imm >= -8 && imm <= 7, \"Invalid loadstore offset ({}). Must be between [-8, 7]\", imm);\n\n const auto imm4 = static_cast(imm) & 0xF;\n uint32_t Instr = 0b1010'0100'0000'0000'1110'0000'0000'0000;\n Instr |= msz << 23;\n Instr |= imm4 << 16;\n Instr |= pg.Idx() << 10;\n Instr |= Encode_rn(rn);\n Instr |= zt.Idx();\n dc32(Instr);\n }\n\n // SVE contiguous non-temporal store (scalar plus immediate)\n void SVEContiguousNontemporalStore(uint32_t msz, ZRegister zt, PRegister pg, Register rn, int32_t imm) {\n LOGMAN_THROW_A_FMT(pg <= PReg::p7, \"Can only use p0-p7 as a governing predicate\");\n LOGMAN_THROW_A_FMT(imm >= -8 && imm <= 7, \"Invalid loadstore offset ({}). Must be between [-8, 7]\", imm);\n\n const auto imm4 = static_cast(imm) & 0xF;\n uint32_t Instr = 0b1110'0100'0001'0000'1110'0000'0000'0000;\n Instr |= msz << 23;\n Instr |= imm4 << 16;\n Instr |= pg.Idx() << 10;\n Instr |= Encode_rn(rn);\n Instr |= zt.Idx();\n dc32(Instr);\n }\n\n void SVEContiguousLoadImm(bool is_store, uint32_t dtype, int32_t imm, PRegister pg, Register rn, ZRegister zt) {\n LOGMAN_THROW_A_FMT(pg <= PReg::p7, \"Can only use p0-p7 as a governing predicate\");\n LOGMAN_THROW_A_FMT(imm >= -8 && imm <= 7, \"Invalid loadstore offset ({}). Must be between [-8, 7]\", imm);\n\n const auto imm4 = static_cast(imm) & 0xF;\n\n uint32_t Instr = 0b1010'0100'0000'0000'1010'0000'0000'0000;\n Instr |= dtype << 21;\n Instr |= imm4 << 16;\n Instr |= pg.Idx() << 10;\n Instr |= Encode_rn(rn);\n Instr |= zt.Idx();\n if (is_store) {\n Instr |= 0x40004000U;\n }\n dc32(Instr);\n }\n\n // zt.b, pg/z, xn, xm\n void SVEContiguousLoadStore(uint32_t b30, uint32_t b13, uint32_t dtype, Register rm, PRegister pg, Register rn, ZRegister zt) {\n LOGMAN_THROW_A_FMT(pg <= PReg::p7, \"Can only use p0-p7 as a governing predicate\");\n\n uint32_t Instr = 0b1010'0100'0000'0000'0100'0000'0000'0000;\n Instr |= b30 << 30;\n Instr |= dtype << 21;\n Instr |= Encode_rm(rm);\n Instr |= b13 << 13;\n Instr |= pg.Idx() << 10;\n Instr |= Encode_rn(rn);\n Instr |= zt.Idx();\n dc32(Instr);\n }\n\n void SVEContiguousLoadStoreMultipleScalar(bool is_store, SubRegSize msz, uint32_t opc, ZRegister zt, PRegister pg, Register rn, Register rm) {\n LOGMAN_THROW_A_FMT(pg <= PReg::p7, \"Can only use p0-p7 as a governing predicate\");\n LOGMAN_THROW_A_FMT(rm != Reg::rsp, \"rm cannot be the stack pointer\");\n\n uint32_t Instr = 0b1010'0100'0000'0000'0000'0000'0000'0000;\n if (is_store) {\n Instr |= 0x40006000U;\n } else {\n Instr |= 0x0000C000U;\n }\n Instr |= FEXCore::ToUnderlying(msz) << 23;\n Instr |= opc << 21;\n Instr |= rm.Idx() << 16;\n Instr |= pg.Idx() << 10;\n Instr |= rn.Idx() << 5;\n Instr |= zt.Idx();\n dc32(Instr);\n }\n\n void SVELoadBroadcastQuadScalarPlusImm(uint32_t msz, uint32_t ssz, ZRegister zt, PRegister pg, Register rn, int imm) {\n LOGMAN_THROW_A_FMT(pg <= PReg::p7, \"Can only use p0-p7 as a governing predicate\");\n\n const auto esize = static_cast(16 << ssz);\n const auto max_imm = (esize << 3) - esize;\n const auto min_imm = -(max_imm + esize);\n\n LOGMAN_THROW_A_FMT((imm % esize) == 0, \"imm ({}) must be a multiple of {}\", imm, esize);\n LOGMAN_THROW_A_FMT(imm >= min_imm && imm <= max_imm, \"imm ({}) must be within [{}, {}]\", imm, min_imm, max_imm);\n\n const auto sanitized_imm = static_cast(imm / esize) & 0b1111;\n\n uint32_t Instr = 0b1010'0100'0000'0000'0010'0000'0000'0000;\n Instr |= msz << 23;\n Instr |= ssz << 21;\n Instr |= sanitized_imm << 16;\n Instr |= pg.Idx() << 10;\n Instr |= rn.Idx() << 5;\n Instr |= zt.Idx();\n dc32(Instr);\n }\n\n void SVELoadBroadcastQuadScalarPlusScalar(uint32_t msz, uint32_t ssz, ZRegister zt, PRegister pg, Register rn, Register rm) {\n LOGMAN_THROW_A_FMT(pg <= PReg::p7, \"Can only use p0-p7 as a governing predicate\");\n LOGMAN_THROW_A_FMT(rm != Reg::rsp, \"rm may not be the stack pointer\");\n\n uint32_t Instr = 0b1010'0100'0000'0000'0000'0000'0000'0000;\n Instr |= msz << 23;\n Instr |= ssz << 21;\n Instr |= rm.Idx() << 16;\n Instr |= pg.Idx() << 10;\n Instr |= rn.Idx() << 5;\n Instr |= zt.Idx();\n dc32(Instr);\n }\n\n void SVELoadAndBroadcastElement(bool is_signed, SubRegSize esize, SubRegSize msize, ZRegister zt, PRegister pg, Register rn, uint32_t imm) {\n LOGMAN_THROW_A_FMT(esize != SubRegSize::i128Bit, \"Cannot use 128-bit elements.\");\n LOGMAN_THROW_A_FMT(pg <= PReg::p7, \"Can only use p0-p7 as a governing predicate\");\n if (is_signed) {\n // The element size needs to be larger than memory size, otherwise you tell\n // me how we're gonna sign extend this bad boy in memory.\n LOGMAN_THROW_A_FMT(esize > msize, \"Signed broadcast element size must be greater than memory size.\");\n }\n\n const auto esize_value = FEXCore::ToUnderlying(esize);\n const auto msize_value = FEXCore::ToUnderlying(msize);\n\n const auto data_size_bytes = 1U << msize_value;\n const auto max_imm = (64U << msize_value) - data_size_bytes;\n LOGMAN_THROW_A_FMT((imm % data_size_bytes) == 0 && imm <= max_imm, \"imm must be a multiple of {} and be within [0, {}]\",\n data_size_bytes, max_imm);\n\n const auto sanitized_imm = imm / data_size_bytes;\n\n auto dtypeh = msize_value;\n auto dtypel = esize_value;\n if (is_signed) {\n // Signed forms of the broadcast instructions are encoded in such a way\n // that msize will always be greater than esize, which, conveniently,\n // works out by just XORing the would-be unsigned dtype values by 3.\n dtypeh ^= 0b11;\n dtypel ^= 0b11;\n }\n // Guards against bogus combinations of element size and memory size values\n // being passed in. Unsigned variants will always have dtypeh be less than\n // or equal to dtypel. The only time this isn't the case is with signed variants.\n LOGMAN_THROW_A_FMT(is_signed == (dtypeh > dtypel),\n \"Invalid element size used with load broadcast instruction \"\n \"(esize: {}, msize: {})\",\n esize_value, msize_value);\n\n uint32_t Instr = 0b1000'0100'0100'0000'1000'0000'0000'0000;\n Instr |= dtypeh << 23;\n Instr |= sanitized_imm << 16;\n Instr |= dtypel << 13;\n Instr |= pg.Idx() << 10;\n Instr |= rn.Idx() << 5;\n Instr |= zt.Idx();\n dc32(Instr);\n }\n\n void SVEIndexGeneration(uint32_t op, SubRegSize size, ZRegister zd, int32_t imm5, int32_t imm5b) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i128Bit, \"INDEX cannot use 128-bit element sizes\");\n\n uint32_t Instr = 0b0000'0100'0010'0000'0100'0000'0000'0000;\n Instr |= op << 10;\n Instr |= FEXCore::ToUnderlying(size) << 22;\n Instr |= (static_cast(imm5b) & 0b11111) << 16;\n Instr |= (static_cast(imm5) & 0b11111) << 5;\n Instr |= zd.Idx();\n dc32(Instr);\n }\n\n void SVEIntegerCompareImm(uint32_t lt, uint32_t ne, uint32_t imm7, SubRegSize size, PRegister pg, ZRegister zn, PRegister pd) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i128Bit, \"Can't use 128-bit size\");\n LOGMAN_THROW_A_FMT(imm7 < 128, \"Invalid imm ({}). Must be within [0, 128]\", imm7);\n LOGMAN_THROW_A_FMT(pg <= PReg::p7, \"Can only use p0-p7 as a governing predicate\");\n\n uint32_t Instr = 0b0010'0100'0010'0000'0000'0000'0000'0000;\n Instr |= FEXCore::ToUnderlying(size) << 22;\n Instr |= imm7 << 14;\n Instr |= lt << 13;\n Instr |= pg.Idx() << 10;\n Instr |= zn.Idx() << 5;\n Instr |= ne << 4;\n Instr |= pd.Idx();\n dc32(Instr);\n }\n\n void SVEIntegerCompareSignedImm(uint32_t op, uint32_t o2, uint32_t ne, int32_t imm5, SubRegSize size, PRegister pg, ZRegister zn, PRegister pd) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i128Bit, \"Can't use 128-bit size\");\n LOGMAN_THROW_A_FMT(imm5 >= -16 && imm5 <= 15, \"Invalid imm ({}). Must be within [-16, 15].\", imm5);\n LOGMAN_THROW_A_FMT(pg <= PReg::p7, \"Can only use p0-p7 as a governing predicate\");\n\n uint32_t Instr = 0b0010'0101'0000'0000'0000'0000'0000'0000;\n Instr |= FEXCore::ToUnderlying(size) << 22;\n Instr |= (static_cast(imm5) & 0b1'1111) << 16;\n Instr |= op << 15;\n Instr |= o2 << 13;\n Instr |= pg.Idx() << 10;\n Instr |= zn.Idx() << 5;\n Instr |= ne << 4;\n Instr |= pd.Idx();\n dc32(Instr);\n }\n\n void SVEFloatCompareVector(uint32_t op, uint32_t o2, uint32_t o3, SubRegSize size, ZRegister zm, PRegister pg, ZRegister zn, PRegister pd) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i128Bit, \"Can't use 128-bit size\");\n LOGMAN_THROW_A_FMT(size != SubRegSize::i8Bit, \"Can't use 8-bit size\");\n LOGMAN_THROW_A_FMT(pg <= PReg::p7, \"Can only use p0-p7 as a governing predicate\");\n\n uint32_t Instr = 0b0110'0101'0000'0000'0100'0000'0000'0000;\n Instr |= FEXCore::ToUnderlying(size) << 22;\n Instr |= zm.Idx() << 16;\n Instr |= op << 15;\n Instr |= o2 << 13;\n Instr |= pg.Idx() << 10;\n Instr |= zn.Idx() << 5;\n Instr |= o3 << 4;\n Instr |= pd.Idx();\n dc32(Instr);\n }\n\n void SVEIntegerMinMaxDifferencePredicated(uint32_t opc, uint32_t U, SubRegSize size, PRegister pg, ZRegister zdn, ZRegister zm, ZRegister zd) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i128Bit, \"Can't use 128-bit size\");\n LOGMAN_THROW_A_FMT(zd == zdn, \"zd needs to equal zdn\");\n LOGMAN_THROW_A_FMT(pg <= PReg::p7, \"Can only use p0-p7 as a governing predicate\");\n\n uint32_t Instr = 0b0000'0100'0000'1000'0000'0000'0000'0000;\n Instr |= FEXCore::ToUnderlying(size) << 22;\n Instr |= opc << 17;\n Instr |= U << 16;\n Instr |= pg.Idx() << 10;\n Instr |= zm.Idx() << 5;\n Instr |= zd.Idx();\n dc32(Instr);\n }\n\n void SVEBitWiseShiftImmediatePred(SubRegSize size, uint32_t opc, uint32_t L, uint32_t U, PRegister pg, ZRegister zd, ZRegister zdn,\n uint32_t Shift) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i128Bit, \"Can't use 128-bit element size\");\n LOGMAN_THROW_A_FMT(zd == zdn, \"zd needs to equal zdn\");\n LOGMAN_THROW_A_FMT(pg <= PReg::p7, \"Can only use p0-p7 as a governing predicate\");\n\n const bool IsLeftShift = L != 0;\n const auto [tszh, tszl_imm3] = EncodeSVEShiftImmediate(size, Shift, IsLeftShift);\n\n uint32_t Instr = 0b0000'0100'0000'0000'1000'0000'0000'0000;\n Instr |= tszh << 22;\n Instr |= opc << 18;\n Instr |= L << 17;\n Instr |= U << 16;\n Instr |= pg.Idx() << 10;\n Instr |= tszl_imm3 << 5;\n Instr |= zd.Idx();\n dc32(Instr);\n }\n\n void SVEBitWiseShiftImmediateUnpred(SubRegSize size, uint32_t opc, ZRegister zd, ZRegister zn, uint32_t Shift) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i128Bit, \"Can't use 128-bit element size\");\n\n const bool IsLeftShift = opc == 0b11;\n const auto [tszh, tszl_imm3] = EncodeSVEShiftImmediate(size, Shift, IsLeftShift);\n\n uint32_t Instr = 0b0000'0100'0010'0000'1001'0000'0000'0000;\n Instr |= tszh << 22;\n Instr |= tszl_imm3 << 16;\n Instr |= opc << 10;\n Instr |= zn.Idx() << 5;\n Instr |= zd.Idx();\n dc32(Instr);\n }\n\n void SVE2BitwiseTernary(uint32_t opc, uint32_t o2, ZRegister zm, ZRegister zk, ZRegister zd, ZRegister zdn) {\n LOGMAN_THROW_A_FMT(zd == zdn, \"zd needs to equal zdn\");\n\n uint32_t Instr = 0b0000'0100'0010'0000'0011'1000'0000'0000;\n Instr |= opc << 22;\n Instr |= zm.Idx() << 16;\n Instr |= o2 << 10;\n Instr |= zk.Idx() << 5;\n Instr |= zdn.Idx();\n dc32(Instr);\n }\n\n void SVEPermuteVector(uint32_t op0, ARMEmitter::ZRegister zd, ARMEmitter::ZRegister zm, uint32_t Imm) {\n constexpr uint32_t Op = 0b0000'0101'0010'0000'000 << 13;\n uint32_t Instr = Op;\n\n Instr |= op0 << 22;\n Instr |= (Imm >> 3) << 16;\n Instr |= (Imm & 0b111) << 10;\n Instr |= zm.Idx() << 5;\n Instr |= zd.Idx();\n dc32(Instr);\n }\n\n void SVEIntegerCompareVector(uint32_t op, uint32_t o2, uint32_t ne, SubRegSize size, ZRegister zm, PRegister pg, ZRegister zn, PRegister pd) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i128Bit, \"Can't use 128-bit element size\");\n LOGMAN_THROW_A_FMT(pg <= PReg::p7, \"Can only use p0-p7 as a governing predicate\");\n\n constexpr uint32_t Op = 0b0010'0100'0000'0000'000 << 13;\n uint32_t Instr = Op;\n\n Instr |= FEXCore::ToUnderlying(size) << 22;\n Instr |= zm.Idx() << 16;\n Instr |= op << 15;\n Instr |= o2 << 13;\n Instr |= pg.Idx() << 10;\n Instr |= zn.Idx() << 5;\n Instr |= ne << 4;\n Instr |= pd.Idx();\n dc32(Instr);\n }\n\n void SVEIntegerCompareVectorWide(uint32_t op, uint32_t o2, uint32_t ne, SubRegSize size, PRegister pd, PRegister pg, ZRegister zn, ZRegister zm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i64Bit, \"Can't use 64-bit element size\");\n SVEIntegerCompareVector(op, o2, ne, size, zm, pg, zn, pd);\n }\n\n void SVE2SaturatingExtractNarrow(SubRegSize size, uint32_t opc, uint32_t T, ZRegister zn, ZRegister zd) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i128Bit && size != SubRegSize::i64Bit, \"Can't use 64/128-bit size\");\n\n // While not necessarily a left shift, we can piggyback off its\n // encoding behavior to encode the tszh and tszl bits.\n const auto [tszh, tszl_imm3] = EncodeSVEShiftImmediate(size, 0, true);\n\n uint32_t Instr = 0b0100'0101'0010'0000'0100'0000'0000'0000;\n Instr |= tszh << 22;\n Instr |= tszl_imm3 << 16;\n Instr |= opc << 11;\n Instr |= T << 10;\n Instr |= zn.Idx() << 5;\n Instr |= zd.Idx();\n dc32(Instr);\n }\n\n void SVE2BitwiseShiftRightNarrow(SubRegSize size, uint32_t shift, uint32_t opc, uint32_t U, uint32_t R, uint32_t T, ZRegister zn, ZRegister zd) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i128Bit && size != SubRegSize::i64Bit, \"Can't use 64/128-bit element size\");\n\n const auto [tszh, tszl_imm3] = EncodeSVEShiftImmediate(size, shift);\n\n uint32_t Instr = 0b0100'0101'0010'0000'0000'0000'0000'0000;\n Instr |= tszh << 22;\n Instr |= tszl_imm3 << 16;\n Instr |= opc << 13;\n Instr |= U << 12;\n Instr |= R << 11;\n Instr |= T << 10;\n Instr |= zn.Idx() << 5;\n Instr |= zd.Idx();\n dc32(Instr);\n }\n\n void SVEFloatUnary(uint32_t opc, SubRegSize size, PRegister pg, ZRegister zn, ZRegister zd) {\n LOGMAN_THROW_A_FMT(pg <= PReg::p7, \"Can only use p0-p7 as a governing predicate\");\n LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), \"Unsupported size in {}\", __func__);\n\n uint32_t Instr = 0b0110'0101'0000'1100'1010'0000'0000'0000;\n Instr |= FEXCore::ToUnderlying(size) << 22;\n Instr |= opc << 16;\n Instr |= pg.Idx() << 10;\n Instr |= zn.Idx() << 5;\n Instr |= zd.Idx();\n dc32(Instr);\n }\n\n void SVE2IntegerMultiplyVectors(uint32_t opc, SubRegSize size, ZRegister zm, ZRegister zn, ZRegister zd) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i128Bit, \"Can't use 128-bit size\");\n\n constexpr uint32_t Op = 0b0000'0100'0010'0000'0110 << 12;\n uint32_t Instr = Op;\n\n Instr |= FEXCore::ToUnderlying(size) << 22;\n Instr |= zm.Idx() << 16;\n Instr |= opc << 10;\n Instr |= zn.Idx() << 5;\n Instr |= zd.Idx();\n dc32(Instr);\n }\n\n void SVEPermuteVectorPredicated(uint32_t opc1, uint32_t opc2, SubRegSize size, ZRegister zd, PRegister pg, ZRegister zn) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i128Bit, \"Can't use 128-bit size\");\n LOGMAN_THROW_A_FMT(pg <= PReg::p7, \"Can only use p0-p7 as a governing predicate\");\n\n uint32_t Instr = 0b0000'0101'0010'0000'1000'0000'0000'0000;\n Instr |= FEXCore::ToUnderlying(size) << 22;\n Instr |= opc1 << 16;\n Instr |= opc2 << 13;\n Instr |= pg.Idx() << 10;\n Instr |= zn.Idx() << 5;\n Instr |= zd.Idx();\n dc32(Instr);\n }\n\n void SVEPropagateBreak(uint32_t opc, uint32_t op2, uint32_t op3, PRegister pd, PRegister pg, PRegister pn, PRegister pm) {\n uint32_t Instr = 0b0010'0101'0000'0000'0000'0000'0000'0000;\n Instr |= opc << 20;\n Instr |= op2 << 14;\n Instr |= op3 << 4;\n Instr |= pm.Idx() << 16;\n Instr |= pg.Idx() << 10;\n Instr |= pn.Idx() << 5;\n Instr |= pd.Idx();\n dc32(Instr);\n }\n\n void SVEPredicateMisc(uint32_t op0, uint32_t op2, uint32_t op3, SubRegSize size, PRegister pd) {\n // Note: op2 combines op1 like [op1:op2], since they're adjacent.\n LOGMAN_THROW_A_FMT(size != SubRegSize::i128Bit, \"Can't use 128-bit size\");\n\n uint32_t Instr = 0b0010'0101'0001'0000'1100'0000'0000'0000;\n Instr |= FEXCore::ToUnderlying(size) << 22;\n Instr |= op0 << 16;\n Instr |= op2 << 9;\n Instr |= op3 << 5;\n Instr |= pd.Idx();\n dc32(Instr);\n }\n\n void SVEIntCompareScalar(uint32_t op1, uint32_t b4, uint32_t op2, SubRegSize size, Register rn, Register rm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i128Bit, \"Can't use 128-bit size\");\n\n uint32_t Instr = 0b0010'0101'0010'0000'0000'0000'0000'0000;\n Instr |= FEXCore::ToUnderlying(size) << 22;\n Instr |= rm.Idx() << 16;\n Instr |= op1 << 10;\n Instr |= rn.Idx() << 5;\n Instr |= b4 << 4;\n Instr |= op2;\n dc32(Instr);\n }\n\n void SVEWriteFFR(uint32_t op0, uint32_t op1, uint32_t op2, uint32_t op3, uint32_t op4) {\n uint32_t Instr = 0b0010'0101'0010'1000'1001'0000'0000'0000;\n Instr |= op0 << 18;\n Instr |= op1 << 16;\n Instr |= op2 << 9;\n Instr |= op3 << 5;\n Instr |= op4;\n dc32(Instr);\n }\n\n void SVEFPUnaryOpsUnpredicated(uint32_t opc, SubRegSize size, ZRegister zd, ZRegister zn) {\n LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), \"SubRegSize must be 16-bit, 32-bit, or 64-bit\");\n\n uint32_t Instr = 0b0110'0101'0000'1000'0011'0000'0000'0000;\n Instr |= FEXCore::ToUnderlying(size) << 22;\n Instr |= opc << 16;\n Instr |= zn.Idx() << 5;\n Instr |= zd.Idx();\n dc32(Instr);\n }\n\n void SVEFPSerialReductionPredicated(uint32_t opc, SubRegSize size, VRegister vd, PRegister pg, VRegister vn, ZRegister zm) {\n LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), \"SubRegSize must be 16-bit, 32-bit, or 64-bit\");\n LOGMAN_THROW_A_FMT(pg <= PReg::p7, \"Can only use p0-p7 as a governing predicate\");\n LOGMAN_THROW_A_FMT(vd == vn, \"vn must be the same as vd\");\n\n uint32_t Instr = 0b0110'0101'0001'1000'0010'0000'0000'0000;\n Instr |= FEXCore::ToUnderlying(size) << 22;\n Instr |= opc << 16;\n Instr |= pg.Idx() << 10;\n Instr |= zm.Idx() << 5;\n Instr |= vd.Idx();\n dc32(Instr);\n }\n\n void SVEFPCompareWithZero(uint32_t eqlt, uint32_t ne, SubRegSize size, PRegister pd, PRegister pg, ZRegister zn) {\n LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), \"SubRegSize must be 16-bit, 32-bit, or 64-bit\");\n LOGMAN_THROW_A_FMT(pg <= PReg::p7, \"Can only use p0-p7 as a governing predicate\");\n\n uint32_t Instr = 0b0110'0101'0001'0000'0010'0000'0000'0000;\n Instr |= FEXCore::ToUnderlying(size) << 22;\n Instr |= eqlt << 16;\n Instr |= pg.Idx() << 10;\n Instr |= zn.Idx() << 5;\n Instr |= ne << 4;\n Instr |= pd.Idx();\n dc32(Instr);\n }\n\n void SVEFPMultiplyAdd(uint32_t opc, SubRegSize size, ZRegister zd, PRegister pg, ZRegister zn, ZRegister zm) {\n // NOTE: opc also includes the op0 bit (bit 15) like op0:opc, since the fields are adjacent\n LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), \"SubRegSize must be 16-bit, 32-bit, or 64-bit\");\n LOGMAN_THROW_A_FMT(pg <= PReg::p7, \"Can only use p0-p7 as a governing predicate\");\n\n uint32_t Instr = 0b0110'0101'0010'0000'0000'0000'0000'0000;\n Instr |= FEXCore::ToUnderlying(size) << 22;\n Instr |= zm.Idx() << 16;\n Instr |= opc << 13;\n Instr |= pg.Idx() << 10;\n Instr |= zn.Idx() << 5;\n Instr |= zd.Idx();\n dc32(Instr);\n }\n\n void SVEFPMultiplyAddIndexed(uint32_t op, SubRegSize size, ZRegister zda, ZRegister zn, ZRegister zm, uint32_t index) {\n LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), \"SubRegSize must be 16-bit, 32-bit, or 64-bit\");\n LOGMAN_THROW_A_FMT((size <= SubRegSize::i32Bit && zm <= ZReg::z7) || (size == SubRegSize::i64Bit && zm <= ZReg::z15),\n \"16-bit and 32-bit indexed variants may only use Zm between z0-z7\\n\"\n \"64-bit variants may only use Zm between z0-z15\");\n\n const auto Underlying = FEXCore::ToUnderlying(size);\n const uint32_t IndexMax = (16 / (1U << Underlying)) - 1;\n LOGMAN_THROW_A_FMT(index <= IndexMax, \"Index must be within 0-{}\", IndexMax);\n\n // Can be bit 20 or 19 depending on whether or not the element size is 64-bit.\n const auto IndexShift = 19 + static_cast(size == SubRegSize::i64Bit);\n\n uint32_t Instr = 0b0110'0100'0010'0000'0000'0000'0000'0000;\n Instr |= Underlying << 22;\n Instr |= (index & 0b1000) << 19;\n Instr |= (index & 0b0111) << IndexShift;\n Instr |= zm.Idx() << 16;\n Instr |= op << 10;\n Instr |= zn.Idx() << 5;\n Instr |= zda.Idx();\n dc32(Instr);\n }\n\n void SVEFPMultiplyAddLongIndexed(uint32_t o2, uint32_t op, uint32_t T, SubRegSize dstsize, ZRegister zda, ZRegister zn, ZRegister zm,\n uint32_t index) {\n LOGMAN_THROW_A_FMT(dstsize == SubRegSize::i32Bit, \"Destination size must be 32-bit.\");\n LOGMAN_THROW_A_FMT(index <= 7, \"Index ({}) must be within [0, 7]\", index);\n LOGMAN_THROW_A_FMT(zm <= ZReg::z7, \"zm (z{}) must be within [z0, z7]\", zm.Idx());\n\n uint32_t Inst = 0b0110'0100'1010'0000'0100'0000'0000'0000;\n Inst |= o2 << 22;\n Inst |= (index & 0b110) << 18;\n Inst |= zm.Idx() << 16;\n Inst |= op << 13;\n Inst |= (index & 0b001) << 11;\n Inst |= T << 10;\n Inst |= zn.Idx() << 5;\n Inst |= zda.Idx();\n dc32(Inst);\n }\n\n void SVEFPMultiplyAddLong(uint32_t o2, uint32_t op, uint32_t T, SubRegSize dstsize, ZRegister zda, ZRegister zn, ZRegister zm) {\n LOGMAN_THROW_A_FMT(dstsize == SubRegSize::i32Bit, \"Destination size must be 32-bit.\");\n\n uint32_t Instr = 0b0110'0100'1010'0000'1000'0000'0000'0000;\n Instr |= o2 << 22;\n Instr |= zm.Idx() << 16;\n Instr |= op << 13;\n Instr |= T << 10;\n Instr |= zn.Idx() << 5;\n Instr |= zda.Idx();\n dc32(Instr);\n }\n\n void SVEFPMatrixMultiplyAccumulate(SubRegSize size, ZRegister zda, ZRegister zn, ZRegister zm) {\n LOGMAN_THROW_A_FMT(size == SubRegSize::i32Bit || size == SubRegSize::i64Bit, \"SubRegSize must be 32-bit or 64-bit\");\n\n uint32_t Instr = 0b0110'0100'0010'0000'1110'0100'0000'0000;\n Instr |= FEXCore::ToUnderlying(size) << 22;\n Instr |= zm.Idx() << 16;\n Instr |= zn.Idx() << 5;\n Instr |= zda.Idx();\n dc32(Instr);\n }\n\n void SVEPredicateCount(uint32_t opc, SubRegSize size, XRegister rd, PRegister pg, PRegister pn) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i128Bit, \"Cannot use 128-bit element size\");\n\n uint32_t Instr = 0b0010'0101'0010'0000'1000'0000'0000'0000;\n Instr |= FEXCore::ToUnderlying(size) << 22;\n Instr |= opc << 16;\n Instr |= pg.Idx() << 10;\n Instr |= pn.Idx() << 5;\n Instr |= rd.Idx();\n\n dc32(Instr);\n }\n\n void SVEElementCount(uint32_t b20, uint32_t op1, SubRegSize size, ZRegister zdn, PredicatePattern pattern, uint32_t imm4) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i128Bit, \"Cannot use 128-bit element size\");\n LOGMAN_THROW_A_FMT(imm4 >= 1 && imm4 <= 16, \"Immediate must be between 1-16 inclusive\");\n\n uint32_t Instr = 0b0000'0100'0010'0000'1100'0000'0000'0000;\n Instr |= FEXCore::ToUnderlying(size) << 22;\n Instr |= b20 << 20;\n Instr |= (imm4 - 1) << 16;\n Instr |= op1 << 10;\n Instr |= FEXCore::ToUnderlying(pattern) << 5;\n Instr |= zdn.Idx();\n dc32(Instr);\n }\n\n void SVEIncDecPredicateCountScalar(uint32_t op0, uint32_t op1, uint32_t opc, uint32_t b16, SubRegSize size, Register rdn, PRegister pm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i128Bit, \"Cannot use 128-bit element size\");\n\n uint32_t Instr = 0b0010'0101'0010'1000'1000'0000'0000'0000;\n Instr |= FEXCore::ToUnderlying(size) << 22;\n Instr |= op0 << 18;\n Instr |= b16 << 16;\n Instr |= op1 << 11;\n Instr |= opc << 9;\n Instr |= pm.Idx() << 5;\n Instr |= rdn.Idx();\n dc32(Instr);\n }\n void SVEIncDecPredicateCountVector(uint32_t op0, uint32_t op1, uint32_t opc, uint32_t b16, SubRegSize size, ZRegister zdn, PRegister pm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i8Bit, \"Cannot use 8-bit element size\");\n SVEIncDecPredicateCountScalar(op0, op1, opc, b16, size, Register {zdn.Idx()}, pm);\n }\n\n void SVE2IntegerPredicated(uint32_t op0, uint32_t op1, SubRegSize size, ZRegister zd, PRegister pg, ZRegister zn) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i128Bit, \"Cannot use 128-bit size\");\n LOGMAN_THROW_A_FMT(pg <= PReg::p7, \"Can only use p0-p7 as a governing predicate\");\n\n uint32_t Instr = 0b0100'0100'0000'0000'0000'0000'0000'0000;\n Instr |= FEXCore::ToUnderlying(size) << 22;\n Instr |= op0 << 16; // Intentionally 16 instead of 17 to handle bit range nicer\n Instr |= op1 << 13;\n Instr |= pg.Idx() << 10;\n Instr |= zn.Idx() << 5;\n Instr |= zd.Idx();\n dc32(Instr);\n }\n\n void SVE2IntegerPairwiseAddAccumulateLong(uint32_t U, SubRegSize size, ZRegister zda, PRegisterMerge pg, ZRegister zn) {\n LOGMAN_THROW_A_FMT(size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit, \"SubRegSize must be 16-bit, \"\n \"32-bit, or 64-bit\");\n SVE2IntegerPredicated((0b0010 << 1) | U, 0b101, size, zda, pg, zn);\n }\n\n void SVE2IntegerUnaryOpsPredicated(uint32_t op0, SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn) {\n SVE2IntegerPredicated(op0, 0b101, size, zd, pg, zn);\n }\n\n void SVE2SaturatingRoundingBitwiseShiftLeft(uint32_t op0, SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n LOGMAN_THROW_A_FMT(zd == zn, \"zn needs to equal zd\");\n SVE2IntegerPredicated(op0, 0b100, size, zd, pg, zm);\n }\n\n void SVE2IntegerHalvingPredicated(uint32_t RSU, SubRegSize size, PRegister pg, ZRegister zd, ZRegister zn, ZRegister zm) {\n LOGMAN_THROW_A_FMT(zd == zn, \"zn needs to equal zd\");\n SVE2IntegerPredicated((0b10 << 3) | RSU, 0b100, size, zd, pg, zm);\n }\n\n void SVEIntegerPairwiseArithmetic(uint32_t opc, uint32_t U, SubRegSize size, PRegister pg, ZRegister zd, ZRegister zn, ZRegister zm) {\n LOGMAN_THROW_A_FMT(zd == zn, \"zn needs to equal zd\");\n SVE2IntegerPredicated((0b10 << 3) | (opc << 1) | U, 0b101, size, zd, pg, zm);\n }\n\n void SVE2IntegerSaturatingAddSub(uint32_t opc, SubRegSize size, ZRegister zd, PRegisterMerge pg, ZRegister zn, ZRegister zm) {\n LOGMAN_THROW_A_FMT(zd == zn, \"zn needs to equal zd\");\n SVE2IntegerPredicated((0b11 << 3) | opc, 0b100, size, zd, pg, zm);\n }\n\n void SVEIntegerMultiplyAddUnpredicated(uint32_t op0, SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i128Bit, \"Cannot use 128-bit element size\");\n\n uint32_t Instr = 0b0100'0100'0000'0000'0000'0000'0000'0000;\n Instr |= FEXCore::ToUnderlying(size) << 22;\n Instr |= zm.Idx() << 16;\n Instr |= op0 << 10;\n Instr |= zn.Idx() << 5;\n Instr |= zd.Idx();\n dc32(Instr);\n }\n\n void SVEIntegerDotProduct(uint32_t op, SubRegSize size, ZRegister zda, ZRegister zn, ZRegister zm, Rotation rot) {\n LOGMAN_THROW_A_FMT(size == SubRegSize::i32Bit || size == SubRegSize::i64Bit, \"Dot product must only use 32-bit or 64-bit element \"\n \"sizes\");\n SVEIntegerComplexMulAdd(op, size, zda, zn, zm, rot);\n }\n\n void SVEIntegerComplexMulAdd(uint32_t op, SubRegSize size, ZRegister zda, ZRegister zn, ZRegister zm, Rotation rot) {\n const auto op0 = op << 2 | FEXCore::ToUnderlying(rot);\n SVEIntegerMultiplyAddUnpredicated(op0, size, zda, zn, zm);\n }\n\n void SVE2SaturatingMulAddInterleaved(uint32_t op0, SubRegSize size, ZRegister zda, ZRegister zn, ZRegister zm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i8Bit, \"Element size may only be 16-bit, 32-bit, or 64-bit\");\n SVEIntegerMultiplyAddUnpredicated(op0, size, zda, zn, zm);\n }\n\n void SVE2IntegerMulAddLong(uint32_t op0, SubRegSize size, ZRegister zda, ZRegister zn, ZRegister zm) {\n SVE2SaturatingMulAddInterleaved(op0, size, zda, zn, zm);\n }\n\n void SVE2WideningIntegerArithmetic(uint32_t op, uint32_t SUT, SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n uint32_t Instr = 0b0100'0101'0000'0000'0000'0000'0000'0000;\n Instr |= FEXCore::ToUnderlying(size) << 22;\n Instr |= zm.Idx() << 16;\n Instr |= op << 13;\n Instr |= SUT << 10;\n Instr |= zn.Idx() << 5;\n Instr |= zd.Idx();\n dc32(Instr);\n }\n\n void SVE2IntegerAddSubLong(uint32_t op, uint32_t SUT, SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i8Bit && size != SubRegSize::i128Bit, \"Can't use 8-bit or 128-bit element size\");\n SVE2WideningIntegerArithmetic(op, SUT, size, zd, zn, zm);\n }\n\n void SVE2IntegerAddSubWide(uint32_t SUT, SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i8Bit && size != SubRegSize::i128Bit, \"Can't use 8-bit or 128-bit element size\");\n SVE2WideningIntegerArithmetic(0b10, SUT, size, zd, zn, zm);\n }\n\n void SVE2IntegerMultiplyLong(uint32_t SUT, SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {\n // PMULLB and PMULLT support the use of 128-bit element sizes (with the SVE2PMULL128 extension)\n if (SUT == 0b010 || SUT == 0b011) {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i8Bit && size != SubRegSize::i32Bit, \"Can't use 8-bit or 32-bit element size\");\n\n // 128-bit variant is encoded as if it were 8-bit (0b00)\n if (size == SubRegSize::i128Bit) {\n size = SubRegSize::i8Bit;\n }\n } else {\n LOGMAN_THROW_A_FMT(size != SubRegSize::i8Bit && size != SubRegSize::i128Bit, \"Can't use 8-bit or 128-bit element size\");\n }\n\n SVE2WideningIntegerArithmetic(0b11, SUT, size, zd, zn, zm);\n }\n\n struct SVEEncodedImmShift {\n uint32_t tszh;\n uint32_t tszl_imm3;\n };\n // Helper for encoding shift immediates that make use of the tszh:tszl and imm3 field.\n static constexpr SVEEncodedImmShift EncodeSVEShiftImmediate(SubRegSize size, uint32_t shift, bool is_left_shift = false) {\n const uint32_t element_size = SubRegSizeInBits(size);\n\n if (is_left_shift) {\n LOGMAN_THROW_A_FMT(shift < element_size, \"Invalid left shift value ({}). Must be within [0, {}]\", shift, element_size - 1);\n } else {\n LOGMAN_THROW_A_FMT(shift > 0 && shift <= element_size, \"Invalid right shift value ({}). Must be within [1, {}]\", shift, element_size);\n }\n\n // Both left and right shifts encodes their shift as if it were\n // expanding the tszh:tszl (tsize) bits to the the left in order to accomodate\n // larger shift values. e.g. (B: tsize=0b0001, H: tsize=0b001x, etc)\n //\n // The difference is in how they're encoded. Left shifts are trivial and\n // encode as element_size_in_bits + shift, which works nicely since\n // the size will just occupy the next bit in tsize leaving the previous\n // one for encoding larger shifts.\n //\n // Right shifts instead encode it like a subtraction. e.g. A shift of 1\n // would encode like (S: tsize=0b0111 imm3=0b111, where 64 - 1 = 63, etc).\n // so the more lower in value the bits are set, the larger the shift.\n const uint32_t encoded_shift = is_left_shift ? element_size + shift : (2 * element_size) - shift;\n\n return {\n .tszh = encoded_shift >> 5,\n .tszl_imm3 = encoded_shift & 0b11111,\n };\n }\n\n // Alias that returns the equivalently sized unsigned type for a floating-point type T.\n template\n requires (std::is_same_v || std::is_same_v)\n using FloatToEquivalentUInt = std::conditional_t, uint32_t, uint64_t>;\n\n#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED\n // Determines if a floating-point value is capable of being converted\n // into an 8-bit immediate. See pseudocode definition of VFPExpandImm\n // in ARM A-profile reference manual for a general overview of how this was derived.\n template\n requires (std::is_same_v || std::is_same_v)\n [[nodiscard]]\n static bool IsValidFPValueForImm8(T value) {\n const uint64_t bits = std::bit_cast>(value);\n const uint64_t datasize_idx = FEXCore::ilog2(sizeof(T)) - 1;\n\n static constexpr std::array mantissa_masks {\n 0x00000000'0000003FULL, // half (bits [5:0])\n 0x00000000'0007FFFFULL, // single (bits [18:0])\n 0x0000FFFF'FFFFFFFFULL, // double (bits [47:0])\n };\n const auto mantissa_mask = mantissa_masks[datasize_idx];\n\n // Relevant mantissa bits must be set to zero\n if ((bits & mantissa_mask) != 0) {\n return false;\n }\n\n static constexpr std::array exponent_masks {\n 0x00000000'00003000ULL, // half (bits [13:12])\n 0x00000000'3E000000ULL, // single (bits [29:25])\n 0x3FC00000'00000000ULL, // double (bits [61:54])\n };\n const auto exponent_mask = exponent_masks[datasize_idx];\n const auto masked_exponent = bits & exponent_mask;\n\n // Relevant exponent bits must either be all set or all cleared.\n if (masked_exponent != 0 && masked_exponent != exponent_mask) {\n return false;\n }\n\n // The two bits before the sign bit must be inverses of each other.\n const auto datasize = 8ULL * sizeof(T);\n const auto inverse = bits ^ (bits << 1);\n const auto inverse_mask = 1ULL << (datasize - 2);\n if ((inverse & inverse_mask) == 0) {\n return false;\n }\n\n return true;\n }\n#endif\n\nprotected:\n static uint32_t FP32ToImm8(float value) {\n#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED\n LOGMAN_THROW_A_FMT(IsValidFPValueForImm8(value), \"Value ({}) cannot be encoded into an 8-bit immediate\", value);\n#endif\n\n const auto bits = std::bit_cast(value);\n const auto sign = (bits & 0x80000000) >> 24;\n const auto expb2 = (bits & 0x20000000) >> 23;\n const auto b5_to_0 = (bits >> 19) & 0x3F;\n\n return sign | expb2 | b5_to_0;\n }\n\n static uint32_t FP64ToImm8(double value) {\n#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED\n LOGMAN_THROW_A_FMT(IsValidFPValueForImm8(value), \"Value ({}) cannot be encoded into an 8-bit immediate\", value);\n#endif\n\n const auto bits = std::bit_cast(value);\n const auto sign = (bits & 0x80000000'00000000) >> 56;\n const auto expb2 = (bits & 0x20000000'00000000) >> 55;\n const auto b5_to_0 = (bits >> 48) & 0x3F;\n\n return static_cast(sign | expb2 | b5_to_0);\n }\n\nprivate:\n // Handling for signed 8-bit immediate shifts (e.g. in cpy/dup)\n struct HandledSImm8Shift {\n int32_t imm;\n uint32_t is_shift;\n };\n static constexpr HandledSImm8Shift HandleSVESImm8Shift(SubRegSize size, int32_t imm) {\n const int32_t imm8_limit = 128;\n const bool is_int8_imm = -imm8_limit <= imm && imm < imm8_limit;\n if (size == SubRegSize::i8Bit) {\n LOGMAN_THROW_A_FMT(is_int8_imm, \"Can't perform LSL #8 shift on 8-bit elements.\");\n }\n\n uint32_t shift = 0;\n if (!is_int8_imm) {\n const int32_t imm16_limit = 32768;\n const bool is_int16_imm = -imm16_limit <= imm && imm < imm16_limit;\n\n LOGMAN_THROW_A_FMT(is_int16_imm, \"Immediate ({}) must be a 16-bit value within [-32768, 32512]\", imm);\n LOGMAN_THROW_A_FMT((imm % 256) == 0, \"Immediate ({}) must be a multiple of 256\", imm);\n\n imm /= 256;\n shift = 1;\n }\n\n return {\n .imm = imm,\n .is_shift = shift,\n };\n }\n\n#ifndef INCLUDED_BY_EMITTER\n}; // struct LoadstoreEmitterOps\n} // namespace ARMEmitter\n#endif\n" }, { "path": "CodeEmitter/CodeEmitter/ScalarOps.inl", "content": "// SPDX-License-Identifier: MIT\n/* Scalar instruction emitters.\n *\n * These contain instruction emitters for scalar ASIMD operations explicitly.\n * Some of these emitter arguments might seem a bit strange at first glance,\n * but is because ARM's instruction encodings for these instructions are a hot mess.\n *\n * Specifically FP16 was an afterthought for these scalar operations, using a `ScalarRegSize` with\n * 16-bit wouldn't encode an FP16 instruction because they are a different instruction class instead.\n *\n * Most FP16 operations instead have their own freestanding implementation using `HRegister` arguments.\n *\n * Meanwhile other FP32 and FP64 instructions will use `ScalarRegSize`, supporting both those sizes.\n *\n * For Scalar integer operations, these instructions will mostly support all `ScalarRegSize` operations.\n * Exceptions to this rule will have asserts in the emitter implementation when misused.\n *\n */\n\n#pragma once\n#ifndef INCLUDED_BY_EMITTER\n#include \nnamespace ARMEmitter {\nstruct EmitterOps : Emitter {\n#endif\n\npublic:\n // Advanced SIMD scalar copy\n void dup(ScalarRegSize size, VRegister rd, VRegister rn, uint32_t Index) {\n const uint32_t SizeImm = FEXCore::ToUnderlying(size);\n const uint32_t IndexShift = SizeImm + 1;\n const uint32_t ElementSize = 1U << SizeImm;\n const uint32_t MaxIndex = 128U / (ElementSize * 8);\n\n LOGMAN_THROW_A_FMT(Index < MaxIndex, \"Index too large. Index={}, Max Index: {}\", Index, MaxIndex);\n\n const uint32_t imm5 = (Index << IndexShift) | ElementSize;\n\n ASIMDScalarCopy(1, 1, imm5, 0b0000, rd, rn);\n }\n\n void mov(ScalarRegSize size, VRegister rd, VRegister rn, uint32_t Index) {\n dup(size, rd, rn, Index);\n }\n\n // Advanced SIMD scalar three same FP16\n void fmulx(HRegister rd, HRegister rn, HRegister rm) {\n ASIMDScalarThreeSameFP16(0, 0, 0b011, rm, rn, rd);\n }\n void fcmeq(HRegister rd, HRegister rn, HRegister rm) {\n ASIMDScalarThreeSameFP16(0, 0, 0b100, rm, rn, rd);\n }\n void frecps(HRegister rd, HRegister rn, HRegister rm) {\n ASIMDScalarThreeSameFP16(0, 0, 0b111, rm, rn, rd);\n }\n void frsqrts(HRegister rd, HRegister rn, HRegister rm) {\n ASIMDScalarThreeSameFP16(0, 1, 0b111, rm, rn, rd);\n }\n void fcmge(HRegister rd, HRegister rn, HRegister rm) {\n ASIMDScalarThreeSameFP16(1, 0, 0b100, rm, rn, rd);\n }\n void facge(HRegister rd, HRegister rn, HRegister rm) {\n ASIMDScalarThreeSameFP16(1, 0, 0b101, rm, rn, rd);\n }\n void fabd(HRegister rd, HRegister rn, HRegister rm) {\n ASIMDScalarThreeSameFP16(1, 1, 0b010, rm, rn, rd);\n }\n void fcmgt(HRegister rd, HRegister rn, HRegister rm) {\n ASIMDScalarThreeSameFP16(1, 1, 0b100, rm, rn, rd);\n }\n void facgt(HRegister rd, HRegister rn, HRegister rm) {\n ASIMDScalarThreeSameFP16(1, 1, 0b101, rm, rn, rd);\n }\n\n // Advanced SIMD scalar two-register miscellaneous FP16\n void fcvtns(HRegister rd, HRegister rn) {\n ASIMDScalarTwoRegMiscFP16(0, 0, 0b11010, rn, rd);\n }\n void fcvtms(HRegister rd, HRegister rn) {\n ASIMDScalarTwoRegMiscFP16(0, 0, 0b11011, rn, rd);\n }\n void fcvtas(HRegister rd, HRegister rn) {\n ASIMDScalarTwoRegMiscFP16(0, 0, 0b11100, rn, rd);\n }\n void scvtf(HRegister rd, HRegister rn) {\n ASIMDScalarTwoRegMiscFP16(0, 0, 0b11101, rn, rd);\n }\n void fcmgt(HRegister rd, HRegister rn) {\n ASIMDScalarTwoRegMiscFP16(0, 1, 0b01100, rn, rd);\n }\n void fcmeq(HRegister rd, HRegister rn) {\n ASIMDScalarTwoRegMiscFP16(0, 1, 0b01101, rn, rd);\n }\n void fcmlt(HRegister rd, HRegister rn) {\n ASIMDScalarTwoRegMiscFP16(0, 1, 0b01110, rn, rd);\n }\n void fcvtps(HRegister rd, HRegister rn) {\n ASIMDScalarTwoRegMiscFP16(0, 1, 0b11010, rn, rd);\n }\n void fcvtzs(HRegister rd, HRegister rn) {\n ASIMDScalarTwoRegMiscFP16(0, 1, 0b11011, rn, rd);\n }\n void frecpe(HRegister rd, HRegister rn) {\n ASIMDScalarTwoRegMiscFP16(0, 1, 0b11101, rn, rd);\n }\n void frecpx(HRegister rd, HRegister rn) {\n ASIMDScalarTwoRegMiscFP16(0, 1, 0b11111, rn, rd);\n }\n void fcvtnu(HRegister rd, HRegister rn) {\n ASIMDScalarTwoRegMiscFP16(1, 0, 0b11010, rn, rd);\n }\n void fcvtmu(HRegister rd, HRegister rn) {\n ASIMDScalarTwoRegMiscFP16(1, 0, 0b11011, rn, rd);\n }\n void fcvtau(HRegister rd, HRegister rn) {\n ASIMDScalarTwoRegMiscFP16(1, 0, 0b11100, rn, rd);\n }\n void ucvtf(HRegister rd, HRegister rn) {\n ASIMDScalarTwoRegMiscFP16(1, 0, 0b11101, rn, rd);\n }\n void fcmge(HRegister rd, HRegister rn) {\n ASIMDScalarTwoRegMiscFP16(1, 1, 0b01100, rn, rd);\n }\n void fcmle(HRegister rd, HRegister rn) {\n ASIMDScalarTwoRegMiscFP16(1, 1, 0b01101, rn, rd);\n }\n void fcvtpu(HRegister rd, HRegister rn) {\n ASIMDScalarTwoRegMiscFP16(1, 1, 0b11010, rn, rd);\n }\n void fcvtzu(HRegister rd, HRegister rn) {\n ASIMDScalarTwoRegMiscFP16(1, 1, 0b11011, rn, rd);\n }\n void frsqrte(HRegister rd, HRegister rn) {\n ASIMDScalarTwoRegMiscFP16(1, 1, 0b11101, rn, rd);\n }\n\n // Advanced SIMD scalar three same extra\n void sqrdmlah(ScalarRegSize size, VRegister rd, VRegister rn, VRegister rm) {\n LOGMAN_THROW_A_FMT(size == ScalarRegSize::i16Bit || size == ScalarRegSize::i32Bit, \"Only supports 16/32-bit\");\n ASIMDScalarThreeSameExtra(1, size, 0b0000, rm, rn, rd);\n }\n void sqrdmlsh(ScalarRegSize size, VRegister rd, VRegister rn, VRegister rm) {\n LOGMAN_THROW_A_FMT(size == ScalarRegSize::i16Bit || size == ScalarRegSize::i32Bit, \"Only supports 16/32-bit\");\n ASIMDScalarThreeSameExtra(1, size, 0b0001, rm, rn, rd);\n }\n\n // Advanced SIMD scalar two-register miscellaneous\n void suqadd(ScalarRegSize size, VRegister rd, VRegister rn) {\n ASIMDScalar2RegMisc(0, 0, size, 0b00011, rd, rn);\n }\n void sqabs(ScalarRegSize size, VRegister rd, VRegister rn) {\n ASIMDScalar2RegMisc(0, 0, size, 0b00111, rd, rn);\n }\n\n ///< Comparison against 0.0\n void cmgt(ScalarRegSize size, VRegister rd, VRegister rn) {\n LOGMAN_THROW_A_FMT(size == ScalarRegSize::i64Bit, \"Only supports 64-bit\");\n ASIMDScalar2RegMisc(0, 0, size, 0b01000, rd, rn);\n }\n ///< Comparison against 0.0\n void cmeq(ScalarRegSize size, VRegister rd, VRegister rn) {\n LOGMAN_THROW_A_FMT(size == ScalarRegSize::i64Bit, \"Only supports 64-bit\");\n ASIMDScalar2RegMisc(0, 0, size, 0b01001, rd, rn);\n }\n\n ///< Comparison against 0.0\n void cmlt(ScalarRegSize size, VRegister rd, VRegister rn) {\n LOGMAN_THROW_A_FMT(size == ScalarRegSize::i64Bit, \"Only supports 64-bit\");\n ASIMDScalar2RegMisc(0, 0, size, 0b01010, rd, rn);\n }\n void abs(ScalarRegSize size, VRegister rd, VRegister rn) {\n LOGMAN_THROW_A_FMT(size == ScalarRegSize::i64Bit, \"Only supports 64-bit\");\n ASIMDScalar2RegMisc(0, 0, size, 0b01011, rd, rn);\n }\n ///< size is destination size.\n void sqxtn(ScalarRegSize size, VRegister rd, VRegister rn) {\n LOGMAN_THROW_A_FMT(size != ScalarRegSize::i64Bit, \"64-bit destination not supported\");\n ASIMDScalar2RegMisc(0, 0, size, 0b10100, rd, rn);\n }\n\n void fcvtns(ScalarRegSize size, VRegister rd, VRegister rn) {\n LOGMAN_THROW_A_FMT(size == ScalarRegSize::i64Bit || size == ScalarRegSize::i32Bit, \"Invalid size selected for float convert\");\n\n const ScalarRegSize ConvertedSize = size == ScalarRegSize::i64Bit ? ScalarRegSize::i16Bit : ScalarRegSize::i8Bit;\n\n ASIMDScalar2RegMisc(0, 0, ConvertedSize, 0b11010, rd, rn);\n }\n void fcvtms(ScalarRegSize size, VRegister rd, VRegister rn) {\n LOGMAN_THROW_A_FMT(size == ScalarRegSize::i64Bit || size == ScalarRegSize::i32Bit, \"Invalid size selected for float convert\");\n\n const ScalarRegSize ConvertedSize = size == ScalarRegSize::i64Bit ? ScalarRegSize::i16Bit : ScalarRegSize::i8Bit;\n\n ASIMDScalar2RegMisc(0, 0, ConvertedSize, 0b11011, rd, rn);\n }\n void fcvtas(ScalarRegSize size, VRegister rd, VRegister rn) {\n LOGMAN_THROW_A_FMT(size == ScalarRegSize::i64Bit || size == ScalarRegSize::i32Bit, \"Invalid size selected for float convert\");\n\n const ScalarRegSize ConvertedSize = size == ScalarRegSize::i64Bit ? ScalarRegSize::i16Bit : ScalarRegSize::i8Bit;\n\n ASIMDScalar2RegMisc(0, 0, ConvertedSize, 0b11100, rd, rn);\n }\n void scvtf(ScalarRegSize size, VRegister rd, VRegister rn) {\n LOGMAN_THROW_A_FMT(size == ScalarRegSize::i64Bit || size == ScalarRegSize::i32Bit, \"Invalid size selected for float convert\");\n\n const ScalarRegSize ConvertedSize = size == ScalarRegSize::i64Bit ? ScalarRegSize::i16Bit : ScalarRegSize::i8Bit;\n\n ASIMDScalar2RegMisc(0, 0, ConvertedSize, 0b11101, rd, rn);\n }\n\n ///< Comparison against 0.0\n void fcmgt(ScalarRegSize size, VRegister rd, VRegister rn) {\n LOGMAN_THROW_A_FMT(size == ScalarRegSize::i64Bit || size == ScalarRegSize::i32Bit, \"Invalid size selected for float compare\");\n ASIMDScalar2RegMisc(0, 0, size, 0b01100, rd, rn);\n }\n ///< Comparison against 0.0\n void fcmeq(ScalarRegSize size, VRegister rd, VRegister rn) {\n LOGMAN_THROW_A_FMT(size == ScalarRegSize::i64Bit || size == ScalarRegSize::i32Bit, \"Invalid size selected for float compare\");\n ASIMDScalar2RegMisc(0, 0, size, 0b01101, rd, rn);\n }\n ///< Comparison against 0.0\n void fcmlt(ScalarRegSize size, VRegister rd, VRegister rn) {\n LOGMAN_THROW_A_FMT(size == ScalarRegSize::i64Bit || size == ScalarRegSize::i32Bit, \"Invalid size selected for float compare\");\n\n ASIMDScalar2RegMisc(0, 0, size, 0b01110, rd, rn);\n }\n void fcvtps(ScalarRegSize size, VRegister rd, VRegister rn) {\n LOGMAN_THROW_A_FMT(size == ScalarRegSize::i64Bit || size == ScalarRegSize::i32Bit, \"Invalid size selected for float convert\");\n\n ASIMDScalar2RegMisc(0, 0, size, 0b11010, rd, rn);\n }\n void fcvtzs(ScalarRegSize size, VRegister rd, VRegister rn) {\n LOGMAN_THROW_A_FMT(size == ScalarRegSize::i64Bit || size == ScalarRegSize::i32Bit, \"Invalid size selected for float convert\");\n\n ASIMDScalar2RegMisc(0, 0, size, 0b11011, rd, rn);\n }\n void frecpe(ScalarRegSize size, VRegister rd, VRegister rn) {\n LOGMAN_THROW_A_FMT(size == ScalarRegSize::i64Bit || size == ScalarRegSize::i32Bit, \"Invalid size selected for float convert\");\n\n ASIMDScalar2RegMisc(0, 0, size, 0b11101, rd, rn);\n }\n void frecpx(ScalarRegSize size, VRegister rd, VRegister rn) {\n LOGMAN_THROW_A_FMT(size == ScalarRegSize::i64Bit || size == ScalarRegSize::i32Bit, \"Invalid size selected for float convert\");\n\n ASIMDScalar2RegMisc(0, 0, size, 0b11111, rd, rn);\n }\n void usqadd(ScalarRegSize size, VRegister rd, VRegister rn) {\n ASIMDScalar2RegMisc(0, 1, size, 0b00011, rd, rn);\n }\n void sqneg(ScalarRegSize size, VRegister rd, VRegister rn) {\n ASIMDScalar2RegMisc(0, 1, size, 0b00111, rd, rn);\n }\n ///< Comparison against 0.0\n void cmge(ScalarRegSize size, VRegister rd, VRegister rn) {\n LOGMAN_THROW_A_FMT(size == ScalarRegSize::i64Bit, \"Only supports 64-bit\");\n ASIMDScalar2RegMisc(0, 1, size, 0b01000, rd, rn);\n }\n ///< Comparison against 0.0\n void cmle(ScalarRegSize size, VRegister rd, VRegister rn) {\n LOGMAN_THROW_A_FMT(size == ScalarRegSize::i64Bit, \"Only supports 64-bit\");\n ASIMDScalar2RegMisc(0, 1, size, 0b01001, rd, rn);\n }\n void neg(ScalarRegSize size, VRegister rd, VRegister rn) {\n LOGMAN_THROW_A_FMT(size == ScalarRegSize::i64Bit, \"Only supports 64-bit\");\n ASIMDScalar2RegMisc(0, 1, size, 0b01011, rd, rn);\n }\n ///< size is destination.\n void sqxtun(ScalarRegSize size, VRegister rd, VRegister rn) {\n LOGMAN_THROW_A_FMT(size != ScalarRegSize::i64Bit, \"64-bit destination not supported\");\n ASIMDScalar2RegMisc(0, 1, size, 0b10010, rd, rn);\n }\n ///< size is destination.\n void uqxtn(ScalarRegSize size, VRegister rd, VRegister rn) {\n LOGMAN_THROW_A_FMT(size != ScalarRegSize::i64Bit, \"64-bit destination not supported\");\n ASIMDScalar2RegMisc(0, 1, size, 0b10100, rd, rn);\n }\n ///< size is destination.\n void fcvtxn(ScalarRegSize size, VRegister rd, VRegister rn) {\n LOGMAN_THROW_A_FMT(size == ScalarRegSize::i32Bit, \"Invalid size selected for float convert\");\n ASIMDScalar2RegMisc(0, 1, ScalarRegSize::i16Bit, 0b10110, rd, rn);\n }\n void fcvtnu(ScalarRegSize size, VRegister rd, VRegister rn) {\n LOGMAN_THROW_A_FMT(size == ScalarRegSize::i64Bit || size == ScalarRegSize::i32Bit, \"Invalid size selected for float convert\");\n\n const ScalarRegSize ConvertedSize = size == ScalarRegSize::i64Bit ? ScalarRegSize::i16Bit : ScalarRegSize::i8Bit;\n\n ASIMDScalar2RegMisc(0, 1, ConvertedSize, 0b11010, rd, rn);\n }\n void fcvtmu(ScalarRegSize size, VRegister rd, VRegister rn) {\n LOGMAN_THROW_A_FMT(size == ScalarRegSize::i64Bit || size == ScalarRegSize::i32Bit, \"Invalid size selected for float convert\");\n\n const ScalarRegSize ConvertedSize = size == ScalarRegSize::i64Bit ? ScalarRegSize::i16Bit : ScalarRegSize::i8Bit;\n\n ASIMDScalar2RegMisc(0, 1, ConvertedSize, 0b11011, rd, rn);\n }\n void fcvtau(ScalarRegSize size, VRegister rd, VRegister rn) {\n LOGMAN_THROW_A_FMT(size == ScalarRegSize::i64Bit || size == ScalarRegSize::i32Bit, \"Invalid size selected for float convert\");\n\n const ScalarRegSize ConvertedSize = size == ScalarRegSize::i64Bit ? ScalarRegSize::i16Bit : ScalarRegSize::i8Bit;\n\n ASIMDScalar2RegMisc(0, 1, ConvertedSize, 0b11100, rd, rn);\n }\n void ucvtf(ScalarRegSize size, VRegister rd, VRegister rn) {\n LOGMAN_THROW_A_FMT(size == ScalarRegSize::i64Bit || size == ScalarRegSize::i32Bit, \"Invalid size selected for float convert\");\n\n const ScalarRegSize ConvertedSize = size == ScalarRegSize::i64Bit ? ScalarRegSize::i16Bit : ScalarRegSize::i8Bit;\n\n ASIMDScalar2RegMisc(0, 1, ConvertedSize, 0b11101, rd, rn);\n }\n ///< Comparison against 0.0\n void fcmge(ScalarRegSize size, VRegister rd, VRegister rn) {\n LOGMAN_THROW_A_FMT(size == ScalarRegSize::i64Bit || size == ScalarRegSize::i32Bit, \"Invalid size selected for float convert\");\n\n ASIMDScalar2RegMisc(0, 1, size, 0b01100, rd, rn);\n }\n ///< Comparison against 0.0\n void fcmle(ScalarRegSize size, VRegister rd, VRegister rn) {\n LOGMAN_THROW_A_FMT(size == ScalarRegSize::i64Bit || size == ScalarRegSize::i32Bit, \"Invalid size selected for float convert\");\n\n ASIMDScalar2RegMisc(0, 1, size, 0b01101, rd, rn);\n }\n void fcvtpu(ScalarRegSize size, VRegister rd, VRegister rn) {\n LOGMAN_THROW_A_FMT(size == ScalarRegSize::i64Bit || size == ScalarRegSize::i32Bit, \"Invalid size selected for float convert\");\n\n ASIMDScalar2RegMisc(0, 1, size, 0b11010, rd, rn);\n }\n void fcvtzu(ScalarRegSize size, VRegister rd, VRegister rn) {\n LOGMAN_THROW_A_FMT(size == ScalarRegSize::i64Bit || size == ScalarRegSize::i32Bit, \"Invalid size selected for float convert\");\n\n ASIMDScalar2RegMisc(0, 1, size, 0b11011, rd, rn);\n }\n void frsqrte(ScalarRegSize size, VRegister rd, VRegister rn) {\n LOGMAN_THROW_A_FMT(size == ScalarRegSize::i64Bit || size == ScalarRegSize::i32Bit, \"Invalid size selected for float convert\");\n\n ASIMDScalar2RegMisc(0, 1, size, 0b11101, rd, rn);\n }\n // Advanced SIMD scalar pairwise\n void addp(ScalarRegSize size, VRegister rd, VRegister rn) {\n LOGMAN_THROW_A_FMT(size == ScalarRegSize::i64Bit, \"Invalid size selected for addp\");\n ASIMDScalar2RegMisc(1, 0, size, 0b11011, rd, rn);\n }\n\n void fmaxnmp(HRegister rd, HRegister rn) {\n ASIMDScalar2RegMisc(1, 0, ScalarRegSize::i8Bit, 0b01100, rd.V(), rn.V());\n }\n void faddp(HRegister rd, HRegister rn) {\n ASIMDScalar2RegMisc(1, 0, ScalarRegSize::i8Bit, 0b01101, rd.V(), rn.V());\n }\n void fmaxp(HRegister rd, HRegister rn) {\n ASIMDScalar2RegMisc(1, 0, ScalarRegSize::i8Bit, 0b01111, rd.V(), rn.V());\n }\n void fminnmp(HRegister rd, HRegister rn) {\n ASIMDScalar2RegMisc(1, 0, ScalarRegSize::i32Bit, 0b01100, rd.V(), rn.V());\n }\n void fminp(HRegister rd, HRegister rn) {\n ASIMDScalar2RegMisc(1, 0, ScalarRegSize::i32Bit, 0b01111, rd.V(), rn.V());\n }\n\n void fmaxnmp(ScalarRegSize size, VRegister rd, VRegister rn) {\n LOGMAN_THROW_A_FMT(size == ScalarRegSize::i64Bit || size == ScalarRegSize::i32Bit, \"Invalid size selected for float convert\");\n\n const ScalarRegSize ConvertedSize = size == ScalarRegSize::i64Bit ? ScalarRegSize::i16Bit : ScalarRegSize::i8Bit;\n\n ASIMDScalar2RegMisc(1, 1, ConvertedSize, 0b01100, rd, rn);\n }\n void faddp(ScalarRegSize size, VRegister rd, VRegister rn) {\n LOGMAN_THROW_A_FMT(size == ScalarRegSize::i64Bit || size == ScalarRegSize::i32Bit, \"Invalid size selected for float convert\");\n\n const ScalarRegSize ConvertedSize = size == ScalarRegSize::i64Bit ? ScalarRegSize::i16Bit : ScalarRegSize::i8Bit;\n\n ASIMDScalar2RegMisc(1, 1, ConvertedSize, 0b01101, rd, rn);\n }\n void fmaxp(ScalarRegSize size, VRegister rd, VRegister rn) {\n LOGMAN_THROW_A_FMT(size == ScalarRegSize::i64Bit || size == ScalarRegSize::i32Bit, \"Invalid size selected for float convert\");\n\n const ScalarRegSize ConvertedSize = size == ScalarRegSize::i64Bit ? ScalarRegSize::i16Bit : ScalarRegSize::i8Bit;\n\n ASIMDScalar2RegMisc(1, 1, ConvertedSize, 0b01111, rd, rn);\n }\n void fminnmp(ScalarRegSize size, VRegister rd, VRegister rn) {\n LOGMAN_THROW_A_FMT(size == ScalarRegSize::i64Bit || size == ScalarRegSize::i32Bit, \"Invalid size selected for float convert\");\n ASIMDScalar2RegMisc(1, 1, size, 0b01100, rd, rn);\n }\n void fminp(ScalarRegSize size, VRegister rd, VRegister rn) {\n LOGMAN_THROW_A_FMT(size == ScalarRegSize::i64Bit || size == ScalarRegSize::i32Bit, \"Invalid size selected for float convert\");\n ASIMDScalar2RegMisc(1, 1, size, 0b01111, rd, rn);\n }\n // Advanced SIMD scalar three different\n ///< size is destination.\n void sqdmlal(ScalarRegSize size, VRegister rd, VRegister rn, VRegister rm) {\n LOGMAN_THROW_A_FMT(size == ScalarRegSize::i64Bit || size == ScalarRegSize::i32Bit, \"Invalid size selected for float convert\");\n const ScalarRegSize ConvertedSize = size == ScalarRegSize::i64Bit ? ScalarRegSize::i32Bit : ScalarRegSize::i16Bit;\n ASIMD3RegDifferent(0, ConvertedSize, 0b1001, rd, rn, rm);\n }\n ///< size is destination.\n void sqdmlsl(ScalarRegSize size, VRegister rd, VRegister rn, VRegister rm) {\n LOGMAN_THROW_A_FMT(size == ScalarRegSize::i64Bit || size == ScalarRegSize::i32Bit, \"Invalid size selected for float convert\");\n const ScalarRegSize ConvertedSize = size == ScalarRegSize::i64Bit ? ScalarRegSize::i32Bit : ScalarRegSize::i16Bit;\n ASIMD3RegDifferent(0, ConvertedSize, 0b1011, rd, rn, rm);\n }\n\n ///< size is destination.\n void sqdmull(ScalarRegSize size, VRegister rd, VRegister rn, VRegister rm) {\n LOGMAN_THROW_A_FMT(size == ScalarRegSize::i64Bit || size == ScalarRegSize::i32Bit, \"Invalid size selected for float convert\");\n const ScalarRegSize ConvertedSize = size == ScalarRegSize::i64Bit ? ScalarRegSize::i32Bit : ScalarRegSize::i16Bit;\n ASIMD3RegDifferent(0, ConvertedSize, 0b1101, rd, rn, rm);\n }\n // Advanced SIMD scalar three same\n void sqadd(ScalarRegSize size, VRegister rd, VRegister rn, VRegister rm) {\n ASIMD3RegSame(0, size, 0b00001, rd, rn, rm);\n }\n void sqsub(ScalarRegSize size, VRegister rd, VRegister rn, VRegister rm) {\n ASIMD3RegSame(0, size, 0b00101, rd, rn, rm);\n }\n void cmgt(ScalarRegSize size, VRegister rd, VRegister rn, VRegister rm) {\n LOGMAN_THROW_A_FMT(size == ScalarRegSize::i64Bit, \"Only supports 64-bit\");\n ASIMD3RegSame(0, size, 0b00110, rd, rn, rm);\n }\n void cmge(ScalarRegSize size, VRegister rd, VRegister rn, VRegister rm) {\n LOGMAN_THROW_A_FMT(size == ScalarRegSize::i64Bit, \"Only supports 64-bit\");\n ASIMD3RegSame(0, size, 0b00111, rd, rn, rm);\n }\n void sshl(ScalarRegSize size, VRegister rd, VRegister rn, VRegister rm) {\n LOGMAN_THROW_A_FMT(size == ScalarRegSize::i64Bit, \"Only supports 64-bit\");\n ASIMD3RegSame(0, size, 0b01000, rd, rn, rm);\n }\n void sqshl(ScalarRegSize size, VRegister rd, VRegister rn, VRegister rm) {\n ASIMD3RegSame(0, size, 0b01001, rd, rn, rm);\n }\n void srshl(ScalarRegSize size, VRegister rd, VRegister rn, VRegister rm) {\n LOGMAN_THROW_A_FMT(size == ScalarRegSize::i64Bit, \"Only supports 64-bit\");\n ASIMD3RegSame(0, size, 0b01010, rd, rn, rm);\n }\n void sqrshl(ScalarRegSize size, VRegister rd, VRegister rn, VRegister rm) {\n ASIMD3RegSame(0, size, 0b01011, rd, rn, rm);\n }\n void add(ScalarRegSize size, VRegister rd, VRegister rn, VRegister rm) {\n LOGMAN_THROW_A_FMT(size == ScalarRegSize::i64Bit, \"Only supports 64-bit\");\n ASIMD3RegSame(0, size, 0b10000, rd, rn, rm);\n }\n void cmtst(ScalarRegSize size, VRegister rd, VRegister rn, VRegister rm) {\n LOGMAN_THROW_A_FMT(size == ScalarRegSize::i64Bit, \"Only supports 64-bit\");\n ASIMD3RegSame(0, size, 0b10001, rd, rn, rm);\n }\n void sqdmulh(ScalarRegSize size, VRegister rd, VRegister rn, VRegister rm) {\n LOGMAN_THROW_A_FMT(size == ScalarRegSize::i32Bit || size == ScalarRegSize::i16Bit, \"Invalid size\");\n ASIMD3RegSame(0, size, 0b10110, rd, rn, rm);\n }\n void fmulx(ScalarRegSize size, VRegister rd, VRegister rn, VRegister rm) {\n LOGMAN_THROW_A_FMT(size == ScalarRegSize::i64Bit || size == ScalarRegSize::i32Bit, \"Invalid size selected for float convert\");\n\n const ScalarRegSize ConvertedSize = size == ScalarRegSize::i64Bit ? ScalarRegSize::i16Bit : ScalarRegSize::i8Bit;\n\n ASIMD3RegSame(0, ConvertedSize, 0b11011, rd, rn, rm);\n }\n void fcmeq(ScalarRegSize size, VRegister rd, VRegister rn, VRegister rm) {\n LOGMAN_THROW_A_FMT(size == ScalarRegSize::i64Bit || size == ScalarRegSize::i32Bit, \"Invalid size selected for float convert\");\n\n const ScalarRegSize ConvertedSize = size == ScalarRegSize::i64Bit ? ScalarRegSize::i16Bit : ScalarRegSize::i8Bit;\n\n ASIMD3RegSame(0, ConvertedSize, 0b11100, rd, rn, rm);\n }\n void frecps(ScalarRegSize size, VRegister rd, VRegister rn, VRegister rm) {\n LOGMAN_THROW_A_FMT(size == ScalarRegSize::i64Bit || size == ScalarRegSize::i32Bit, \"Invalid size selected for float convert\");\n\n const ScalarRegSize ConvertedSize = size == ScalarRegSize::i64Bit ? ScalarRegSize::i16Bit : ScalarRegSize::i8Bit;\n\n ASIMD3RegSame(0, ConvertedSize, 0b11111, rd, rn, rm);\n }\n void frsqrts(ScalarRegSize size, VRegister rd, VRegister rn, VRegister rm) {\n LOGMAN_THROW_A_FMT(size == ScalarRegSize::i64Bit || size == ScalarRegSize::i32Bit, \"Invalid size selected for float convert\");\n ASIMD3RegSame(0, size, 0b11111, rd, rn, rm);\n }\n void uqadd(ScalarRegSize size, VRegister rd, VRegister rn, VRegister rm) {\n ASIMD3RegSame(1, size, 0b00001, rd, rn, rm);\n }\n void uqsub(ScalarRegSize size, VRegister rd, VRegister rn, VRegister rm) {\n ASIMD3RegSame(1, size, 0b00101, rd, rn, rm);\n }\n void cmhi(ScalarRegSize size, VRegister rd, VRegister rn, VRegister rm) {\n LOGMAN_THROW_A_FMT(size == ScalarRegSize::i64Bit, \"Only supports 64-bit\");\n ASIMD3RegSame(1, size, 0b00110, rd, rn, rm);\n }\n void cmhs(ScalarRegSize size, VRegister rd, VRegister rn, VRegister rm) {\n LOGMAN_THROW_A_FMT(size == ScalarRegSize::i64Bit, \"Only supports 64-bit\");\n ASIMD3RegSame(1, size, 0b00111, rd, rn, rm);\n }\n void ushl(ScalarRegSize size, VRegister rd, VRegister rn, VRegister rm) {\n LOGMAN_THROW_A_FMT(size == ScalarRegSize::i64Bit, \"Only supports 64-bit\");\n ASIMD3RegSame(1, size, 0b01000, rd, rn, rm);\n }\n void uqshl(ScalarRegSize size, VRegister rd, VRegister rn, VRegister rm) {\n ASIMD3RegSame(1, size, 0b01001, rd, rn, rm);\n }\n void urshl(ScalarRegSize size, VRegister rd, VRegister rn, VRegister rm) {\n LOGMAN_THROW_A_FMT(size == ScalarRegSize::i64Bit, \"Only supports 64-bit\");\n ASIMD3RegSame(1, size, 0b01010, rd, rn, rm);\n }\n void uqrshl(ScalarRegSize size, VRegister rd, VRegister rn, VRegister rm) {\n ASIMD3RegSame(1, size, 0b01011, rd, rn, rm);\n }\n void sub(ScalarRegSize size, VRegister rd, VRegister rn, VRegister rm) {\n LOGMAN_THROW_A_FMT(size == ScalarRegSize::i64Bit, \"Only supports 64-bit\");\n ASIMD3RegSame(1, size, 0b10000, rd, rn, rm);\n }\n void cmeq(ScalarRegSize size, VRegister rd, VRegister rn, VRegister rm) {\n LOGMAN_THROW_A_FMT(size == ScalarRegSize::i64Bit, \"Only supports 64-bit\");\n ASIMD3RegSame(1, size, 0b10001, rd, rn, rm);\n }\n void sqrdmulh(ScalarRegSize size, VRegister rd, VRegister rn, VRegister rm) {\n LOGMAN_THROW_A_FMT(size == ScalarRegSize::i32Bit || size == ScalarRegSize::i16Bit, \"Invalid size\");\n ASIMD3RegSame(1, size, 0b10110, rd, rn, rm);\n }\n void fcmge(ScalarRegSize size, VRegister rd, VRegister rn, VRegister rm) {\n LOGMAN_THROW_A_FMT(size == ScalarRegSize::i64Bit || size == ScalarRegSize::i32Bit, \"Invalid size selected for float convert\");\n\n const ScalarRegSize ConvertedSize = size == ScalarRegSize::i64Bit ? ScalarRegSize::i16Bit : ScalarRegSize::i8Bit;\n\n ASIMD3RegSame(1, ConvertedSize, 0b11100, rd, rn, rm);\n }\n void facge(ScalarRegSize size, VRegister rd, VRegister rn, VRegister rm) {\n LOGMAN_THROW_A_FMT(size == ScalarRegSize::i64Bit || size == ScalarRegSize::i32Bit, \"Invalid size selected for float convert\");\n\n const ScalarRegSize ConvertedSize = size == ScalarRegSize::i64Bit ? ScalarRegSize::i16Bit : ScalarRegSize::i8Bit;\n\n ASIMD3RegSame(1, ConvertedSize, 0b11101, rd, rn, rm);\n }\n void fabd(ScalarRegSize size, VRegister rd, VRegister rn, VRegister rm) {\n LOGMAN_THROW_A_FMT(size == ScalarRegSize::i64Bit || size == ScalarRegSize::i32Bit, \"Invalid size selected for float convert\");\n ASIMD3RegSame(1, size, 0b11010, rd, rn, rm);\n }\n void fcmgt(ScalarRegSize size, VRegister rd, VRegister rn, VRegister rm) {\n LOGMAN_THROW_A_FMT(size == ScalarRegSize::i64Bit || size == ScalarRegSize::i32Bit, \"Invalid size selected for float convert\");\n ASIMD3RegSame(1, size, 0b11100, rd, rn, rm);\n }\n void facgt(ScalarRegSize size, VRegister rd, VRegister rn, VRegister rm) {\n LOGMAN_THROW_A_FMT(size == ScalarRegSize::i64Bit || size == ScalarRegSize::i32Bit, \"Invalid size selected for float convert\");\n ASIMD3RegSame(1, size, 0b11101, rd, rn, rm);\n }\n // Advanced SIMD scalar shift by immediate\n void sshr(ScalarRegSize size, VRegister rd, VRegister rn, uint32_t Shift) {\n LOGMAN_THROW_A_FMT(Shift > 0 && Shift < 64, \"Invalid shift for sshr\");\n LOGMAN_THROW_A_FMT(size == ARMEmitter::ScalarRegSize::i64Bit, \"Invalid size selected for sshr\");\n const size_t SubregSizeInBits = ScalarRegSizeInBits(size);\n // Shift encoded in immh:immb, but inverted with 128-bit source\n // shift = (esize * 2) - immh:immb\n const uint32_t InvertedShift = (SubregSizeInBits * 2) - Shift;\n const uint32_t immh = InvertedShift >> 3;\n const uint32_t immb = InvertedShift & 0b111;\n ASIMDScalarShiftByImm(0, immh, immb, 0b00000, rd, rn);\n }\n void ssra(ScalarRegSize size, VRegister rd, VRegister rn, uint32_t Shift) {\n LOGMAN_THROW_A_FMT(Shift > 0 && Shift < 64, \"Invalid shift for sshr\");\n LOGMAN_THROW_A_FMT(size == ARMEmitter::ScalarRegSize::i64Bit, \"Invalid size selected for sshr\");\n const size_t SubregSizeInBits = ScalarRegSizeInBits(size);\n // Shift encoded in immh:immb, but inverted with 128-bit source\n // shift = (esize * 2) - immh:immb\n const uint32_t InvertedShift = (SubregSizeInBits * 2) - Shift;\n const uint32_t immh = InvertedShift >> 3;\n const uint32_t immb = InvertedShift & 0b111;\n ASIMDScalarShiftByImm(0, immh, immb, 0b00010, rd, rn);\n }\n void srshr(ScalarRegSize size, VRegister rd, VRegister rn, uint32_t Shift) {\n LOGMAN_THROW_A_FMT(Shift > 0 && Shift < 64, \"Invalid shift for sshr\");\n LOGMAN_THROW_A_FMT(size == ARMEmitter::ScalarRegSize::i64Bit, \"Invalid size selected for sshr\");\n const size_t SubregSizeInBits = ScalarRegSizeInBits(size);\n // Shift encoded in immh:immb, but inverted with 128-bit source\n // shift = (esize * 2) - immh:immb\n const uint32_t InvertedShift = (SubregSizeInBits * 2) - Shift;\n const uint32_t immh = InvertedShift >> 3;\n const uint32_t immb = InvertedShift & 0b111;\n ASIMDScalarShiftByImm(0, immh, immb, 0b00100, rd, rn);\n }\n void srsra(ScalarRegSize size, VRegister rd, VRegister rn, uint32_t Shift) {\n LOGMAN_THROW_A_FMT(Shift > 0 && Shift < 64, \"Invalid shift for sshr\");\n LOGMAN_THROW_A_FMT(size == ARMEmitter::ScalarRegSize::i64Bit, \"Invalid size selected for sshr\");\n const size_t SubregSizeInBits = ScalarRegSizeInBits(size);\n // Shift encoded in immh:immb, but inverted with 128-bit source\n // shift = (esize * 2) - immh:immb\n const uint32_t InvertedShift = (SubregSizeInBits * 2) - Shift;\n const uint32_t immh = InvertedShift >> 3;\n const uint32_t immb = InvertedShift & 0b111;\n ASIMDScalarShiftByImm(0, immh, immb, 0b00110, rd, rn);\n }\n void shl(ScalarRegSize size, VRegister rd, VRegister rn, uint32_t Shift) {\n LOGMAN_THROW_A_FMT(Shift > 0 && Shift < 64, \"Invalid shift for sshr\");\n LOGMAN_THROW_A_FMT(size == ARMEmitter::ScalarRegSize::i64Bit, \"Invalid size selected for sshr\");\n // Shift encoded a bit weirdly.\n // shift = immh:immb - elementsize but immh is /also/ used for element size.\n const uint32_t immh = 1 << FEXCore::ToUnderlying(size) | (Shift >> 3);\n const uint32_t immb = Shift & 0b111;\n ASIMDScalarShiftByImm(0, immh, immb, 0b01010, rd, rn);\n }\n void sqshl(ScalarRegSize size, VRegister rd, VRegister rn, uint32_t Shift) {\n LOGMAN_THROW_A_FMT(Shift > 0 && Shift < ScalarRegSizeInBits(size), \"Invalid shift for sshr\");\n // Shift encoded a bit weirdly.\n // shift = immh:immb - elementsize but immh is /also/ used for element size.\n const uint32_t immh = 1 << FEXCore::ToUnderlying(size) | (Shift >> 3);\n const uint32_t immb = Shift & 0b111;\n ASIMDScalarShiftByImm(0, immh, immb, 0b01110, rd, rn);\n }\n ///< size is destination\n void sqshrn(ScalarRegSize size, VRegister rd, VRegister rn, uint32_t Shift) {\n LOGMAN_THROW_A_FMT(Shift > 0 && Shift < ScalarRegSizeInBits(size), \"Invalid shift for sshr\");\n LOGMAN_THROW_A_FMT(size != ARMEmitter::ScalarRegSize::i64Bit, \"Invalid size selected for sqshrn\");\n const size_t SubregSizeInBits = ScalarRegSizeInBits(size);\n // Shift encoded in immh:immb, but inverted with 128-bit source\n // shift = (esize * 2) - immh:immb\n const uint32_t InvertedShift = (SubregSizeInBits * 2) - Shift;\n const uint32_t immh = InvertedShift >> 3;\n const uint32_t immb = InvertedShift & 0b111;\n ASIMDScalarShiftByImm(0, immh, immb, 0b10010, rd, rn);\n }\n void sqrshrn(ScalarRegSize size, VRegister rd, VRegister rn, uint32_t Shift) {\n LOGMAN_THROW_A_FMT(Shift > 0 && Shift < ScalarRegSizeInBits(size), \"Invalid shift for sshr\");\n LOGMAN_THROW_A_FMT(size != ARMEmitter::ScalarRegSize::i64Bit, \"Invalid size selected for sqshrn\");\n const size_t SubregSizeInBits = ScalarRegSizeInBits(size);\n // Shift encoded in immh:immb, but inverted with 128-bit source\n // shift = (esize * 2) - immh:immb\n const uint32_t InvertedShift = (SubregSizeInBits * 2) - Shift;\n const uint32_t immh = InvertedShift >> 3;\n const uint32_t immb = InvertedShift & 0b111;\n ASIMDScalarShiftByImm(0, immh, immb, 0b10011, rd, rn);\n }\n // TODO: SCVTF, FCVTZS\n void ushr(ScalarRegSize size, VRegister rd, VRegister rn, uint32_t Shift) {\n LOGMAN_THROW_A_FMT(Shift > 0 && Shift < 64, \"Invalid shift for sshr\");\n LOGMAN_THROW_A_FMT(size == ARMEmitter::ScalarRegSize::i64Bit, \"Invalid size selected for sshr\");\n const size_t SubregSizeInBits = ScalarRegSizeInBits(size);\n // Shift encoded in immh:immb, but inverted with 128-bit source\n // shift = (esize * 2) - immh:immb\n const uint32_t InvertedShift = (SubregSizeInBits * 2) - Shift;\n const uint32_t immh = InvertedShift >> 3;\n const uint32_t immb = InvertedShift & 0b111;\n ASIMDScalarShiftByImm(1, immh, immb, 0b00000, rd, rn);\n }\n void usra(ScalarRegSize size, VRegister rd, VRegister rn, uint32_t Shift) {\n LOGMAN_THROW_A_FMT(Shift > 0 && Shift < 64, \"Invalid shift for sshr\");\n LOGMAN_THROW_A_FMT(size == ARMEmitter::ScalarRegSize::i64Bit, \"Invalid size selected for sshr\");\n const size_t SubregSizeInBits = ScalarRegSizeInBits(size);\n // Shift encoded in immh:immb, but inverted with 128-bit source\n // shift = (esize * 2) - immh:immb\n const uint32_t InvertedShift = (SubregSizeInBits * 2) - Shift;\n const uint32_t immh = InvertedShift >> 3;\n const uint32_t immb = InvertedShift & 0b111;\n ASIMDScalarShiftByImm(1, immh, immb, 0b00010, rd, rn);\n }\n void urshr(ScalarRegSize size, VRegister rd, VRegister rn, uint32_t Shift) {\n LOGMAN_THROW_A_FMT(Shift > 0 && Shift < 64, \"Invalid shift for sshr\");\n LOGMAN_THROW_A_FMT(size == ARMEmitter::ScalarRegSize::i64Bit, \"Invalid size selected for sshr\");\n const size_t SubregSizeInBits = ScalarRegSizeInBits(size);\n // Shift encoded in immh:immb, but inverted with 128-bit source\n // shift = (esize * 2) - immh:immb\n const uint32_t InvertedShift = (SubregSizeInBits * 2) - Shift;\n const uint32_t immh = InvertedShift >> 3;\n const uint32_t immb = InvertedShift & 0b111;\n ASIMDScalarShiftByImm(1, immh, immb, 0b00100, rd, rn);\n }\n void ursra(ScalarRegSize size, VRegister rd, VRegister rn, uint32_t Shift) {\n LOGMAN_THROW_A_FMT(Shift > 0 && Shift < 64, \"Invalid shift for sshr\");\n LOGMAN_THROW_A_FMT(size == ARMEmitter::ScalarRegSize::i64Bit, \"Invalid size selected for sshr\");\n const size_t SubregSizeInBits = ScalarRegSizeInBits(size);\n // Shift encoded in immh:immb, but inverted with 128-bit source\n // shift = (esize * 2) - immh:immb\n const uint32_t InvertedShift = (SubregSizeInBits * 2) - Shift;\n const uint32_t immh = InvertedShift >> 3;\n const uint32_t immb = InvertedShift & 0b111;\n ASIMDScalarShiftByImm(1, immh, immb, 0b00110, rd, rn);\n }\n void sri(ScalarRegSize size, VRegister rd, VRegister rn, uint32_t Shift) {\n LOGMAN_THROW_A_FMT(Shift > 0 && Shift < 64, \"Invalid shift for sshr\");\n LOGMAN_THROW_A_FMT(size == ARMEmitter::ScalarRegSize::i64Bit, \"Invalid size selected for sshr\");\n const size_t SubregSizeInBits = ScalarRegSizeInBits(size);\n // Shift encoded in immh:immb, but inverted with 128-bit source\n // shift = (esize * 2) - immh:immb\n const uint32_t InvertedShift = (SubregSizeInBits * 2) - Shift;\n const uint32_t immh = InvertedShift >> 3;\n const uint32_t immb = InvertedShift & 0b111;\n ASIMDScalarShiftByImm(1, immh, immb, 0b01000, rd, rn);\n }\n void sli(ScalarRegSize size, VRegister rd, VRegister rn, uint32_t Shift) {\n LOGMAN_THROW_A_FMT(Shift > 0 && Shift < 64, \"Invalid shift for sshr\");\n LOGMAN_THROW_A_FMT(size == ARMEmitter::ScalarRegSize::i64Bit, \"Invalid size selected for sshr\");\n // Shift encoded a bit weirdly.\n // shift = immh:immb - elementsize but immh is /also/ used for element size.\n const uint32_t immh = 1 << FEXCore::ToUnderlying(size) | (Shift >> 3);\n const uint32_t immb = Shift & 0b111;\n ASIMDScalarShiftByImm(1, immh, immb, 0b01010, rd, rn);\n }\n void sqshlu(ScalarRegSize size, VRegister rd, VRegister rn, uint32_t Shift) {\n LOGMAN_THROW_A_FMT(Shift > 0 && Shift < ScalarRegSizeInBits(size), \"Invalid shift for sshr\");\n // Shift encoded a bit weirdly.\n // shift = immh:immb - elementsize but immh is /also/ used for element size.\n const uint32_t immh = 1 << FEXCore::ToUnderlying(size) | (Shift >> 3);\n const uint32_t immb = Shift & 0b111;\n ASIMDScalarShiftByImm(1, immh, immb, 0b01100, rd, rn);\n }\n void uqshl(ScalarRegSize size, VRegister rd, VRegister rn, uint32_t Shift) {\n LOGMAN_THROW_A_FMT(Shift > 0 && Shift < ScalarRegSizeInBits(size), \"Invalid shift for sshr\");\n // Shift encoded a bit weirdly.\n // shift = immh:immb - elementsize but immh is /also/ used for element size.\n const uint32_t immh = 1 << FEXCore::ToUnderlying(size) | (Shift >> 3);\n const uint32_t immb = Shift & 0b111;\n ASIMDScalarShiftByImm(1, immh, immb, 0b01110, rd, rn);\n }\n ///< size is destination.\n void sqshrun(ScalarRegSize size, VRegister rd, VRegister rn, uint32_t Shift) {\n LOGMAN_THROW_A_FMT(Shift > 0 && Shift < ScalarRegSizeInBits(size), \"Invalid shift for sshr\");\n LOGMAN_THROW_A_FMT(size != ARMEmitter::ScalarRegSize::i64Bit, \"Invalid size selected for sqshrun\");\n const size_t SubregSizeInBits = ScalarRegSizeInBits(size);\n // Shift encoded in immh:immb, but inverted with 128-bit source\n // shift = (esize * 2) - immh:immb\n const uint32_t InvertedShift = (SubregSizeInBits * 2) - Shift;\n const uint32_t immh = InvertedShift >> 3;\n const uint32_t immb = InvertedShift & 0b111;\n ASIMDScalarShiftByImm(1, immh, immb, 0b10000, rd, rn);\n }\n ///< size is destination.\n void sqrshrun(ScalarRegSize size, VRegister rd, VRegister rn, uint32_t Shift) {\n LOGMAN_THROW_A_FMT(Shift > 0 && Shift < ScalarRegSizeInBits(size), \"Invalid shift for sshr\");\n LOGMAN_THROW_A_FMT(size != ARMEmitter::ScalarRegSize::i64Bit, \"Invalid size selected for sqrshrun\");\n const size_t SubregSizeInBits = ScalarRegSizeInBits(size);\n // Shift encoded in immh:immb, but inverted with 128-bit source\n // shift = (esize * 2) - immh:immb\n const uint32_t InvertedShift = (SubregSizeInBits * 2) - Shift;\n const uint32_t immh = InvertedShift >> 3;\n const uint32_t immb = InvertedShift & 0b111;\n ASIMDScalarShiftByImm(1, immh, immb, 0b10001, rd, rn);\n }\n ///< size is destination.\n void uqshrn(ScalarRegSize size, VRegister rd, VRegister rn, uint32_t Shift) {\n LOGMAN_THROW_A_FMT(Shift > 0 && Shift < ScalarRegSizeInBits(size), \"Invalid shift for sshr\");\n LOGMAN_THROW_A_FMT(size != ARMEmitter::ScalarRegSize::i64Bit, \"Invalid size selected for sqrshrun\");\n const size_t SubregSizeInBits = ScalarRegSizeInBits(size);\n // Shift encoded in immh:immb, but inverted with 128-bit source\n // shift = (esize * 2) - immh:immb\n const uint32_t InvertedShift = (SubregSizeInBits * 2) - Shift;\n const uint32_t immh = InvertedShift >> 3;\n const uint32_t immb = InvertedShift & 0b111;\n ASIMDScalarShiftByImm(1, immh, immb, 0b10010, rd, rn);\n }\n ///< size is destination.\n void uqrshrn(ScalarRegSize size, VRegister rd, VRegister rn, uint32_t Shift) {\n LOGMAN_THROW_A_FMT(Shift > 0 && Shift < ScalarRegSizeInBits(size), \"Invalid shift for sshr\");\n LOGMAN_THROW_A_FMT(size != ARMEmitter::ScalarRegSize::i64Bit, \"Invalid size selected for sqrshrun\");\n const size_t SubregSizeInBits = ScalarRegSizeInBits(size);\n // Shift encoded in immh:immb, but inverted with 128-bit source\n // shift = (esize * 2) - immh:immb\n const uint32_t InvertedShift = (SubregSizeInBits * 2) - Shift;\n const uint32_t immh = InvertedShift >> 3;\n const uint32_t immb = InvertedShift & 0b111;\n ASIMDScalarShiftByImm(1, immh, immb, 0b10011, rd, rn);\n }\n\n // TODO: UCVTF, FCVTZU\n\n // Advanced SIMD scalar x indexed element\n void sqdmlal(ScalarRegSize size, VRegister rd, VRegister rn, VRegister rm, uint32_t index) {\n LOGMAN_THROW_A_FMT(size != ScalarRegSize::i64Bit, \"Scalar size must not be 64-bit\");\n ASIMDScalarXIndexedElement(0, size, 0b0011, rm, rn, rd, index);\n }\n void sqdmlsl(ScalarRegSize size, VRegister rd, VRegister rn, VRegister rm, uint32_t index) {\n LOGMAN_THROW_A_FMT(size != ScalarRegSize::i64Bit, \"Scalar size must not be 64-bit\");\n ASIMDScalarXIndexedElement(0, size, 0b0111, rm, rn, rd, index);\n }\n void sqdmull(ScalarRegSize size, VRegister rd, VRegister rn, VRegister rm, uint32_t index) {\n LOGMAN_THROW_A_FMT(size != ScalarRegSize::i64Bit, \"Scalar size must not be 64-bit\");\n ASIMDScalarXIndexedElement(0, size, 0b1011, rm, rn, rd, index);\n }\n void sqdmulh(ScalarRegSize size, VRegister rd, VRegister rn, VRegister rm, uint32_t index) {\n LOGMAN_THROW_A_FMT(size != ScalarRegSize::i64Bit, \"Scalar size must not be 64-bit\");\n ASIMDScalarXIndexedElement(0, size, 0b1100, rm, rn, rd, index);\n }\n void sqrdmulh(ScalarRegSize size, VRegister rd, VRegister rn, VRegister rm, uint32_t index) {\n LOGMAN_THROW_A_FMT(size != ScalarRegSize::i64Bit, \"Scalar size must not be 64-bit\");\n ASIMDScalarXIndexedElement(0, size, 0b1101, rm, rn, rd, index);\n }\n void fmla(ScalarRegSize size, VRegister rd, VRegister rn, VRegister rm, uint32_t index) {\n ASIMDScalarXIndexedElement(0, size, 0b0001, rm, rn, rd, index);\n }\n void fmls(ScalarRegSize size, VRegister rd, VRegister rn, VRegister rm, uint32_t index) {\n ASIMDScalarXIndexedElement(0, size, 0b0101, rm, rn, rd, index);\n }\n void fmul(ScalarRegSize size, VRegister rd, VRegister rn, VRegister rm, uint32_t index) {\n ASIMDScalarXIndexedElement(0, size, 0b1001, rm, rn, rd, index);\n }\n void sqrdmlah(ScalarRegSize size, VRegister rd, VRegister rn, VRegister rm, uint32_t index) {\n LOGMAN_THROW_A_FMT(size != ScalarRegSize::i64Bit, \"Scalar size must not be 64-bit\");\n ASIMDScalarXIndexedElement(1, size, 0b1101, rm, rn, rd, index);\n }\n void sqrdmlsh(ScalarRegSize size, VRegister rd, VRegister rn, VRegister rm, uint32_t index) {\n LOGMAN_THROW_A_FMT(size != ScalarRegSize::i64Bit, \"Scalar size must not be 64-bit\");\n ASIMDScalarXIndexedElement(1, size, 0b1111, rm, rn, rd, index);\n }\n void fmulx(ScalarRegSize size, VRegister rd, VRegister rn, VRegister rm, uint32_t index) {\n ASIMDScalarXIndexedElement(1, size, 0b1001, rm, rn, rd, index);\n }\n\n // Floating-point data-processing (1 source)\n void fmov(ScalarRegSize size, VRegister rd, VRegister rn) {\n Float1Source(size, 0, 0, 0b000000, rd, rn);\n }\n void fabs(ScalarRegSize size, VRegister rd, VRegister rn) {\n Float1Source(size, 0, 0, 0b000001, rd, rn);\n }\n void fneg(ScalarRegSize size, VRegister rd, VRegister rn) {\n Float1Source(size, 0, 0, 0b000010, rd, rn);\n }\n void fsqrt(ScalarRegSize size, VRegister rd, VRegister rn) {\n Float1Source(size, 0, 0, 0b000011, rd, rn);\n }\n void frintn(ScalarRegSize size, VRegister rd, VRegister rn) {\n Float1Source(size, 0, 0, 0b001000, rd, rn);\n }\n void frintp(ScalarRegSize size, VRegister rd, VRegister rn) {\n Float1Source(size, 0, 0, 0b001001, rd, rn);\n }\n void frintm(ScalarRegSize size, VRegister rd, VRegister rn) {\n Float1Source(size, 0, 0, 0b001010, rd, rn);\n }\n void frintz(ScalarRegSize size, VRegister rd, VRegister rn) {\n Float1Source(size, 0, 0, 0b001011, rd, rn);\n }\n void frinta(ScalarRegSize size, VRegister rd, VRegister rn) {\n Float1Source(size, 0, 0, 0b001100, rd, rn);\n }\n void frintx(ScalarRegSize size, VRegister rd, VRegister rn) {\n Float1Source(size, 0, 0, 0b001110, rd, rn);\n }\n void frinti(ScalarRegSize size, VRegister rd, VRegister rn) {\n Float1Source(size, 0, 0, 0b001111, rd, rn);\n }\n void frint32z(ScalarRegSize size, VRegister rd, VRegister rn) {\n Float1Source(size, 0, 0, 0b010000, rd, rn);\n }\n void frint32x(ScalarRegSize size, VRegister rd, VRegister rn) {\n Float1Source(size, 0, 0, 0b010001, rd, rn);\n }\n void frint64z(ScalarRegSize size, VRegister rd, VRegister rn) {\n Float1Source(size, 0, 0, 0b010010, rd, rn);\n }\n void frint64x(ScalarRegSize size, VRegister rd, VRegister rn) {\n Float1Source(size, 0, 0, 0b010011, rd, rn);\n }\n\n void fmov(SRegister rd, SRegister rn) {\n Float1Source(0, 0, 0b00, 0b000000, rd.V(), rn.V());\n }\n void fabs(SRegister rd, SRegister rn) {\n Float1Source(0, 0, 0b00, 0b000001, rd.V(), rn.V());\n }\n void fneg(SRegister rd, SRegister rn) {\n Float1Source(0, 0, 0b00, 0b000010, rd.V(), rn.V());\n }\n void fsqrt(SRegister rd, SRegister rn) {\n Float1Source(0, 0, 0b00, 0b000011, rd.V(), rn.V());\n }\n void fcvt(DRegister rd, SRegister rn) {\n Float1Source(0, 0, 0b00, 0b000101, rd.V(), rn.V());\n }\n void fcvt(HRegister rd, SRegister rn) {\n Float1Source(0, 0, 0b00, 0b000111, rd.V(), rn.V());\n }\n void frintn(SRegister rd, SRegister rn) {\n Float1Source(0, 0, 0b00, 0b001000, rd.V(), rn.V());\n }\n void frintp(SRegister rd, SRegister rn) {\n Float1Source(0, 0, 0b00, 0b001001, rd.V(), rn.V());\n }\n void frintm(SRegister rd, SRegister rn) {\n Float1Source(0, 0, 0b00, 0b001010, rd.V(), rn.V());\n }\n void frintz(SRegister rd, SRegister rn) {\n Float1Source(0, 0, 0b00, 0b001011, rd.V(), rn.V());\n }\n void frinta(SRegister rd, SRegister rn) {\n Float1Source(0, 0, 0b00, 0b001100, rd.V(), rn.V());\n }\n void frintx(SRegister rd, SRegister rn) {\n Float1Source(0, 0, 0b00, 0b001110, rd.V(), rn.V());\n }\n void frinti(SRegister rd, SRegister rn) {\n Float1Source(0, 0, 0b00, 0b001111, rd.V(), rn.V());\n }\n void frint32z(SRegister rd, SRegister rn) {\n Float1Source(0, 0, 0b00, 0b010000, rd.V(), rn.V());\n }\n void frint32x(SRegister rd, SRegister rn) {\n Float1Source(0, 0, 0b00, 0b010001, rd.V(), rn.V());\n }\n void frint64z(SRegister rd, SRegister rn) {\n Float1Source(0, 0, 0b00, 0b010010, rd.V(), rn.V());\n }\n void frint64x(SRegister rd, SRegister rn) {\n Float1Source(0, 0, 0b00, 0b010011, rd.V(), rn.V());\n }\n\n void fmov(DRegister rd, DRegister rn) {\n Float1Source(0, 0, 0b01, 0b000000, rd.V(), rn.V());\n }\n void fabs(DRegister rd, DRegister rn) {\n Float1Source(0, 0, 0b01, 0b000001, rd.V(), rn.V());\n }\n void fneg(DRegister rd, DRegister rn) {\n Float1Source(0, 0, 0b01, 0b000010, rd.V(), rn.V());\n }\n void fsqrt(DRegister rd, DRegister rn) {\n Float1Source(0, 0, 0b01, 0b000011, rd.V(), rn.V());\n }\n void fcvt(SRegister rd, DRegister rn) {\n Float1Source(0, 0, 0b01, 0b000100, rd.V(), rn.V());\n }\n void bfcvt(HRegister rd, SRegister rn) {\n Float1Source(0, 0, 0b01, 0b000110, rd.V(), rn.V());\n }\n void fcvt(HRegister rd, DRegister rn) {\n Float1Source(0, 0, 0b01, 0b000111, rd.V(), rn.V());\n }\n void frintn(DRegister rd, DRegister rn) {\n Float1Source(0, 0, 0b01, 0b001000, rd.V(), rn.V());\n }\n void frintp(DRegister rd, DRegister rn) {\n Float1Source(0, 0, 0b01, 0b001001, rd.V(), rn.V());\n }\n void frintm(DRegister rd, DRegister rn) {\n Float1Source(0, 0, 0b01, 0b001010, rd.V(), rn.V());\n }\n void frintz(DRegister rd, DRegister rn) {\n Float1Source(0, 0, 0b01, 0b001011, rd.V(), rn.V());\n }\n void frinta(DRegister rd, DRegister rn) {\n Float1Source(0, 0, 0b01, 0b001100, rd.V(), rn.V());\n }\n void frintx(DRegister rd, DRegister rn) {\n Float1Source(0, 0, 0b01, 0b001110, rd.V(), rn.V());\n }\n void frinti(DRegister rd, DRegister rn) {\n Float1Source(0, 0, 0b01, 0b001111, rd.V(), rn.V());\n }\n void frint32z(DRegister rd, DRegister rn) {\n Float1Source(0, 0, 0b01, 0b010000, rd.V(), rn.V());\n }\n void frint32x(DRegister rd, DRegister rn) {\n Float1Source(0, 0, 0b01, 0b010001, rd.V(), rn.V());\n }\n void frint64z(DRegister rd, DRegister rn) {\n Float1Source(0, 0, 0b01, 0b010010, rd.V(), rn.V());\n }\n void frint64x(DRegister rd, DRegister rn) {\n Float1Source(0, 0, 0b01, 0b010011, rd.V(), rn.V());\n }\n\n void fmov(HRegister rd, HRegister rn) {\n Float1Source(0, 0, 0b11, 0b000000, rd.V(), rn.V());\n }\n void fabs(HRegister rd, HRegister rn) {\n Float1Source(0, 0, 0b11, 0b000001, rd.V(), rn.V());\n }\n void fneg(HRegister rd, HRegister rn) {\n Float1Source(0, 0, 0b11, 0b000010, rd.V(), rn.V());\n }\n void fsqrt(HRegister rd, HRegister rn) {\n Float1Source(0, 0, 0b11, 0b000011, rd.V(), rn.V());\n }\n void fcvt(SRegister rd, HRegister rn) {\n Float1Source(0, 0, 0b11, 0b000100, rd.V(), rn.V());\n }\n void fcvt(DRegister rd, HRegister rn) {\n Float1Source(0, 0, 0b11, 0b000101, rd.V(), rn.V());\n }\n void frintn(HRegister rd, HRegister rn) {\n Float1Source(0, 0, 0b11, 0b001000, rd.V(), rn.V());\n }\n void frintp(HRegister rd, HRegister rn) {\n Float1Source(0, 0, 0b11, 0b001001, rd.V(), rn.V());\n }\n void frintm(HRegister rd, HRegister rn) {\n Float1Source(0, 0, 0b11, 0b001010, rd.V(), rn.V());\n }\n void frintz(HRegister rd, HRegister rn) {\n Float1Source(0, 0, 0b11, 0b001011, rd.V(), rn.V());\n }\n void frinta(HRegister rd, HRegister rn) {\n Float1Source(0, 0, 0b11, 0b001100, rd.V(), rn.V());\n }\n void frintx(HRegister rd, HRegister rn) {\n Float1Source(0, 0, 0b11, 0b001110, rd.V(), rn.V());\n }\n void frinti(HRegister rd, HRegister rn) {\n Float1Source(0, 0, 0b11, 0b001111, rd.V(), rn.V());\n }\n\n // Floating-point compare\n void fcmp(ScalarRegSize Size, VRegister rn, VRegister rm) {\n LOGMAN_THROW_A_FMT(Size != ScalarRegSize::i8Bit, \"8-bit destination not supported\");\n\n const auto ConvertedSize = Size == ARMEmitter::ScalarRegSize::i64Bit ? 0b01 :\n Size == ARMEmitter::ScalarRegSize::i32Bit ? 0b00 :\n Size == ARMEmitter::ScalarRegSize::i16Bit ? 0b11 :\n 0;\n\n FloatCompare(0, 0, ConvertedSize, 0b00, 0b00000, rn, rm);\n }\n\n void fcmp(SRegister rn, SRegister rm) {\n FloatCompare(0, 0, 0b00, 0b00, 0b00000, rn.V(), rm.V());\n }\n ///< Compare to #0.0\n void fcmp(SRegister rn) {\n FloatCompare(0, 0, 0b00, 0b00, 0b01000, rn.V(), VReg::v0);\n }\n void fcmpe(SRegister rn, SRegister rm) {\n FloatCompare(0, 0, 0b00, 0b00, 0b10000, rn.V(), rm.V());\n }\n\n ///< Compare to #0.0\n void fcmpe(SRegister rn) {\n FloatCompare(0, 0, 0b00, 0b00, 0b11000, rn.V(), VReg::v0);\n }\n void fcmp(DRegister rn, DRegister rm) {\n FloatCompare(0, 0, 0b01, 0b00, 0b00000, rn.V(), rm.V());\n }\n\n ///< Compare to #0.0\n void fcmp(DRegister rn) {\n FloatCompare(0, 0, 0b01, 0b00, 0b01000, rn.V(), VReg::v0);\n }\n void fcmpe(DRegister rn, DRegister rm) {\n FloatCompare(0, 0, 0b01, 0b00, 0b10000, rn.V(), rm.V());\n }\n\n ///< Compare to #0.0\n void fcmpe(DRegister rn) {\n FloatCompare(0, 0, 0b01, 0b00, 0b11000, rn.V(), VReg::v0);\n }\n void fcmp(HRegister rn, HRegister rm) {\n FloatCompare(0, 0, 0b11, 0b00, 0b00000, rn.V(), rm.V());\n }\n\n ///< Compare to #0.0\n void fcmp(HRegister rn) {\n FloatCompare(0, 0, 0b11, 0b00, 0b01000, rn.V(), VReg::v0);\n }\n void fcmpe(HRegister rn, HRegister rm) {\n FloatCompare(0, 0, 0b11, 0b00, 0b10000, rn.V(), rm.V());\n }\n\n ///< Compare to #0.0\n void fcmpe(HRegister rn) {\n FloatCompare(0, 0, 0b11, 0b00, 0b11000, rn.V(), VReg::v0);\n }\n\n // Floating-point immediate\n void fmov(ARMEmitter::ScalarRegSize size, ARMEmitter::VRegister rd, float Value) {\n uint32_t M = 0;\n uint32_t S = 0;\n uint32_t ptype;\n uint32_t imm8;\n uint32_t imm5 = 0b0'0000;\n if (size == ARMEmitter::ScalarRegSize::i16Bit) {\n LOGMAN_MSG_A_FMT(\"Unsupported\");\n FEX_UNREACHABLE;\n } else if (size == ARMEmitter::ScalarRegSize::i32Bit) {\n ptype = 0b00;\n imm8 = FP32ToImm8(Value);\n } else if (size == ARMEmitter::ScalarRegSize::i64Bit) {\n ptype = 0b01;\n imm8 = FP64ToImm8(Value);\n } else {\n FEX_UNREACHABLE;\n }\n\n FloatScalarImmediate(M, S, ptype, imm8, imm5, rd);\n }\n\n void FloatScalarImmediate(uint32_t M, uint32_t S, uint32_t ptype, uint32_t imm8, uint32_t imm5, ARMEmitter::VRegister rd) {\n constexpr uint32_t Op = 0b0001'1110'0010'0000'0001'00 << 10;\n uint32_t Instr = Op;\n\n Instr |= M << 31;\n Instr |= S << 29;\n Instr |= ptype << 22;\n Instr |= imm8 << 13;\n Instr |= imm5 << 5;\n Instr |= rd.Idx();\n dc32(Instr);\n }\n\n // Floating-point conditional compare\n void fccmp(SRegister rn, SRegister rm, StatusFlags flags, Condition Cond) {\n FloatConditionalCompare(0, 0, 0b00, 0b0, rn.V(), rm.V(), flags, Cond);\n }\n void fccmpe(SRegister rn, SRegister rm, StatusFlags flags, Condition Cond) {\n FloatConditionalCompare(0, 0, 0b00, 0b1, rn.V(), rm.V(), flags, Cond);\n }\n void fccmp(DRegister rn, DRegister rm, StatusFlags flags, Condition Cond) {\n FloatConditionalCompare(0, 0, 0b01, 0b0, rn.V(), rm.V(), flags, Cond);\n }\n void fccmpe(DRegister rn, DRegister rm, StatusFlags flags, Condition Cond) {\n FloatConditionalCompare(0, 0, 0b01, 0b1, rn.V(), rm.V(), flags, Cond);\n }\n void fccmp(HRegister rn, HRegister rm, StatusFlags flags, Condition Cond) {\n FloatConditionalCompare(0, 0, 0b11, 0b0, rn.V(), rm.V(), flags, Cond);\n }\n void fccmpe(HRegister rn, HRegister rm, StatusFlags flags, Condition Cond) {\n FloatConditionalCompare(0, 0, 0b11, 0b1, rn.V(), rm.V(), flags, Cond);\n }\n\n // Floating-point data-processing (2 source)\n void fmul(ScalarRegSize size, VRegister rd, VRegister rn, VRegister rm) {\n Float2Source(size, 0, 0, 0b0000, rd, rn, rm);\n }\n void fdiv(ScalarRegSize size, VRegister rd, VRegister rn, VRegister rm) {\n Float2Source(size, 0, 0, 0b0001, rd, rn, rm);\n }\n void fadd(ScalarRegSize size, VRegister rd, VRegister rn, VRegister rm) {\n Float2Source(size, 0, 0, 0b0010, rd, rn, rm);\n }\n void fsub(ScalarRegSize size, VRegister rd, VRegister rn, VRegister rm) {\n Float2Source(size, 0, 0, 0b0011, rd, rn, rm);\n }\n void fmax(ScalarRegSize size, VRegister rd, VRegister rn, VRegister rm) {\n Float2Source(size, 0, 0, 0b0100, rd, rn, rm);\n }\n void fmin(ScalarRegSize size, VRegister rd, VRegister rn, VRegister rm) {\n Float2Source(size, 0, 0, 0b0101, rd, rn, rm);\n }\n void fmaxnm(ScalarRegSize size, VRegister rd, VRegister rn, VRegister rm) {\n Float2Source(size, 0, 0, 0b0110, rd, rn, rm);\n }\n void fminnm(ScalarRegSize size, VRegister rd, VRegister rn, VRegister rm) {\n Float2Source(size, 0, 0, 0b0111, rd, rn, rm);\n }\n void fnmul(ScalarRegSize size, VRegister rd, VRegister rn, VRegister rm) {\n Float2Source(size, 0, 0, 0b1000, rd, rn, rm);\n }\n\n void fmul(SRegister rd, SRegister rn, SRegister rm) {\n Float2Source(0, 0, 0b00, 0b0000, rd.V(), rn.V(), rm.V());\n }\n void fdiv(SRegister rd, SRegister rn, SRegister rm) {\n Float2Source(0, 0, 0b00, 0b0001, rd.V(), rn.V(), rm.V());\n }\n void fadd(SRegister rd, SRegister rn, SRegister rm) {\n Float2Source(0, 0, 0b00, 0b0010, rd.V(), rn.V(), rm.V());\n }\n void fsub(SRegister rd, SRegister rn, SRegister rm) {\n Float2Source(0, 0, 0b00, 0b0011, rd.V(), rn.V(), rm.V());\n }\n void fmax(SRegister rd, SRegister rn, SRegister rm) {\n Float2Source(0, 0, 0b00, 0b0100, rd.V(), rn.V(), rm.V());\n }\n void fmin(SRegister rd, SRegister rn, SRegister rm) {\n Float2Source(0, 0, 0b00, 0b0101, rd.V(), rn.V(), rm.V());\n }\n void fmaxnm(SRegister rd, SRegister rn, SRegister rm) {\n Float2Source(0, 0, 0b00, 0b0110, rd.V(), rn.V(), rm.V());\n }\n void fminnm(SRegister rd, SRegister rn, SRegister rm) {\n Float2Source(0, 0, 0b00, 0b0111, rd.V(), rn.V(), rm.V());\n }\n void fnmul(SRegister rd, SRegister rn, SRegister rm) {\n Float2Source(0, 0, 0b00, 0b1000, rd.V(), rn.V(), rm.V());\n }\n\n void fmul(DRegister rd, DRegister rn, DRegister rm) {\n Float2Source(0, 0, 0b01, 0b0000, rd.V(), rn.V(), rm.V());\n }\n void fdiv(DRegister rd, DRegister rn, DRegister rm) {\n Float2Source(0, 0, 0b01, 0b0001, rd.V(), rn.V(), rm.V());\n }\n void fadd(DRegister rd, DRegister rn, DRegister rm) {\n Float2Source(0, 0, 0b01, 0b0010, rd.V(), rn.V(), rm.V());\n }\n void fsub(DRegister rd, DRegister rn, DRegister rm) {\n Float2Source(0, 0, 0b01, 0b0011, rd.V(), rn.V(), rm.V());\n }\n void fmax(DRegister rd, DRegister rn, DRegister rm) {\n Float2Source(0, 0, 0b01, 0b0100, rd.V(), rn.V(), rm.V());\n }\n void fmin(DRegister rd, DRegister rn, DRegister rm) {\n Float2Source(0, 0, 0b01, 0b0101, rd.V(), rn.V(), rm.V());\n }\n void fmaxnm(DRegister rd, DRegister rn, DRegister rm) {\n Float2Source(0, 0, 0b01, 0b0110, rd.V(), rn.V(), rm.V());\n }\n void fminnm(DRegister rd, DRegister rn, DRegister rm) {\n Float2Source(0, 0, 0b01, 0b0111, rd.V(), rn.V(), rm.V());\n }\n void fnmul(DRegister rd, DRegister rn, DRegister rm) {\n Float2Source(0, 0, 0b01, 0b1000, rd.V(), rn.V(), rm.V());\n }\n\n void fmul(HRegister rd, HRegister rn, HRegister rm) {\n Float2Source(0, 0, 0b11, 0b0000, rd.V(), rn.V(), rm.V());\n }\n void fdiv(HRegister rd, HRegister rn, HRegister rm) {\n Float2Source(0, 0, 0b11, 0b0001, rd.V(), rn.V(), rm.V());\n }\n void fadd(HRegister rd, HRegister rn, HRegister rm) {\n Float2Source(0, 0, 0b11, 0b0010, rd.V(), rn.V(), rm.V());\n }\n void fsub(HRegister rd, HRegister rn, HRegister rm) {\n Float2Source(0, 0, 0b11, 0b0011, rd.V(), rn.V(), rm.V());\n }\n void fmax(HRegister rd, HRegister rn, HRegister rm) {\n Float2Source(0, 0, 0b11, 0b0100, rd.V(), rn.V(), rm.V());\n }\n void fmin(HRegister rd, HRegister rn, HRegister rm) {\n Float2Source(0, 0, 0b11, 0b0101, rd.V(), rn.V(), rm.V());\n }\n void fmaxnm(HRegister rd, HRegister rn, HRegister rm) {\n Float2Source(0, 0, 0b11, 0b0110, rd.V(), rn.V(), rm.V());\n }\n void fminnm(HRegister rd, HRegister rn, HRegister rm) {\n Float2Source(0, 0, 0b11, 0b0111, rd.V(), rn.V(), rm.V());\n }\n void fnmul(HRegister rd, HRegister rn, HRegister rm) {\n Float2Source(0, 0, 0b11, 0b1000, rd.V(), rn.V(), rm.V());\n }\n\n // Floating-point conditional select\n void fcsel(ScalarRegSize size, VRegister rd, VRegister rn, VRegister rm, Condition Cond) {\n LOGMAN_THROW_A_FMT(size == ScalarRegSize::i16Bit || size == ScalarRegSize::i64Bit || size == ScalarRegSize::i32Bit,\n \"Invalid size selected for {}\", __func__);\n\n const uint32_t ConvertedSize = size == ScalarRegSize::i64Bit ? 0b01 : size == ScalarRegSize::i32Bit ? 0b00 : 0b11;\n\n FloatConditionalSelect(0, 0, ConvertedSize, rd, rn, rm, Cond);\n }\n\n void fcsel(SRegister rd, SRegister rn, SRegister rm, Condition Cond) {\n FloatConditionalSelect(0, 0, 0b00, rd.V(), rn.V(), rm.V(), Cond);\n }\n void fcsel(DRegister rd, DRegister rn, DRegister rm, Condition Cond) {\n FloatConditionalSelect(0, 0, 0b01, rd.V(), rn.V(), rm.V(), Cond);\n }\n void fcsel(HRegister rd, HRegister rn, HRegister rm, Condition Cond) {\n FloatConditionalSelect(0, 0, 0b11, rd.V(), rn.V(), rm.V(), Cond);\n }\n\n // Floating-point data-processing (3 source)\n void fmadd(SRegister rd, SRegister rn, SRegister rm, SRegister ra) {\n Float3Source(0, 0, 0b00, 0, 0, rd.V(), rn.V(), rm.V(), ra.V());\n }\n void fmsub(SRegister rd, SRegister rn, SRegister rm, SRegister ra) {\n Float3Source(0, 0, 0b00, 0, 1, rd.V(), rn.V(), rm.V(), ra.V());\n }\n void fnmadd(SRegister rd, SRegister rn, SRegister rm, SRegister ra) {\n Float3Source(0, 0, 0b00, 1, 0, rd.V(), rn.V(), rm.V(), ra.V());\n }\n void fnmsub(SRegister rd, SRegister rn, SRegister rm, SRegister ra) {\n Float3Source(0, 0, 0b00, 1, 1, rd.V(), rn.V(), rm.V(), ra.V());\n }\n\n void fmadd(DRegister rd, DRegister rn, DRegister rm, DRegister ra) {\n Float3Source(0, 0, 0b01, 0, 0, rd.V(), rn.V(), rm.V(), ra.V());\n }\n void fmsub(DRegister rd, DRegister rn, DRegister rm, DRegister ra) {\n Float3Source(0, 0, 0b01, 0, 1, rd.V(), rn.V(), rm.V(), ra.V());\n }\n void fnmadd(DRegister rd, DRegister rn, DRegister rm, DRegister ra) {\n Float3Source(0, 0, 0b01, 1, 0, rd.V(), rn.V(), rm.V(), ra.V());\n }\n void fnmsub(DRegister rd, DRegister rn, DRegister rm, DRegister ra) {\n Float3Source(0, 0, 0b01, 1, 1, rd.V(), rn.V(), rm.V(), ra.V());\n }\n\n void fmadd(HRegister rd, HRegister rn, HRegister rm, HRegister ra) {\n Float3Source(0, 0, 0b11, 0, 0, rd.V(), rn.V(), rm.V(), ra.V());\n }\n void fmsub(HRegister rd, HRegister rn, HRegister rm, HRegister ra) {\n Float3Source(0, 0, 0b11, 0, 1, rd.V(), rn.V(), rm.V(), ra.V());\n }\n void fnmadd(HRegister rd, HRegister rn, HRegister rm, HRegister ra) {\n Float3Source(0, 0, 0b11, 1, 0, rd.V(), rn.V(), rm.V(), ra.V());\n }\n void fnmsub(HRegister rd, HRegister rn, HRegister rm, HRegister ra) {\n Float3Source(0, 0, 0b11, 1, 1, rd.V(), rn.V(), rm.V(), ra.V());\n }\n\nprivate:\n // Advanced SIMD scalar copy\n void ASIMDScalarCopy(uint32_t Q, uint32_t b28, uint32_t imm5, uint32_t imm4, VRegister rd, VRegister rn) {\n uint32_t Instr = 0b0000'1110'0000'0000'0000'01U << 10;\n Instr |= Q << 30;\n Instr |= b28 << 28;\n Instr |= imm5 << 16;\n Instr |= imm4 << 11;\n Instr |= Encode_rn(rn);\n Instr |= Encode_rd(rd);\n dc32(Instr);\n }\n\n // Advanced SIMD scalar three same FP16\n void ASIMDScalarThreeSameFP16(uint32_t U, uint32_t a, uint32_t opcode, HRegister rm, HRegister rn, HRegister rd) {\n uint32_t Instr = 0b0101'1110'0100'0000'0000'0100'0000'0000;\n\n Instr |= U << 29;\n Instr |= a << 23;\n Instr |= rm.Idx() << 16;\n Instr |= opcode << 11;\n Instr |= rn.Idx() << 5;\n Instr |= rd.Idx();\n dc32(Instr);\n }\n // Advanced SIMD scalar two-register miscellaneous FP16\n void ASIMDScalarTwoRegMiscFP16(uint32_t U, uint32_t a, uint32_t opcode, HRegister rn, HRegister rd) {\n uint32_t Instr = 0b0101'1110'0111'1000'0000'1000'0000'0000;\n\n Instr |= U << 29;\n Instr |= a << 23;\n Instr |= opcode << 12;\n Instr |= rn.Idx() << 5;\n Instr |= rd.Idx();\n dc32(Instr);\n }\n\n // Advanced SIMD scalar three same extra\n void ASIMDScalarThreeSameExtra(uint32_t U, ScalarRegSize size, uint32_t opcode, VRegister rm, VRegister rn, VRegister rd) {\n uint32_t Instr = 0b0101'1110'0000'0000'1000'0100'0000'0000;\n Instr |= U << 29;\n Instr |= FEXCore::ToUnderlying(size) << 22;\n Instr |= rm.Idx() << 16;\n Instr |= opcode << 11;\n Instr |= rn.Idx() << 5;\n Instr |= rd.Idx();\n dc32(Instr);\n }\n\n // Advanced SIMD scalar two-register miscellaneous\n void ASIMDScalar2RegMisc(uint32_t b20, uint32_t U, ScalarRegSize size, uint32_t opcode, VRegister rd, VRegister rn) {\n uint32_t Instr = 0b0101'1110'0010'0000'0000'1000'0000'0000;\n\n Instr |= U << 29;\n Instr |= FEXCore::ToUnderlying(size) << 22;\n Instr |= b20 << 20;\n Instr |= opcode << 12;\n Instr |= rn.Idx() << 5;\n Instr |= rd.Idx();\n dc32(Instr);\n }\n\n // Advanced SIMD scalar three different\n void ASIMD3RegDifferent(uint32_t U, ScalarRegSize size, uint32_t opcode, VRegister rd, VRegister rn, VRegister rm) {\n uint32_t Instr = 0b0101'1110'0010'0000'0000'0000'0000'0000;\n\n Instr |= U << 29;\n Instr |= FEXCore::ToUnderlying(size) << 22;\n Instr |= Encode_rm(rm);\n Instr |= opcode << 12;\n Instr |= Encode_rn(rn);\n Instr |= Encode_rd(rd);\n dc32(Instr);\n }\n // Advanced SIMD scalar three same\n void ASIMD3RegSame(uint32_t U, ScalarRegSize size, uint32_t opcode, VRegister rd, VRegister rn, VRegister rm) {\n uint32_t Instr = 0b0101'1110'0010'0000'0000'0100'0000'0000;\n\n Instr |= U << 29;\n Instr |= FEXCore::ToUnderlying(size) << 22;\n Instr |= Encode_rm(rm);\n Instr |= opcode << 11;\n Instr |= Encode_rn(rn);\n Instr |= Encode_rd(rd);\n dc32(Instr);\n }\n // Advanced SIMD scalar shift by immediate\n void ASIMDScalarShiftByImm(uint32_t U, uint32_t immh, uint32_t immb, uint32_t opcode, VRegister rd, VRegister rn) {\n uint32_t Instr = 0b0101'1111'0000'0000'0000'0100'0000'0000;\n\n Instr |= U << 29;\n Instr |= immh << 19;\n Instr |= immb << 16;\n Instr |= opcode << 11;\n Instr |= Encode_rn(rn);\n Instr |= Encode_rd(rd);\n dc32(Instr);\n }\n\n // Advanced SIMD scalar x indexed element\n void ASIMDScalarXIndexedElement(uint32_t U, ScalarRegSize size, uint32_t opcode, VRegister rm, VRegister rn, VRegister rd, uint32_t index) {\n LOGMAN_THROW_A_FMT(size != ScalarRegSize::i8Bit, \"Scalar size must not be 8-bit\");\n\n const auto invalid_bound = 16U >> FEXCore::ToUnderlying(size);\n LOGMAN_THROW_A_FMT(index < invalid_bound, \"Index ({}) must be within [0-{}]\", index, invalid_bound - 1);\n\n uint32_t Instr = 0b0101'1111'0000'0000'0000'0000'0000'0000;\n\n // FMUL/FMLA/FMLS indexed variants deal with size differently.\n if (opcode == 0b0001 || opcode == 0b0101 || opcode == 0b1001) {\n // Unlike other instructions in the group, 16-bit is encoded as zero\n // and 32/64-bit are encoded with the top bit always set to one.\n if (size != ScalarRegSize::i16Bit) {\n Instr |= (0b10 | (FEXCore::ToUnderlying(size) & 1)) << 22;\n }\n } else {\n Instr |= FEXCore::ToUnderlying(size) << 22;\n }\n\n uint32_t H = 0;\n uint32_t LM = 0;\n if (size == ScalarRegSize::i16Bit) {\n LOGMAN_THROW_A_FMT(rm <= VReg::v15, \"rm ({}) must be within [v0-v15]\", rm.Idx());\n H = (index >> 2) & 1;\n LM = index & 0b11;\n } else if (size == ScalarRegSize::i32Bit) {\n H = (index >> 1) & 1;\n LM = (index & 0b01) << 1;\n } else {\n H = index & 1;\n }\n\n Instr |= U << 29;\n Instr |= LM << 20;\n Instr |= rm.Idx() << 16;\n Instr |= opcode << 12;\n Instr |= H << 11;\n Instr |= rn.Idx() << 5;\n Instr |= rd.Idx();\n dc32(Instr);\n }\n\n // Floating-point data-processing (1 source)\n void Float1Source(uint32_t M, uint32_t S, uint32_t ptype, uint32_t opcode, VRegister rd, VRegister rn) {\n uint32_t Instr = 0b0001'1110'0010'0000'0100'0000'0000'0000;\n\n Instr |= M << 31;\n Instr |= S << 29;\n Instr |= ptype << 22;\n Instr |= opcode << 15;\n Instr |= Encode_rn(rn);\n Instr |= Encode_rd(rd);\n\n dc32(Instr);\n }\n void Float1Source(ScalarRegSize size, uint32_t M, uint32_t S, uint32_t opcode, VRegister rd, VRegister rn) {\n LOGMAN_THROW_A_FMT(size == ScalarRegSize::i16Bit || size == ScalarRegSize::i64Bit || size == ScalarRegSize::i32Bit,\n \"Invalid size selected for {}\", __func__);\n\n const uint32_t ConvertedSize = size == ScalarRegSize::i64Bit ? 0b01 : size == ScalarRegSize::i32Bit ? 0b00 : 0b11;\n\n Float1Source(M, S, ConvertedSize, opcode, rd, rn);\n }\n\n // Floating-point compare\n void FloatCompare(uint32_t M, uint32_t S, uint32_t ftype, uint32_t op, uint32_t opcode2, VRegister rn, VRegister rm) {\n uint32_t Instr = 0b0001'1110'0010'0000'0010'0000'0000'0000;\n\n Instr |= M << 31;\n Instr |= S << 29;\n Instr |= ftype << 22;\n Instr |= Encode_rm(rm);\n Instr |= op << 14;\n Instr |= Encode_rn(rn);\n Instr |= opcode2;\n\n dc32(Instr);\n }\n // Floating-point immediate\n // XXX:\n // Floating-point conditional compare\n void FloatConditionalCompare(uint32_t M, uint32_t S, uint32_t ptype, uint32_t op, VRegister rn, VRegister rm, StatusFlags flags, Condition Cond) {\n uint32_t Instr = 0b0001'1110'0010'0000'0000'0100'0000'0000;\n\n Instr |= M << 31;\n Instr |= S << 29;\n Instr |= ptype << 22;\n Instr |= Encode_rm(rm);\n Instr |= FEXCore::ToUnderlying(Cond) << 12;\n Instr |= Encode_rn(rn);\n Instr |= op << 4;\n Instr |= FEXCore::ToUnderlying(flags);\n\n dc32(Instr);\n }\n // Floating-point data-processing (2 source)\n\n void Float2Source(uint32_t M, uint32_t S, uint32_t ptype, uint32_t opcode, VRegister rd, VRegister rn, VRegister rm) {\n uint32_t Instr = 0b0001'1110'0010'0000'0000'1000'0000'0000;\n\n Instr |= M << 31;\n Instr |= S << 29;\n Instr |= ptype << 22;\n Instr |= Encode_rm(rm);\n Instr |= opcode << 12;\n Instr |= Encode_rn(rn);\n Instr |= Encode_rd(rd);\n\n dc32(Instr);\n }\n\n void Float2Source(ScalarRegSize size, uint32_t M, uint32_t S, uint32_t opcode, VRegister rd, VRegister rn, VRegister rm) {\n LOGMAN_THROW_A_FMT(size == ScalarRegSize::i16Bit || size == ScalarRegSize::i64Bit || size == ScalarRegSize::i32Bit,\n \"Invalid size selected for {}\", __func__);\n\n const uint32_t ConvertedSize = size == ScalarRegSize::i64Bit ? 0b01 : size == ScalarRegSize::i32Bit ? 0b00 : 0b11;\n\n Float2Source(M, S, ConvertedSize, opcode, rd, rn, rm);\n }\n\n // Floating-point conditional select\n void FloatConditionalSelect(uint32_t M, uint32_t S, uint32_t ptype, VRegister rd, VRegister rn, VRegister rm, Condition Cond) {\n uint32_t Instr = 0b0001'1110'0010'0000'0000'1100'0000'0000;\n\n Instr |= M << 31;\n Instr |= S << 29;\n Instr |= ptype << 22;\n Instr |= rm.Idx() << 16;\n Instr |= FEXCore::ToUnderlying(Cond) << 12;\n Instr |= rn.Idx() << 5;\n Instr |= rd.Idx();\n dc32(Instr);\n }\n\n // Floating-point data-processing (3 source)\n void Float3Source(uint32_t M, uint32_t S, uint32_t ptype, uint32_t o1, uint32_t o0, VRegister rd, VRegister rn, VRegister rm, VRegister ra) {\n uint32_t Instr = 0b0001'1111'0000'0000'0000'0000'0000'0000;\n\n Instr |= M << 31;\n Instr |= S << 29;\n Instr |= ptype << 22;\n Instr |= o1 << 21;\n Instr |= Encode_rm(rm);\n Instr |= o0 << 15;\n Instr |= Encode_ra(ra);\n Instr |= Encode_rn(rn);\n Instr |= Encode_rd(rd);\n dc32(Instr);\n }\n\n#ifndef INCLUDED_BY_EMITTER\n}; // struct LoadstoreEmitterOps\n} // namespace ARMEmitter\n#endif\n" }, { "path": "CodeEmitter/CodeEmitter/SystemOps.inl", "content": "// SPDX-License-Identifier: MIT\n/* System instruction emitters.\n *\n * This is mostly a mashup of various instruction types.\n * Nothing follows an explicit pattern since they are mostly different.\n */\n\n#pragma once\n#ifndef INCLUDED_BY_EMITTER\n#include \nnamespace ARMEmitter {\nstruct EmitterOps : Emitter {\n#endif\n\npublic:\n // Reserved\n void udf(uint32_t Imm) {\n LOGMAN_THROW_A_FMT(Imm < 0x1'0000, \"Immediate needs to be 16-bit\");\n dc32(Imm);\n }\n\n // System with result\n // TODO: SYSL\n // System Instruction\n // TODO: AT\n // TODO: CFP\n // TODO: CPP\n void dc(ARMEmitter::DataCacheOperation DCOp, ARMEmitter::Register rt) {\n constexpr uint32_t Op = 0b1101'0101'0000'1000'0111 << 12;\n SystemInstruction(Op, 0, FEXCore::ToUnderlying(DCOp), rt);\n }\n // TODO: DVP\n // TODO: IC\n // TODO: TLBI\n\n // Exception generation\n void svc(uint32_t Imm) {\n ExceptionGeneration(0b000, 0b000, 0b01, Imm);\n }\n void hvc(uint32_t Imm) {\n ExceptionGeneration(0b000, 0b000, 0b10, Imm);\n }\n void smc(uint32_t Imm) {\n ExceptionGeneration(0b000, 0b000, 0b11, Imm);\n }\n void brk(uint32_t Imm) {\n ExceptionGeneration(0b001, 0b000, 0b00, Imm);\n }\n void hlt(uint32_t Imm) {\n ExceptionGeneration(0b010, 0b000, 0b00, Imm);\n }\n void tcancel(uint32_t Imm) {\n ExceptionGeneration(0b011, 0b000, 0b00, Imm);\n }\n void dcps1(uint32_t Imm) {\n ExceptionGeneration(0b101, 0b000, 0b01, Imm);\n }\n void dcps2(uint32_t Imm) {\n ExceptionGeneration(0b101, 0b000, 0b10, Imm);\n }\n void dcps3(uint32_t Imm) {\n ExceptionGeneration(0b101, 0b000, 0b11, Imm);\n }\n // System instructions with register argument\n void wfet(ARMEmitter::Register rt) {\n SystemInstructionWithReg(0b0000, 0b000, rt);\n }\n void wfit(ARMEmitter::Register rt) {\n SystemInstructionWithReg(0b0000, 0b001, rt);\n }\n\n // Hints\n void nop() {\n Hint(ARMEmitter::HintRegister::NOP);\n }\n void yield() {\n Hint(ARMEmitter::HintRegister::YIELD);\n }\n void wfe() {\n Hint(ARMEmitter::HintRegister::WFE);\n }\n void wfi() {\n Hint(ARMEmitter::HintRegister::WFI);\n }\n void sev() {\n Hint(ARMEmitter::HintRegister::SEV);\n }\n void sevl() {\n Hint(ARMEmitter::HintRegister::SEVL);\n }\n void dgh() {\n Hint(ARMEmitter::HintRegister::DGH);\n }\n void csdb() {\n Hint(ARMEmitter::HintRegister::CSDB);\n }\n\n // Barriers\n void clrex(uint32_t imm = 15) {\n LOGMAN_THROW_A_FMT(imm < 16, \"Immediate out of range\");\n Barrier(ARMEmitter::BarrierRegister::CLREX, imm);\n }\n void dsb(ARMEmitter::BarrierScope Scope) {\n Barrier(ARMEmitter::BarrierRegister::DSB, FEXCore::ToUnderlying(Scope));\n }\n void dmb(ARMEmitter::BarrierScope Scope) {\n Barrier(ARMEmitter::BarrierRegister::DMB, FEXCore::ToUnderlying(Scope));\n }\n void isb() {\n Barrier(ARMEmitter::BarrierRegister::ISB, FEXCore::ToUnderlying(ARMEmitter::BarrierScope::SY));\n }\n void sb() {\n Barrier(ARMEmitter::BarrierRegister::SB, 0);\n }\n void tcommit() {\n Barrier(ARMEmitter::BarrierRegister::TCOMMIT, 0);\n }\n\n // System register move\n void msr(ARMEmitter::SystemRegister reg, ARMEmitter::Register rt) {\n constexpr uint32_t Op = 0b1101'0101'0001 << 20;\n SystemRegisterMove(Op, rt, reg);\n }\n\n void mrs(ARMEmitter::Register rd, ARMEmitter::SystemRegister reg) {\n constexpr uint32_t Op = 0b1101'0101'0011 << 20;\n SystemRegisterMove(Op, rd, reg);\n }\n\nprivate:\n\n // Exception Generation\n void ExceptionGeneration(uint32_t opc, uint32_t op2, uint32_t LL, uint32_t Imm) {\n LOGMAN_THROW_A_FMT((Imm & 0xFFFF'0000) == 0, \"Imm amount too large\");\n\n uint32_t Instr = 0b1101'0100 << 24;\n\n Instr |= opc << 21;\n Instr |= Imm << 5;\n Instr |= op2 << 2;\n Instr |= LL;\n\n dc32(Instr);\n }\n\n // System instructions with register argument\n void SystemInstructionWithReg(uint32_t CRm, uint32_t op2, ARMEmitter::Register rt) {\n uint32_t Instr = 0b1101'0101'0000'0011'0001 << 12;\n\n Instr |= CRm << 8;\n Instr |= op2 << 5;\n Instr |= Encode_rt(rt);\n dc32(Instr);\n }\n\n // Hints\n void Hint(ARMEmitter::HintRegister Reg) {\n uint32_t Instr = 0b1101'0101'0000'0011'0010'0000'0001'1111U;\n Instr |= FEXCore::ToUnderlying(Reg);\n dc32(Instr);\n }\n // Barriers\n void Barrier(ARMEmitter::BarrierRegister Reg, uint32_t CRm) {\n uint32_t Instr = 0b1101'0101'0000'0011'0011'0000'0001'1111U;\n Instr |= CRm << 8;\n Instr |= FEXCore::ToUnderlying(Reg);\n dc32(Instr);\n }\n\n // System Instruction\n void SystemInstruction(uint32_t Op, uint32_t L, uint32_t SubOp, ARMEmitter::Register rt) {\n uint32_t Instr = Op;\n\n Instr |= L << 21;\n Instr |= SubOp;\n Instr |= Encode_rt(rt);\n\n dc32(Instr);\n }\n\n // System register move\n void SystemRegisterMove(uint32_t Op, ARMEmitter::Register rt, ARMEmitter::SystemRegister reg) {\n uint32_t Instr = Op;\n\n Instr |= FEXCore::ToUnderlying(reg);\n Instr |= Encode_rt(rt);\n\n dc32(Instr);\n }\n\n#ifndef INCLUDED_BY_EMITTER\n}; // struct LoadstoreEmitterOps\n} // namespace ARMEmitter\n#endif\n" }, { "path": "CodeEmitter/CodeEmitter/VixlUtils.inl", "content": "// Collection of utilities from vixl.\n// Following is the vixl license.\n// Copyright 2015, VIXL authors\n// All rights reserved.\n//\n// Redistribution and use in source and binary forms, with or without\n// modification, are permitted provided that the following conditions are met:\n//\n// * Redistributions of source code must retain the above copyright notice,\n// this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright notice,\n// this list of conditions and the following disclaimer in the documentation\n// and/or other materials provided with the distribution.\n// * Neither the name of ARM Limited nor the names of its contributors may be\n// used to endorse or promote products derived from this software without\n// specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS \"AS IS\" AND\n// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\n// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\n// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE\n// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL\n// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR\n// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER\n// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,\n// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\n\n// Test if a given value can be encoded in the immediate field of a logical\n// instruction.\n// If it can be encoded, the function returns true, and values pointed to by n,\n// imm_s and imm_r are updated with immediates encoded in the format required\n// by the corresponding fields in the logical instruction.\n// If it can not be encoded, the function returns false, and the values pointed\n// to by n, imm_s and imm_r are undefined.\nstatic bool IsImmLogical(uint64_t value, unsigned width, unsigned* n = nullptr, unsigned* imm_s = nullptr, unsigned* imm_r = nullptr) {\n [[maybe_unused]] constexpr auto kBRegSize = 8;\n [[maybe_unused]] constexpr auto kHRegSize = 16;\n [[maybe_unused]] constexpr auto kSRegSize = 32;\n [[maybe_unused]] constexpr auto kDRegSize = 64;\n\n constexpr auto kWRegSize = 32;\n\n LOGMAN_THROW_A_FMT((width == kBRegSize) || (width == kHRegSize) || (width == kSRegSize) || (width == kDRegSize), \"Unexpected imm size\");\n\n bool negate = false;\n\n // Logical immediates are encoded using parameters n, imm_s and imm_r using\n // the following table:\n //\n // N imms immr size S R\n // 1 ssssss rrrrrr 64 UInt(ssssss) UInt(rrrrrr)\n // 0 0sssss xrrrrr 32 UInt(sssss) UInt(rrrrr)\n // 0 10ssss xxrrrr 16 UInt(ssss) UInt(rrrr)\n // 0 110sss xxxrrr 8 UInt(sss) UInt(rrr)\n // 0 1110ss xxxxrr 4 UInt(ss) UInt(rr)\n // 0 11110s xxxxxr 2 UInt(s) UInt(r)\n // (s bits must not be all set)\n //\n // A pattern is constructed of size bits, where the least significant S+1 bits\n // are set. The pattern is rotated right by R, and repeated across a 32 or\n // 64-bit value, depending on destination register width.\n //\n // Put another way: the basic format of a logical immediate is a single\n // contiguous stretch of 1 bits, repeated across the whole word at intervals\n // given by a power of 2. To identify them quickly, we first locate the\n // lowest stretch of 1 bits, then the next 1 bit above that; that combination\n // is different for every logical immediate, so it gives us all the\n // information we need to identify the only logical immediate that our input\n // could be, and then we simply check if that's the value we actually have.\n //\n // (The rotation parameter does give the possibility of the stretch of 1 bits\n // going 'round the end' of the word. To deal with that, we observe that in\n // any situation where that happens the bitwise NOT of the value is also a\n // valid logical immediate. So we simply invert the input whenever its low bit\n // is set, and then we know that the rotated case can't arise.)\n\n if (value & 1) {\n // If the low bit is 1, negate the value, and set a flag to remember that we\n // did (so that we can adjust the return values appropriately).\n negate = true;\n value = ~value;\n }\n\n if (width <= kWRegSize) {\n // To handle 8/16/32-bit logical immediates, the very easiest thing is to repeat\n // the input value to fill a 64-bit word. The correct encoding of that as a\n // logical immediate will also be the correct encoding of the value.\n\n // Avoid making the assumption that the most-significant 56/48/32 bits are zero by\n // shifting the value left and duplicating it.\n for (unsigned bits = width; bits <= kWRegSize; bits *= 2) {\n value <<= bits;\n uint64_t mask = (UINT64_C(1) << bits) - 1;\n value |= ((value >> bits) & mask);\n }\n }\n\n // The basic analysis idea: imagine our input word looks like this.\n //\n // 0011111000111110001111100011111000111110001111100011111000111110\n // c b a\n // |<--d-->|\n //\n // We find the lowest set bit (as an actual power-of-2 value, not its index)\n // and call it a. Then we add a to our original number, which wipes out the\n // bottommost stretch of set bits and replaces it with a 1 carried into the\n // next zero bit. Then we look for the new lowest set bit, which is in\n // position b, and subtract it, so now our number is just like the original\n // but with the lowest stretch of set bits completely gone. Now we find the\n // lowest set bit again, which is position c in the diagram above. Then we'll\n // measure the distance d between bit positions a and c (using CLZ), and that\n // tells us that the only valid logical immediate that could possibly be equal\n // to this number is the one in which a stretch of bits running from a to just\n // below b is replicated every d bits.\n uint64_t a = LowestSetBit(value);\n uint64_t value_plus_a = value + a;\n uint64_t b = LowestSetBit(value_plus_a);\n uint64_t value_plus_a_minus_b = value_plus_a - b;\n uint64_t c = LowestSetBit(value_plus_a_minus_b);\n\n int d, clz_a, out_n;\n uint64_t mask;\n\n if (c != 0) {\n // The general case, in which there is more than one stretch of set bits.\n // Compute the repeat distance d, and set up a bitmask covering the basic\n // unit of repetition (i.e. a word with the bottom d bits set). Also, in all\n // of these cases the N bit of the output will be zero.\n clz_a = std::countl_zero(a);\n int clz_c = std::countl_zero(c);\n d = clz_a - clz_c;\n mask = ((UINT64_C(1) << d) - 1);\n out_n = 0;\n } else {\n // Handle degenerate cases.\n //\n // If any of those 'find lowest set bit' operations didn't find a set bit at\n // all, then the word will have been zero thereafter, so in particular the\n // last lowest_set_bit operation will have returned zero. So we can test for\n // all the special case conditions in one go by seeing if c is zero.\n if (a == 0) {\n // The input was zero (or all 1 bits, which will come to here too after we\n // inverted it at the start of the function), for which we just return\n // false.\n return false;\n } else {\n // Otherwise, if c was zero but a was not, then there's just one stretch\n // of set bits in our word, meaning that we have the trivial case of\n // d == 64 and only one 'repetition'. Set up all the same variables as in\n // the general case above, and set the N bit in the output.\n clz_a = std::countl_zero(a);\n d = 64;\n mask = ~UINT64_C(0);\n out_n = 1;\n }\n }\n\n // If the repeat period d is not a power of two, it can't be encoded.\n if (!std::has_single_bit(uint32_t(d))) {\n return false;\n }\n\n if (((b - a) & ~mask) != 0) {\n // If the bit stretch (b - a) does not fit within the mask derived from the\n // repeat period, then fail.\n return false;\n }\n\n // The only possible option is b - a repeated every d bits. Now we're going to\n // actually construct the valid logical immediate derived from that\n // specification, and see if it equals our original input.\n //\n // To repeat a value every d bits, we multiply it by a number of the form\n // (1 + 2^d + 2^(2d) + ...), i.e. 0x0001000100010001 or similar. These can\n // be derived using a table lookup on CLZ(d).\n static const uint64_t multipliers[] = {\n 0x0000000000000001UL, 0x0000000100000001UL, 0x0001000100010001UL, 0x0101010101010101UL, 0x1111111111111111UL, 0x5555555555555555UL,\n };\n uint64_t multiplier = multipliers[std::countl_zero(uint64_t(d)) - 57];\n uint64_t candidate = (b - a) * multiplier;\n\n if (value != candidate) {\n // The candidate pattern doesn't match our input value, so fail.\n return false;\n }\n\n // We have a match! This is a valid logical immediate, so now we have to\n // construct the bits and pieces of the instruction encoding that generates\n // it.\n\n // Count the set bits in our basic stretch. The special case of clz(0) == -1\n // makes the answer come out right for stretches that reach the very top of\n // the word (e.g. numbers like 0xffffc00000000000).\n int clz_b = (b == 0) ? -1 : std::countl_zero(b);\n int s = clz_a - clz_b;\n\n // Decide how many bits to rotate right by, to put the low bit of that basic\n // stretch in position a.\n int r;\n if (negate) {\n // If we inverted the input right at the start of this function, here's\n // where we compensate: the number of set bits becomes the number of clear\n // bits, and the rotation count is based on position b rather than position\n // a (since b is the location of the 'lowest' 1 bit after inversion).\n s = d - s;\n r = (clz_b + 1) & (d - 1);\n } else {\n r = (clz_a + 1) & (d - 1);\n }\n\n // Now we're done, except for having to encode the S output in such a way that\n // it gives both the number of set bits and the length of the repeated\n // segment. The s field is encoded like this:\n //\n // imms size S\n // ssssss 64 UInt(ssssss)\n // 0sssss 32 UInt(sssss)\n // 10ssss 16 UInt(ssss)\n // 110sss 8 UInt(sss)\n // 1110ss 4 UInt(ss)\n // 11110s 2 UInt(s)\n //\n // So we 'or' (2 * -d) with our computed s to form imms.\n if (n != nullptr) {\n *n = out_n;\n }\n if (imm_s != nullptr) {\n *imm_s = ((2 * -d) | (s - 1)) & 0x3f;\n }\n if (imm_r != nullptr) {\n *imm_r = r;\n }\n\n return true;\n}\n\nstatic inline bool IsIntN(unsigned n, int64_t x) {\n if (n == 64) {\n return true;\n }\n int64_t limit = INT64_C(1) << (n - 1);\n return (-limit <= x) && (x < limit);\n}\n\nstatic inline bool IsUintN(unsigned n, int64_t x) {\n // Convert to an unsigned integer to avoid implementation-defined behavior.\n return !(static_cast(x) >> n);\n}\n\n// clang-format off\n#define INT_1_TO_32_LIST(V) \\\nV(1) V(2) V(3) V(4) V(5) V(6) V(7) V(8) \\\nV(9) V(10) V(11) V(12) V(13) V(14) V(15) V(16) \\\nV(17) V(18) V(19) V(20) V(21) V(22) V(23) V(24) \\\nV(25) V(26) V(27) V(28) V(29) V(30) V(31) V(32)\n\n#define INT_33_TO_63_LIST(V) \\\nV(33) V(34) V(35) V(36) V(37) V(38) V(39) V(40) \\\nV(41) V(42) V(43) V(44) V(45) V(46) V(47) V(48) \\\nV(49) V(50) V(51) V(52) V(53) V(54) V(55) V(56) \\\nV(57) V(58) V(59) V(60) V(61) V(62) V(63)\n\n#define INT_1_TO_63_LIST(V) INT_1_TO_32_LIST(V) INT_33_TO_63_LIST(V)\n\n// clang-format on\n\n#define DECLARE_IS_INT_N(N) \\\n static inline bool IsInt##N(int64_t x) { \\\n return IsIntN(N, x); \\\n }\n\n#define DECLARE_IS_UINT_N(N) \\\n static inline bool IsUint##N(int64_t x) { \\\n return IsUintN(N, x); \\\n }\n\nINT_1_TO_63_LIST(DECLARE_IS_INT_N)\nINT_1_TO_63_LIST(DECLARE_IS_UINT_N)\n\n#undef DECLARE_IS_INT_N\n#undef DECLARE_IS_UINT_N\n\nprivate:\n\n// Some compilers dislike negating unsigned integers,\n// so we provide an equivalent.\ntemplate\nstatic inline T UnsignedNegate(T value) {\n static_assert(std::is_unsigned::value);\n return ~value + 1;\n}\n\nstatic inline uint64_t LowestSetBit(uint64_t value) {\n return value & UnsignedNegate(value);\n}\n\npublic:\n" }, { "path": "Data/AppConfig/CMakeLists.txt", "content": "file(GLOB CONFIG_SOURCES CONFIGURE_DEPENDS *.json)\nfile(GLOB GEN_CONFIG_SOURCES CONFIGURE_DEPENDS *.json.in)\n\n# Any application configuration json file gets installed\nforeach(CONFIG_SRC ${CONFIG_SOURCES})\n install(FILES ${CONFIG_SRC}\n DESTINATION ${DATA_DIRECTORY}/AppConfig/\n COMPONENT Runtime)\nendforeach()\n\n# Any configuration file json file that needs to be generated\n# First generate then install it\nforeach(GEN_CONFIG_SRC ${GEN_CONFIG_SOURCES})\n # Get the filename only component\n get_filename_component(CONFIG_NAME ${GEN_CONFIG_SRC} NAME_WLE)\n\n # Configure it\n configure_file(${GEN_CONFIG_SRC} ${CMAKE_BINARY_DIR}/Data/AppConfig/${CONFIG_NAME})\n\n # Then install the configured json\n install(FILES ${CMAKE_BINARY_DIR}/Data/AppConfig/${CONFIG_NAME}\n DESTINATION ${DATA_DIRECTORY}/AppConfig/\n COMPONENT Runtime)\nendforeach()\n" }, { "path": "Data/AppConfig/client.json", "content": "{\n \"Config\": {\n \"HideHypervisorBit\": \"1\"\n }\n}\n" }, { "path": "Data/AppConfig/steamwebhelper.json", "content": "{\n \"Comment\": \"Bypasses libGL's glX and instead sends GLX requests directly via xcb\",\n \"ThunksDB\": {\n \"GL\": 0\n }\n}\n" }, { "path": "Data/CI/FEXLinuxTestsThunks.json", "content": "{\n \"ThunksDB\": {\n \"fex_thunk_test\": 1\n }\n}\n" }, { "path": "Data/CI/GLThunks.json", "content": "{\n \"ThunksDB\": {\n \"GL\": 1\n }\n}\n" }, { "path": "Data/CI/VulkanThunks.json", "content": "{\n \"ThunksDB\": {\n \"Vulkan\": 1\n }\n}\n" }, { "path": "Data/CMake/FindZycore.cmake", "content": "# SPDX-License-Identifier: MIT\n\nif (CMAKE_CROSSCOMPILING)\n return()\nendif()\n\ninclude(FindPackageHandleStandardArgs)\n\nfind_package(Zycore QUIET CONFIG)\n\nif (Zycore_CONSIDERED_CONFIGS)\n find_package_handle_standard_args(Zycore CONFIG_MODE)\nelse()\n find_package(PkgConfig QUIET)\n pkg_search_module(Zycore QUIET IMPORTED_TARGET zycore)\n find_package_handle_standard_args(Zycore\n REQUIRED_VARS zycore_LINK_LIBRARIES\n VERSION_VAR zycore_VERSION)\n\n if (TARGET PkgConfig::zycore)\n add_library(Zycore::Zycore ALIAS PkgConfig::zycore)\n endif()\nendif()\n" }, { "path": "Data/CMake/FindZydis.cmake", "content": "# SPDX-License-Identifier: MIT\n\nif (CMAKE_CROSSCOMPILING)\n return()\nendif()\n\ninclude(FindPackageHandleStandardArgs)\n\nfind_package(Zydis QUIET CONFIG)\n\nif (Zydis_CONSIDERED_CONFIGS)\n find_package_handle_standard_args(Zydis CONFIG_MODE)\nelse()\n find_package(PkgConfig QUIET)\n pkg_search_module(Zydis QUIET IMPORTED_TARGET zydis)\n find_package_handle_standard_args(Zydis\n REQUIRED_VARS zydis_LINK_LIBRARIES\n VERSION_VAR zydis_VERSION)\n\n if (TARGET PkgConfig::zydis)\n add_library(Zydis::Zydis ALIAS PkgConfig::zydis)\n endif()\nendif()\n" }, { "path": "Data/CMake/Findxxhash.cmake", "content": "# SPDX-License-Identifier: MIT\n\ninclude(FindPackageHandleStandardArgs)\n\nfind_package(PkgConfig QUIET)\npkg_search_module(xxhash QUIET IMPORTED_TARGET xxhash libxxhash)\nfind_package_handle_standard_args(xxhash\n REQUIRED_VARS xxhash_LINK_LIBRARIES\n VERSION_VAR xxhash_VERSION\n)\n\nif (xxhash_FOUND AND NOT TARGET xxHash::xxhash)\n if (TARGET PkgConfig::xxhash)\n add_library(xxHash::xxhash ALIAS PkgConfig::xxhash)\n else()\n add_library(xxHash::xxhash ALIAS xxhash)\n endif()\nendif()\n" }, { "path": "Data/CMake/LinkerGC.cmake", "content": "# SPDX-License-Identifier: MIT\n\n# This applies some common linker options that reduce code size and linking time in Release mode. Namely:\n# --gc-sections: Linktime garbage collection, discards unused sections from the final output\n# --strip-all : Similar to running `strip`, discards the symbol table from the final output\n# --as-needed : Only includes libraries that are actually needed in the final output.\n\nmacro(LinkerGC target)\n if (CMAKE_BUILD_TYPE MATCHES \"RELEASE\")\n target_link_options(${target} PRIVATE\n \"LINKER:--gc-sections\"\n \"LINKER:--strip-all\"\n \"LINKER:--as-needed\")\n endif()\nendmacro()\n" }, { "path": "Data/CMake/cmake_uninstall.cmake.in", "content": "if(NOT EXISTS \"@CMAKE_BINARY_DIR@/install_manifest.txt\")\n message(FATAL_ERROR \"Cannot find install manifest: @CMAKE_BINARY_DIR@/install_manifest.txt\")\nendif()\n\nfile(READ \"@CMAKE_BINARY_DIR@/install_manifest.txt\" files)\nstring(REGEX REPLACE \"\\n\" \";\" files \"${files}\")\nforeach(file ${files})\n message(STATUS \"Uninstalling $ENV{DESTDIR}${file}\")\n if(IS_SYMLINK \"$ENV{DESTDIR}${file}\" OR EXISTS \"$ENV{DESTDIR}${file}\")\n exec_program(\n \"@CMAKE_COMMAND@\" ARGS \"-E remove \\\"$ENV{DESTDIR}${file}\\\"\"\n OUTPUT_VARIABLE rm_out\n RETURN_VALUE rm_retval\n )\n if(NOT \"${rm_retval}\" STREQUAL 0)\n message(FATAL_ERROR \"Problem when removing $ENV{DESTDIR}${file}\")\n endif()\n else(IS_SYMLINK \"$ENV{DESTDIR}${file}\" OR EXISTS \"$ENV{DESTDIR}${file}\")\n message(STATUS \"File $ENV{DESTDIR}${file} does not exist.\")\n endif()\nendforeach()\n" }, { "path": "Data/CMake/toolchain_aarch64.cmake", "content": "# This is a reference AArch64 cross compile script\n# Pass in to cmake when building:\n# eg: cmake --toolchain ../Data/CMake/toolchain_aarch64.cmake ..\nif (NOT DEFINED ENV{SYSROOT})\n message(FATAL_ERROR \"Need to have SYSROOT environment variable set\")\nendif()\n\nset(CMAKE_SYSTEM_NAME Linux)\nset(CMAKE_SYSTEM_PROCESSOR aarch64)\nset(CMAKE_CROSSCOMPILING TRUE)\n\n# Target triple needs to match the binutils exactly\nset(TARGET_TRIPLE aarch64-linux-gnu)\nset(CMAKE_C_COMPILER \"clang\")\nset(CMAKE_CXX_COMPILER \"clang++\")\nset(CMAKE_C_COMPILER_AR \"llvm-ar\")\nset(CMAKE_CXX_COMPILER_AR \"llvm-ar\")\nset(CMAKE_C_COMPILER_RANLIB \"llvm-ranlib\")\nset(CMAKE_CXX_COMPILER_RANLIB \"llvm-ranlib\")\nset(CMAKE_LINKER \"ld.lld\")\n\nset(CMAKE_C_COMPILER_TARGET ${TARGET_TRIPLE})\nset(CMAKE_CXX_COMPILER_TARGET ${TARGET_TRIPLE})\n\n# Set the environment variable SYSROOT to the aarch64 rootfs\nset(CMAKE_FIND_ROOT_PATH \"$ENV{SYSROOT}\")\nset(CMAKE_SYSROOT \"$ENV{SYSROOT}\")\n\nlist(APPEND CMAKE_PREFIX_PATH \"$ENV{SYSROOT}/usr/lib/${TARGET_TRIPLE}/cmake/\")\n\nset(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)\n\nset(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)\nset(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)\nset(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)\n" }, { "path": "Data/CMake/toolchain_mingw.cmake", "content": "set(MINGW_TRIPLE \"\" CACHE STRING \"MinGW compiler target architecture triple\")\n\nset(CMAKE_RC_COMPILER ${MINGW_TRIPLE}-windres)\nset(CMAKE_C_COMPILER ${MINGW_TRIPLE}-clang)\nset(CMAKE_CXX_COMPILER ${MINGW_TRIPLE}-clang++)\nset(CMAKE_DLLTOOL ${MINGW_TRIPLE}-dlltool)\nset(CMAKE_AR ${MINGW_TRIPLE}-ar)\n\n# Compile everything as static to avoid requiring the MinGW runtime libraries, force page aligned sections so that\n# debug symbols work correctly, and disable loop alignment to workaround an LLVM bug\n# (https://github.com/llvm/llvm-project/issues/47432)\nset(CMAKE_SHARED_LINKER_FLAGS_INIT \"-static -static-libgcc -static-libstdc++ -Wl,--file-alignment=4096,/mllvm:-align-loops=1\")\nset(CMAKE_EXE_LINKER_FLAGS_INIT \"-static -static-libgcc -static-libstdc++ -Wl,--file-alignment=4096,/mllvm:-align-loops=1\")\nset(CMAKE_C_STANDARD_LIBRARIES \"\" CACHE STRING \"\" FORCE)\nset(CMAKE_CXX_STANDARD_LIBRARIES \"\" CACHE STRING \"\" FORCE)\nset(CMAKE_STANDARD_LIBRARIES \"\" CACHE STRING \"\" FORCE)\nset(CMAKE_SYSTEM_NAME Windows)\nset(CMAKE_SYSTEM_PROCESSOR ${MINGW_TRIPLE})\n\nset(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)\n" }, { "path": "Data/CMake/toolchain_x86_32.cmake", "content": "option(ENABLE_CLANG_THUNKS \"Enable building thunks with clang\" FALSE)\n\nset(CMAKE_SYSTEM_PROCESSOR i686)\n\nif (ENABLE_CLANG_THUNKS)\n message(STATUS \"Enabling thunk clang building. Force enabling LLD as well\")\n\n set(CMAKE_EXE_LINKER_FLAGS_INIT \"-fuse-ld=lld\")\n set(CMAKE_MODULE_LINKER_FLAGS_INIT \"-fuse-ld=lld\")\n set(CMAKE_SHARED_LINKER_FLAGS_INIT \"-fuse-ld=lld\")\n set(CMAKE_C_COMPILER clang)\n set(CMAKE_CXX_COMPILER clang++)\n set(CLANG_FLAGS \"-target i686-linux-gnu -msse2 -mfpmath=sse\")\n\n set(CMAKE_C_FLAGS \"${CMAKE_C_FLAGS} ${CLANG_FLAGS}\")\n set(CMAKE_CXX_FLAGS \"${CMAKE_CXX_FLAGS} ${CLANG_FLAGS}\")\nelse()\n set(CMAKE_C_COMPILER x86_64-linux-gnu-gcc -m32)\n set(CMAKE_CXX_COMPILER x86_64-linux-gnu-g++ -m32)\nendif()\n" }, { "path": "Data/CMake/toolchain_x86_64.cmake", "content": "option(ENABLE_CLANG_THUNKS \"Enable building thunks with clang\" FALSE)\n\nset(CMAKE_SYSTEM_PROCESSOR x86_64)\n\nif (ENABLE_CLANG_THUNKS)\n message(STATUS \"Enabling thunk clang building. Force enabling LLD as well\")\n\n set(CMAKE_EXE_LINKER_FLAGS_INIT \"-fuse-ld=lld\")\n set(CMAKE_MODULE_LINKER_FLAGS_INIT \"-fuse-ld=lld\")\n set(CMAKE_SHARED_LINKER_FLAGS_INIT \"-fuse-ld=lld\")\n set(CMAKE_C_COMPILER clang)\n set(CMAKE_CXX_COMPILER clang++)\n set(CLANG_FLAGS \"-target x86_64-linux-gnu\")\n\n set(CMAKE_C_FLAGS \"${CMAKE_C_FLAGS} ${CLANG_FLAGS}\")\n set(CMAKE_CXX_FLAGS \"${CMAKE_CXX_FLAGS} ${CLANG_FLAGS}\")\nelse()\n set(CMAKE_C_COMPILER x86_64-linux-gnu-gcc)\n set(CMAKE_CXX_COMPILER x86_64-linux-gnu-g++)\nendif()\n" }, { "path": "Data/CMake/version_to_variables.cmake", "content": "# Extracts a version from the passed in version string in the form of \"..\".\n# If a part of the version is missing then it gets set as zero.\n# Version variables returned in:\n# ${Package}_VERSION_MAJOR\n# ${Package}_VERSION_MINOR\n# ${Package}_VERSION_PATCH\nfunction(version_to_variables VERSION _Package)\n string(REPLACE \".\" \";\" VERSION_LIST \"${VERSION}\")\n list (LENGTH VERSION_LIST VERSION_LEN)\n if (${VERSION_LEN} GREATER 0)\n list(GET VERSION_LIST 0 VERSION_MAJOR)\n set(${_Package}_VERSION_MAJOR ${VERSION_MAJOR} PARENT_SCOPE)\n else()\n set(${_Package}_VERSION_MAJOR 0 PARENT_SCOPE)\n endif()\n\n if (${VERSION_LEN} GREATER 1)\n list(GET VERSION_LIST 1 VERSION_MINOR)\n set(${_Package}_VERSION_MINOR ${VERSION_MINOR} PARENT_SCOPE)\n else()\n set(${_Package}_VERSION_MINOR 0 PARENT_SCOPE)\n endif()\n\n if (${VERSION_LEN} GREATER 2)\n list(GET VERSION_LIST 2 VERSION_PATCH)\n set(${_Package}_VERSION_PATCH ${VERSION_PATCH} PARENT_SCOPE)\n else()\n set(${_Package}_VERSION_PATCH 0 PARENT_SCOPE)\n endif()\nendfunction()\n" }, { "path": "Data/Dockerfile", "content": "# --- Stage 1: Builder ---\nFROM ubuntu:22.04 as builder\n\nRUN DEBIAN_FRONTEND=\"noninteractive\" apt-get update\nRUN DEBIAN_FRONTEND=\"noninteractive\" apt install -y cmake \\\nclang-13 llvm-13 nasm ninja-build pkg-config \\\nlibcap-dev libglfw3-dev libepoxy-dev python3-dev libsdl2-dev \\\npython3 linux-headers-generic \\\ngit qtbase5-dev qtdeclarative5-dev lld\n\nRUN git clone --recurse-submodules https://github.com/FEX-Emu/FEX.git\nWORKDIR /FEX\nRUN mkdir build\n\nARG CC=clang-13\nARG CXX=clang++-13\nRUN cmake -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_BUILD_TYPE=Release -DUSE_LINKER=lld -DENABLE_LTO=True -DBUILD_TESTING=False -DENABLE_ASSERTIONS=False -G Ninja .\nRUN ninja\n\nWORKDIR /FEX/build\n\n# --- Stage 2: Runner ---\nFROM builder as runner\n\nRUN DEBIAN_FRONTEND=\"noninteractive\" apt-get update\nRUN DEBIAN_FRONTEND=\"noninteractive\" apt install -y \\\nlibcap-dev libglfw3-dev libepoxy-dev\n\nCOPY --from=builder /FEX/Bin/* /usr/bin/\n\nWORKDIR /\n" }, { "path": "Data/ThunksDB.json", "content": "{\n \"DB\": {\n \"GL\": {\n \"Library\" : \"libGL-guest.so\",\n \"Overlay\": [\n \"@PREFIX_LIB@/libGL.so\",\n \"@PREFIX_LIB@/libGL.so.1\",\n \"@PREFIX_LIB@/libGL.so.1.2.0\",\n \"@PREFIX_LIB@/libGL.so.1.7.0\"\n ]\n },\n \"Vulkan\": {\n \"Library\": \"libvulkan-guest.so\",\n \"Overlay\": [\n \"@PREFIX_LIB@/libvulkan.so\",\n \"@PREFIX_LIB@/libvulkan.so.1\",\n \"@HOME@/.local/share/Steam/ubuntu12_32/steam-runtime/pinned_libs_64/libvulkan.so.1\"\n ]\n },\n \"drm\": {\n \"Library\": \"libdrm-guest.so\",\n \"Overlay\": [\n \"@PREFIX_LIB@/libdrm.so\",\n \"@PREFIX_LIB@/libdrm.so.2\",\n \"@PREFIX_LIB@/libdrm.so.2.4.0\"\n ]\n },\n \"asound\": {\n \"Library\": \"libasound-guest.so\",\n \"Overlay\": [\n \"@PREFIX_LIB@/libasound.so\",\n \"@PREFIX_LIB@/libasound.so.2\",\n \"@PREFIX_LIB@/libasound.so.2.0.0\"\n ]\n },\n \"fex_thunk_test\": {\n \"Library\": \"libfex_thunk_test-guest.so\",\n \"Overlay\": [\n \"@PREFIX_LIB@/libfex_thunk_test.so\"\n ]\n },\n \"WaylandClient\": {\n \"Library\" : \"libwayland-client-guest.so\",\n \"Overlay\": [\n \"@PREFIX_LIB@/libwayland-client.so\",\n \"@PREFIX_LIB@/libwayland-client.so.0\",\n \"@PREFIX_LIB@/libwayland-client.so.0.20.0\"\n ]\n }\n }\n}\n" }, { "path": "Data/binfmts/CMakeLists.txt", "content": "function(GenBinFmt Name)\n # Get the filename only component\n get_filename_component(FMT_NAME ${Name} NAME_WE)\n\n # Configure it\n configure_file(${Name} ${CMAKE_BINARY_DIR}/Data/binfmts/${FMT_NAME})\n\n # Then install the configured binfmt\n install(FILES ${CMAKE_BINARY_DIR}/Data/binfmts/${FMT_NAME}\n DESTINATION ${CMAKE_INSTALL_PREFIX}/share/binfmts/\n COMPONENT Runtime)\nendfunction()\n\nif (NOT USE_LEGACY_BINFMTMISC)\n configure_file(FEX-x86.conf.in ${CMAKE_BINARY_DIR}/Data/binfmts/FEX-x86.conf)\n configure_file(FEX-x86_64.conf.in ${CMAKE_BINARY_DIR}/Data/binfmts/FEX-x86_64.conf)\n\n install(FILES ${CMAKE_BINARY_DIR}/Data/binfmts/FEX-x86.conf ${CMAKE_BINARY_DIR}/Data/binfmts/FEX-x86_64.conf\n DESTINATION ${CMAKE_INSTALL_PREFIX}/lib/binfmt.d/\n COMPONENT Runtime)\nelse()\n GenBinFmt(FEX-x86.in)\n GenBinFmt(FEX-x86_64.in)\nendif()\n" }, { "path": "Data/binfmts/FEX-x86.conf.in", "content": ":FEX-x86:M:0:\\x7fELF\\x01\\x01\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\\x00\\x03\\x00:\\xff\\xff\\xff\\xff\\xff\\xfe\\xfe\\x00\\x00\\x00\\x00\\xff\\xff\\xff\\xff\\xff\\xfe\\xff\\xff\\xff:@CMAKE_INSTALL_PREFIX@/bin/FEX:POCF\n" }, { "path": "Data/binfmts/FEX-x86.in", "content": "package fex\ninterpreter @CMAKE_INSTALL_PREFIX@/bin/FEX\nmagic \\x7fELF\\x01\\x01\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\\x00\\x03\\x00\noffset 0\nmask \\xff\\xff\\xff\\xff\\xff\\xfe\\xfe\\x00\\x00\\x00\\x00\\xff\\xff\\xff\\xff\\xff\\xfe\\xff\\xff\\xff\ncredentials yes\nfix_binary yes\npreserve yes\n" }, { "path": "Data/binfmts/FEX-x86_64.conf.in", "content": ":FEX-x86_64:M:0:\\x7fELF\\x02\\x01\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\\x00\\x3e\\x00:\\xff\\xff\\xff\\xff\\xff\\xfe\\xfe\\x00\\x00\\x00\\x00\\xff\\xff\\xff\\xff\\xff\\xfe\\xff\\xff\\xff:@CMAKE_INSTALL_PREFIX@/bin/FEX:POCF\n" }, { "path": "Data/binfmts/FEX-x86_64.in", "content": "package fex\ninterpreter @CMAKE_INSTALL_PREFIX@/bin/FEX\nmagic \\x7fELF\\x02\\x01\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\\x00\\x3e\\x00\noffset 0\nmask \\xff\\xff\\xff\\xff\\xff\\xfe\\xfe\\x00\\x00\\x00\\x00\\xff\\xff\\xff\\xff\\xff\\xfe\\xff\\xff\\xff\ncredentials yes\nfix_binary yes\npreserve yes\n" }, { "path": "Data/nix/FEXLinuxTests/shell.nix", "content": "{ pkgs ? import { } }:\n\nlet\n pkgsCross32 = pkgs.pkgsCross.gnu32;\n pkgsCross64 = pkgs.pkgsCross.gnu64;\n\n gcc32 = pkgs.writeText \"toolchain_nix_gcc_x86_32.txt\" ''\n set(CMAKE_SYSTEM_PROCESSOR i686)\n set(CMAKE_C_COMPILER ${pkgsCross32.buildPackages.gcc}/bin/i686-unknown-linux-gnu-gcc)\n set(CMAKE_CXX_COMPILER ${pkgsCross32.buildPackages.gcc}/bin/i686-unknown-linux-gnu-g++)\n '';\n\n gcc64 = pkgs.writeText \"toolchain_nix_gcc_x86_64.txt\" ''\n set(CMAKE_SYSTEM_PROCESSOR x86_64)\n set(CMAKE_C_COMPILER ${pkgsCross64.buildPackages.gcc}/bin/x86_64-unknown-linux-gnu-gcc)\n set(CMAKE_CXX_COMPILER ${pkgsCross64.buildPackages.gcc}/bin/x86_64-unknown-linux-gnu-g++)\n '';\nin\npkgs.mkShell {\n buildInputs = [\n pkgsCross64.buildPackages.clang\n pkgsCross32.buildPackages.clang\n ];\n\n shellHook = ''\n if [[ $- == *i* ]]; then\n echo \"toolchain32: ${gcc32}\"\n echo \"toolchain64: ${gcc64}\"\n echo \"\"\n echo \"Use \\$FEX_CMAKE_TOOLCHAINS to configure CMake.\"\n fi\n '';\n\n FEX_CMAKE_TOOLCHAINS = \"-DX86_32_TOOLCHAIN_FILE=${gcc32} -DX86_64_TOOLCHAIN_FILE=${gcc64}\";\n}\n" }, { "path": "Data/nix/LibraryForwarding/shell.nix", "content": "{ pkgs ? import { } }:\n\nlet\n pkgsCross32 = pkgs.pkgsCross.gnu32;\n pkgsCross64 = pkgs.pkgsCross.gnu64;\n\n devRootFS = pkgs.buildEnv {\n name = \"fex-dev-rootfs\";\n paths = [\n pkgsCross64.stdenv.cc.libc_dev\n pkgsCross32.stdenv.cc.libc_dev\n pkgsCross64.stdenv.cc.cc\n pkgsCross32.stdenv.cc.cc\n\n pkgs.alsa-lib.dev\n pkgs.libdrm.dev\n pkgs.libGL.dev\n pkgs.wayland.dev\n pkgs.xorg.libX11.dev\n pkgs.xorg.libxcb.dev\n pkgs.xorg.libXrandr.dev\n pkgs.xorg.libXrender.dev\n pkgs.xorg.xorgproto\n ];\n ignoreCollisions = true;\n pathsToLink = [\n \"/include\"\n \"/lib\"\n ];\n\n postBuild = ''\n mkdir -p $out/usr\n ln -s $out/include $out/usr/\n '';\n };\n\n toolchain32 = pkgs.writeText \"toolchain_nix_x86_32.txt\" ''\n set(CMAKE_EXE_LINKER_FLAGS_INIT \"-fuse-ld=lld\")\n set(CMAKE_MODULE_LINKER_FLAGS_INIT \"-fuse-ld=lld\")\n set(CMAKE_SHARED_LINKER_FLAGS_INIT \"-fuse-ld=lld\")\n set(CMAKE_SYSTEM_PROCESSOR i686)\n set(CMAKE_C_COMPILER clang)\n set(CMAKE_CXX_COMPILER clang++)\n set(CMAKE_C_COMPILER ${pkgsCross32.buildPackages.clang}/bin/i686-unknown-linux-gnu-clang)\n set(CMAKE_CXX_COMPILER ${pkgsCross32.buildPackages.clang}/bin/i686-unknown-linux-gnu-clang++)\n set(CLANG_FLAGS \"-nodefaultlibs -nostartfiles -lstdc++ -target i686-linux-gnu -msse2 -mfpmath=sse --sysroot=${devRootFS} -iwithsysroot/include\")\n set(CMAKE_C_FLAGS \"''${CMAKE_C_FLAGS} ''${CLANG_FLAGS}\")\n set(CMAKE_CXX_FLAGS \"''${CMAKE_CXX_FLAGS} ''${CLANG_FLAGS}\")\n '';\n\n toolchain64 = pkgs.writeText \"toolchain_nix_x86_64.txt\" ''\n set(CMAKE_EXE_LINKER_FLAGS_INIT \"-fuse-ld=lld\")\n set(CMAKE_MODULE_LINKER_FLAGS_INIT \"-fuse-ld=lld\")\n set(CMAKE_SHARED_LINKER_FLAGS_INIT \"-fuse-ld=lld\")\n set(CMAKE_SYSTEM_PROCESSOR x86_64)\n set(CMAKE_C_COMPILER clang)\n set(CMAKE_CXX_COMPILER clang++)\n set(CMAKE_C_COMPILER ${pkgsCross64.buildPackages.clang}/bin/x86_64-unknown-linux-gnu-clang)\n set(CMAKE_CXX_COMPILER ${pkgsCross64.buildPackages.clang}/bin/x86_64-unknown-linux-gnu-clang++)\n set(CLANG_FLAGS \"-nodefaultlibs -nostartfiles -lstdc++ -target x86_64-linux-gnu --sysroot=${devRootFS} -iwithsysroot/usr/include\")\n set(CMAKE_C_FLAGS \"''${CMAKE_C_FLAGS} ''${CLANG_FLAGS}\")\n set(CMAKE_CXX_FLAGS \"''${CMAKE_CXX_FLAGS} ''${CLANG_FLAGS}\")\n '';\nin\npkgs.mkShell {\n buildInputs = [\n pkgsCross64.buildPackages.clang\n pkgsCross32.buildPackages.clang\n ];\n\n shellHook = ''\n if [[ $- == *i* ]]; then\n echo \"Set up dev RootFS at ${devRootFS}\"\n echo \"toolchain32: ${toolchain32}\"\n echo \"toolchain64: ${toolchain64}\"\n echo \"\"\n echo \"Use \\$FEX_CMAKE_TOOLCHAINS to configure CMake.\"\n fi\n '';\n\n FEX_CMAKE_TOOLCHAINS = \"-DX86_32_TOOLCHAIN_FILE=${toolchain32} -DX86_64_TOOLCHAIN_FILE=${toolchain64} -DX86_DEV_ROOTFS=${devRootFS}\";\n ROOTFS = \"${devRootFS}\";\n}\n" }, { "path": "Data/nix/WineOnArm/shell.nix", "content": "{ pkgs ? import { } }:\n\nlet\n toolchain = pkgs.fetchzip {\n url = \"https://github.com/bylaws/llvm-mingw/releases/download/20250920/llvm-mingw-20250920-ucrt-ubuntu-22.04-aarch64.tar.xz\";\n sha256 = \"sha256-LaojKjC8KzY+soW5u6eoDoXE3qtYk9Ejr7M3enTqRAE=\";\n };\n\n cmakeToolchainFile = pkgs.substitute {\n # Use absolute paths that are discoverable outside of the nix shell\n src = ../../CMake/toolchain_mingw.cmake;\n substitutions = [\"--replace-fail\" \"\\${MINGW_TRIPLE}-\" \"${toolchain}/bin/\\${MINGW_TRIPLE}-\"];\n };\n\n mesonCrossFile = pkgs.writeText \"crossfile_llvm_mingw.txt\" ''\n [binaries]\n ar = '${toolchain}/bin/arm64ec-w64-mingw32-ar'\n c = '${toolchain}/bin/arm64ec-w64-mingw32-gcc'\n cpp = '${toolchain}/bin/arm64ec-w64-mingw32-g++'\n ld = '${toolchain}/bin/arm64ec-w64-mingw32-ld'\n windres = '${toolchain}/bin/arm64ec-w64-mingw32-windres'\n strip = '${toolchain}/bin/strip'\n widl = '${toolchain}/bin/arm64ec-w64-mingw32-widl'\n pkgconfig = 'aarch64-linux-gnu-pkg-config'\n [host_machine]\n system = 'windows'\n cpu_family = 'aarch64'\n cpu = 'aarch64'\n endian = 'little'\n '';\nin\npkgs.mkShell {\n buildInputs = [\n toolchain\n ];\n\n shellHook = ''\n if [[ $- == *i* ]]; then\n echo \"llvm-mingw set up at ${toolchain}.\"\n echo \"\"\n echo \"To configure DXVK/vkd3d-proton: meson setup \\$FEX_MESON_CROSSFILE\"\n echo \"\"\n echo \"To configure 32-bit FEX build: cmake \\$FEX_CMAKE_TOOLCHAIN_WOW64\"\n echo \"To configure 64-bit FEX build: cmake \\$FEX_CMAKE_TOOLCHAIN_ARM64EC\"\n fi\n '';\n\n # E.g. cmake $FEX_CMAKE_TOOLCHAIN_ARM64EC -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr -DENABLE_LTO=False -DBUILD_TESTING=False\n FEX_CMAKE_TOOLCHAIN_ARM64EC = \"--toolchain ${cmakeToolchainFile} -DMINGW_TRIPLE=arm64ec-w64-mingw32 -DCMAKE_INSTALL_LIBDIR=/usr/lib/wine/aarch64-windows\";\n FEX_CMAKE_TOOLCHAIN_WOW64 = \"--toolchain ${cmakeToolchainFile} -DMINGW_TRIPLE=aarch64-w64-mingw32 -DCMAKE_INSTALL_LIBDIR=/usr/lib/wine/aarch64-windows\";\n FEX_MESON_CROSSFILE = \"--cross-file ${mesonCrossFile}\";\n}\n" }, { "path": "Data/nix/cmake_configure_woa32.sh", "content": "#! /usr/bin/env nix-shell\n#! nix-shell -i bash WineOnArm/shell.nix\n\n# Helper script to configure CMake for building FEX as library for emulation\n# of 32-bit applications in Wine/Proton.\n# The required cross-toolchains will be set up and managed by nix.\n\nif [ $# -eq 0 ]\nthen\n echo \"Expected CMake argument list\"\n exit 1\nfi\n\nif [ -f CMakeCache.txt ]\nthen\n echo \"Expected empty build folder\"\n exit 1\nfi\n\nset -o xtrace\ncmake $FEX_CMAKE_TOOLCHAIN_WOW64 -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr -DENABLE_LTO=False -DBUILD_TESTING=False $@\n" }, { "path": "Data/nix/cmake_configure_woa64.sh", "content": "#! /usr/bin/env nix-shell\n#! nix-shell -i bash WineOnArm/shell.nix\n\n# Helper script to configure CMake for building FEX as library for emulation\n# of 64-bit applications in Wine/Proton\n# Nix is used to install and manage the required cross-toolchains.\n\nif [ $# -eq 0 ]\nthen\n echo \"Expected CMake argument list\"\n exit 1\nfi\n\nif [ -f CMakeCache.txt ]\nthen\n echo \"Expected empty build folder\"\n exit 1\nfi\n\nset -o xtrace\ncmake $FEX_CMAKE_TOOLCHAIN_ARM64EC -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr -DENABLE_LTO=False -DBUILD_TESTING=False $@\n" }, { "path": "Data/nix/cmake_enable_flt.sh", "content": "#! /usr/bin/env nix-shell\n#! nix-shell -i bash FEXLinuxTests/shell.nix\n\n# Helper script to configure CMake for building FEXLinuxTests.\n# Nix is used to install and manage the required cross-toolchains.\n\nif [ ! -f CMakeCache.txt ]\nthen\n echo \"Must be run from a pre-configured CMake build folder\"\n exit 1\nfi\n\n# Remove previous build to ensure the new toolchain is applied\nrm -rf unittests/FEXLinuxTests\n\nset -o xtrace\ncmake . $FEX_CMAKE_TOOLCHAINS -DBUILD_TESTING=ON -DBUILD_FEX_LINUX_TESTS=ON\n" }, { "path": "Data/nix/cmake_enable_libfwd.sh", "content": "# Helper script to configure CMake for library forwarding in FEX.\n# Nix is used to install and manage the required cross-toolchains.\n\nif [ ! -f CMakeCache.txt ]\nthen\n echo \"Must be run from a pre-configured CMake build folder\"\n exit 1\nfi\n\n# Remove previous build to ensure the new toolchain is applied\nrm -rf guest-libs guest-libs-32 Guest Guest_32\n\n# Set clang executable path manually since the one from the nix store\n# will be picked up otherwise\nCLANG_EXEC_PATH=\"\"\nif ! grep -q CLANG_EXEC_PATH CMakeCache.txt\nthen\n CLANG_EXEC_PATH=\"-DCLANG_EXEC_PATH=`which clang`\"\nfi\n\nnix-shell `dirname -- \"$0\"`/LibraryForwarding/shell.nix \\\n --run \"set -o xtrace; cmake . \\$FEX_CMAKE_TOOLCHAINS -DBUILD_THUNKS=ON $CLANG_EXEC_PATH; set +o xtrace\"\n" }, { "path": "External/.clang-format", "content": "DisableFormat: true\n" }, { "path": "External/SoftFloat-3e/CMakeLists.txt", "content": "\nadd_library(softfloat_3e STATIC\n # F80 support\n src/extF80_add.c\n src/extF80_div.c\n src/extF80_sub.c\n src/extF80_mul.c\n src/extF80_rem.c\n src/extF80_sqrt.c\n src/extF80_le.c\n src/extF80_to_i32.c\n src/extF80_to_i64.c\n src/extF80_to_ui64.c\n src/extF80_to_f32.c\n src/extF80_to_f64.c\n src/i32_to_extF80.c\n src/ui64_to_extF80.c\n src/extF80_to_f128.c\n src/f128_to_extF80.c\n\n # F128 support\n src/f128_add.c\n src/f128_div.c\n src/f128_eq.c\n src/f128_eq_signaling.c\n src/f128_isSignalingNaN.c\n src/f128_le.c\n src/f128_le_quiet.c\n src/f128_lt.c\n src/f128_lt_quiet.c\n src/f128_mulAdd.c\n src/f128_mul.c\n src/f128_rem.c\n src/f128_sqrt.c\n src/f128_sub.c\n src/f128_to_f16.c\n src/f128_to_f32.c\n src/f128_to_f64.c\n src/f128_to_i32.c\n src/f128_to_i64.c\n src/f128_to_ui32.c\n src/f128_to_ui64.c\n src/s_addMagsF128.c\n src/s_subMagsF128.c\n src/s_normRoundPackToF128.c\n src/s_roundPackToF128.c\n src/s_propagateNaNF128UI.c\n\n # Conversion\n src/f32_to_f128.c\n src/i32_to_f128.c\n\n src/s_roundToUI64.c\n src/s_f128UIToCommonNaN.c\n src/s_commonNaNToF128UI.c\n src/s_normSubnormalF128Sig.c\n src/s_roundToI32.c\n src/s_roundToI64.c\n src/s_roundPackToF32.c\n src/s_addMagsExtF80.c\n src/s_extF80UIToCommonNaN.c\n src/s_commonNaNToF32UI.c\n src/s_commonNaNToF64UI.c\n src/s_roundPackToF64.c\n src/s_propagateNaNExtF80UI.c\n src/s_roundPackToExtF80.c\n src/s_normSubnormalExtF80Sig.c\n src/s_subMagsExtF80.c\n src/s_shiftRightJam128.c\n src/s_shiftRightJam128Extra.c\n src/s_normRoundPackToExtF80.c\n src/s_approxRecip_1Ks.c\n src/s_approxRecipSqrt32_1.c\n src/s_approxRecipSqrt_1Ks.c\n src/softfloat_raiseFlags.c\n src/f64_to_extF80.c\n src/s_commonNaNToExtF80UI.c\n src/s_normSubnormalF64Sig.c\n src/s_f64UIToCommonNaN.c\n src/extF80_roundToInt.c\n src/extF80_eq.c\n src/extF80_lt.c\n src/f32_to_extF80.c\n src/s_normSubnormalF32Sig.c\n src/s_f32UIToCommonNaN.c)\n\nif (ARCHITECTURE_arm64 AND HAS_CLANG_PRESERVE_ALL)\n list(APPEND DEFINES \"-DFEXCORE_PRESERVE_ALL_ATTR=__attribute__((preserve_all));-DFEXCORE_HAS_PRESERVE_ALL_ATTR=1\")\nelse()\n list(APPEND DEFINES \"-DFEXCORE_PRESERVE_ALL_ATTR=;-DFEXCORE_HAS_PRESERVE_ALL_ATTR=0\")\nendif()\n\nlist(APPEND DEFINES \"-DSOFTFLOAT_BUILTIN_CLZ=1;-DINLINE=static inline;-DINLINE_LEVEL=4;-DSOFTFLOAT_FAST_INT64=1;-DSOFTFLOAT_FAST_DIV32TO16=1;-DSOFTFLOAT_FAST_DIV64TO32=1\")\n\ntarget_include_directories(softfloat_3e PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include/)\ntarget_include_directories(softfloat_3e PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include/SoftFloat-3e/)\ntarget_compile_definitions(softfloat_3e PUBLIC ${DEFINES})\n" }, { "path": "External/SoftFloat-3e/include/SoftFloat-3e/opts-GCC.h", "content": "\r\n/*============================================================================\r\n\r\nThis C header file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2017 The Regents of the University of California. All rights\r\nreserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#ifndef opts_GCC_h\r\n#define opts_GCC_h 1\r\n\r\n#ifdef INLINE\r\n\r\n#include \r\n#include \"primitiveTypes.h\"\r\n\r\n#ifdef SOFTFLOAT_BUILTIN_CLZ\r\n\r\nINLINE uint_fast8_t softfloat_countLeadingZeros16( uint16_t a )\r\n { return a ? __builtin_clz( a ) - 16 : 16; }\r\n#define softfloat_countLeadingZeros16 softfloat_countLeadingZeros16\r\n\r\nINLINE uint_fast8_t softfloat_countLeadingZeros32( uint32_t a )\r\n { return a ? __builtin_clz( a ) : 32; }\r\n#define softfloat_countLeadingZeros32 softfloat_countLeadingZeros32\r\n\r\nINLINE uint_fast8_t softfloat_countLeadingZeros64( uint64_t a )\r\n { return a ? __builtin_clzll( a ) : 64; }\r\n#define softfloat_countLeadingZeros64 softfloat_countLeadingZeros64\r\n\r\n#endif\r\n\r\n#ifdef SOFTFLOAT_INTRINSIC_INT128\r\n\r\nINLINE struct uint128 softfloat_mul64ByShifted32To128( uint64_t a, uint32_t b )\r\n{\r\n union { unsigned __int128 ui; struct uint128 s; } uZ;\r\n uZ.ui = (unsigned __int128) a * ((uint_fast64_t) b<<32);\r\n return uZ.s;\r\n}\r\n#define softfloat_mul64ByShifted32To128 softfloat_mul64ByShifted32To128\r\n\r\nINLINE struct uint128 softfloat_mul64To128( uint64_t a, uint64_t b )\r\n{\r\n union { unsigned __int128 ui; struct uint128 s; } uZ;\r\n uZ.ui = (unsigned __int128) a * b;\r\n return uZ.s;\r\n}\r\n#define softfloat_mul64To128 softfloat_mul64To128\r\n\r\nINLINE\r\nstruct uint128 softfloat_mul128By32( uint64_t a64, uint64_t a0, uint32_t b )\r\n{\r\n union { unsigned __int128 ui; struct uint128 s; } uZ;\r\n uZ.ui = ((unsigned __int128) a64<<64 | a0) * b;\r\n return uZ.s;\r\n}\r\n#define softfloat_mul128By32 softfloat_mul128By32\r\n\r\nINLINE\r\nvoid\r\n softfloat_mul128To256M(\r\n uint64_t a64, uint64_t a0, uint64_t b64, uint64_t b0, uint64_t *zPtr )\r\n{\r\n unsigned __int128 z0, mid1, mid, z128;\r\n z0 = (unsigned __int128) a0 * b0;\r\n mid1 = (unsigned __int128) a64 * b0;\r\n mid = mid1 + (unsigned __int128) a0 * b64;\r\n z128 = (unsigned __int128) a64 * b64;\r\n z128 += (unsigned __int128) (mid < mid1)<<64 | mid>>64;\r\n mid <<= 64;\r\n z0 += mid;\r\n z128 += (z0 < mid);\r\n zPtr[indexWord( 4, 0 )] = z0;\r\n zPtr[indexWord( 4, 1 )] = z0>>64;\r\n zPtr[indexWord( 4, 2 )] = z128;\r\n zPtr[indexWord( 4, 3 )] = z128>>64;\r\n}\r\n#define softfloat_mul128To256M softfloat_mul128To256M\r\n\r\n#endif\r\n\r\n#endif\r\n\r\n#endif\r\n\r\n" }, { "path": "External/SoftFloat-3e/include/SoftFloat-3e/platform.h", "content": "\r\n/*============================================================================\r\n\r\nThis C header file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014, 2015, 2016, 2017 The Regents of the\r\nUniversity of California. All rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n/*----------------------------------------------------------------------------\r\n*----------------------------------------------------------------------------*/\r\n#define LITTLEENDIAN 1\r\n\r\n/*----------------------------------------------------------------------------\r\n*----------------------------------------------------------------------------*/\r\n#define SOFTFLOAT_BUILTIN_CLZ 1\r\n#define SOFTFLOAT_INTRINSIC_INT128 1\r\n#define SOFTFLOAT_FAST_INT64 1\r\n#include \"opts-GCC.h\"\r\n\r\n" }, { "path": "External/SoftFloat-3e/include/SoftFloat-3e/primitiveTypes.h", "content": "\r\n/*============================================================================\r\n\r\nThis C header file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014 The Regents of the University of California.\r\nAll rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#ifndef primitiveTypes_h\r\n#define primitiveTypes_h 1\r\n\r\n#include \r\n\r\n#ifdef SOFTFLOAT_FAST_INT64\r\n\r\n#ifdef LITTLEENDIAN\r\nstruct uint128 { uint64_t v0, v64; };\r\nstruct uint64_extra { uint64_t extra, v; };\r\nstruct uint128_extra { uint64_t extra; struct uint128 v; };\r\n#else\r\nstruct uint128 { uint64_t v64, v0; };\r\nstruct uint64_extra { uint64_t v, extra; };\r\nstruct uint128_extra { struct uint128 v; uint64_t extra; };\r\n#endif\r\n\r\n#endif\r\n\r\n/*----------------------------------------------------------------------------\r\n| These macros are used to isolate the differences in word order between big-\r\n| endian and little-endian platforms.\r\n*----------------------------------------------------------------------------*/\r\n#ifdef LITTLEENDIAN\r\n#define wordIncr 1\r\n#define indexWord( total, n ) (n)\r\n#define indexWordHi( total ) ((total) - 1)\r\n#define indexWordLo( total ) 0\r\n#define indexMultiword( total, m, n ) (n)\r\n#define indexMultiwordHi( total, n ) ((total) - (n))\r\n#define indexMultiwordLo( total, n ) 0\r\n#define indexMultiwordHiBut( total, n ) (n)\r\n#define indexMultiwordLoBut( total, n ) 0\r\n#define INIT_UINTM4( v3, v2, v1, v0 ) { v0, v1, v2, v3 }\r\n#else\r\n#define wordIncr -1\r\n#define indexWord( total, n ) ((total) - 1 - (n))\r\n#define indexWordHi( total ) 0\r\n#define indexWordLo( total ) ((total) - 1)\r\n#define indexMultiword( total, m, n ) ((total) - 1 - (m))\r\n#define indexMultiwordHi( total, n ) 0\r\n#define indexMultiwordLo( total, n ) ((total) - (n))\r\n#define indexMultiwordHiBut( total, n ) 0\r\n#define indexMultiwordLoBut( total, n ) (n)\r\n#define INIT_UINTM4( v3, v2, v1, v0 ) { v3, v2, v1, v0 }\r\n#endif\r\n\r\n#endif\r\n\r\n" }, { "path": "External/SoftFloat-3e/include/SoftFloat-3e/softfloat.h", "content": "\r\n/*============================================================================\r\n\r\nThis C header file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014, 2015, 2016, 2017 The Regents of the\r\nUniversity of California. All rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n\r\n/*============================================================================\r\n| Note: If SoftFloat is made available as a general library for programs to\r\n| use, it is strongly recommended that a platform-specific version of this\r\n| header, \"softfloat.h\", be created that folds in \"softfloat_types.h\" and that\r\n| eliminates all dependencies on compile-time macros.\r\n*============================================================================*/\r\n\r\n\r\n#ifndef softfloat_h\r\n#define softfloat_h 1\r\n\r\n#include \r\n#include \r\n#include \"softfloat_types.h\"\r\n\r\n/*----------------------------------------------------------------------------\r\n| Routine to raise any or all of the software floating-point exception flags.\r\n*----------------------------------------------------------------------------*/\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nvoid softfloat_raiseFlags( struct softfloat_state *, uint_fast8_t );\r\n\r\n/*----------------------------------------------------------------------------\r\n| Integer-to-floating-point conversion routines.\r\n*----------------------------------------------------------------------------*/\r\nfloat16_t ui32_to_f16( uint32_t );\r\nfloat32_t ui32_to_f32( uint32_t );\r\nfloat64_t ui32_to_f64( uint32_t );\r\n#ifdef SOFTFLOAT_FAST_INT64\r\nextFloat80_t ui32_to_extF80( uint32_t );\r\nfloat128_t ui32_to_f128( uint32_t );\r\n#endif\r\nvoid ui32_to_extF80M( uint32_t, extFloat80_t * );\r\nvoid ui32_to_f128M( uint32_t, float128_t * );\r\nfloat16_t ui64_to_f16( uint64_t );\r\nfloat32_t ui64_to_f32( uint64_t );\r\nfloat64_t ui64_to_f64( uint64_t );\r\n#ifdef SOFTFLOAT_FAST_INT64\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nextFloat80_t ui64_to_extF80( uint64_t );\r\nfloat128_t ui64_to_f128( uint64_t );\r\n#endif\r\nvoid ui64_to_extF80M( uint64_t, extFloat80_t * );\r\nvoid ui64_to_f128M( uint64_t, float128_t * );\r\nfloat16_t i32_to_f16( int32_t );\r\nfloat32_t i32_to_f32( int32_t );\r\nfloat64_t i32_to_f64( int32_t );\r\n#ifdef SOFTFLOAT_FAST_INT64\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nextFloat80_t i32_to_extF80( int32_t );\r\nfloat128_t i32_to_f128( int32_t );\r\n#endif\r\nvoid i32_to_extF80M( int32_t, extFloat80_t * );\r\nvoid i32_to_f128M( int32_t, float128_t * );\r\nfloat16_t i64_to_f16( int64_t );\r\nfloat32_t i64_to_f32( int64_t );\r\nfloat64_t i64_to_f64( int64_t );\r\n#ifdef SOFTFLOAT_FAST_INT64\r\nextFloat80_t i64_to_extF80( int64_t );\r\nfloat128_t i64_to_f128( int64_t );\r\n#endif\r\nvoid i64_to_extF80M( int64_t, extFloat80_t * );\r\nvoid i64_to_f128M( int64_t, float128_t * );\r\n\r\n/*----------------------------------------------------------------------------\r\n| 16-bit (half-precision) floating-point operations.\r\n*----------------------------------------------------------------------------*/\r\nuint_fast32_t f16_to_ui32( float16_t, uint_fast8_t, bool );\r\nuint_fast64_t f16_to_ui64( float16_t, uint_fast8_t, bool );\r\nint_fast32_t f16_to_i32( float16_t, uint_fast8_t, bool );\r\nint_fast64_t f16_to_i64( float16_t, uint_fast8_t, bool );\r\nuint_fast32_t f16_to_ui32_r_minMag( float16_t, bool );\r\nuint_fast64_t f16_to_ui64_r_minMag( float16_t, bool );\r\nint_fast32_t f16_to_i32_r_minMag( float16_t, bool );\r\nint_fast64_t f16_to_i64_r_minMag( float16_t, bool );\r\nfloat32_t f16_to_f32( float16_t );\r\nfloat64_t f16_to_f64( float16_t );\r\n#ifdef SOFTFLOAT_FAST_INT64\r\nextFloat80_t f16_to_extF80( float16_t );\r\nfloat128_t f16_to_f128( float16_t );\r\n#endif\r\nvoid f16_to_extF80M( float16_t, extFloat80_t * );\r\nvoid f16_to_f128M( float16_t, float128_t * );\r\nfloat16_t f16_roundToInt( float16_t, uint_fast8_t, bool );\r\nfloat16_t f16_add( float16_t, float16_t );\r\nfloat16_t f16_sub( float16_t, float16_t );\r\nfloat16_t f16_mul( float16_t, float16_t );\r\nfloat16_t f16_mulAdd( float16_t, float16_t, float16_t );\r\nfloat16_t f16_div( float16_t, float16_t );\r\nfloat16_t f16_rem( float16_t, float16_t );\r\nfloat16_t f16_sqrt( float16_t );\r\nbool f16_eq( float16_t, float16_t );\r\nbool f16_le( float16_t, float16_t );\r\nbool f16_lt( float16_t, float16_t );\r\nbool f16_eq_signaling( float16_t, float16_t );\r\nbool f16_le_quiet( float16_t, float16_t );\r\nbool f16_lt_quiet( float16_t, float16_t );\r\nbool f16_isSignalingNaN( float16_t );\r\n\r\n/*----------------------------------------------------------------------------\r\n| 32-bit (single-precision) floating-point operations.\r\n*----------------------------------------------------------------------------*/\r\nuint_fast32_t f32_to_ui32( float32_t, uint_fast8_t, bool );\r\nuint_fast64_t f32_to_ui64( float32_t, uint_fast8_t, bool );\r\nint_fast32_t f32_to_i32( float32_t, uint_fast8_t, bool );\r\nint_fast64_t f32_to_i64( float32_t, uint_fast8_t, bool );\r\nuint_fast32_t f32_to_ui32_r_minMag( float32_t, bool );\r\nuint_fast64_t f32_to_ui64_r_minMag( float32_t, bool );\r\nint_fast32_t f32_to_i32_r_minMag( float32_t, bool );\r\nint_fast64_t f32_to_i64_r_minMag( float32_t, bool );\r\nfloat16_t f32_to_f16( float32_t );\r\nfloat64_t f32_to_f64( float32_t );\r\n#ifdef SOFTFLOAT_FAST_INT64\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nextFloat80_t f32_to_extF80( struct softfloat_state *, float32_t );\r\nfloat128_t f32_to_f128( struct softfloat_state *, float32_t );\r\n#endif\r\nvoid f32_to_extF80M( float32_t, extFloat80_t * );\r\nvoid f32_to_f128M( float32_t, float128_t * );\r\nfloat32_t f32_roundToInt( float32_t, uint_fast8_t, bool );\r\nfloat32_t f32_add( float32_t, float32_t );\r\nfloat32_t f32_sub( float32_t, float32_t );\r\nfloat32_t f32_mul( float32_t, float32_t );\r\nfloat32_t f32_mulAdd( float32_t, float32_t, float32_t );\r\nfloat32_t f32_div( float32_t, float32_t );\r\nfloat32_t f32_rem( float32_t, float32_t );\r\nfloat32_t f32_sqrt( float32_t );\r\nbool f32_eq( float32_t, float32_t );\r\nbool f32_le( float32_t, float32_t );\r\nbool f32_lt( float32_t, float32_t );\r\nbool f32_eq_signaling( float32_t, float32_t );\r\nbool f32_le_quiet( float32_t, float32_t );\r\nbool f32_lt_quiet( float32_t, float32_t );\r\nbool f32_isSignalingNaN( float32_t );\r\n\r\n/*----------------------------------------------------------------------------\r\n| 64-bit (double-precision) floating-point operations.\r\n*----------------------------------------------------------------------------*/\r\nuint_fast32_t f64_to_ui32( float64_t, uint_fast8_t, bool );\r\nuint_fast64_t f64_to_ui64( float64_t, uint_fast8_t, bool );\r\nint_fast32_t f64_to_i32( float64_t, uint_fast8_t, bool );\r\nint_fast64_t f64_to_i64( float64_t, uint_fast8_t, bool );\r\nuint_fast32_t f64_to_ui32_r_minMag( float64_t, bool );\r\nuint_fast64_t f64_to_ui64_r_minMag( float64_t, bool );\r\nint_fast32_t f64_to_i32_r_minMag( float64_t, bool );\r\nint_fast64_t f64_to_i64_r_minMag( float64_t, bool );\r\nfloat16_t f64_to_f16( float64_t );\r\nfloat32_t f64_to_f32( float64_t );\r\n#ifdef SOFTFLOAT_FAST_INT64\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nextFloat80_t f64_to_extF80( struct softfloat_state *, float64_t );\r\nfloat128_t f64_to_f128( float64_t );\r\n#endif\r\nvoid f64_to_extF80M( float64_t, extFloat80_t * );\r\nvoid f64_to_f128M( float64_t, float128_t * );\r\nfloat64_t f64_roundToInt( float64_t, uint_fast8_t, bool );\r\nfloat64_t f64_add( float64_t, float64_t );\r\nfloat64_t f64_sub( float64_t, float64_t );\r\nfloat64_t f64_mul( float64_t, float64_t );\r\nfloat64_t f64_mulAdd( float64_t, float64_t, float64_t );\r\nfloat64_t f64_div( float64_t, float64_t );\r\nfloat64_t f64_rem( float64_t, float64_t );\r\nfloat64_t f64_sqrt( float64_t );\r\nbool f64_eq( float64_t, float64_t );\r\nbool f64_le( float64_t, float64_t );\r\nbool f64_lt( float64_t, float64_t );\r\nbool f64_eq_signaling( float64_t, float64_t );\r\nbool f64_le_quiet( float64_t, float64_t );\r\nbool f64_lt_quiet( float64_t, float64_t );\r\nbool f64_isSignalingNaN( float64_t );\r\n\r\n/*----------------------------------------------------------------------------\r\n| 80-bit extended double-precision floating-point operations.\r\n*----------------------------------------------------------------------------*/\r\n#ifdef SOFTFLOAT_FAST_INT64\r\nuint_fast32_t extF80_to_ui32( extFloat80_t, uint_fast8_t, bool );\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nuint_fast64_t extF80_to_ui64( struct softfloat_state *, extFloat80_t, uint_fast8_t, bool );\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nint_fast32_t extF80_to_i32( struct softfloat_state *, extFloat80_t, uint_fast8_t, bool );\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nint_fast64_t extF80_to_i64( struct softfloat_state *, extFloat80_t, uint_fast8_t, bool );\r\nuint_fast32_t extF80_to_ui32_r_minMag( extFloat80_t, bool );\r\nuint_fast64_t extF80_to_ui64_r_minMag( extFloat80_t, bool );\r\nint_fast32_t extF80_to_i32_r_minMag( extFloat80_t, bool );\r\nint_fast64_t extF80_to_i64_r_minMag( extFloat80_t, bool );\r\nfloat16_t extF80_to_f16( extFloat80_t );\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nfloat32_t extF80_to_f32( struct softfloat_state *, extFloat80_t );\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nfloat64_t extF80_to_f64( struct softfloat_state *, extFloat80_t );\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nfloat128_t extF80_to_f128( struct softfloat_state *, extFloat80_t );\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nextFloat80_t extF80_roundToInt( struct softfloat_state *, extFloat80_t, uint_fast8_t, bool );\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nextFloat80_t extF80_add( struct softfloat_state *, extFloat80_t, extFloat80_t );\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nextFloat80_t extF80_sub( struct softfloat_state *, extFloat80_t, extFloat80_t );\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nextFloat80_t extF80_mul( struct softfloat_state *, extFloat80_t, extFloat80_t );\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nextFloat80_t extF80_div( struct softfloat_state *, extFloat80_t, extFloat80_t );\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nextFloat80_t extF80_rem( struct softfloat_state *, extFloat80_t, extFloat80_t );\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nextFloat80_t extF80_sqrt( struct softfloat_state *, extFloat80_t );\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nbool extF80_eq( struct softfloat_state *, extFloat80_t, extFloat80_t );\r\nbool extF80_le( struct softfloat_state *, extFloat80_t, extFloat80_t );\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nbool extF80_lt( struct softfloat_state *, extFloat80_t, extFloat80_t );\r\nbool extF80_eq_signaling( extFloat80_t, extFloat80_t );\r\nbool extF80_le_quiet( extFloat80_t, extFloat80_t );\r\nbool extF80_lt_quiet( extFloat80_t, extFloat80_t );\r\nbool extF80_isSignalingNaN( extFloat80_t );\r\nstatic inline extFloat80_t extF80_complement_sign(extFloat80_t a) {\r\n a.signExp ^= 1ULL << 15;\r\n return a;\r\n}\r\n#endif\r\nuint_fast32_t extF80M_to_ui32( const extFloat80_t *, uint_fast8_t, bool );\r\nuint_fast64_t extF80M_to_ui64( const extFloat80_t *, uint_fast8_t, bool );\r\nint_fast32_t extF80M_to_i32( const extFloat80_t *, uint_fast8_t, bool );\r\nint_fast64_t extF80M_to_i64( const extFloat80_t *, uint_fast8_t, bool );\r\nuint_fast32_t extF80M_to_ui32_r_minMag( const extFloat80_t *, bool );\r\nuint_fast64_t extF80M_to_ui64_r_minMag( const extFloat80_t *, bool );\r\nint_fast32_t extF80M_to_i32_r_minMag( const extFloat80_t *, bool );\r\nint_fast64_t extF80M_to_i64_r_minMag( const extFloat80_t *, bool );\r\nfloat16_t extF80M_to_f16( const extFloat80_t * );\r\nfloat32_t extF80M_to_f32( const extFloat80_t * );\r\nfloat64_t extF80M_to_f64( const extFloat80_t * );\r\nvoid extF80M_to_f128M( const extFloat80_t *, float128_t * );\r\nvoid\r\n extF80M_roundToInt(\r\n const extFloat80_t *, uint_fast8_t, bool, extFloat80_t * );\r\nvoid extF80M_add( const extFloat80_t *, const extFloat80_t *, extFloat80_t * );\r\nvoid extF80M_sub( const extFloat80_t *, const extFloat80_t *, extFloat80_t * );\r\nvoid extF80M_mul( const extFloat80_t *, const extFloat80_t *, extFloat80_t * );\r\nvoid extF80M_div( const extFloat80_t *, const extFloat80_t *, extFloat80_t * );\r\nvoid extF80M_rem( const extFloat80_t *, const extFloat80_t *, extFloat80_t * );\r\nvoid extF80M_sqrt( const extFloat80_t *, extFloat80_t * );\r\nbool extF80M_eq( const extFloat80_t *, const extFloat80_t * );\r\nbool extF80M_le( const extFloat80_t *, const extFloat80_t * );\r\nbool extF80M_lt( const extFloat80_t *, const extFloat80_t * );\r\nbool extF80M_eq_signaling( const extFloat80_t *, const extFloat80_t * );\r\nbool extF80M_le_quiet( const extFloat80_t *, const extFloat80_t * );\r\nbool extF80M_lt_quiet( const extFloat80_t *, const extFloat80_t * );\r\nbool extF80M_isSignalingNaN( const extFloat80_t * );\r\n\r\n/*----------------------------------------------------------------------------\r\n| 128-bit (quadruple-precision) floating-point operations.\r\n*----------------------------------------------------------------------------*/\r\n#ifdef SOFTFLOAT_FAST_INT64\r\nuint_fast32_t f128_to_ui32( struct softfloat_state *, float128_t, uint_fast8_t, bool );\r\nuint_fast64_t f128_to_ui64( struct softfloat_state *, float128_t, uint_fast8_t, bool );\r\nint_fast32_t f128_to_i32( struct softfloat_state *, float128_t, uint_fast8_t, bool );\r\nint_fast64_t f128_to_i64( struct softfloat_state *, float128_t, uint_fast8_t, bool );\r\nuint_fast32_t f128_to_ui32_r_minMag( float128_t, bool );\r\nuint_fast64_t f128_to_ui64_r_minMag( float128_t, bool );\r\nint_fast32_t f128_to_i32_r_minMag( float128_t, bool );\r\nint_fast64_t f128_to_i64_r_minMag( float128_t, bool );\r\nfloat16_t f128_to_f16( struct softfloat_state *, float128_t );\r\nfloat32_t f128_to_f32( struct softfloat_state *, float128_t );\r\nfloat64_t f128_to_f64( struct softfloat_state *, float128_t );\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nextFloat80_t f128_to_extF80( struct softfloat_state *, float128_t );\r\nfloat128_t f128_roundToInt( float128_t, uint_fast8_t, bool );\r\nfloat128_t f128_add( struct softfloat_state *, float128_t, float128_t );\r\nfloat128_t f128_sub( struct softfloat_state *, float128_t, float128_t );\r\nfloat128_t f128_mul( struct softfloat_state *, float128_t, float128_t );\r\nfloat128_t f128_mulAdd( struct softfloat_state *, float128_t, float128_t, float128_t );\r\nfloat128_t f128_div( struct softfloat_state *, float128_t, float128_t );\r\nfloat128_t f128_rem( struct softfloat_state *, float128_t, float128_t );\r\nfloat128_t f128_sqrt( struct softfloat_state *, float128_t );\r\nbool f128_eq( struct softfloat_state *, float128_t, float128_t );\r\nbool f128_le( struct softfloat_state *, float128_t, float128_t );\r\nbool f128_lt( struct softfloat_state *, float128_t, float128_t );\r\nbool f128_eq_signaling( struct softfloat_state *, float128_t, float128_t );\r\nbool f128_le_quiet( struct softfloat_state *, float128_t, float128_t );\r\nbool f128_lt_quiet( struct softfloat_state *, float128_t, float128_t );\r\nbool f128_isSignalingNaN( float128_t );\r\nstatic inline float128_t f128_complement_sign(float128_t a) {\r\n a.v[1] ^= 1ULL << 63;\r\n return a;\r\n}\r\n#endif\r\nuint_fast32_t f128M_to_ui32( const float128_t *, uint_fast8_t, bool );\r\nuint_fast64_t f128M_to_ui64( const float128_t *, uint_fast8_t, bool );\r\nint_fast32_t f128M_to_i32( const float128_t *, uint_fast8_t, bool );\r\nint_fast64_t f128M_to_i64( const float128_t *, uint_fast8_t, bool );\r\nuint_fast32_t f128M_to_ui32_r_minMag( const float128_t *, bool );\r\nuint_fast64_t f128M_to_ui64_r_minMag( const float128_t *, bool );\r\nint_fast32_t f128M_to_i32_r_minMag( const float128_t *, bool );\r\nint_fast64_t f128M_to_i64_r_minMag( const float128_t *, bool );\r\nfloat16_t f128M_to_f16( const float128_t * );\r\nfloat32_t f128M_to_f32( const float128_t * );\r\nfloat64_t f128M_to_f64( const float128_t * );\r\nvoid f128M_to_extF80M( const float128_t *, extFloat80_t * );\r\nvoid f128M_roundToInt( const float128_t *, uint_fast8_t, bool, float128_t * );\r\nvoid f128M_add( const float128_t *, const float128_t *, float128_t * );\r\nvoid f128M_sub( const float128_t *, const float128_t *, float128_t * );\r\nvoid f128M_mul( const float128_t *, const float128_t *, float128_t * );\r\nvoid\r\n f128M_mulAdd(\r\n const float128_t *, const float128_t *, const float128_t *, float128_t *\r\n );\r\nvoid f128M_div( const float128_t *, const float128_t *, float128_t * );\r\nvoid f128M_rem( const float128_t *, const float128_t *, float128_t * );\r\nvoid f128M_sqrt( const float128_t *, float128_t * );\r\nbool f128M_eq( const float128_t *, const float128_t * );\r\nbool f128M_le( const float128_t *, const float128_t * );\r\nbool f128M_lt( const float128_t *, const float128_t * );\r\nbool f128M_eq_signaling( const float128_t *, const float128_t * );\r\nbool f128M_le_quiet( const float128_t *, const float128_t * );\r\nbool f128M_lt_quiet( const float128_t *, const float128_t * );\r\nbool f128M_isSignalingNaN( const float128_t * );\r\n\r\n#endif\r\n\r\n" }, { "path": "External/SoftFloat-3e/include/SoftFloat-3e/softfloat_types.h", "content": "\r\n/*============================================================================\r\n\r\nThis C header file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014, 2015, 2017 The Regents of the University of\r\nCalifornia. All rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#ifndef softfloat_types_h\r\n#define softfloat_types_h 1\r\n\r\n#include \r\n\r\n/*----------------------------------------------------------------------------\r\n| Types used to pass 16-bit, 32-bit, 64-bit, and 128-bit floating-point\r\n| arguments and results to/from functions. These types must be exactly\r\n| 16 bits, 32 bits, 64 bits, and 128 bits in size, respectively. Where a\r\n| platform has \"native\" support for IEEE-Standard floating-point formats,\r\n| the types below may, if desired, be defined as aliases for the native types\r\n| (typically 'float' and 'double', and possibly 'long double').\r\n*----------------------------------------------------------------------------*/\r\ntypedef struct { uint16_t v; } float16_t;\r\ntypedef struct { uint32_t v; } float32_t;\r\ntypedef struct { uint64_t v; } float64_t;\r\ntypedef struct { uint64_t v[2]; } float128_t;\r\n\r\n/*----------------------------------------------------------------------------\r\n| The format of an 80-bit extended floating-point number in memory. This\r\n| structure must contain a 16-bit field named 'signExp' and a 64-bit field\r\n| named 'signif'.\r\n*----------------------------------------------------------------------------*/\r\n#ifdef LITTLEENDIAN\r\nstruct extFloat80M { uint64_t signif; uint16_t signExp; };\r\n#else\r\nstruct extFloat80M { uint16_t signExp; uint64_t signif; };\r\n#endif\r\n\r\n/*----------------------------------------------------------------------------\r\n| The type used to pass 80-bit extended floating-point arguments and\r\n| results to/from functions. This type must have size identical to\r\n| 'struct extFloat80M'. Type 'extFloat80_t' can be defined as an alias for\r\n| 'struct extFloat80M'. Alternatively, if a platform has \"native\" support\r\n| for IEEE-Standard 80-bit extended floating-point, it may be possible,\r\n| if desired, to define 'extFloat80_t' as an alias for the native type\r\n| (presumably either 'long double' or a nonstandard compiler-intrinsic type).\r\n| In that case, the 'signif' and 'signExp' fields of 'struct extFloat80M'\r\n| must align exactly with the locations in memory of the sign, exponent, and\r\n| significand of the native type.\r\n*----------------------------------------------------------------------------*/\r\ntypedef struct extFloat80M extFloat80_t;\r\n\r\nenum {\r\n softfloat_tininess_beforeRounding = 0,\r\n softfloat_tininess_afterRounding = 1\r\n};\r\n\r\nenum {\r\n softfloat_round_near_even = 0,\r\n softfloat_round_minMag = 1,\r\n softfloat_round_min = 2,\r\n softfloat_round_max = 3,\r\n softfloat_round_near_maxMag = 4,\r\n softfloat_round_odd = 6\r\n};\r\n\r\nenum {\r\n softfloat_flag_inexact = 1,\r\n softfloat_flag_underflow = 2,\r\n softfloat_flag_overflow = 4,\r\n softfloat_flag_infinite = 8,\r\n softfloat_flag_invalid = 16\r\n};\r\n\r\nstruct softfloat_state {\r\n/*----------------------------------------------------------------------------\r\n| Software floating-point underflow tininess-detection mode.\r\n*----------------------------------------------------------------------------*/\r\nuint8_t detectTininess; /* = init_detectTininess */\r\n/*----------------------------------------------------------------------------\r\n| Software floating-point rounding mode. (Mode \"odd\" is supported only if\r\n| SoftFloat is compiled with macro 'SOFTFLOAT_ROUND_ODD' defined.)\r\n*----------------------------------------------------------------------------*/\r\nuint8_t roundingMode; /* = softfloat_round_near_even */\r\n\r\n/*----------------------------------------------------------------------------\r\n| Software floating-point exception flags.\r\n*----------------------------------------------------------------------------*/\r\nuint8_t exceptionFlags; /* = 0 */\r\n\r\n/*----------------------------------------------------------------------------\r\n| Rounding precision for 80-bit extended double-precision floating-point.\r\n| Valid values are 32, 64, and 80.\r\n*----------------------------------------------------------------------------*/\r\nuint8_t roundingPrecision; /* = 80 */\r\n};\r\n\r\n#endif\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/extF80_add.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014 The Regents of the University of California.\r\nAll rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \r\n#include \"platform.h\"\r\n#include \"internals.h\"\r\n#include \"softfloat.h\"\r\n\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nextFloat80_t extF80_add( struct softfloat_state *state, extFloat80_t a, extFloat80_t b )\r\n{\r\n union { struct extFloat80M s; extFloat80_t f; } uA;\r\n uint_fast16_t uiA64;\r\n uint_fast64_t uiA0;\r\n bool signA;\r\n union { struct extFloat80M s; extFloat80_t f; } uB;\r\n uint_fast16_t uiB64;\r\n uint_fast64_t uiB0;\r\n bool signB;\r\n extFloat80_t\r\n (*magsFuncPtr)(\r\n struct softfloat_state *, uint_fast16_t, uint_fast64_t, uint_fast16_t, uint_fast64_t, bool );\r\n\r\n uA.f = a;\r\n uiA64 = uA.s.signExp;\r\n uiA0 = uA.s.signif;\r\n signA = signExtF80UI64( uiA64 );\r\n uB.f = b;\r\n uiB64 = uB.s.signExp;\r\n uiB0 = uB.s.signif;\r\n signB = signExtF80UI64( uiB64 );\r\n magsFuncPtr =\r\n (signA == signB) ? softfloat_addMagsExtF80 : softfloat_subMagsExtF80;\r\n return (*magsFuncPtr)( state, uiA64, uiA0, uiB64, uiB0, signA );\r\n}\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/extF80_div.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014 The Regents of the University of California.\r\nAll rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \r\n#include \"platform.h\"\r\n#include \"internals.h\"\r\n#include \"specialize.h\"\r\n#include \"softfloat.h\"\r\n\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nextFloat80_t extF80_div( struct softfloat_state *state, extFloat80_t a, extFloat80_t b )\r\n{\r\n union { struct extFloat80M s; extFloat80_t f; } uA;\r\n uint_fast16_t uiA64;\r\n uint_fast64_t uiA0;\r\n bool signA;\r\n int_fast32_t expA;\r\n uint_fast64_t sigA;\r\n union { struct extFloat80M s; extFloat80_t f; } uB;\r\n uint_fast16_t uiB64;\r\n uint_fast64_t uiB0;\r\n bool signB;\r\n int_fast32_t expB;\r\n uint_fast64_t sigB;\r\n bool signZ;\r\n struct exp32_sig64 normExpSig;\r\n int_fast32_t expZ;\r\n struct uint128 rem;\r\n uint_fast32_t recip32;\r\n uint_fast64_t sigZ;\r\n int ix;\r\n uint_fast64_t q64;\r\n uint_fast32_t q;\r\n struct uint128 term;\r\n uint_fast64_t sigZExtra;\r\n struct uint128 uiZ;\r\n uint_fast16_t uiZ64;\r\n uint_fast64_t uiZ0;\r\n union { struct extFloat80M s; extFloat80_t f; } uZ;\r\n\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n uA.f = a;\r\n uiA64 = uA.s.signExp;\r\n uiA0 = uA.s.signif;\r\n signA = signExtF80UI64( uiA64 );\r\n expA = expExtF80UI64( uiA64 );\r\n sigA = uiA0;\r\n uB.f = b;\r\n uiB64 = uB.s.signExp;\r\n uiB0 = uB.s.signif;\r\n signB = signExtF80UI64( uiB64 );\r\n expB = expExtF80UI64( uiB64 );\r\n sigB = uiB0;\r\n signZ = signA ^ signB;\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n if ( expA == 0x7FFF ) {\r\n if ( sigA & UINT64_C( 0x7FFFFFFFFFFFFFFF ) ) goto propagateNaN;\r\n if ( expB == 0x7FFF ) {\r\n if ( sigB & UINT64_C( 0x7FFFFFFFFFFFFFFF ) ) goto propagateNaN;\r\n goto invalid;\r\n }\r\n goto infinity;\r\n }\r\n if ( expB == 0x7FFF ) {\r\n if ( sigB & UINT64_C( 0x7FFFFFFFFFFFFFFF ) ) goto propagateNaN;\r\n goto zero;\r\n }\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n if ( ! expB ) expB = 1;\r\n if ( ! (sigB & UINT64_C( 0x8000000000000000 )) ) {\r\n if ( ! sigB ) {\r\n if ( ! sigA ) goto invalid;\r\n softfloat_raiseFlags( state, softfloat_flag_infinite );\r\n goto infinity;\r\n }\r\n normExpSig = softfloat_normSubnormalExtF80Sig( sigB );\r\n expB += normExpSig.exp;\r\n sigB = normExpSig.sig;\r\n }\r\n if ( ! expA ) expA = 1;\r\n if ( ! (sigA & UINT64_C( 0x8000000000000000 )) ) {\r\n if ( ! sigA ) goto zero;\r\n normExpSig = softfloat_normSubnormalExtF80Sig( sigA );\r\n expA += normExpSig.exp;\r\n sigA = normExpSig.sig;\r\n }\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n expZ = expA - expB + 0x3FFF;\r\n if ( sigA < sigB ) {\r\n --expZ;\r\n rem = softfloat_shortShiftLeft128( 0, sigA, 32 );\r\n } else {\r\n rem = softfloat_shortShiftLeft128( 0, sigA, 31 );\r\n }\r\n recip32 = softfloat_approxRecip32_1( sigB>>32 );\r\n sigZ = 0;\r\n ix = 2;\r\n for (;;) {\r\n q64 = (uint_fast64_t) (uint32_t) (rem.v64>>2) * recip32;\r\n q = (q64 + 0x80000000)>>32;\r\n --ix;\r\n if ( ix < 0 ) break;\r\n rem = softfloat_shortShiftLeft128( rem.v64, rem.v0, 29 );\r\n term = softfloat_mul64ByShifted32To128( sigB, q );\r\n rem = softfloat_sub128( rem.v64, rem.v0, term.v64, term.v0 );\r\n if ( rem.v64 & UINT64_C( 0x8000000000000000 ) ) {\r\n --q;\r\n rem = softfloat_add128( rem.v64, rem.v0, sigB>>32, sigB<<32 );\r\n }\r\n sigZ = (sigZ<<29) + q;\r\n }\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n if ( ((q + 1) & 0x3FFFFF) < 2 ) {\r\n rem = softfloat_shortShiftLeft128( rem.v64, rem.v0, 29 );\r\n term = softfloat_mul64ByShifted32To128( sigB, q );\r\n rem = softfloat_sub128( rem.v64, rem.v0, term.v64, term.v0 );\r\n term = softfloat_shortShiftLeft128( 0, sigB, 32 );\r\n if ( rem.v64 & UINT64_C( 0x8000000000000000 ) ) {\r\n --q;\r\n rem = softfloat_add128( rem.v64, rem.v0, term.v64, term.v0 );\r\n } else if ( softfloat_le128( term.v64, term.v0, rem.v64, rem.v0 ) ) {\r\n ++q;\r\n rem = softfloat_sub128( rem.v64, rem.v0, term.v64, term.v0 );\r\n }\r\n if ( rem.v64 | rem.v0 ) q |= 1;\r\n }\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n sigZ = (sigZ<<6) + (q>>23);\r\n sigZExtra = (uint64_t) ((uint_fast64_t) q<<41);\r\n return\r\n softfloat_roundPackToExtF80(\r\n state, signZ, expZ, sigZ, sigZExtra, state->roundingPrecision );\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n propagateNaN:\r\n uiZ = softfloat_propagateNaNExtF80UI( state, uiA64, uiA0, uiB64, uiB0 );\r\n uiZ64 = uiZ.v64;\r\n uiZ0 = uiZ.v0;\r\n goto uiZ;\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n invalid:\r\n softfloat_raiseFlags( state, softfloat_flag_invalid );\r\n uiZ64 = defaultNaNExtF80UI64;\r\n uiZ0 = defaultNaNExtF80UI0;\r\n goto uiZ;\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n infinity:\r\n uiZ64 = packToExtF80UI64( signZ, 0x7FFF );\r\n uiZ0 = UINT64_C( 0x8000000000000000 );\r\n goto uiZ;\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n zero:\r\n uiZ64 = packToExtF80UI64( signZ, 0 );\r\n uiZ0 = 0;\r\n uiZ:\r\n uZ.s.signExp = uiZ64;\r\n uZ.s.signif = uiZ0;\r\n return uZ.f;\r\n\r\n}\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/extF80_eq.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014 The Regents of the University of California.\r\nAll rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \r\n#include \"platform.h\"\r\n#include \"internals.h\"\r\n#include \"specialize.h\"\r\n#include \"softfloat.h\"\r\n\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nbool extF80_eq( struct softfloat_state *state, extFloat80_t a, extFloat80_t b )\r\n{\r\n union { struct extFloat80M s; extFloat80_t f; } uA;\r\n uint_fast16_t uiA64;\r\n uint_fast64_t uiA0;\r\n union { struct extFloat80M s; extFloat80_t f; } uB;\r\n uint_fast16_t uiB64;\r\n uint_fast64_t uiB0;\r\n\r\n uA.f = a;\r\n uiA64 = uA.s.signExp;\r\n uiA0 = uA.s.signif;\r\n uB.f = b;\r\n uiB64 = uB.s.signExp;\r\n uiB0 = uB.s.signif;\r\n if ( isNaNExtF80UI( uiA64, uiA0 ) || isNaNExtF80UI( uiB64, uiB0 ) ) {\r\n if (\r\n softfloat_isSigNaNExtF80UI( uiA64, uiA0 )\r\n || softfloat_isSigNaNExtF80UI( uiB64, uiB0 )\r\n ) {\r\n softfloat_raiseFlags( state, softfloat_flag_invalid );\r\n }\r\n return false;\r\n }\r\n return\r\n (uiA0 == uiB0)\r\n && ((uiA64 == uiB64) || (! uiA0 && ! ((uiA64 | uiB64) & 0x7FFF)));\r\n\r\n}\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/extF80_le.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014 The Regents of the University of California.\r\nAll rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \r\n#include \"platform.h\"\r\n#include \"internals.h\"\r\n#include \"specialize.h\"\r\n#include \"softfloat.h\"\r\n\r\nbool extF80_le( struct softfloat_state *state, extFloat80_t a, extFloat80_t b )\r\n{\r\n union { struct extFloat80M s; extFloat80_t f; } uA;\r\n uint_fast16_t uiA64;\r\n uint_fast64_t uiA0;\r\n union { struct extFloat80M s; extFloat80_t f; } uB;\r\n uint_fast16_t uiB64;\r\n uint_fast64_t uiB0;\r\n bool signA, signB;\r\n\r\n uA.f = a;\r\n uiA64 = uA.s.signExp;\r\n uiA0 = uA.s.signif;\r\n uB.f = b;\r\n uiB64 = uB.s.signExp;\r\n uiB0 = uB.s.signif;\r\n if ( isNaNExtF80UI( uiA64, uiA0 ) || isNaNExtF80UI( uiB64, uiB0 ) ) {\r\n softfloat_raiseFlags( state, softfloat_flag_invalid );\r\n return false;\r\n }\r\n signA = signExtF80UI64( uiA64 );\r\n signB = signExtF80UI64( uiB64 );\r\n return\r\n (signA != signB)\r\n ? signA || ! (((uiA64 | uiB64) & 0x7FFF) | uiA0 | uiB0)\r\n : ((uiA64 == uiB64) && (uiA0 == uiB0))\r\n || (signA ^ softfloat_lt128( uiA64, uiA0, uiB64, uiB0 ));\r\n\r\n}\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/extF80_lt.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014 The Regents of the University of California.\r\nAll rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \r\n#include \"platform.h\"\r\n#include \"internals.h\"\r\n#include \"specialize.h\"\r\n#include \"softfloat.h\"\r\n\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nbool extF80_lt( struct softfloat_state *state, extFloat80_t a, extFloat80_t b )\r\n{\r\n union { struct extFloat80M s; extFloat80_t f; } uA;\r\n uint_fast16_t uiA64;\r\n uint_fast64_t uiA0;\r\n union { struct extFloat80M s; extFloat80_t f; } uB;\r\n uint_fast16_t uiB64;\r\n uint_fast64_t uiB0;\r\n bool signA, signB;\r\n\r\n uA.f = a;\r\n uiA64 = uA.s.signExp;\r\n uiA0 = uA.s.signif;\r\n uB.f = b;\r\n uiB64 = uB.s.signExp;\r\n uiB0 = uB.s.signif;\r\n if ( isNaNExtF80UI( uiA64, uiA0 ) || isNaNExtF80UI( uiB64, uiB0 ) ) {\r\n softfloat_raiseFlags( state, softfloat_flag_invalid );\r\n return false;\r\n }\r\n signA = signExtF80UI64( uiA64 );\r\n signB = signExtF80UI64( uiB64 );\r\n return\r\n (signA != signB)\r\n ? signA && (((uiA64 | uiB64) & 0x7FFF) | uiA0 | uiB0)\r\n : ((uiA64 != uiB64) || (uiA0 != uiB0))\r\n && (signA ^ softfloat_lt128( uiA64, uiA0, uiB64, uiB0 ));\r\n\r\n}\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/extF80_mul.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014 The Regents of the University of California.\r\nAll rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \r\n#include \"platform.h\"\r\n#include \"internals.h\"\r\n#include \"specialize.h\"\r\n#include \"softfloat.h\"\r\n\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nextFloat80_t extF80_mul( struct softfloat_state *state, extFloat80_t a, extFloat80_t b )\r\n{\r\n union { struct extFloat80M s; extFloat80_t f; } uA;\r\n uint_fast16_t uiA64;\r\n uint_fast64_t uiA0;\r\n bool signA;\r\n int_fast32_t expA;\r\n uint_fast64_t sigA;\r\n union { struct extFloat80M s; extFloat80_t f; } uB;\r\n uint_fast16_t uiB64;\r\n uint_fast64_t uiB0;\r\n bool signB;\r\n int_fast32_t expB;\r\n uint_fast64_t sigB;\r\n bool signZ;\r\n uint_fast64_t magBits;\r\n struct exp32_sig64 normExpSig;\r\n int_fast32_t expZ;\r\n struct uint128 sig128Z, uiZ;\r\n uint_fast16_t uiZ64;\r\n uint_fast64_t uiZ0;\r\n union { struct extFloat80M s; extFloat80_t f; } uZ;\r\n\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n uA.f = a;\r\n uiA64 = uA.s.signExp;\r\n uiA0 = uA.s.signif;\r\n signA = signExtF80UI64( uiA64 );\r\n expA = expExtF80UI64( uiA64 );\r\n sigA = uiA0;\r\n uB.f = b;\r\n uiB64 = uB.s.signExp;\r\n uiB0 = uB.s.signif;\r\n signB = signExtF80UI64( uiB64 );\r\n expB = expExtF80UI64( uiB64 );\r\n sigB = uiB0;\r\n signZ = signA ^ signB;\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n if ( expA == 0x7FFF ) {\r\n if (\r\n (sigA & UINT64_C( 0x7FFFFFFFFFFFFFFF ))\r\n || ((expB == 0x7FFF) && (sigB & UINT64_C( 0x7FFFFFFFFFFFFFFF )))\r\n ) {\r\n goto propagateNaN;\r\n }\r\n magBits = expB | sigB;\r\n goto infArg;\r\n }\r\n if ( expB == 0x7FFF ) {\r\n if ( sigB & UINT64_C( 0x7FFFFFFFFFFFFFFF ) ) goto propagateNaN;\r\n magBits = expA | sigA;\r\n goto infArg;\r\n }\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n if ( ! expA ) expA = 1;\r\n if ( ! (sigA & UINT64_C( 0x8000000000000000 )) ) {\r\n if ( ! sigA ) goto zero;\r\n normExpSig = softfloat_normSubnormalExtF80Sig( sigA );\r\n expA += normExpSig.exp;\r\n sigA = normExpSig.sig;\r\n }\r\n if ( ! expB ) expB = 1;\r\n if ( ! (sigB & UINT64_C( 0x8000000000000000 )) ) {\r\n if ( ! sigB ) goto zero;\r\n normExpSig = softfloat_normSubnormalExtF80Sig( sigB );\r\n expB += normExpSig.exp;\r\n sigB = normExpSig.sig;\r\n }\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n expZ = expA + expB - 0x3FFE;\r\n sig128Z = softfloat_mul64To128( sigA, sigB );\r\n if ( sig128Z.v64 < UINT64_C( 0x8000000000000000 ) ) {\r\n --expZ;\r\n sig128Z =\r\n softfloat_add128(\r\n sig128Z.v64, sig128Z.v0, sig128Z.v64, sig128Z.v0 );\r\n }\r\n return\r\n softfloat_roundPackToExtF80(\r\n state, signZ, expZ, sig128Z.v64, sig128Z.v0, state->roundingPrecision );\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n propagateNaN:\r\n uiZ = softfloat_propagateNaNExtF80UI( state, uiA64, uiA0, uiB64, uiB0 );\r\n uiZ64 = uiZ.v64;\r\n uiZ0 = uiZ.v0;\r\n goto uiZ;\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n infArg:\r\n if ( ! magBits ) {\r\n softfloat_raiseFlags( state, softfloat_flag_invalid );\r\n uiZ64 = defaultNaNExtF80UI64;\r\n uiZ0 = defaultNaNExtF80UI0;\r\n } else {\r\n uiZ64 = packToExtF80UI64( signZ, 0x7FFF );\r\n uiZ0 = UINT64_C( 0x8000000000000000 );\r\n }\r\n goto uiZ;\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n zero:\r\n uiZ64 = packToExtF80UI64( signZ, 0 );\r\n uiZ0 = 0;\r\n uiZ:\r\n uZ.s.signExp = uiZ64;\r\n uZ.s.signif = uiZ0;\r\n return uZ.f;\r\n\r\n}\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/extF80_rem.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014, 2015, 2016, 2017 The Regents of the\r\nUniversity of California. All rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \r\n#include \"platform.h\"\r\n#include \"internals.h\"\r\n#include \"specialize.h\"\r\n#include \"softfloat.h\"\r\n\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nextFloat80_t extF80_rem( struct softfloat_state *state, extFloat80_t a, extFloat80_t b )\r\n{\r\n union { struct extFloat80M s; extFloat80_t f; } uA;\r\n uint_fast16_t uiA64;\r\n uint_fast64_t uiA0;\r\n bool signA;\r\n int_fast32_t expA;\r\n uint_fast64_t sigA;\r\n union { struct extFloat80M s; extFloat80_t f; } uB;\r\n uint_fast16_t uiB64;\r\n uint_fast64_t uiB0;\r\n int_fast32_t expB;\r\n uint_fast64_t sigB;\r\n struct exp32_sig64 normExpSig;\r\n int_fast32_t expDiff;\r\n struct uint128 rem, shiftedSigB;\r\n uint_fast32_t q, recip32;\r\n uint_fast64_t q64;\r\n struct uint128 term, altRem, meanRem;\r\n bool signRem;\r\n struct uint128 uiZ;\r\n uint_fast16_t uiZ64;\r\n uint_fast64_t uiZ0;\r\n union { struct extFloat80M s; extFloat80_t f; } uZ;\r\n\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n uA.f = a;\r\n uiA64 = uA.s.signExp;\r\n uiA0 = uA.s.signif;\r\n signA = signExtF80UI64( uiA64 );\r\n expA = expExtF80UI64( uiA64 );\r\n sigA = uiA0;\r\n uB.f = b;\r\n uiB64 = uB.s.signExp;\r\n uiB0 = uB.s.signif;\r\n expB = expExtF80UI64( uiB64 );\r\n sigB = uiB0;\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n if ( expA == 0x7FFF ) {\r\n if (\r\n (sigA & UINT64_C( 0x7FFFFFFFFFFFFFFF ))\r\n || ((expB == 0x7FFF) && (sigB & UINT64_C( 0x7FFFFFFFFFFFFFFF )))\r\n ) {\r\n goto propagateNaN;\r\n }\r\n goto invalid;\r\n }\r\n if ( expB == 0x7FFF ) {\r\n if ( sigB & UINT64_C( 0x7FFFFFFFFFFFFFFF ) ) goto propagateNaN;\r\n /*--------------------------------------------------------------------\r\n | Argument b is an infinity. Doubling `expB' is an easy way to ensure\r\n | that `expDiff' later is less than -1, which will result in returning\r\n | a canonicalized version of argument a.\r\n *--------------------------------------------------------------------*/\r\n expB += expB;\r\n }\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n if ( ! expB ) expB = 1;\r\n if ( ! (sigB & UINT64_C( 0x8000000000000000 )) ) {\r\n if ( ! sigB ) goto invalid;\r\n normExpSig = softfloat_normSubnormalExtF80Sig( sigB );\r\n expB += normExpSig.exp;\r\n sigB = normExpSig.sig;\r\n }\r\n if ( ! expA ) expA = 1;\r\n if ( ! (sigA & UINT64_C( 0x8000000000000000 )) ) {\r\n if ( ! sigA ) {\r\n expA = 0;\r\n goto copyA;\r\n }\r\n normExpSig = softfloat_normSubnormalExtF80Sig( sigA );\r\n expA += normExpSig.exp;\r\n sigA = normExpSig.sig;\r\n }\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n expDiff = expA - expB;\r\n if ( expDiff < -1 ) goto copyA;\r\n rem = softfloat_shortShiftLeft128( 0, sigA, 32 );\r\n shiftedSigB = softfloat_shortShiftLeft128( 0, sigB, 32 );\r\n if ( expDiff < 1 ) {\r\n if ( expDiff ) {\r\n --expB;\r\n shiftedSigB = softfloat_shortShiftLeft128( 0, sigB, 33 );\r\n q = 0;\r\n } else {\r\n q = (sigB <= sigA);\r\n if ( q ) {\r\n rem =\r\n softfloat_sub128(\r\n rem.v64, rem.v0, shiftedSigB.v64, shiftedSigB.v0 );\r\n }\r\n }\r\n } else {\r\n recip32 = softfloat_approxRecip32_1( sigB>>32 );\r\n expDiff -= 30;\r\n for (;;) {\r\n q64 = (uint_fast64_t) (uint32_t) (rem.v64>>2) * recip32;\r\n if ( expDiff < 0 ) break;\r\n q = (q64 + 0x80000000)>>32;\r\n rem = softfloat_shortShiftLeft128( rem.v64, rem.v0, 29 );\r\n term = softfloat_mul64ByShifted32To128( sigB, q );\r\n rem = softfloat_sub128( rem.v64, rem.v0, term.v64, term.v0 );\r\n if ( rem.v64 & UINT64_C( 0x8000000000000000 ) ) {\r\n rem =\r\n softfloat_add128(\r\n rem.v64, rem.v0, shiftedSigB.v64, shiftedSigB.v0 );\r\n }\r\n expDiff -= 29;\r\n }\r\n /*--------------------------------------------------------------------\r\n | (`expDiff' cannot be less than -29 here.)\r\n *--------------------------------------------------------------------*/\r\n q = (uint32_t) (q64>>32)>>(~expDiff & 31);\r\n rem = softfloat_shortShiftLeft128( rem.v64, rem.v0, expDiff + 30 );\r\n term = softfloat_mul64ByShifted32To128( sigB, q );\r\n rem = softfloat_sub128( rem.v64, rem.v0, term.v64, term.v0 );\r\n if ( rem.v64 & UINT64_C( 0x8000000000000000 ) ) {\r\n altRem =\r\n softfloat_add128(\r\n rem.v64, rem.v0, shiftedSigB.v64, shiftedSigB.v0 );\r\n goto selectRem;\r\n }\r\n }\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n do {\r\n altRem = rem;\r\n ++q;\r\n rem =\r\n softfloat_sub128(\r\n rem.v64, rem.v0, shiftedSigB.v64, shiftedSigB.v0 );\r\n } while ( ! (rem.v64 & UINT64_C( 0x8000000000000000 )) );\r\n selectRem:\r\n meanRem = softfloat_add128( rem.v64, rem.v0, altRem.v64, altRem.v0 );\r\n if (\r\n (meanRem.v64 & UINT64_C( 0x8000000000000000 ))\r\n || (! (meanRem.v64 | meanRem.v0) && (q & 1))\r\n ) {\r\n rem = altRem;\r\n }\r\n signRem = signA;\r\n if ( rem.v64 & UINT64_C( 0x8000000000000000 ) ) {\r\n signRem = ! signRem;\r\n rem = softfloat_sub128( 0, 0, rem.v64, rem.v0 );\r\n }\r\n return\r\n softfloat_normRoundPackToExtF80(\r\n state, signRem, rem.v64 | rem.v0 ? expB + 32 : 0, rem.v64, rem.v0, 80 );\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n propagateNaN:\r\n uiZ = softfloat_propagateNaNExtF80UI( state, uiA64, uiA0, uiB64, uiB0 );\r\n uiZ64 = uiZ.v64;\r\n uiZ0 = uiZ.v0;\r\n goto uiZ;\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n invalid:\r\n softfloat_raiseFlags( state, softfloat_flag_invalid );\r\n uiZ64 = defaultNaNExtF80UI64;\r\n uiZ0 = defaultNaNExtF80UI0;\r\n goto uiZ;\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n copyA:\r\n if ( expA < 1 ) {\r\n sigA >>= 1 - expA;\r\n expA = 0;\r\n }\r\n uiZ64 = packToExtF80UI64( signA, expA );\r\n uiZ0 = sigA;\r\n uiZ:\r\n uZ.s.signExp = uiZ64;\r\n uZ.s.signif = uiZ0;\r\n return uZ.f;\r\n\r\n}\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/extF80_roundToInt.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014, 2017 The Regents of the University of\r\nCalifornia. All rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \r\n#include \"platform.h\"\r\n#include \"internals.h\"\r\n#include \"specialize.h\"\r\n#include \"softfloat.h\"\r\n\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nextFloat80_t\r\n extF80_roundToInt( struct softfloat_state *state, extFloat80_t a, uint_fast8_t roundingMode, bool exact )\r\n{\r\n union { struct extFloat80M s; extFloat80_t f; } uA;\r\n uint_fast16_t uiA64, signUI64;\r\n int_fast32_t exp;\r\n uint_fast64_t sigA;\r\n uint_fast16_t uiZ64;\r\n uint_fast64_t sigZ;\r\n struct exp32_sig64 normExpSig;\r\n struct uint128 uiZ;\r\n uint_fast64_t lastBitMask, roundBitsMask;\r\n union { struct extFloat80M s; extFloat80_t f; } uZ;\r\n\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n uA.f = a;\r\n uiA64 = uA.s.signExp;\r\n signUI64 = uiA64 & packToExtF80UI64( 1, 0 );\r\n exp = expExtF80UI64( uiA64 );\r\n sigA = uA.s.signif;\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n if ( !(sigA & UINT64_C( 0x8000000000000000 )) && (exp != 0x7FFF) ) {\r\n if ( !sigA ) {\r\n uiZ64 = signUI64;\r\n sigZ = 0;\r\n goto uiZ;\r\n }\r\n normExpSig = softfloat_normSubnormalExtF80Sig( sigA );\r\n exp += normExpSig.exp;\r\n sigA = normExpSig.sig;\r\n }\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n if ( 0x403E <= exp ) {\r\n if ( exp == 0x7FFF ) {\r\n if ( sigA & UINT64_C( 0x7FFFFFFFFFFFFFFF ) ) {\r\n uiZ = softfloat_propagateNaNExtF80UI( state, uiA64, sigA, 0, 0 );\r\n uiZ64 = uiZ.v64;\r\n sigZ = uiZ.v0;\r\n goto uiZ;\r\n }\r\n sigZ = UINT64_C( 0x8000000000000000 );\r\n } else {\r\n sigZ = sigA;\r\n }\r\n uiZ64 = signUI64 | exp;\r\n goto uiZ;\r\n }\r\n if ( exp <= 0x3FFE ) {\r\n if ( exact ) state->exceptionFlags |= softfloat_flag_inexact;\r\n switch ( roundingMode ) {\r\n case softfloat_round_near_even:\r\n if ( !(sigA & UINT64_C( 0x7FFFFFFFFFFFFFFF )) ) break;\r\n __attribute__((fallthrough));\r\n case softfloat_round_near_maxMag:\r\n if ( exp == 0x3FFE ) goto mag1;\r\n break;\r\n case softfloat_round_min:\r\n if ( signUI64 ) goto mag1;\r\n break;\r\n case softfloat_round_max:\r\n if ( !signUI64 ) goto mag1;\r\n break;\r\n#ifdef SOFTFLOAT_ROUND_ODD\r\n case softfloat_round_odd:\r\n goto mag1;\r\n#endif\r\n }\r\n uiZ64 = signUI64;\r\n sigZ = 0;\r\n goto uiZ;\r\n mag1:\r\n uiZ64 = signUI64 | 0x3FFF;\r\n sigZ = UINT64_C( 0x8000000000000000 );\r\n goto uiZ;\r\n }\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n uiZ64 = signUI64 | exp;\r\n lastBitMask = (uint_fast64_t) 1<<(0x403E - exp);\r\n roundBitsMask = lastBitMask - 1;\r\n sigZ = sigA;\r\n if ( roundingMode == softfloat_round_near_maxMag ) {\r\n sigZ += lastBitMask>>1;\r\n } else if ( roundingMode == softfloat_round_near_even ) {\r\n sigZ += lastBitMask>>1;\r\n if ( !(sigZ & roundBitsMask) ) sigZ &= ~lastBitMask;\r\n } else if (\r\n roundingMode == (signUI64 ? softfloat_round_min : softfloat_round_max)\r\n ) {\r\n sigZ += roundBitsMask;\r\n }\r\n sigZ &= ~roundBitsMask;\r\n if ( !sigZ ) {\r\n ++uiZ64;\r\n sigZ = UINT64_C( 0x8000000000000000 );\r\n }\r\n if ( sigZ != sigA ) {\r\n#ifdef SOFTFLOAT_ROUND_ODD\r\n if ( roundingMode == softfloat_round_odd ) sigZ |= lastBitMask;\r\n#endif\r\n if ( exact ) state->exceptionFlags |= softfloat_flag_inexact;\r\n }\r\n uiZ:\r\n uZ.s.signExp = uiZ64;\r\n uZ.s.signif = sigZ;\r\n return uZ.f;\r\n\r\n}\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/extF80_sqrt.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014, 2015, 2016, 2017 The Regents of the\r\nUniversity of California. All rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \r\n#include \"platform.h\"\r\n#include \"internals.h\"\r\n#include \"specialize.h\"\r\n#include \"softfloat.h\"\r\n\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nextFloat80_t extF80_sqrt( struct softfloat_state *state, extFloat80_t a )\r\n{\r\n union { struct extFloat80M s; extFloat80_t f; } uA;\r\n uint_fast16_t uiA64;\r\n uint_fast64_t uiA0;\r\n bool signA;\r\n int_fast32_t expA;\r\n uint_fast64_t sigA;\r\n struct uint128 uiZ;\r\n uint_fast16_t uiZ64;\r\n uint_fast64_t uiZ0;\r\n struct exp32_sig64 normExpSig;\r\n int_fast32_t expZ;\r\n uint_fast32_t sig32A, recipSqrt32, sig32Z;\r\n struct uint128 rem;\r\n uint_fast64_t q, x64, sigZ;\r\n struct uint128 y, term;\r\n uint_fast64_t sigZExtra;\r\n union { struct extFloat80M s; extFloat80_t f; } uZ;\r\n\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n uA.f = a;\r\n uiA64 = uA.s.signExp;\r\n uiA0 = uA.s.signif;\r\n signA = signExtF80UI64( uiA64 );\r\n expA = expExtF80UI64( uiA64 );\r\n sigA = uiA0;\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n if ( expA == 0x7FFF ) {\r\n if ( sigA & UINT64_C( 0x7FFFFFFFFFFFFFFF ) ) {\r\n uiZ = softfloat_propagateNaNExtF80UI( state, uiA64, uiA0, 0, 0 );\r\n uiZ64 = uiZ.v64;\r\n uiZ0 = uiZ.v0;\r\n goto uiZ;\r\n }\r\n if ( ! signA ) return a;\r\n goto invalid;\r\n }\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n if ( signA ) {\r\n if ( ! sigA ) goto zero;\r\n goto invalid;\r\n }\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n if ( ! expA ) expA = 1;\r\n if ( ! (sigA & UINT64_C( 0x8000000000000000 )) ) {\r\n if ( ! sigA ) goto zero;\r\n normExpSig = softfloat_normSubnormalExtF80Sig( sigA );\r\n expA += normExpSig.exp;\r\n sigA = normExpSig.sig;\r\n }\r\n /*------------------------------------------------------------------------\r\n | (`sig32Z' is guaranteed to be a lower bound on the square root of\r\n | `sig32A', which makes `sig32Z' also a lower bound on the square root of\r\n | `sigA'.)\r\n *------------------------------------------------------------------------*/\r\n expZ = ((expA - 0x3FFF)>>1) + 0x3FFF;\r\n expA &= 1;\r\n sig32A = sigA>>32;\r\n recipSqrt32 = softfloat_approxRecipSqrt32_1( expA, sig32A );\r\n sig32Z = ((uint_fast64_t) sig32A * recipSqrt32)>>32;\r\n if ( expA ) {\r\n sig32Z >>= 1;\r\n rem = softfloat_shortShiftLeft128( 0, sigA, 61 );\r\n } else {\r\n rem = softfloat_shortShiftLeft128( 0, sigA, 62 );\r\n }\r\n rem.v64 -= (uint_fast64_t) sig32Z * sig32Z;\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n q = ((uint32_t) (rem.v64>>2) * (uint_fast64_t) recipSqrt32)>>32;\r\n x64 = (uint_fast64_t) sig32Z<<32;\r\n sigZ = x64 + (q<<3);\r\n y = softfloat_shortShiftLeft128( rem.v64, rem.v0, 29 );\r\n /*------------------------------------------------------------------------\r\n | (Repeating this loop is a rare occurrence.)\r\n *------------------------------------------------------------------------*/\r\n for (;;) {\r\n term = softfloat_mul64ByShifted32To128( x64 + sigZ, q );\r\n rem = softfloat_sub128( y.v64, y.v0, term.v64, term.v0 );\r\n if ( ! (rem.v64 & UINT64_C( 0x8000000000000000 )) ) break;\r\n --q;\r\n sigZ -= 1<<3;\r\n }\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n q = (((rem.v64>>2) * recipSqrt32)>>32) + 2;\r\n x64 = sigZ;\r\n sigZ = (sigZ<<1) + (q>>25);\r\n sigZExtra = (uint64_t) (q<<39);\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n if ( (q & 0xFFFFFF) <= 2 ) {\r\n q &= ~(uint_fast64_t) 0xFFFF;\r\n sigZExtra = (uint64_t) (q<<39);\r\n term = softfloat_mul64ByShifted32To128( x64 + (q>>27), q );\r\n x64 = (uint32_t) (q<<5) * (uint_fast64_t) (uint32_t) q;\r\n term = softfloat_add128( term.v64, term.v0, 0, x64 );\r\n rem = softfloat_shortShiftLeft128( rem.v64, rem.v0, 28 );\r\n rem = softfloat_sub128( rem.v64, rem.v0, term.v64, term.v0 );\r\n if ( rem.v64 & UINT64_C( 0x8000000000000000 ) ) {\r\n if ( ! sigZExtra ) --sigZ;\r\n --sigZExtra;\r\n } else {\r\n if ( rem.v64 | rem.v0 ) sigZExtra |= 1;\r\n }\r\n }\r\n return\r\n softfloat_roundPackToExtF80(\r\n state, 0, expZ, sigZ, sigZExtra, state->roundingPrecision );\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n invalid:\r\n softfloat_raiseFlags( state, softfloat_flag_invalid );\r\n uiZ64 = defaultNaNExtF80UI64;\r\n uiZ0 = defaultNaNExtF80UI0;\r\n goto uiZ;\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n zero:\r\n uiZ64 = packToExtF80UI64( signA, 0 );\r\n uiZ0 = 0;\r\n uiZ:\r\n uZ.s.signExp = uiZ64;\r\n uZ.s.signif = uiZ0;\r\n return uZ.f;\r\n\r\n}\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/extF80_sub.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014 The Regents of the University of California.\r\nAll rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \r\n#include \"platform.h\"\r\n#include \"internals.h\"\r\n#include \"softfloat.h\"\r\n\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nextFloat80_t extF80_sub( struct softfloat_state *state, extFloat80_t a, extFloat80_t b )\r\n{\r\n union { struct extFloat80M s; extFloat80_t f; } uA;\r\n uint_fast16_t uiA64;\r\n uint_fast64_t uiA0;\r\n bool signA;\r\n union { struct extFloat80M s; extFloat80_t f; } uB;\r\n uint_fast16_t uiB64;\r\n uint_fast64_t uiB0;\r\n bool signB;\r\n#if ! defined INLINE_LEVEL || (INLINE_LEVEL < 2)\r\n extFloat80_t\r\n (*magsFuncPtr)(\r\n struct softfloat_state *, uint_fast16_t, uint_fast64_t, uint_fast16_t, uint_fast64_t, bool );\r\n#endif\r\n\r\n uA.f = a;\r\n uiA64 = uA.s.signExp;\r\n uiA0 = uA.s.signif;\r\n signA = signExtF80UI64( uiA64 );\r\n uB.f = b;\r\n uiB64 = uB.s.signExp;\r\n uiB0 = uB.s.signif;\r\n signB = signExtF80UI64( uiB64 );\r\n#if defined INLINE_LEVEL && (2 <= INLINE_LEVEL)\r\n if ( signA == signB ) {\r\n return softfloat_subMagsExtF80( state, uiA64, uiA0, uiB64, uiB0, signA );\r\n } else {\r\n return softfloat_addMagsExtF80( state, uiA64, uiA0, uiB64, uiB0, signA );\r\n }\r\n#else\r\n magsFuncPtr =\r\n (signA == signB) ? softfloat_subMagsExtF80 : softfloat_addMagsExtF80;\r\n return (*magsFuncPtr)( state, uiA64, uiA0, uiB64, uiB0, signA );\r\n#endif\r\n\r\n}\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/extF80_to_f128.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014 The Regents of the University of California.\r\nAll rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \r\n#include \"platform.h\"\r\n#include \"internals.h\"\r\n#include \"specialize.h\"\r\n#include \"softfloat.h\"\r\n\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nfloat128_t extF80_to_f128( struct softfloat_state *state, extFloat80_t a )\r\n{\r\n union { struct extFloat80M s; extFloat80_t f; } uA;\r\n uint_fast16_t uiA64;\r\n uint_fast64_t uiA0;\r\n uint_fast16_t exp;\r\n uint_fast64_t frac;\r\n struct commonNaN commonNaN;\r\n struct uint128 uiZ;\r\n bool sign;\r\n struct uint128 frac128;\r\n union ui128_f128 uZ;\r\n\r\n uA.f = a;\r\n uiA64 = uA.s.signExp;\r\n uiA0 = uA.s.signif;\r\n exp = expExtF80UI64( uiA64 );\r\n frac = uiA0 & UINT64_C( 0x7FFFFFFFFFFFFFFF );\r\n if ( (exp == 0x7FFF) && frac ) {\r\n softfloat_extF80UIToCommonNaN( state, uiA64, uiA0, &commonNaN );\r\n uiZ = softfloat_commonNaNToF128UI( &commonNaN );\r\n } else {\r\n sign = signExtF80UI64( uiA64 );\r\n frac128 = softfloat_shortShiftLeft128( 0, frac, 49 );\r\n uiZ.v64 = packToF128UI64( sign, exp, frac128.v64 );\r\n uiZ.v0 = frac128.v0;\r\n }\r\n uZ.ui = uiZ;\r\n return uZ.f;\r\n\r\n}\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/extF80_to_f32.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014, 2015 The Regents of the University of\r\nCalifornia. All rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \r\n#include \"platform.h\"\r\n#include \"internals.h\"\r\n#include \"specialize.h\"\r\n#include \"softfloat.h\"\r\n\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nfloat32_t extF80_to_f32( struct softfloat_state *state, extFloat80_t a )\r\n{\r\n union { struct extFloat80M s; extFloat80_t f; } uA;\r\n uint_fast16_t uiA64;\r\n uint_fast64_t uiA0;\r\n bool sign;\r\n int_fast32_t exp;\r\n uint_fast64_t sig;\r\n struct commonNaN commonNaN;\r\n uint_fast32_t uiZ, sig32;\r\n union ui32_f32 uZ;\r\n\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n uA.f = a;\r\n uiA64 = uA.s.signExp;\r\n uiA0 = uA.s.signif;\r\n sign = signExtF80UI64( uiA64 );\r\n exp = expExtF80UI64( uiA64 );\r\n sig = uiA0;\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n if ( exp == 0x7FFF ) {\r\n if ( sig & UINT64_C( 0x7FFFFFFFFFFFFFFF ) ) {\r\n softfloat_extF80UIToCommonNaN( state, uiA64, uiA0, &commonNaN );\r\n uiZ = softfloat_commonNaNToF32UI( &commonNaN );\r\n } else {\r\n uiZ = packToF32UI( sign, 0xFF, 0 );\r\n }\r\n goto uiZ;\r\n }\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n sig32 = softfloat_shortShiftRightJam64( sig, 33 );\r\n if ( ! (exp | sig32) ) {\r\n uiZ = packToF32UI( sign, 0, 0 );\r\n goto uiZ;\r\n }\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n exp -= 0x3F81;\r\n if ( sizeof (int_fast16_t) < sizeof (int_fast32_t) ) {\r\n if ( exp < -0x1000 ) exp = -0x1000;\r\n }\r\n return softfloat_roundPackToF32( state, sign, exp, sig32 );\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n uiZ:\r\n uZ.ui = uiZ;\r\n return uZ.f;\r\n\r\n}\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/extF80_to_f64.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014, 2015 The Regents of the University of\r\nCalifornia. All rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \r\n#include \"platform.h\"\r\n#include \"internals.h\"\r\n#include \"specialize.h\"\r\n#include \"softfloat.h\"\r\n\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nfloat64_t extF80_to_f64( struct softfloat_state *state, extFloat80_t a )\r\n{\r\n union { struct extFloat80M s; extFloat80_t f; } uA;\r\n uint_fast16_t uiA64;\r\n uint_fast64_t uiA0;\r\n bool sign;\r\n int_fast32_t exp;\r\n uint_fast64_t sig;\r\n struct commonNaN commonNaN;\r\n uint_fast64_t uiZ;\r\n union ui64_f64 uZ;\r\n\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n uA.f = a;\r\n uiA64 = uA.s.signExp;\r\n uiA0 = uA.s.signif;\r\n sign = signExtF80UI64( uiA64 );\r\n exp = expExtF80UI64( uiA64 );\r\n sig = uiA0;\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n if ( ! (exp | sig) ) {\r\n uiZ = packToF64UI( sign, 0, 0 );\r\n goto uiZ;\r\n }\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n if ( exp == 0x7FFF ) {\r\n if ( sig & UINT64_C( 0x7FFFFFFFFFFFFFFF ) ) {\r\n softfloat_extF80UIToCommonNaN( state, uiA64, uiA0, &commonNaN );\r\n uiZ = softfloat_commonNaNToF64UI( &commonNaN );\r\n } else {\r\n uiZ = packToF64UI( sign, 0x7FF, 0 );\r\n }\r\n goto uiZ;\r\n }\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n sig = softfloat_shortShiftRightJam64( sig, 1 );\r\n exp -= 0x3C01;\r\n if ( sizeof (int_fast16_t) < sizeof (int_fast32_t) ) {\r\n if ( exp < -0x1000 ) exp = -0x1000;\r\n }\r\n return softfloat_roundPackToF64( state, sign, exp, sig );\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n uiZ:\r\n uZ.ui = uiZ;\r\n return uZ.f;\r\n\r\n}\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/extF80_to_i32.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014, 2015, 2016, 2017 The Regents of the\r\nUniversity of California. All rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \r\n#include \"platform.h\"\r\n#include \"internals.h\"\r\n#include \"specialize.h\"\r\n#include \"softfloat.h\"\r\n\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nint_fast32_t\r\n extF80_to_i32( struct softfloat_state *state, extFloat80_t a, uint_fast8_t roundingMode, bool exact )\r\n{\r\n union { struct extFloat80M s; extFloat80_t f; } uA;\r\n uint_fast16_t uiA64;\r\n bool sign;\r\n int_fast32_t exp;\r\n uint_fast64_t sig;\r\n int_fast32_t shiftDist;\r\n\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n uA.f = a;\r\n uiA64 = uA.s.signExp;\r\n sign = signExtF80UI64( uiA64 );\r\n exp = expExtF80UI64( uiA64 );\r\n sig = uA.s.signif;\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n#if (i32_fromNaN != i32_fromPosOverflow) || (i32_fromNaN != i32_fromNegOverflow)\r\n if ( (exp == 0x7FFF) && (sig & UINT64_C( 0x7FFFFFFFFFFFFFFF )) ) {\r\n#if (i32_fromNaN == i32_fromPosOverflow)\r\n sign = 0;\r\n#elif (i32_fromNaN == i32_fromNegOverflow)\r\n sign = 1;\r\n#else\r\n softfloat_raiseFlags( state, softfloat_flag_invalid );\r\n return i32_fromNaN;\r\n#endif\r\n }\r\n#endif\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n shiftDist = 0x4032 - exp;\r\n if ( shiftDist <= 0 ) shiftDist = 1;\r\n sig = softfloat_shiftRightJam64( sig, shiftDist );\r\n return softfloat_roundToI32( state, sign, sig, roundingMode, exact );\r\n\r\n}\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/extF80_to_i64.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014, 2015, 2016, 2017 The Regents of the\r\nUniversity of California. All rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \r\n#include \"platform.h\"\r\n#include \"internals.h\"\r\n#include \"specialize.h\"\r\n#include \"softfloat.h\"\r\n\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nint_fast64_t\r\n extF80_to_i64( struct softfloat_state *state, extFloat80_t a, uint_fast8_t roundingMode, bool exact )\r\n{\r\n union { struct extFloat80M s; extFloat80_t f; } uA;\r\n uint_fast16_t uiA64;\r\n bool sign;\r\n int_fast32_t exp;\r\n uint_fast64_t sig;\r\n int_fast32_t shiftDist;\r\n uint_fast64_t sigExtra;\r\n struct uint64_extra sig64Extra;\r\n\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n uA.f = a;\r\n uiA64 = uA.s.signExp;\r\n sign = signExtF80UI64( uiA64 );\r\n exp = expExtF80UI64( uiA64 );\r\n sig = uA.s.signif;\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n shiftDist = 0x403E - exp;\r\n if ( shiftDist <= 0 ) {\r\n /*--------------------------------------------------------------------\r\n *--------------------------------------------------------------------*/\r\n if ( shiftDist ) {\r\n softfloat_raiseFlags( state, softfloat_flag_invalid );\r\n return\r\n (exp == 0x7FFF) && (sig & UINT64_C( 0x7FFFFFFFFFFFFFFF ))\r\n ? i64_fromNaN\r\n : sign ? i64_fromNegOverflow : i64_fromPosOverflow;\r\n }\r\n /*--------------------------------------------------------------------\r\n *--------------------------------------------------------------------*/\r\n sigExtra = 0;\r\n } else {\r\n /*--------------------------------------------------------------------\r\n *--------------------------------------------------------------------*/\r\n sig64Extra = softfloat_shiftRightJam64Extra( sig, 0, shiftDist );\r\n sig = sig64Extra.v;\r\n sigExtra = sig64Extra.extra;\r\n }\r\n return softfloat_roundToI64( state, sign, sig, sigExtra, roundingMode, exact );\r\n\r\n}\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/extF80_to_ui64.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014, 2015, 2016, 2017 The Regents of the\r\nUniversity of California. All rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \r\n#include \"platform.h\"\r\n#include \"internals.h\"\r\n#include \"specialize.h\"\r\n#include \"softfloat.h\"\r\n\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nuint_fast64_t\r\n extF80_to_ui64( struct softfloat_state *state, extFloat80_t a, uint_fast8_t roundingMode, bool exact )\r\n{\r\n union { struct extFloat80M s; extFloat80_t f; } uA;\r\n uint_fast16_t uiA64;\r\n bool sign;\r\n int_fast32_t exp;\r\n uint_fast64_t sig;\r\n int_fast32_t shiftDist;\r\n uint_fast64_t sigExtra;\r\n struct uint64_extra sig64Extra;\r\n\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n uA.f = a;\r\n uiA64 = uA.s.signExp;\r\n sign = signExtF80UI64( uiA64 );\r\n exp = expExtF80UI64( uiA64 );\r\n sig = uA.s.signif;\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n shiftDist = 0x403E - exp;\r\n if ( shiftDist < 0 ) {\r\n softfloat_raiseFlags( state, softfloat_flag_invalid );\r\n return\r\n (exp == 0x7FFF) && (sig & UINT64_C( 0x7FFFFFFFFFFFFFFF ))\r\n ? ui64_fromNaN\r\n : sign ? ui64_fromNegOverflow : ui64_fromPosOverflow;\r\n }\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n sigExtra = 0;\r\n if ( shiftDist ) {\r\n sig64Extra = softfloat_shiftRightJam64Extra( sig, 0, shiftDist );\r\n sig = sig64Extra.v;\r\n sigExtra = sig64Extra.extra;\r\n }\r\n return softfloat_roundToUI64( state, sign, sig, sigExtra, roundingMode, exact );\r\n\r\n}\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/f128_add.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014 The Regents of the University of California.\r\nAll rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \r\n#include \"platform.h\"\r\n#include \"internals.h\"\r\n#include \"softfloat.h\"\r\n\r\nfloat128_t f128_add( struct softfloat_state *state, float128_t a, float128_t b )\r\n{\r\n union ui128_f128 uA;\r\n uint_fast64_t uiA64, uiA0;\r\n bool signA;\r\n union ui128_f128 uB;\r\n uint_fast64_t uiB64, uiB0;\r\n bool signB;\r\n#if ! defined INLINE_LEVEL || (INLINE_LEVEL < 2)\r\n float128_t\r\n (*magsFuncPtr)(\r\n uint_fast64_t, uint_fast64_t, uint_fast64_t, uint_fast64_t, bool );\r\n#endif\r\n\r\n uA.f = a;\r\n uiA64 = uA.ui.v64;\r\n uiA0 = uA.ui.v0;\r\n signA = signF128UI64( uiA64 );\r\n uB.f = b;\r\n uiB64 = uB.ui.v64;\r\n uiB0 = uB.ui.v0;\r\n signB = signF128UI64( uiB64 );\r\n#if defined INLINE_LEVEL && (2 <= INLINE_LEVEL)\r\n if ( signA == signB ) {\r\n return softfloat_addMagsF128( state, uiA64, uiA0, uiB64, uiB0, signA );\r\n } else {\r\n return softfloat_subMagsF128( state, uiA64, uiA0, uiB64, uiB0, signA );\r\n }\r\n#else\r\n magsFuncPtr =\r\n (signA == signB) ? softfloat_addMagsF128 : softfloat_subMagsF128;\r\n return (*magsFuncPtr)( uiA64, uiA0, uiB64, uiB0, signA );\r\n#endif\r\n\r\n}\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/f128_div.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014 The Regents of the University of California.\r\nAll rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \r\n#include \"platform.h\"\r\n#include \"internals.h\"\r\n#include \"specialize.h\"\r\n#include \"softfloat.h\"\r\n\r\nfloat128_t f128_div( struct softfloat_state *state, float128_t a, float128_t b )\r\n{\r\n union ui128_f128 uA;\r\n uint_fast64_t uiA64, uiA0;\r\n bool signA;\r\n int_fast32_t expA;\r\n struct uint128 sigA;\r\n union ui128_f128 uB;\r\n uint_fast64_t uiB64, uiB0;\r\n bool signB;\r\n int_fast32_t expB;\r\n struct uint128 sigB;\r\n bool signZ;\r\n struct exp32_sig128 normExpSig;\r\n int_fast32_t expZ;\r\n struct uint128 rem;\r\n uint_fast32_t recip32;\r\n int ix;\r\n uint_fast64_t q64;\r\n uint_fast32_t q;\r\n struct uint128 term;\r\n uint_fast32_t qs[3];\r\n uint_fast64_t sigZExtra;\r\n struct uint128 sigZ, uiZ;\r\n union ui128_f128 uZ;\r\n\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n uA.f = a;\r\n uiA64 = uA.ui.v64;\r\n uiA0 = uA.ui.v0;\r\n signA = signF128UI64( uiA64 );\r\n expA = expF128UI64( uiA64 );\r\n sigA.v64 = fracF128UI64( uiA64 );\r\n sigA.v0 = uiA0;\r\n uB.f = b;\r\n uiB64 = uB.ui.v64;\r\n uiB0 = uB.ui.v0;\r\n signB = signF128UI64( uiB64 );\r\n expB = expF128UI64( uiB64 );\r\n sigB.v64 = fracF128UI64( uiB64 );\r\n sigB.v0 = uiB0;\r\n signZ = signA ^ signB;\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n if ( expA == 0x7FFF ) {\r\n if ( sigA.v64 | sigA.v0 ) goto propagateNaN;\r\n if ( expB == 0x7FFF ) {\r\n if ( sigB.v64 | sigB.v0 ) goto propagateNaN;\r\n goto invalid;\r\n }\r\n goto infinity;\r\n }\r\n if ( expB == 0x7FFF ) {\r\n if ( sigB.v64 | sigB.v0 ) goto propagateNaN;\r\n goto zero;\r\n }\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n if ( ! expB ) {\r\n if ( ! (sigB.v64 | sigB.v0) ) {\r\n if ( ! (expA | sigA.v64 | sigA.v0) ) goto invalid;\r\n softfloat_raiseFlags( state, softfloat_flag_infinite );\r\n goto infinity;\r\n }\r\n normExpSig = softfloat_normSubnormalF128Sig( sigB.v64, sigB.v0 );\r\n expB = normExpSig.exp;\r\n sigB = normExpSig.sig;\r\n }\r\n if ( ! expA ) {\r\n if ( ! (sigA.v64 | sigA.v0) ) goto zero;\r\n normExpSig = softfloat_normSubnormalF128Sig( sigA.v64, sigA.v0 );\r\n expA = normExpSig.exp;\r\n sigA = normExpSig.sig;\r\n }\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n expZ = expA - expB + 0x3FFE;\r\n sigA.v64 |= UINT64_C( 0x0001000000000000 );\r\n sigB.v64 |= UINT64_C( 0x0001000000000000 );\r\n rem = sigA;\r\n if ( softfloat_lt128( sigA.v64, sigA.v0, sigB.v64, sigB.v0 ) ) {\r\n --expZ;\r\n rem = softfloat_add128( sigA.v64, sigA.v0, sigA.v64, sigA.v0 );\r\n }\r\n recip32 = softfloat_approxRecip32_1( sigB.v64>>17 );\r\n ix = 3;\r\n for (;;) {\r\n q64 = (uint_fast64_t) (uint32_t) (rem.v64>>19) * recip32;\r\n q = (q64 + 0x80000000)>>32;\r\n --ix;\r\n if ( ix < 0 ) break;\r\n rem = softfloat_shortShiftLeft128( rem.v64, rem.v0, 29 );\r\n term = softfloat_mul128By32( sigB.v64, sigB.v0, q );\r\n rem = softfloat_sub128( rem.v64, rem.v0, term.v64, term.v0 );\r\n if ( rem.v64 & UINT64_C( 0x8000000000000000 ) ) {\r\n --q;\r\n rem = softfloat_add128( rem.v64, rem.v0, sigB.v64, sigB.v0 );\r\n }\r\n qs[ix] = q;\r\n }\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n if ( ((q + 1) & 7) < 2 ) {\r\n rem = softfloat_shortShiftLeft128( rem.v64, rem.v0, 29 );\r\n term = softfloat_mul128By32( sigB.v64, sigB.v0, q );\r\n rem = softfloat_sub128( rem.v64, rem.v0, term.v64, term.v0 );\r\n if ( rem.v64 & UINT64_C( 0x8000000000000000 ) ) {\r\n --q;\r\n rem = softfloat_add128( rem.v64, rem.v0, sigB.v64, sigB.v0 );\r\n } else if ( softfloat_le128( sigB.v64, sigB.v0, rem.v64, rem.v0 ) ) {\r\n ++q;\r\n rem = softfloat_sub128( rem.v64, rem.v0, sigB.v64, sigB.v0 );\r\n }\r\n if ( rem.v64 | rem.v0 ) q |= 1;\r\n }\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n sigZExtra = (uint64_t) ((uint_fast64_t) q<<60);\r\n term = softfloat_shortShiftLeft128( 0, qs[1], 54 );\r\n sigZ =\r\n softfloat_add128(\r\n (uint_fast64_t) qs[2]<<19, ((uint_fast64_t) qs[0]<<25) + (q>>4),\r\n term.v64, term.v0\r\n );\r\n return\r\n softfloat_roundPackToF128( state, signZ, expZ, sigZ.v64, sigZ.v0, sigZExtra );\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n propagateNaN:\r\n uiZ = softfloat_propagateNaNF128UI( state, uiA64, uiA0, uiB64, uiB0 );\r\n goto uiZ;\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n invalid:\r\n softfloat_raiseFlags( state, softfloat_flag_invalid );\r\n uiZ.v64 = defaultNaNF128UI64;\r\n uiZ.v0 = defaultNaNF128UI0;\r\n goto uiZ;\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n infinity:\r\n uiZ.v64 = packToF128UI64( signZ, 0x7FFF, 0 );\r\n goto uiZ0;\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n zero:\r\n uiZ.v64 = packToF128UI64( signZ, 0, 0 );\r\n uiZ0:\r\n uiZ.v0 = 0;\r\n uiZ:\r\n uZ.ui = uiZ;\r\n return uZ.f;\r\n\r\n}\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/f128_eq.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014 The Regents of the University of California.\r\nAll rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \r\n#include \"platform.h\"\r\n#include \"internals.h\"\r\n#include \"specialize.h\"\r\n#include \"softfloat.h\"\r\n\r\nbool f128_eq( struct softfloat_state *state, float128_t a, float128_t b )\r\n{\r\n union ui128_f128 uA;\r\n uint_fast64_t uiA64, uiA0;\r\n union ui128_f128 uB;\r\n uint_fast64_t uiB64, uiB0;\r\n\r\n uA.f = a;\r\n uiA64 = uA.ui.v64;\r\n uiA0 = uA.ui.v0;\r\n uB.f = b;\r\n uiB64 = uB.ui.v64;\r\n uiB0 = uB.ui.v0;\r\n if ( isNaNF128UI( uiA64, uiA0 ) || isNaNF128UI( uiB64, uiB0 ) ) {\r\n if (\r\n softfloat_isSigNaNF128UI( uiA64, uiA0 )\r\n || softfloat_isSigNaNF128UI( uiB64, uiB0 )\r\n ) {\r\n softfloat_raiseFlags( state, softfloat_flag_invalid );\r\n }\r\n return false;\r\n }\r\n return\r\n (uiA0 == uiB0)\r\n && ( (uiA64 == uiB64)\r\n || (! uiA0 && ! ((uiA64 | uiB64) & UINT64_C( 0x7FFFFFFFFFFFFFFF )))\r\n );\r\n\r\n}\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/f128_eq_signaling.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014 The Regents of the University of California.\r\nAll rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \r\n#include \"platform.h\"\r\n#include \"internals.h\"\r\n#include \"softfloat.h\"\r\n\r\nbool f128_eq_signaling( struct softfloat_state *state, float128_t a, float128_t b )\r\n{\r\n union ui128_f128 uA;\r\n uint_fast64_t uiA64, uiA0;\r\n union ui128_f128 uB;\r\n uint_fast64_t uiB64, uiB0;\r\n\r\n uA.f = a;\r\n uiA64 = uA.ui.v64;\r\n uiA0 = uA.ui.v0;\r\n uB.f = b;\r\n uiB64 = uB.ui.v64;\r\n uiB0 = uB.ui.v0;\r\n if ( isNaNF128UI( uiA64, uiA0 ) || isNaNF128UI( uiB64, uiB0 ) ) {\r\n softfloat_raiseFlags( state, softfloat_flag_invalid );\r\n return false;\r\n }\r\n return\r\n (uiA0 == uiB0)\r\n && ( (uiA64 == uiB64)\r\n || (! uiA0 && ! ((uiA64 | uiB64) & UINT64_C( 0x7FFFFFFFFFFFFFFF )))\r\n );\r\n\r\n}\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/f128_isSignalingNaN.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014 The Regents of the University of California.\r\nAll rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \"platform.h\"\r\n#include \"internals.h\"\r\n#include \"specialize.h\"\r\n#include \"softfloat.h\"\r\n\r\nbool f128_isSignalingNaN( float128_t a )\r\n{\r\n union ui128_f128 uA;\r\n\r\n uA.f = a;\r\n return softfloat_isSigNaNF128UI( uA.ui.v64, uA.ui.v0 );\r\n\r\n}\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/f128_le.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014 The Regents of the University of California.\r\nAll rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \r\n#include \"platform.h\"\r\n#include \"internals.h\"\r\n#include \"softfloat.h\"\r\n\r\nbool f128_le( struct softfloat_state *state, float128_t a, float128_t b )\r\n{\r\n union ui128_f128 uA;\r\n uint_fast64_t uiA64, uiA0;\r\n union ui128_f128 uB;\r\n uint_fast64_t uiB64, uiB0;\r\n bool signA, signB;\r\n\r\n uA.f = a;\r\n uiA64 = uA.ui.v64;\r\n uiA0 = uA.ui.v0;\r\n uB.f = b;\r\n uiB64 = uB.ui.v64;\r\n uiB0 = uB.ui.v0;\r\n if ( isNaNF128UI( uiA64, uiA0 ) || isNaNF128UI( uiB64, uiB0 ) ) {\r\n softfloat_raiseFlags( state, softfloat_flag_invalid );\r\n return false;\r\n }\r\n signA = signF128UI64( uiA64 );\r\n signB = signF128UI64( uiB64 );\r\n return\r\n (signA != signB)\r\n ? signA\r\n || ! (((uiA64 | uiB64) & UINT64_C( 0x7FFFFFFFFFFFFFFF ))\r\n | uiA0 | uiB0)\r\n : ((uiA64 == uiB64) && (uiA0 == uiB0))\r\n || (signA ^ softfloat_lt128( uiA64, uiA0, uiB64, uiB0 ));\r\n\r\n}\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/f128_le_quiet.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014 The Regents of the University of California.\r\nAll rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \r\n#include \"platform.h\"\r\n#include \"internals.h\"\r\n#include \"specialize.h\"\r\n#include \"softfloat.h\"\r\n\r\nbool f128_le_quiet( struct softfloat_state *state, float128_t a, float128_t b )\r\n{\r\n union ui128_f128 uA;\r\n uint_fast64_t uiA64, uiA0;\r\n union ui128_f128 uB;\r\n uint_fast64_t uiB64, uiB0;\r\n bool signA, signB;\r\n\r\n uA.f = a;\r\n uiA64 = uA.ui.v64;\r\n uiA0 = uA.ui.v0;\r\n uB.f = b;\r\n uiB64 = uB.ui.v64;\r\n uiB0 = uB.ui.v0;\r\n if ( isNaNF128UI( uiA64, uiA0 ) || isNaNF128UI( uiB64, uiB0 ) ) {\r\n if (\r\n softfloat_isSigNaNF128UI( uiA64, uiA0 )\r\n || softfloat_isSigNaNF128UI( uiB64, uiB0 )\r\n ) {\r\n softfloat_raiseFlags( state, softfloat_flag_invalid );\r\n }\r\n return false;\r\n }\r\n signA = signF128UI64( uiA64 );\r\n signB = signF128UI64( uiB64 );\r\n return\r\n (signA != signB)\r\n ? signA\r\n || ! (((uiA64 | uiB64) & UINT64_C( 0x7FFFFFFFFFFFFFFF ))\r\n | uiA0 | uiB0)\r\n : ((uiA64 == uiB64) && (uiA0 == uiB0))\r\n || (signA ^ softfloat_lt128( uiA64, uiA0, uiB64, uiB0 ));\r\n\r\n}\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/f128_lt.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014 The Regents of the University of California.\r\nAll rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \r\n#include \"platform.h\"\r\n#include \"internals.h\"\r\n#include \"softfloat.h\"\r\n\r\nbool f128_lt( struct softfloat_state *state, float128_t a, float128_t b )\r\n{\r\n union ui128_f128 uA;\r\n uint_fast64_t uiA64, uiA0;\r\n union ui128_f128 uB;\r\n uint_fast64_t uiB64, uiB0;\r\n bool signA, signB;\r\n\r\n uA.f = a;\r\n uiA64 = uA.ui.v64;\r\n uiA0 = uA.ui.v0;\r\n uB.f = b;\r\n uiB64 = uB.ui.v64;\r\n uiB0 = uB.ui.v0;\r\n if ( isNaNF128UI( uiA64, uiA0 ) || isNaNF128UI( uiB64, uiB0 ) ) {\r\n softfloat_raiseFlags( state, softfloat_flag_invalid );\r\n return false;\r\n }\r\n signA = signF128UI64( uiA64 );\r\n signB = signF128UI64( uiB64 );\r\n return\r\n (signA != signB)\r\n ? signA\r\n && (((uiA64 | uiB64) & UINT64_C( 0x7FFFFFFFFFFFFFFF ))\r\n | uiA0 | uiB0)\r\n : ((uiA64 != uiB64) || (uiA0 != uiB0))\r\n && (signA ^ softfloat_lt128( uiA64, uiA0, uiB64, uiB0 ));\r\n\r\n}\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/f128_lt_quiet.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014 The Regents of the University of California.\r\nAll rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \r\n#include \"platform.h\"\r\n#include \"internals.h\"\r\n#include \"specialize.h\"\r\n#include \"softfloat.h\"\r\n\r\nbool f128_lt_quiet( struct softfloat_state *state, float128_t a, float128_t b )\r\n{\r\n union ui128_f128 uA;\r\n uint_fast64_t uiA64, uiA0;\r\n union ui128_f128 uB;\r\n uint_fast64_t uiB64, uiB0;\r\n bool signA, signB;\r\n\r\n uA.f = a;\r\n uiA64 = uA.ui.v64;\r\n uiA0 = uA.ui.v0;\r\n uB.f = b;\r\n uiB64 = uB.ui.v64;\r\n uiB0 = uB.ui.v0;\r\n if ( isNaNF128UI( uiA64, uiA0 ) || isNaNF128UI( uiB64, uiB0 ) ) {\r\n if (\r\n softfloat_isSigNaNF128UI( uiA64, uiA0 )\r\n || softfloat_isSigNaNF128UI( uiB64, uiB0 )\r\n ) {\r\n softfloat_raiseFlags( state, softfloat_flag_invalid );\r\n }\r\n return false;\r\n }\r\n signA = signF128UI64( uiA64 );\r\n signB = signF128UI64( uiB64 );\r\n return\r\n (signA != signB)\r\n ? signA\r\n && (((uiA64 | uiB64) & UINT64_C( 0x7FFFFFFFFFFFFFFF ))\r\n | uiA0 | uiB0)\r\n : ((uiA64 != uiB64) || (uiA0 != uiB0))\r\n && (signA ^ softfloat_lt128( uiA64, uiA0, uiB64, uiB0 ));\r\n\r\n}\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/f128_mul.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014 The Regents of the University of California.\r\nAll rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \r\n#include \"platform.h\"\r\n#include \"internals.h\"\r\n#include \"specialize.h\"\r\n#include \"softfloat.h\"\r\n\r\nfloat128_t f128_mul( struct softfloat_state *state, float128_t a, float128_t b )\r\n{\r\n union ui128_f128 uA;\r\n uint_fast64_t uiA64, uiA0;\r\n bool signA;\r\n int_fast32_t expA;\r\n struct uint128 sigA;\r\n union ui128_f128 uB;\r\n uint_fast64_t uiB64, uiB0;\r\n bool signB;\r\n int_fast32_t expB;\r\n struct uint128 sigB;\r\n bool signZ;\r\n uint_fast64_t magBits;\r\n struct exp32_sig128 normExpSig;\r\n int_fast32_t expZ;\r\n uint64_t sig256Z[4];\r\n uint_fast64_t sigZExtra;\r\n struct uint128 sigZ;\r\n struct uint128_extra sig128Extra;\r\n struct uint128 uiZ;\r\n union ui128_f128 uZ;\r\n\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n uA.f = a;\r\n uiA64 = uA.ui.v64;\r\n uiA0 = uA.ui.v0;\r\n signA = signF128UI64( uiA64 );\r\n expA = expF128UI64( uiA64 );\r\n sigA.v64 = fracF128UI64( uiA64 );\r\n sigA.v0 = uiA0;\r\n uB.f = b;\r\n uiB64 = uB.ui.v64;\r\n uiB0 = uB.ui.v0;\r\n signB = signF128UI64( uiB64 );\r\n expB = expF128UI64( uiB64 );\r\n sigB.v64 = fracF128UI64( uiB64 );\r\n sigB.v0 = uiB0;\r\n signZ = signA ^ signB;\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n if ( expA == 0x7FFF ) {\r\n if (\r\n (sigA.v64 | sigA.v0) || ((expB == 0x7FFF) && (sigB.v64 | sigB.v0))\r\n ) {\r\n goto propagateNaN;\r\n }\r\n magBits = expB | sigB.v64 | sigB.v0;\r\n goto infArg;\r\n }\r\n if ( expB == 0x7FFF ) {\r\n if ( sigB.v64 | sigB.v0 ) goto propagateNaN;\r\n magBits = expA | sigA.v64 | sigA.v0;\r\n goto infArg;\r\n }\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n if ( ! expA ) {\r\n if ( ! (sigA.v64 | sigA.v0) ) goto zero;\r\n normExpSig = softfloat_normSubnormalF128Sig( sigA.v64, sigA.v0 );\r\n expA = normExpSig.exp;\r\n sigA = normExpSig.sig;\r\n }\r\n if ( ! expB ) {\r\n if ( ! (sigB.v64 | sigB.v0) ) goto zero;\r\n normExpSig = softfloat_normSubnormalF128Sig( sigB.v64, sigB.v0 );\r\n expB = normExpSig.exp;\r\n sigB = normExpSig.sig;\r\n }\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n expZ = expA + expB - 0x4000;\r\n sigA.v64 |= UINT64_C( 0x0001000000000000 );\r\n sigB = softfloat_shortShiftLeft128( sigB.v64, sigB.v0, 16 );\r\n softfloat_mul128To256M( sigA.v64, sigA.v0, sigB.v64, sigB.v0, sig256Z );\r\n sigZExtra = sig256Z[indexWord( 4, 1 )] | (sig256Z[indexWord( 4, 0 )] != 0);\r\n sigZ =\r\n softfloat_add128(\r\n sig256Z[indexWord( 4, 3 )], sig256Z[indexWord( 4, 2 )],\r\n sigA.v64, sigA.v0\r\n );\r\n if ( UINT64_C( 0x0002000000000000 ) <= sigZ.v64 ) {\r\n ++expZ;\r\n sig128Extra =\r\n softfloat_shortShiftRightJam128Extra(\r\n sigZ.v64, sigZ.v0, sigZExtra, 1 );\r\n sigZ = sig128Extra.v;\r\n sigZExtra = sig128Extra.extra;\r\n }\r\n return\r\n softfloat_roundPackToF128( state, signZ, expZ, sigZ.v64, sigZ.v0, sigZExtra );\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n propagateNaN:\r\n uiZ = softfloat_propagateNaNF128UI( state, uiA64, uiA0, uiB64, uiB0 );\r\n goto uiZ;\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n infArg:\r\n if ( ! magBits ) {\r\n softfloat_raiseFlags( state, softfloat_flag_invalid );\r\n uiZ.v64 = defaultNaNF128UI64;\r\n uiZ.v0 = defaultNaNF128UI0;\r\n goto uiZ;\r\n }\r\n uiZ.v64 = packToF128UI64( signZ, 0x7FFF, 0 );\r\n goto uiZ0;\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n zero:\r\n uiZ.v64 = packToF128UI64( signZ, 0, 0 );\r\n uiZ0:\r\n uiZ.v0 = 0;\r\n uiZ:\r\n uZ.ui = uiZ;\r\n return uZ.f;\r\n\r\n}\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/f128_mulAdd.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014 The Regents of the University of California.\r\nAll rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \"platform.h\"\r\n#include \"internals.h\"\r\n#include \"softfloat.h\"\r\n\r\nfloat128_t f128_mulAdd( struct softfloat_state *state, float128_t a, float128_t b, float128_t c )\r\n{\r\n union ui128_f128 uA;\r\n uint_fast64_t uiA64, uiA0;\r\n union ui128_f128 uB;\r\n uint_fast64_t uiB64, uiB0;\r\n union ui128_f128 uC;\r\n uint_fast64_t uiC64, uiC0;\r\n\r\n uA.f = a;\r\n uiA64 = uA.ui.v64;\r\n uiA0 = uA.ui.v0;\r\n uB.f = b;\r\n uiB64 = uB.ui.v64;\r\n uiB0 = uB.ui.v0;\r\n uC.f = c;\r\n uiC64 = uC.ui.v64;\r\n uiC0 = uC.ui.v0;\r\n return softfloat_mulAddF128( uiA64, uiA0, uiB64, uiB0, uiC64, uiC0, 0 );\r\n\r\n}\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/f128_rem.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014, 2015, 2016 The Regents of the University of\r\nCalifornia. All rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \r\n#include \"platform.h\"\r\n#include \"internals.h\"\r\n#include \"specialize.h\"\r\n#include \"softfloat.h\"\r\n\r\nfloat128_t f128_rem( struct softfloat_state *state, float128_t a, float128_t b )\r\n{\r\n union ui128_f128 uA;\r\n uint_fast64_t uiA64, uiA0;\r\n bool signA;\r\n int_fast32_t expA;\r\n struct uint128 sigA;\r\n union ui128_f128 uB;\r\n uint_fast64_t uiB64, uiB0;\r\n int_fast32_t expB;\r\n struct uint128 sigB;\r\n struct exp32_sig128 normExpSig;\r\n struct uint128 rem;\r\n int_fast32_t expDiff;\r\n uint_fast32_t q, recip32;\r\n uint_fast64_t q64;\r\n struct uint128 term, altRem, meanRem;\r\n bool signRem;\r\n struct uint128 uiZ;\r\n union ui128_f128 uZ;\r\n\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n uA.f = a;\r\n uiA64 = uA.ui.v64;\r\n uiA0 = uA.ui.v0;\r\n signA = signF128UI64( uiA64 );\r\n expA = expF128UI64( uiA64 );\r\n sigA.v64 = fracF128UI64( uiA64 );\r\n sigA.v0 = uiA0;\r\n uB.f = b;\r\n uiB64 = uB.ui.v64;\r\n uiB0 = uB.ui.v0;\r\n expB = expF128UI64( uiB64 );\r\n sigB.v64 = fracF128UI64( uiB64 );\r\n sigB.v0 = uiB0;\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n if ( expA == 0x7FFF ) {\r\n if (\r\n (sigA.v64 | sigA.v0) || ((expB == 0x7FFF) && (sigB.v64 | sigB.v0))\r\n ) {\r\n goto propagateNaN;\r\n }\r\n goto invalid;\r\n }\r\n if ( expB == 0x7FFF ) {\r\n if ( sigB.v64 | sigB.v0 ) goto propagateNaN;\r\n return a;\r\n }\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n if ( ! expB ) {\r\n if ( ! (sigB.v64 | sigB.v0) ) goto invalid;\r\n normExpSig = softfloat_normSubnormalF128Sig( sigB.v64, sigB.v0 );\r\n expB = normExpSig.exp;\r\n sigB = normExpSig.sig;\r\n }\r\n if ( ! expA ) {\r\n if ( ! (sigA.v64 | sigA.v0) ) return a;\r\n normExpSig = softfloat_normSubnormalF128Sig( sigA.v64, sigA.v0 );\r\n expA = normExpSig.exp;\r\n sigA = normExpSig.sig;\r\n }\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n sigA.v64 |= UINT64_C( 0x0001000000000000 );\r\n sigB.v64 |= UINT64_C( 0x0001000000000000 );\r\n rem = sigA;\r\n expDiff = expA - expB;\r\n if ( expDiff < 1 ) {\r\n if ( expDiff < -1 ) return a;\r\n if ( expDiff ) {\r\n --expB;\r\n sigB = softfloat_add128( sigB.v64, sigB.v0, sigB.v64, sigB.v0 );\r\n q = 0;\r\n } else {\r\n q = softfloat_le128( sigB.v64, sigB.v0, rem.v64, rem.v0 );\r\n if ( q ) {\r\n rem = softfloat_sub128( rem.v64, rem.v0, sigB.v64, sigB.v0 );\r\n }\r\n }\r\n } else {\r\n recip32 = softfloat_approxRecip32_1( sigB.v64>>17 );\r\n expDiff -= 30;\r\n for (;;) {\r\n q64 = (uint_fast64_t) (uint32_t) (rem.v64>>19) * recip32;\r\n if ( expDiff < 0 ) break;\r\n q = (q64 + 0x80000000)>>32;\r\n rem = softfloat_shortShiftLeft128( rem.v64, rem.v0, 29 );\r\n term = softfloat_mul128By32( sigB.v64, sigB.v0, q );\r\n rem = softfloat_sub128( rem.v64, rem.v0, term.v64, term.v0 );\r\n if ( rem.v64 & UINT64_C( 0x8000000000000000 ) ) {\r\n rem = softfloat_add128( rem.v64, rem.v0, sigB.v64, sigB.v0 );\r\n }\r\n expDiff -= 29;\r\n }\r\n /*--------------------------------------------------------------------\r\n | (`expDiff' cannot be less than -29 here.)\r\n *--------------------------------------------------------------------*/\r\n q = (uint32_t) (q64>>32)>>(~expDiff & 31);\r\n rem = softfloat_shortShiftLeft128( rem.v64, rem.v0, expDiff + 30 );\r\n term = softfloat_mul128By32( sigB.v64, sigB.v0, q );\r\n rem = softfloat_sub128( rem.v64, rem.v0, term.v64, term.v0 );\r\n if ( rem.v64 & UINT64_C( 0x8000000000000000 ) ) {\r\n altRem = softfloat_add128( rem.v64, rem.v0, sigB.v64, sigB.v0 );\r\n goto selectRem;\r\n }\r\n }\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n do {\r\n altRem = rem;\r\n ++q;\r\n rem = softfloat_sub128( rem.v64, rem.v0, sigB.v64, sigB.v0 );\r\n } while ( ! (rem.v64 & UINT64_C( 0x8000000000000000 )) );\r\n selectRem:\r\n meanRem = softfloat_add128( rem.v64, rem.v0, altRem.v64, altRem.v0 );\r\n if (\r\n (meanRem.v64 & UINT64_C( 0x8000000000000000 ))\r\n || (! (meanRem.v64 | meanRem.v0) && (q & 1))\r\n ) {\r\n rem = altRem;\r\n }\r\n signRem = signA;\r\n if ( rem.v64 & UINT64_C( 0x8000000000000000 ) ) {\r\n signRem = ! signRem;\r\n rem = softfloat_sub128( 0, 0, rem.v64, rem.v0 );\r\n }\r\n return softfloat_normRoundPackToF128( state, signRem, expB - 1, rem.v64, rem.v0 );\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n propagateNaN:\r\n uiZ = softfloat_propagateNaNF128UI( state, uiA64, uiA0, uiB64, uiB0 );\r\n goto uiZ;\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n invalid:\r\n softfloat_raiseFlags( state, softfloat_flag_invalid );\r\n uiZ.v64 = defaultNaNF128UI64;\r\n uiZ.v0 = defaultNaNF128UI0;\r\n uiZ:\r\n uZ.ui = uiZ;\r\n return uZ.f;\r\n\r\n}\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/f128_sqrt.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014, 2015, 2016, 2017 The Regents of the\r\nUniversity of California. All rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \r\n#include \"platform.h\"\r\n#include \"internals.h\"\r\n#include \"specialize.h\"\r\n#include \"softfloat.h\"\r\n\r\nfloat128_t f128_sqrt( struct softfloat_state *state, float128_t a )\r\n{\r\n union ui128_f128 uA;\r\n uint_fast64_t uiA64, uiA0;\r\n bool signA;\r\n int_fast32_t expA;\r\n struct uint128 sigA, uiZ;\r\n struct exp32_sig128 normExpSig;\r\n int_fast32_t expZ;\r\n uint_fast32_t sig32A, recipSqrt32, sig32Z;\r\n struct uint128 rem;\r\n uint32_t qs[3];\r\n uint_fast32_t q;\r\n uint_fast64_t x64, sig64Z;\r\n struct uint128 y, term;\r\n uint_fast64_t sigZExtra;\r\n struct uint128 sigZ;\r\n union ui128_f128 uZ;\r\n\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n uA.f = a;\r\n uiA64 = uA.ui.v64;\r\n uiA0 = uA.ui.v0;\r\n signA = signF128UI64( uiA64 );\r\n expA = expF128UI64( uiA64 );\r\n sigA.v64 = fracF128UI64( uiA64 );\r\n sigA.v0 = uiA0;\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n if ( expA == 0x7FFF ) {\r\n if ( sigA.v64 | sigA.v0 ) {\r\n uiZ = softfloat_propagateNaNF128UI( state, uiA64, uiA0, 0, 0 );\r\n goto uiZ;\r\n }\r\n if ( ! signA ) return a;\r\n goto invalid;\r\n }\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n if ( signA ) {\r\n if ( ! (expA | sigA.v64 | sigA.v0) ) return a;\r\n goto invalid;\r\n }\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n if ( ! expA ) {\r\n if ( ! (sigA.v64 | sigA.v0) ) return a;\r\n normExpSig = softfloat_normSubnormalF128Sig( sigA.v64, sigA.v0 );\r\n expA = normExpSig.exp;\r\n sigA = normExpSig.sig;\r\n }\r\n /*------------------------------------------------------------------------\r\n | (`sig32Z' is guaranteed to be a lower bound on the square root of\r\n | `sig32A', which makes `sig32Z' also a lower bound on the square root of\r\n | `sigA'.)\r\n *------------------------------------------------------------------------*/\r\n expZ = ((expA - 0x3FFF)>>1) + 0x3FFE;\r\n expA &= 1;\r\n sigA.v64 |= UINT64_C( 0x0001000000000000 );\r\n sig32A = sigA.v64>>17;\r\n recipSqrt32 = softfloat_approxRecipSqrt32_1( expA, sig32A );\r\n sig32Z = ((uint_fast64_t) sig32A * recipSqrt32)>>32;\r\n if ( expA ) {\r\n sig32Z >>= 1;\r\n rem = softfloat_shortShiftLeft128( sigA.v64, sigA.v0, 12 );\r\n } else {\r\n rem = softfloat_shortShiftLeft128( sigA.v64, sigA.v0, 13 );\r\n }\r\n qs[2] = sig32Z;\r\n rem.v64 -= (uint_fast64_t) sig32Z * sig32Z;\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n q = ((uint32_t) (rem.v64>>2) * (uint_fast64_t) recipSqrt32)>>32;\r\n x64 = (uint_fast64_t) sig32Z<<32;\r\n sig64Z = x64 + ((uint_fast64_t) q<<3);\r\n y = softfloat_shortShiftLeft128( rem.v64, rem.v0, 29 );\r\n /*------------------------------------------------------------------------\r\n | (Repeating this loop is a rare occurrence.)\r\n *------------------------------------------------------------------------*/\r\n for (;;) {\r\n term = softfloat_mul64ByShifted32To128( x64 + sig64Z, q );\r\n rem = softfloat_sub128( y.v64, y.v0, term.v64, term.v0 );\r\n if ( ! (rem.v64 & UINT64_C( 0x8000000000000000 )) ) break;\r\n --q;\r\n sig64Z -= 1<<3;\r\n }\r\n qs[1] = q;\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n q = ((rem.v64>>2) * recipSqrt32)>>32;\r\n y = softfloat_shortShiftLeft128( rem.v64, rem.v0, 29 );\r\n sig64Z <<= 1;\r\n /*------------------------------------------------------------------------\r\n | (Repeating this loop is a rare occurrence.)\r\n *------------------------------------------------------------------------*/\r\n for (;;) {\r\n term = softfloat_shortShiftLeft128( 0, sig64Z, 32 );\r\n term = softfloat_add128( term.v64, term.v0, 0, (uint_fast64_t) q<<6 );\r\n term = softfloat_mul128By32( term.v64, term.v0, q );\r\n rem = softfloat_sub128( y.v64, y.v0, term.v64, term.v0 );\r\n if ( ! (rem.v64 & UINT64_C( 0x8000000000000000 )) ) break;\r\n --q;\r\n }\r\n qs[0] = q;\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n q = (((rem.v64>>2) * recipSqrt32)>>32) + 2;\r\n sigZExtra = (uint64_t) ((uint_fast64_t) q<<59);\r\n term = softfloat_shortShiftLeft128( 0, qs[1], 53 );\r\n sigZ =\r\n softfloat_add128(\r\n (uint_fast64_t) qs[2]<<18, ((uint_fast64_t) qs[0]<<24) + (q>>5),\r\n term.v64, term.v0\r\n );\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n if ( (q & 0xF) <= 2 ) {\r\n q &= ~3;\r\n sigZExtra = (uint64_t) ((uint_fast64_t) q<<59);\r\n y = softfloat_shortShiftLeft128( sigZ.v64, sigZ.v0, 6 );\r\n y.v0 |= sigZExtra>>58;\r\n term = softfloat_sub128( y.v64, y.v0, 0, q );\r\n y = softfloat_mul64ByShifted32To128( term.v0, q );\r\n term = softfloat_mul64ByShifted32To128( term.v64, q );\r\n term = softfloat_add128( term.v64, term.v0, 0, y.v64 );\r\n rem = softfloat_shortShiftLeft128( rem.v64, rem.v0, 20 );\r\n term = softfloat_sub128( term.v64, term.v0, rem.v64, rem.v0 );\r\n /*--------------------------------------------------------------------\r\n | The concatenation of `term' and `y.v0' is now the negative remainder\r\n | (3 words altogether).\r\n *--------------------------------------------------------------------*/\r\n if ( term.v64 & UINT64_C( 0x8000000000000000 ) ) {\r\n sigZExtra |= 1;\r\n } else {\r\n if ( term.v64 | term.v0 | y.v0 ) {\r\n if ( sigZExtra ) {\r\n --sigZExtra;\r\n } else {\r\n sigZ = softfloat_sub128( sigZ.v64, sigZ.v0, 0, 1 );\r\n sigZExtra = ~0;\r\n }\r\n }\r\n }\r\n }\r\n return softfloat_roundPackToF128( state, 0, expZ, sigZ.v64, sigZ.v0, sigZExtra );\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n invalid:\r\n softfloat_raiseFlags( state, softfloat_flag_invalid );\r\n uiZ.v64 = defaultNaNF128UI64;\r\n uiZ.v0 = defaultNaNF128UI0;\r\n uiZ:\r\n uZ.ui = uiZ;\r\n return uZ.f;\r\n\r\n}\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/f128_sub.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014 The Regents of the University of California.\r\nAll rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \r\n#include \"platform.h\"\r\n#include \"internals.h\"\r\n#include \"softfloat.h\"\r\n\r\nfloat128_t f128_sub( struct softfloat_state *state, float128_t a, float128_t b )\r\n{\r\n union ui128_f128 uA;\r\n uint_fast64_t uiA64, uiA0;\r\n bool signA;\r\n union ui128_f128 uB;\r\n uint_fast64_t uiB64, uiB0;\r\n bool signB;\r\n#if ! defined INLINE_LEVEL || (INLINE_LEVEL < 2)\r\n float128_t\r\n (*magsFuncPtr)(\r\n uint_fast64_t, uint_fast64_t, uint_fast64_t, uint_fast64_t, bool );\r\n#endif\r\n\r\n uA.f = a;\r\n uiA64 = uA.ui.v64;\r\n uiA0 = uA.ui.v0;\r\n signA = signF128UI64( uiA64 );\r\n uB.f = b;\r\n uiB64 = uB.ui.v64;\r\n uiB0 = uB.ui.v0;\r\n signB = signF128UI64( uiB64 );\r\n#if defined INLINE_LEVEL && (2 <= INLINE_LEVEL)\r\n if ( signA == signB ) {\r\n return softfloat_subMagsF128( state, uiA64, uiA0, uiB64, uiB0, signA );\r\n } else {\r\n return softfloat_addMagsF128( state, uiA64, uiA0, uiB64, uiB0, signA );\r\n }\r\n#else\r\n magsFuncPtr =\r\n (signA == signB) ? softfloat_subMagsF128 : softfloat_addMagsF128;\r\n return (*magsFuncPtr)( uiA64, uiA0, uiB64, uiB0, signA );\r\n#endif\r\n\r\n}\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/f128_to_extF80.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014, 2015 The Regents of the University of\r\nCalifornia. All rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \r\n#include \"platform.h\"\r\n#include \"internals.h\"\r\n#include \"specialize.h\"\r\n#include \"softfloat.h\"\r\n\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nextFloat80_t f128_to_extF80( struct softfloat_state *state, float128_t a )\r\n{\r\n union ui128_f128 uA;\r\n uint_fast64_t uiA64, uiA0;\r\n bool sign;\r\n int_fast32_t exp;\r\n uint_fast64_t frac64, frac0;\r\n struct commonNaN commonNaN;\r\n struct uint128 uiZ;\r\n uint_fast16_t uiZ64;\r\n uint_fast64_t uiZ0;\r\n struct exp32_sig128 normExpSig;\r\n struct uint128 sig128;\r\n union { struct extFloat80M s; extFloat80_t f; } uZ;\r\n\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n uA.f = a;\r\n uiA64 = uA.ui.v64;\r\n uiA0 = uA.ui.v0;\r\n sign = signF128UI64( uiA64 );\r\n exp = expF128UI64( uiA64 );\r\n frac64 = fracF128UI64( uiA64 );\r\n frac0 = uiA0;\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n if ( exp == 0x7FFF ) {\r\n if ( frac64 | frac0 ) {\r\n softfloat_f128UIToCommonNaN( state, uiA64, uiA0, &commonNaN );\r\n uiZ = softfloat_commonNaNToExtF80UI( &commonNaN );\r\n uiZ64 = uiZ.v64;\r\n uiZ0 = uiZ.v0;\r\n } else {\r\n uiZ64 = packToExtF80UI64( sign, 0x7FFF );\r\n uiZ0 = UINT64_C( 0x8000000000000000 );\r\n }\r\n goto uiZ;\r\n }\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n if ( ! exp ) {\r\n if ( ! (frac64 | frac0) ) {\r\n uiZ64 = packToExtF80UI64( sign, 0 );\r\n uiZ0 = 0;\r\n goto uiZ;\r\n }\r\n normExpSig = softfloat_normSubnormalF128Sig( frac64, frac0 );\r\n exp = normExpSig.exp;\r\n frac64 = normExpSig.sig.v64;\r\n frac0 = normExpSig.sig.v0;\r\n }\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n sig128 =\r\n softfloat_shortShiftLeft128(\r\n frac64 | UINT64_C( 0x0001000000000000 ), frac0, 15 );\r\n return softfloat_roundPackToExtF80( state, sign, exp, sig128.v64, sig128.v0, 80 );\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n uiZ:\r\n uZ.s.signExp = uiZ64;\r\n uZ.s.signif = uiZ0;\r\n return uZ.f;\r\n\r\n}\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/f128_to_f16.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014, 2015 The Regents of the University of\r\nCalifornia. All rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \r\n#include \"platform.h\"\r\n#include \"internals.h\"\r\n#include \"specialize.h\"\r\n#include \"softfloat.h\"\r\n\r\nfloat16_t f128_to_f16( struct softfloat_state *state, float128_t a )\r\n{\r\n union ui128_f128 uA;\r\n uint_fast64_t uiA64, uiA0;\r\n bool sign;\r\n int_fast32_t exp;\r\n uint_fast64_t frac64;\r\n struct commonNaN commonNaN;\r\n uint_fast16_t uiZ, frac16;\r\n union ui16_f16 uZ;\r\n\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n uA.f = a;\r\n uiA64 = uA.ui.v64;\r\n uiA0 = uA.ui.v0;\r\n sign = signF128UI64( uiA64 );\r\n exp = expF128UI64( uiA64 );\r\n frac64 = fracF128UI64( uiA64 ) | (uiA0 != 0);\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n if ( exp == 0x7FFF ) {\r\n if ( frac64 ) {\r\n softfloat_f128UIToCommonNaN( state, uiA64, uiA0, &commonNaN );\r\n uiZ = softfloat_commonNaNToF16UI( &commonNaN );\r\n } else {\r\n uiZ = packToF16UI( sign, 0x1F, 0 );\r\n }\r\n goto uiZ;\r\n }\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n frac16 = softfloat_shortShiftRightJam64( frac64, 34 );\r\n if ( ! (exp | frac16) ) {\r\n uiZ = packToF16UI( sign, 0, 0 );\r\n goto uiZ;\r\n }\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n exp -= 0x3FF1;\r\n if ( sizeof (int_fast16_t) < sizeof (int_fast32_t) ) {\r\n if ( exp < -0x40 ) exp = -0x40;\r\n }\r\n return softfloat_roundPackToF16( sign, exp, frac16 | 0x4000 );\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n uiZ:\r\n uZ.ui = uiZ;\r\n return uZ.f;\r\n\r\n}\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/f128_to_f32.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014, 2015 The Regents of the University of\r\nCalifornia. All rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \r\n#include \"platform.h\"\r\n#include \"internals.h\"\r\n#include \"specialize.h\"\r\n#include \"softfloat.h\"\r\n\r\nfloat32_t f128_to_f32( struct softfloat_state *state, float128_t a )\r\n{\r\n union ui128_f128 uA;\r\n uint_fast64_t uiA64, uiA0;\r\n bool sign;\r\n int_fast32_t exp;\r\n uint_fast64_t frac64;\r\n struct commonNaN commonNaN;\r\n uint_fast32_t uiZ, frac32;\r\n union ui32_f32 uZ;\r\n\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n uA.f = a;\r\n uiA64 = uA.ui.v64;\r\n uiA0 = uA.ui.v0;\r\n sign = signF128UI64( uiA64 );\r\n exp = expF128UI64( uiA64 );\r\n frac64 = fracF128UI64( uiA64 ) | (uiA0 != 0);\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n if ( exp == 0x7FFF ) {\r\n if ( frac64 ) {\r\n softfloat_f128UIToCommonNaN( state, uiA64, uiA0, &commonNaN );\r\n uiZ = softfloat_commonNaNToF32UI( &commonNaN );\r\n } else {\r\n uiZ = packToF32UI( sign, 0xFF, 0 );\r\n }\r\n goto uiZ;\r\n }\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n frac32 = softfloat_shortShiftRightJam64( frac64, 18 );\r\n if ( ! (exp | frac32) ) {\r\n uiZ = packToF32UI( sign, 0, 0 );\r\n goto uiZ;\r\n }\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n exp -= 0x3F81;\r\n if ( sizeof (int_fast16_t) < sizeof (int_fast32_t) ) {\r\n if ( exp < -0x1000 ) exp = -0x1000;\r\n }\r\n return softfloat_roundPackToF32( state, sign, exp, frac32 | 0x40000000 );\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n uiZ:\r\n uZ.ui = uiZ;\r\n return uZ.f;\r\n\r\n}\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/f128_to_f64.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014, 2015 The Regents of the University of\r\nCalifornia. All rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \r\n#include \"platform.h\"\r\n#include \"internals.h\"\r\n#include \"specialize.h\"\r\n#include \"softfloat.h\"\r\n\r\nfloat64_t f128_to_f64( struct softfloat_state *state, float128_t a )\r\n{\r\n union ui128_f128 uA;\r\n uint_fast64_t uiA64, uiA0;\r\n bool sign;\r\n int_fast32_t exp;\r\n uint_fast64_t frac64, frac0;\r\n struct commonNaN commonNaN;\r\n uint_fast64_t uiZ;\r\n struct uint128 frac128;\r\n union ui64_f64 uZ;\r\n\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n uA.f = a;\r\n uiA64 = uA.ui.v64;\r\n uiA0 = uA.ui.v0;\r\n sign = signF128UI64( uiA64 );\r\n exp = expF128UI64( uiA64 );\r\n frac64 = fracF128UI64( uiA64 );\r\n frac0 = uiA0;\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n if ( exp == 0x7FFF ) {\r\n if ( frac64 | frac0 ) {\r\n softfloat_f128UIToCommonNaN( state, uiA64, uiA0, &commonNaN );\r\n uiZ = softfloat_commonNaNToF64UI( &commonNaN );\r\n } else {\r\n uiZ = packToF64UI( sign, 0x7FF, 0 );\r\n }\r\n goto uiZ;\r\n }\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n frac128 = softfloat_shortShiftLeft128( frac64, frac0, 14 );\r\n frac64 = frac128.v64 | (frac128.v0 != 0);\r\n if ( ! (exp | frac64) ) {\r\n uiZ = packToF64UI( sign, 0, 0 );\r\n goto uiZ;\r\n }\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n exp -= 0x3C01;\r\n if ( sizeof (int_fast16_t) < sizeof (int_fast32_t) ) {\r\n if ( exp < -0x1000 ) exp = -0x1000;\r\n }\r\n return\r\n softfloat_roundPackToF64(\r\n state, sign, exp, frac64 | UINT64_C( 0x4000000000000000 ) );\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n uiZ:\r\n uZ.ui = uiZ;\r\n return uZ.f;\r\n\r\n}\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/f128_to_i32.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014, 2015, 2016, 2017 The Regents of the\r\nUniversity of California. All rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \r\n#include \"platform.h\"\r\n#include \"internals.h\"\r\n#include \"specialize.h\"\r\n#include \"softfloat.h\"\r\n\r\nint_fast32_t f128_to_i32( struct softfloat_state *state, float128_t a, uint_fast8_t roundingMode, bool exact )\r\n{\r\n union ui128_f128 uA;\r\n uint_fast64_t uiA64, uiA0;\r\n bool sign;\r\n int_fast32_t exp;\r\n uint_fast64_t sig64, sig0;\r\n int_fast32_t shiftDist;\r\n\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n uA.f = a;\r\n uiA64 = uA.ui.v64;\r\n uiA0 = uA.ui.v0;\r\n sign = signF128UI64( uiA64 );\r\n exp = expF128UI64( uiA64 );\r\n sig64 = fracF128UI64( uiA64 );\r\n sig0 = uiA0;\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n#if (i32_fromNaN != i32_fromPosOverflow) || (i32_fromNaN != i32_fromNegOverflow)\r\n if ( (exp == 0x7FFF) && (sig64 | sig0) ) {\r\n#if (i32_fromNaN == i32_fromPosOverflow)\r\n sign = 0;\r\n#elif (i32_fromNaN == i32_fromNegOverflow)\r\n sign = 1;\r\n#else\r\n softfloat_raiseFlags( softfloat_flag_invalid );\r\n return i32_fromNaN;\r\n#endif\r\n }\r\n#endif\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n if ( exp ) sig64 |= UINT64_C( 0x0001000000000000 );\r\n sig64 |= (sig0 != 0);\r\n shiftDist = 0x4023 - exp;\r\n if ( 0 < shiftDist ) sig64 = softfloat_shiftRightJam64( sig64, shiftDist );\r\n return softfloat_roundToI32( state, sign, sig64, roundingMode, exact );\r\n\r\n}\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/f128_to_i64.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014, 2015, 2016, 2017 The Regents of the\r\nUniversity of California. All rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \r\n#include \"platform.h\"\r\n#include \"internals.h\"\r\n#include \"specialize.h\"\r\n#include \"softfloat.h\"\r\n\r\nint_fast64_t f128_to_i64( struct softfloat_state *state, float128_t a, uint_fast8_t roundingMode, bool exact )\r\n{\r\n union ui128_f128 uA;\r\n uint_fast64_t uiA64, uiA0;\r\n bool sign;\r\n int_fast32_t exp;\r\n uint_fast64_t sig64, sig0;\r\n int_fast32_t shiftDist;\r\n struct uint128 sig128;\r\n struct uint64_extra sigExtra;\r\n\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n uA.f = a;\r\n uiA64 = uA.ui.v64;\r\n uiA0 = uA.ui.v0;\r\n sign = signF128UI64( uiA64 );\r\n exp = expF128UI64( uiA64 );\r\n sig64 = fracF128UI64( uiA64 );\r\n sig0 = uiA0;\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n shiftDist = 0x402F - exp;\r\n if ( shiftDist <= 0 ) {\r\n /*--------------------------------------------------------------------\r\n *--------------------------------------------------------------------*/\r\n if ( shiftDist < -15 ) {\r\n softfloat_raiseFlags( state, softfloat_flag_invalid );\r\n return\r\n (exp == 0x7FFF) && (sig64 | sig0) ? i64_fromNaN\r\n : sign ? i64_fromNegOverflow : i64_fromPosOverflow;\r\n }\r\n /*--------------------------------------------------------------------\r\n *--------------------------------------------------------------------*/\r\n sig64 |= UINT64_C( 0x0001000000000000 );\r\n if ( shiftDist ) {\r\n sig128 = softfloat_shortShiftLeft128( sig64, sig0, -shiftDist );\r\n sig64 = sig128.v64;\r\n sig0 = sig128.v0;\r\n }\r\n } else {\r\n /*--------------------------------------------------------------------\r\n *--------------------------------------------------------------------*/\r\n if ( exp ) sig64 |= UINT64_C( 0x0001000000000000 );\r\n sigExtra = softfloat_shiftRightJam64Extra( sig64, sig0, shiftDist );\r\n sig64 = sigExtra.v;\r\n sig0 = sigExtra.extra;\r\n }\r\n return softfloat_roundToI64( state, sign, sig64, sig0, roundingMode, exact );\r\n\r\n}\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/f128_to_ui32.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014, 2015, 2016, 2017 The Regents of the\r\nUniversity of California. All rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \r\n#include \"platform.h\"\r\n#include \"internals.h\"\r\n#include \"specialize.h\"\r\n#include \"softfloat.h\"\r\n\r\nuint_fast32_t\r\n f128_to_ui32( struct softfloat_state *state, float128_t a, uint_fast8_t roundingMode, bool exact )\r\n{\r\n union ui128_f128 uA;\r\n uint_fast64_t uiA64, uiA0;\r\n bool sign;\r\n int_fast32_t exp;\r\n uint_fast64_t sig64;\r\n int_fast32_t shiftDist;\r\n\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n uA.f = a;\r\n uiA64 = uA.ui.v64;\r\n uiA0 = uA.ui.v0;\r\n sign = signF128UI64( uiA64 );\r\n exp = expF128UI64( uiA64 );\r\n sig64 = fracF128UI64( uiA64 ) | (uiA0 != 0);\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n#if (ui32_fromNaN != ui32_fromPosOverflow) || (ui32_fromNaN != ui32_fromNegOverflow)\r\n if ( (exp == 0x7FFF) && sig64 ) {\r\n#if (ui32_fromNaN == ui32_fromPosOverflow)\r\n sign = 0;\r\n#elif (ui32_fromNaN == ui32_fromNegOverflow)\r\n sign = 1;\r\n#else\r\n softfloat_raiseFlags( softfloat_flag_invalid );\r\n return ui32_fromNaN;\r\n#endif\r\n }\r\n#endif\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n if ( exp ) sig64 |= UINT64_C( 0x0001000000000000 );\r\n shiftDist = 0x4023 - exp;\r\n if ( 0 < shiftDist ) {\r\n sig64 = softfloat_shiftRightJam64( sig64, shiftDist );\r\n }\r\n return softfloat_roundToUI32( sign, sig64, roundingMode, exact );\r\n\r\n}\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/f128_to_ui64.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014, 2015, 2016, 2017 The Regents of the\r\nUniversity of California. All rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \r\n#include \"platform.h\"\r\n#include \"internals.h\"\r\n#include \"specialize.h\"\r\n#include \"softfloat.h\"\r\n\r\nuint_fast64_t\r\n f128_to_ui64( struct softfloat_state *state, float128_t a, uint_fast8_t roundingMode, bool exact )\r\n{\r\n union ui128_f128 uA;\r\n uint_fast64_t uiA64, uiA0;\r\n bool sign;\r\n int_fast32_t exp;\r\n uint_fast64_t sig64, sig0;\r\n int_fast32_t shiftDist;\r\n struct uint128 sig128;\r\n struct uint64_extra sigExtra;\r\n\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n uA.f = a;\r\n uiA64 = uA.ui.v64;\r\n uiA0 = uA.ui.v0;\r\n sign = signF128UI64( uiA64 );\r\n exp = expF128UI64( uiA64 );\r\n sig64 = fracF128UI64( uiA64 );\r\n sig0 = uiA0;\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n shiftDist = 0x402F - exp;\r\n if ( shiftDist <= 0 ) {\r\n /*--------------------------------------------------------------------\r\n *--------------------------------------------------------------------*/\r\n if ( shiftDist < -15 ) {\r\n softfloat_raiseFlags( state, softfloat_flag_invalid );\r\n return\r\n (exp == 0x7FFF) && (sig64 | sig0) ? ui64_fromNaN\r\n : sign ? ui64_fromNegOverflow : ui64_fromPosOverflow;\r\n }\r\n /*--------------------------------------------------------------------\r\n *--------------------------------------------------------------------*/\r\n sig64 |= UINT64_C( 0x0001000000000000 );\r\n if ( shiftDist ) {\r\n sig128 = softfloat_shortShiftLeft128( sig64, sig0, -shiftDist );\r\n sig64 = sig128.v64;\r\n sig0 = sig128.v0;\r\n }\r\n } else {\r\n /*--------------------------------------------------------------------\r\n *--------------------------------------------------------------------*/\r\n if ( exp ) sig64 |= UINT64_C( 0x0001000000000000 );\r\n sigExtra = softfloat_shiftRightJam64Extra( sig64, sig0, shiftDist );\r\n sig64 = sigExtra.v;\r\n sig0 = sigExtra.extra;\r\n }\r\n return softfloat_roundToUI64( state, sign, sig64, sig0, roundingMode, exact );\r\n\r\n}\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/f32_to_extF80.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014, 2015 The Regents of the University of\r\nCalifornia. All rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \r\n#include \"platform.h\"\r\n#include \"internals.h\"\r\n#include \"specialize.h\"\r\n#include \"softfloat.h\"\r\n\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nextFloat80_t f32_to_extF80( struct softfloat_state *state, float32_t a )\r\n{\r\n union ui32_f32 uA;\r\n uint_fast32_t uiA;\r\n bool sign;\r\n int_fast16_t exp;\r\n uint_fast32_t frac;\r\n struct commonNaN commonNaN;\r\n struct uint128 uiZ;\r\n uint_fast16_t uiZ64;\r\n uint_fast64_t uiZ0;\r\n struct exp16_sig32 normExpSig;\r\n union { struct extFloat80M s; extFloat80_t f; } uZ;\r\n\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n uA.f = a;\r\n uiA = uA.ui;\r\n sign = signF32UI( uiA );\r\n exp = expF32UI( uiA );\r\n frac = fracF32UI( uiA );\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n if ( exp == 0xFF ) {\r\n if ( frac ) {\r\n softfloat_f32UIToCommonNaN( state, uiA, &commonNaN );\r\n uiZ = softfloat_commonNaNToExtF80UI( &commonNaN );\r\n uiZ64 = uiZ.v64;\r\n uiZ0 = uiZ.v0;\r\n } else {\r\n uiZ64 = packToExtF80UI64( sign, 0x7FFF );\r\n uiZ0 = UINT64_C( 0x8000000000000000 );\r\n }\r\n goto uiZ;\r\n }\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n if ( ! exp ) {\r\n if ( ! frac ) {\r\n uiZ64 = packToExtF80UI64( sign, 0 );\r\n uiZ0 = 0;\r\n goto uiZ;\r\n }\r\n normExpSig = softfloat_normSubnormalF32Sig( frac );\r\n exp = normExpSig.exp;\r\n frac = normExpSig.sig;\r\n }\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n uiZ64 = packToExtF80UI64( sign, exp + 0x3F80 );\r\n uiZ0 = (uint_fast64_t) (frac | 0x00800000)<<40;\r\n uiZ:\r\n uZ.s.signExp = uiZ64;\r\n uZ.s.signif = uiZ0;\r\n return uZ.f;\r\n\r\n}\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/f32_to_f128.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014, 2015 The Regents of the University of\r\nCalifornia. All rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \r\n#include \"platform.h\"\r\n#include \"internals.h\"\r\n#include \"specialize.h\"\r\n#include \"softfloat.h\"\r\n\r\nfloat128_t f32_to_f128( struct softfloat_state *state, float32_t a )\r\n{\r\n union ui32_f32 uA;\r\n uint_fast32_t uiA;\r\n bool sign;\r\n int_fast16_t exp;\r\n uint_fast32_t frac;\r\n struct commonNaN commonNaN;\r\n struct uint128 uiZ;\r\n struct exp16_sig32 normExpSig;\r\n union ui128_f128 uZ;\r\n\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n uA.f = a;\r\n uiA = uA.ui;\r\n sign = signF32UI( uiA );\r\n exp = expF32UI( uiA );\r\n frac = fracF32UI( uiA );\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n if ( exp == 0xFF ) {\r\n if ( frac ) {\r\n softfloat_f32UIToCommonNaN( state, uiA, &commonNaN );\r\n uiZ = softfloat_commonNaNToF128UI( &commonNaN );\r\n } else {\r\n uiZ.v64 = packToF128UI64( sign, 0x7FFF, 0 );\r\n uiZ.v0 = 0;\r\n }\r\n goto uiZ;\r\n }\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n if ( ! exp ) {\r\n if ( ! frac ) {\r\n uiZ.v64 = packToF128UI64( sign, 0, 0 );\r\n uiZ.v0 = 0;\r\n goto uiZ;\r\n }\r\n normExpSig = softfloat_normSubnormalF32Sig( frac );\r\n exp = normExpSig.exp - 1;\r\n frac = normExpSig.sig;\r\n }\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n uiZ.v64 = packToF128UI64( sign, exp + 0x3F80, (uint_fast64_t) frac<<25 );\r\n uiZ.v0 = 0;\r\n uiZ:\r\n uZ.ui = uiZ;\r\n return uZ.f;\r\n\r\n}\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/f64_to_extF80.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014, 2015 The Regents of the University of\r\nCalifornia. All rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \r\n#include \"platform.h\"\r\n#include \"internals.h\"\r\n#include \"specialize.h\"\r\n#include \"softfloat.h\"\r\n\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nextFloat80_t f64_to_extF80( struct softfloat_state *state, float64_t a )\r\n{\r\n union ui64_f64 uA;\r\n uint_fast64_t uiA;\r\n bool sign;\r\n int_fast16_t exp;\r\n uint_fast64_t frac;\r\n struct commonNaN commonNaN;\r\n struct uint128 uiZ;\r\n uint_fast16_t uiZ64;\r\n uint_fast64_t uiZ0;\r\n struct exp16_sig64 normExpSig;\r\n union { struct extFloat80M s; extFloat80_t f; } uZ;\r\n\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n uA.f = a;\r\n uiA = uA.ui;\r\n sign = signF64UI( uiA );\r\n exp = expF64UI( uiA );\r\n frac = fracF64UI( uiA );\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n if ( exp == 0x7FF ) {\r\n if ( frac ) {\r\n softfloat_f64UIToCommonNaN( state, uiA, &commonNaN );\r\n uiZ = softfloat_commonNaNToExtF80UI( &commonNaN );\r\n uiZ64 = uiZ.v64;\r\n uiZ0 = uiZ.v0;\r\n } else {\r\n uiZ64 = packToExtF80UI64( sign, 0x7FFF );\r\n uiZ0 = UINT64_C( 0x8000000000000000 );\r\n }\r\n goto uiZ;\r\n }\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n if ( ! exp ) {\r\n if ( ! frac ) {\r\n uiZ64 = packToExtF80UI64( sign, 0 );\r\n uiZ0 = 0;\r\n goto uiZ;\r\n }\r\n normExpSig = softfloat_normSubnormalF64Sig( frac );\r\n exp = normExpSig.exp;\r\n frac = normExpSig.sig;\r\n }\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n uiZ64 = packToExtF80UI64( sign, exp + 0x3C00 );\r\n uiZ0 = (frac | UINT64_C( 0x0010000000000000 ))<<11;\r\n uiZ:\r\n uZ.s.signExp = uiZ64;\r\n uZ.s.signif = uiZ0;\r\n return uZ.f;\r\n\r\n}\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/i32_to_extF80.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014, 2015, 2016 The Regents of the University of\r\nCalifornia. All rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \r\n#include \"platform.h\"\r\n#include \"internals.h\"\r\n#include \"softfloat.h\"\r\n\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nextFloat80_t i32_to_extF80( int32_t a )\r\n{\r\n uint_fast16_t uiZ64;\r\n uint_fast32_t absA;\r\n bool sign;\r\n int_fast8_t shiftDist;\r\n union { struct extFloat80M s; extFloat80_t f; } uZ;\r\n\r\n uiZ64 = 0;\r\n absA = 0;\r\n if ( a ) {\r\n sign = (a < 0);\r\n absA = sign ? -(uint_fast32_t) a : (uint_fast32_t) a;\r\n shiftDist = softfloat_countLeadingZeros32( absA );\r\n uiZ64 = packToExtF80UI64( sign, 0x401E - shiftDist );\r\n absA <<= shiftDist;\r\n }\r\n uZ.s.signExp = uiZ64;\r\n uZ.s.signif = (uint_fast64_t) absA<<32;\r\n return uZ.f;\r\n\r\n}\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/i32_to_f128.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014, 2015, 2016 The Regents of the University of\r\nCalifornia. All rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \"platform.h\"\r\n#include \"internals.h\"\r\n#include \"softfloat.h\"\r\n\r\nfloat128_t i32_to_f128( int32_t a )\r\n{\r\n uint_fast64_t uiZ64;\r\n bool sign;\r\n uint_fast32_t absA;\r\n int_fast8_t shiftDist;\r\n union ui128_f128 uZ;\r\n\r\n uiZ64 = 0;\r\n if ( a ) {\r\n sign = (a < 0);\r\n absA = sign ? -(uint_fast32_t) a : (uint_fast32_t) a;\r\n shiftDist = softfloat_countLeadingZeros32( absA ) + 17;\r\n uiZ64 =\r\n packToF128UI64(\r\n sign, 0x402E - shiftDist, (uint_fast64_t) absA<\r\n#include \r\n#include \"primitives.h\"\r\n#include \"softfloat_types.h\"\r\n\r\nunion ui16_f16 { uint16_t ui; float16_t f; };\r\nunion ui32_f32 { uint32_t ui; float32_t f; };\r\nunion ui64_f64 { uint64_t ui; float64_t f; };\r\n\r\n#ifdef SOFTFLOAT_FAST_INT64\r\nunion extF80M_extF80 { struct extFloat80M fM; extFloat80_t f; };\r\nunion ui128_f128 { struct uint128 ui; float128_t f; };\r\n#endif\r\n\r\nenum {\r\n softfloat_mulAdd_subC = 1,\r\n softfloat_mulAdd_subProd = 2\r\n};\r\n\r\n/*----------------------------------------------------------------------------\r\n*----------------------------------------------------------------------------*/\r\nuint_fast32_t softfloat_roundToUI32( bool, uint_fast64_t, uint_fast8_t, bool );\r\n\r\n#ifdef SOFTFLOAT_FAST_INT64\r\nuint_fast64_t\r\n softfloat_roundToUI64(\r\n struct softfloat_state *, bool, uint_fast64_t, uint_fast64_t, uint_fast8_t, bool );\r\n#else\r\nuint_fast64_t softfloat_roundMToUI64( bool, uint32_t *, uint_fast8_t, bool );\r\n#endif\r\n\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nint_fast32_t softfloat_roundToI32( struct softfloat_state *, bool, uint_fast64_t, uint_fast8_t, bool );\r\n\r\n#ifdef SOFTFLOAT_FAST_INT64\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nint_fast64_t\r\n softfloat_roundToI64(\r\n struct softfloat_state *, bool, uint_fast64_t, uint_fast64_t, uint_fast8_t, bool );\r\n#else\r\nint_fast64_t softfloat_roundMToI64( bool, uint32_t *, uint_fast8_t, bool );\r\n#endif\r\n\r\n/*----------------------------------------------------------------------------\r\n*----------------------------------------------------------------------------*/\r\n#define signF16UI( a ) ((bool) ((uint16_t) (a)>>15))\r\n#define expF16UI( a ) ((int_fast8_t) ((a)>>10) & 0x1F)\r\n#define fracF16UI( a ) ((a) & 0x03FF)\r\n#define packToF16UI( sign, exp, sig ) (((uint16_t) (sign)<<15) + ((uint16_t) (exp)<<10) + (sig))\r\n\r\n#define isNaNF16UI( a ) (((~(a) & 0x7C00) == 0) && ((a) & 0x03FF))\r\n\r\nstruct exp8_sig16 { int_fast8_t exp; uint_fast16_t sig; };\r\nstruct exp8_sig16 softfloat_normSubnormalF16Sig( uint_fast16_t );\r\n\r\nfloat16_t softfloat_roundPackToF16( bool, int_fast16_t, uint_fast16_t );\r\nfloat16_t softfloat_normRoundPackToF16( bool, int_fast16_t, uint_fast16_t );\r\n\r\nfloat16_t softfloat_addMagsF16( uint_fast16_t, uint_fast16_t );\r\nfloat16_t softfloat_subMagsF16( uint_fast16_t, uint_fast16_t );\r\nfloat16_t\r\n softfloat_mulAddF16(\r\n uint_fast16_t, uint_fast16_t, uint_fast16_t, uint_fast8_t );\r\n\r\n/*----------------------------------------------------------------------------\r\n*----------------------------------------------------------------------------*/\r\n#define signF32UI( a ) ((bool) ((uint32_t) (a)>>31))\r\n#define expF32UI( a ) ((int_fast16_t) ((a)>>23) & 0xFF)\r\n#define fracF32UI( a ) ((a) & 0x007FFFFF)\r\n#define packToF32UI( sign, exp, sig ) (((uint32_t) (sign)<<31) + ((uint32_t) (exp)<<23) + (sig))\r\n\r\n#define isNaNF32UI( a ) (((~(a) & 0x7F800000) == 0) && ((a) & 0x007FFFFF))\r\n\r\nstruct exp16_sig32 { int_fast16_t exp; uint_fast32_t sig; };\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nstruct exp16_sig32 softfloat_normSubnormalF32Sig( uint_fast32_t );\r\n\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nfloat32_t softfloat_roundPackToF32( struct softfloat_state *, bool, int_fast16_t, uint_fast32_t );\r\nfloat32_t softfloat_normRoundPackToF32( bool, int_fast16_t, uint_fast32_t );\r\n\r\nfloat32_t softfloat_addMagsF32( uint_fast32_t, uint_fast32_t );\r\nfloat32_t softfloat_subMagsF32( uint_fast32_t, uint_fast32_t );\r\nfloat32_t\r\n softfloat_mulAddF32(\r\n uint_fast32_t, uint_fast32_t, uint_fast32_t, uint_fast8_t );\r\n\r\n/*----------------------------------------------------------------------------\r\n*----------------------------------------------------------------------------*/\r\n#define signF64UI( a ) ((bool) ((uint64_t) (a)>>63))\r\n#define expF64UI( a ) ((int_fast16_t) ((a)>>52) & 0x7FF)\r\n#define fracF64UI( a ) ((a) & UINT64_C( 0x000FFFFFFFFFFFFF ))\r\n#define packToF64UI( sign, exp, sig ) ((uint64_t) (((uint_fast64_t) (sign)<<63) + ((uint_fast64_t) (exp)<<52) + (sig)))\r\n\r\n#define isNaNF64UI( a ) (((~(a) & UINT64_C( 0x7FF0000000000000 )) == 0) && ((a) & UINT64_C( 0x000FFFFFFFFFFFFF )))\r\n\r\nstruct exp16_sig64 { int_fast16_t exp; uint_fast64_t sig; };\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nstruct exp16_sig64 softfloat_normSubnormalF64Sig( uint_fast64_t );\r\n\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nfloat64_t softfloat_roundPackToF64( struct softfloat_state *, bool, int_fast16_t, uint_fast64_t );\r\nfloat64_t softfloat_normRoundPackToF64( bool, int_fast16_t, uint_fast64_t );\r\n\r\nfloat64_t softfloat_addMagsF64( uint_fast64_t, uint_fast64_t, bool );\r\nfloat64_t softfloat_subMagsF64( uint_fast64_t, uint_fast64_t, bool );\r\nfloat64_t\r\n softfloat_mulAddF64(\r\n uint_fast64_t, uint_fast64_t, uint_fast64_t, uint_fast8_t );\r\n\r\n/*----------------------------------------------------------------------------\r\n*----------------------------------------------------------------------------*/\r\n#define signExtF80UI64( a64 ) ((bool) ((uint16_t) (a64)>>15))\r\n#define expExtF80UI64( a64 ) ((a64) & 0x7FFF)\r\n#define packToExtF80UI64( sign, exp ) ((uint_fast16_t) (sign)<<15 | (exp))\r\n\r\n#define isNaNExtF80UI( a64, a0 ) ((((a64) & 0x7FFF) == 0x7FFF) && ((a0) & UINT64_C( 0x7FFFFFFFFFFFFFFF )))\r\n\r\n#ifdef SOFTFLOAT_FAST_INT64\r\n\r\n/*----------------------------------------------------------------------------\r\n*----------------------------------------------------------------------------*/\r\n\r\nstruct exp32_sig64 { int_fast32_t exp; uint64_t sig; };\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nstruct exp32_sig64 softfloat_normSubnormalExtF80Sig( uint_fast64_t );\r\n\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nextFloat80_t\r\n softfloat_roundPackToExtF80(\r\n struct softfloat_state *, bool, int_fast32_t, uint_fast64_t, uint_fast64_t, uint_fast8_t );\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nextFloat80_t\r\n softfloat_normRoundPackToExtF80(\r\n struct softfloat_state *, bool, int_fast32_t, uint_fast64_t, uint_fast64_t, uint_fast8_t );\r\n\r\nextFloat80_t\r\n softfloat_addMagsExtF80(\r\n struct softfloat_state *, uint_fast16_t, uint_fast64_t, uint_fast16_t, uint_fast64_t, bool );\r\nextFloat80_t\r\n softfloat_subMagsExtF80(\r\n struct softfloat_state *, uint_fast16_t, uint_fast64_t, uint_fast16_t, uint_fast64_t, bool );\r\n\r\n/*----------------------------------------------------------------------------\r\n*----------------------------------------------------------------------------*/\r\n#define signF128UI64( a64 ) ((bool) ((uint64_t) (a64)>>63))\r\n#define expF128UI64( a64 ) ((int_fast32_t) ((a64)>>48) & 0x7FFF)\r\n#define fracF128UI64( a64 ) ((a64) & UINT64_C( 0x0000FFFFFFFFFFFF ))\r\n#define packToF128UI64( sign, exp, sig64 ) (((uint_fast64_t) (sign)<<63) + ((uint_fast64_t) (exp)<<48) + (sig64))\r\n\r\n#define isNaNF128UI( a64, a0 ) (((~(a64) & UINT64_C( 0x7FFF000000000000 )) == 0) && (a0 || ((a64) & UINT64_C( 0x0000FFFFFFFFFFFF ))))\r\n\r\nstruct exp32_sig128 { int_fast32_t exp; struct uint128 sig; };\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nstruct exp32_sig128\r\n softfloat_normSubnormalF128Sig( uint_fast64_t, uint_fast64_t );\r\n\r\nfloat128_t\r\n softfloat_roundPackToF128(\r\n struct softfloat_state *,\r\n bool, int_fast32_t, uint_fast64_t, uint_fast64_t, uint_fast64_t );\r\nfloat128_t\r\n softfloat_normRoundPackToF128(\r\n struct softfloat_state *,\r\n bool, int_fast32_t, uint_fast64_t, uint_fast64_t );\r\n\r\nfloat128_t\r\n softfloat_addMagsF128(\r\n struct softfloat_state *,\r\n uint_fast64_t, uint_fast64_t, uint_fast64_t, uint_fast64_t, bool );\r\nfloat128_t\r\n softfloat_subMagsF128(\r\n struct softfloat_state *,\r\n uint_fast64_t, uint_fast64_t, uint_fast64_t, uint_fast64_t, bool );\r\nfloat128_t\r\n softfloat_mulAddF128(\r\n uint_fast64_t,\r\n uint_fast64_t,\r\n uint_fast64_t,\r\n uint_fast64_t,\r\n uint_fast64_t,\r\n uint_fast64_t,\r\n uint_fast8_t\r\n );\r\n\r\n#else\r\n\r\n/*----------------------------------------------------------------------------\r\n*----------------------------------------------------------------------------*/\r\n\r\nbool\r\n softfloat_tryPropagateNaNExtF80M(\r\n const struct extFloat80M *,\r\n const struct extFloat80M *,\r\n struct extFloat80M *\r\n );\r\nvoid softfloat_invalidExtF80M( struct extFloat80M * );\r\n\r\nint softfloat_normExtF80SigM( uint64_t * );\r\n\r\nvoid\r\n softfloat_roundPackMToExtF80M(\r\n bool, int32_t, uint32_t *, uint_fast8_t, struct extFloat80M * );\r\nvoid\r\n softfloat_normRoundPackMToExtF80M(\r\n bool, int32_t, uint32_t *, uint_fast8_t, struct extFloat80M * );\r\n\r\nvoid\r\n softfloat_addExtF80M(\r\n const struct extFloat80M *,\r\n const struct extFloat80M *,\r\n struct extFloat80M *,\r\n bool\r\n );\r\n\r\nint\r\n softfloat_compareNonnormExtF80M(\r\n const struct extFloat80M *, const struct extFloat80M * );\r\n\r\n/*----------------------------------------------------------------------------\r\n*----------------------------------------------------------------------------*/\r\n#define signF128UI96( a96 ) ((bool) ((uint32_t) (a96)>>31))\r\n#define expF128UI96( a96 ) ((int32_t) ((a96)>>16) & 0x7FFF)\r\n#define fracF128UI96( a96 ) ((a96) & 0x0000FFFF)\r\n#define packToF128UI96( sign, exp, sig96 ) (((uint32_t) (sign)<<31) + ((uint32_t) (exp)<<16) + (sig96))\r\n\r\nbool softfloat_isNaNF128M( const uint32_t * );\r\n\r\nbool\r\n softfloat_tryPropagateNaNF128M(\r\n const uint32_t *, const uint32_t *, uint32_t * );\r\nvoid softfloat_invalidF128M( uint32_t * );\r\n\r\nint softfloat_shiftNormSigF128M( const uint32_t *, uint_fast8_t, uint32_t * );\r\n\r\nvoid softfloat_roundPackMToF128M( bool, int32_t, uint32_t *, uint32_t * );\r\nvoid softfloat_normRoundPackMToF128M( bool, int32_t, uint32_t *, uint32_t * );\r\n\r\nvoid\r\n softfloat_addF128M( const uint32_t *, const uint32_t *, uint32_t *, bool );\r\nvoid\r\n softfloat_mulAddF128M(\r\n const uint32_t *,\r\n const uint32_t *,\r\n const uint32_t *,\r\n uint32_t *,\r\n uint_fast8_t\r\n );\r\n\r\n#endif\r\n\r\n#endif\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/primitives.h", "content": "\r\n/*============================================================================\r\n\r\nThis C header file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014, 2015, 2016, 2017 The Regents of the\r\nUniversity of California. All rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#ifndef primitives_h\r\n#define primitives_h 1\r\n\r\n#include \r\n#include \r\n#include \"primitiveTypes.h\"\r\n\r\n#ifndef softfloat_shortShiftRightJam64\r\n/*----------------------------------------------------------------------------\r\n| Shifts 'a' right by the number of bits given in 'dist', which must be in\r\n| the range 1 to 63. If any nonzero bits are shifted off, they are \"jammed\"\r\n| into the least-significant bit of the shifted value by setting the least-\r\n| significant bit to 1. This shifted-and-jammed value is returned.\r\n*----------------------------------------------------------------------------*/\r\n#if defined INLINE_LEVEL && (2 <= INLINE_LEVEL)\r\nINLINE\r\nuint64_t softfloat_shortShiftRightJam64( uint64_t a, uint_fast8_t dist )\r\n { return a>>dist | ((a & (((uint_fast64_t) 1<>dist | ((uint32_t) (a<<(-dist & 31)) != 0) : (a != 0);\r\n}\r\n#else\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nuint32_t softfloat_shiftRightJam32( uint32_t a, uint_fast16_t dist );\r\n#endif\r\n#endif\r\n\r\n#ifndef softfloat_shiftRightJam64\r\n/*----------------------------------------------------------------------------\r\n| Shifts 'a' right by the number of bits given in 'dist', which must not\r\n| be zero. If any nonzero bits are shifted off, they are \"jammed\" into the\r\n| least-significant bit of the shifted value by setting the least-significant\r\n| bit to 1. This shifted-and-jammed value is returned.\r\n| The value of 'dist' can be arbitrarily large. In particular, if 'dist' is\r\n| greater than 64, the result will be either 0 or 1, depending on whether 'a'\r\n| is zero or nonzero.\r\n*----------------------------------------------------------------------------*/\r\n#if defined INLINE_LEVEL && (3 <= INLINE_LEVEL)\r\nINLINE uint64_t softfloat_shiftRightJam64( uint64_t a, uint_fast32_t dist )\r\n{\r\n return\r\n (dist < 63) ? a>>dist | ((uint64_t) (a<<(-dist & 63)) != 0) : (a != 0);\r\n}\r\n#else\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nuint64_t softfloat_shiftRightJam64( uint64_t a, uint_fast32_t dist );\r\n#endif\r\n#endif\r\n\r\n/*----------------------------------------------------------------------------\r\n| A constant table that translates an 8-bit unsigned integer (the array index)\r\n| into the number of leading 0 bits before the most-significant 1 of that\r\n| integer. For integer zero (index 0), the corresponding table element is 8.\r\n*----------------------------------------------------------------------------*/\r\nextern const uint_least8_t softfloat_countLeadingZeros8[256];\r\n\r\n#ifndef softfloat_countLeadingZeros16\r\n/*----------------------------------------------------------------------------\r\n| Returns the number of leading 0 bits before the most-significant 1 bit of\r\n| 'a'. If 'a' is zero, 16 is returned.\r\n*----------------------------------------------------------------------------*/\r\n#if defined INLINE_LEVEL && (2 <= INLINE_LEVEL)\r\nINLINE uint_fast8_t softfloat_countLeadingZeros16( uint16_t a )\r\n{\r\n uint_fast8_t count = 8;\r\n if ( 0x100 <= a ) {\r\n count = 0;\r\n a >>= 8;\r\n }\r\n count += softfloat_countLeadingZeros8[a];\r\n return count;\r\n}\r\n#else\r\nuint_fast8_t softfloat_countLeadingZeros16( uint16_t a );\r\n#endif\r\n#endif\r\n\r\n#ifndef softfloat_countLeadingZeros32\r\n/*----------------------------------------------------------------------------\r\n| Returns the number of leading 0 bits before the most-significant 1 bit of\r\n| 'a'. If 'a' is zero, 32 is returned.\r\n*----------------------------------------------------------------------------*/\r\n#if defined INLINE_LEVEL && (3 <= INLINE_LEVEL)\r\nINLINE uint_fast8_t softfloat_countLeadingZeros32( uint32_t a )\r\n{\r\n uint_fast8_t count = 0;\r\n if ( a < 0x10000 ) {\r\n count = 16;\r\n a <<= 16;\r\n }\r\n if ( a < 0x1000000 ) {\r\n count += 8;\r\n a <<= 8;\r\n }\r\n count += softfloat_countLeadingZeros8[a>>24];\r\n return count;\r\n}\r\n#else\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nuint_fast8_t softfloat_countLeadingZeros32( uint32_t a );\r\n#endif\r\n#endif\r\n\r\n#ifndef softfloat_countLeadingZeros64\r\n/*----------------------------------------------------------------------------\r\n| Returns the number of leading 0 bits before the most-significant 1 bit of\r\n| 'a'. If 'a' is zero, 64 is returned.\r\n*----------------------------------------------------------------------------*/\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nuint_fast8_t softfloat_countLeadingZeros64( uint64_t a );\r\n#endif\r\n\r\nextern const uint16_t softfloat_approxRecip_1k0s[16];\r\nextern const uint16_t softfloat_approxRecip_1k1s[16];\r\n\r\n#ifndef softfloat_approxRecip32_1\r\n/*----------------------------------------------------------------------------\r\n| Returns an approximation to the reciprocal of the number represented by 'a',\r\n| where 'a' is interpreted as an unsigned fixed-point number with one integer\r\n| bit and 31 fraction bits. The 'a' input must be \"normalized\", meaning that\r\n| its most-significant bit (bit 31) must be 1. Thus, if A is the value of\r\n| the fixed-point interpretation of 'a', then 1 <= A < 2. The returned value\r\n| is interpreted as a pure unsigned fraction, having no integer bits and 32\r\n| fraction bits. The approximation returned is never greater than the true\r\n| reciprocal 1/A, and it differs from the true reciprocal by at most 2.006 ulp\r\n| (units in the last place).\r\n*----------------------------------------------------------------------------*/\r\n#ifdef SOFTFLOAT_FAST_DIV64TO32\r\n#define softfloat_approxRecip32_1( a ) ((uint32_t) (UINT64_C( 0x7FFFFFFFFFFFFFFF ) / (uint32_t) (a)))\r\n#else\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nuint32_t softfloat_approxRecip32_1( uint32_t a );\r\n#endif\r\n#endif\r\n\r\nextern const uint16_t softfloat_approxRecipSqrt_1k0s[16];\r\nextern const uint16_t softfloat_approxRecipSqrt_1k1s[16];\r\n\r\n#ifndef softfloat_approxRecipSqrt32_1\r\n/*----------------------------------------------------------------------------\r\n| Returns an approximation to the reciprocal of the square root of the number\r\n| represented by 'a', where 'a' is interpreted as an unsigned fixed-point\r\n| number either with one integer bit and 31 fraction bits or with two integer\r\n| bits and 30 fraction bits. The format of 'a' is determined by 'oddExpA',\r\n| which must be either 0 or 1. If 'oddExpA' is 1, 'a' is interpreted as\r\n| having one integer bit, and if 'oddExpA' is 0, 'a' is interpreted as having\r\n| two integer bits. The 'a' input must be \"normalized\", meaning that its\r\n| most-significant bit (bit 31) must be 1. Thus, if A is the value of the\r\n| fixed-point interpretation of 'a', it follows that 1 <= A < 2 when 'oddExpA'\r\n| is 1, and 2 <= A < 4 when 'oddExpA' is 0.\r\n| The returned value is interpreted as a pure unsigned fraction, having\r\n| no integer bits and 32 fraction bits. The approximation returned is never\r\n| greater than the true reciprocal 1/sqrt(A), and it differs from the true\r\n| reciprocal by at most 2.06 ulp (units in the last place). The approximation\r\n| returned is also always within the range 0.5 to 1; thus, the most-\r\n| significant bit of the result is always set.\r\n*----------------------------------------------------------------------------*/\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nuint32_t softfloat_approxRecipSqrt32_1( unsigned int oddExpA, uint32_t a );\r\n#endif\r\n\r\n#ifdef SOFTFLOAT_FAST_INT64\r\n\r\n/*----------------------------------------------------------------------------\r\n| The following functions are needed only when 'SOFTFLOAT_FAST_INT64' is\r\n| defined.\r\n*----------------------------------------------------------------------------*/\r\n\r\n#ifndef softfloat_eq128\r\n/*----------------------------------------------------------------------------\r\n| Returns true if the 128-bit unsigned integer formed by concatenating 'a64'\r\n| and 'a0' is equal to the 128-bit unsigned integer formed by concatenating\r\n| 'b64' and 'b0'.\r\n*----------------------------------------------------------------------------*/\r\n#if defined INLINE_LEVEL && (1 <= INLINE_LEVEL)\r\nINLINE\r\nbool softfloat_eq128( uint64_t a64, uint64_t a0, uint64_t b64, uint64_t b0 )\r\n { return (a64 == b64) && (a0 == b0); }\r\n#else\r\nbool softfloat_eq128( uint64_t a64, uint64_t a0, uint64_t b64, uint64_t b0 );\r\n#endif\r\n#endif\r\n\r\n#ifndef softfloat_le128\r\n/*----------------------------------------------------------------------------\r\n| Returns true if the 128-bit unsigned integer formed by concatenating 'a64'\r\n| and 'a0' is less than or equal to the 128-bit unsigned integer formed by\r\n| concatenating 'b64' and 'b0'.\r\n*----------------------------------------------------------------------------*/\r\n#if defined INLINE_LEVEL && (2 <= INLINE_LEVEL)\r\nINLINE\r\nbool softfloat_le128( uint64_t a64, uint64_t a0, uint64_t b64, uint64_t b0 )\r\n { return (a64 < b64) || ((a64 == b64) && (a0 <= b0)); }\r\n#else\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nbool softfloat_le128( uint64_t a64, uint64_t a0, uint64_t b64, uint64_t b0 );\r\n#endif\r\n#endif\r\n\r\n#ifndef softfloat_lt128\r\n/*----------------------------------------------------------------------------\r\n| Returns true if the 128-bit unsigned integer formed by concatenating 'a64'\r\n| and 'a0' is less than the 128-bit unsigned integer formed by concatenating\r\n| 'b64' and 'b0'.\r\n*----------------------------------------------------------------------------*/\r\n#if defined INLINE_LEVEL && (2 <= INLINE_LEVEL)\r\nINLINE\r\nbool softfloat_lt128( uint64_t a64, uint64_t a0, uint64_t b64, uint64_t b0 )\r\n { return (a64 < b64) || ((a64 == b64) && (a0 < b0)); }\r\n#else\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nbool softfloat_lt128( uint64_t a64, uint64_t a0, uint64_t b64, uint64_t b0 );\r\n#endif\r\n#endif\r\n\r\n#ifndef softfloat_shortShiftLeft128\r\n/*----------------------------------------------------------------------------\r\n| Shifts the 128 bits formed by concatenating 'a64' and 'a0' left by the\r\n| number of bits given in 'dist', which must be in the range 1 to 63.\r\n*----------------------------------------------------------------------------*/\r\n#if defined INLINE_LEVEL && (2 <= INLINE_LEVEL)\r\nINLINE\r\nstruct uint128\r\n softfloat_shortShiftLeft128( uint64_t a64, uint64_t a0, uint_fast8_t dist )\r\n{\r\n struct uint128 z;\r\n z.v64 = a64<>(-dist & 63);\r\n z.v0 = a0<>dist;\r\n z.v0 = a64<<(-dist & 63) | a0>>dist;\r\n return z;\r\n}\r\n#else\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nstruct uint128\r\n softfloat_shortShiftRight128( uint64_t a64, uint64_t a0, uint_fast8_t dist );\r\n#endif\r\n#endif\r\n\r\n#ifndef softfloat_shortShiftRightJam64Extra\r\n/*----------------------------------------------------------------------------\r\n| This function is the same as 'softfloat_shiftRightJam64Extra' (below),\r\n| except that 'dist' must be in the range 1 to 63.\r\n*----------------------------------------------------------------------------*/\r\n#if defined INLINE_LEVEL && (2 <= INLINE_LEVEL)\r\nINLINE\r\nstruct uint64_extra\r\n softfloat_shortShiftRightJam64Extra(\r\n uint64_t a, uint64_t extra, uint_fast8_t dist )\r\n{\r\n struct uint64_extra z;\r\n z.v = a>>dist;\r\n z.extra = a<<(-dist & 63) | (extra != 0);\r\n return z;\r\n}\r\n#else\r\nstruct uint64_extra\r\n softfloat_shortShiftRightJam64Extra(\r\n uint64_t a, uint64_t extra, uint_fast8_t dist );\r\n#endif\r\n#endif\r\n\r\n#ifndef softfloat_shortShiftRightJam128\r\n/*----------------------------------------------------------------------------\r\n| Shifts the 128 bits formed by concatenating 'a64' and 'a0' right by the\r\n| number of bits given in 'dist', which must be in the range 1 to 63. If any\r\n| nonzero bits are shifted off, they are \"jammed\" into the least-significant\r\n| bit of the shifted value by setting the least-significant bit to 1. This\r\n| shifted-and-jammed value is returned.\r\n*----------------------------------------------------------------------------*/\r\n#if defined INLINE_LEVEL && (3 <= INLINE_LEVEL)\r\nINLINE\r\nstruct uint128\r\n softfloat_shortShiftRightJam128(\r\n uint64_t a64, uint64_t a0, uint_fast8_t dist )\r\n{\r\n uint_fast8_t negDist = -dist;\r\n struct uint128 z;\r\n z.v64 = a64>>dist;\r\n z.v0 =\r\n a64<<(negDist & 63) | a0>>dist\r\n | ((uint64_t) (a0<<(negDist & 63)) != 0);\r\n return z;\r\n}\r\n#else\r\nstruct uint128\r\n softfloat_shortShiftRightJam128(\r\n uint64_t a64, uint64_t a0, uint_fast8_t dist );\r\n#endif\r\n#endif\r\n\r\n#ifndef softfloat_shortShiftRightJam128Extra\r\n/*----------------------------------------------------------------------------\r\n| This function is the same as 'softfloat_shiftRightJam128Extra' (below),\r\n| except that 'dist' must be in the range 1 to 63.\r\n*----------------------------------------------------------------------------*/\r\n#if defined INLINE_LEVEL && (3 <= INLINE_LEVEL)\r\nINLINE\r\nstruct uint128_extra\r\n softfloat_shortShiftRightJam128Extra(\r\n uint64_t a64, uint64_t a0, uint64_t extra, uint_fast8_t dist )\r\n{\r\n uint_fast8_t negDist = -dist;\r\n struct uint128_extra z;\r\n z.v.v64 = a64>>dist;\r\n z.v.v0 = a64<<(negDist & 63) | a0>>dist;\r\n z.extra = a0<<(negDist & 63) | (extra != 0);\r\n return z;\r\n}\r\n#else\r\nstruct uint128_extra\r\n softfloat_shortShiftRightJam128Extra(\r\n uint64_t a64, uint64_t a0, uint64_t extra, uint_fast8_t dist );\r\n#endif\r\n#endif\r\n\r\n#ifndef softfloat_shiftRightJam64Extra\r\n/*----------------------------------------------------------------------------\r\n| Shifts the 128 bits formed by concatenating 'a' and 'extra' right by 64\r\n| _plus_ the number of bits given in 'dist', which must not be zero. This\r\n| shifted value is at most 64 nonzero bits and is returned in the 'v' field\r\n| of the 'struct uint64_extra' result. The 64-bit 'extra' field of the result\r\n| contains a value formed as follows from the bits that were shifted off: The\r\n| _last_ bit shifted off is the most-significant bit of the 'extra' field, and\r\n| the other 63 bits of the 'extra' field are all zero if and only if _all_but_\r\n| _the_last_ bits shifted off were all zero.\r\n| (This function makes more sense if 'a' and 'extra' are considered to form\r\n| an unsigned fixed-point number with binary point between 'a' and 'extra'.\r\n| This fixed-point value is shifted right by the number of bits given in\r\n| 'dist', and the integer part of this shifted value is returned in the 'v'\r\n| field of the result. The fractional part of the shifted value is modified\r\n| as described above and returned in the 'extra' field of the result.)\r\n*----------------------------------------------------------------------------*/\r\n#if defined INLINE_LEVEL && (4 <= INLINE_LEVEL)\r\nINLINE\r\nstruct uint64_extra\r\n softfloat_shiftRightJam64Extra(\r\n uint64_t a, uint64_t extra, uint_fast32_t dist )\r\n{\r\n struct uint64_extra z;\r\n if ( dist < 64 ) {\r\n z.v = a>>dist;\r\n z.extra = a<<(-dist & 63);\r\n } else {\r\n z.v = 0;\r\n z.extra = (dist == 64) ? a : (a != 0);\r\n }\r\n z.extra |= (extra != 0);\r\n return z;\r\n}\r\n#else\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nstruct uint64_extra\r\n softfloat_shiftRightJam64Extra(\r\n uint64_t a, uint64_t extra, uint_fast32_t dist );\r\n#endif\r\n#endif\r\n\r\n#ifndef softfloat_shiftRightJam128\r\n/*----------------------------------------------------------------------------\r\n| Shifts the 128 bits formed by concatenating 'a64' and 'a0' right by the\r\n| number of bits given in 'dist', which must not be zero. If any nonzero bits\r\n| are shifted off, they are \"jammed\" into the least-significant bit of the\r\n| shifted value by setting the least-significant bit to 1. This shifted-and-\r\n| jammed value is returned.\r\n| The value of 'dist' can be arbitrarily large. In particular, if 'dist' is\r\n| greater than 128, the result will be either 0 or 1, depending on whether the\r\n| original 128 bits are all zeros.\r\n*----------------------------------------------------------------------------*/\r\nstruct uint128\r\n softfloat_shiftRightJam128( uint64_t a64, uint64_t a0, uint_fast32_t dist );\r\n#endif\r\n\r\n#ifndef softfloat_shiftRightJam128Extra\r\n/*----------------------------------------------------------------------------\r\n| Shifts the 192 bits formed by concatenating 'a64', 'a0', and 'extra' right\r\n| by 64 _plus_ the number of bits given in 'dist', which must not be zero.\r\n| This shifted value is at most 128 nonzero bits and is returned in the 'v'\r\n| field of the 'struct uint128_extra' result. The 64-bit 'extra' field of the\r\n| result contains a value formed as follows from the bits that were shifted\r\n| off: The _last_ bit shifted off is the most-significant bit of the 'extra'\r\n| field, and the other 63 bits of the 'extra' field are all zero if and only\r\n| if _all_but_the_last_ bits shifted off were all zero.\r\n| (This function makes more sense if 'a64', 'a0', and 'extra' are considered\r\n| to form an unsigned fixed-point number with binary point between 'a0' and\r\n| 'extra'. This fixed-point value is shifted right by the number of bits\r\n| given in 'dist', and the integer part of this shifted value is returned\r\n| in the 'v' field of the result. The fractional part of the shifted value\r\n| is modified as described above and returned in the 'extra' field of the\r\n| result.)\r\n*----------------------------------------------------------------------------*/\r\nstruct uint128_extra\r\n softfloat_shiftRightJam128Extra(\r\n uint64_t a64, uint64_t a0, uint64_t extra, uint_fast32_t dist );\r\n#endif\r\n\r\n#ifndef softfloat_shiftRightJam256M\r\n/*----------------------------------------------------------------------------\r\n| Shifts the 256-bit unsigned integer pointed to by 'aPtr' right by the number\r\n| of bits given in 'dist', which must not be zero. If any nonzero bits are\r\n| shifted off, they are \"jammed\" into the least-significant bit of the shifted\r\n| value by setting the least-significant bit to 1. This shifted-and-jammed\r\n| value is stored at the location pointed to by 'zPtr'. Each of 'aPtr' and\r\n| 'zPtr' points to an array of four 64-bit elements that concatenate in the\r\n| platform's normal endian order to form a 256-bit integer.\r\n| The value of 'dist' can be arbitrarily large. In particular, if 'dist'\r\n| is greater than 256, the stored result will be either 0 or 1, depending on\r\n| whether the original 256 bits are all zeros.\r\n*----------------------------------------------------------------------------*/\r\nvoid\r\n softfloat_shiftRightJam256M(\r\n const uint64_t *aPtr, uint_fast32_t dist, uint64_t *zPtr );\r\n#endif\r\n\r\n#ifndef softfloat_add128\r\n/*----------------------------------------------------------------------------\r\n| Returns the sum of the 128-bit integer formed by concatenating 'a64' and\r\n| 'a0' and the 128-bit integer formed by concatenating 'b64' and 'b0'. The\r\n| addition is modulo 2^128, so any carry out is lost.\r\n*----------------------------------------------------------------------------*/\r\n#if defined INLINE_LEVEL && (2 <= INLINE_LEVEL)\r\nINLINE\r\nstruct uint128\r\n softfloat_add128( uint64_t a64, uint64_t a0, uint64_t b64, uint64_t b0 )\r\n{\r\n struct uint128 z;\r\n z.v0 = a0 + b0;\r\n z.v64 = a64 + b64 + (z.v0 < a0);\r\n return z;\r\n}\r\n#else\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nstruct uint128\r\n softfloat_add128( uint64_t a64, uint64_t a0, uint64_t b64, uint64_t b0 );\r\n#endif\r\n#endif\r\n\r\n#ifndef softfloat_add256M\r\n/*----------------------------------------------------------------------------\r\n| Adds the two 256-bit integers pointed to by 'aPtr' and 'bPtr'. The addition\r\n| is modulo 2^256, so any carry out is lost. The sum is stored at the\r\n| location pointed to by 'zPtr'. Each of 'aPtr', 'bPtr', and 'zPtr' points to\r\n| an array of four 64-bit elements that concatenate in the platform's normal\r\n| endian order to form a 256-bit integer.\r\n*----------------------------------------------------------------------------*/\r\nvoid\r\n softfloat_add256M(\r\n const uint64_t *aPtr, const uint64_t *bPtr, uint64_t *zPtr );\r\n#endif\r\n\r\n#ifndef softfloat_sub128\r\n/*----------------------------------------------------------------------------\r\n| Returns the difference of the 128-bit integer formed by concatenating 'a64'\r\n| and 'a0' and the 128-bit integer formed by concatenating 'b64' and 'b0'.\r\n| The subtraction is modulo 2^128, so any borrow out (carry out) is lost.\r\n*----------------------------------------------------------------------------*/\r\n#if defined INLINE_LEVEL && (2 <= INLINE_LEVEL)\r\nINLINE\r\nstruct uint128\r\n softfloat_sub128( uint64_t a64, uint64_t a0, uint64_t b64, uint64_t b0 )\r\n{\r\n struct uint128 z;\r\n z.v0 = a0 - b0;\r\n z.v64 = a64 - b64;\r\n z.v64 -= (a0 < b0);\r\n return z;\r\n}\r\n#else\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nstruct uint128\r\n softfloat_sub128( uint64_t a64, uint64_t a0, uint64_t b64, uint64_t b0 );\r\n#endif\r\n#endif\r\n\r\n#ifndef softfloat_sub256M\r\n/*----------------------------------------------------------------------------\r\n| Subtracts the 256-bit integer pointed to by 'bPtr' from the 256-bit integer\r\n| pointed to by 'aPtr'. The addition is modulo 2^256, so any borrow out\r\n| (carry out) is lost. The difference is stored at the location pointed to\r\n| by 'zPtr'. Each of 'aPtr', 'bPtr', and 'zPtr' points to an array of four\r\n| 64-bit elements that concatenate in the platform's normal endian order to\r\n| form a 256-bit integer.\r\n*----------------------------------------------------------------------------*/\r\nvoid\r\n softfloat_sub256M(\r\n const uint64_t *aPtr, const uint64_t *bPtr, uint64_t *zPtr );\r\n#endif\r\n\r\n#ifndef softfloat_mul64ByShifted32To128\r\n/*----------------------------------------------------------------------------\r\n| Returns the 128-bit product of 'a', 'b', and 2^32.\r\n*----------------------------------------------------------------------------*/\r\n#if defined INLINE_LEVEL && (3 <= INLINE_LEVEL)\r\nINLINE struct uint128 softfloat_mul64ByShifted32To128( uint64_t a, uint32_t b )\r\n{\r\n uint_fast64_t mid;\r\n struct uint128 z;\r\n mid = (uint_fast64_t) (uint32_t) a * b;\r\n z.v0 = mid<<32;\r\n z.v64 = (uint_fast64_t) (uint32_t) (a>>32) * b + (mid>>32);\r\n return z;\r\n}\r\n#else\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nstruct uint128 softfloat_mul64ByShifted32To128( uint64_t a, uint32_t b );\r\n#endif\r\n#endif\r\n\r\n#ifndef softfloat_mul64To128\r\n/*----------------------------------------------------------------------------\r\n| Returns the 128-bit product of 'a' and 'b'.\r\n*----------------------------------------------------------------------------*/\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nstruct uint128 softfloat_mul64To128( uint64_t a, uint64_t b );\r\n#endif\r\n\r\n#ifndef softfloat_mul128By32\r\n/*----------------------------------------------------------------------------\r\n| Returns the product of the 128-bit integer formed by concatenating 'a64' and\r\n| 'a0', multiplied by 'b'. The multiplication is modulo 2^128; any overflow\r\n| bits are discarded.\r\n*----------------------------------------------------------------------------*/\r\n#if defined INLINE_LEVEL && (4 <= INLINE_LEVEL)\r\nINLINE\r\nstruct uint128 softfloat_mul128By32( uint64_t a64, uint64_t a0, uint32_t b )\r\n{\r\n struct uint128 z;\r\n uint_fast64_t mid;\r\n uint_fast32_t carry;\r\n z.v0 = a0 * b;\r\n mid = (uint_fast64_t) (uint32_t) (a0>>32) * b;\r\n carry = (uint32_t) ((uint_fast32_t) (z.v0>>32) - (uint_fast32_t) mid);\r\n z.v64 = a64 * b + (uint_fast32_t) ((mid + carry)>>32);\r\n return z;\r\n}\r\n#else\r\nstruct uint128 softfloat_mul128By32( uint64_t a64, uint64_t a0, uint32_t b );\r\n#endif\r\n#endif\r\n\r\n#ifndef softfloat_mul128To256M\r\n/*----------------------------------------------------------------------------\r\n| Multiplies the 128-bit unsigned integer formed by concatenating 'a64' and\r\n| 'a0' by the 128-bit unsigned integer formed by concatenating 'b64' and\r\n| 'b0'. The 256-bit product is stored at the location pointed to by 'zPtr'.\r\n| Argument 'zPtr' points to an array of four 64-bit elements that concatenate\r\n| in the platform's normal endian order to form a 256-bit integer.\r\n*----------------------------------------------------------------------------*/\r\nvoid\r\n softfloat_mul128To256M(\r\n uint64_t a64, uint64_t a0, uint64_t b64, uint64_t b0, uint64_t *zPtr );\r\n#endif\r\n\r\n#else\r\n\r\n/*----------------------------------------------------------------------------\r\n| The following functions are needed only when 'SOFTFLOAT_FAST_INT64' is not\r\n| defined.\r\n*----------------------------------------------------------------------------*/\r\n\r\n#ifndef softfloat_compare96M\r\n/*----------------------------------------------------------------------------\r\n| Compares the two 96-bit unsigned integers pointed to by 'aPtr' and 'bPtr'.\r\n| Returns -1 if the first integer (A) is less than the second (B); returns 0\r\n| if the two integers are equal; and returns +1 if the first integer (A)\r\n| is greater than the second (B). (The result is thus the signum of A - B.)\r\n| Each of 'aPtr' and 'bPtr' points to an array of three 32-bit elements that\r\n| concatenate in the platform's normal endian order to form a 96-bit integer.\r\n*----------------------------------------------------------------------------*/\r\nint_fast8_t softfloat_compare96M( const uint32_t *aPtr, const uint32_t *bPtr );\r\n#endif\r\n\r\n#ifndef softfloat_compare128M\r\n/*----------------------------------------------------------------------------\r\n| Compares the two 128-bit unsigned integers pointed to by 'aPtr' and 'bPtr'.\r\n| Returns -1 if the first integer (A) is less than the second (B); returns 0\r\n| if the two integers are equal; and returns +1 if the first integer (A)\r\n| is greater than the second (B). (The result is thus the signum of A - B.)\r\n| Each of 'aPtr' and 'bPtr' points to an array of four 32-bit elements that\r\n| concatenate in the platform's normal endian order to form a 128-bit integer.\r\n*----------------------------------------------------------------------------*/\r\nint_fast8_t\r\n softfloat_compare128M( const uint32_t *aPtr, const uint32_t *bPtr );\r\n#endif\r\n\r\n#ifndef softfloat_shortShiftLeft64To96M\r\n/*----------------------------------------------------------------------------\r\n| Extends 'a' to 96 bits and shifts the value left by the number of bits given\r\n| in 'dist', which must be in the range 1 to 31. The result is stored at the\r\n| location pointed to by 'zPtr'. Argument 'zPtr' points to an array of three\r\n| 32-bit elements that concatenate in the platform's normal endian order to\r\n| form a 96-bit integer.\r\n*----------------------------------------------------------------------------*/\r\n#if defined INLINE_LEVEL && (2 <= INLINE_LEVEL)\r\nINLINE\r\nvoid\r\n softfloat_shortShiftLeft64To96M(\r\n uint64_t a, uint_fast8_t dist, uint32_t *zPtr )\r\n{\r\n zPtr[indexWord( 3, 0 )] = (uint32_t) a<>= 32 - dist;\r\n zPtr[indexWord( 3, 2 )] = a>>32;\r\n zPtr[indexWord( 3, 1 )] = a;\r\n}\r\n#else\r\nvoid\r\n softfloat_shortShiftLeft64To96M(\r\n uint64_t a, uint_fast8_t dist, uint32_t *zPtr );\r\n#endif\r\n#endif\r\n\r\n#ifndef softfloat_shortShiftLeftM\r\n/*----------------------------------------------------------------------------\r\n| Shifts the N-bit unsigned integer pointed to by 'aPtr' left by the number\r\n| of bits given in 'dist', where N = 'size_words' * 32. The value of 'dist'\r\n| must be in the range 1 to 31. Any nonzero bits shifted off are lost. The\r\n| shifted N-bit result is stored at the location pointed to by 'zPtr'. Each\r\n| of 'aPtr' and 'zPtr' points to a 'size_words'-long array of 32-bit elements\r\n| that concatenate in the platform's normal endian order to form an N-bit\r\n| integer.\r\n*----------------------------------------------------------------------------*/\r\nvoid\r\n softfloat_shortShiftLeftM(\r\n uint_fast8_t size_words,\r\n const uint32_t *aPtr,\r\n uint_fast8_t dist,\r\n uint32_t *zPtr\r\n );\r\n#endif\r\n\r\n#ifndef softfloat_shortShiftLeft96M\r\n/*----------------------------------------------------------------------------\r\n| This function or macro is the same as 'softfloat_shortShiftLeftM' with\r\n| 'size_words' = 3 (N = 96).\r\n*----------------------------------------------------------------------------*/\r\n#define softfloat_shortShiftLeft96M( aPtr, dist, zPtr ) softfloat_shortShiftLeftM( 3, aPtr, dist, zPtr )\r\n#endif\r\n\r\n#ifndef softfloat_shortShiftLeft128M\r\n/*----------------------------------------------------------------------------\r\n| This function or macro is the same as 'softfloat_shortShiftLeftM' with\r\n| 'size_words' = 4 (N = 128).\r\n*----------------------------------------------------------------------------*/\r\n#define softfloat_shortShiftLeft128M( aPtr, dist, zPtr ) softfloat_shortShiftLeftM( 4, aPtr, dist, zPtr )\r\n#endif\r\n\r\n#ifndef softfloat_shortShiftLeft160M\r\n/*----------------------------------------------------------------------------\r\n| This function or macro is the same as 'softfloat_shortShiftLeftM' with\r\n| 'size_words' = 5 (N = 160).\r\n*----------------------------------------------------------------------------*/\r\n#define softfloat_shortShiftLeft160M( aPtr, dist, zPtr ) softfloat_shortShiftLeftM( 5, aPtr, dist, zPtr )\r\n#endif\r\n\r\n#ifndef softfloat_shiftLeftM\r\n/*----------------------------------------------------------------------------\r\n| Shifts the N-bit unsigned integer pointed to by 'aPtr' left by the number\r\n| of bits given in 'dist', where N = 'size_words' * 32. The value of 'dist'\r\n| must not be zero. Any nonzero bits shifted off are lost. The shifted\r\n| N-bit result is stored at the location pointed to by 'zPtr'. Each of 'aPtr'\r\n| and 'zPtr' points to a 'size_words'-long array of 32-bit elements that\r\n| concatenate in the platform's normal endian order to form an N-bit integer.\r\n| The value of 'dist' can be arbitrarily large. In particular, if 'dist' is\r\n| greater than N, the stored result will be 0.\r\n*----------------------------------------------------------------------------*/\r\nvoid\r\n softfloat_shiftLeftM(\r\n uint_fast8_t size_words,\r\n const uint32_t *aPtr,\r\n uint32_t dist,\r\n uint32_t *zPtr\r\n );\r\n#endif\r\n\r\n#ifndef softfloat_shiftLeft96M\r\n/*----------------------------------------------------------------------------\r\n| This function or macro is the same as 'softfloat_shiftLeftM' with\r\n| 'size_words' = 3 (N = 96).\r\n*----------------------------------------------------------------------------*/\r\n#define softfloat_shiftLeft96M( aPtr, dist, zPtr ) softfloat_shiftLeftM( 3, aPtr, dist, zPtr )\r\n#endif\r\n\r\n#ifndef softfloat_shiftLeft128M\r\n/*----------------------------------------------------------------------------\r\n| This function or macro is the same as 'softfloat_shiftLeftM' with\r\n| 'size_words' = 4 (N = 128).\r\n*----------------------------------------------------------------------------*/\r\n#define softfloat_shiftLeft128M( aPtr, dist, zPtr ) softfloat_shiftLeftM( 4, aPtr, dist, zPtr )\r\n#endif\r\n\r\n#ifndef softfloat_shiftLeft160M\r\n/*----------------------------------------------------------------------------\r\n| This function or macro is the same as 'softfloat_shiftLeftM' with\r\n| 'size_words' = 5 (N = 160).\r\n*----------------------------------------------------------------------------*/\r\n#define softfloat_shiftLeft160M( aPtr, dist, zPtr ) softfloat_shiftLeftM( 5, aPtr, dist, zPtr )\r\n#endif\r\n\r\n#ifndef softfloat_shortShiftRightM\r\n/*----------------------------------------------------------------------------\r\n| Shifts the N-bit unsigned integer pointed to by 'aPtr' right by the number\r\n| of bits given in 'dist', where N = 'size_words' * 32. The value of 'dist'\r\n| must be in the range 1 to 31. Any nonzero bits shifted off are lost. The\r\n| shifted N-bit result is stored at the location pointed to by 'zPtr'. Each\r\n| of 'aPtr' and 'zPtr' points to a 'size_words'-long array of 32-bit elements\r\n| that concatenate in the platform's normal endian order to form an N-bit\r\n| integer.\r\n*----------------------------------------------------------------------------*/\r\nvoid\r\n softfloat_shortShiftRightM(\r\n uint_fast8_t size_words,\r\n const uint32_t *aPtr,\r\n uint_fast8_t dist,\r\n uint32_t *zPtr\r\n );\r\n#endif\r\n\r\n#ifndef softfloat_shortShiftRight128M\r\n/*----------------------------------------------------------------------------\r\n| This function or macro is the same as 'softfloat_shortShiftRightM' with\r\n| 'size_words' = 4 (N = 128).\r\n*----------------------------------------------------------------------------*/\r\n#define softfloat_shortShiftRight128M( aPtr, dist, zPtr ) softfloat_shortShiftRightM( 4, aPtr, dist, zPtr )\r\n#endif\r\n\r\n#ifndef softfloat_shortShiftRight160M\r\n/*----------------------------------------------------------------------------\r\n| This function or macro is the same as 'softfloat_shortShiftRightM' with\r\n| 'size_words' = 5 (N = 160).\r\n*----------------------------------------------------------------------------*/\r\n#define softfloat_shortShiftRight160M( aPtr, dist, zPtr ) softfloat_shortShiftRightM( 5, aPtr, dist, zPtr )\r\n#endif\r\n\r\n#ifndef softfloat_shortShiftRightJamM\r\n/*----------------------------------------------------------------------------\r\n| Shifts the N-bit unsigned integer pointed to by 'aPtr' right by the number\r\n| of bits given in 'dist', where N = 'size_words' * 32. The value of 'dist'\r\n| must be in the range 1 to 31. If any nonzero bits are shifted off, they are\r\n| \"jammed\" into the least-significant bit of the shifted value by setting the\r\n| least-significant bit to 1. This shifted-and-jammed N-bit result is stored\r\n| at the location pointed to by 'zPtr'. Each of 'aPtr' and 'zPtr' points\r\n| to a 'size_words'-long array of 32-bit elements that concatenate in the\r\n| platform's normal endian order to form an N-bit integer.\r\n*----------------------------------------------------------------------------*/\r\nvoid\r\n softfloat_shortShiftRightJamM(\r\n uint_fast8_t, const uint32_t *, uint_fast8_t, uint32_t * );\r\n#endif\r\n\r\n#ifndef softfloat_shortShiftRightJam160M\r\n/*----------------------------------------------------------------------------\r\n| This function or macro is the same as 'softfloat_shortShiftRightJamM' with\r\n| 'size_words' = 5 (N = 160).\r\n*----------------------------------------------------------------------------*/\r\n#define softfloat_shortShiftRightJam160M( aPtr, dist, zPtr ) softfloat_shortShiftRightJamM( 5, aPtr, dist, zPtr )\r\n#endif\r\n\r\n#ifndef softfloat_shiftRightM\r\n/*----------------------------------------------------------------------------\r\n| Shifts the N-bit unsigned integer pointed to by 'aPtr' right by the number\r\n| of bits given in 'dist', where N = 'size_words' * 32. The value of 'dist'\r\n| must not be zero. Any nonzero bits shifted off are lost. The shifted\r\n| N-bit result is stored at the location pointed to by 'zPtr'. Each of 'aPtr'\r\n| and 'zPtr' points to a 'size_words'-long array of 32-bit elements that\r\n| concatenate in the platform's normal endian order to form an N-bit integer.\r\n| The value of 'dist' can be arbitrarily large. In particular, if 'dist' is\r\n| greater than N, the stored result will be 0.\r\n*----------------------------------------------------------------------------*/\r\nvoid\r\n softfloat_shiftRightM(\r\n uint_fast8_t size_words,\r\n const uint32_t *aPtr,\r\n uint32_t dist,\r\n uint32_t *zPtr\r\n );\r\n#endif\r\n\r\n#ifndef softfloat_shiftRight96M\r\n/*----------------------------------------------------------------------------\r\n| This function or macro is the same as 'softfloat_shiftRightM' with\r\n| 'size_words' = 3 (N = 96).\r\n*----------------------------------------------------------------------------*/\r\n#define softfloat_shiftRight96M( aPtr, dist, zPtr ) softfloat_shiftRightM( 3, aPtr, dist, zPtr )\r\n#endif\r\n\r\n#ifndef softfloat_shiftRightJamM\r\n/*----------------------------------------------------------------------------\r\n| Shifts the N-bit unsigned integer pointed to by 'aPtr' right by the number\r\n| of bits given in 'dist', where N = 'size_words' * 32. The value of 'dist'\r\n| must not be zero. If any nonzero bits are shifted off, they are \"jammed\"\r\n| into the least-significant bit of the shifted value by setting the least-\r\n| significant bit to 1. This shifted-and-jammed N-bit result is stored\r\n| at the location pointed to by 'zPtr'. Each of 'aPtr' and 'zPtr' points\r\n| to a 'size_words'-long array of 32-bit elements that concatenate in the\r\n| platform's normal endian order to form an N-bit integer.\r\n| The value of 'dist' can be arbitrarily large. In particular, if 'dist'\r\n| is greater than N, the stored result will be either 0 or 1, depending on\r\n| whether the original N bits are all zeros.\r\n*----------------------------------------------------------------------------*/\r\nvoid\r\n softfloat_shiftRightJamM(\r\n uint_fast8_t size_words,\r\n const uint32_t *aPtr,\r\n uint32_t dist,\r\n uint32_t *zPtr\r\n );\r\n#endif\r\n\r\n#ifndef softfloat_shiftRightJam96M\r\n/*----------------------------------------------------------------------------\r\n| This function or macro is the same as 'softfloat_shiftRightJamM' with\r\n| 'size_words' = 3 (N = 96).\r\n*----------------------------------------------------------------------------*/\r\n#define softfloat_shiftRightJam96M( aPtr, dist, zPtr ) softfloat_shiftRightJamM( 3, aPtr, dist, zPtr )\r\n#endif\r\n\r\n#ifndef softfloat_shiftRightJam128M\r\n/*----------------------------------------------------------------------------\r\n| This function or macro is the same as 'softfloat_shiftRightJamM' with\r\n| 'size_words' = 4 (N = 128).\r\n*----------------------------------------------------------------------------*/\r\n#define softfloat_shiftRightJam128M( aPtr, dist, zPtr ) softfloat_shiftRightJamM( 4, aPtr, dist, zPtr )\r\n#endif\r\n\r\n#ifndef softfloat_shiftRightJam160M\r\n/*----------------------------------------------------------------------------\r\n| This function or macro is the same as 'softfloat_shiftRightJamM' with\r\n| 'size_words' = 5 (N = 160).\r\n*----------------------------------------------------------------------------*/\r\n#define softfloat_shiftRightJam160M( aPtr, dist, zPtr ) softfloat_shiftRightJamM( 5, aPtr, dist, zPtr )\r\n#endif\r\n\r\n#ifndef softfloat_addM\r\n/*----------------------------------------------------------------------------\r\n| Adds the two N-bit integers pointed to by 'aPtr' and 'bPtr', where N =\r\n| 'size_words' * 32. The addition is modulo 2^N, so any carry out is lost.\r\n| The N-bit sum is stored at the location pointed to by 'zPtr'. Each of\r\n| 'aPtr', 'bPtr', and 'zPtr' points to a 'size_words'-long array of 32-bit\r\n| elements that concatenate in the platform's normal endian order to form an\r\n| N-bit integer.\r\n*----------------------------------------------------------------------------*/\r\nvoid\r\n softfloat_addM(\r\n uint_fast8_t size_words,\r\n const uint32_t *aPtr,\r\n const uint32_t *bPtr,\r\n uint32_t *zPtr\r\n );\r\n#endif\r\n\r\n#ifndef softfloat_add96M\r\n/*----------------------------------------------------------------------------\r\n| This function or macro is the same as 'softfloat_addM' with 'size_words'\r\n| = 3 (N = 96).\r\n*----------------------------------------------------------------------------*/\r\n#define softfloat_add96M( aPtr, bPtr, zPtr ) softfloat_addM( 3, aPtr, bPtr, zPtr )\r\n#endif\r\n\r\n#ifndef softfloat_add128M\r\n/*----------------------------------------------------------------------------\r\n| This function or macro is the same as 'softfloat_addM' with 'size_words'\r\n| = 4 (N = 128).\r\n*----------------------------------------------------------------------------*/\r\n#define softfloat_add128M( aPtr, bPtr, zPtr ) softfloat_addM( 4, aPtr, bPtr, zPtr )\r\n#endif\r\n\r\n#ifndef softfloat_add160M\r\n/*----------------------------------------------------------------------------\r\n| This function or macro is the same as 'softfloat_addM' with 'size_words'\r\n| = 5 (N = 160).\r\n*----------------------------------------------------------------------------*/\r\n#define softfloat_add160M( aPtr, bPtr, zPtr ) softfloat_addM( 5, aPtr, bPtr, zPtr )\r\n#endif\r\n\r\n#ifndef softfloat_addCarryM\r\n/*----------------------------------------------------------------------------\r\n| Adds the two N-bit unsigned integers pointed to by 'aPtr' and 'bPtr', where\r\n| N = 'size_words' * 32, plus 'carry', which must be either 0 or 1. The N-bit\r\n| sum (modulo 2^N) is stored at the location pointed to by 'zPtr', and any\r\n| carry out is returned as the result. Each of 'aPtr', 'bPtr', and 'zPtr'\r\n| points to a 'size_words'-long array of 32-bit elements that concatenate in\r\n| the platform's normal endian order to form an N-bit integer.\r\n*----------------------------------------------------------------------------*/\r\nuint_fast8_t\r\n softfloat_addCarryM(\r\n uint_fast8_t size_words,\r\n const uint32_t *aPtr,\r\n const uint32_t *bPtr,\r\n uint_fast8_t carry,\r\n uint32_t *zPtr\r\n );\r\n#endif\r\n\r\n#ifndef softfloat_addComplCarryM\r\n/*----------------------------------------------------------------------------\r\n| This function or macro is the same as 'softfloat_addCarryM', except that\r\n| the value of the unsigned integer pointed to by 'bPtr' is bit-wise completed\r\n| before the addition.\r\n*----------------------------------------------------------------------------*/\r\nuint_fast8_t\r\n softfloat_addComplCarryM(\r\n uint_fast8_t size_words,\r\n const uint32_t *aPtr,\r\n const uint32_t *bPtr,\r\n uint_fast8_t carry,\r\n uint32_t *zPtr\r\n );\r\n#endif\r\n\r\n#ifndef softfloat_addComplCarry96M\r\n/*----------------------------------------------------------------------------\r\n| This function or macro is the same as 'softfloat_addComplCarryM' with\r\n| 'size_words' = 3 (N = 96).\r\n*----------------------------------------------------------------------------*/\r\n#define softfloat_addComplCarry96M( aPtr, bPtr, carry, zPtr ) softfloat_addComplCarryM( 3, aPtr, bPtr, carry, zPtr )\r\n#endif\r\n\r\n#ifndef softfloat_negXM\r\n/*----------------------------------------------------------------------------\r\n| Replaces the N-bit unsigned integer pointed to by 'zPtr' by the\r\n| 2s-complement of itself, where N = 'size_words' * 32. Argument 'zPtr'\r\n| points to a 'size_words'-long array of 32-bit elements that concatenate in\r\n| the platform's normal endian order to form an N-bit integer.\r\n*----------------------------------------------------------------------------*/\r\nvoid softfloat_negXM( uint_fast8_t size_words, uint32_t *zPtr );\r\n#endif\r\n\r\n#ifndef softfloat_negX96M\r\n/*----------------------------------------------------------------------------\r\n| This function or macro is the same as 'softfloat_negXM' with 'size_words'\r\n| = 3 (N = 96).\r\n*----------------------------------------------------------------------------*/\r\n#define softfloat_negX96M( zPtr ) softfloat_negXM( 3, zPtr )\r\n#endif\r\n\r\n#ifndef softfloat_negX128M\r\n/*----------------------------------------------------------------------------\r\n| This function or macro is the same as 'softfloat_negXM' with 'size_words'\r\n| = 4 (N = 128).\r\n*----------------------------------------------------------------------------*/\r\n#define softfloat_negX128M( zPtr ) softfloat_negXM( 4, zPtr )\r\n#endif\r\n\r\n#ifndef softfloat_negX160M\r\n/*----------------------------------------------------------------------------\r\n| This function or macro is the same as 'softfloat_negXM' with 'size_words'\r\n| = 5 (N = 160).\r\n*----------------------------------------------------------------------------*/\r\n#define softfloat_negX160M( zPtr ) softfloat_negXM( 5, zPtr )\r\n#endif\r\n\r\n#ifndef softfloat_negX256M\r\n/*----------------------------------------------------------------------------\r\n| This function or macro is the same as 'softfloat_negXM' with 'size_words'\r\n| = 8 (N = 256).\r\n*----------------------------------------------------------------------------*/\r\n#define softfloat_negX256M( zPtr ) softfloat_negXM( 8, zPtr )\r\n#endif\r\n\r\n#ifndef softfloat_sub1XM\r\n/*----------------------------------------------------------------------------\r\n| Subtracts 1 from the N-bit integer pointed to by 'zPtr', where N =\r\n| 'size_words' * 32. The subtraction is modulo 2^N, so any borrow out (carry\r\n| out) is lost. Argument 'zPtr' points to a 'size_words'-long array of 32-bit\r\n| elements that concatenate in the platform's normal endian order to form an\r\n| N-bit integer.\r\n*----------------------------------------------------------------------------*/\r\nvoid softfloat_sub1XM( uint_fast8_t size_words, uint32_t *zPtr );\r\n#endif\r\n\r\n#ifndef softfloat_sub1X96M\r\n/*----------------------------------------------------------------------------\r\n| This function or macro is the same as 'softfloat_sub1XM' with 'size_words'\r\n| = 3 (N = 96).\r\n*----------------------------------------------------------------------------*/\r\n#define softfloat_sub1X96M( zPtr ) softfloat_sub1XM( 3, zPtr )\r\n#endif\r\n\r\n#ifndef softfloat_sub1X160M\r\n/*----------------------------------------------------------------------------\r\n| This function or macro is the same as 'softfloat_sub1XM' with 'size_words'\r\n| = 5 (N = 160).\r\n*----------------------------------------------------------------------------*/\r\n#define softfloat_sub1X160M( zPtr ) softfloat_sub1XM( 5, zPtr )\r\n#endif\r\n\r\n#ifndef softfloat_subM\r\n/*----------------------------------------------------------------------------\r\n| Subtracts the two N-bit integers pointed to by 'aPtr' and 'bPtr', where N =\r\n| 'size_words' * 32. The subtraction is modulo 2^N, so any borrow out (carry\r\n| out) is lost. The N-bit difference is stored at the location pointed to by\r\n| 'zPtr'. Each of 'aPtr', 'bPtr', and 'zPtr' points to a 'size_words'-long\r\n| array of 32-bit elements that concatenate in the platform's normal endian\r\n| order to form an N-bit integer.\r\n*----------------------------------------------------------------------------*/\r\nvoid\r\n softfloat_subM(\r\n uint_fast8_t size_words,\r\n const uint32_t *aPtr,\r\n const uint32_t *bPtr,\r\n uint32_t *zPtr\r\n );\r\n#endif\r\n\r\n#ifndef softfloat_sub96M\r\n/*----------------------------------------------------------------------------\r\n| This function or macro is the same as 'softfloat_subM' with 'size_words'\r\n| = 3 (N = 96).\r\n*----------------------------------------------------------------------------*/\r\n#define softfloat_sub96M( aPtr, bPtr, zPtr ) softfloat_subM( 3, aPtr, bPtr, zPtr )\r\n#endif\r\n\r\n#ifndef softfloat_sub128M\r\n/*----------------------------------------------------------------------------\r\n| This function or macro is the same as 'softfloat_subM' with 'size_words'\r\n| = 4 (N = 128).\r\n*----------------------------------------------------------------------------*/\r\n#define softfloat_sub128M( aPtr, bPtr, zPtr ) softfloat_subM( 4, aPtr, bPtr, zPtr )\r\n#endif\r\n\r\n#ifndef softfloat_sub160M\r\n/*----------------------------------------------------------------------------\r\n| This function or macro is the same as 'softfloat_subM' with 'size_words'\r\n| = 5 (N = 160).\r\n*----------------------------------------------------------------------------*/\r\n#define softfloat_sub160M( aPtr, bPtr, zPtr ) softfloat_subM( 5, aPtr, bPtr, zPtr )\r\n#endif\r\n\r\n#ifndef softfloat_mul64To128M\r\n/*----------------------------------------------------------------------------\r\n| Multiplies 'a' and 'b' and stores the 128-bit product at the location\r\n| pointed to by 'zPtr'. Argument 'zPtr' points to an array of four 32-bit\r\n| elements that concatenate in the platform's normal endian order to form a\r\n| 128-bit integer.\r\n*----------------------------------------------------------------------------*/\r\nvoid softfloat_mul64To128M( uint64_t a, uint64_t b, uint32_t *zPtr );\r\n#endif\r\n\r\n#ifndef softfloat_mul128MTo256M\r\n/*----------------------------------------------------------------------------\r\n| Multiplies the two 128-bit unsigned integers pointed to by 'aPtr' and\r\n| 'bPtr', and stores the 256-bit product at the location pointed to by 'zPtr'.\r\n| Each of 'aPtr' and 'bPtr' points to an array of four 32-bit elements that\r\n| concatenate in the platform's normal endian order to form a 128-bit integer.\r\n| Argument 'zPtr' points to an array of eight 32-bit elements that concatenate\r\n| to form a 256-bit integer.\r\n*----------------------------------------------------------------------------*/\r\nvoid\r\n softfloat_mul128MTo256M(\r\n const uint32_t *aPtr, const uint32_t *bPtr, uint32_t *zPtr );\r\n#endif\r\n\r\n#ifndef softfloat_remStepMBy32\r\n/*----------------------------------------------------------------------------\r\n| Performs a \"remainder reduction step\" as follows: Arguments 'remPtr' and\r\n| 'bPtr' both point to N-bit unsigned integers, where N = 'size_words' * 32.\r\n| Defining R and B as the values of those integers, the expression (R<<'dist')\r\n| - B * q is computed modulo 2^N, and the N-bit result is stored at the\r\n| location pointed to by 'zPtr'. Each of 'remPtr', 'bPtr', and 'zPtr' points\r\n| to a 'size_words'-long array of 32-bit elements that concatenate in the\r\n| platform's normal endian order to form an N-bit integer.\r\n*----------------------------------------------------------------------------*/\r\nvoid\r\n softfloat_remStepMBy32(\r\n uint_fast8_t size_words,\r\n const uint32_t *remPtr,\r\n uint_fast8_t dist,\r\n const uint32_t *bPtr,\r\n uint32_t q,\r\n uint32_t *zPtr\r\n );\r\n#endif\r\n\r\n#ifndef softfloat_remStep96MBy32\r\n/*----------------------------------------------------------------------------\r\n| This function or macro is the same as 'softfloat_remStepMBy32' with\r\n| 'size_words' = 3 (N = 96).\r\n*----------------------------------------------------------------------------*/\r\n#define softfloat_remStep96MBy32( remPtr, dist, bPtr, q, zPtr ) softfloat_remStepMBy32( 3, remPtr, dist, bPtr, q, zPtr )\r\n#endif\r\n\r\n#ifndef softfloat_remStep128MBy32\r\n/*----------------------------------------------------------------------------\r\n| This function or macro is the same as 'softfloat_remStepMBy32' with\r\n| 'size_words' = 4 (N = 128).\r\n*----------------------------------------------------------------------------*/\r\n#define softfloat_remStep128MBy32( remPtr, dist, bPtr, q, zPtr ) softfloat_remStepMBy32( 4, remPtr, dist, bPtr, q, zPtr )\r\n#endif\r\n\r\n#ifndef softfloat_remStep160MBy32\r\n/*----------------------------------------------------------------------------\r\n| This function or macro is the same as 'softfloat_remStepMBy32' with\r\n| 'size_words' = 5 (N = 160).\r\n*----------------------------------------------------------------------------*/\r\n#define softfloat_remStep160MBy32( remPtr, dist, bPtr, q, zPtr ) softfloat_remStepMBy32( 5, remPtr, dist, bPtr, q, zPtr )\r\n#endif\r\n\r\n#endif\r\n\r\n#endif\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/s_add128.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014, 2015 The Regents of the University of\r\nCalifornia. All rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \"platform.h\"\r\n#include \"primitiveTypes.h\"\r\n\r\n#ifndef softfloat_add128\r\n\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nstruct uint128\r\n softfloat_add128( uint64_t a64, uint64_t a0, uint64_t b64, uint64_t b0 )\r\n{\r\n struct uint128 z;\r\n\r\n z.v0 = a0 + b0;\r\n z.v64 = a64 + b64 + (z.v0 < a0);\r\n return z;\r\n\r\n}\r\n\r\n#endif\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/s_addMagsExtF80.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014 The Regents of the University of California.\r\nAll rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \r\n#include \"platform.h\"\r\n#include \"internals.h\"\r\n#include \"specialize.h\"\r\n#include \"softfloat.h\"\r\n\r\nextFloat80_t\r\n softfloat_addMagsExtF80(\r\n struct softfloat_state *state,\r\n uint_fast16_t uiA64,\r\n uint_fast64_t uiA0,\r\n uint_fast16_t uiB64,\r\n uint_fast64_t uiB0,\r\n bool signZ\r\n )\r\n{\r\n int_fast32_t expA;\r\n uint_fast64_t sigA;\r\n int_fast32_t expB;\r\n uint_fast64_t sigB;\r\n int_fast32_t expDiff;\r\n uint_fast16_t uiZ64;\r\n uint_fast64_t uiZ0, sigZ, sigZExtra;\r\n struct exp32_sig64 normExpSig;\r\n int_fast32_t expZ;\r\n struct uint64_extra sig64Extra;\r\n struct uint128 uiZ;\r\n union { struct extFloat80M s; extFloat80_t f; } uZ;\r\n\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n expA = expExtF80UI64( uiA64 );\r\n sigA = uiA0;\r\n expB = expExtF80UI64( uiB64 );\r\n sigB = uiB0;\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n expDiff = expA - expB;\r\n if ( ! expDiff ) {\r\n if ( expA == 0x7FFF ) {\r\n if ( (sigA | sigB) & UINT64_C( 0x7FFFFFFFFFFFFFFF ) ) {\r\n goto propagateNaN;\r\n }\r\n uiZ64 = uiA64;\r\n uiZ0 = uiA0;\r\n goto uiZ;\r\n }\r\n sigZ = sigA + sigB;\r\n sigZExtra = 0;\r\n if ( ! expA ) {\r\n normExpSig = softfloat_normSubnormalExtF80Sig( sigZ );\r\n expZ = normExpSig.exp + 1;\r\n sigZ = normExpSig.sig;\r\n goto roundAndPack;\r\n }\r\n expZ = expA;\r\n goto shiftRight1;\r\n }\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n if ( expDiff < 0 ) {\r\n if ( expB == 0x7FFF ) {\r\n if ( sigB & UINT64_C( 0x7FFFFFFFFFFFFFFF ) ) goto propagateNaN;\r\n uiZ64 = packToExtF80UI64( signZ, 0x7FFF );\r\n uiZ0 = uiB0;\r\n goto uiZ;\r\n }\r\n expZ = expB;\r\n if ( ! expA ) {\r\n ++expDiff;\r\n sigZExtra = 0;\r\n if ( ! expDiff ) goto newlyAligned;\r\n }\r\n sig64Extra = softfloat_shiftRightJam64Extra( sigA, 0, -expDiff );\r\n sigA = sig64Extra.v;\r\n sigZExtra = sig64Extra.extra;\r\n } else {\r\n if ( expA == 0x7FFF ) {\r\n if ( sigA & UINT64_C( 0x7FFFFFFFFFFFFFFF ) ) goto propagateNaN;\r\n uiZ64 = uiA64;\r\n uiZ0 = uiA0;\r\n goto uiZ;\r\n }\r\n expZ = expA;\r\n if ( ! expB ) {\r\n --expDiff;\r\n sigZExtra = 0;\r\n if ( ! expDiff ) goto newlyAligned;\r\n }\r\n sig64Extra = softfloat_shiftRightJam64Extra( sigB, 0, expDiff );\r\n sigB = sig64Extra.v;\r\n sigZExtra = sig64Extra.extra;\r\n }\r\n newlyAligned:\r\n sigZ = sigA + sigB;\r\n if ( sigZ & UINT64_C( 0x8000000000000000 ) ) goto roundAndPack;\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n shiftRight1:\r\n sig64Extra = softfloat_shortShiftRightJam64Extra( sigZ, sigZExtra, 1 );\r\n sigZ = sig64Extra.v | UINT64_C( 0x8000000000000000 );\r\n sigZExtra = sig64Extra.extra;\r\n ++expZ;\r\n roundAndPack:\r\n return\r\n softfloat_roundPackToExtF80(\r\n state, signZ, expZ, sigZ, sigZExtra, state->roundingPrecision );\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n propagateNaN:\r\n uiZ = softfloat_propagateNaNExtF80UI( state, uiA64, uiA0, uiB64, uiB0 );\r\n uiZ64 = uiZ.v64;\r\n uiZ0 = uiZ.v0;\r\n uiZ:\r\n uZ.s.signExp = uiZ64;\r\n uZ.s.signif = uiZ0;\r\n return uZ.f;\r\n\r\n}\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/s_addMagsF128.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014 The Regents of the University of California.\r\nAll rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \r\n#include \"platform.h\"\r\n#include \"internals.h\"\r\n#include \"specialize.h\"\r\n\r\nfloat128_t\r\n softfloat_addMagsF128(\r\n struct softfloat_state *state,\r\n uint_fast64_t uiA64,\r\n uint_fast64_t uiA0,\r\n uint_fast64_t uiB64,\r\n uint_fast64_t uiB0,\r\n bool signZ\r\n )\r\n{\r\n int_fast32_t expA;\r\n struct uint128 sigA;\r\n int_fast32_t expB;\r\n struct uint128 sigB;\r\n int_fast32_t expDiff;\r\n struct uint128 uiZ, sigZ;\r\n int_fast32_t expZ;\r\n uint_fast64_t sigZExtra;\r\n struct uint128_extra sig128Extra;\r\n union ui128_f128 uZ;\r\n\r\n expA = expF128UI64( uiA64 );\r\n sigA.v64 = fracF128UI64( uiA64 );\r\n sigA.v0 = uiA0;\r\n expB = expF128UI64( uiB64 );\r\n sigB.v64 = fracF128UI64( uiB64 );\r\n sigB.v0 = uiB0;\r\n expDiff = expA - expB;\r\n if ( ! expDiff ) {\r\n if ( expA == 0x7FFF ) {\r\n if ( sigA.v64 | sigA.v0 | sigB.v64 | sigB.v0 ) goto propagateNaN;\r\n uiZ.v64 = uiA64;\r\n uiZ.v0 = uiA0;\r\n goto uiZ;\r\n }\r\n sigZ = softfloat_add128( sigA.v64, sigA.v0, sigB.v64, sigB.v0 );\r\n if ( ! expA ) {\r\n uiZ.v64 = packToF128UI64( signZ, 0, sigZ.v64 );\r\n uiZ.v0 = sigZ.v0;\r\n goto uiZ;\r\n }\r\n expZ = expA;\r\n sigZ.v64 |= UINT64_C( 0x0002000000000000 );\r\n sigZExtra = 0;\r\n goto shiftRight1;\r\n }\r\n if ( expDiff < 0 ) {\r\n if ( expB == 0x7FFF ) {\r\n if ( sigB.v64 | sigB.v0 ) goto propagateNaN;\r\n uiZ.v64 = packToF128UI64( signZ, 0x7FFF, 0 );\r\n uiZ.v0 = 0;\r\n goto uiZ;\r\n }\r\n expZ = expB;\r\n if ( expA ) {\r\n sigA.v64 |= UINT64_C( 0x0001000000000000 );\r\n } else {\r\n ++expDiff;\r\n sigZExtra = 0;\r\n if ( ! expDiff ) goto newlyAligned;\r\n }\r\n sig128Extra =\r\n softfloat_shiftRightJam128Extra( sigA.v64, sigA.v0, 0, -expDiff );\r\n sigA = sig128Extra.v;\r\n sigZExtra = sig128Extra.extra;\r\n } else {\r\n if ( expA == 0x7FFF ) {\r\n if ( sigA.v64 | sigA.v0 ) goto propagateNaN;\r\n uiZ.v64 = uiA64;\r\n uiZ.v0 = uiA0;\r\n goto uiZ;\r\n }\r\n expZ = expA;\r\n if ( expB ) {\r\n sigB.v64 |= UINT64_C( 0x0001000000000000 );\r\n } else {\r\n --expDiff;\r\n sigZExtra = 0;\r\n if ( ! expDiff ) goto newlyAligned;\r\n }\r\n sig128Extra =\r\n softfloat_shiftRightJam128Extra( sigB.v64, sigB.v0, 0, expDiff );\r\n sigB = sig128Extra.v;\r\n sigZExtra = sig128Extra.extra;\r\n }\r\n newlyAligned:\r\n sigZ =\r\n softfloat_add128(\r\n sigA.v64 | UINT64_C( 0x0001000000000000 ),\r\n sigA.v0,\r\n sigB.v64,\r\n sigB.v0\r\n );\r\n --expZ;\r\n if ( sigZ.v64 < UINT64_C( 0x0002000000000000 ) ) goto roundAndPack;\r\n ++expZ;\r\n shiftRight1:\r\n sig128Extra =\r\n softfloat_shortShiftRightJam128Extra(\r\n sigZ.v64, sigZ.v0, sigZExtra, 1 );\r\n sigZ = sig128Extra.v;\r\n sigZExtra = sig128Extra.extra;\r\n roundAndPack:\r\n return\r\n softfloat_roundPackToF128( state, signZ, expZ, sigZ.v64, sigZ.v0, sigZExtra );\r\n propagateNaN:\r\n uiZ = softfloat_propagateNaNF128UI( state, uiA64, uiA0, uiB64, uiB0 );\r\n uiZ:\r\n uZ.ui = uiZ;\r\n return uZ.f;\r\n\r\n}\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/s_approxRecip32_1.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014, 2015, 2016 The Regents of the University of\r\nCalifornia. All rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \"platform.h\"\r\n\r\n#ifndef softfloat_approxRecip32_1\r\n\r\nextern const uint16_t softfloat_approxRecip_1k0s[16];\r\nextern const uint16_t softfloat_approxRecip_1k1s[16];\r\n\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nuint32_t softfloat_approxRecip32_1( uint32_t a )\r\n{\r\n int index;\r\n uint16_t eps, r0;\r\n uint32_t sigma0;\r\n uint_fast32_t r;\r\n uint32_t sqrSigma0;\r\n\r\n index = a>>27 & 0xF;\r\n eps = (uint16_t) (a>>11);\r\n r0 = softfloat_approxRecip_1k0s[index]\r\n - ((softfloat_approxRecip_1k1s[index] * (uint_fast32_t) eps)>>20);\r\n sigma0 = ~(uint_fast32_t) ((r0 * (uint_fast64_t) a)>>7);\r\n r = ((uint_fast32_t) r0<<16) + ((r0 * (uint_fast64_t) sigma0)>>24);\r\n sqrSigma0 = ((uint_fast64_t) sigma0 * sigma0)>>32;\r\n r += ((uint32_t) r * (uint_fast64_t) sqrSigma0)>>48;\r\n return r;\r\n\r\n}\r\n\r\n#endif\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/s_approxRecipSqrt32_1.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014, 2015, 2016 The Regents of the University of\r\nCalifornia. All rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \"platform.h\"\r\n\r\n#ifndef softfloat_approxRecipSqrt32_1\r\n\r\nextern const uint16_t softfloat_approxRecipSqrt_1k0s[];\r\nextern const uint16_t softfloat_approxRecipSqrt_1k1s[];\r\n\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nuint32_t softfloat_approxRecipSqrt32_1( unsigned int oddExpA, uint32_t a )\r\n{\r\n int index;\r\n uint16_t eps, r0;\r\n uint_fast32_t ESqrR0;\r\n uint32_t sigma0;\r\n uint_fast32_t r;\r\n uint32_t sqrSigma0;\r\n\r\n index = (a>>27 & 0xE) + oddExpA;\r\n eps = (uint16_t) (a>>12);\r\n r0 = softfloat_approxRecipSqrt_1k0s[index]\r\n - ((softfloat_approxRecipSqrt_1k1s[index] * (uint_fast32_t) eps)\r\n >>20);\r\n ESqrR0 = (uint_fast32_t) r0 * r0;\r\n if ( ! oddExpA ) ESqrR0 <<= 1;\r\n sigma0 = ~(uint_fast32_t) (((uint32_t) ESqrR0 * (uint_fast64_t) a)>>23);\r\n r = ((uint_fast32_t) r0<<16) + ((r0 * (uint_fast64_t) sigma0)>>25);\r\n sqrSigma0 = ((uint_fast64_t) sigma0 * sigma0)>>32;\r\n r += ((uint32_t) ((r>>1) + (r>>3) - ((uint_fast32_t) r0<<14))\r\n * (uint_fast64_t) sqrSigma0)\r\n >>48;\r\n if ( ! (r & 0x80000000) ) r = 0x80000000;\r\n return r;\r\n\r\n}\r\n\r\n#endif\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/s_approxRecipSqrt_1Ks.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014, 2015, 2016 The Regents of the University of\r\nCalifornia. All rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \"platform.h\"\r\n#include \"primitives.h\"\r\n\r\nconst uint16_t softfloat_approxRecipSqrt_1k0s[16] = {\r\n 0xB4C9, 0xFFAB, 0xAA7D, 0xF11C, 0xA1C5, 0xE4C7, 0x9A43, 0xDA29,\r\n 0x93B5, 0xD0E5, 0x8DED, 0xC8B7, 0x88C6, 0xC16D, 0x8424, 0xBAE1\r\n};\r\nconst uint16_t softfloat_approxRecipSqrt_1k1s[16] = {\r\n 0xA5A5, 0xEA42, 0x8C21, 0xC62D, 0x788F, 0xAA7F, 0x6928, 0x94B6,\r\n 0x5CC7, 0x8335, 0x52A6, 0x74E2, 0x4A3E, 0x68FE, 0x432B, 0x5EFD\r\n};\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/s_approxRecip_1Ks.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014, 2015, 2016 The Regents of the University of\r\nCalifornia. All rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \"platform.h\"\r\n#include \"primitives.h\"\r\n\r\nconst uint16_t softfloat_approxRecip_1k0s[16] = {\r\n 0xFFC4, 0xF0BE, 0xE363, 0xD76F, 0xCCAD, 0xC2F0, 0xBA16, 0xB201,\r\n 0xAA97, 0xA3C6, 0x9D7A, 0x97A6, 0x923C, 0x8D32, 0x887E, 0x8417\r\n};\r\nconst uint16_t softfloat_approxRecip_1k1s[16] = {\r\n 0xF0F1, 0xD62C, 0xBFA1, 0xAC77, 0x9C0A, 0x8DDB, 0x8185, 0x76BA,\r\n 0x6D3B, 0x64D4, 0x5D5C, 0x56B1, 0x50B6, 0x4B55, 0x4679, 0x4211\r\n};\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/s_commonNaNToExtF80UI.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014 The Regents of the University of California.\r\nAll rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \"platform.h\"\r\n#include \"primitives.h\"\r\n#include \"specialize.h\"\r\n\r\n/*----------------------------------------------------------------------------\r\n| Converts the common NaN pointed to by `aPtr' into an 80-bit extended\r\n| floating-point NaN, and returns the bit pattern of this value as an unsigned\r\n| integer.\r\n*----------------------------------------------------------------------------*/\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nstruct uint128 softfloat_commonNaNToExtF80UI( const struct commonNaN *aPtr )\r\n{\r\n struct uint128 uiZ;\r\n\r\n uiZ.v64 = (uint_fast16_t) aPtr->sign<<15 | 0x7FFF;\r\n uiZ.v0 = UINT64_C( 0xC000000000000000 ) | aPtr->v64>>1;\r\n return uiZ;\r\n\r\n}\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/s_commonNaNToF128UI.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014 The Regents of the University of California.\r\nAll rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \"platform.h\"\r\n#include \"primitives.h\"\r\n#include \"specialize.h\"\r\n\r\n/*----------------------------------------------------------------------------\r\n| Converts the common NaN pointed to by `aPtr' into a 128-bit floating-point\r\n| NaN, and returns the bit pattern of this value as an unsigned integer.\r\n*----------------------------------------------------------------------------*/\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nstruct uint128 softfloat_commonNaNToF128UI( const struct commonNaN *aPtr )\r\n{\r\n struct uint128 uiZ;\r\n\r\n uiZ = softfloat_shortShiftRight128( aPtr->v64, aPtr->v0, 16 );\r\n uiZ.v64 |= (uint_fast64_t) aPtr->sign<<63 | UINT64_C( 0x7FFF800000000000 );\r\n return uiZ;\r\n\r\n}\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/s_commonNaNToF32UI.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014 The Regents of the University of California.\r\nAll rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \"platform.h\"\r\n#include \"specialize.h\"\r\n\r\n/*----------------------------------------------------------------------------\r\n| Converts the common NaN pointed to by `aPtr' into a 32-bit floating-point\r\n| NaN, and returns the bit pattern of this value as an unsigned integer.\r\n*----------------------------------------------------------------------------*/\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nuint_fast32_t softfloat_commonNaNToF32UI( const struct commonNaN *aPtr )\r\n{\r\n\r\n return (uint_fast32_t) aPtr->sign<<31 | 0x7FC00000 | aPtr->v64>>41;\r\n\r\n}\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/s_commonNaNToF64UI.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014 The Regents of the University of California.\r\nAll rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \"platform.h\"\r\n#include \"specialize.h\"\r\n\r\n/*----------------------------------------------------------------------------\r\n| Converts the common NaN pointed to by `aPtr' into a 64-bit floating-point\r\n| NaN, and returns the bit pattern of this value as an unsigned integer.\r\n*----------------------------------------------------------------------------*/\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nuint_fast64_t softfloat_commonNaNToF64UI( const struct commonNaN *aPtr )\r\n{\r\n\r\n return\r\n (uint_fast64_t) aPtr->sign<<63 | UINT64_C( 0x7FF8000000000000 )\r\n | aPtr->v64>>12;\r\n\r\n}\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/s_countLeadingZeros32.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014, 2015 The Regents of the University of\r\nCalifornia. All rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \"platform.h\"\r\n\r\n#ifndef softfloat_countLeadingZeros32\r\n\r\n#define softfloat_countLeadingZeros32 softfloat_countLeadingZeros32\r\n#include \"primitives.h\"\r\n\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nuint_fast8_t softfloat_countLeadingZeros32( uint32_t a )\r\n{\r\n uint_fast8_t count;\r\n\r\n count = 0;\r\n if ( a < 0x10000 ) {\r\n count = 16;\r\n a <<= 16;\r\n }\r\n if ( a < 0x1000000 ) {\r\n count += 8;\r\n a <<= 8;\r\n }\r\n count += softfloat_countLeadingZeros8[a>>24];\r\n return count;\r\n\r\n}\r\n\r\n#endif\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/s_countLeadingZeros64.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014, 2015 The Regents of the University of\r\nCalifornia. All rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \"platform.h\"\r\n\r\n#ifndef softfloat_countLeadingZeros64\r\n\r\n#define softfloat_countLeadingZeros64 softfloat_countLeadingZeros64\r\n#include \"primitives.h\"\r\n\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nuint_fast8_t softfloat_countLeadingZeros64( uint64_t a )\r\n{\r\n uint_fast8_t count;\r\n uint32_t a32;\r\n\r\n count = 0;\r\n a32 = a>>32;\r\n if ( ! a32 ) {\r\n count = 32;\r\n a32 = a;\r\n }\r\n /*------------------------------------------------------------------------\r\n | From here, result is current count + count leading zeros of `a32'.\r\n *------------------------------------------------------------------------*/\r\n if ( a32 < 0x10000 ) {\r\n count += 16;\r\n a32 <<= 16;\r\n }\r\n if ( a32 < 0x1000000 ) {\r\n count += 8;\r\n a32 <<= 8;\r\n }\r\n count += softfloat_countLeadingZeros8[a32>>24];\r\n return count;\r\n\r\n}\r\n\r\n#endif\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/s_countLeadingZeros8.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014 The Regents of the University of California.\r\nAll rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \"platform.h\"\r\n#include \"primitives.h\"\r\n\r\nconst uint_least8_t softfloat_countLeadingZeros8[256] = {\r\n 8, 7, 6, 6, 5, 5, 5, 5, 4, 4, 4, 4, 4, 4, 4, 4,\r\n 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,\r\n 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\r\n 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\r\n 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\r\n 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\r\n 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\r\n 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\r\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\r\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\r\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\r\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\r\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\r\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\r\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\r\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0\r\n};\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/s_extF80UIToCommonNaN.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014 The Regents of the University of California.\r\nAll rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \"platform.h\"\r\n#include \"specialize.h\"\r\n#include \"softfloat.h\"\r\n\r\n/*----------------------------------------------------------------------------\r\n| Assuming the unsigned integer formed from concatenating `uiA64' and `uiA0'\r\n| has the bit pattern of an 80-bit extended floating-point NaN, converts\r\n| this NaN to the common NaN form, and stores the resulting common NaN at the\r\n| location pointed to by `zPtr'. If the NaN is a signaling NaN, the invalid\r\n| exception is raised.\r\n*----------------------------------------------------------------------------*/\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nvoid\r\n softfloat_extF80UIToCommonNaN(\r\n struct softfloat_state *state, uint_fast16_t uiA64, uint_fast64_t uiA0, struct commonNaN *zPtr )\r\n{\r\n\r\n if ( softfloat_isSigNaNExtF80UI( uiA64, uiA0 ) ) {\r\n softfloat_raiseFlags( state, softfloat_flag_invalid );\r\n }\r\n zPtr->sign = uiA64>>15;\r\n zPtr->v64 = uiA0<<1;\r\n zPtr->v0 = 0;\r\n\r\n}\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/s_f128UIToCommonNaN.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014 The Regents of the University of California.\r\nAll rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \"platform.h\"\r\n#include \"primitives.h\"\r\n#include \"specialize.h\"\r\n#include \"softfloat.h\"\r\n\r\n/*----------------------------------------------------------------------------\r\n| Assuming the unsigned integer formed from concatenating `uiA64' and `uiA0'\r\n| has the bit pattern of a 128-bit floating-point NaN, converts this NaN to\r\n| the common NaN form, and stores the resulting common NaN at the location\r\n| pointed to by `zPtr'. If the NaN is a signaling NaN, the invalid exception\r\n| is raised.\r\n*----------------------------------------------------------------------------*/\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nvoid\r\n softfloat_f128UIToCommonNaN(\r\n struct softfloat_state *state, uint_fast64_t uiA64, uint_fast64_t uiA0, struct commonNaN *zPtr )\r\n{\r\n struct uint128 NaNSig;\r\n\r\n if ( softfloat_isSigNaNF128UI( uiA64, uiA0 ) ) {\r\n softfloat_raiseFlags( state, softfloat_flag_invalid );\r\n }\r\n NaNSig = softfloat_shortShiftLeft128( uiA64, uiA0, 16 );\r\n zPtr->sign = uiA64>>63;\r\n zPtr->v64 = NaNSig.v64;\r\n zPtr->v0 = NaNSig.v0;\r\n\r\n}\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/s_f32UIToCommonNaN.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014 The Regents of the University of California.\r\nAll rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \"platform.h\"\r\n#include \"specialize.h\"\r\n#include \"softfloat.h\"\r\n\r\n/*----------------------------------------------------------------------------\r\n| Assuming `uiA' has the bit pattern of a 32-bit floating-point NaN, converts\r\n| this NaN to the common NaN form, and stores the resulting common NaN at the\r\n| location pointed to by `zPtr'. If the NaN is a signaling NaN, the invalid\r\n| exception is raised.\r\n*----------------------------------------------------------------------------*/\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nvoid softfloat_f32UIToCommonNaN( struct softfloat_state *state, uint_fast32_t uiA, struct commonNaN *zPtr )\r\n{\r\n\r\n if ( softfloat_isSigNaNF32UI( uiA ) ) {\r\n softfloat_raiseFlags( state, softfloat_flag_invalid );\r\n }\r\n zPtr->sign = uiA>>31;\r\n zPtr->v64 = (uint_fast64_t) uiA<<41;\r\n zPtr->v0 = 0;\r\n\r\n}\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/s_f64UIToCommonNaN.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014 The Regents of the University of California.\r\nAll rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \"platform.h\"\r\n#include \"specialize.h\"\r\n#include \"softfloat.h\"\r\n\r\n/*----------------------------------------------------------------------------\r\n| Assuming `uiA' has the bit pattern of a 64-bit floating-point NaN, converts\r\n| this NaN to the common NaN form, and stores the resulting common NaN at the\r\n| location pointed to by `zPtr'. If the NaN is a signaling NaN, the invalid\r\n| exception is raised.\r\n*----------------------------------------------------------------------------*/\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nvoid softfloat_f64UIToCommonNaN( struct softfloat_state *state, uint_fast64_t uiA, struct commonNaN *zPtr )\r\n{\r\n\r\n if ( softfloat_isSigNaNF64UI( uiA ) ) {\r\n softfloat_raiseFlags( state, softfloat_flag_invalid );\r\n }\r\n zPtr->sign = uiA>>63;\r\n zPtr->v64 = uiA<<12;\r\n zPtr->v0 = 0;\r\n\r\n}\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/s_le128.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014, 2015 The Regents of the University of\r\nCalifornia. All rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \r\n#include \"platform.h\"\r\n\r\n#ifndef softfloat_le128\r\n\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nbool softfloat_le128( uint64_t a64, uint64_t a0, uint64_t b64, uint64_t b0 )\r\n{\r\n\r\n return (a64 < b64) || ((a64 == b64) && (a0 <= b0));\r\n\r\n}\r\n\r\n#endif\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/s_lt128.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014, 2015 The Regents of the University of\r\nCalifornia. All rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \r\n#include \"platform.h\"\r\n\r\n#ifndef softfloat_lt128\r\n\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nbool softfloat_lt128( uint64_t a64, uint64_t a0, uint64_t b64, uint64_t b0 )\r\n{\r\n\r\n return (a64 < b64) || ((a64 == b64) && (a0 < b0));\r\n\r\n}\r\n\r\n#endif\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/s_mul64ByShifted32To128.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014, 2015 The Regents of the University of\r\nCalifornia. All rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \"platform.h\"\r\n#include \"primitiveTypes.h\"\r\n\r\n#ifndef softfloat_mul64ByShifted32To128\r\n\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nstruct uint128 softfloat_mul64ByShifted32To128( uint64_t a, uint32_t b )\r\n{\r\n uint_fast64_t mid;\r\n struct uint128 z;\r\n\r\n mid = (uint_fast64_t) (uint32_t) a * b;\r\n z.v0 = mid<<32;\r\n z.v64 = (uint_fast64_t) (uint32_t) (a>>32) * b + (mid>>32);\r\n return z;\r\n\r\n}\r\n\r\n#endif\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/s_mul64To128.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014, 2015 The Regents of the University of\r\nCalifornia. All rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \"platform.h\"\r\n#include \"primitiveTypes.h\"\r\n\r\n#ifndef softfloat_mul64To128\r\n\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nstruct uint128 softfloat_mul64To128( uint64_t a, uint64_t b )\r\n{\r\n uint32_t a32, a0, b32, b0;\r\n struct uint128 z;\r\n uint64_t mid1, mid;\r\n\r\n a32 = a>>32;\r\n a0 = a;\r\n b32 = b>>32;\r\n b0 = b;\r\n z.v0 = (uint_fast64_t) a0 * b0;\r\n mid1 = (uint_fast64_t) a32 * b0;\r\n mid = mid1 + (uint_fast64_t) a0 * b32;\r\n z.v64 = (uint_fast64_t) a32 * b32;\r\n z.v64 += (uint_fast64_t) (mid < mid1)<<32 | mid>>32;\r\n mid <<= 32;\r\n z.v0 += mid;\r\n z.v64 += (z.v0 < mid);\r\n return z;\r\n\r\n}\r\n\r\n#endif\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/s_normRoundPackToExtF80.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014, 2015, 2016 The Regents of the University of\r\nCalifornia. All rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \r\n#include \"platform.h\"\r\n#include \"internals.h\"\r\n\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nextFloat80_t\r\n softfloat_normRoundPackToExtF80(\r\n struct softfloat_state *state,\r\n bool sign,\r\n int_fast32_t exp,\r\n uint_fast64_t sig,\r\n uint_fast64_t sigExtra,\r\n uint_fast8_t roundingPrecision\r\n )\r\n{\r\n int_fast8_t shiftDist;\r\n struct uint128 sig128;\r\n\r\n if ( ! sig ) {\r\n exp -= 64;\r\n sig = sigExtra;\r\n sigExtra = 0;\r\n }\r\n shiftDist = softfloat_countLeadingZeros64( sig );\r\n exp -= shiftDist;\r\n if ( shiftDist ) {\r\n sig128 = softfloat_shortShiftLeft128( sig, sigExtra, shiftDist );\r\n sig = sig128.v64;\r\n sigExtra = sig128.v0;\r\n }\r\n return\r\n softfloat_roundPackToExtF80(\r\n state, sign, exp, sig, sigExtra, roundingPrecision );\r\n\r\n}\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/s_normRoundPackToF128.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014, 2015, 2016 The Regents of the University of\r\nCalifornia. All rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \r\n#include \"platform.h\"\r\n#include \"internals.h\"\r\n\r\nfloat128_t\r\n softfloat_normRoundPackToF128(\r\n struct softfloat_state *state,\r\n bool sign, int_fast32_t exp, uint_fast64_t sig64, uint_fast64_t sig0 )\r\n{\r\n int_fast8_t shiftDist;\r\n struct uint128 sig128;\r\n union ui128_f128 uZ;\r\n uint_fast64_t sigExtra;\r\n struct uint128_extra sig128Extra;\r\n\r\n if ( ! sig64 ) {\r\n exp -= 64;\r\n sig64 = sig0;\r\n sig0 = 0;\r\n }\r\n shiftDist = softfloat_countLeadingZeros64( sig64 ) - 15;\r\n exp -= shiftDist;\r\n if ( 0 <= shiftDist ) {\r\n if ( shiftDist ) {\r\n sig128 = softfloat_shortShiftLeft128( sig64, sig0, shiftDist );\r\n sig64 = sig128.v64;\r\n sig0 = sig128.v0;\r\n }\r\n if ( (uint32_t) exp < 0x7FFD ) {\r\n uZ.ui.v64 = packToF128UI64( sign, sig64 | sig0 ? exp : 0, sig64 );\r\n uZ.ui.v0 = sig0;\r\n return uZ.f;\r\n }\r\n sigExtra = 0;\r\n } else {\r\n sig128Extra =\r\n softfloat_shortShiftRightJam128Extra( sig64, sig0, 0, -shiftDist );\r\n sig64 = sig128Extra.v.v64;\r\n sig0 = sig128Extra.v.v0;\r\n sigExtra = sig128Extra.extra;\r\n }\r\n return softfloat_roundPackToF128( state, sign, exp, sig64, sig0, sigExtra );\r\n\r\n}\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/s_normSubnormalExtF80Sig.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014, 2015, 2016 The Regents of the University of\r\nCalifornia. All rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \"platform.h\"\r\n#include \"internals.h\"\r\n\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nstruct exp32_sig64 softfloat_normSubnormalExtF80Sig( uint_fast64_t sig )\r\n{\r\n int_fast8_t shiftDist;\r\n struct exp32_sig64 z;\r\n\r\n shiftDist = softfloat_countLeadingZeros64( sig );\r\n z.exp = -shiftDist;\r\n z.sig = sig<\r\n#include \"platform.h\"\r\n#include \"internals.h\"\r\n\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nstruct exp32_sig128\r\n softfloat_normSubnormalF128Sig( uint_fast64_t sig64, uint_fast64_t sig0 )\r\n{\r\n int_fast8_t shiftDist;\r\n struct exp32_sig128 z;\r\n\r\n if ( ! sig64 ) {\r\n shiftDist = softfloat_countLeadingZeros64( sig0 ) - 15;\r\n z.exp = -63 - shiftDist;\r\n if ( shiftDist < 0 ) {\r\n z.sig.v64 = sig0>>-shiftDist;\r\n z.sig.v0 = sig0<<(shiftDist & 63);\r\n } else {\r\n z.sig.v64 = sig0<\r\n#include \"platform.h\"\r\n#include \"internals.h\"\r\n\r\nint softfloat_normSubnormalF128SigM( uint32_t *sigPtr )\r\n{\r\n const uint32_t *ptr;\r\n int_fast16_t shiftDist;\r\n uint32_t wordSig;\r\n\r\n ptr = sigPtr + indexWordHi( 4 );\r\n shiftDist = 0;\r\n for (;;) {\r\n wordSig = *ptr;\r\n if ( wordSig ) break;\r\n shiftDist += 32;\r\n if ( 128 <= shiftDist ) return 1;\r\n ptr -= wordIncr;\r\n }\r\n shiftDist += softfloat_countLeadingZeros32( wordSig ) - 15;\r\n if ( shiftDist ) softfloat_shiftLeft128M( sigPtr, shiftDist, sigPtr );\r\n return 1 - shiftDist;\r\n\r\n}\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/s_normSubnormalF32Sig.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014, 2015, 2016 The Regents of the University of\r\nCalifornia. All rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \"platform.h\"\r\n#include \"internals.h\"\r\n\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nstruct exp16_sig32 softfloat_normSubnormalF32Sig( uint_fast32_t sig )\r\n{\r\n int_fast8_t shiftDist;\r\n struct exp16_sig32 z;\r\n\r\n shiftDist = softfloat_countLeadingZeros32( sig ) - 8;\r\n z.exp = 1 - shiftDist;\r\n z.sig = sig<\r\n#include \"platform.h\"\r\n#include \"internals.h\"\r\n\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nstruct exp16_sig64 softfloat_normSubnormalF64Sig( uint_fast64_t sig )\r\n{\r\n int_fast8_t shiftDist;\r\n struct exp16_sig64 z;\r\n\r\n shiftDist = softfloat_countLeadingZeros64( sig ) - 11;\r\n z.exp = 1 - shiftDist;\r\n z.sig = sig<\r\n#include \r\n#include \"platform.h\"\r\n#include \"internals.h\"\r\n#include \"specialize.h\"\r\n#include \"softfloat.h\"\r\n\r\n/*----------------------------------------------------------------------------\r\n| Interpreting the unsigned integer formed from concatenating 'uiA64' and\r\n| 'uiA0' as an 80-bit extended floating-point value, and likewise interpreting\r\n| the unsigned integer formed from concatenating 'uiB64' and 'uiB0' as another\r\n| 80-bit extended floating-point value, and assuming at least on of these\r\n| floating-point values is a NaN, returns the bit pattern of the combined NaN\r\n| result. If either original floating-point value is a signaling NaN, the\r\n| invalid exception is raised.\r\n*----------------------------------------------------------------------------*/\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nstruct uint128\r\n softfloat_propagateNaNExtF80UI(\r\n struct softfloat_state *state,\r\n uint_fast16_t uiA64,\r\n uint_fast64_t uiA0,\r\n uint_fast16_t uiB64,\r\n uint_fast64_t uiB0\r\n )\r\n{\r\n bool isSigNaNA, isSigNaNB;\r\n uint_fast64_t uiNonsigA0, uiNonsigB0;\r\n uint_fast16_t uiMagA64, uiMagB64;\r\n struct uint128 uiZ;\r\n\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n isSigNaNA = softfloat_isSigNaNExtF80UI( uiA64, uiA0 );\r\n isSigNaNB = softfloat_isSigNaNExtF80UI( uiB64, uiB0 );\r\n /*------------------------------------------------------------------------\r\n | Make NaNs non-signaling.\r\n *------------------------------------------------------------------------*/\r\n uiNonsigA0 = uiA0 | UINT64_C( 0xC000000000000000 );\r\n uiNonsigB0 = uiB0 | UINT64_C( 0xC000000000000000 );\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n if ( isSigNaNA | isSigNaNB ) {\r\n softfloat_raiseFlags( state, softfloat_flag_invalid );\r\n if ( isSigNaNA ) {\r\n if ( isSigNaNB ) goto returnLargerMag;\r\n if ( isNaNExtF80UI( uiB64, uiB0 ) ) goto returnB;\r\n goto returnA;\r\n } else {\r\n if ( isNaNExtF80UI( uiA64, uiA0 ) ) goto returnA;\r\n goto returnB;\r\n }\r\n }\r\n returnLargerMag:\r\n uiMagA64 = uiA64 & 0x7FFF;\r\n uiMagB64 = uiB64 & 0x7FFF;\r\n if ( uiMagA64 < uiMagB64 ) goto returnB;\r\n if ( uiMagB64 < uiMagA64 ) goto returnA;\r\n if ( uiA0 < uiB0 ) goto returnB;\r\n if ( uiB0 < uiA0 ) goto returnA;\r\n if ( uiA64 < uiB64 ) goto returnA;\r\n returnB:\r\n uiZ.v64 = uiB64;\r\n uiZ.v0 = uiNonsigB0;\r\n return uiZ;\r\n returnA:\r\n uiZ.v64 = uiA64;\r\n uiZ.v0 = uiNonsigA0;\r\n return uiZ;\r\n\r\n}\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/s_propagateNaNF128UI.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014 The Regents of the University of California.\r\nAll rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \r\n#include \"platform.h\"\r\n#include \"internals.h\"\r\n#include \"specialize.h\"\r\n#include \"softfloat.h\"\r\n\r\n/*----------------------------------------------------------------------------\r\n| Interpreting the unsigned integer formed from concatenating `uiA64' and\r\n| `uiA0' as a 128-bit floating-point value, and likewise interpreting the\r\n| unsigned integer formed from concatenating `uiB64' and `uiB0' as another\r\n| 128-bit floating-point value, and assuming at least on of these floating-\r\n| point values is a NaN, returns the bit pattern of the combined NaN result.\r\n| If either original floating-point value is a signaling NaN, the invalid\r\n| exception is raised.\r\n*----------------------------------------------------------------------------*/\r\nstruct uint128\r\n softfloat_propagateNaNF128UI(\r\n struct softfloat_state *state,\r\n uint_fast64_t uiA64,\r\n uint_fast64_t uiA0,\r\n uint_fast64_t uiB64,\r\n uint_fast64_t uiB0\r\n )\r\n{\r\n bool isSigNaNA;\r\n struct uint128 uiZ;\r\n\r\n isSigNaNA = softfloat_isSigNaNF128UI( uiA64, uiA0 );\r\n if ( isSigNaNA || softfloat_isSigNaNF128UI( uiB64, uiB0 ) ) {\r\n softfloat_raiseFlags( state, softfloat_flag_invalid );\r\n if ( isSigNaNA ) goto returnNonsigA;\r\n }\r\n if ( isNaNF128UI( uiA64, uiA0 ) ) {\r\n returnNonsigA:\r\n uiZ.v64 = uiA64;\r\n uiZ.v0 = uiA0;\r\n } else {\r\n uiZ.v64 = uiB64;\r\n uiZ.v0 = uiB0;\r\n }\r\n uiZ.v64 |= UINT64_C( 0x0000800000000000 );\r\n return uiZ;\r\n\r\n}\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/s_roundPackToExtF80.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014, 2015, 2017 The Regents of the University of\r\nCalifornia. All rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \r\n#include \"platform.h\"\r\n#include \"internals.h\"\r\n#include \"softfloat.h\"\r\n\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nextFloat80_t\r\n softfloat_roundPackToExtF80(\r\n struct softfloat_state *state,\r\n bool sign,\r\n int_fast32_t exp,\r\n uint_fast64_t sig,\r\n uint_fast64_t sigExtra,\r\n uint_fast8_t roundingPrecision\r\n )\r\n{\r\n uint_fast8_t roundingMode;\r\n bool roundNearEven;\r\n uint_fast64_t roundIncrement, roundMask, roundBits;\r\n bool isTiny, doIncrement;\r\n struct uint64_extra sig64Extra;\r\n union { struct extFloat80M s; extFloat80_t f; } uZ;\r\n\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n roundingMode = state->roundingMode;\r\n roundNearEven = (roundingMode == softfloat_round_near_even);\r\n if ( roundingPrecision == 80 ) goto precision80;\r\n if ( roundingPrecision == 64 ) {\r\n roundIncrement = UINT64_C( 0x0000000000000400 );\r\n roundMask = UINT64_C( 0x00000000000007FF );\r\n } else if ( roundingPrecision == 32 ) {\r\n roundIncrement = UINT64_C( 0x0000008000000000 );\r\n roundMask = UINT64_C( 0x000000FFFFFFFFFF );\r\n } else {\r\n goto precision80;\r\n }\r\n sig |= (sigExtra != 0);\r\n if ( ! roundNearEven && (roundingMode != softfloat_round_near_maxMag) ) {\r\n roundIncrement =\r\n (roundingMode\r\n == (sign ? softfloat_round_min : softfloat_round_max))\r\n ? roundMask\r\n : 0;\r\n }\r\n roundBits = sig & roundMask;\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n if ( 0x7FFD <= (uint32_t) (exp - 1) ) {\r\n if ( exp <= 0 ) {\r\n /*----------------------------------------------------------------\r\n *----------------------------------------------------------------*/\r\n isTiny =\r\n (state->detectTininess\r\n == softfloat_tininess_beforeRounding)\r\n || (exp < 0)\r\n || (sig <= (uint64_t) (sig + roundIncrement));\r\n sig = softfloat_shiftRightJam64( sig, 1 - exp );\r\n roundBits = sig & roundMask;\r\n if ( roundBits ) {\r\n if ( isTiny ) softfloat_raiseFlags( state, softfloat_flag_underflow );\r\n state->exceptionFlags |= softfloat_flag_inexact;\r\n#ifdef SOFTFLOAT_ROUND_ODD\r\n if ( roundingMode == softfloat_round_odd ) {\r\n sig |= roundMask + 1;\r\n }\r\n#endif\r\n }\r\n sig += roundIncrement;\r\n exp = ((sig & UINT64_C( 0x8000000000000000 )) != 0);\r\n roundIncrement = roundMask + 1;\r\n if ( roundNearEven && (roundBits<<1 == roundIncrement) ) {\r\n roundMask |= roundIncrement;\r\n }\r\n sig &= ~roundMask;\r\n goto packReturn;\r\n }\r\n if (\r\n (0x7FFE < exp)\r\n || ((exp == 0x7FFE) && ((uint64_t) (sig + roundIncrement) < sig))\r\n ) {\r\n goto overflow;\r\n }\r\n }\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n if ( roundBits ) {\r\n state->exceptionFlags |= softfloat_flag_inexact;\r\n#ifdef SOFTFLOAT_ROUND_ODD\r\n if ( roundingMode == softfloat_round_odd ) {\r\n sig = (sig & ~roundMask) | (roundMask + 1);\r\n goto packReturn;\r\n }\r\n#endif\r\n }\r\n sig = (uint64_t) (sig + roundIncrement);\r\n if ( sig < roundIncrement ) {\r\n ++exp;\r\n sig = UINT64_C( 0x8000000000000000 );\r\n }\r\n roundIncrement = roundMask + 1;\r\n if ( roundNearEven && (roundBits<<1 == roundIncrement) ) {\r\n roundMask |= roundIncrement;\r\n }\r\n sig &= ~roundMask;\r\n goto packReturn;\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n precision80:\r\n doIncrement = (UINT64_C( 0x8000000000000000 ) <= sigExtra);\r\n if ( ! roundNearEven && (roundingMode != softfloat_round_near_maxMag) ) {\r\n doIncrement =\r\n (roundingMode\r\n == (sign ? softfloat_round_min : softfloat_round_max))\r\n && sigExtra;\r\n }\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n if ( 0x7FFD <= (uint32_t) (exp - 1) ) {\r\n if ( exp <= 0 ) {\r\n /*----------------------------------------------------------------\r\n *----------------------------------------------------------------*/\r\n isTiny =\r\n (state->detectTininess\r\n == softfloat_tininess_beforeRounding)\r\n || (exp < 0)\r\n || ! doIncrement\r\n || (sig < UINT64_C( 0xFFFFFFFFFFFFFFFF ));\r\n sig64Extra =\r\n softfloat_shiftRightJam64Extra( sig, sigExtra, 1 - exp );\r\n exp = 0;\r\n sig = sig64Extra.v;\r\n sigExtra = sig64Extra.extra;\r\n if ( sigExtra ) {\r\n if ( isTiny ) softfloat_raiseFlags( state, softfloat_flag_underflow );\r\n state->exceptionFlags |= softfloat_flag_inexact;\r\n#ifdef SOFTFLOAT_ROUND_ODD\r\n if ( roundingMode == softfloat_round_odd ) {\r\n sig |= 1;\r\n goto packReturn;\r\n }\r\n#endif\r\n }\r\n doIncrement = (UINT64_C( 0x8000000000000000 ) <= sigExtra);\r\n if (\r\n ! roundNearEven\r\n && (roundingMode != softfloat_round_near_maxMag)\r\n ) {\r\n doIncrement =\r\n (roundingMode\r\n == (sign ? softfloat_round_min : softfloat_round_max))\r\n && sigExtra;\r\n }\r\n if ( doIncrement ) {\r\n ++sig;\r\n sig &=\r\n ~(uint_fast64_t)\r\n (! (sigExtra & UINT64_C( 0x7FFFFFFFFFFFFFFF ))\r\n & roundNearEven);\r\n exp = ((sig & UINT64_C( 0x8000000000000000 )) != 0);\r\n }\r\n goto packReturn;\r\n }\r\n if (\r\n (0x7FFE < exp)\r\n || ((exp == 0x7FFE) && (sig == UINT64_C( 0xFFFFFFFFFFFFFFFF ))\r\n && doIncrement)\r\n ) {\r\n /*----------------------------------------------------------------\r\n *----------------------------------------------------------------*/\r\n roundMask = 0;\r\n overflow:\r\n softfloat_raiseFlags(\r\n state, softfloat_flag_overflow | softfloat_flag_inexact );\r\n if (\r\n roundNearEven\r\n || (roundingMode == softfloat_round_near_maxMag)\r\n || (roundingMode\r\n == (sign ? softfloat_round_min : softfloat_round_max))\r\n ) {\r\n exp = 0x7FFF;\r\n sig = UINT64_C( 0x8000000000000000 );\r\n } else {\r\n exp = 0x7FFE;\r\n sig = ~roundMask;\r\n }\r\n goto packReturn;\r\n }\r\n }\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n if ( sigExtra ) {\r\n state->exceptionFlags |= softfloat_flag_inexact;\r\n#ifdef SOFTFLOAT_ROUND_ODD\r\n if ( roundingMode == softfloat_round_odd ) {\r\n sig |= 1;\r\n goto packReturn;\r\n }\r\n#endif\r\n }\r\n if ( doIncrement ) {\r\n ++sig;\r\n if ( ! sig ) {\r\n ++exp;\r\n sig = UINT64_C( 0x8000000000000000 );\r\n } else {\r\n sig &=\r\n ~(uint_fast64_t)\r\n (! (sigExtra & UINT64_C( 0x7FFFFFFFFFFFFFFF ))\r\n & roundNearEven);\r\n }\r\n }\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n packReturn:\r\n uZ.s.signExp = packToExtF80UI64( sign, exp );\r\n uZ.s.signif = sig;\r\n return uZ.f;\r\n\r\n}\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/s_roundPackToF128.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014, 2015, 2017 The Regents of the University of\r\nCalifornia. All rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \r\n#include \"platform.h\"\r\n#include \"internals.h\"\r\n#include \"softfloat.h\"\r\n\r\nfloat128_t\r\n softfloat_roundPackToF128(\r\n struct softfloat_state *state,\r\n bool sign,\r\n int_fast32_t exp,\r\n uint_fast64_t sig64,\r\n uint_fast64_t sig0,\r\n uint_fast64_t sigExtra\r\n )\r\n{\r\n uint_fast8_t roundingMode;\r\n bool roundNearEven, doIncrement, isTiny;\r\n struct uint128_extra sig128Extra;\r\n uint_fast64_t uiZ64, uiZ0;\r\n struct uint128 sig128;\r\n union ui128_f128 uZ;\r\n\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n roundingMode = state->roundingMode;\r\n roundNearEven = (roundingMode == softfloat_round_near_even);\r\n doIncrement = (UINT64_C( 0x8000000000000000 ) <= sigExtra);\r\n if ( ! roundNearEven && (roundingMode != softfloat_round_near_maxMag) ) {\r\n doIncrement =\r\n (roundingMode\r\n == (sign ? softfloat_round_min : softfloat_round_max))\r\n && sigExtra;\r\n }\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n if ( 0x7FFD <= (uint32_t) exp ) {\r\n if ( exp < 0 ) {\r\n /*----------------------------------------------------------------\r\n *----------------------------------------------------------------*/\r\n isTiny =\r\n (state->detectTininess\r\n == softfloat_tininess_beforeRounding)\r\n || (exp < -1)\r\n || ! doIncrement\r\n || softfloat_lt128(\r\n sig64,\r\n sig0,\r\n UINT64_C( 0x0001FFFFFFFFFFFF ),\r\n UINT64_C( 0xFFFFFFFFFFFFFFFF )\r\n );\r\n sig128Extra =\r\n softfloat_shiftRightJam128Extra( sig64, sig0, sigExtra, -exp );\r\n sig64 = sig128Extra.v.v64;\r\n sig0 = sig128Extra.v.v0;\r\n sigExtra = sig128Extra.extra;\r\n exp = 0;\r\n if ( isTiny && sigExtra ) {\r\n softfloat_raiseFlags( state, softfloat_flag_underflow );\r\n }\r\n doIncrement = (UINT64_C( 0x8000000000000000 ) <= sigExtra);\r\n if (\r\n ! roundNearEven\r\n && (roundingMode != softfloat_round_near_maxMag)\r\n ) {\r\n doIncrement =\r\n (roundingMode\r\n == (sign ? softfloat_round_min : softfloat_round_max))\r\n && sigExtra;\r\n }\r\n } else if (\r\n (0x7FFD < exp)\r\n || ((exp == 0x7FFD)\r\n && softfloat_eq128( \r\n sig64,\r\n sig0,\r\n UINT64_C( 0x0001FFFFFFFFFFFF ),\r\n UINT64_C( 0xFFFFFFFFFFFFFFFF )\r\n )\r\n && doIncrement)\r\n ) {\r\n /*----------------------------------------------------------------\r\n *----------------------------------------------------------------*/\r\n softfloat_raiseFlags(\r\n state, softfloat_flag_overflow | softfloat_flag_inexact );\r\n if (\r\n roundNearEven\r\n || (roundingMode == softfloat_round_near_maxMag)\r\n || (roundingMode\r\n == (sign ? softfloat_round_min : softfloat_round_max))\r\n ) {\r\n uiZ64 = packToF128UI64( sign, 0x7FFF, 0 );\r\n uiZ0 = 0;\r\n } else {\r\n uiZ64 =\r\n packToF128UI64(\r\n sign, 0x7FFE, UINT64_C( 0x0000FFFFFFFFFFFF ) );\r\n uiZ0 = UINT64_C( 0xFFFFFFFFFFFFFFFF );\r\n }\r\n goto uiZ;\r\n }\r\n }\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n if ( sigExtra ) {\r\n state->exceptionFlags |= softfloat_flag_inexact;\r\n#ifdef SOFTFLOAT_ROUND_ODD\r\n if ( roundingMode == softfloat_round_odd ) {\r\n sig0 |= 1;\r\n goto packReturn;\r\n }\r\n#endif\r\n }\r\n if ( doIncrement ) {\r\n sig128 = softfloat_add128( sig64, sig0, 0, 1 );\r\n sig64 = sig128.v64;\r\n sig0 =\r\n sig128.v0\r\n & ~(uint64_t)\r\n (! (sigExtra & UINT64_C( 0x7FFFFFFFFFFFFFFF ))\r\n & roundNearEven);\r\n } else {\r\n if ( ! (sig64 | sig0) ) exp = 0;\r\n }\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n uiZ64 = packToF128UI64( sign, exp, sig64 );\r\n uiZ0 = sig0;\r\n uiZ:\r\n uZ.ui.v64 = uiZ64;\r\n uZ.ui.v0 = uiZ0;\r\n return uZ.f;\r\n\r\n}\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/s_roundPackToF32.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014, 2015, 2017 The Regents of the University of\r\nCalifornia. All rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \r\n#include \"platform.h\"\r\n#include \"internals.h\"\r\n#include \"softfloat.h\"\r\n\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nfloat32_t\r\n softfloat_roundPackToF32( struct softfloat_state *state, bool sign, int_fast16_t exp, uint_fast32_t sig )\r\n{\r\n uint_fast8_t roundingMode;\r\n bool roundNearEven;\r\n uint_fast8_t roundIncrement, roundBits;\r\n bool isTiny;\r\n uint_fast32_t uiZ;\r\n union ui32_f32 uZ;\r\n\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n roundingMode = state->roundingMode;\r\n roundNearEven = (roundingMode == softfloat_round_near_even);\r\n roundIncrement = 0x40;\r\n if ( ! roundNearEven && (roundingMode != softfloat_round_near_maxMag) ) {\r\n roundIncrement =\r\n (roundingMode\r\n == (sign ? softfloat_round_min : softfloat_round_max))\r\n ? 0x7F\r\n : 0;\r\n }\r\n roundBits = sig & 0x7F;\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n if ( 0xFD <= (unsigned int) exp ) {\r\n if ( exp < 0 ) {\r\n /*----------------------------------------------------------------\r\n *----------------------------------------------------------------*/\r\n isTiny =\r\n (state->detectTininess == softfloat_tininess_beforeRounding)\r\n || (exp < -1) || (sig + roundIncrement < 0x80000000);\r\n sig = softfloat_shiftRightJam32( sig, -exp );\r\n exp = 0;\r\n roundBits = sig & 0x7F;\r\n if ( isTiny && roundBits ) {\r\n softfloat_raiseFlags( state, softfloat_flag_underflow );\r\n }\r\n } else if ( (0xFD < exp) || (0x80000000 <= sig + roundIncrement) ) {\r\n /*----------------------------------------------------------------\r\n *----------------------------------------------------------------*/\r\n softfloat_raiseFlags(\r\n state, softfloat_flag_overflow | softfloat_flag_inexact );\r\n uiZ = packToF32UI( sign, 0xFF, 0 ) - ! roundIncrement;\r\n goto uiZ;\r\n }\r\n }\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n sig = (sig + roundIncrement)>>7;\r\n if ( roundBits ) {\r\n state->exceptionFlags |= softfloat_flag_inexact;\r\n#ifdef SOFTFLOAT_ROUND_ODD\r\n if ( roundingMode == softfloat_round_odd ) {\r\n sig |= 1;\r\n goto packReturn;\r\n }\r\n#endif\r\n }\r\n sig &= ~(uint_fast32_t) (! (roundBits ^ 0x40) & roundNearEven);\r\n if ( ! sig ) exp = 0;\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n#ifdef SOFTFLOAT_ROUND_ODD\r\n packReturn:\r\n#endif\r\n uiZ = packToF32UI( sign, exp, sig );\r\n uiZ:\r\n uZ.ui = uiZ;\r\n return uZ.f;\r\n\r\n}\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/s_roundPackToF64.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014, 2015, 2017 The Regents of the University of\r\nCalifornia. All rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \r\n#include \"platform.h\"\r\n#include \"internals.h\"\r\n#include \"softfloat.h\"\r\n\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nfloat64_t\r\n softfloat_roundPackToF64( struct softfloat_state *state, bool sign, int_fast16_t exp, uint_fast64_t sig )\r\n{\r\n uint_fast8_t roundingMode;\r\n bool roundNearEven;\r\n uint_fast16_t roundIncrement, roundBits;\r\n bool isTiny;\r\n uint_fast64_t uiZ;\r\n union ui64_f64 uZ;\r\n\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n roundingMode = state->roundingMode;\r\n roundNearEven = (roundingMode == softfloat_round_near_even);\r\n roundIncrement = 0x200;\r\n if ( ! roundNearEven && (roundingMode != softfloat_round_near_maxMag) ) {\r\n roundIncrement =\r\n (roundingMode\r\n == (sign ? softfloat_round_min : softfloat_round_max))\r\n ? 0x3FF\r\n : 0;\r\n }\r\n roundBits = sig & 0x3FF;\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n if ( 0x7FD <= (uint16_t) exp ) {\r\n if ( exp < 0 ) {\r\n /*----------------------------------------------------------------\r\n *----------------------------------------------------------------*/\r\n isTiny =\r\n (state->detectTininess == softfloat_tininess_beforeRounding)\r\n || (exp < -1)\r\n || (sig + roundIncrement < UINT64_C( 0x8000000000000000 ));\r\n sig = softfloat_shiftRightJam64( sig, -exp );\r\n exp = 0;\r\n roundBits = sig & 0x3FF;\r\n if ( isTiny && roundBits ) {\r\n softfloat_raiseFlags( state, softfloat_flag_underflow );\r\n }\r\n } else if (\r\n (0x7FD < exp)\r\n || (UINT64_C( 0x8000000000000000 ) <= sig + roundIncrement)\r\n ) {\r\n /*----------------------------------------------------------------\r\n *----------------------------------------------------------------*/\r\n softfloat_raiseFlags(\r\n state, softfloat_flag_overflow | softfloat_flag_inexact );\r\n uiZ = packToF64UI( sign, 0x7FF, 0 ) - ! roundIncrement;\r\n goto uiZ;\r\n }\r\n }\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n sig = (sig + roundIncrement)>>10;\r\n if ( roundBits ) {\r\n state->exceptionFlags |= softfloat_flag_inexact;\r\n#ifdef SOFTFLOAT_ROUND_ODD\r\n if ( roundingMode == softfloat_round_odd ) {\r\n sig |= 1;\r\n goto packReturn;\r\n }\r\n#endif\r\n }\r\n sig &= ~(uint_fast64_t) (! (roundBits ^ 0x200) & roundNearEven);\r\n if ( ! sig ) exp = 0;\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n#ifdef SOFTFLOAT_ROUND_ODD\r\n packReturn:\r\n#endif\r\n uiZ = packToF64UI( sign, exp, sig );\r\n uiZ:\r\n uZ.ui = uiZ;\r\n return uZ.f;\r\n\r\n}\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/s_roundToI32.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014, 2015, 2016, 2017 The Regents of the\r\nUniversity of California. All rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \r\n#include \"platform.h\"\r\n#include \"internals.h\"\r\n#include \"specialize.h\"\r\n#include \"softfloat.h\"\r\n\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nint_fast32_t\r\n softfloat_roundToI32(\r\n struct softfloat_state *state, bool sign, uint_fast64_t sig, uint_fast8_t roundingMode, bool exact )\r\n{\r\n uint_fast16_t roundIncrement, roundBits;\r\n uint_fast32_t sig32;\r\n union { uint32_t ui; int32_t i; } uZ;\r\n int_fast32_t z;\r\n\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n roundIncrement = 0x800;\r\n if (\r\n (roundingMode != softfloat_round_near_maxMag)\r\n && (roundingMode != softfloat_round_near_even)\r\n ) {\r\n roundIncrement = 0;\r\n if ( \r\n sign\r\n ? (roundingMode == softfloat_round_min)\r\n#ifdef SOFTFLOAT_ROUND_ODD\r\n || (roundingMode == softfloat_round_odd)\r\n#endif\r\n : (roundingMode == softfloat_round_max)\r\n ) {\r\n roundIncrement = 0xFFF;\r\n }\r\n }\r\n roundBits = sig & 0xFFF;\r\n sig += roundIncrement;\r\n if ( sig & UINT64_C( 0xFFFFF00000000000 ) ) goto invalid;\r\n sig32 = sig>>12;\r\n if (\r\n (roundBits == 0x800) && (roundingMode == softfloat_round_near_even)\r\n ) {\r\n sig32 &= ~(uint_fast32_t) 1;\r\n }\r\n uZ.ui = sign ? -sig32 : sig32;\r\n z = uZ.i;\r\n if ( z && ((z < 0) ^ sign) ) goto invalid;\r\n if ( roundBits ) {\r\n#ifdef SOFTFLOAT_ROUND_ODD\r\n if ( roundingMode == softfloat_round_odd ) z |= 1;\r\n#endif\r\n if ( exact ) state->exceptionFlags |= softfloat_flag_inexact;\r\n }\r\n return z;\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n invalid:\r\n softfloat_raiseFlags( state, softfloat_flag_invalid );\r\n return sign ? i32_fromNegOverflow : i32_fromPosOverflow;\r\n\r\n}\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/s_roundToI64.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014, 2015, 2016, 2017 The Regents of the\r\nUniversity of California. All rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \r\n#include \"platform.h\"\r\n#include \"internals.h\"\r\n#include \"specialize.h\"\r\n#include \"softfloat.h\"\r\n\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nint_fast64_t\r\n softfloat_roundToI64(\r\n struct softfloat_state *state,\r\n bool sign,\r\n uint_fast64_t sig,\r\n uint_fast64_t sigExtra,\r\n uint_fast8_t roundingMode,\r\n bool exact\r\n )\r\n{\r\n union { uint64_t ui; int64_t i; } uZ;\r\n int_fast64_t z;\r\n\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n if (\r\n (roundingMode == softfloat_round_near_maxMag)\r\n || (roundingMode == softfloat_round_near_even)\r\n ) {\r\n if ( UINT64_C( 0x8000000000000000 ) <= sigExtra ) goto increment;\r\n } else {\r\n if (\r\n sigExtra\r\n && (sign\r\n ? (roundingMode == softfloat_round_min)\r\n#ifdef SOFTFLOAT_ROUND_ODD\r\n || (roundingMode == softfloat_round_odd)\r\n#endif\r\n : (roundingMode == softfloat_round_max))\r\n ) {\r\n increment:\r\n ++sig;\r\n if ( !sig ) goto invalid;\r\n if (\r\n (sigExtra == UINT64_C( 0x8000000000000000 ))\r\n && (roundingMode == softfloat_round_near_even)\r\n ) {\r\n sig &= ~(uint_fast64_t) 1;\r\n }\r\n }\r\n }\r\n uZ.ui = sign ? -sig : sig;\r\n z = uZ.i;\r\n if ( z && ((z < 0) ^ sign) ) goto invalid;\r\n if ( sigExtra ) {\r\n#ifdef SOFTFLOAT_ROUND_ODD\r\n if ( roundingMode == softfloat_round_odd ) z |= 1;\r\n#endif\r\n if ( exact ) state->exceptionFlags |= softfloat_flag_inexact;\r\n }\r\n return z;\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n invalid:\r\n softfloat_raiseFlags( state, softfloat_flag_invalid );\r\n return sign ? i64_fromNegOverflow : i64_fromPosOverflow;\r\n\r\n}\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/s_roundToUI64.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014, 2015, 2016, 2017 The Regents of the\r\nUniversity of California. All rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \r\n#include \"platform.h\"\r\n#include \"internals.h\"\r\n#include \"specialize.h\"\r\n#include \"softfloat.h\"\r\n\r\nuint_fast64_t\r\n softfloat_roundToUI64(\r\n struct softfloat_state *state,\r\n bool sign,\r\n uint_fast64_t sig,\r\n uint_fast64_t sigExtra,\r\n uint_fast8_t roundingMode,\r\n bool exact\r\n )\r\n{\r\n\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n if (\r\n (roundingMode == softfloat_round_near_maxMag)\r\n || (roundingMode == softfloat_round_near_even)\r\n ) {\r\n if ( UINT64_C( 0x8000000000000000 ) <= sigExtra ) goto increment;\r\n } else {\r\n if ( sign ) {\r\n if ( !(sig | sigExtra) ) return 0;\r\n if ( roundingMode == softfloat_round_min ) goto invalid;\r\n#ifdef SOFTFLOAT_ROUND_ODD\r\n if ( roundingMode == softfloat_round_odd ) goto invalid;\r\n#endif\r\n } else {\r\n if ( (roundingMode == softfloat_round_max) && sigExtra ) {\r\n increment:\r\n ++sig;\r\n if ( !sig ) goto invalid;\r\n if ( \r\n (sigExtra == UINT64_C( 0x8000000000000000 ))\r\n && (roundingMode == softfloat_round_near_even)\r\n ) {\r\n sig &= ~(uint_fast64_t) 1;\r\n }\r\n }\r\n }\r\n }\r\n if ( sign && sig ) goto invalid;\r\n if ( sigExtra ) {\r\n#ifdef SOFTFLOAT_ROUND_ODD\r\n if ( roundingMode == softfloat_round_odd ) sig |= 1;\r\n#endif\r\n if ( exact ) state->exceptionFlags |= softfloat_flag_inexact;\r\n }\r\n return sig;\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n invalid:\r\n softfloat_raiseFlags( state, softfloat_flag_invalid );\r\n return sign ? ui64_fromNegOverflow : ui64_fromPosOverflow;\r\n\r\n}\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/s_shiftRightJam128.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014, 2015, 2016 The Regents of the University of\r\nCalifornia. All rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \"platform.h\"\r\n#include \"primitiveTypes.h\"\r\n\r\n#ifndef softfloat_shiftRightJam128\r\n\r\nstruct uint128\r\n softfloat_shiftRightJam128( uint64_t a64, uint64_t a0, uint_fast32_t dist )\r\n{\r\n uint_fast8_t u8NegDist;\r\n struct uint128 z;\r\n\r\n if ( dist < 64 ) {\r\n u8NegDist = -dist;\r\n z.v64 = a64>>dist;\r\n z.v0 =\r\n a64<<(u8NegDist & 63) | a0>>dist\r\n | ((uint64_t) (a0<<(u8NegDist & 63)) != 0);\r\n } else {\r\n z.v64 = 0;\r\n z.v0 =\r\n (dist < 127)\r\n ? a64>>(dist & 63)\r\n | (((a64 & (((uint_fast64_t) 1<<(dist & 63)) - 1)) | a0)\r\n != 0)\r\n : ((a64 | a0) != 0);\r\n }\r\n return z;\r\n\r\n}\r\n\r\n#endif\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/s_shiftRightJam128Extra.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014, 2015, 2016 The Regents of the University of\r\nCalifornia. All rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \"platform.h\"\r\n#include \"primitiveTypes.h\"\r\n\r\n#ifndef softfloat_shiftRightJam128Extra\r\n\r\nstruct uint128_extra\r\n softfloat_shiftRightJam128Extra(\r\n uint64_t a64, uint64_t a0, uint64_t extra, uint_fast32_t dist )\r\n{\r\n uint_fast8_t u8NegDist;\r\n struct uint128_extra z;\r\n\r\n u8NegDist = -dist;\r\n if ( dist < 64 ) {\r\n z.v.v64 = a64>>dist;\r\n z.v.v0 = a64<<(u8NegDist & 63) | a0>>dist;\r\n z.extra = a0<<(u8NegDist & 63);\r\n } else {\r\n z.v.v64 = 0;\r\n if ( dist == 64 ) {\r\n z.v.v0 = a64;\r\n z.extra = a0;\r\n } else {\r\n extra |= a0;\r\n if ( dist < 128 ) {\r\n z.v.v0 = a64>>(dist & 63);\r\n z.extra = a64<<(u8NegDist & 63);\r\n } else {\r\n z.v.v0 = 0;\r\n z.extra = (dist == 128) ? a64 : (a64 != 0);\r\n }\r\n }\r\n }\r\n z.extra |= (extra != 0);\r\n return z;\r\n\r\n}\r\n\r\n#endif\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/s_shiftRightJam32.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014, 2015, 2016 The Regents of the University of\r\nCalifornia. All rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \"platform.h\"\r\n\r\n#ifndef softfloat_shiftRightJam32\r\n\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nuint32_t softfloat_shiftRightJam32( uint32_t a, uint_fast16_t dist )\r\n{\r\n\r\n return\r\n (dist < 31) ? a>>dist | ((uint32_t) (a<<(-dist & 31)) != 0) : (a != 0);\r\n\r\n}\r\n\r\n#endif\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/s_shiftRightJam64.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014, 2015, 2016 The Regents of the University of\r\nCalifornia. All rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \"platform.h\"\r\n\r\n#ifndef softfloat_shiftRightJam64\r\n\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nuint64_t softfloat_shiftRightJam64( uint64_t a, uint_fast32_t dist )\r\n{\r\n\r\n return\r\n (dist < 63) ? a>>dist | ((uint64_t) (a<<(-dist & 63)) != 0) : (a != 0);\r\n\r\n}\r\n\r\n#endif\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/s_shiftRightJam64Extra.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014, 2015, 2016 The Regents of the University of\r\nCalifornia. All rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \"platform.h\"\r\n#include \"primitiveTypes.h\"\r\n\r\n#ifndef softfloat_shiftRightJam64Extra\r\n\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nstruct uint64_extra\r\n softfloat_shiftRightJam64Extra(\r\n uint64_t a, uint64_t extra, uint_fast32_t dist )\r\n{\r\n struct uint64_extra z;\r\n\r\n if ( dist < 64 ) {\r\n z.v = a>>dist;\r\n z.extra = a<<(-dist & 63);\r\n } else {\r\n z.v = 0;\r\n z.extra = (dist == 64) ? a : (a != 0);\r\n }\r\n z.extra |= (extra != 0);\r\n return z;\r\n\r\n}\r\n\r\n#endif\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/s_shortShiftLeft128.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014, 2015, 2016 The Regents of the University of\r\nCalifornia. All rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \"platform.h\"\r\n#include \"primitiveTypes.h\"\r\n\r\n#ifndef softfloat_shortShiftLeft128\r\n\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nstruct uint128\r\n softfloat_shortShiftLeft128( uint64_t a64, uint64_t a0, uint_fast8_t dist )\r\n{\r\n struct uint128 z;\r\n\r\n z.v64 = a64<>(-dist & 63);\r\n z.v0 = a0<\r\n#include \"platform.h\"\r\n#include \"primitiveTypes.h\"\r\n\r\n#ifndef softfloat_shortShiftRight128\r\n\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nstruct uint128\r\n softfloat_shortShiftRight128( uint64_t a64, uint64_t a0, uint_fast8_t dist )\r\n{\r\n struct uint128 z;\r\n\r\n z.v64 = a64>>dist;\r\n z.v0 = a64<<(-dist & 63) | a0>>dist;\r\n return z;\r\n\r\n}\r\n\r\n#endif\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/s_shortShiftRightJam64.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014, 2015, 2016 The Regents of the University of\r\nCalifornia. All rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \"platform.h\"\r\n\r\n#ifndef softfloat_shortShiftRightJam64\r\n\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nuint64_t softfloat_shortShiftRightJam64( uint64_t a, uint_fast8_t dist )\r\n{\r\n\r\n return a>>dist | ((a & (((uint_fast64_t) 1<\r\n#include \"platform.h\"\r\n#include \"primitiveTypes.h\"\r\n\r\n#ifndef softfloat_shortShiftRightJam64Extra\r\n\r\nstruct uint64_extra\r\n softfloat_shortShiftRightJam64Extra(\r\n uint64_t a, uint64_t extra, uint_fast8_t dist )\r\n{\r\n struct uint64_extra z;\r\n\r\n z.v = a>>dist;\r\n z.extra = a<<(-dist & 63) | (extra != 0);\r\n return z;\r\n\r\n}\r\n\r\n#endif\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/s_sub128.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014, 2015 The Regents of the University of\r\nCalifornia. All rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \"platform.h\"\r\n#include \"primitiveTypes.h\"\r\n\r\n#ifndef softfloat_sub128\r\n\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nstruct uint128\r\n softfloat_sub128( uint64_t a64, uint64_t a0, uint64_t b64, uint64_t b0 )\r\n{\r\n struct uint128 z;\r\n\r\n z.v0 = a0 - b0;\r\n z.v64 = a64 - b64 - (a0 < b0);\r\n return z;\r\n\r\n}\r\n\r\n#endif\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/s_subMagsExtF80.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014, 2015 The Regents of the University of\r\nCalifornia. All rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \r\n#include \"platform.h\"\r\n#include \"internals.h\"\r\n#include \"specialize.h\"\r\n#include \"softfloat.h\"\r\n\r\nextFloat80_t\r\n softfloat_subMagsExtF80(\r\n struct softfloat_state *state,\r\n uint_fast16_t uiA64,\r\n uint_fast64_t uiA0,\r\n uint_fast16_t uiB64,\r\n uint_fast64_t uiB0,\r\n bool signZ\r\n )\r\n{\r\n int_fast32_t expA;\r\n uint_fast64_t sigA;\r\n int_fast32_t expB;\r\n uint_fast64_t sigB;\r\n int_fast32_t expDiff;\r\n uint_fast16_t uiZ64;\r\n uint_fast64_t uiZ0;\r\n int_fast32_t expZ;\r\n uint_fast64_t sigExtra;\r\n struct uint128 sig128, uiZ;\r\n union { struct extFloat80M s; extFloat80_t f; } uZ;\r\n\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n expA = expExtF80UI64( uiA64 );\r\n sigA = uiA0;\r\n expB = expExtF80UI64( uiB64 );\r\n sigB = uiB0;\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n expDiff = expA - expB;\r\n if ( 0 < expDiff ) goto expABigger;\r\n if ( expDiff < 0 ) goto expBBigger;\r\n if ( expA == 0x7FFF ) {\r\n if ( (sigA | sigB) & UINT64_C( 0x7FFFFFFFFFFFFFFF ) ) {\r\n goto propagateNaN;\r\n }\r\n softfloat_raiseFlags( state, softfloat_flag_invalid );\r\n uiZ64 = defaultNaNExtF80UI64;\r\n uiZ0 = defaultNaNExtF80UI0;\r\n goto uiZ;\r\n }\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n expZ = expA;\r\n if ( ! expZ ) expZ = 1;\r\n sigExtra = 0;\r\n if ( sigB < sigA ) goto aBigger;\r\n if ( sigA < sigB ) goto bBigger;\r\n uiZ64 =\r\n packToExtF80UI64( (state->roundingMode == softfloat_round_min), 0 );\r\n uiZ0 = 0;\r\n goto uiZ;\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n expBBigger:\r\n if ( expB == 0x7FFF ) {\r\n if ( sigB & UINT64_C( 0x7FFFFFFFFFFFFFFF ) ) goto propagateNaN;\r\n uiZ64 = packToExtF80UI64( signZ ^ 1, 0x7FFF );\r\n uiZ0 = UINT64_C( 0x8000000000000000 );\r\n goto uiZ;\r\n }\r\n if ( ! expA ) {\r\n ++expDiff;\r\n sigExtra = 0;\r\n if ( ! expDiff ) goto newlyAlignedBBigger;\r\n }\r\n sig128 = softfloat_shiftRightJam128( sigA, 0, -expDiff );\r\n sigA = sig128.v64;\r\n sigExtra = sig128.v0;\r\n newlyAlignedBBigger:\r\n expZ = expB;\r\n bBigger:\r\n signZ = ! signZ;\r\n sig128 = softfloat_sub128( sigB, 0, sigA, sigExtra );\r\n goto normRoundPack;\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n expABigger:\r\n if ( expA == 0x7FFF ) {\r\n if ( sigA & UINT64_C( 0x7FFFFFFFFFFFFFFF ) ) goto propagateNaN;\r\n uiZ64 = uiA64;\r\n uiZ0 = uiA0;\r\n goto uiZ;\r\n }\r\n if ( ! expB ) {\r\n --expDiff;\r\n sigExtra = 0;\r\n if ( ! expDiff ) goto newlyAlignedABigger;\r\n }\r\n sig128 = softfloat_shiftRightJam128( sigB, 0, expDiff );\r\n sigB = sig128.v64;\r\n sigExtra = sig128.v0;\r\n newlyAlignedABigger:\r\n expZ = expA;\r\n aBigger:\r\n sig128 = softfloat_sub128( sigA, 0, sigB, sigExtra );\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n normRoundPack:\r\n return\r\n softfloat_normRoundPackToExtF80(\r\n state, signZ, expZ, sig128.v64, sig128.v0, state->roundingPrecision );\r\n /*------------------------------------------------------------------------\r\n *------------------------------------------------------------------------*/\r\n propagateNaN:\r\n uiZ = softfloat_propagateNaNExtF80UI( state, uiA64, uiA0, uiB64, uiB0 );\r\n uiZ64 = uiZ.v64;\r\n uiZ0 = uiZ.v0;\r\n uiZ:\r\n uZ.s.signExp = uiZ64;\r\n uZ.s.signif = uiZ0;\r\n return uZ.f;\r\n\r\n}\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/s_subMagsF128.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014, 2015 The Regents of the University of\r\nCalifornia. All rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \r\n#include \"platform.h\"\r\n#include \"internals.h\"\r\n#include \"specialize.h\"\r\n#include \"softfloat.h\"\r\n\r\nfloat128_t\r\n softfloat_subMagsF128(\r\n struct softfloat_state *state,\r\n uint_fast64_t uiA64,\r\n uint_fast64_t uiA0,\r\n uint_fast64_t uiB64,\r\n uint_fast64_t uiB0,\r\n bool signZ\r\n )\r\n{\r\n int_fast32_t expA;\r\n struct uint128 sigA;\r\n int_fast32_t expB;\r\n struct uint128 sigB, sigZ;\r\n int_fast32_t expDiff, expZ;\r\n struct uint128 uiZ;\r\n union ui128_f128 uZ;\r\n\r\n expA = expF128UI64( uiA64 );\r\n sigA.v64 = fracF128UI64( uiA64 );\r\n sigA.v0 = uiA0;\r\n expB = expF128UI64( uiB64 );\r\n sigB.v64 = fracF128UI64( uiB64 );\r\n sigB.v0 = uiB0;\r\n sigA = softfloat_shortShiftLeft128( sigA.v64, sigA.v0, 4 );\r\n sigB = softfloat_shortShiftLeft128( sigB.v64, sigB.v0, 4 );\r\n expDiff = expA - expB;\r\n if ( 0 < expDiff ) goto expABigger;\r\n if ( expDiff < 0 ) goto expBBigger;\r\n if ( expA == 0x7FFF ) {\r\n if ( sigA.v64 | sigA.v0 | sigB.v64 | sigB.v0 ) goto propagateNaN;\r\n softfloat_raiseFlags( state, softfloat_flag_invalid );\r\n uiZ.v64 = defaultNaNF128UI64;\r\n uiZ.v0 = defaultNaNF128UI0;\r\n goto uiZ;\r\n }\r\n expZ = expA;\r\n if ( ! expZ ) expZ = 1;\r\n if ( sigB.v64 < sigA.v64 ) goto aBigger;\r\n if ( sigA.v64 < sigB.v64 ) goto bBigger;\r\n if ( sigB.v0 < sigA.v0 ) goto aBigger;\r\n if ( sigA.v0 < sigB.v0 ) goto bBigger;\r\n uiZ.v64 =\r\n packToF128UI64(\r\n (state->roundingMode == softfloat_round_min), 0, 0 );\r\n uiZ.v0 = 0;\r\n goto uiZ;\r\n expBBigger:\r\n if ( expB == 0x7FFF ) {\r\n if ( sigB.v64 | sigB.v0 ) goto propagateNaN;\r\n uiZ.v64 = packToF128UI64( signZ ^ 1, 0x7FFF, 0 );\r\n uiZ.v0 = 0;\r\n goto uiZ;\r\n }\r\n if ( expA ) {\r\n sigA.v64 |= UINT64_C( 0x0010000000000000 );\r\n } else {\r\n ++expDiff;\r\n if ( ! expDiff ) goto newlyAlignedBBigger;\r\n }\r\n sigA = softfloat_shiftRightJam128( sigA.v64, sigA.v0, -expDiff );\r\n newlyAlignedBBigger:\r\n expZ = expB;\r\n sigB.v64 |= UINT64_C( 0x0010000000000000 );\r\n bBigger:\r\n signZ = ! signZ;\r\n sigZ = softfloat_sub128( sigB.v64, sigB.v0, sigA.v64, sigA.v0 );\r\n goto normRoundPack;\r\n expABigger:\r\n if ( expA == 0x7FFF ) {\r\n if ( sigA.v64 | sigA.v0 ) goto propagateNaN;\r\n uiZ.v64 = uiA64;\r\n uiZ.v0 = uiA0;\r\n goto uiZ;\r\n }\r\n if ( expB ) {\r\n sigB.v64 |= UINT64_C( 0x0010000000000000 );\r\n } else {\r\n --expDiff;\r\n if ( ! expDiff ) goto newlyAlignedABigger;\r\n }\r\n sigB = softfloat_shiftRightJam128( sigB.v64, sigB.v0, expDiff );\r\n newlyAlignedABigger:\r\n expZ = expA;\r\n sigA.v64 |= UINT64_C( 0x0010000000000000 );\r\n aBigger:\r\n sigZ = softfloat_sub128( sigA.v64, sigA.v0, sigB.v64, sigB.v0 );\r\n normRoundPack:\r\n return softfloat_normRoundPackToF128( state, signZ, expZ - 5, sigZ.v64, sigZ.v0 );\r\n propagateNaN:\r\n uiZ = softfloat_propagateNaNF128UI( state, uiA64, uiA0, uiB64, uiB0 );\r\n uiZ:\r\n uZ.ui = uiZ;\r\n return uZ.f;\r\n\r\n}\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/softfloat_raiseFlags.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014 The Regents of the University of California.\r\nAll rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \"platform.h\"\r\n#include \"softfloat.h\"\r\n\r\n/*----------------------------------------------------------------------------\r\n| Raises the exceptions specified by `flags'. Floating-point traps can be\r\n| defined here if desired. It is currently not possible for such a trap\r\n| to substitute a result value. If traps are not implemented, this routine\r\n| should be simply `softfloat_exceptionFlags |= flags;'.\r\n*----------------------------------------------------------------------------*/\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nvoid softfloat_raiseFlags( struct softfloat_state *state, uint_fast8_t flags )\r\n{\r\n\r\n state->exceptionFlags |= flags;\r\n\r\n}\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/specialize.h", "content": "\r\n/*============================================================================\r\n\r\nThis C header file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014, 2015, 2016, 2018 The Regents of the\r\nUniversity of California. All rights reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#ifndef specialize_h\r\n#define specialize_h 1\r\n\r\n#include \r\n#include \r\n#include \"primitiveTypes.h\"\r\n#include \"softfloat.h\"\r\n\r\n/*----------------------------------------------------------------------------\r\n| Default value for 'softfloat_detectTininess'.\r\n*----------------------------------------------------------------------------*/\r\n#define init_detectTininess softfloat_tininess_afterRounding\r\n\r\n/*----------------------------------------------------------------------------\r\n| The values to return on conversions to 32-bit integer formats that raise an\r\n| invalid exception.\r\n*----------------------------------------------------------------------------*/\r\n#define ui32_fromPosOverflow 0xFFFFFFFF\r\n#define ui32_fromNegOverflow 0xFFFFFFFF\r\n#define ui32_fromNaN 0xFFFFFFFF\r\n#define i32_fromPosOverflow (-0x7FFFFFFF - 1)\r\n#define i32_fromNegOverflow (-0x7FFFFFFF - 1)\r\n#define i32_fromNaN (-0x7FFFFFFF - 1)\r\n\r\n/*----------------------------------------------------------------------------\r\n| The values to return on conversions to 64-bit integer formats that raise an\r\n| invalid exception.\r\n*----------------------------------------------------------------------------*/\r\n#define ui64_fromPosOverflow UINT64_C( 0xFFFFFFFFFFFFFFFF )\r\n#define ui64_fromNegOverflow UINT64_C( 0xFFFFFFFFFFFFFFFF )\r\n#define ui64_fromNaN UINT64_C( 0xFFFFFFFFFFFFFFFF )\r\n#define i64_fromPosOverflow (-INT64_C( 0x7FFFFFFFFFFFFFFF ) - 1)\r\n#define i64_fromNegOverflow (-INT64_C( 0x7FFFFFFFFFFFFFFF ) - 1)\r\n#define i64_fromNaN (-INT64_C( 0x7FFFFFFFFFFFFFFF ) - 1)\r\n\r\n/*----------------------------------------------------------------------------\r\n| \"Common NaN\" structure, used to transfer NaN representations from one format\r\n| to another.\r\n*----------------------------------------------------------------------------*/\r\nstruct commonNaN {\r\n bool sign;\r\n#ifdef LITTLEENDIAN\r\n uint64_t v0, v64;\r\n#else\r\n uint64_t v64, v0;\r\n#endif\r\n};\r\n\r\n/*----------------------------------------------------------------------------\r\n| The bit pattern for a default generated 16-bit floating-point NaN.\r\n*----------------------------------------------------------------------------*/\r\n#define defaultNaNF16UI 0xFE00\r\n\r\n/*----------------------------------------------------------------------------\r\n| Returns true when 16-bit unsigned integer 'uiA' has the bit pattern of a\r\n| 16-bit floating-point signaling NaN.\r\n| Note: This macro evaluates its argument more than once.\r\n*----------------------------------------------------------------------------*/\r\n#define softfloat_isSigNaNF16UI( uiA ) ((((uiA) & 0x7E00) == 0x7C00) && ((uiA) & 0x01FF))\r\n\r\n/*----------------------------------------------------------------------------\r\n| Assuming 'uiA' has the bit pattern of a 16-bit floating-point NaN, converts\r\n| this NaN to the common NaN form, and stores the resulting common NaN at the\r\n| location pointed to by 'zPtr'. If the NaN is a signaling NaN, the invalid\r\n| exception is raised.\r\n*----------------------------------------------------------------------------*/\r\nvoid softfloat_f16UIToCommonNaN( uint_fast16_t uiA, struct commonNaN *zPtr );\r\n\r\n/*----------------------------------------------------------------------------\r\n| Converts the common NaN pointed to by 'aPtr' into a 16-bit floating-point\r\n| NaN, and returns the bit pattern of this value as an unsigned integer.\r\n*----------------------------------------------------------------------------*/\r\nuint_fast16_t softfloat_commonNaNToF16UI( const struct commonNaN *aPtr );\r\n\r\n/*----------------------------------------------------------------------------\r\n| Interpreting 'uiA' and 'uiB' as the bit patterns of two 16-bit floating-\r\n| point values, at least one of which is a NaN, returns the bit pattern of\r\n| the combined NaN result. If either 'uiA' or 'uiB' has the pattern of a\r\n| signaling NaN, the invalid exception is raised.\r\n*----------------------------------------------------------------------------*/\r\nuint_fast16_t\r\n softfloat_propagateNaNF16UI( uint_fast16_t uiA, uint_fast16_t uiB );\r\n\r\n/*----------------------------------------------------------------------------\r\n| The bit pattern for a default generated 32-bit floating-point NaN.\r\n*----------------------------------------------------------------------------*/\r\n#define defaultNaNF32UI 0xFFC00000\r\n\r\n/*----------------------------------------------------------------------------\r\n| Returns true when 32-bit unsigned integer 'uiA' has the bit pattern of a\r\n| 32-bit floating-point signaling NaN.\r\n| Note: This macro evaluates its argument more than once.\r\n*----------------------------------------------------------------------------*/\r\n#define softfloat_isSigNaNF32UI( uiA ) ((((uiA) & 0x7FC00000) == 0x7F800000) && ((uiA) & 0x003FFFFF))\r\n\r\n/*----------------------------------------------------------------------------\r\n| Assuming 'uiA' has the bit pattern of a 32-bit floating-point NaN, converts\r\n| this NaN to the common NaN form, and stores the resulting common NaN at the\r\n| location pointed to by 'zPtr'. If the NaN is a signaling NaN, the invalid\r\n| exception is raised.\r\n*----------------------------------------------------------------------------*/\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nvoid softfloat_f32UIToCommonNaN( struct softfloat_state *, uint_fast32_t uiA, struct commonNaN *zPtr );\r\n\r\n/*----------------------------------------------------------------------------\r\n| Converts the common NaN pointed to by 'aPtr' into a 32-bit floating-point\r\n| NaN, and returns the bit pattern of this value as an unsigned integer.\r\n*----------------------------------------------------------------------------*/\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nuint_fast32_t softfloat_commonNaNToF32UI( const struct commonNaN *aPtr );\r\n\r\n/*----------------------------------------------------------------------------\r\n| Interpreting 'uiA' and 'uiB' as the bit patterns of two 32-bit floating-\r\n| point values, at least one of which is a NaN, returns the bit pattern of\r\n| the combined NaN result. If either 'uiA' or 'uiB' has the pattern of a\r\n| signaling NaN, the invalid exception is raised.\r\n*----------------------------------------------------------------------------*/\r\nuint_fast32_t\r\n softfloat_propagateNaNF32UI( uint_fast32_t uiA, uint_fast32_t uiB );\r\n\r\n/*----------------------------------------------------------------------------\r\n| The bit pattern for a default generated 64-bit floating-point NaN.\r\n*----------------------------------------------------------------------------*/\r\n#define defaultNaNF64UI UINT64_C( 0xFFF8000000000000 )\r\n\r\n/*----------------------------------------------------------------------------\r\n| Returns true when 64-bit unsigned integer 'uiA' has the bit pattern of a\r\n| 64-bit floating-point signaling NaN.\r\n| Note: This macro evaluates its argument more than once.\r\n*----------------------------------------------------------------------------*/\r\n#define softfloat_isSigNaNF64UI( uiA ) ((((uiA) & UINT64_C( 0x7FF8000000000000 )) == UINT64_C( 0x7FF0000000000000 )) && ((uiA) & UINT64_C( 0x0007FFFFFFFFFFFF )))\r\n\r\n/*----------------------------------------------------------------------------\r\n| Assuming 'uiA' has the bit pattern of a 64-bit floating-point NaN, converts\r\n| this NaN to the common NaN form, and stores the resulting common NaN at the\r\n| location pointed to by 'zPtr'. If the NaN is a signaling NaN, the invalid\r\n| exception is raised.\r\n*----------------------------------------------------------------------------*/\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nvoid softfloat_f64UIToCommonNaN( struct softfloat_state *, uint_fast64_t uiA, struct commonNaN *zPtr );\r\n\r\n/*----------------------------------------------------------------------------\r\n| Converts the common NaN pointed to by 'aPtr' into a 64-bit floating-point\r\n| NaN, and returns the bit pattern of this value as an unsigned integer.\r\n*----------------------------------------------------------------------------*/\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nuint_fast64_t softfloat_commonNaNToF64UI( const struct commonNaN *aPtr );\r\n\r\n/*----------------------------------------------------------------------------\r\n| Interpreting 'uiA' and 'uiB' as the bit patterns of two 64-bit floating-\r\n| point values, at least one of which is a NaN, returns the bit pattern of\r\n| the combined NaN result. If either 'uiA' or 'uiB' has the pattern of a\r\n| signaling NaN, the invalid exception is raised.\r\n*----------------------------------------------------------------------------*/\r\nuint_fast64_t\r\n softfloat_propagateNaNF64UI( uint_fast64_t uiA, uint_fast64_t uiB );\r\n\r\n/*----------------------------------------------------------------------------\r\n| The bit pattern for a default generated 80-bit extended floating-point NaN.\r\n*----------------------------------------------------------------------------*/\r\n#define defaultNaNExtF80UI64 0xFFFF\r\n#define defaultNaNExtF80UI0 UINT64_C( 0xC000000000000000 )\r\n\r\n/*----------------------------------------------------------------------------\r\n| Returns true when the 80-bit unsigned integer formed from concatenating\r\n| 16-bit 'uiA64' and 64-bit 'uiA0' has the bit pattern of an 80-bit extended\r\n| floating-point signaling NaN.\r\n| Note: This macro evaluates its arguments more than once.\r\n*----------------------------------------------------------------------------*/\r\n#define softfloat_isSigNaNExtF80UI( uiA64, uiA0 ) ((((uiA64) & 0x7FFF) == 0x7FFF) && ! ((uiA0) & UINT64_C( 0x4000000000000000 )) && ((uiA0) & UINT64_C( 0x3FFFFFFFFFFFFFFF )))\r\n\r\n#ifdef SOFTFLOAT_FAST_INT64\r\n\r\n/*----------------------------------------------------------------------------\r\n| The following functions are needed only when 'SOFTFLOAT_FAST_INT64' is\r\n| defined.\r\n*----------------------------------------------------------------------------*/\r\n\r\n/*----------------------------------------------------------------------------\r\n| Assuming the unsigned integer formed from concatenating 'uiA64' and 'uiA0'\r\n| has the bit pattern of an 80-bit extended floating-point NaN, converts\r\n| this NaN to the common NaN form, and stores the resulting common NaN at the\r\n| location pointed to by 'zPtr'. If the NaN is a signaling NaN, the invalid\r\n| exception is raised.\r\n*----------------------------------------------------------------------------*/\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nvoid\r\n softfloat_extF80UIToCommonNaN(\r\n struct softfloat_state *, uint_fast16_t uiA64, uint_fast64_t uiA0, struct commonNaN *zPtr );\r\n\r\n/*----------------------------------------------------------------------------\r\n| Converts the common NaN pointed to by 'aPtr' into an 80-bit extended\r\n| floating-point NaN, and returns the bit pattern of this value as an unsigned\r\n| integer.\r\n*----------------------------------------------------------------------------*/\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nstruct uint128 softfloat_commonNaNToExtF80UI( const struct commonNaN *aPtr );\r\n\r\n/*----------------------------------------------------------------------------\r\n| Interpreting the unsigned integer formed from concatenating 'uiA64' and\r\n| 'uiA0' as an 80-bit extended floating-point value, and likewise interpreting\r\n| the unsigned integer formed from concatenating 'uiB64' and 'uiB0' as another\r\n| 80-bit extended floating-point value, and assuming at least on of these\r\n| floating-point values is a NaN, returns the bit pattern of the combined NaN\r\n| result. If either original floating-point value is a signaling NaN, the\r\n| invalid exception is raised.\r\n*----------------------------------------------------------------------------*/\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nstruct uint128\r\n softfloat_propagateNaNExtF80UI(\r\n struct softfloat_state *,\r\n uint_fast16_t uiA64,\r\n uint_fast64_t uiA0,\r\n uint_fast16_t uiB64,\r\n uint_fast64_t uiB0\r\n );\r\n\r\n/*----------------------------------------------------------------------------\r\n| The bit pattern for a default generated 128-bit floating-point NaN.\r\n*----------------------------------------------------------------------------*/\r\n#define defaultNaNF128UI64 UINT64_C( 0xFFFF800000000000 )\r\n#define defaultNaNF128UI0 UINT64_C( 0 )\r\n\r\n/*----------------------------------------------------------------------------\r\n| Returns true when the 128-bit unsigned integer formed from concatenating\r\n| 64-bit 'uiA64' and 64-bit 'uiA0' has the bit pattern of a 128-bit floating-\r\n| point signaling NaN.\r\n| Note: This macro evaluates its arguments more than once.\r\n*----------------------------------------------------------------------------*/\r\n#define softfloat_isSigNaNF128UI( uiA64, uiA0 ) ((((uiA64) & UINT64_C( 0x7FFF800000000000 )) == UINT64_C( 0x7FFF000000000000 )) && ((uiA0) || ((uiA64) & UINT64_C( 0x00007FFFFFFFFFFF ))))\r\n\r\n/*----------------------------------------------------------------------------\r\n| Assuming the unsigned integer formed from concatenating 'uiA64' and 'uiA0'\r\n| has the bit pattern of a 128-bit floating-point NaN, converts this NaN to\r\n| the common NaN form, and stores the resulting common NaN at the location\r\n| pointed to by 'zPtr'. If the NaN is a signaling NaN, the invalid exception\r\n| is raised.\r\n*----------------------------------------------------------------------------*/\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nvoid\r\n softfloat_f128UIToCommonNaN(\r\n struct softfloat_state *, uint_fast64_t uiA64, uint_fast64_t uiA0, struct commonNaN *zPtr );\r\n\r\n/*----------------------------------------------------------------------------\r\n| Converts the common NaN pointed to by 'aPtr' into a 128-bit floating-point\r\n| NaN, and returns the bit pattern of this value as an unsigned integer.\r\n*----------------------------------------------------------------------------*/\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nstruct uint128 softfloat_commonNaNToF128UI( const struct commonNaN * );\r\n\r\n/*----------------------------------------------------------------------------\r\n| Interpreting the unsigned integer formed from concatenating 'uiA64' and\r\n| 'uiA0' as a 128-bit floating-point value, and likewise interpreting the\r\n| unsigned integer formed from concatenating 'uiB64' and 'uiB0' as another\r\n| 128-bit floating-point value, and assuming at least on of these floating-\r\n| point values is a NaN, returns the bit pattern of the combined NaN result.\r\n| If either original floating-point value is a signaling NaN, the invalid\r\n| exception is raised.\r\n*----------------------------------------------------------------------------*/\r\nstruct uint128\r\n softfloat_propagateNaNF128UI(\r\n struct softfloat_state *,\r\n uint_fast64_t uiA64,\r\n uint_fast64_t uiA0,\r\n uint_fast64_t uiB64,\r\n uint_fast64_t uiB0\r\n );\r\n\r\n#else\r\n\r\n/*----------------------------------------------------------------------------\r\n| The following functions are needed only when 'SOFTFLOAT_FAST_INT64' is not\r\n| defined.\r\n*----------------------------------------------------------------------------*/\r\n\r\n/*----------------------------------------------------------------------------\r\n| Assuming the 80-bit extended floating-point value pointed to by 'aSPtr' is\r\n| a NaN, converts this NaN to the common NaN form, and stores the resulting\r\n| common NaN at the location pointed to by 'zPtr'. If the NaN is a signaling\r\n| NaN, the invalid exception is raised.\r\n*----------------------------------------------------------------------------*/\r\nvoid\r\n softfloat_extF80MToCommonNaN(\r\n const struct extFloat80M *aSPtr, struct commonNaN *zPtr );\r\n\r\n/*----------------------------------------------------------------------------\r\n| Converts the common NaN pointed to by 'aPtr' into an 80-bit extended\r\n| floating-point NaN, and stores this NaN at the location pointed to by\r\n| 'zSPtr'.\r\n*----------------------------------------------------------------------------*/\r\nvoid\r\n softfloat_commonNaNToExtF80M(\r\n const struct commonNaN *aPtr, struct extFloat80M *zSPtr );\r\n\r\n/*----------------------------------------------------------------------------\r\n| Assuming at least one of the two 80-bit extended floating-point values\r\n| pointed to by 'aSPtr' and 'bSPtr' is a NaN, stores the combined NaN result\r\n| at the location pointed to by 'zSPtr'. If either original floating-point\r\n| value is a signaling NaN, the invalid exception is raised.\r\n*----------------------------------------------------------------------------*/\r\nvoid\r\n softfloat_propagateNaNExtF80M(\r\n const struct extFloat80M *aSPtr,\r\n const struct extFloat80M *bSPtr,\r\n struct extFloat80M *zSPtr\r\n );\r\n\r\n/*----------------------------------------------------------------------------\r\n| The bit pattern for a default generated 128-bit floating-point NaN.\r\n*----------------------------------------------------------------------------*/\r\n#define defaultNaNF128UI96 0xFFFF8000\r\n#define defaultNaNF128UI64 0\r\n#define defaultNaNF128UI32 0\r\n#define defaultNaNF128UI0 0\r\n\r\n/*----------------------------------------------------------------------------\r\n| Assuming the 128-bit floating-point value pointed to by 'aWPtr' is a NaN,\r\n| converts this NaN to the common NaN form, and stores the resulting common\r\n| NaN at the location pointed to by 'zPtr'. If the NaN is a signaling NaN,\r\n| the invalid exception is raised. Argument 'aWPtr' points to an array of\r\n| four 32-bit elements that concatenate in the platform's normal endian order\r\n| to form a 128-bit floating-point value.\r\n*----------------------------------------------------------------------------*/\r\nvoid\r\n softfloat_f128MToCommonNaN( const uint32_t *aWPtr, struct commonNaN *zPtr );\r\n\r\n/*----------------------------------------------------------------------------\r\n| Converts the common NaN pointed to by 'aPtr' into a 128-bit floating-point\r\n| NaN, and stores this NaN at the location pointed to by 'zWPtr'. Argument\r\n| 'zWPtr' points to an array of four 32-bit elements that concatenate in the\r\n| platform's normal endian order to form a 128-bit floating-point value.\r\n*----------------------------------------------------------------------------*/\r\nvoid\r\n softfloat_commonNaNToF128M( const struct commonNaN *aPtr, uint32_t *zWPtr );\r\n\r\n/*----------------------------------------------------------------------------\r\n| Assuming at least one of the two 128-bit floating-point values pointed to by\r\n| 'aWPtr' and 'bWPtr' is a NaN, stores the combined NaN result at the location\r\n| pointed to by 'zWPtr'. If either original floating-point value is a\r\n| signaling NaN, the invalid exception is raised. Each of 'aWPtr', 'bWPtr',\r\n| and 'zWPtr' points to an array of four 32-bit elements that concatenate in\r\n| the platform's normal endian order to form a 128-bit floating-point value.\r\n*----------------------------------------------------------------------------*/\r\nvoid\r\n softfloat_propagateNaNF128M(\r\n const uint32_t *aWPtr, const uint32_t *bWPtr, uint32_t *zWPtr );\r\n\r\n#endif\r\n\r\n#endif\r\n\r\n" }, { "path": "External/SoftFloat-3e/src/ui64_to_extF80.c", "content": "\r\n/*============================================================================\r\n\r\nThis C source file is part of the SoftFloat IEEE Floating-Point Arithmetic\r\nPackage, Release 3e, by John R. Hauser.\r\n\r\nCopyright 2011, 2012, 2013, 2014, 2015, 2016 The Regents of the University of\r\nCalifornia. All Rights Reserved.\r\n\r\nRedistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n 1. Redistributions of source code must retain the above copyright notice,\r\n this list of conditions, and the following disclaimer.\r\n\r\n 2. Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions, and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n 3. Neither the name of the University nor the names of its contributors may\r\n be used to endorse or promote products derived from this software without\r\n specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS \"AS IS\", AND ANY\r\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY\r\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\n=============================================================================*/\r\n\r\n#include \r\n#include \"platform.h\"\r\n#include \"internals.h\"\r\n#include \"softfloat.h\"\r\n\r\nFEXCORE_PRESERVE_ALL_ATTR\r\nextFloat80_t ui64_to_extF80( uint64_t a )\r\n{\r\n uint_fast16_t uiZ64;\r\n int_fast8_t shiftDist;\r\n union { struct extFloat80M s; extFloat80_t f; } uZ;\r\n\r\n uiZ64 = 0;\r\n if ( a ) {\r\n shiftDist = softfloat_countLeadingZeros64( a );\r\n uiZ64 = 0x403E - shiftDist;\r\n a <<= shiftDist;\r\n }\r\n uZ.s.signExp = uiZ64;\r\n uZ.s.signif = a;\r\n return uZ.f;\r\n\r\n}\r\n\r\n" }, { "path": "External/cephes/CMakeLists.txt", "content": "add_library(cephes_128bit STATIC\n src/128bit/Impl.cpp\n src/128bit/atanll.c\n src/128bit/constll.c\n src/128bit/exp2ll.c\n src/128bit/floorll.c\n src/128bit/log2ll.c\n src/128bit/mtherr.c\n src/128bit/polevll.c\n src/128bit/sinll.c\n src/128bit/tanll.c)\n\n# 128-bit library\ntarget_link_libraries(cephes_128bit softfloat_3e)\ntarget_include_directories(cephes_128bit PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include/)\ntarget_compile_options(cephes_128bit PRIVATE -fno-builtin)\n" }, { "path": "External/cephes/LICENSE", "content": "The cephes math library is BSD licensed.\nThe source can be accessed from https://www.netlib.org/cephes/\n\nOriginal license from https://www.netlib.org/cephes/readme :\n> Some software in this archive may be from the book _Methods and\n> Programs for Mathematical Functions_ (Prentice-Hall or Simon & Schuster\n> International, 1989) or from the Cephes Mathematical Library, a\n> commercial product. In either event, it is copyrighted by the author.\n> What you see here may be used freely but it comes with no support or\n> guarantee.\n>\n> The two known misprints in the book are repaired here in the\n> source listings for the gamma function and the incomplete beta\n> integral.\n>\n>\n> Stephen L. Moshier\n> moshier@na-net.ornl.gov\n\nThe author was e-mailed and they allowed it to be relicensed under BSD.\nResources:\nhttps://bugs.gentoo.org/687276\nhttps://lists.debian.org/debian-legal/2004/12/msg00295.html\nhttps://github.com/deepmind/torch-cephes/blob/master/LICENSE.txt\nhttps://github.com/nearform/node-cephes/blob/master/LICENSE\n\nE-mail snippit from torch-cephes source:\n\nReturn-Path: \nX-Original-To: julien@cornebise.com\nDelivered-To: julien@cornebise.com\nReceived: from atl4mhob11.myregisteredsite.com (atl4mhob11.myregisteredsite.com [209.17.115.49])\n by cornebise.com (Postfix) with ESMTP id D47B139FC0\n for ; Fri, 25 Oct 2013 16:32:40 +0200 (CEST)\nReceived: from mailpod1.hostingplatform.com ([10.30.71.116])\n by atl4mhob11.myregisteredsite.com (8.14.4/8.14.4) with ESMTP id r9PEWcwQ003543\n for ; Fri, 25 Oct 2013 10:32:38 -0400\nReceived: (qmail 11948 invoked by uid 0); 25 Oct 2013 12:36:20 -0000\nX-TCPREMOTEIP: 76.24.25.74\nX-Authenticated-UID: steve@moshier.net\nReceived: from unknown (HELO d510.local) (steve@moshier.net@76.24.25.74)\n by 0 with ESMTPA; 25 Oct 2013 12:36:20 -0000\nDate: Fri, 25 Oct 2013 08:36:19 -0400 (EDT)\nFrom: Stephen Moshier \nX-X-Sender: steve@d510\nTo: Julien Cornebise \nSubject: Re: Cephes: permission to wrap+distribute for Lua\nIn-Reply-To: <52653AD3.1010004@cornebise.com>\nMessage-ID: \nReferences: <52653AD3.1010004@cornebise.com>\nUser-Agent: Alpine 2.02 (DEB 1266 2009-07-14)\nMIME-Version: 1.0\nContent-Type: TEXT/PLAIN; charset=US-ASCII; format=flowed\n\n\nJulien, thank you for writing.\nBSD license is fine, modification is OK.\nThere are more build scripts available in the web site distributions than\nthere are on the Netlib. I think there is an update to Planck's radiation\nfunction that I haven't sent to Netlib yet. But Netlib is a more stable\nsite, so it is better to cite that as a reference.\n\n\nOn Mon, 21 Oct 2013, Julien Cornebise wrote:\n\n> -----BEGIN PGP SIGNED MESSAGE-----\n> Hash: SHA1\n>\n> Dear Mr Moshier\n>\n> I am a researcher in mathematics and machine learning in London, and\n> am writing about your awesome Cephes library, whom I found at the\n> heart of Scipy.\n>\n> It is so useful that, with your permission, I would like to wrap it\n> for Lua and Torch (a machine learning overlay to Lua, specialized in\n> neural nets, see http://www.torch.ch). I would like to distribute it\n> as a package for Torch, including your source code along the wrapping\n> code.\n> This wouldbe a public package, distributed under BSD License. I have\n> put a first draft on github:\n> https://github.com/jucor/torch-cephes\n>\n> Hence my three questions, please:\n>\n> 1/ How would you like to be acknowledged, beyond the comments that are\n> already in your code? Do you have any standard header/disclaimer that\n> I could add to the documentation?\n>\n> 2/ At the moment, your code is left untouched. However, if I ever need\n> to modify bits of the code, what are the conditions/restrictions?\n> Nothing huge -- I definitely do not want to mess with it: I was\n> planning to use the natural completion of some functions on the\n> completed real line (e.g. CDF returing 1 when called with \"infinity\",\n> or quantiles returning -Infinity when called with 0), either natively\n> if supported, or by setting a specific flag via mtherr().\n>\n> 3/ I am currently using the source from Netlib. Do you recommend using\n> the source from your website instead ?\n>\n> Thank you very much for your attention,\n> and, more importantly, for the time and effort your poured into Cephes.\n>\n> Best regards,\n>\n> Julien Cornebise, Ph.D.\n> London, UK\n> http://www.cornebise.com/julien\n> -----BEGIN PGP SIGNATURE-----\n> Version: GnuPG v1.4.14 (Darwin)\n> Comment: GPGTools - http://gpgtools.org\n> Comment: Using GnuPG with Thunderbird - http://www.enigmail.net/\n>\n> iEYEARECAAYFAlJlOtEACgkQKYR3gC0rw/gIpQCfZKu6+iDh9ghhm6QfsLXnldKN\n> BuIAn2zZHu1c/IrRAevhjM7N7xGg0LHO\n> =WeP5\n> -----END PGP SIGNATURE-----\n\n" }, { "path": "External/cephes/include/cephes_128bit.h", "content": "#pragma once\n\nextern \"C\" {\n#include \"SoftFloat-3e/platform.h\"\n#include \"SoftFloat-3e/softfloat.h\"\n}\n\nnamespace FEXCore::cephes_128bit {\n float128_t atan2l(float128_t y, float128_t x);\n float128_t cosl(float128_t x);\n float128_t exp2l(float128_t x);\n float128_t log2l(float128_t x);\n float128_t sinl(float128_t x);\n float128_t tanl(float128_t x);\n}\n" }, { "path": "External/cephes/src/128bit/Impl.cpp", "content": "#include \"cephes_128bit.h\"\n\nextern \"C\" {\n// cephes_128bit functions\nfloat128_t cephes_f128_atan2l(float128_t y, float128_t x);\nfloat128_t cephes_f128_cosl(float128_t x);\nfloat128_t cephes_f128_exp2l(float128_t x);\nfloat128_t cephes_f128_log2l(float128_t x);\nfloat128_t cephes_f128_sinl(float128_t x);\nfloat128_t cephes_f128_tanl(float128_t x);\n}\n\nnamespace FEXCore::cephes_128bit {\n float128_t atan2l(float128_t y, float128_t x) {\n return cephes_f128_atan2l(y, x);\n }\n float128_t cosl(float128_t x) {\n return cephes_f128_cosl(x);\n }\n float128_t exp2l(float128_t x) {\n return cephes_f128_exp2l(x);\n }\n float128_t log2l(float128_t x) {\n return cephes_f128_log2l(x);\n }\n float128_t sinl(float128_t x) {\n return cephes_f128_sinl(x);\n }\n float128_t tanl(float128_t x) {\n return cephes_f128_tanl(x);\n }\n}\n" }, { "path": "External/cephes/src/128bit/atanll.c", "content": "/*\t\t\t\t\t\t\tatanl.c\n *\n *\tInverse circular tangent, 128-bit float128_t precision\n * (arctangent)\n *\n *\n *\n * SYNOPSIS:\n *\n * float128_t x, y, atanl();\n *\n * y = atanl( x );\n *\n *\n *\n * DESCRIPTION:\n *\n * Returns radian angle between -pi/2 and +pi/2 whose tangent\n * is x.\n *\n * Range reduction is from four intervals into the interval\n * from zero to tan( pi/8 ). The approximant uses a rational\n * function of degree 3/4 of the form x + x**3 P(x)/Q(x).\n *\n *\n *\n * ACCURACY:\n *\n * Relative error:\n * arithmetic domain # trials peak rms\n * IEEE -10, 10 100,000 2.6e-34 6.5e-35\n *\n */\n\f/*\t\t\t\t\t\t\tatan2l()\n *\n *\tQuadrant correct inverse circular tangent,\n *\tfloat128_t precision\n *\n *\n *\n * SYNOPSIS:\n *\n * float128_t x, y, z, atan2l();\n *\n * z = atan2l( y, x );\n *\n *\n *\n * DESCRIPTION:\n *\n * Returns radian angle whose tangent is y/x.\n * Define compile time symbol ANSIC = 1 for ANSI standard,\n * range -PI < z <= +PI, args (y,x); else ANSIC = 0 for range\n * 0 to 2PI, args (x,y).\n *\n *\n *\n * ACCURACY:\n *\n * Relative error:\n * arithmetic domain # trials peak rms\n * IEEE -10, 10 100,000 3.2e-34 5.9e-35\n * See atan.c.\n *\n */\n\f\n/*\t\t\t\t\t\t\tatan.c */\n\n\n/*\nCephes Math Library Release 2.2: December, 1990\nCopyright 1984, 1990 by Stephen L. Moshier\nDirect inquiries to 30 Frost Street, Cambridge, MA 02140\n*/\n\n#include \"mconf.h\"\n\n/* arctan(x) = x + x^3 P(x^2)\n * Theoretical peak relative error = 3.0e-36\n * relative peak error spread = 6.6e-8\n */\nstatic float128_t P[9] = {\n{0xf3f0105b1dae46bfULL, 0xbff45be85838aa26ULL}, // -6.635810778635296712545011270011752799963E-4L,\n{0x529a2bf25f15874bULL, 0xbffec0f17ae68a18ULL}, // -8.768423468036849091777415076702113400070E-1L,\n{0x3054a2e7144e265cULL, 0xc00397b0dc1f4d10ULL}, // -2.548067867495502632615671450650071218995E1L,\n{0x1e19d6b8c5cd9e65ULL, 0xc006f38d4e47779aULL}, // -2.497759878476618348858065206895055957104E2L,\n{0x69dcb1e41a413bddULL, 0xc0091f0a8586c642ULL}, // -1.148164399808514330375280133523543970854E3L,\n{0x501d0f5157516744ULL, 0xc00a5d08ba650145ULL}, // -2.792272753241044941703278827346430350236E3L,\n{0x16f18bf3f5b4b987ULL, 0xc00ace087656cfbeULL}, // -3.696264445691821235400930243493001671932E3L,\n{0x2966de608cbf9696ULL, 0xc00a3a5a8d629fc7ULL}, // -2.514829758941713674909996882101723647996E3L,\n{0xeb77db69572ecd22ULL, 0xc0085807a6c98431ULL}, // -6.880597774405940432145577545328795037141E2L\n};\nstatic float128_t Q[8] = {\n/* 1.000000000000000000000000000000000000000E0L, */\n{0x0cc994a760137543ULL, 0x40041d4c974b22bcULL}, // 3.566239794444800849656497338030115886153E1L,\n{0xa5b186c10b6a065eULL, 0x4007aed5b7e20c37ULL}, // 4.308348370818927353321556740027020068897E2L,\n{0x8711ebf202296129ULL, 0x400a37d5c6fdd0cdULL}, // 2.494680540950601626662048893678584497900E3L,\n{0x02d59339ee4eee21ULL, 0x400bef892855649eULL}, // 7.928572347062145288093560392463784743935E3L,\n{0xd9b903b0950fefb3ULL, 0x400cc7c8d1c45b09ULL}, // 1.458510242529987155225086911411015961174E4L,\n{0x174d6e0dae833752ULL, 0x400ce38f8ba0a897ULL}, // 1.547394317752562611786521896296215170819E4L,\n{0xfcbdd5dddcf7c68cULL, 0x400c1277f99a3d1aULL}, // 8.782996876218210302516194604424986107121E3L,\n{0x7099e48f01631a53ULL, 0x400a0205bd172325ULL}, // 2.064179332321782129643673263598686441900E3L\n};\n\n/* tan( 3*pi/8 ) */\nstatic float128_t T3P8 = {0x6484597d89b3754bULL, 0x40003504f333f9deULL};\n\n/* tan( pi/8 ) */\nstatic float128_t TP8 = {0x2422cbec4d9baa56ULL, 0x3ffda827999fcef3ULL};\n\nstatic const float128_t zero = {0, 0};\nstatic const float128_t one = {0, 0x3fff000000000000ULL};\n__attribute__((unused)) static const float128_t f_2_p0 = {0x0000000000000000ULL, 0x4000000000000000ULL};\n__attribute__((unused)) static const float128_t f_3_p0 = {0x0000000000000000ULL, 0x4000800000000000ULL};\n\nfloat128_t cephes_f128_atanl(float128_t x)\n{\nstruct softfloat_state state = {};\nfloat128_t y, z;\nshort sign;\n\n/* make argument positive and save the sign */\nsign = 1;\nif( f128_lt(&state, x, zero) )\n\t{\n\tsign = -1;\n\tx = f128_complement_sign(x);\n\t}\n\n/* range reduction */\n// if( x > T3P8 )\n\nif( f128_lt(&state, T3P8, x) )\n\t{\n\ty = F128_PIO2L;\n\tx = f128_complement_sign( f128_div(&state, one, x));\n\t}\n\nelse if( f128_lt(&state, TP8, x) )\n\t{\n\ty = F128_PIO4L;\n\tx = f128_div(&state, f128_sub(&state, x, one), f128_add(&state, x, one));\n\t}\nelse\n\ty = zero;\n\n/* rational form in x**2 */\nz = f128_mul(&state, x, x);\ny = f128_add(&state, f128_add(&state, y, f128_mul(&state, f128_mul(&state, f128_div(&state, cephes_f128_polevll( z, P, 8 ), cephes_f128_p1evll( z, Q, 8 ) ), z), x)), x);\n\nif( sign < 0 )\n\ty = f128_complement_sign(y);\n\nreturn(y);\n}\n\f\n/*\t\t\t\t\t\t\tatan2\t*/\n\n\n\n#if ANSIC\nfloat128_t cephes_f128_atan2l( float128_t y, float128_t x )\n#else\nfloat128_t cephes_f128_atan2l( float128_t x, float128_t y )\n#endif\n{\nstruct softfloat_state state = {};\nfloat128_t z, w;\nshort code;\n\n\ncode = 0;\nw = zero;\n\nif( f128_lt(&state, x, zero) )\n\tcode = 2;\nif( f128_lt(&state, y, zero) )\n\tcode |= 1;\n\nif( f128_eq(&state, x, zero) )\n\t{\n\tif( code & 1 )\n\t\t{\n#if ANSIC\n\t\treturn( f128_complement_sign(F128_PIO2L) );\n#else\n\t\treturn( f128_mul(&state, f_3_p0, F128_PIO2L) );\n#endif\n\t\t}\n\tif( f128_eq(&state, y, zero) )\n\t\treturn zero;\n\treturn( F128_PIO2L );\n\t}\n\nif( f128_eq(&state, y, zero) )\n\t{\n\tif( code & 2 )\n\t\treturn( F128_PIL );\n\treturn zero;\n\t}\n\n\nswitch( code )\n\t{\n#if ANSIC\n\tcase 0:\n\tcase 1: w = zero; break;\n\tcase 2: w = F128_PIL; break;\n\tcase 3: w = f128_complement_sign(F128_PIL); break;\n#else\n\tcase 0: w = zero; break;\n\tcase 1: w = f128_mul(&state, f_2_p0, F128_PIL); break;\n\tcase 2:\n\tcase 3: w = F128_PIL; break;\n#endif\n\t}\n\nz = cephes_f128_atanl( f128_div(&state, y, x) );\n\nreturn f128_add(&state, w, z );\n}\n" }, { "path": "External/cephes/src/128bit/constll.c", "content": "#include \"mconf.h\"\n\n/* (1 - 2^-113) 2^16384 */\nfloat128_t F128_MAXNUML = {0xffffffffffffffffULL, 0x7ffeffffffffffffULL}; //1.189731495357231765085759326628007016196469e4932L;\n\n/* 2^-113 */\nfloat128_t F128_MACHEPL = {0x0000000000000000ULL, 0x3f8e000000000000ULL}; // 9.629649721936179265279889712924636592690508e-35L;\n\n/* (1 + 2^-112) 2^-16382 */\nfloat128_t F128_UFTHRESHL = {0x0000000000000001ULL, 0x0001000000000000ULL}; // 3.362103143112093506262677817321753250115591e-4932L;\n\n/* 2^-16494 */\nfloat128_t F128_MINNUML = {0x0000000000000001ULL, 0x0000000000000000ULL}; // 6.475175119438025110924438958227646552499569e-4966L;\n\n/* ln(MAXNUM) */\nfloat128_t F128_MAXLOGL = {0xf35793c7673007e6ULL, 0x400c62e42fefa39eULL}; // 1.1356523406294143949491931077970764891253E4L;\n\n/* ln(MINNUM) */\nfloat128_t F128_MINLOGL = {0x2c89d24d65e96274ULL, 0xc00c654628220780ULL}; // -1.143276959615573793352782661133116431383730e4L;\n\n/* ln(UFTHRESH) */\n/* float128_t F128_MINLOGL = -1.135513711193302405887309661372784853802025e4L; */\n\nfloat128_t F128_PIL = {0x8469898cc51701b8ULL, 0x4000921fb54442d1ULL}; // 3.141592653589793238462643383279502884197169L;\n\nfloat128_t F128_PIO2L = {0x8469898cc51701b8ULL, 0x3fff921fb54442d1ULL}; // 1.570796326794896619231321691639751442098585L;\n\nfloat128_t F128_PIO4L = {0x8469898cc51701b8ULL, 0x3ffe921fb54442d1ULL}; // 0.7853981633974483096156608458198757210492923L;\n\nfloat128_t F128_LOGE2L = {0xf35793c7673007e6ULL, 0x3ffe62e42fefa39eULL}; // 0.6931471805599453094172321214581765680755001L;\n\nfloat128_t F128_LOG2EL = {0xe1777d0ffda0d23aULL, 0x3fff71547652b82fULL}; // 1.442695040888963407359924681001892137426646L;\n\nfloat128_t F128_INFINITYL = {0x0000000000000000ULL, 0x7fff000000000000ULL}; // 1.0L / 0.0L;\n" }, { "path": "External/cephes/src/128bit/exp2ll.c", "content": "/*\t\t\t\t\t\t\texp2l.c\n *\n *\tBase 2 exponential function, 128-bit float128_t precision\n *\n *\n *\n * SYNOPSIS:\n *\n * float128_t x, y, exp2l();\n *\n * y = exp2l( x );\n *\n *\n *\n * DESCRIPTION:\n *\n * Returns 2 raised to the x power.\n *\n * Range reduction is accomplished by separating the argument\n * into an integer k and fraction f such that\n * x k f\n * 2 = 2 2.\n *\n * A Pade' form\n *\n * 1 + 2x P(x**2) / (Q(x**2) - x P(x**2) )\n *\n * approximates 2**x in the basic range [-0.5, 0.5].\n *\n *\n * ACCURACY:\n *\n * Relative error:\n * arithmetic domain # trials peak rms\n * IEEE +-16300 100,000 2.0e-34 4.8e-35\n *\n *\n * See exp.c for comments on error amplification.\n *\n *\n * ERROR MESSAGES:\n *\n * message condition value returned\n * exp2l underflow x < -16382 0.0\n * exp2l overflow x >= 16384 MAXNUM\n *\n */\n\f\n\n/*\nCephes Math Library Release 2.2: January, 1991\nCopyright 1984, 1991 by Stephen L. Moshier\nDirect inquiries to 30 Frost Street, Cambridge, MA 02140\n*/\n\n#include \"mconf.h\"\nstatic char fname[] = {\"exp2l\"};\n\n/* Pade' coefficients for 2^x - 1\n Theoretical peak relative error = 1.4e-40,\n relative peak error spread = 6.8e-14\n */\nstatic float128_t P[5] = {\n {0x3008ca100ca13471ULL, 0x40063d6f2f556577ULL}, // 1.587171580015525194694938306936721666031E2L,\n {0x9fac10fe43d72769ULL, 0x40122e00e88b4606ULL}, // 6.185032670011643762127954396427045467506E5L,\n {0x4c22cf0c6c7a8fc7ULL, 0x401c0eb996d98ba4ULL}, // 5.677513871931844661829755443994214173883E8L,\n {0x4acd9b1339dda08aULL, 0x40241d19e728a6beULL}, // 1.530625323728429161131811299626419117557E11L,\n {0xae406b996488ba7aULL, 0x402a0840400c1c84ULL}, // 9.079594442980146270952372234833529694788E12L\n};\nstatic float128_t Q[5] = {\n/* 1.000000000000000000000000000000000000000E0L, */\n {0xcf48c9db239c2189ULL, 0x400c827029417a6aULL}, // 1.236602014442099053716561665053645270207E4L,\n {0xb20f61f9a3c778b9ULL, 0x40174d9860120d5dULL}, // 2.186249607051644894762167991800811827835E7L,\n {0x9f361a3e85f209ceULL, 0x4020457bc8296e4eULL}, // 1.092141473886177435056423606755843616331E10L,\n {0x2dcf78c66f0a65ddULL, 0x40275b0c5bcbd7a7ULL}, // 1.490560994263653042761789432690793026977E12L,\n {0x4e4a9905cf9c9235ULL, 0x402b7d3bcb89794eULL}, // 2.619817175234089411411070339065679229869E13L\n};\n\nstatic const float128_t MAXL2 = {0x0000000000000000ULL, 0x400d000000000000ULL};\nstatic const float128_t MINL2 = {0x0000000000000000ULL, 0xc00cfff000000000ULL};\nstatic const float128_t zero = {0, 0};\nstatic const float128_t f_0_p5 = {0, 0x3ffe000000000000ULL};\nstatic const float128_t one = {0, 0x3fff000000000000ULL};\n\nextern float128_t F128_MAXNUML;\n\nfloat128_t cephes_f128_exp2l(float128_t x) {\nstruct softfloat_state state = {};\nfloat128_t px, xx;\nint n;\n\nif( f128_le(&state, MAXL2, x))\n\t{\n\tmtherr( fname, OVERFLOW );\n\treturn( F128_MAXNUML );\n\t}\n\nif(f128_lt(&state, x, MINL2) )\n\t{\n\tmtherr( fname, UNDERFLOW );\n\treturn zero;\n\t}\n\nxx = x;\t/* save x */\n/* separate into integer and fractional parts */\npx = cephes_f128_floorl(f128_add(&state, x, f_0_p5));\nn = f128_to_i32(&state, px, softfloat_round_near_even, true);\nx = f128_sub(&state, x, px);\n\n/* rational approximation\n * exp2(x) = 1.0 + 2xP(xx)/(Q(xx) - P(xx))\n * where xx = x**2\n */\nxx = f128_mul(&state, x, x);\npx = f128_mul(&state, x, cephes_f128_polevll( xx, P, 4 ));\nx = f128_div(&state, px, f128_sub(&state, cephes_f128_p1evll( xx, Q, 5 ), px));\nx = f128_add(&state, one, cephes_f128_ldexpl( x, 1 ));\n\n/* scale by power of 2 */\nx = cephes_f128_ldexpl( x, n );\nreturn(x);\n}\n" }, { "path": "External/cephes/src/128bit/floorll.c", "content": "/* ceill()\n * floorl()\n * frexpl()\n * ldexpl()\n * fabsl()\n *\t\t\t\t\t\t\tsignbitl()\n *\t\t\t\t\t\t\tisnanl()\n *\t\t\t\t\t\t\tisfinitel()\n *\n * Floating point numeric utilities\n *\n *\n *\n * SYNOPSIS:\n *\n * float128_t x, y;\n * float128_t ceill(), floorl(), frexpl(), ldexpl(), fabsl();\n * int signbitl(), isnanl(), isfinitel();\n * int expnt, n;\n *\n * y = floorl(x);\n * y = ceill(x);\n * y = frexpl( x, &expnt );\n * y = ldexpl( x, n );\n * y = fabsl( x );\n *\n *\n *\n * DESCRIPTION:\n *\n * All four routines return a float128_t precision floating point\n * result.\n *\n * floorl() returns the largest integer less than or equal to x.\n * It truncates toward minus infinity.\n *\n * ceill() returns the smallest integer greater than or equal\n * to x. It truncates toward plus infinity.\n *\n * frexpl() extracts the exponent from x. It returns an integer\n * power of two to expnt and the significand between 0.5 and 1\n * to y. Thus x = y * 2**expn.\n *\n * ldexpl() multiplies x by 2**n.\n *\n * fabsl() returns the absolute value of its argument.\n *\n * signbitl(x) returns 1 if the sign bit of x is 1, else 0.\n *\n * These functions are part of the standard C run time library\n * for some but not all C compilers. The ones supplied are\n * written in C for IEEE arithmetic. They should\n * be used only if your compiler library does not already have\n * them.\n *\n * The IEEE versions assume that denormal numbers are implemented\n * in the arithmetic. Some modifications will be required if\n * the arithmetic has abrupt rather than gradual underflow.\n */\n\f\n\n/*\nCephes Math Library Release 2.2: July, 1992\nCopyright 1984, 1987, 1988, 1992 by Stephen L. Moshier\nDirect inquiries to 30 Frost Street, Cambridge, MA 02140\n*/\n\n#include \"mconf.h\"\n#define DENORMAL 1\n\n#ifdef UNK\nchar *unkmsg = \"ceill(), floorl(), frexpl(), ldexpl() must be rewritten!\\n\";\n#undef UNK\n#define MIEEE 1\n#define EXPOFS 0\n#endif\n\n#ifdef IBMPC\n#define NBITS 113\n#define EXPOFS 7\n#endif\n\n#ifdef MIEEE\n#define NBITS 113\n#define EXPOFS 0\n#endif\n\nextern float128_t F128_MAXNUML;\n\n\nstatic const float128_t zero = {0, 0};\nstatic const float128_t f_0_p5 = {0, 0x3ffe000000000000ULL};\nstatic const float128_t one = {0, 0x3fff000000000000ULL};\nstatic const float128_t neg_one = {0, 0xbfff000000000000ULL};\nstatic const float128_t f_2_p0 = {0, 0x4000000000000000ULL};\n\nfloat128_t cephes_f128_fabsl(float128_t x)\n{\nstruct softfloat_state state = {};\n\nif( f128_lt(&state, x, zero) )\n return f128_sub(&state, zero, x );\nelse\n return( x );\n}\n\n\n\nfloat128_t cephes_f128_ceill(float128_t x)\n{\nfloat128_t y;\n\n#ifdef UNK\nmtherr( \"ceill\", DOMAIN );\nreturn(0.0L);\n#endif\n\nstruct softfloat_state state = {};\ny = cephes_f128_floorl(x);\nif( f128_lt(&state, y, x) )\n y = f128_add(&state, y, one);\nreturn(y);\n}\n\n\n\n\n/* Bit clearing masks: */\n\nstatic unsigned short bmask[] = {\n0xffff,\n0xfffe,\n0xfffc,\n0xfff8,\n0xfff0,\n0xffe0,\n0xffc0,\n0xff80,\n0xff00,\n0xfe00,\n0xfc00,\n0xf800,\n0xf000,\n0xe000,\n0xc000,\n0x8000,\n0x0000,\n};\n\n\n\nfloat128_t cephes_f128_floorl(float128_t x)\n{\nunion\n {\n float128_t y;\n unsigned short sh[8];\n } u;\nint e, j;\n\n#ifdef UNK\nmtherr( \"floor\", DOMAIN );\nreturn(0.0L);\n#endif\n\nstruct softfloat_state state = {};\nu.y = x;\n/* find the exponent (power of 2) */\ne = (u.sh[EXPOFS] & 0x7fff) - 0x3fff;\n\nif( e < 0 )\n {\n if( f128_lt(&state, u.y, zero) )\n return neg_one;\n else\n return zero;\n }\n\n#ifdef IBMPC\nj = 0;\n#endif\n\n#ifdef MIEEE\nj = 7;\n#endif\n\ne = (NBITS - 1) - e;\n/* clean out 16 bits at a time */\nwhile( e >= 16 )\n {\n#ifdef IBMPC\n u.sh[j++] = 0;\n#endif\n\n#ifdef MIEEE\n u.sh[j--] = 0;\n#endif\n e -= 16;\n }\n\n/* clear the remaining bits */\nif( e > 0 )\n u.sh[j] &= bmask[e];\n\nif( f128_lt(&state, x, zero) && !f128_eq(&state, u.y, x) )\n u.y = f128_sub(&state, u.y, one);;\n\nreturn(u.y);\n}\n\n\n\nfloat128_t cephes_f128_frexpl( float128_t x, int *pw2 )\n{\nunion\n {\n float128_t y;\n unsigned short sh[8];\n } u;\nint i, k;\n\nstruct softfloat_state state = {};\nu.y = x;\n\n#ifdef UNK\nmtherr( \"frexp\", DOMAIN );\nreturn(0.0L);\n#endif\n\n/* find the exponent (power of 2) */\ni = u.sh[EXPOFS] & 0x7fff;\n\nif( i == 0 )\n {\n if( f128_eq(&state, u.y, zero))\n {\n *pw2 = 0;\n return zero;\n }\n/* Number is denormal or zero */\n#if DENORMAL\n/* Handle denormal number. */\ndo\n {\n u.y = f128_mul(&state, u.y, f_2_p0);\n i -= 1;\n k = u.sh[EXPOFS] & 0x7fff;\n }\nwhile( (k == 0) && (i > -115) );\ni = i + k;\n#else\n *pw2 = 0;\n return(0.0L);\n#endif /* DENORMAL */\n }\n\n*pw2 = i - 0x3ffe;\nu.sh[EXPOFS] = 0x3ffe;\nreturn( u.y );\n}\n\n\nfloat128_t cephes_f128_ldexpl( float128_t x, int pw2 )\n{\nunion\n {\n float128_t y;\n unsigned short sh[8];\n } u;\nlong e;\n\n#ifdef UNK\nmtherr( \"ldexp\", DOMAIN );\nreturn zero;\n#endif\n\nstruct softfloat_state state = {};\nu.y = x;\nwhile( (e = (u.sh[EXPOFS] & 0x7fffL)) == 0 )\n {\n#if DENORMAL\n if( f128_eq(&state, u.y, zero))\n {\n return zero;\n }\n/* Input is denormal. */\n if( pw2 > 0 )\n {\n u.y = f128_mul(&state, u.y, f_2_p0);\n pw2 -= 1;\n }\n if( pw2 < 0 )\n {\n if( pw2 < -113 )\n return zero;\n u.y = f128_sub(&state, u.y, f_0_p5);\n pw2 += 1;\n }\n if( pw2 == 0 )\n return(u.y);\n#else\n return zero;\n#endif\n }\n\ne = e + pw2;\n\n/* Handle overflow */\nif( e > 0x7ffeL )\n {\n e = u.sh[EXPOFS];\n u.y = zero;\n u.sh[EXPOFS] = e | 0x7fff;\n return( u.y );\n }\nu.sh[EXPOFS] &= 0x8000;\n/* Handle denormalized results */\nif( e < 1 )\n {\n#if DENORMAL\n if( e < -113 )\n return zero;\n u.sh[EXPOFS] |= 1;\n while( e < 1 )\n {\n u.y = f128_sub(&state, u.y, f_0_p5);\n e += 1;\n }\n e = 0;\n#else\n return zero;\n#endif\n }\n\nu.sh[EXPOFS] |= e & 0x7fff;\nreturn(u.y);\n}\n\n/* Return 1 if x is a number that is Not a Number, else return 0. */\n\nint cephes_f128_isnanl(float128_t x)\n{\n#ifdef NANS\nunion\n\t{\n\tfloat128_t d;\n\tunsigned short s[8];\n\tunsigned int i[4];\n\t} u;\n\nu.d = x;\n\nif( sizeof(int) == 4 )\n\t{\n#ifdef IBMPC\t \n\tif( ((u.s[7] & 0x7fff) == 0x7fff)\n\t && ((u.i[3] & 0x7fff) | u.i[2] | u.i[1] | u.i[0]))\n\t\treturn 1;\n#endif\n#ifdef MIEEE\n\tif( ((u.i[0] & 0x7fff0000) == 0x7fff0000)\n\t && ((u.i[0] & 0x7fff) | u.i[1] | u.i[2] | u.i[3]))\n\t\treturn 1;\n#endif\n\treturn(0);\n\t}\nelse\n\t{ /* size int not 4 */\n#ifdef IBMPC\n\tif( (u.s[7] & 0x7fff) == 0x7fff)\n\t\t{\n\t\tif((u.s[6] & 0x7fff) | u.s[5] | u.s[4] | u.s[3] | u.s[2] | u.s[1] | u.s[0])\n\t\t\treturn(1);\n\t\t}\n#endif\n#ifdef MIEEE\n\tif( (u.s[0] & 0x7fff) == 0x7fff)\n\t\t{\n\t\tif((u.s[1] & 0x7fff) | (u.s[2] & 0x7fff) | u.s[3] | u.s[4] | u.s[5] | u.s[6] | u.s[7])\n\t\t\treturn(1);\n\t\t}\n#endif\n\treturn(0);\n\t} /* size int not 4 */\n\n#else\n/* No NANS. */\nreturn(0);\n#endif\n}\n\n\n/* Return 1 if x is not infinite and is not a NaN. */\n\nint cephes_f128_isfinitel(float128_t x)\n{\n#ifdef INFINITIES\nunion\n\t{\n\tfloat128_t d;\n\tunsigned short s[8];\n\tunsigned int i[4];\n\t} u;\n\nu.d = x;\n\nif( sizeof(int) == 4 )\n\t{\n#ifdef IBMPC\n\tif( (u.s[7] & 0x7fff) != 0x7fff)\n\t\treturn 1;\n#endif\n#ifdef MIEEE\n\tif( (u.i[0] & 0x7fff0000) != 0x7fff0000)\n\t\treturn 1;\n#endif\n\treturn(0);\n\t}\nelse\n\t{\n#ifdef IBMPC\n\tif( (u.s[7] & 0x7fff) != 0x7fff)\n\t\treturn 1;\n#endif\n#ifdef MIEEE\n\tif( (u.s[0] & 0x7fff) != 0x7fff)\n\t\treturn 1;\n#endif\n\treturn(0);\n\t}\n#else\n/* No INFINITY. */\nreturn(1);\n#endif\n}\n\n\n/* Return 1 if the sign bit of x is 1, else 0. */\n\nint cephes_f128_signbitl(float128_t x)\n{\nunion\n\t{\n\tfloat128_t d;\n\tshort s[8];\n\tint i[4];\n\t} u;\n\nu.d = x;\n\nif( sizeof(int) == 4 )\n\t{\n#ifdef IBMPC\n\treturn( u.s[7] < 0 );\n#endif\n#ifdef DEC\nerror no such DEC format\n#endif\n#ifdef MIEEE\n\treturn( u.i[0] < 0 );\n#endif\n\t}\nelse\n\t{\n#ifdef IBMPC\n\treturn( u.s[7] < 0 );\n#endif\n#ifdef DEC\nerror no such DEC format\n#endif\n#ifdef MIEEE\n\treturn( u.s[0] < 0 );\n#endif\n\t}\n}\n" }, { "path": "External/cephes/src/128bit/log2ll.c", "content": "/*\t\t\t\t\t\t\tcephes_f128_log2l.c\n *\n *\tBase 2 logarithm, float128_t precision\n *\n *\n *\n * SYNOPSIS:\n *\n * float128_t x, y, cephes_f128_log2l();\n *\n * y = cephes_f128_log2l( x );\n *\n *\n *\n * DESCRIPTION:\n *\n * Returns the base 2 logarithm of x.\n *\n * The argument is separated into its exponent and fractional\n * parts. If the exponent is between -1 and +1, the (natural)\n * logarithm of the fraction is approximated by\n *\n * log(1+x) = x - 0.5 x**2 + x**3 P(x)/Q(x).\n *\n * Otherwise, setting z = 2(x-1)/x+1),\n * \n * log(x) = z + z**3 P(z)/Q(z).\n *\n *\n *\n * ACCURACY:\n *\n * Relative error:\n * arithmetic domain # trials peak rms\n * IEEE 0.5, 2.0 100,000 1.3e-34 4.5e-35\n * IEEE exp(+-10000) 100,000 9.6e-35 4.0e-35\n *\n * In the tests over the interval exp(+-10000), the logarithms\n * of the random arguments were uniformly distributed over\n * [-10000, +10000].\n *\n * ERROR MESSAGES:\n *\n * log singularity: x = 0; returns MINLOG\n * log domain: x < 0; returns MINLOG\n */\n\f\n/*\nCephes Math Library Release 2.2: January, 1991\nCopyright 1984, 1991 by Stephen L. Moshier\nDirect inquiries to 30 Frost Street, Cambridge, MA 02140\n*/\n\n#include \"mconf.h\"\nstatic char fname[] = {\"cephes_f128_log2l\"};\n\n/* Coefficients for ln(1+x) = x - x**2/2 + x**3 P(x)/Q(x)\n * 1/sqrt(2) <= x < sqrt(2)\n * Theoretical peak relative error = 5.3e-37,\n * relative peak error spread = 2.3e-14\n */\nstatic float128_t P[13] = {\n\t{0x95434922008560fcULL, 0x3feb9d04a0d6ed82ULL}, // 1.538612243596254322971797716843006400388E-6L\n\t{0x2e9cb5e91a8c2fa0ULL, 0x3ffdffd7e21347ccULL}, // 4.998469661968096229986658302195402690910E-1L\n\t{0x674c43ea62a592e7ULL, 0x400373615178fe96ULL}, // 2.321125933898420063925789532045674660756E1L\n\t{0xfa539715d5fd0560ULL, 0x40079b73a8639c28ULL}, // 4.114517881637811823002128927449878962058E2L\n\t{0x5ec5c60d38b7fa2aULL, 0x400ade1e79b3ae12ULL}, // 3.824952356185897735160588078446136783779E3L\n\t{0x6369f0cada64eeecULL, 0x400d4ca24f0550cfULL}, // 2.128857716871515081352991964243375186031E4L\n\t{0x115104b644c1f464ULL, 0x400f28a791822d40ULL}, // 7.594356839258970405033155585486712125861E4L\n\t{0x95ec43488121aff8ULL, 0x40105f196a49f171ULL}, // 1.797628303815655343403735250238293741397E5L\n\t{0xa2484b7171ab5034ULL, 0x401116caba9f2757ULL}, // 2.854829159639697837788887080758954924001E5L\n\t{0xe49b2bf8646a8a1eULL, 0x401125a72eb05ba7ULL}, // 3.007007295140399532324943111654767187848E5L\n\t{0x17ac5c737d1b8ad4ULL, 0x4010897ca319418dULL}, // 2.014652742082537582487669938141683759923E5L\n\t{0x9ff15925da76d408ULL, 0x400f2f8f8bfbf9a1ULL}, // 7.771154681358524243729929227226708890930E4L\n\t{0xe740b8544d79077cULL, 0x400c9a7dcad5d0efULL}, // 1.313572404063446165910279910527789794488E4L\n};\nstatic float128_t Q[12] = {\n/* 1.000000000000000000000000000000000000000E0L, */\n{0x4a2113daac8d7fa5ULL,0x40048322fbda4d3fULL}, // 4.839208193348159620282142911143429644326E1L,\n{0x9efb2fe2c778f56fULL,0x4008c73f14777e56ULL}, // 9.104928120962988414618126155557301584078E2L,\n{0xf23a98d434d3a705ULL,0x400c1dd933ea5565ULL}, // 9.147150349299596453976674231612674085381E3L,\n{0x4b44059a3b76f461ULL,0x400eb5f4d77aed02ULL}, // 5.605842085972455027590989944010492125825E4L,\n{0x2962234d48fff0bcULL,0x4010b71bb67f5effULL}, // 2.248234257620569139969141618556349415120E5L,\n{0xe673c713bcf24ee3ULL,0x40122b6c5ddac3b8ULL}, // 6.132189329546557743179177159925690841200E5L,\n{0x34d8d36e8de37c71ULL,0x40131ab83fa3b03bULL}, // 1.158019977462989115839826904108208787040E6L,\n{0x061338bb0e95b314ULL,0x401371d8273f762aULL}, // 1.514882452993549494932585972882995548426E6L,\n{0xe379b5d8e7071d74ULL,0x401348fbe89d38e2ULL}, // 1.347518538384329112529391120390701166528E6L,\n{0x412eafafea233277ULL,0x40127bc5211688c1ULL}, // 7.777690340007566932935753241556479363645E5L,\n{0x16378fd2514ba129ULL,0x40110088814003eaULL}, // 2.626900195321832660448791748036714883242E5L,\n{0xed708a3f3a1ac5caULL,0x400e33de58205cb3ULL}, // 3.940717212190338497730839731583397586124E4L\n};\n\n/* Coefficients for log(x) = z + z^3 P(z^2)/Q(z^2),\n * where z = 2(x-1)/(x+1)\n * 1/sqrt(2) <= x < sqrt(2)\n * Theoretical peak relative error = 1.1e-35,\n * relative peak error spread 1.1e-9\n */\nstatic float128_t R[6] = {\n\t{0x68479d54e4ced708ULL, 0xbffec40a1c874f5aULL}, // -8.828896441624934385266096344596648080902E-1L,\n\t{0x565b5611a30df628ULL, 0x40054247b533971eULL}, // 8.057002716646055371965756206836056074715E1L,\n\t{0xb690eddd457e03b0ULL, 0xc009fa1350a9210eULL}, // -2.024301798136027039250415126250455056397E3L,\n\t{0xea1230d4dc2a41c8ULL, 0x400d4020cbb3c4edULL}, // 2.048819892795278657810231591630928516206E4L,\n\t{0x388e5d3ae806c32aULL, 0xc00f5eac94780e23ULL}, // -8.977257995689735303686582344659576526998E4L,\n\t{0x6802a6fb3250b4fdULL, 0x401014fab5e2e8c1ULL}, // 1.418134209872192732479751274970992665513E5L\n};\nstatic float128_t S[6] = {\n/* 1.000000000000000000000000000000000000000E0L, */\n {0x2575cd7cadd52c63ULL, 0xc005da8b34108b63ULL}, // -1.186359407982897997337150403816839480438E2L,\n {0x9022bf51e9d20aecULL, 0x400af3d0db24df08ULL}, // 3.998526750980007367835804959888064681098E3L,\n {0xeb27fc1032bb267dULL, 0xc00ec11ad77cc51cULL}, // -5.748542087379434595104154610899551484314E4L,\n {0xaeec5bd6a5211cbdULL, 0x401186c6f13df72eULL}, // 4.001557694070773974936904547424676279307E5L,\n {0xee9e91e4b3020178ULL, 0xc013455371e04bc5ULL}, // -1.332535117259762928288745111081235577029E6L,\n {0x1c03fa78cb791730ULL, 0x40139f7810d45d22ULL}, // 1.701761051846631278975701529965589676574E6L\n};\n/* log2(e) - 1 */\nstatic const float128_t LOG2EA = {0x85ddf43ff68348eaULL, 0x3ffdc551d94ae0bfULL};\n\nstatic const float128_t SQRTH = {0xc908b2fb1366ea95ULL, 0x3ffe6a09e667f3bcULL};\nstatic const float128_t zero = {0, 0};\nstatic const float128_t f_0_p5 = {0, 0x3ffe000000000000ULL};\nstatic const float128_t one = {0, 0x3fff000000000000ULL};\n\nstatic const float128_t indeterminate = {0x0000000000000000ULL, 0xc00d000000000000ULL};\n\nfloat128_t cephes_f128_log2l(float128_t x) {\nVOLATILE float128_t z;\nfloat128_t y;\nint e;\n\nstruct softfloat_state state = {};\n\n/* Test for domain */\nif( f128_le(&state, x, zero) )\n\t{\n\tif( f128_eq(&state, x, zero) )\n\t\tmtherr( fname, SING );\n\telse\n\t\tmtherr( fname, DOMAIN );\n\treturn indeterminate;\n\t}\n\n/* separate mantissa from exponent */\n\n/* Note, frexp is used so that denormal numbers\n * will be handled properly.\n */\nx = cephes_f128_frexpl( x, &e );\n\n\n/* logarithm using log(x) = z + z**3 P(z)/Q(z),\n * where z = 2(x-1)/x+1)\n */\nif( (e > 2) || (e < -2) )\n{\nif( f128_lt(&state, x, SQRTH) )\n\t{ /* 2( 2x-1 )/( 2x+1 ) */\n\te -= 1;\n\tz = f128_sub(&state, x, f_0_p5);\n\ty = f128_add(&state, f128_mul(&state, f_0_p5, z), f_0_p5);\n\t}\t\nelse\n\t{ /* 2 (x-1)/(x+1) */\n\tz = f128_sub(&state, x, f_0_p5);\n\tz = f128_sub(&state, z, f_0_p5);\n\ty = f128_add(&state, f128_mul(&state, f_0_p5, x), f_0_p5);\n\t}\nx = f128_div(&state, z, y);\nz = f128_mul(&state, x, x);\ny = f128_mul(&state, x,\n f128_div(&state, f128_mul(&state, z, cephes_f128_polevll( z, R, 5 )), cephes_f128_p1evll( z, S, 6 ) ));\ngoto done;\n}\n\n\n/* logarithm using log(1+x) = x - .5x**2 + x**3 P(x)/Q(x) */\n\nif( f128_lt(&state, x, SQRTH) )\n\t{\n\te -= 1;\n\tx = f128_sub(&state, cephes_f128_ldexpl( x, 1 ), one); /* 2x - 1 */\n\t}\nelse\n\t{\n\tx = f128_sub(&state, x, one);\n\t}\nz = f128_mul(&state, x, x);\ny = f128_mul(&state, x,\n f128_div(&state, f128_mul(&state, z, cephes_f128_polevll( x, P, 12 )), cephes_f128_p1evll( x, Q, 12 )));\ny = f128_sub(&state, y, cephes_f128_ldexpl( z, -1 )); /* -0.5x^2 + ... */\n\ndone:\n\n/* Multiply log of fraction by log2(e)\n * and base 2 exponent by 1\n *\n * ***CAUTION***\n *\n * This sequence of operations is critical and it may\n * be horribly defeated by some compiler optimizers.\n */\nz = f128_mul(&state, y, LOG2EA);\nz = f128_add(&state, z, f128_mul(&state, x, LOG2EA));\nz = f128_add(&state, z, y);\nz = f128_add(&state, z, x);\nz = f128_add(&state, z, i32_to_f128(e));\nreturn( z );\n}\n\n" }, { "path": "External/cephes/src/128bit/mconf.h", "content": "/*\t\t\t\t\t\t\tmconf.h\n *\n *\tCommon include file for math routines\n *\n *\n *\n * SYNOPSIS:\n *\n * #include \"mconf.h\"\n *\n *\n *\n * DESCRIPTION:\n *\n * This file contains definitions for error codes that are\n * passed to the common error handling routine mtherr()\n * (which see).\n *\n * The file also includes a conditional assembly definition\n * for the type of computer arithmetic (IEEE, DEC, Motorola\n * IEEE, or UNKnown).\n * \n * For Digital Equipment PDP-11 and VAX computers, certain\n * IBM systems, and others that use numbers with a 56-bit\n * significand, the symbol DEC should be defined. In this\n * mode, most floating point constants are given as arrays\n * of octal integers to eliminate decimal to binary conversion\n * errors that might be introduced by the compiler.\n *\n * For little-endian computers, such as IBM PC, that follow the\n * IEEE Standard for Binary Floating Point Arithmetic (ANSI/IEEE\n * Std 754-1985), the symbol IBMPC should be defined. These\n * numbers have 53-bit significands. In this mode, constants\n * are provided as arrays of hexadecimal 16 bit integers.\n *\n * Big-endian IEEE format is denoted MIEEE. On some RISC\n * systems such as Sun SPARC, double precision constants\n * must be stored on 8-byte address boundaries. Since integer\n * arrays may be aligned differently, the MIEEE configuration\n * may fail on such machines.\n *\n * To accommodate other types of computer arithmetic, all\n * constants are also provided in a normal decimal radix\n * which one can hope are correctly converted to a suitable\n * format by the available C language compiler. To invoke\n * this mode, define the symbol UNK.\n *\n * An important difference among these modes is a predefined\n * set of machine arithmetic constants for each. The numbers\n * MACHEP (the machine roundoff error), MAXNUM (largest number\n * represented), and several other parameters are preset by\n * the configuration symbol. Check the file const.c to\n * ensure that these values are correct for your computer.\n *\n * Configurations NANS, INFINITIES, MINUSZERO, and DENORMAL\n * may fail on many systems. Verify that they are supposed\n * to work on your computer.\n */\n\f\n/*\nCephes Math Library Release 2.3: June, 1995\nCopyright 1984, 1987, 1989, 1995 by Stephen L. Moshier\n*/\n\n\n/* Constant definitions for math error conditions\n */\n\n#include \"SoftFloat-3e/platform.h\"\n#include \"SoftFloat-3e/softfloat.h\"\n\n#define DOMAIN\t\t1\t/* argument domain error */\n#define SING\t\t2\t/* argument singularity */\n#define OVERFLOW\t3\t/* overflow range error */\n#define UNDERFLOW\t4\t/* underflow range error */\n#define TLOSS\t\t5\t/* total loss of precision */\n#define PLOSS\t\t6\t/* partial loss of precision */\n\n#define EDOM\t\t33\n#define ERANGE\t\t34\n\n/* Complex numeral. */\ntypedef struct\n\t{\n\tdouble r;\n\tdouble i;\n\t} cmplx;\n\ntypedef struct\n\t{\n\tfloat r;\n\tfloat i;\n\t} cmplxf;\n\n/* Long double complex numeral. */\n\ntypedef struct\n\t{\n\tfloat128_t r;\n\tfloat128_t i;\n\t} cmplxl;\n\n\n/* Type of computer arithmetic */\n\n/* PDP-11, Pro350, VAX:\n */\n/* #define DEC 1 */\n\n/* Intel IEEE, low order words come first:\n */\n#define IBMPC 1\n\n/* Motorola IEEE, high order words come first\n * (Sun 680x0 workstation):\n */\n/* #define MIEEE 1 */\n\n/* UNKnown arithmetic, invokes coefficients given in\n * normal decimal format. Beware of range boundary\n * problems (MACHEP, MAXLOG, etc. in const.c) and\n * roundoff problems in pow.c:\n * (Sun SPARCstation)\n */\n/* #define UNK 1 */\n\n/* If you define UNK, then be sure to set BIGENDIAN properly. */\n/* #define BIGENDIAN 1 */\n\n/* Define this `volatile' if your compiler thinks\n * that floating point arithmetic obeys the associative\n * and distributive laws. It will defeat some optimizations\n * (but probably not enough of them).\n *\n * #define VOLATILE volatile\n */\n#define VOLATILE\n\n/* For 12-byte long doubles on an i386, pad a 16-bit short 0\n * to the end of real constants initialized by integer arrays.\n *\n * #define XPD 0,\n *\n * Otherwise, the type is 10 bytes long and XPD should be\n * defined blank (e.g., Microsoft C).\n *\n * #define XPD\n */\n#define XPD 0,\n\n/* Define to support tiny denormal numbers, else undefine. */\n#define DENORMAL 1\n\n/* Define to ask for infinity support, else undefine. */\n#define INFINITIES 1\n\n/* Define to ask for support of numbers that are Not-a-Number,\n else undefine. This may automatically define INFINITIES in some files. */\n#define NANS 1\n\n/* Define to distinguish between -0.0 and +0.0. */\n#define MINUSZERO 1\n\n/* Define 1 for ANSI C atan2() function\n and ANSI prototypes for float arguments.\n See atan.c and clog.c. */\n#define ANSIC 1\n\n/* Variable for error reporting. See mtherr.c. */\nextern int merror;\n\n/* Forward declarations */\nextern float128_t F128_MINLOGL;\n\nextern float128_t F128_MAXNUML;\nextern float128_t F128_PIL;\nextern float128_t F128_PIO2L, F128_PIO4L;\n\nfloat128_t cephes_f128_atanl(float128_t x);\n#if ANSIC\nfloat128_t cephes_f128_atan2l( float128_t y, float128_t x );\n#else\nfloat128_t cephes_f128_atan2l( float128_t x, float128_t y );\n#endif\nfloat128_t cephes_f128_ceill(float128_t x);\nfloat128_t cephes_f128_cosl(float128_t x);\nfloat128_t cephes_f128_fabsl(float128_t x);\nfloat128_t cephes_f128_floorl(float128_t x);\nfloat128_t cephes_f128_frexpl( float128_t x, int *pw2 );\nint cephes_f128_isfinitel(float128_t x);\nint cephes_f128_isnanl(float128_t x);\nfloat128_t cephes_f128_ldexpl( float128_t x, int pw2 );\nfloat128_t cephes_f128_polevll( float128_t x, void *PP, int n );\nfloat128_t cephes_f128_p1evll( float128_t x, void *PP, int n );\nint cephes_f128_signbitl(float128_t x);\nfloat128_t cephes_f128_sinl(float128_t x);\nint mtherr( char *name, int code );\n\n/* Public symbol declarations */\nfloat128_t cephes_f128_log2l(float128_t x);\n" }, { "path": "External/cephes/src/128bit/mtherr.c", "content": "/*\t\t\t\t\t\t\tmtherr.c\n *\n *\tLibrary common error handling routine\n *\n *\n *\n * SYNOPSIS:\n *\n * char *fctnam;\n * int code;\n * int mtherr();\n *\n * mtherr( fctnam, code );\n *\n *\n *\n * DESCRIPTION:\n *\n * This routine may be called to report one of the following\n * error conditions (in the include file mconf.h).\n * \n * Mnemonic Value Significance\n *\n * DOMAIN 1 argument domain error\n * SING 2 function singularity\n * OVERFLOW 3 overflow range error\n * UNDERFLOW 4 underflow range error\n * TLOSS 5 total loss of precision\n * PLOSS 6 partial loss of precision\n * EDOM 33 Unix domain error code\n * ERANGE 34 Unix range error code\n *\n * The default version of the file prints the function name,\n * passed to it by the pointer fctnam, followed by the\n * error condition. The display is directed to the standard\n * output device. The routine then returns to the calling\n * program. Users may wish to modify the program to abort by\n * calling exit() under severe error conditions such as domain\n * errors.\n *\n * Since all error conditions pass control to this function,\n * the display may be easily changed, eliminated, or directed\n * to an error logging device.\n *\n * SEE ALSO:\n *\n * mconf.h\n *\n */\n\n/*\nCephes Math Library Release 2.0: April, 1987\nCopyright 1984, 1987 by Stephen L. Moshier\nDirect inquiries to 30 Frost Street, Cambridge, MA 02140\n*/\n\n#include \"mconf.h\"\n\nint merror = 0;\n\nint mtherr( char *name, int code )\n{\n#if 0\n/* Display string passed by calling program,\n * which is supposed to be the name of the\n * function in which the error occurred:\n */\nprintf( \"\\n%s \", name );\n\n/* Set global error message word */\nmerror = code;\n\n/* Display error message defined\n * by the code argument.\n */\nif( (code <= 0) || (code >= 7) )\n\tcode = 0;\nprintf( \"%s error\\n\", ermsg[code] );\n\n#endif\n/* Return to calling\n * program\n */\nreturn( 0 );\n}\n" }, { "path": "External/cephes/src/128bit/polevll.c", "content": "/*\t\t\t\t\t\t\tpolevll.c\n *\t\t\t\t\t\t\tp1evll.c\n *\n *\tEvaluate polynomial\n *\n *\n *\n * SYNOPSIS:\n *\n * int N;\n * float128_t x, y, coef[N+1], polevl[];\n *\n * y = polevll( x, coef, N );\n *\n *\n *\n * DESCRIPTION:\n *\n * Evaluates polynomial of degree N:\n *\n * 2 N\n * y = C + C x + C x +...+ C x\n * 0 1 2 N\n *\n * Coefficients are stored in reverse order:\n *\n * coef[0] = C , ..., coef[N] = C .\n * N 0\n *\n * The function p1evll() assumes that coef[N] = 1.0 and is\n * omitted from the array. Its calling arguments are\n * otherwise the same as polevll().\n *\n *\n * SPEED:\n *\n * In the interest of speed, there are no checks for out\n * of bounds arithmetic. This routine is used by most of\n * the functions in the library. Depending on available\n * equipment features, the user may wish to rewrite the\n * program in microcode or assembly language.\n *\n */\n\f\n\n/*\nCephes Math Library Release 2.2: July, 1992\nCopyright 1984, 1987, 1988, 1992 by Stephen L. Moshier\nDirect inquiries to 30 Frost Street, Cambridge, MA 02140\n*/\n#include \"mconf.h\"\n\n\n/* Polynomial evaluator:\n * P[0] x^n + P[1] x^(n-1) + ... + P[n]\n */\nfloat128_t cephes_f128_polevll( float128_t x, void *PP, int n )\n{\n\nstruct softfloat_state state = {};\nregister float128_t y;\nfloat128_t *P;\n\nP = (float128_t *) PP;\ny = *P++;\ndo\n\t{\n\ty = f128_add(&state, f128_mul(&state, y, x), *P++);\n\t}\nwhile( --n );\nreturn(y);\n}\n\n\n\n/* Polynomial evaluator:\n * x^n + P[0] x^(n-1) + P[1] x^(n-2) + ... + P[n]\n */\nfloat128_t cephes_f128_p1evll( float128_t x, void *PP, int n )\n{\nstruct softfloat_state state = {};\nregister float128_t y;\nfloat128_t *P;\n\nP = (float128_t *) PP;\nn -= 1;\ny = f128_add(&state, x, *P++);\ndo\n\t{\n\ty = f128_add(&state, f128_mul(&state, y, x), *P++);\n\t}\nwhile( --n );\nreturn( y );\n}\n" }, { "path": "External/cephes/src/128bit/sinll.c", "content": "/*\t\t\t\t\t\t\tsinl.c\n *\n *\tCircular sine, float128_t precision\n *\n *\n *\n * SYNOPSIS:\n *\n * float128_t x, y, sinl();\n *\n * y = sinl( x );\n *\n *\n *\n * DESCRIPTION:\n *\n * Range reduction is into intervals of pi/4. The reduction\n * error is nearly eliminated by contriving an extended precision\n * modular arithmetic.\n *\n * Two polynomial approximating functions are employed.\n * Between 0 and pi/4 the sine is approximated by the Cody\n * and Waite polynomial form\n * x + x^3 P(x^2) .\n * Between pi/4 and pi/2 the cosine is represented as\n * 1 - .5 x^2 + x^4 Q(x^2) .\n *\n *\n * ACCURACY:\n *\n * Relative error:\n * arithmetic domain # trials peak rms\n * IEEE +-3.6e16 100,000 2.0e-34 5.3e-35\n *\n * ERROR MESSAGES:\n *\n * message condition value returned\n * sin total loss x > 2^55 0.0\n *\n */\n\f/*\t\t\t\t\t\t\tcosl.c\n *\n *\tCircular cosine, float128_t precision\n *\n *\n *\n * SYNOPSIS:\n *\n * float128_t x, y, cosl();\n *\n * y = cosl( x );\n *\n *\n *\n * DESCRIPTION:\n *\n * Range reduction is into intervals of pi/4. The reduction\n * error is nearly eliminated by contriving an extended precision\n * modular arithmetic.\n *\n * Two polynomial approximating functions are employed.\n * Between 0 and pi/4 the cosine is approximated by\n * 1 - .5 x^2 + x^4 Q(x^2) .\n * Between pi/4 and pi/2 the sine is represented by the Cody\n * and Waite polynomial form\n * x + x^3 P(x^2) .\n *\n *\n * ACCURACY:\n *\n * Relative error:\n * arithmetic domain # trials peak rms\n * IEEE +-3.6e16 100,000 2.0e-34 5.2e-35\n *\n * ERROR MESSAGES:\n *\n * message condition value returned\n * cos total loss x > 2^55 0.0\n */\n\f\n/*\t\t\t\t\t\t\tsin.c\t*/\n\n/*\nCephes Math Library Release 2.2: December, 1990\nCopyright 1985, 1990 by Stephen L. Moshier\nDirect inquiries to 30 Frost Street, Cambridge, MA 02140\n*/\n\n#include \"mconf.h\"\n\n/* sin(x) = x + x^3 P(x^2)\n * Theoretical peak relative error = 5.6e-39\n * relative peak error spread = 1.7e-9\n */\n\nstatic float128_t sincof[12] = {\n{0x07424c0cc240ddd5ULL, 0x3fab3d6c15b6d187ULL}, // 6.410290407010279602425714995528976754871E-26L,\n{0x0f48760e659301d0ULL, 0xbfb47619a65f0be7ULL}, // -3.868105354403065333804959405965295962871E-23L,\n{0xcb791f8ea7c13184ULL, 0x3fbd71b8ee9a64e1ULL}, // 1.957294039628045847156851410307133941611E-20L,\n{0x0b420eabbeb9d9bcULL, 0xbfc62f49b467cdf7ULL}, // -8.220635246181818130416407184286068307901E-18L,\n{0x4be70cee4054eef9ULL, 0x3fce952c77030ab5ULL}, // 2.811457254345322887443598804951004537784E-15L,\n{0xe782874b38cbd281ULL, 0xbfd6ae7f3e733b81ULL}, // -7.647163731819815869711749952353081768709E-13L,\n{0x97c83627668fe57cULL, 0x3fde6124613a86d0ULL}, // 1.605904383682161459812515654720205050216E-10L,\n{0x38fe73eef2ec94cdULL, 0xbfe5ae64567f544eULL}, // -2.505210838544171877505034150892770940116E-8L,\n{0x38faac1c6f6fa52aULL, 0x3fec71de3a556c73ULL}, // 2.755731922398589065255731765498970284004E-6L,\n{0xa01a01a019fc52ccULL, 0xbff2a01a01a01a01ULL}, // -1.984126984126984126984126984045294307281E-4L,\n{0x1111111111111083ULL, 0x3ff8111111111111ULL}, // 8.333333333333333333333333333333119885283E-3L,\n{0x5555555555555555ULL, 0xbffc555555555555ULL}, // -1.666666666666666666666666666666666647199E-1L\n};\n/* cos(x) = 1 - .5 x^2 + x^2 (x^2 P(x^2))\n * Theoretical peak relative error = 2.1e-37,\n * relative peak error spread = 1.4e-8\n */\nstatic float128_t coscof[11] = {\n{0x86919a6fdf15a4b3ULL, 0x3fafefc8801eb0a1ULL}, // 1.601961934248327059668321782499768648351E-24L,\n{0x902367b3281c9510ULL, 0xbfb90ce245980e11ULL}, // -8.896621117922334603659240022184527001401E-22L,\n{0xcf5102d043ad399aULL, 0x3fc1e542b8eb4f0dULL}, // 4.110317451243694098169570731967589555498E-19L,\n{0xa8272970c73ab5ffULL, 0xbfca6827863b2960ULL}, // -1.561920696747074515985647487260202922160E-16L,\n{0xf9016edb75d1fb52ULL, 0x3fd2ae7f3e733b51ULL}, // 4.779477332386900932514186378501779328195E-14L,\n{0xc3e862188c1c1f15ULL, 0xbfda93974a8c07c9ULL}, // -1.147074559772972328629102981460088437917E-11L,\n{0x7b517ff3abf58399ULL, 0x3fe21eed8eff8d89ULL}, // 2.087675698786809897637922200570559726116E-9L,\n{0xc72eef5d4453f45cULL, 0xbfe927e4fb7789f5ULL}, // -2.755731922398589065255365968070684102298E-7L,\n{0xa01a019fdf56450dULL, 0x3fefa01a01a01a01ULL}, // 2.480158730158730158730158440896461945271E-5L,\n{0x6c16c16c16b76e10ULL, 0xbff56c16c16c16c1ULL}, // -1.388888888888888888888888888765724370132E-3L,\n{0x55555555555553fdULL, 0x3ffa555555555555ULL}, // 4.166666666666666666666666666666459301466E-2L\n};\n/*\nstatic float128_t DP1 = 7.853981554508209228515625E-1L;\nstatic float128_t DP2 = 7.94662735614792836713604629039764404296875E-9L;\nstatic float128_t DP3 = 3.0616169978683829430651648306875026455243736148E-17L;\nstatic float128_t lossth = 5.49755813888e11L;\n*/\nstatic float128_t DP1 =\n{0x8400000000000000ULL, 0x3ffe921fb54442d1ULL};\n //7.853981633974483067550664827649598009884357452392578125E-1L;\nstatic float128_t DP2 =\n{0xe000000000000000ULL, 0x3fc4a62633145c06ULL};\n //2.8605943630549158983813312792950660807511260829685741796657E-18L;\nstatic float128_t DP3 =\n{0xa67cc74020bbea64ULL, 0x3f8bcd129024e088ULL};\n //2.1679525325309452561992610065108379921905808E-35L;\n\nstatic const float128_t lossth = {0x0000000000000000ULL, 0x4036000000000000ULL}; // 3.6028797018963968E16L; /* 2^55 */\nstatic const float128_t zero = {0, 0};\nstatic const float128_t one = {0, 0x3fff000000000000ULL};\n\nfloat128_t cephes_f128_sinl(float128_t x)\n{\nstruct softfloat_state state = {};\nfloat128_t y, z, zz;\nint j, sign;\n\n/* make argument positive but save the sign */\nsign = 1;\nif( f128_lt(&state, x, zero) )\n\t{\n\tx = f128_complement_sign(x);\n\tsign = -1;\n\t}\n\nif( f128_lt(&state, lossth, x))\n\t{\n\tmtherr( \"sinl\", TLOSS );\n\treturn zero;\n\t}\n\ny = cephes_f128_floorl( f128_div(&state, x, F128_PIO4L) ); /* integer part of x/PIO4 */\n\n/* strip high bits of integer part to prevent integer overflow */\nz = cephes_f128_ldexpl( y, -4 );\nz = cephes_f128_floorl(z); /* integer part of y/8 */\nz = f128_sub(&state, y, cephes_f128_ldexpl( z, 4 )); /* y - 16 * (y/16) */\n\nj = f128_to_i32(&state, z, softfloat_round_near_even, true); /* convert to integer for tests on the phase angle */\n/* map zeros to origin */\nif( j & 1 )\n\t{\n\tj += 1;\n\ty = f128_add(&state, y, one);\n\t}\nj = j & 07; /* octant modulo 360 degrees */\n/* reflect in x axis */\nif( j > 3)\n\t{\n\tsign = -sign;\n\tj -= 4;\n\t}\n\n/* Extended precision modular arithmetic */\n// z = ((x - y * DP1) - y * DP2) - y * DP3;\n {\n float128_t tmp1 = f128_mul(&state, y, DP1);\n float128_t tmp2 = f128_mul(&state, y, DP2);\n float128_t tmp3 = f128_mul(&state, y, DP3);\n float128_t tmp4 = f128_sub(&state, x, tmp1);\n float128_t tmp5 = f128_sub(&state, tmp4, tmp2);\n z = f128_sub(&state, tmp5, tmp3);\n }\n\nz = f128_sub(&state, f128_sub(&state, f128_sub(&state, x, f128_mul(&state, y, DP1)), f128_mul(&state, y, DP2)), f128_mul(&state, y, DP3));\n\nzz = f128_mul(&state, z, z);\nif( (j==1) || (j==2) )\n\t{\n // y = 1.0L - ldexpl(zz,-1) + zz * zz * polevll( zz, coscof, 10 );\n float128_t tmp1 = f128_mul(&state, zz, zz);\n float128_t tmp2 = f128_mul(&state, tmp1, cephes_f128_polevll( zz, coscof, 10 ));\n float128_t tmp3 = f128_sub(&state, one, cephes_f128_ldexpl(zz,-1));\n y = f128_add(&state, tmp3, tmp2);\n\t}\nelse\n\t{\n // y = z + z * (zz * polevll( zz, sincof, 11 ));\n float128_t tmp1 = f128_mul(&state, zz, cephes_f128_polevll( zz, sincof, 11 ));\n float128_t tmp2 = f128_mul(&state, z, tmp1);\n y = f128_add(&state, z, tmp2);\n\t}\n\nif(sign < 0)\n\ty = f128_complement_sign(y);\n\nreturn(y);\n}\n\n\n\n\n\nfloat128_t cephes_f128_cosl(float128_t x)\n{\nstruct softfloat_state state = {};\nfloat128_t y, z, zz;\nlong i;\nint j, sign;\n\n/* make argument positive */\nsign = 1;\nif( f128_lt(&state, x, zero) )\n\tx = f128_complement_sign(x);\n\n\nif( f128_lt(&state, lossth, x))\n\t{\n\tmtherr( \"cosl\", TLOSS );\n\treturn zero;\n\t}\n\ny = cephes_f128_floorl( f128_div(&state, x, F128_PIO4L));\nz = cephes_f128_ldexpl( y, -4 );\nz = cephes_f128_floorl(z);\t\t/* integer part of y/8 */\nz = f128_sub(&state, y, cephes_f128_ldexpl( z, 4 )); /* y - 16 * (y/16) */\n\n/* integer and fractional part modulo one octant */\ni = f128_to_i32(&state, z, softfloat_round_near_even, true);\nif( i & 1 )\t/* map zeros to origin */\n\t{\n\ti += 1;\n\ty = f128_add(&state, y, one);\n\t}\nj = i & 07;\nif( j > 3)\n\t{\n\tj -=4;\n\tsign = -sign;\n\t}\n\nif( j > 1 )\n\tsign = -sign;\n\n/* Extended precision modular arithmetic */\n// z = ((x - y * DP1) - y * DP2) - y * DP3;\n {\n float128_t tmp1 = f128_mul(&state, y, DP1);\n float128_t tmp2 = f128_mul(&state, y, DP2);\n float128_t tmp3 = f128_mul(&state, y, DP3);\n float128_t tmp4 = f128_sub(&state, x, tmp1);\n float128_t tmp5 = f128_sub(&state, tmp4, tmp2);\n z = f128_sub(&state, tmp5, tmp3);\n }\n\nzz = f128_mul(&state, z, z);\nif( (j==1) || (j==2) )\n\t{\n // y = z + z * (zz * polevll( zz, sincof, 11 ));\n float128_t tmp1 = f128_mul(&state, zz, cephes_f128_polevll( zz, sincof, 11 ));\n float128_t tmp2 = f128_mul(&state, z, tmp1);\n y = f128_add(&state, z, tmp2);\n\t}\nelse\n\t{\n // y = 1.0L - ldexpl(zz,-1) + zz * zz * polevll( zz, coscof, 10 );\n float128_t tmp1 = f128_mul(&state, zz, zz);\n float128_t tmp2 = f128_mul(&state, tmp1, cephes_f128_polevll( zz, coscof, 10 ));\n float128_t tmp3 = f128_sub(&state, one, cephes_f128_ldexpl(zz,-1));\n y = f128_add(&state, tmp3, tmp2);\n\t}\n\nif(sign < 0)\n\ty = f128_complement_sign(y);\n\nreturn(y);\n}\n" }, { "path": "External/cephes/src/128bit/tanll.c", "content": "/*\t\t\t\t\t\t\ttanl.c\n *\n *\tCircular tangent, 128-bit float128_t precision\n *\n *\n *\n * SYNOPSIS:\n *\n * float128_t x, y, tanl();\n *\n * y = tanl( x );\n *\n *\n *\n * DESCRIPTION:\n *\n * Returns the circular tangent of the radian argument x.\n *\n * Range reduction is modulo pi/4. A rational function\n * x + x**3 P(x**2)/Q(x**2)\n * is employed in the basic interval [0, pi/4].\n *\n *\n *\n * ACCURACY:\n *\n * Relative error:\n * arithmetic domain # trials peak rms\n * IEEE +-3.6e16 100,000 3.0e-34 7.2e-35\n *\n * ERROR MESSAGES:\n *\n * message condition value returned\n * tan total loss x > 2^55 0.0\n *\n */\n\f/*\t\t\t\t\t\t\tcotl.c\n *\n *\tCircular cotangent, float128_t precision\n *\n *\n *\n * SYNOPSIS:\n *\n * float128_t x, y, cotl();\n *\n * y = cotl( x );\n *\n *\n *\n * DESCRIPTION:\n *\n * Returns the circular cotangent of the radian argument x.\n *\n * Range reduction is modulo pi/4. A rational function\n * x + x**3 P(x**2)/Q(x**2)\n * is employed in the basic interval [0, pi/4].\n *\n *\n *\n * ACCURACY:\n *\n * Relative error:\n * arithmetic domain # trials peak rms\n * IEEE +-3.6e16 100,000 2.9e-34 7.2e-35\n *\n *\n * ERROR MESSAGES:\n *\n * message condition value returned\n * cot total loss x > 2^55 0.0\n * cot singularity x = 0 MAXNUM\n *\n */\n\f\n/*\nCephes Math Library Release 2.2: December, 1990\nCopyright 1984, 1990 by Stephen L. Moshier\nDirect inquiries to 30 Frost Street, Cambridge, MA 02140\n*/\n\n#include \"mconf.h\"\n\n/* tan(x) = x + x^3 P(x^2)\n * 0 <= |x| <= pi/4\n * Theoretical peak relative error = 4.3e-38\n * relative peak error spread = 6.1e-11\n */\nstatic float128_t P[6] = {\n {0x09978dc7ae2a2f4bULL, 0xbffefa5d486820e2ULL}, // -9.889929415807650724957118893791829849557E-1L,\n {0x52a017b1ca7c4799ULL, 0x40093e130edd1294ULL}, // 1.272297782199996882828849455156962260810E3L,\n {0x8857161b398b3c53ULL, 0xc0119f024bdcc6c3ULL}, // -4.249691853501233575668486667664718192660E5L,\n {0xcc299261a6616b83ULL, 0x401889b0ed404622ULL}, // 5.160188250214037865511600561074819366815E7L,\n {0x37d9311de4cdbf04ULL, 0xc01e1304fe4d6331ULL}, // -2.307030822693734879744223131873392503321E9L,\n {0x6e9f0eac6b638a9aULL, 0x4021ada98af62f83ULL}, // 2.883414728874239697964612246732416606301E10L\n};\nstatic float128_t Q[6] = {\n/* 1.000000000000000000000000000000000000000E0L, */\n {0xeb01d728f7d3bb04ULL, 0xc009494f98d3c1caULL}, // -1.317243702830553658702531997959756728291E3L,\n {0xcdd312b4ac46a6cdULL, 0x4011ba538d331a98ULL}, // 4.529422062441341616231663543669583527923E5L,\n {0x2a1a6372eebd73a1ULL, 0xc018b57281a9f10bULL}, // -5.733709132766856723608447733926138506824E7L,\n {0x3e9defb0e348fbe5ULL, 0x401e48d6025d9b41ULL}, // 2.758476078803232151774723646710890525496E9L,\n {0x7cd82869db5580d1ULL, 0xc022355d0fdbd24eULL}, // -4.152206921457208101480801635640958361612E10L,\n {0x92f74b01508aa7f3ULL, 0x4023423f2838a3a2ULL}, // 8.650244186622719093893836740197250197602E10L\n};\n\nstatic float128_t DP1 =\n{0x8400000000000000ULL, 0x3ffe921fb54442d1ULL};\n //7.853981633974483067550664827649598009884357452392578125E-1L;\nstatic float128_t DP2 =\n{0xe000000000000000ULL, 0x3fc4a62633145c06ULL};\n //2.8605943630549158983813312792950660807511260829685741796657E-18L;\nstatic float128_t DP3 =\n{0xa67cc74020bbea64ULL, 0x3f8bcd129024e088ULL};\n // 2.1679525325309452561992610065108379921905808E-35L;\n\nstatic const float128_t lossth = {0x0000000000000000ULL, 0x4036000000000000ULL}; // 3.6028797018963968E16L; /* 2^55 */\n\nstatic const float128_t zero = {0, 0};\nstatic const float128_t one = {0, 0x3fff000000000000ULL};\nstatic const float128_t neg_one = {0, 0xbfff000000000000ULL};\n\nstatic const float128_t max_quad = {0x35d511e976394d7aULL, 0x3fbc79ca10c92422ULL};\n\nstatic float128_t tancotl( struct softfloat_state *state, float128_t xx, int cotflg );\n\nfloat128_t cephes_f128_tanl(float128_t x)\n{\nstruct softfloat_state state = {};\nreturn( tancotl(&state, x,0) );\n}\n\n\nfloat128_t cotl(float128_t x)\n{\nstruct softfloat_state state = {};\n\nif( f128_eq(&state, x, zero) )\n\t{\n\tmtherr( \"cotl\", SING );\n\treturn( F128_MAXNUML );\n\t}\nreturn( tancotl(&state, x,1) );\n}\n\n\nstatic float128_t tancotl( struct softfloat_state *state, float128_t xx, int cotflg )\n{\nfloat128_t x, y, z, zz;\nint j, sign;\n\n/* make argument positive but save the sign */\n// if (xx < 0.0L)\nif( f128_lt(state, xx, zero) )\n\t{\n\tx = f128_sub(state, zero, xx);\n\tsign = -1;\n\t}\nelse\n\t{\n\tx = xx;\n\tsign = 1;\n\t}\n\n//if (x > lossth)\nif (f128_lt(state, lossth, x))\n\t{\n\tif( cotflg )\n\t\tmtherr( \"cotl\", TLOSS );\n\telse\n\t\tmtherr( \"tanl\", TLOSS );\n\treturn zero;\n\t}\n\n/* compute x mod PIO4 */\ny = cephes_f128_floorl( f128_div(state, x, F128_PIO4L));\n\n/* strip high bits of integer part */\nz = cephes_f128_ldexpl( y, -4 );\nz = cephes_f128_floorl(z);\t\t/* integer part of y/16 */\nz = f128_sub(state, y, cephes_f128_ldexpl( z, 4 )); /* y - 16 * (y/16) */\n\n/* integer and fractional part modulo one octant */\nj = f128_to_i32(state, z, softfloat_round_near_even, true);\n\n/* map zeros and singularities to origin */\nif( j & 1 )\n\t{\n\tj += 1;\n\ty = f128_add(state, y, one);\n\t}\n\nz = f128_sub(state, f128_sub(state, f128_sub(state, x, f128_mul(state, y, DP1)), f128_mul(state, y, DP2)), f128_mul(state, y, DP3));\n\nzz = f128_mul(state, z, z);\n\n// if( zz > 1.0e-20L )\nif (f128_lt(state, max_quad, zz))\n{\n\ty = f128_add(state, z, f128_mul(state, z, f128_div(state, f128_mul(state, zz, cephes_f128_polevll( zz, P, 5 )), cephes_f128_p1evll(zz, Q, 6))));\n}\nelse\n{\n\ty = z;\n}\n\t\nif( j & 2 )\n\t{\n\tif( cotflg )\n y = f128_complement_sign(y);\n\telse\n\t\ty = f128_div(state, neg_one, y);\n\t}\nelse\n\t{\n\tif( cotflg )\n\t\ty = f128_div(state, one, y);\n\t}\n\nif( sign < 0 )\n y = f128_complement_sign(y);\n\nreturn( y );\n}\n" }, { "path": "External/code-format-helper/code-format-helper.py", "content": "#!/usr/bin/env python3\n#\n# ====- code-format-helper, runs code formatters from the ci or in a hook --*- python -*--==#\n#\n# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.\n# See https://llvm.org/LICENSE.txt for license information.\n# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception\n#\n# ==--------------------------------------------------------------------------------------==#\n\nimport argparse\nimport os\nimport subprocess\nimport sys\nfrom typing import List, Optional\n\n\"\"\"\nThis script is run by GitHub actions to ensure that the code in PR's conform to\nthe coding style of LLVM. It can also be installed as a pre-commit git hook to\ncheck the coding style before submitting it. The canonical source of this script\nis in the LLVM source tree under llvm/utils/git.\n\nFor C/C++ code it uses clang-format.\n\nYou can learn more about the LLVM coding style on llvm.org:\nhttps://llvm.org/docs/CodingStandards.html\n\nYou can install this script as a git hook by symlinking it to the .git/hooks\ndirectory:\n\nln -s $(pwd)/llvm/utils/git/code-format-helper.py .git/hooks/pre-commit\n\nYou can control the exact path to clang-format with the following\nenvironment variable: $CLANG_FORMAT_PATH.\n\"\"\"\n\n\nclass FormatArgs:\n start_rev: str = None\n end_rev: str = None\n repo: str = None\n changed_files: List[str] = []\n token: str = None\n verbose: bool = True\n issue_number: int = 0\n write_comment_to_file: str = None\n\n def __init__(self, args: argparse.Namespace = None) -> None:\n if not args is None:\n self.start_rev = args.start_rev\n self.end_rev = args.end_rev\n self.repo = args.repo\n self.token = args.token\n self.changed_files = args.changed_files\n self.issue_number = args.issue_number\n self.write_comment_to_file = args.write_comment_to_file\n\n\nclass FormatHelper:\n COMMENT_TAG = \"\"\n name: str\n friendly_name: str\n comment: dict = None\n\n @property\n def comment_tag(self) -> str:\n return self.COMMENT_TAG.replace(\"fmt\", self.name)\n\n @property\n def instructions(self) -> str:\n raise NotImplementedError()\n\n def has_tool(self) -> bool:\n raise NotImplementedError()\n\n def format_run(self, changed_files: List[str], args: FormatArgs) -> Optional[str]:\n raise NotImplementedError()\n\n def pr_comment_text_for_diff(self, diff: str) -> str:\n return f\"\"\"\n:warning: {self.friendly_name}, {self.name} found issues in your code. :warning:\n\n
\n\nYou can test this locally with the following command:\n\n\n``````````bash\n{self.instructions}\n``````````\n\n
\n\n
\n\nView the diff from {self.name} here.\n\n\n``````````diff\n{diff}\n``````````\n\n
\n\"\"\"\n\n # TODO: any type should be replaced with the correct github type, but it requires refactoring to\n # not require the github module to be installed everywhere.\n def find_comment(self, pr: any) -> any:\n for comment in pr.as_issue().get_comments():\n if self.comment_tag in comment.body:\n return comment\n return None\n\n def update_pr(self, comment_text: str, args: FormatArgs, create_new: bool) -> None:\n import github\n from github import IssueComment, PullRequest\n\n repo = github.Github(args.token).get_repo(args.repo)\n pr = repo.get_issue(args.issue_number).as_pull_request()\n\n comment_text = self.comment_tag + \"\\n\\n\" + comment_text\n\n existing_comment = self.find_comment(pr)\n\n if args.write_comment_to_file:\n if create_new or existing_comment:\n self.comment = {\"body\": comment_text}\n if existing_comment:\n self.comment[\"id\"] = existing_comment.id\n return\n\n if existing_comment:\n existing_comment.edit(comment_text)\n elif create_new:\n pr.as_issue().create_comment(comment_text)\n\n def run(self, changed_files: List[str], args: FormatArgs) -> bool:\n changed_files = [arg for arg in changed_files if \"third-party\" not in arg]\n diff = self.format_run(changed_files, args)\n should_update_gh = args.token is not None and args.repo is not None\n\n if diff is None:\n if should_update_gh:\n comment_text = (\n \":white_check_mark: With the latest revision \"\n f\"this PR passed the {self.friendly_name}.\"\n )\n self.update_pr(comment_text, args, create_new=False)\n return True\n elif len(diff) > 0:\n if should_update_gh:\n comment_text = self.pr_comment_text_for_diff(diff)\n self.update_pr(comment_text, args, create_new=True)\n else:\n print(\n f\"Warning: {self.friendly_name}, {self.name} detected \"\n \"some issues with your code formatting...\"\n )\n return False\n else:\n # The formatter failed but didn't output a diff (e.g. some sort of\n # infrastructure failure).\n comment_text = (\n f\":warning: The {self.friendly_name} failed without printing \"\n \"a diff. Check the logs for stderr output. :warning:\"\n )\n self.update_pr(comment_text, args, create_new=False)\n return False\n\n\nclass ClangFormatHelper(FormatHelper):\n name = \"git-clang-format\"\n friendly_name = \"C/C++ code formatter\"\n\n\n @property\n def instructions(self) -> str:\n return \" \".join(self.cf_cmd)\n\n def should_include_extensionless_file(self, path: str) -> bool:\n return path.startswith(\"libcxx/include\")\n\n def filter_changed_files(self, changed_files: List[str]) -> List[str]:\n filtered_files = []\n for path in changed_files:\n _, ext = os.path.splitext(path)\n if ext in (\".cpp\", \".c\", \".h\", \".hpp\", \".hxx\", \".cxx\", \".inc\", \".cppm\"):\n filtered_files.append(path)\n elif ext == \"\" and self.should_include_extensionless_file(path):\n filtered_files.append(path)\n return filtered_files\n\n @property\n def clang_fmt_path(self) -> str:\n if \"CLANG_FORMAT_PATH\" in os.environ:\n return os.environ[\"CLANG_FORMAT_PATH\"]\n return \"git-clang-format-19\"\n\n def has_tool(self) -> bool:\n cmd = [self.clang_fmt_path, \"-h\"]\n proc = None\n try:\n proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)\n except:\n return False\n return proc.returncode == 0\n\n def format_run(self, changed_files: List[str], args: FormatArgs) -> Optional[str]:\n cpp_files = self.filter_changed_files(changed_files)\n if not cpp_files:\n return None\n\n cf_cmd = [\n self.clang_fmt_path,\n \"--binary=clang-format-19\",\n \"--diff\",\n ]\n\n if args.start_rev and args.end_rev:\n cf_cmd.append(args.start_rev)\n cf_cmd.append(args.end_rev)\n\n cf_cmd.append(\"--\")\n cf_cmd += cpp_files\n\n if args.verbose:\n print(f\"Running: {' '.join(cf_cmd)}\")\n self.cf_cmd = cf_cmd\n proc = subprocess.run(cf_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)\n sys.stdout.write(proc.stderr.decode(\"utf-8\"))\n\n if proc.returncode != 0:\n # formatting needed, or the command otherwise failed\n if args.verbose:\n print(f\"error: {self.name} exited with code {proc.returncode}\")\n # Print the diff in the log so that it is viewable there\n print(proc.stdout.decode(\"utf-8\"))\n return proc.stdout.decode(\"utf-8\")\n else:\n return None\n\nALL_FORMATTERS = [ClangFormatHelper()]\n\ndef hook_main():\n # fill out args\n args = FormatArgs()\n args.verbose = False\n\n # find the changed files\n cmd = [\"git\", \"diff\", \"--cached\", \"--name-only\", \"--diff-filter=d\"]\n proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)\n output = proc.stdout.decode(\"utf-8\")\n for line in output.splitlines():\n args.changed_files.append(line)\n\n failed_fmts = []\n for fmt in ALL_FORMATTERS:\n if fmt.has_tool():\n if not fmt.run(args.changed_files, args):\n failed_fmts.append(fmt.name)\n if fmt.comment:\n comments.append(fmt.comment)\n else:\n print(f\"Couldn't find {fmt.name}, can't check \" + fmt.friendly_name.lower())\n\n if len(failed_fmts) > 0:\n sys.exit(1)\n\n sys.exit(0)\n\n\nif __name__ == \"__main__\":\n script_path = os.path.abspath(__file__)\n if \".git/hooks\" in script_path:\n hook_main()\n sys.exit(0)\n\n parser = argparse.ArgumentParser()\n parser.add_argument(\n \"--token\", type=str, required=False, help=\"GitHub authentication token\"\n )\n parser.add_argument(\n \"--repo\",\n type=str,\n default=os.getenv(\"GITHUB_REPOSITORY\", \"llvm/llvm-project\"),\n help=\"The GitHub repository that we are working with in the form of / (e.g. llvm/llvm-project)\",\n )\n parser.add_argument(\"--issue-number\", type=int, required=True)\n parser.add_argument(\n \"--start-rev\",\n type=str,\n required=True,\n help=\"Compute changes from this revision.\",\n )\n parser.add_argument(\n \"--end-rev\", type=str, required=True, help=\"Compute changes to this revision\"\n )\n parser.add_argument(\n \"--changed-files\",\n type=str,\n help=\"Comma separated list of files that has been changed\",\n )\n parser.add_argument(\n \"--write-comment-to-file\",\n type=str,\n help=\"Don't post comments on the PR, instead write the comments and metadata a file\",\n )\n\n args = FormatArgs(parser.parse_args())\n\n changed_files = []\n if args.changed_files:\n changed_files = args.changed_files.split(\",\")\n\n failed_formatters = []\n comments = []\n for fmt in ALL_FORMATTERS:\n if not fmt.run(changed_files, args):\n failed_formatters.append(fmt.name)\n if fmt.comment:\n comments.append(fmt.comment)\n\n if len(comments):\n with open(args.write_comment_to_file, \"w\") as f:\n import json\n\n json.dump(comments, f)\n\n if len(failed_formatters) > 0:\n print(f\"error: some formatters failed: {' '.join(failed_formatters)}\")\n sys.exit(1)\n" }, { "path": "External/code-format-helper/requirements_formatting.txt", "content": "#\n# This file is autogenerated by pip-compile with Python 3.13\n# by the following command:\n#\n# pip-compile --generate-hashes --output-file=requirements_formatting.txt --strip-extras requirements_formatting.txt.in\n#\nblack==26.3.1 \\\n --hash=sha256:0126ae5b7c09957da2bdbd91a9ba1207453feada9e9fe51992848658c6c8e01c \\\n --hash=sha256:0f76ff19ec5297dd8e66eb64deda23631e642c9393ab592826fd4bdc97a4bce7 \\\n --hash=sha256:28ef38aee69e4b12fda8dba75e21f9b4f979b490c8ac0baa7cb505369ac9e1ff \\\n --hash=sha256:2bd5aa94fc267d38bb21a70d7410a89f1a1d318841855f698746f8e7f51acd1b \\\n --hash=sha256:2c50f5063a9641c7eed7795014ba37b0f5fa227f3d408b968936e24bc0566b07 \\\n --hash=sha256:2d6bfaf7fd0993b420bed691f20f9492d53ce9a2bcccea4b797d34e947318a78 \\\n --hash=sha256:41cd2012d35b47d589cb8a16faf8a32ef7a336f56356babd9fcf70939ad1897f \\\n --hash=sha256:474c27574d6d7037c1bc875a81d9be0a9a4f9ee95e62800dab3cfaadbf75acd5 \\\n --hash=sha256:5602bdb96d52d2d0672f24f6ffe5218795736dd34807fd0fd55ccd6bf206168b \\\n --hash=sha256:5e9d0d86df21f2e1677cc4bd090cd0e446278bcbbe49bf3659c308c3e402843e \\\n --hash=sha256:5ed0ca58586c8d9a487352a96b15272b7fa55d139fc8496b519e78023a8dab0a \\\n --hash=sha256:6c54a4a82e291a1fee5137371ab488866b7c86a3305af4026bdd4dc78642e1ac \\\n --hash=sha256:6e131579c243c98f35bce64a7e08e87fb2d610544754675d4a0e73a070a5aa3a \\\n --hash=sha256:855822d90f884905362f602880ed8b5df1b7e3ee7d0db2502d4388a954cc8c54 \\\n --hash=sha256:86a8b5035fce64f5dcd1b794cf8ec4d31fe458cf6ce3986a30deb434df82a1d2 \\\n --hash=sha256:8a33d657f3276328ce00e4d37fe70361e1ec7614da5d7b6e78de5426cb56332f \\\n --hash=sha256:92c0ec1f2cc149551a2b7b47efc32c866406b6891b0ee4625e95967c8f4acfb1 \\\n --hash=sha256:9a5e9f45e5d5e1c5b5c29b3bd4265dcc90e8b92cf4534520896ed77f791f4da5 \\\n --hash=sha256:afc622538b430aa4c8c853f7f63bc582b3b8030fd8c80b70fb5fa5b834e575c2 \\\n --hash=sha256:b07fc0dab849d24a80a29cfab8d8a19187d1c4685d8a5e6385a5ce323c1f015f \\\n --hash=sha256:b5e6f89631eb88a7302d416594a32faeee9fb8fb848290da9d0a5f2903519fc1 \\\n --hash=sha256:bf9bf162ed91a26f1adba8efda0b573bc6924ec1408a52cc6f82cb73ec2b142c \\\n --hash=sha256:c7e72339f841b5a237ff14f7d3880ddd0fc7f98a1199e8c4327f9a4f478c1839 \\\n --hash=sha256:ddb113db38838eb9f043623ba274cfaf7d51d5b0c22ecb30afe58b1bb8322983 \\\n --hash=sha256:dfdd51fc3e64ea4f35873d1b3fb25326773d55d2329ff8449139ebaad7357efb \\\n --hash=sha256:f1cd08e99d2f9317292a311dfe578fd2a24b15dbce97792f9c4d752275c1fa56 \\\n --hash=sha256:f89f2ab047c76a9c03f78d0d66ca519e389519902fa27e7a91117ef7611c0568\n # via\n # -r requirements_formatting.txt.in\n # darker\ncertifi==2025.7.14 \\\n --hash=sha256:6b31f564a415d79ee77df69d757bb49a5bb53bd9f756cbbe24394ffd6fc1f4b2 \\\n --hash=sha256:8ea99dbdfaaf2ba2f9bac77b9249ef62ec5218e7c2b2e903378ed5fccf765995\n # via\n # -r requirements_formatting.txt.in\n # requests\ncffi==2.0.0 \\\n --hash=sha256:00bdf7acc5f795150faa6957054fbbca2439db2f775ce831222b66f192f03beb \\\n --hash=sha256:07b271772c100085dd28b74fa0cd81c8fb1a3ba18b21e03d7c27f3436a10606b \\\n --hash=sha256:087067fa8953339c723661eda6b54bc98c5625757ea62e95eb4898ad5e776e9f \\\n --hash=sha256:0a1527a803f0a659de1af2e1fd700213caba79377e27e4693648c2923da066f9 \\\n --hash=sha256:0cf2d91ecc3fcc0625c2c530fe004f82c110405f101548512cce44322fa8ac44 \\\n --hash=sha256:0f6084a0ea23d05d20c3edcda20c3d006f9b6f3fefeac38f59262e10cef47ee2 \\\n --hash=sha256:12873ca6cb9b0f0d3a0da705d6086fe911591737a59f28b7936bdfed27c0d47c \\\n --hash=sha256:19f705ada2530c1167abacb171925dd886168931e0a7b78f5bffcae5c6b5be75 \\\n --hash=sha256:1cd13c99ce269b3ed80b417dcd591415d3372bcac067009b6e0f59c7d4015e65 \\\n --hash=sha256:1e3a615586f05fc4065a8b22b8152f0c1b00cdbc60596d187c2a74f9e3036e4e \\\n --hash=sha256:1f72fb8906754ac8a2cc3f9f5aaa298070652a0ffae577e0ea9bd480dc3c931a \\\n --hash=sha256:1fc9ea04857caf665289b7a75923f2c6ed559b8298a1b8c49e59f7dd95c8481e \\\n --hash=sha256:203a48d1fb583fc7d78a4c6655692963b860a417c0528492a6bc21f1aaefab25 \\\n --hash=sha256:2081580ebb843f759b9f617314a24ed5738c51d2aee65d31e02f6f7a2b97707a \\\n --hash=sha256:21d1152871b019407d8ac3985f6775c079416c282e431a4da6afe7aefd2bccbe \\\n --hash=sha256:24b6f81f1983e6df8db3adc38562c83f7d4a0c36162885ec7f7b77c7dcbec97b \\\n --hash=sha256:256f80b80ca3853f90c21b23ee78cd008713787b1b1e93eae9f3d6a7134abd91 \\\n --hash=sha256:28a3a209b96630bca57cce802da70c266eb08c6e97e5afd61a75611ee6c64592 \\\n --hash=sha256:2c8f814d84194c9ea681642fd164267891702542f028a15fc97d4674b6206187 \\\n --hash=sha256:2de9a304e27f7596cd03d16f1b7c72219bd944e99cc52b84d0145aefb07cbd3c \\\n --hash=sha256:38100abb9d1b1435bc4cc340bb4489635dc2f0da7456590877030c9b3d40b0c1 \\\n --hash=sha256:3925dd22fa2b7699ed2617149842d2e6adde22b262fcbfada50e3d195e4b3a94 \\\n --hash=sha256:3e17ed538242334bf70832644a32a7aae3d83b57567f9fd60a26257e992b79ba \\\n --hash=sha256:3e837e369566884707ddaf85fc1744b47575005c0a229de3327f8f9a20f4efeb \\\n --hash=sha256:3f4d46d8b35698056ec29bca21546e1551a205058ae1a181d871e278b0b28165 \\\n --hash=sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529 \\\n --hash=sha256:45d5e886156860dc35862657e1494b9bae8dfa63bf56796f2fb56e1679fc0bca \\\n --hash=sha256:4647afc2f90d1ddd33441e5b0e85b16b12ddec4fca55f0d9671fef036ecca27c \\\n --hash=sha256:4671d9dd5ec934cb9a73e7ee9676f9362aba54f7f34910956b84d727b0d73fb6 \\\n --hash=sha256:53f77cbe57044e88bbd5ed26ac1d0514d2acf0591dd6bb02a3ae37f76811b80c \\\n --hash=sha256:5eda85d6d1879e692d546a078b44251cdd08dd1cfb98dfb77b670c97cee49ea0 \\\n --hash=sha256:5fed36fccc0612a53f1d4d9a816b50a36702c28a2aa880cb8a122b3466638743 \\\n --hash=sha256:61d028e90346df14fedc3d1e5441df818d095f3b87d286825dfcbd6459b7ef63 \\\n --hash=sha256:66f011380d0e49ed280c789fbd08ff0d40968ee7b665575489afa95c98196ab5 \\\n --hash=sha256:6824f87845e3396029f3820c206e459ccc91760e8fa24422f8b0c3d1731cbec5 \\\n --hash=sha256:6c6c373cfc5c83a975506110d17457138c8c63016b563cc9ed6e056a82f13ce4 \\\n --hash=sha256:6d02d6655b0e54f54c4ef0b94eb6be0607b70853c45ce98bd278dc7de718be5d \\\n --hash=sha256:6d50360be4546678fc1b79ffe7a66265e28667840010348dd69a314145807a1b \\\n --hash=sha256:730cacb21e1bdff3ce90babf007d0a0917cc3e6492f336c2f0134101e0944f93 \\\n --hash=sha256:737fe7d37e1a1bffe70bd5754ea763a62a066dc5913ca57e957824b72a85e205 \\\n --hash=sha256:74a03b9698e198d47562765773b4a8309919089150a0bb17d829ad7b44b60d27 \\\n --hash=sha256:7553fb2090d71822f02c629afe6042c299edf91ba1bf94951165613553984512 \\\n --hash=sha256:7a66c7204d8869299919db4d5069a82f1561581af12b11b3c9f48c584eb8743d \\\n --hash=sha256:7cc09976e8b56f8cebd752f7113ad07752461f48a58cbba644139015ac24954c \\\n --hash=sha256:81afed14892743bbe14dacb9e36d9e0e504cd204e0b165062c488942b9718037 \\\n --hash=sha256:8941aaadaf67246224cee8c3803777eed332a19d909b47e29c9842ef1e79ac26 \\\n --hash=sha256:89472c9762729b5ae1ad974b777416bfda4ac5642423fa93bd57a09204712322 \\\n --hash=sha256:8ea985900c5c95ce9db1745f7933eeef5d314f0565b27625d9a10ec9881e1bfb \\\n --hash=sha256:8eca2a813c1cb7ad4fb74d368c2ffbbb4789d377ee5bb8df98373c2cc0dee76c \\\n --hash=sha256:92b68146a71df78564e4ef48af17551a5ddd142e5190cdf2c5624d0c3ff5b2e8 \\\n --hash=sha256:9332088d75dc3241c702d852d4671613136d90fa6881da7d770a483fd05248b4 \\\n --hash=sha256:94698a9c5f91f9d138526b48fe26a199609544591f859c870d477351dc7b2414 \\\n --hash=sha256:9a67fc9e8eb39039280526379fb3a70023d77caec1852002b4da7e8b270c4dd9 \\\n --hash=sha256:9de40a7b0323d889cf8d23d1ef214f565ab154443c42737dfe52ff82cf857664 \\\n --hash=sha256:a05d0c237b3349096d3981b727493e22147f934b20f6f125a3eba8f994bec4a9 \\\n --hash=sha256:afb8db5439b81cf9c9d0c80404b60c3cc9c3add93e114dcae767f1477cb53775 \\\n --hash=sha256:b18a3ed7d5b3bd8d9ef7a8cb226502c6bf8308df1525e1cc676c3680e7176739 \\\n --hash=sha256:b1e74d11748e7e98e2f426ab176d4ed720a64412b6a15054378afdb71e0f37dc \\\n --hash=sha256:b21e08af67b8a103c71a250401c78d5e0893beff75e28c53c98f4de42f774062 \\\n --hash=sha256:b4c854ef3adc177950a8dfc81a86f5115d2abd545751a304c5bcf2c2c7283cfe \\\n --hash=sha256:b882b3df248017dba09d6b16defe9b5c407fe32fc7c65a9c69798e6175601be9 \\\n --hash=sha256:baf5215e0ab74c16e2dd324e8ec067ef59e41125d3eade2b863d294fd5035c92 \\\n --hash=sha256:c649e3a33450ec82378822b3dad03cc228b8f5963c0c12fc3b1e0ab940f768a5 \\\n --hash=sha256:c654de545946e0db659b3400168c9ad31b5d29593291482c43e3564effbcee13 \\\n --hash=sha256:c6638687455baf640e37344fe26d37c404db8b80d037c3d29f58fe8d1c3b194d \\\n --hash=sha256:c8d3b5532fc71b7a77c09192b4a5a200ea992702734a2e9279a37f2478236f26 \\\n --hash=sha256:cb527a79772e5ef98fb1d700678fe031e353e765d1ca2d409c92263c6d43e09f \\\n --hash=sha256:cf364028c016c03078a23b503f02058f1814320a56ad535686f90565636a9495 \\\n --hash=sha256:d48a880098c96020b02d5a1f7d9251308510ce8858940e6fa99ece33f610838b \\\n --hash=sha256:d68b6cef7827e8641e8ef16f4494edda8b36104d79773a334beaa1e3521430f6 \\\n --hash=sha256:d9b29c1f0ae438d5ee9acb31cadee00a58c46cc9c0b2f9038c6b0b3470877a8c \\\n --hash=sha256:d9b97165e8aed9272a6bb17c01e3cc5871a594a446ebedc996e2397a1c1ea8ef \\\n --hash=sha256:da68248800ad6320861f129cd9c1bf96ca849a2771a59e0344e88681905916f5 \\\n --hash=sha256:da902562c3e9c550df360bfa53c035b2f241fed6d9aef119048073680ace4a18 \\\n --hash=sha256:dbd5c7a25a7cb98f5ca55d258b103a2054f859a46ae11aaf23134f9cc0d356ad \\\n --hash=sha256:dd4f05f54a52fb558f1ba9f528228066954fee3ebe629fc1660d874d040ae5a3 \\\n --hash=sha256:de8dad4425a6ca6e4e5e297b27b5c824ecc7581910bf9aee86cb6835e6812aa7 \\\n --hash=sha256:e11e82b744887154b182fd3e7e8512418446501191994dbf9c9fc1f32cc8efd5 \\\n --hash=sha256:e6e73b9e02893c764e7e8d5bb5ce277f1a009cd5243f8228f75f842bf937c534 \\\n --hash=sha256:f73b96c41e3b2adedc34a7356e64c8eb96e03a3782b535e043a986276ce12a49 \\\n --hash=sha256:f93fd8e5c8c0a4aa1f424d6173f14a892044054871c771f8566e4008eaa359d2 \\\n --hash=sha256:fc33c5141b55ed366cfaad382df24fe7dcbc686de5be719b207bb248e3053dc5 \\\n --hash=sha256:fc7de24befaeae77ba923797c7c87834c73648a05a4bde34b3b7e5588973a453 \\\n --hash=sha256:fe562eb1a64e67dd297ccc4f5addea2501664954f2692b69a76449ec7913ecbf\n # via\n # cryptography\n # pynacl\ncharset-normalizer==3.2.0 \\\n --hash=sha256:04e57ab9fbf9607b77f7d057974694b4f6b142da9ed4a199859d9d4d5c63fe96 \\\n --hash=sha256:09393e1b2a9461950b1c9a45d5fd251dc7c6f228acab64da1c9c0165d9c7765c \\\n --hash=sha256:0b87549028f680ca955556e3bd57013ab47474c3124dc069faa0b6545b6c9710 \\\n --hash=sha256:1000fba1057b92a65daec275aec30586c3de2401ccdcd41f8a5c1e2c87078706 \\\n --hash=sha256:1249cbbf3d3b04902ff081ffbb33ce3377fa6e4c7356f759f3cd076cc138d020 \\\n --hash=sha256:1920d4ff15ce893210c1f0c0e9d19bfbecb7983c76b33f046c13a8ffbd570252 \\\n --hash=sha256:193cbc708ea3aca45e7221ae58f0fd63f933753a9bfb498a3b474878f12caaad \\\n --hash=sha256:1a100c6d595a7f316f1b6f01d20815d916e75ff98c27a01ae817439ea7726329 \\\n --hash=sha256:1f30b48dd7fa1474554b0b0f3fdfdd4c13b5c737a3c6284d3cdc424ec0ffff3a \\\n --hash=sha256:203f0c8871d5a7987be20c72442488a0b8cfd0f43b7973771640fc593f56321f \\\n --hash=sha256:246de67b99b6851627d945db38147d1b209a899311b1305dd84916f2b88526c6 \\\n --hash=sha256:2dee8e57f052ef5353cf608e0b4c871aee320dd1b87d351c28764fc0ca55f9f4 \\\n --hash=sha256:2efb1bd13885392adfda4614c33d3b68dee4921fd0ac1d3988f8cbb7d589e72a \\\n --hash=sha256:2f4ac36d8e2b4cc1aa71df3dd84ff8efbe3bfb97ac41242fbcfc053c67434f46 \\\n --hash=sha256:3170c9399da12c9dc66366e9d14da8bf7147e1e9d9ea566067bbce7bb74bd9c2 \\\n --hash=sha256:3b1613dd5aee995ec6d4c69f00378bbd07614702a315a2cf6c1d21461fe17c23 \\\n --hash=sha256:3bb3d25a8e6c0aedd251753a79ae98a093c7e7b471faa3aa9a93a81431987ace \\\n --hash=sha256:3bb7fda7260735efe66d5107fb7e6af6a7c04c7fce9b2514e04b7a74b06bf5dd \\\n --hash=sha256:41b25eaa7d15909cf3ac4c96088c1f266a9a93ec44f87f1d13d4a0e86c81b982 \\\n --hash=sha256:45de3f87179c1823e6d9e32156fb14c1927fcc9aba21433f088fdfb555b77c10 \\\n --hash=sha256:46fb8c61d794b78ec7134a715a3e564aafc8f6b5e338417cb19fe9f57a5a9bf2 \\\n --hash=sha256:48021783bdf96e3d6de03a6e39a1171ed5bd7e8bb93fc84cc649d11490f87cea \\\n --hash=sha256:4957669ef390f0e6719db3613ab3a7631e68424604a7b448f079bee145da6e09 \\\n --hash=sha256:5e86d77b090dbddbe78867a0275cb4df08ea195e660f1f7f13435a4649e954e5 \\\n --hash=sha256:6339d047dab2780cc6220f46306628e04d9750f02f983ddb37439ca47ced7149 \\\n --hash=sha256:681eb3d7e02e3c3655d1b16059fbfb605ac464c834a0c629048a30fad2b27489 \\\n --hash=sha256:6c409c0deba34f147f77efaa67b8e4bb83d2f11c8806405f76397ae5b8c0d1c9 \\\n --hash=sha256:7095f6fbfaa55defb6b733cfeb14efaae7a29f0b59d8cf213be4e7ca0b857b80 \\\n --hash=sha256:70c610f6cbe4b9fce272c407dd9d07e33e6bf7b4aa1b7ffb6f6ded8e634e3592 \\\n --hash=sha256:72814c01533f51d68702802d74f77ea026b5ec52793c791e2da806a3844a46c3 \\\n --hash=sha256:7a4826ad2bd6b07ca615c74ab91f32f6c96d08f6fcc3902ceeedaec8cdc3bcd6 \\\n --hash=sha256:7c70087bfee18a42b4040bb9ec1ca15a08242cf5867c58726530bdf3945672ed \\\n --hash=sha256:855eafa5d5a2034b4621c74925d89c5efef61418570e5ef9b37717d9c796419c \\\n --hash=sha256:8700f06d0ce6f128de3ccdbc1acaea1ee264d2caa9ca05daaf492fde7c2a7200 \\\n --hash=sha256:89f1b185a01fe560bc8ae5f619e924407efca2191b56ce749ec84982fc59a32a \\\n --hash=sha256:8b2c760cfc7042b27ebdb4a43a4453bd829a5742503599144d54a032c5dc7e9e \\\n --hash=sha256:8c2f5e83493748286002f9369f3e6607c565a6a90425a3a1fef5ae32a36d749d \\\n --hash=sha256:8e098148dd37b4ce3baca71fb394c81dc5d9c7728c95df695d2dca218edf40e6 \\\n --hash=sha256:94aea8eff76ee6d1cdacb07dd2123a68283cb5569e0250feab1240058f53b623 \\\n --hash=sha256:95eb302ff792e12aba9a8b8f8474ab229a83c103d74a750ec0bd1c1eea32e669 \\\n --hash=sha256:9bd9b3b31adcb054116447ea22caa61a285d92e94d710aa5ec97992ff5eb7cf3 \\\n --hash=sha256:9e608aafdb55eb9f255034709e20d5a83b6d60c054df0802fa9c9883d0a937aa \\\n --hash=sha256:a103b3a7069b62f5d4890ae1b8f0597618f628b286b03d4bc9195230b154bfa9 \\\n --hash=sha256:a386ebe437176aab38c041de1260cd3ea459c6ce5263594399880bbc398225b2 \\\n --hash=sha256:a38856a971c602f98472050165cea2cdc97709240373041b69030be15047691f \\\n --hash=sha256:a401b4598e5d3f4a9a811f3daf42ee2291790c7f9d74b18d75d6e21dda98a1a1 \\\n --hash=sha256:a7647ebdfb9682b7bb97e2a5e7cb6ae735b1c25008a70b906aecca294ee96cf4 \\\n --hash=sha256:aaf63899c94de41fe3cf934601b0f7ccb6b428c6e4eeb80da72c58eab077b19a \\\n --hash=sha256:b0dac0ff919ba34d4df1b6131f59ce95b08b9065233446be7e459f95554c0dc8 \\\n --hash=sha256:baacc6aee0b2ef6f3d308e197b5d7a81c0e70b06beae1f1fcacffdbd124fe0e3 \\\n --hash=sha256:bf420121d4c8dce6b889f0e8e4ec0ca34b7f40186203f06a946fa0276ba54029 \\\n --hash=sha256:c04a46716adde8d927adb9457bbe39cf473e1e2c2f5d0a16ceb837e5d841ad4f \\\n --hash=sha256:c0b21078a4b56965e2b12f247467b234734491897e99c1d51cee628da9786959 \\\n --hash=sha256:c1c76a1743432b4b60ab3358c937a3fe1341c828ae6194108a94c69028247f22 \\\n --hash=sha256:c4983bf937209c57240cff65906b18bb35e64ae872da6a0db937d7b4af845dd7 \\\n --hash=sha256:c4fb39a81950ec280984b3a44f5bd12819953dc5fa3a7e6fa7a80db5ee853952 \\\n --hash=sha256:c57921cda3a80d0f2b8aec7e25c8aa14479ea92b5b51b6876d975d925a2ea346 \\\n --hash=sha256:c8063cf17b19661471ecbdb3df1c84f24ad2e389e326ccaf89e3fb2484d8dd7e \\\n --hash=sha256:ccd16eb18a849fd8dcb23e23380e2f0a354e8daa0c984b8a732d9cfaba3a776d \\\n --hash=sha256:cd6dbe0238f7743d0efe563ab46294f54f9bc8f4b9bcf57c3c666cc5bc9d1299 \\\n --hash=sha256:d62e51710986674142526ab9f78663ca2b0726066ae26b78b22e0f5e571238dd \\\n --hash=sha256:db901e2ac34c931d73054d9797383d0f8009991e723dab15109740a63e7f902a \\\n --hash=sha256:e03b8895a6990c9ab2cdcd0f2fe44088ca1c65ae592b8f795c3294af00a461c3 \\\n --hash=sha256:e1c8a2f4c69e08e89632defbfabec2feb8a8d99edc9f89ce33c4b9e36ab63037 \\\n --hash=sha256:e4b749b9cc6ee664a3300bb3a273c1ca8068c46be705b6c31cf5d276f8628a94 \\\n --hash=sha256:e6a5bf2cba5ae1bb80b154ed68a3cfa2fa00fde979a7f50d6598d3e17d9ac20c \\\n --hash=sha256:e857a2232ba53ae940d3456f7533ce6ca98b81917d47adc3c7fd55dad8fab858 \\\n --hash=sha256:ee4006268ed33370957f55bf2e6f4d263eaf4dc3cfc473d1d90baff6ed36ce4a \\\n --hash=sha256:eef9df1eefada2c09a5e7a40991b9fc6ac6ef20b1372abd48d2794a316dc0449 \\\n --hash=sha256:f058f6963fd82eb143c692cecdc89e075fa0828db2e5b291070485390b2f1c9c \\\n --hash=sha256:f25c229a6ba38a35ae6e25ca1264621cc25d4d38dca2942a7fce0b67a4efe918 \\\n --hash=sha256:f2a1d0fd4242bd8643ce6f98927cf9c04540af6efa92323e9d3124f57727bfc1 \\\n --hash=sha256:f7560358a6811e52e9c4d142d497f1a6e10103d3a6881f18d04dbce3729c0e2c \\\n --hash=sha256:f779d3ad205f108d14e99bb3859aa7dd8e9c68874617c72354d7ecaec2a054ac \\\n --hash=sha256:f87f746ee241d30d6ed93969de31e5ffd09a2961a051e60ae6bddde9ec3583aa\n # via requests\nclick==8.1.7 \\\n --hash=sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28 \\\n --hash=sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de\n # via black\ncryptography==46.0.5 \\\n --hash=sha256:02f547fce831f5096c9a567fd41bc12ca8f11df260959ecc7c3202555cc47a72 \\\n --hash=sha256:039917b0dc418bb9f6edce8a906572d69e74bd330b0b3fea4f79dab7f8ddd235 \\\n --hash=sha256:1abfdb89b41c3be0365328a410baa9df3ff8a9110fb75e7b52e66803ddabc9a9 \\\n --hash=sha256:2ae6971afd6246710480e3f15824ed3029a60fc16991db250034efd0b9fb4356 \\\n --hash=sha256:2b7a67c9cd56372f3249b39699f2ad479f6991e62ea15800973b956f4b73e257 \\\n --hash=sha256:351695ada9ea9618b3500b490ad54c739860883df6c1f555e088eaf25b1bbaad \\\n --hash=sha256:38946c54b16c885c72c4f59846be9743d699eee2b69b6988e0a00a01f46a61a4 \\\n --hash=sha256:3b4995dc971c9fb83c25aa44cf45f02ba86f71ee600d81091c2f0cbae116b06c \\\n --hash=sha256:3ce58ba46e1bc2aac4f7d9290223cead56743fa6ab94a5d53292ffaac6a91614 \\\n --hash=sha256:3ee190460e2fbe447175cda91b88b84ae8322a104fc27766ad09428754a618ed \\\n --hash=sha256:4108d4c09fbbf2789d0c926eb4152ae1760d5a2d97612b92d508d96c861e4d31 \\\n --hash=sha256:420d0e909050490d04359e7fdb5ed7e667ca5c3c402b809ae2563d7e66a92229 \\\n --hash=sha256:47fb8a66058b80e509c47118ef8a75d14c455e81ac369050f20ba0d23e77fee0 \\\n --hash=sha256:4c3341037c136030cb46e4b1e17b7418ea4cbd9dd207e4a6f3b2b24e0d4ac731 \\\n --hash=sha256:4d7e3d356b8cd4ea5aff04f129d5f66ebdc7b6f8eae802b93739ed520c47c79b \\\n --hash=sha256:4d8ae8659ab18c65ced284993c2265910f6c9e650189d4e3f68445ef82a810e4 \\\n --hash=sha256:4e817a8920bfbcff8940ecfd60f23d01836408242b30f1a708d93198393a80b4 \\\n --hash=sha256:50bfb6925eff619c9c023b967d5b77a54e04256c4281b0e21336a130cd7fc263 \\\n --hash=sha256:556e106ee01aa13484ce9b0239bca667be5004efb0aabbed28d353df86445595 \\\n --hash=sha256:582f5fcd2afa31622f317f80426a027f30dc792e9c80ffee87b993200ea115f1 \\\n --hash=sha256:5be7bf2fb40769e05739dd0046e7b26f9d4670badc7b032d6ce4db64dddc0678 \\\n --hash=sha256:60ee7e19e95104d4c03871d7d7dfb3d22ef8a9b9c6778c94e1c8fcc8365afd48 \\\n --hash=sha256:61aa400dce22cb001a98014f647dc21cda08f7915ceb95df0c9eaf84b4b6af76 \\\n --hash=sha256:68f68d13f2e1cb95163fa3b4db4bf9a159a418f5f6e7242564fc75fcae667fd0 \\\n --hash=sha256:7d1f30a86d2757199cb2d56e48cce14deddf1f9c95f1ef1b64ee91ea43fe2e18 \\\n --hash=sha256:7d731d4b107030987fd61a7f8ab512b25b53cef8f233a97379ede116f30eb67d \\\n --hash=sha256:803812e111e75d1aa73690d2facc295eaefd4439be1023fefc4995eaea2af90d \\\n --hash=sha256:80a8d7bfdf38f87ca30a5391c0c9ce4ed2926918e017c29ddf643d0ed2778ea1 \\\n --hash=sha256:8293f3dea7fc929ef7240796ba231413afa7b68ce38fd21da2995549f5961981 \\\n --hash=sha256:8456928655f856c6e1533ff59d5be76578a7157224dbd9ce6872f25055ab9ab7 \\\n --hash=sha256:890bcb4abd5a2d3f852196437129eb3667d62630333aacc13dfd470fad3aaa82 \\\n --hash=sha256:94a76daa32eb78d61339aff7952ea819b1734b46f73646a07decb40e5b3448e2 \\\n --hash=sha256:9f16fbdf4da055efb21c22d81b89f155f02ba420558db21288b3d0035bafd5f4 \\\n --hash=sha256:a3d1fae9863299076f05cb8a778c467578262fae09f9dc0ee9b12eb4268ce663 \\\n --hash=sha256:a3d507bb6a513ca96ba84443226af944b0f7f47dcc9a399d110cd6146481d24c \\\n --hash=sha256:abace499247268e3757271b2f1e244b36b06f8515cf27c4d49468fc9eb16e93d \\\n --hash=sha256:ba2a27ff02f48193fc4daeadf8ad2590516fa3d0adeeb34336b96f7fa64c1e3a \\\n --hash=sha256:bc84e875994c3b445871ea7181d424588171efec3e185dced958dad9e001950a \\\n --hash=sha256:bfd56bb4b37ed4f330b82402f6f435845a5f5648edf1ad497da51a8452d5d62d \\\n --hash=sha256:c18ff11e86df2e28854939acde2d003f7984f721eba450b56a200ad90eeb0e6b \\\n --hash=sha256:c3bcce8521d785d510b2aad26ae2c966092b7daa8f45dd8f44734a104dc0bc1a \\\n --hash=sha256:c4143987a42a2397f2fc3b4d7e3a7d313fbe684f67ff443999e803dd75a76826 \\\n --hash=sha256:c69fd885df7d089548a42d5ec05be26050ebcd2283d89b3d30676eb32ff87dee \\\n --hash=sha256:ced80795227d70549a411a4ab66e8ce307899fad2220ce5ab2f296e687eacde9 \\\n --hash=sha256:d66e421495fdb797610a08f43b05269e0a5ea7f5e652a89bfd5a7d3c1dee3648 \\\n --hash=sha256:d861ee9e76ace6cf36a6a89b959ec08e7bc2493ee39d07ffe5acb23ef46d27da \\\n --hash=sha256:e9251e3be159d1020c4030bd2e5f84d6a43fe54b6c19c12f51cde9542a2817b2 \\\n --hash=sha256:f145bba11b878005c496e93e257c1e88f154d278d2638e6450d17e0f31e558d2 \\\n --hash=sha256:fe346b143ff9685e40192a4960938545c699054ba11d4f9029f94751e3f71d87\n # via\n # -r requirements_formatting.txt.in\n # pyjwt\ndarker==2.1.1 \\\n --hash=sha256:a6e6a682c0604e76fe9aec7650e96a944f517563c69b28fcc076db9d957d98ea \\\n --hash=sha256:ead701414c45359fc0312bc285614d3285fc135476d43f3bc08d989ee19d9020\n # via -r requirements_formatting.txt.in\ndarkgraylib==1.2.1 \\\n --hash=sha256:60c59de69842367ce0c78c32c451fa8e9d29500e681312d9864a7416bcdb7792 \\\n --hash=sha256:a5dd6a2015a470d9047278cdd01a91ccb1d746675f8fd4562b3b5f6b8cbda930\n # via\n # darker\n # graylint\ndeprecated==1.2.14 \\\n --hash=sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c \\\n --hash=sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3\n # via pygithub\ngraylint==1.1.1 \\\n --hash=sha256:0fd8e02972ca03d0ef2bf0adea76b5343efcd492d7afb5f658f3e3a724f55a36 \\\n --hash=sha256:b7e0eab6c159684dbf5ef84e942c3340f6a6549b02a3d11b1a1763cc4f8f0593\n # via darker\nidna==3.10 \\\n --hash=sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9 \\\n --hash=sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3\n # via\n # -r requirements_formatting.txt.in\n # requests\nmypy-extensions==1.0.0 \\\n --hash=sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d \\\n --hash=sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782\n # via black\npackaging==23.1 \\\n --hash=sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61 \\\n --hash=sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f\n # via black\npathspec==1.0.4 \\\n --hash=sha256:0210e2ae8a21a9137c0d470578cb0e595af87edaa6ebf12ff176f14a02e0e645 \\\n --hash=sha256:fb6ae2fd4e7c921a165808a552060e722767cfa526f99ca5156ed2ce45a5c723\n # via black\nplatformdirs==3.10.0 \\\n --hash=sha256:b45696dab2d7cc691a3226759c0d3b00c47c8b6e293d96f6436f733303f77f6d \\\n --hash=sha256:d7c24979f292f916dc9cbf8648319032f551ea8c49a4c9bf2fb556a02070ec1d\n # via black\npycparser==2.21 \\\n --hash=sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9 \\\n --hash=sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206\n # via cffi\npygithub==2.6.1 \\\n --hash=sha256:6f2fa6d076ccae475f9fc392cc6cdbd54db985d4f69b8833a28397de75ed6ca3 \\\n --hash=sha256:b5c035392991cca63959e9453286b41b54d83bf2de2daa7d7ff7e4312cebf3bf\n # via -r requirements_formatting.txt.in\npyjwt==2.12.1 \\\n --hash=sha256:28ca37c070cad8ba8cd9790cd940535d40274d22f80ab87f3ac6a713e6e8454c \\\n --hash=sha256:c74a7a2adf861c04d002db713dd85f84beb242228e671280bf709d765b03672b\n # via\n # -r requirements_formatting.txt.in\n # pygithub\npynacl==1.6.2 \\\n --hash=sha256:018494d6d696ae03c7e656e5e74cdfd8ea1326962cc401bcf018f1ed8436811c \\\n --hash=sha256:04316d1fc625d860b6c162fff704eb8426b1a8bcd3abacea11142cbd99a6b574 \\\n --hash=sha256:22de65bb9010a725b0dac248f353bb072969c94fa8d6b1f34b87d7953cf7bbe4 \\\n --hash=sha256:26bfcd00dcf2cf160f122186af731ae30ab120c18e8375684ec2670dccd28130 \\\n --hash=sha256:2fef529ef3ee487ad8113d287a593fa26f48ee3620d92ecc6f1d09ea38e0709b \\\n --hash=sha256:320ef68a41c87547c91a8b58903c9caa641ab01e8512ce291085b5fe2fcb7590 \\\n --hash=sha256:3bffb6d0f6becacb6526f8f42adfb5efb26337056ee0831fb9a7044d1a964444 \\\n --hash=sha256:44081faff368d6c5553ccf55322ef2819abb40e25afaec7e740f159f74813634 \\\n --hash=sha256:46065496ab748469cdd999246d17e301b2c24ae2fdf739132e580a0e94c94a87 \\\n --hash=sha256:5811c72b473b2f38f7e2a3dc4f8642e3a3e9b5e7317266e4ced1fba85cae41aa \\\n --hash=sha256:622d7b07cc5c02c666795792931b50c91f3ce3c2649762efb1ef0d5684c81594 \\\n --hash=sha256:62985f233210dee6548c223301b6c25440852e13d59a8b81490203c3227c5ba0 \\\n --hash=sha256:68be3a09455743ff9505491220b64440ced8973fe930f270c8e07ccfa25b1f9e \\\n --hash=sha256:834a43af110f743a754448463e8fd61259cd4ab5bbedcf70f9dabad1d28a394c \\\n --hash=sha256:8845c0631c0be43abdd865511c41eab235e0be69c81dc66a50911594198679b0 \\\n --hash=sha256:8a66d6fb6ae7661c58995f9c6435bda2b1e68b54b598a6a10247bfcdadac996c \\\n --hash=sha256:8b097553b380236d51ed11356c953bf8ce36a29a3e596e934ecabe76c985a577 \\\n --hash=sha256:a84bf1c20339d06dc0c85d9aea9637a24f718f375d861b2668b2f9f96fa51145 \\\n --hash=sha256:a9f9932d8d2811ce1a8ffa79dcbdf3970e7355b5c8eb0c1a881a57e7f7d96e88 \\\n --hash=sha256:bc4a36b28dd72fb4845e5d8f9760610588a96d5a51f01d84d8c6ff9849968c14 \\\n --hash=sha256:c8a231e36ec2cab018c4ad4358c386e36eede0319a0c41fed24f840b1dac59f6 \\\n --hash=sha256:c949ea47e4206af7c8f604b8278093b674f7c79ed0d4719cc836902bf4517465 \\\n --hash=sha256:d071c6a9a4c94d79eb665db4ce5cedc537faf74f2355e4d502591d850d3913c0 \\\n --hash=sha256:d29bfe37e20e015a7d8b23cfc8bd6aa7909c92a1b8f41ee416bbb3e79ef182b2 \\\n --hash=sha256:fe9847ca47d287af41e82be1dd5e23023d3c31a951da134121ab02e42ac218c9\n # via\n # -r requirements_formatting.txt.in\n # pygithub\npytokens==0.4.1 \\\n --hash=sha256:0fc71786e629cef478cbf29d7ea1923299181d0699dbe7c3c0f4a583811d9fc1 \\\n --hash=sha256:11edda0942da80ff58c4408407616a310adecae1ddd22eef8c692fe266fa5009 \\\n --hash=sha256:140709331e846b728475786df8aeb27d24f48cbcf7bcd449f8de75cae7a45083 \\\n --hash=sha256:24afde1f53d95348b5a0eb19488661147285ca4dd7ed752bbc3e1c6242a304d1 \\\n --hash=sha256:26cef14744a8385f35d0e095dc8b3a7583f6c953c2e3d269c7f82484bf5ad2de \\\n --hash=sha256:27b83ad28825978742beef057bfe406ad6ed524b2d28c252c5de7b4a6dd48fa2 \\\n --hash=sha256:292052fe80923aae2260c073f822ceba21f3872ced9a68bb7953b348e561179a \\\n --hash=sha256:29d1d8fb1030af4d231789959f21821ab6325e463f0503a61d204343c9b355d1 \\\n --hash=sha256:2a44ed93ea23415c54f3face3b65ef2b844d96aeb3455b8a69b3df6beab6acc5 \\\n --hash=sha256:30f51edd9bb7f85c748979384165601d028b84f7bd13fe14d3e065304093916a \\\n --hash=sha256:34bcc734bd2f2d5fe3b34e7b3c0116bfb2397f2d9666139988e7a3eb5f7400e3 \\\n --hash=sha256:3ad72b851e781478366288743198101e5eb34a414f1d5627cdd585ca3b25f1db \\\n --hash=sha256:3f901fe783e06e48e8cbdc82d631fca8f118333798193e026a50ce1b3757ea68 \\\n --hash=sha256:42f144f3aafa5d92bad964d471a581651e28b24434d184871bd02e3a0d956037 \\\n --hash=sha256:4a14d5f5fc78ce85e426aa159489e2d5961acf0e47575e08f35584009178e321 \\\n --hash=sha256:4a58d057208cb9075c144950d789511220b07636dd2e4708d5645d24de666bdc \\\n --hash=sha256:4e691d7f5186bd2842c14813f79f8884bb03f5995f0575272009982c5ac6c0f7 \\\n --hash=sha256:5502408cab1cb18e128570f8d598981c68a50d0cbd7c61312a90507cd3a1276f \\\n --hash=sha256:584c80c24b078eec1e227079d56dc22ff755e0ba8654d8383b2c549107528918 \\\n --hash=sha256:5ad948d085ed6c16413eb5fec6b3e02fa00dc29a2534f088d3302c47eb59adf9 \\\n --hash=sha256:670d286910b531c7b7e3c0b453fd8156f250adb140146d234a82219459b9640c \\\n --hash=sha256:682fa37ff4d8e95f7df6fe6fe6a431e8ed8e788023c6bcc0f0880a12eab80ad1 \\\n --hash=sha256:6d6c4268598f762bc8e91f5dbf2ab2f61f7b95bdc07953b602db879b3c8c18e1 \\\n --hash=sha256:79fc6b8699564e1f9b521582c35435f1bd32dd06822322ec44afdeba666d8cb3 \\\n --hash=sha256:8bdb9d0ce90cbf99c525e75a2fa415144fd570a1ba987380190e8b786bc6ef9b \\\n --hash=sha256:8fcb9ba3709ff77e77f1c7022ff11d13553f3c30299a9fe246a166903e9091eb \\\n --hash=sha256:941d4343bf27b605e9213b26bfa1c4bf197c9c599a9627eb7305b0defcfe40c1 \\\n --hash=sha256:967cf6e3fd4adf7de8fc73cd3043754ae79c36475c1c11d514fc72cf5490094a \\\n --hash=sha256:970b08dd6b86058b6dc07efe9e98414f5102974716232d10f32ff39701e841c4 \\\n --hash=sha256:97f50fd18543be72da51dd505e2ed20d2228c74e0464e4262e4899797803d7fa \\\n --hash=sha256:9bd7d7f544d362576be74f9d5901a22f317efc20046efe2034dced238cbbfe78 \\\n --hash=sha256:add8bf86b71a5d9fb5b89f023a80b791e04fba57960aa790cc6125f7f1d39dfe \\\n --hash=sha256:b35d7e5ad269804f6697727702da3c517bb8a5228afa450ab0fa787732055fc9 \\\n --hash=sha256:b49750419d300e2b5a3813cf229d4e5a4c728dae470bcc89867a9ad6f25a722d \\\n --hash=sha256:d31b97b3de0f61571a124a00ffe9a81fb9939146c122c11060725bd5aea79975 \\\n --hash=sha256:d70e77c55ae8380c91c0c18dea05951482e263982911fc7410b1ffd1dadd3440 \\\n --hash=sha256:d9907d61f15bf7261d7e775bd5d7ee4d2930e04424bab1972591918497623a16 \\\n --hash=sha256:da5baeaf7116dced9c6bb76dc31ba04a2dc3695f3d9f74741d7910122b456edc \\\n --hash=sha256:dc74c035f9bfca0255c1af77ddd2d6ae8419012805453e4b0e7513e17904545d \\\n --hash=sha256:dcafc12c30dbaf1e2af0490978352e0c4041a7cde31f4f81435c2a5e8b9cabb6 \\\n --hash=sha256:ee44d0f85b803321710f9239f335aafe16553b39106384cef8e6de40cb4ef2f6 \\\n --hash=sha256:f66a6bbe741bd431f6d741e617e0f39ec7257ca1f89089593479347cc4d13324\n # via black\nrequests==2.32.4 \\\n --hash=sha256:27babd3cda2a6d50b30443204ee89830707d396671944c998b5975b031ac2b2c \\\n --hash=sha256:27d0316682c8a29834d3264820024b62a36942083d52caf2f14c0591336d3422\n # via\n # -r requirements_formatting.txt.in\n # pygithub\ntoml==0.10.2 \\\n --hash=sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b \\\n --hash=sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f\n # via\n # darker\n # darkgraylib\ntyping-extensions==4.14.1 \\\n --hash=sha256:38b39f4aeeab64884ce9f74c94263ef78f3c22467c8724005483154c26648d36 \\\n --hash=sha256:d1e1e3b58374dc93031d6eda2420a48ea44a36c2b4766a4fdeb3710755731d76\n # via pygithub\nurllib3==2.6.3 \\\n --hash=sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed \\\n --hash=sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4\n # via\n # -r requirements_formatting.txt.in\n # pygithub\n # requests\nwrapt==1.15.0 \\\n --hash=sha256:02fce1852f755f44f95af51f69d22e45080102e9d00258053b79367d07af39c0 \\\n --hash=sha256:077ff0d1f9d9e4ce6476c1a924a3332452c1406e59d90a2cf24aeb29eeac9420 \\\n --hash=sha256:078e2a1a86544e644a68422f881c48b84fef6d18f8c7a957ffd3f2e0a74a0d4a \\\n --hash=sha256:0970ddb69bba00670e58955f8019bec4a42d1785db3faa043c33d81de2bf843c \\\n --hash=sha256:1286eb30261894e4c70d124d44b7fd07825340869945c79d05bda53a40caa079 \\\n --hash=sha256:21f6d9a0d5b3a207cdf7acf8e58d7d13d463e639f0c7e01d82cdb671e6cb7923 \\\n --hash=sha256:230ae493696a371f1dbffaad3dafbb742a4d27a0afd2b1aecebe52b740167e7f \\\n --hash=sha256:26458da5653aa5b3d8dc8b24192f574a58984c749401f98fff994d41d3f08da1 \\\n --hash=sha256:2cf56d0e237280baed46f0b5316661da892565ff58309d4d2ed7dba763d984b8 \\\n --hash=sha256:2e51de54d4fb8fb50d6ee8327f9828306a959ae394d3e01a1ba8b2f937747d86 \\\n --hash=sha256:2fbfbca668dd15b744418265a9607baa970c347eefd0db6a518aaf0cfbd153c0 \\\n --hash=sha256:38adf7198f8f154502883242f9fe7333ab05a5b02de7d83aa2d88ea621f13364 \\\n --hash=sha256:3a8564f283394634a7a7054b7983e47dbf39c07712d7b177b37e03f2467a024e \\\n --hash=sha256:3abbe948c3cbde2689370a262a8d04e32ec2dd4f27103669a45c6929bcdbfe7c \\\n --hash=sha256:3bbe623731d03b186b3d6b0d6f51865bf598587c38d6f7b0be2e27414f7f214e \\\n --hash=sha256:40737a081d7497efea35ab9304b829b857f21558acfc7b3272f908d33b0d9d4c \\\n --hash=sha256:41d07d029dd4157ae27beab04d22b8e261eddfc6ecd64ff7000b10dc8b3a5727 \\\n --hash=sha256:46ed616d5fb42f98630ed70c3529541408166c22cdfd4540b88d5f21006b0eff \\\n --hash=sha256:493d389a2b63c88ad56cdc35d0fa5752daac56ca755805b1b0c530f785767d5e \\\n --hash=sha256:4ff0d20f2e670800d3ed2b220d40984162089a6e2c9646fdb09b85e6f9a8fc29 \\\n --hash=sha256:54accd4b8bc202966bafafd16e69da9d5640ff92389d33d28555c5fd4f25ccb7 \\\n --hash=sha256:56374914b132c702aa9aa9959c550004b8847148f95e1b824772d453ac204a72 \\\n --hash=sha256:578383d740457fa790fdf85e6d346fda1416a40549fe8db08e5e9bd281c6a475 \\\n --hash=sha256:58d7a75d731e8c63614222bcb21dd992b4ab01a399f1f09dd82af17bbfc2368a \\\n --hash=sha256:5c5aa28df055697d7c37d2099a7bc09f559d5053c3349b1ad0c39000e611d317 \\\n --hash=sha256:5fc8e02f5984a55d2c653f5fea93531e9836abbd84342c1d1e17abc4a15084c2 \\\n --hash=sha256:63424c681923b9f3bfbc5e3205aafe790904053d42ddcc08542181a30a7a51bd \\\n --hash=sha256:64b1df0f83706b4ef4cfb4fb0e4c2669100fd7ecacfb59e091fad300d4e04640 \\\n --hash=sha256:74934ebd71950e3db69960a7da29204f89624dde411afbfb3b4858c1409b1e98 \\\n --hash=sha256:75669d77bb2c071333417617a235324a1618dba66f82a750362eccbe5b61d248 \\\n --hash=sha256:75760a47c06b5974aa5e01949bf7e66d2af4d08cb8c1d6516af5e39595397f5e \\\n --hash=sha256:76407ab327158c510f44ded207e2f76b657303e17cb7a572ffe2f5a8a48aa04d \\\n --hash=sha256:76e9c727a874b4856d11a32fb0b389afc61ce8aaf281ada613713ddeadd1cfec \\\n --hash=sha256:77d4c1b881076c3ba173484dfa53d3582c1c8ff1f914c6461ab70c8428b796c1 \\\n --hash=sha256:780c82a41dc493b62fc5884fb1d3a3b81106642c5c5c78d6a0d4cbe96d62ba7e \\\n --hash=sha256:7dc0713bf81287a00516ef43137273b23ee414fe41a3c14be10dd95ed98a2df9 \\\n --hash=sha256:7eebcdbe3677e58dd4c0e03b4f2cfa346ed4049687d839adad68cc38bb559c92 \\\n --hash=sha256:896689fddba4f23ef7c718279e42f8834041a21342d95e56922e1c10c0cc7afb \\\n --hash=sha256:96177eb5645b1c6985f5c11d03fc2dbda9ad24ec0f3a46dcce91445747e15094 \\\n --hash=sha256:96e25c8603a155559231c19c0349245eeb4ac0096fe3c1d0be5c47e075bd4f46 \\\n --hash=sha256:9d37ac69edc5614b90516807de32d08cb8e7b12260a285ee330955604ed9dd29 \\\n --hash=sha256:9ed6aa0726b9b60911f4aed8ec5b8dd7bf3491476015819f56473ffaef8959bd \\\n --hash=sha256:a487f72a25904e2b4bbc0817ce7a8de94363bd7e79890510174da9d901c38705 \\\n --hash=sha256:a4cbb9ff5795cd66f0066bdf5947f170f5d63a9274f99bdbca02fd973adcf2a8 \\\n --hash=sha256:a74d56552ddbde46c246b5b89199cb3fd182f9c346c784e1a93e4dc3f5ec9975 \\\n --hash=sha256:a89ce3fd220ff144bd9d54da333ec0de0399b52c9ac3d2ce34b569cf1a5748fb \\\n --hash=sha256:abd52a09d03adf9c763d706df707c343293d5d106aea53483e0ec8d9e310ad5e \\\n --hash=sha256:abd8f36c99512755b8456047b7be10372fca271bf1467a1caa88db991e7c421b \\\n --hash=sha256:af5bd9ccb188f6a5fdda9f1f09d9f4c86cc8a539bd48a0bfdc97723970348418 \\\n --hash=sha256:b02f21c1e2074943312d03d243ac4388319f2456576b2c6023041c4d57cd7019 \\\n --hash=sha256:b06fa97478a5f478fb05e1980980a7cdf2712015493b44d0c87606c1513ed5b1 \\\n --hash=sha256:b0724f05c396b0a4c36a3226c31648385deb6a65d8992644c12a4963c70326ba \\\n --hash=sha256:b130fe77361d6771ecf5a219d8e0817d61b236b7d8b37cc045172e574ed219e6 \\\n --hash=sha256:b56d5519e470d3f2fe4aa7585f0632b060d532d0696c5bdfb5e8319e1d0f69a2 \\\n --hash=sha256:b67b819628e3b748fd3c2192c15fb951f549d0f47c0449af0764d7647302fda3 \\\n --hash=sha256:ba1711cda2d30634a7e452fc79eabcadaffedf241ff206db2ee93dd2c89a60e7 \\\n --hash=sha256:bbeccb1aa40ab88cd29e6c7d8585582c99548f55f9b2581dfc5ba68c59a85752 \\\n --hash=sha256:bd84395aab8e4d36263cd1b9308cd504f6cf713b7d6d3ce25ea55670baec5416 \\\n --hash=sha256:c99f4309f5145b93eca6e35ac1a988f0dc0a7ccf9ccdcd78d3c0adf57224e62f \\\n --hash=sha256:ca1cccf838cd28d5a0883b342474c630ac48cac5df0ee6eacc9c7290f76b11c1 \\\n --hash=sha256:cd525e0e52a5ff16653a3fc9e3dd827981917d34996600bbc34c05d048ca35cc \\\n --hash=sha256:cdb4f085756c96a3af04e6eca7f08b1345e94b53af8921b25c72f096e704e145 \\\n --hash=sha256:ce42618f67741d4697684e501ef02f29e758a123aa2d669e2d964ff734ee00ee \\\n --hash=sha256:d06730c6aed78cee4126234cf2d071e01b44b915e725a6cb439a879ec9754a3a \\\n --hash=sha256:d5fe3e099cf07d0fb5a1e23d399e5d4d1ca3e6dfcbe5c8570ccff3e9208274f7 \\\n --hash=sha256:d6bcbfc99f55655c3d93feb7ef3800bd5bbe963a755687cbf1f490a71fb7794b \\\n --hash=sha256:d787272ed958a05b2c86311d3a4135d3c2aeea4fc655705f074130aa57d71653 \\\n --hash=sha256:e169e957c33576f47e21864cf3fc9ff47c223a4ebca8960079b8bd36cb014fd0 \\\n --hash=sha256:e20076a211cd6f9b44a6be58f7eeafa7ab5720eb796975d0c03f05b47d89eb90 \\\n --hash=sha256:e826aadda3cae59295b95343db8f3d965fb31059da7de01ee8d1c40a60398b29 \\\n --hash=sha256:eef4d64c650f33347c1f9266fa5ae001440b232ad9b98f1f43dfe7a79435c0a6 \\\n --hash=sha256:f2e69b3ed24544b0d3dbe2c5c0ba5153ce50dcebb576fdc4696d52aa22db6034 \\\n --hash=sha256:f87ec75864c37c4c6cb908d282e1969e79763e0d9becdfe9fe5473b7bb1e5f09 \\\n --hash=sha256:fbec11614dba0424ca72f4e8ba3c420dba07b4a7c206c8c8e4e73f2e98f4c559 \\\n --hash=sha256:fd69666217b62fa5d7c6aa88e507493a34dec4fa20c5bd925e4bc12fce586639\n # via deprecated\n" }, { "path": "External/code-format-helper/requirements_formatting.txt.in", "content": "black>=26.3.1\ndarker==2.1.1\nPyGithub==2.6.1\ncryptography>=46.0.5\nurllib3>=2.6.3\nrequests>=2.32.4\nidna>=3.7\ncertifi>=2024.7.4\nPyNaCl>=1.6.2\nPyJWT>=2.12.1\n" }, { "path": "External/tiny-json/CMakeLists.txt", "content": "set(NAME tiny-json)\nset(SRCS tiny-json.c)\nadd_library(${NAME} STATIC ${SRCS})\n\ntarget_include_directories(${NAME} PUBLIC ${CMAKE_CURRENT_LIST_DIR})\nadd_library(${NAME}::${NAME} ALIAS ${NAME})" }, { "path": "External/tiny-json/LICENSE", "content": "MIT License\n\nCopyright (c) 2018 Rafa Garcia\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n" }, { "path": "External/tiny-json/tiny-json.c", "content": "\n/*\n\n\n \n Licensed under the MIT License .\n SPDX-License-Identifier: MIT\n Copyright (c) 2016-2018 Rafa Garcia .\n\n Permission is hereby granted, free of charge, to any person obtaining a copy\n of this software and associated documentation files (the \"Software\"), to deal\n in the Software without restriction, including without limitation the rights\n to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n copies of the Software, and to permit persons to whom the Software is\n furnished to do so, subject to the following conditions:\n\n The above copyright notice and this permission notice shall be included in all\n copies or substantial portions of the Software.\n\n THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\n SOFTWARE.\n \n*/\n\n#include \n#include \n#include \n#include // For NULL\n#include \"tiny-json.h\"\n\n/** Structure to handle a heap of JSON properties. */\ntypedef struct jsonStaticPool_s {\n json_t* const mem; /**< Pointer to array of json properties. */\n unsigned int const qty; /**< Length of the array of json properties. */\n unsigned int nextFree; /**< The index of the next free json property. */\n jsonPool_t pool;\n} jsonStaticPool_t;\n\n/* Search a property by its name in a JSON object. */\njson_t const* json_getProperty( json_t const* obj, char const* property ) {\n json_t const* sibling;\n for( sibling = obj->u.c.child; sibling; sibling = sibling->sibling )\n if ( sibling->name && !strcmp( sibling->name, property ) )\n return sibling;\n return 0;\n}\n\n/* Search a property by its name in a JSON object and return its value. */\nchar const* json_getPropertyValue( json_t const* obj, char const* property ) {\n\tjson_t const* field = json_getProperty( obj, property );\n\tif ( !field ) return 0;\n jsonType_t type = json_getType( field );\n if ( JSON_ARRAY >= type ) return 0;\n\treturn json_getValue( field );\n}\n\n/* Internal prototypes: */\nstatic char* goBlank( char* str );\nstatic char* goNum( char* str );\nstatic json_t* poolInit( jsonPool_t* pool );\nstatic json_t* poolAlloc( jsonPool_t* pool );\nstatic char* objValue( char* ptr, json_t* obj, jsonPool_t* pool );\nstatic char* setToNull( char* ch );\nstatic bool isEndOfPrimitive( char ch );\n\n/* Parse a string to get a json. */\njson_t const* json_createWithPool( char *str, jsonPool_t *pool ) {\n char* ptr = goBlank( str );\n if ( !ptr || *ptr != '{' ) return 0;\n json_t* obj = pool->init( pool );\n obj->name = 0;\n obj->sibling = 0;\n obj->u.c.child = 0;\n ptr = objValue( ptr, obj, pool );\n if ( !ptr ) return 0;\n return obj;\n}\n\n/* Parse a string to get a json. */\njson_t const* json_create( char* str, json_t mem[], unsigned int qty ) {\n jsonStaticPool_t spool = {\n .mem = mem,\n .qty = qty,\n .pool = {\n .init = poolInit,\n .alloc = poolAlloc\n }\n };\n return json_createWithPool( str, &spool.pool );\n}\n\n/** Get a special character with its escape character. Examples:\n * 'b' -> '\\b', 'n' -> '\\n', 't' -> '\\t'\n * @param ch The escape character.\n * @return The character code. */\nstatic char getEscape( char ch ) {\n static struct { char ch; char code; } const pair[] = {\n { '\\\"', '\\\"' }, { '\\\\', '\\\\' },\n { '/', '/' }, { 'b', '\\b' },\n { 'f', '\\f' }, { 'n', '\\n' },\n { 'r', '\\r' }, { 't', '\\t' },\n };\n unsigned int i;\n for( i = 0; i < sizeof pair / sizeof *pair; ++i )\n if ( pair[i].ch == ch )\n return pair[i].code;\n return '\\0';\n}\n\n/** Parse 4 characters.\n * @Param str Pointer to first digit.\n * @retval '?' If the four characters are hexadecimal digits.\n * @retcal '\\0' In other cases. */\nstatic unsigned char getCharFromUnicode( unsigned char const* str ) {\n unsigned int i;\n for( i = 0; i < 4; ++i )\n if ( !isxdigit( str[i] ) )\n return '\\0';\n return '?';\n}\n\n/** Parse a string and replace the scape characters by their meaning characters.\n * This parser stops when finds the character '\\\"'. Then replaces '\\\"' by '\\0'.\n * @param str Pointer to first character.\n * @retval Pointer to first non white space after the string. If success.\n * @retval Null pointer if any error occur. */\nstatic char* parseString( char* str ) {\n unsigned char* head = (unsigned char*)str;\n unsigned char* tail = (unsigned char*)str;\n for( ; *head >= ' '; ++head, ++tail ) {\n if ( *head == '\\\"' ) {\n *tail = '\\0';\n return (char*)++head;\n }\n if ( *head == '\\\\' ) {\n if ( *++head == 'u' ) {\n char const ch = getCharFromUnicode( ++head );\n if ( ch == '\\0' ) return 0;\n *tail = ch;\n head += 3;\n }\n else {\n char const esc = getEscape( *head );\n if ( esc == '\\0' ) return 0;\n *tail = esc;\n }\n }\n else *tail = *head;\n }\n return 0;\n}\n\n/** Parse a string to get the name of a property.\n * @param str Pointer to first character.\n * @param property The property to assign the name.\n * @retval Pointer to first of property value. If success.\n * @retval Null pointer if any error occur. */\nstatic char* propertyName( char* ptr, json_t* property ) {\n property->name = ++ptr;\n ptr = parseString( ptr );\n if ( !ptr ) return 0;\n ptr = goBlank( ptr );\n if ( !ptr ) return 0;\n if ( *ptr++ != ':' ) return 0;\n return goBlank( ptr );\n}\n\n/** Parse a string to get the value of a property when its type is JSON_TEXT.\n * @param str Pointer to first character ('\\\"').\n * @param property The property to assign the name.\n * @retval Pointer to first non white space after the string. If success.\n * @retval Null pointer if any error occur. */\nstatic char* textValue( char* ptr, json_t* property ) {\n ++property->u.value;\n ptr = parseString( ++ptr );\n if ( !ptr ) return 0;\n property->type = JSON_TEXT;\n return ptr;\n}\n\n/** Compare two strings until get the null character in the second one.\n * @param ptr sub string\n * @param str main string\n * @retval Pointer to next character.\n * @retval Null pointer if any error occur. */\nstatic char* checkStr( char* ptr, char const* str ) {\n while( *str )\n if ( *ptr++ != *str++ )\n return 0;\n return ptr;\n}\n\n/** Parser a string to get a primitive value.\n * If the first character after the value is different of '}' or ']' is set to '\\0'.\n * @param str Pointer to first character.\n * @param property Property handler to set the value and the type, (true, false or null).\n * @param value String with the primitive literal.\n * @param type The code of the type. ( JSON_BOOLEAN or JSON_NULL )\n * @retval Pointer to first non white space after the string. If success.\n * @retval Null pointer if any error occur. */\nstatic char* primitiveValue( char* ptr, json_t* property, char const* value, jsonType_t type ) {\n ptr = checkStr( ptr, value );\n if ( !ptr || !isEndOfPrimitive( *ptr ) ) return 0;\n ptr = setToNull( ptr );\n property->type = type;\n return ptr;\n}\n\n/** Parser a string to get a true value.\n * If the first character after the value is different of '}' or ']' is set to '\\0'.\n * @param str Pointer to first character.\n * @param property Property handler to set the value and the type, (true, false or null).\n * @retval Pointer to first non white space after the string. If success.\n * @retval Null pointer if any error occur. */\nstatic char* trueValue( char* ptr, json_t* property ) {\n return primitiveValue( ptr, property, \"true\", JSON_BOOLEAN );\n}\n\n/** Parser a string to get a false value.\n * If the first character after the value is different of '}' or ']' is set to '\\0'.\n * @param str Pointer to first character.\n * @param property Property handler to set the value and the type, (true, false or null).\n * @retval Pointer to first non white space after the string. If success.\n * @retval Null pointer if any error occur. */\nstatic char* falseValue( char* ptr, json_t* property ) {\n return primitiveValue( ptr, property, \"false\", JSON_BOOLEAN );\n}\n\n/** Parser a string to get a null value.\n * If the first character after the value is different of '}' or ']' is set to '\\0'.\n * @param str Pointer to first character.\n * @param property Property handler to set the value and the type, (true, false or null).\n * @retval Pointer to first non white space after the string. If success.\n * @retval Null pointer if any error occur. */\nstatic char* nullValue( char* ptr, json_t* property ) {\n return primitiveValue( ptr, property, \"null\", JSON_NULL );\n}\n\n/** Analyze the exponential part of a real number.\n * @param str Pointer to first character.\n * @retval Pointer to first non numerical after the string. If success.\n * @retval Null pointer if any error occur. */\nstatic char* expValue( char* ptr ) {\n if ( *ptr == '-' || *ptr == '+' ) ++ptr;\n if ( !isdigit( *ptr ) ) return 0;\n ptr = goNum( ++ptr );\n return ptr;\n}\n\n/** Analyze the decimal part of a real number.\n * @param str Pointer to first character.\n * @retval Pointer to first non numerical after the string. If success.\n * @retval Null pointer if any error occur. */\nstatic char* fraqValue( char* ptr ) {\n if ( !isdigit( *ptr ) ) return 0;\n ptr = goNum( ++ptr );\n if ( !ptr ) return 0;\n return ptr;\n}\n\n/** Parser a string to get a numerical value.\n * If the first character after the value is different of '}' or ']' is set to '\\0'.\n * @param str Pointer to first character.\n * @param property Property handler to set the value and the type: JSON_REAL or JSON_INTEGER.\n * @retval Pointer to first non white space after the string. If success.\n * @retval Null pointer if any error occur. */\nstatic char* numValue( char* ptr, json_t* property ) {\n if ( *ptr == '-' ) ++ptr;\n if ( !isdigit( *ptr ) ) return 0;\n if ( *ptr != '0' ) {\n ptr = goNum( ptr );\n if ( !ptr ) return 0;\n }\n else if ( isdigit( *++ptr ) ) return 0;\n property->type = JSON_INTEGER;\n if ( *ptr == '.' ) {\n ptr = fraqValue( ++ptr );\n if ( !ptr ) return 0;\n property->type = JSON_REAL;\n }\n if ( *ptr == 'e' || *ptr == 'E' ) {\n ptr = expValue( ++ptr );\n if ( !ptr ) return 0;\n property->type = JSON_REAL;\n }\n if ( !isEndOfPrimitive( *ptr ) ) return 0;\n if ( JSON_INTEGER == property->type ) {\n char const* value = property->u.value;\n bool const negative = *value == '-';\n static char const min[] = \"-9223372036854775808\";\n static char const max[] = \"9223372036854775807\";\n unsigned int const maxdigits = ( negative? sizeof min: sizeof max ) - 1;\n unsigned int const len = ptr - value;\n if ( len > maxdigits ) return 0;\n if ( len == maxdigits ) {\n char const tmp = *ptr;\n *ptr = '\\0';\n char const* const threshold = negative ? min: max;\n if ( 0 > strcmp( threshold, value ) ) return 0;\n *ptr = tmp;\n }\n }\n ptr = setToNull( ptr );\n return ptr;\n}\n\n/** Add a property to a JSON object or array.\n * @param obj The handler of the JSON object or array.\n * @param property The handler of the property to be added. */\nstatic void add( json_t* obj, json_t* property ) {\n property->sibling = 0;\n if ( !obj->u.c.child ){\n\t obj->u.c.child = property;\n\t obj->u.c.last_child = property;\n } else {\n\t obj->u.c.last_child->sibling = property;\n\t obj->u.c.last_child = property;\n }\n}\n\n/** Parser a string to get a json object value.\n * @param str Pointer to first character.\n * @param pool The handler of a json pool for creating json instances.\n * @retval Pointer to first character after the value. If success.\n * @retval Null pointer if any error occur. */\nstatic char* objValue( char* ptr, json_t* obj, jsonPool_t* pool ) {\n obj->type = JSON_OBJ;\n obj->u.c.child = 0;\n obj->sibling = 0;\n ptr++;\n for(;;) {\n ptr = goBlank( ptr );\n if ( !ptr ) return 0;\n if ( *ptr == ',' ) {\n ++ptr;\n continue;\n }\n char const endchar = ( obj->type == JSON_OBJ )? '}': ']';\n if ( *ptr == endchar ) {\n *ptr = '\\0';\n json_t* parentObj = obj->sibling;\n if ( !parentObj ) return ++ptr;\n obj->sibling = 0;\n obj = parentObj;\n ++ptr;\n continue;\n }\n json_t* property = pool->alloc( pool );\n if ( !property ) return 0;\n if( obj->type != JSON_ARRAY ) {\n if ( *ptr != '\\\"' ) return 0;\n ptr = propertyName( ptr, property );\n if ( !ptr ) return 0;\n }\n else property->name = 0;\n add( obj, property );\n property->u.value = ptr;\n switch( *ptr ) {\n case '{':\n property->type = JSON_OBJ;\n property->u.c.child = 0;\n property->sibling = obj;\n obj = property;\n ++ptr;\n break;\n case '[':\n property->type = JSON_ARRAY;\n property->u.c.child = 0;\n property->sibling = obj;\n obj = property;\n ++ptr;\n break;\n case '\\\"': ptr = textValue( ptr, property ); break;\n case 't': ptr = trueValue( ptr, property ); break;\n case 'f': ptr = falseValue( ptr, property ); break;\n case 'n': ptr = nullValue( ptr, property ); break;\n default: ptr = numValue( ptr, property ); break;\n }\n if ( !ptr ) return 0;\n }\n}\n\n/** Initialize a json pool.\n * @param pool The handler of the pool.\n * @return a instance of a json. */\nstatic json_t* poolInit( jsonPool_t* pool ) {\n jsonStaticPool_t *spool = json_containerOf( pool, jsonStaticPool_t, pool );\n spool->nextFree = 1;\n return spool->mem;\n}\n\n/** Create an instance of a json from a pool.\n * @param pool The handler of the pool.\n * @retval The handler of the new instance if success.\n * @retval Null pointer if the pool was empty. */\nstatic json_t* poolAlloc( jsonPool_t* pool ) {\n jsonStaticPool_t *spool = json_containerOf( pool, jsonStaticPool_t, pool );\n if ( spool->nextFree >= spool->qty ) return 0;\n return spool->mem + spool->nextFree++;\n}\n\n/** Checks whether an character belongs to set.\n * @param ch Character value to be checked.\n * @param set Set of characters. It is just a null-terminated string.\n * @return true or false there is membership or not. */\nstatic bool isOneOfThem( char ch, char const* set ) {\n while( *set != '\\0' )\n if ( ch == *set++ )\n return true;\n return false;\n}\n\n/** Increases a pointer while it points to a character that belongs to a set.\n * @param str The initial pointer value.\n * @param set Set of characters. It is just a null-terminated string.\n * @return The final pointer value or null pointer if the null character was found. */\nstatic char* goWhile( char* str, char const* set ) {\n for(; *str != '\\0'; ++str ) {\n if ( !isOneOfThem( *str, set ) )\n return str;\n }\n return 0;\n}\n\n/** Set of characters that defines a blank. */\nstatic char const* const blank = \" \\n\\r\\t\\f\";\n\n/** Increases a pointer while it points to a white space character.\n * @param str The initial pointer value.\n * @return The final pointer value or null pointer if the null character was found. */\nstatic char* goBlank( char* str ) {\n return goWhile( str, blank );\n}\n\n/** Increases a pointer while it points to a decimal digit character.\n * @param str The initial pointer value.\n * @return The final pointer value or null pointer if the null character was found. */\nstatic char* goNum( char* str ) {\n for( ; *str != '\\0'; ++str ) {\n if ( !isdigit( *str ) )\n return str;\n }\n return 0;\n}\n\n/** Set of characters that defines the end of an array or a JSON object. */\nstatic char const* const endofblock = \"}]\";\n\n/** Set a char to '\\0' and increase its pointer if the char is different to '}' or ']'.\n * @param ch Pointer to character.\n * @return Final value pointer. */\nstatic char* setToNull( char* ch ) {\n if ( !isOneOfThem( *ch, endofblock ) ) *ch++ = '\\0';\n return ch;\n}\n\n/** Indicate if a character is the end of a primitive value. */\nstatic bool isEndOfPrimitive( char ch ) {\n return ch == ',' || isOneOfThem( ch, blank ) || isOneOfThem( ch, endofblock );\n}\n\n/** Add a character at the end of a string.\n * @param dest Pointer to the null character of the string\n * @param ch Value to be added.\n * @return Pointer to the null character of the destination string. */\nstatic char* chtoa( char* dest, char ch ) {\n *dest = ch;\n *++dest = '\\0';\n return dest;\n}\n\n/** Copy a null-terminated string.\n * @param dest Destination memory block.\n * @param src Source string.\n * @return Pointer to the null character of the destination string. */\nstatic char* atoa( char* dest, char const* src ) {\n for( ; *src != '\\0'; ++dest, ++src )\n *dest = *src;\n *dest = '\\0';\n return dest;\n}\n\n/* Open a JSON object in a JSON string. */\nchar* json_objOpen( char* dest, char const* name ) {\n if ( NULL == name )\n dest = chtoa( dest, '{' );\n else {\n dest = chtoa( dest, '\\\"' );\n dest = atoa( dest, name );\n dest = atoa( dest, \"\\\":{\" );\n }\n return dest;\n}\n\n/* Close a JSON object in a JSON string. */\nchar* json_objClose( char* dest ) {\n if ( dest[-1] == ',' )\n --dest;\n return atoa( dest, \"},\" );\n}\n\n/* Open an array in a JSON string. */\nchar* json_arrOpen( char* dest, char const* name ) {\n if ( NULL == name )\n dest = chtoa( dest, '[' );\n else {\n dest = chtoa( dest, '\\\"' );\n dest = atoa( dest, name );\n dest = atoa( dest, \"\\\":[\" );\n }\n return dest;\n}\n\n/* Close an array in a JSON string. */\nchar* json_arrClose( char* dest ) {\n if ( dest[-1] == ',' )\n --dest;\n return atoa( dest, \"],\" );\n}\n\n/** Add the name of a text property.\n * @param dest Destination memory.\n * @param name The name of the property.\n * @return Pointer to the next char. */\nstatic char* strname( char* dest, char const* name ) {\n dest = chtoa( dest, '\\\"' );\n if ( NULL != name ) {\n dest = atoa( dest, name );\n dest = atoa( dest, \"\\\":\\\"\" );\n }\n return dest;\n}\n\n/** Get the hexadecimal digit of the least significant nibble of a integer. */\nstatic int nibbletoch( int nibble ) {\n return \"0123456789ABCDEF\"[ nibble % 16u ];\n}\n\n/** Get the escape character of a non-printable.\n * @param ch Character source.\n * @return The escape character or null character if error. */\nstatic int escape( int ch ) {\n static struct { char code; char ch; } const pair[] = {\n { '\\\"', '\\\"' }, { '\\\\', '\\\\' }, { '/', '/' }, { 'b', '\\b' },\n { 'f', '\\f' }, { 'n', '\\n' }, { 'r', '\\r' }, { 't', '\\t' },\n };\n for( int i = 0; i < sizeof pair / sizeof *pair; ++i )\n if ( ch == pair[i].ch )\n return pair[i].code;\n return '\\0';\n}\n\n/** Copy a null-terminated string inserting escape characters if needed.\n * @param dest Destination memory block.\n * @param src Source string.\n * @return Pointer to the null character of the destination string. */\nstatic char* atoesc( char* dest, char const* src ) {\n for( ; *src != '\\0'; ++dest, ++src ) {\n if ( *src >= ' ' && *src != '\\\"' && *src != '\\\\' && *src != '/' )\n *dest = *src;\n else {\n *dest++ = '\\\\';\n int const esc = escape( *src );\n if ( esc )\n *dest = esc;\n else {\n *dest++ = 'u';\n *dest++ = '0';\n *dest++ = '0';\n *dest++ = nibbletoch( *src / 16 );\n *dest++ = nibbletoch( *src );\n }\n }\n }\n *dest = '\\0';\n return dest;\n}\n\n/* Add a text property in a JSON string. */\nchar* json_str( char* dest, char const* name, char const* value ) {\n dest = strname( dest, name );\n dest = atoesc( dest, value );\n dest = atoa( dest, \"\\\",\" );\n return dest;\n}\n\n/** Add the name of a primitive property.\n * @param dest Destination memory.\n * @param name The name of the property.\n * @return Pointer to the next char. */\nstatic char* primitivename( char* dest, char const* name ) {\n if( NULL == name )\n return dest;\n dest = chtoa( dest, '\\\"' );\n dest = atoa( dest, name );\n dest = atoa( dest, \"\\\":\" );\n return dest;\n}\n\n/* Add a boolean property in a JSON string. */\nchar* json_bool( char* dest, char const* name, int value ) {\n dest = primitivename( dest, name );\n dest = atoa( dest, value ? \"true,\" : \"false,\" );\n return dest;\n}\n\n/* Add a null property in a JSON string. */\nchar* json_null( char* dest, char const* name ) {\n dest = primitivename( dest, name );\n dest = atoa( dest, \"null,\" );\n return dest;\n}\n\n/* Used to finish the root JSON object. After call json_objClose(). */\nchar* json_end( char* dest ) {\n if ( ',' == dest[-1] ) {\n dest[-1] = '\\0';\n --dest;\n }\n return dest;\n}\n\n#define ALL_TYPES \\\n X( json_int, int, \"%d\" ) \\\n X( json_long, long, \"%ld\" ) \\\n X( json_uint, unsigned int, \"%u\" ) \\\n X( json_ulong, unsigned long, \"%lu\" ) \\\n X( json_verylong, long long, \"%lld\" ) \\\n X( json_double, double, \"%g\" ) \\\n\n\n#define json_num( funcname, type, fmt ) \\\nchar* funcname( char* dest, char const* name, type value ) { \\\n dest = primitivename( dest, name ); \\\n dest += sprintf( dest, fmt, value ); \\\n dest = chtoa( dest, ',' ); \\\n return dest; \\\n}\n\n#define X( name, type, fmt ) json_num( name, type, fmt )\nALL_TYPES\n#undef X\n" }, { "path": "External/tiny-json/tiny-json.h", "content": "\n/*\n\n\n \n Licensed under the MIT License .\n SPDX-License-Identifier: MIT\n Copyright (c) 2016-2018 Rafa Garcia .\n\n Permission is hereby granted, free of charge, to any person obtaining a copy\n of this software and associated documentation files (the \"Software\"), to deal\n in the Software without restriction, including without limitation the rights\n to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n copies of the Software, and to permit persons to whom the Software is\n furnished to do so, subject to the following conditions:\n\n The above copyright notice and this permission notice shall be included in all\n copies or substantial portions of the Software.\n\n THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\n SOFTWARE.\n \n*/\n\n#ifndef _TINY_JSON_H_\n#define\t_TINY_JSON_H_\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\n#include \n#include \n#include \n#include \n\n#define json_containerOf( ptr, type, member ) \\\n ((type*)( (char*)ptr - offsetof( type, member ) ))\n\n/** @defgroup tinyJson Tiny JSON parser.\n * @{ */\n\n/** Enumeration of codes of supported JSON properties types. */\ntypedef enum {\n JSON_OBJ, JSON_ARRAY, JSON_TEXT, JSON_BOOLEAN,\n JSON_INTEGER, JSON_REAL, JSON_NULL\n} jsonType_t;\n\n/** Structure to handle JSON properties. */\ntypedef struct json_s {\n struct json_s* sibling;\n const char* name;\n union {\n const char* value;\n struct {\n struct json_s* child;\n struct json_s* last_child;\n } c;\n } u;\n jsonType_t type;\n} json_t;\n\n/** Parse a string to get a json.\n * @param str String pointer with a JSON object. It will be modified.\n * @param mem Array of json properties to allocate.\n * @param qty Number of elements of mem.\n * @retval Null pointer if any was wrong in the parse process.\n * @retval If the parser process was successfully a valid handler of a json.\n * This property is always unnamed and its type is JSON_OBJ. */\nconst json_t* json_create(char* str, json_t mem[], unsigned int qty);\n\n/** Get the name of a json property.\n * @param json A valid handler of a json property.\n * @retval Pointer to null-terminated if property has name.\n * @retval Null pointer if the property is unnamed. */\nstatic inline const char* json_getName(const json_t* json) {\n return json->name;\n}\n\n/** Get the value of a json property.\n * The type of property cannot be JSON_OBJ or JSON_ARRAY.\n * @param json A valid handler of a json property.\n * @return Pointer to null-terminated string with the value. */\nstatic inline const char* json_getValue(const json_t* property) {\n return property->u.value;\n}\n\n/** Get the type of a json property.\n * @param json A valid handler of a json property.\n * @return The code of type.*/\nstatic inline jsonType_t json_getType(const json_t* json) {\n return json->type;\n}\n\n/** Get the next sibling of a JSON property that is within a JSON object or array.\n * @param json A valid handler of a json property.\n * @retval The handler of the next sibling if found.\n * @retval Null pointer if the json property is the last one. */\nstatic inline const json_t* json_getSibling(const json_t* json) {\n return json->sibling;\n}\n\n/** Search a property by its name in a JSON object.\n * @param obj A valid handler of a json object. Its type must be JSON_OBJ.\n * @param property The name of property to get.\n * @retval The handler of the json property if found.\n * @retval Null pointer if not found. */\nconst json_t* json_getProperty(const json_t* obj, const char* property);\n\n\n/** Search a property by its name in a JSON object and return its value.\n * @param obj A valid handler of a json object. Its type must be JSON_OBJ.\n * @param property The name of property to get.\n * @retval If found a pointer to null-terminated string with the value.\n * @retval Null pointer if not found or it is an array or an object. */\nconst char* json_getPropertyValue(const json_t* obj, const char* property);\n\n/** Get the first property of a JSON object or array.\n * @param json A valid handler of a json property.\n * Its type must be JSON_OBJ or JSON_ARRAY.\n * @retval The handler of the first property if there is.\n * @retval Null pointer if the json object has not properties. */\nstatic inline const json_t* json_getChild(const json_t* json) {\n return json->u.c.child;\n}\n\n/** Get the value of a json boolean property.\n * @param property A valid handler of a json object. Its type must be JSON_BOOLEAN.\n * @return The value stdbool. */\nstatic inline bool json_getBoolean(const json_t* property) {\n return *property->u.value == 't';\n}\n\n/** Get the value of a json integer property.\n * @param property A valid handler of a json object. Its type must be JSON_INTEGER.\n * @return The value stdint. */\nstatic inline int64_t json_getInteger(const json_t* property) {\n return atoll( property->u.value );\n}\n\n/** Get the value of a json real property.\n * @param property A valid handler of a json object. Its type must be JSON_REAL.\n * @return The value. */\nstatic inline double json_getReal(const json_t* property) {\n return atof( property->u.value );\n}\n\n\n/** Structure to handle a heap of JSON properties. */\ntypedef struct jsonPool_s jsonPool_t;\nstruct jsonPool_s {\n json_t* (*init)( jsonPool_t* pool );\n json_t* (*alloc)( jsonPool_t* pool );\n};\n\n/** Parse a string to get a json.\n * @param str String pointer with a JSON object. It will be modified.\n * @param pool Custom json pool pointer.\n * @retval Null pointer if any was wrong in the parse process.\n * @retval If the parser process was successfully a valid handler of a json.\n * This property is always unnamed and its type is JSON_OBJ. */\nconst json_t* json_createWithPool(char* str, jsonPool_t* pool);\n\n/** @ } */\n\n/** @defgroup makejoson Make JSON.\n * @{ */\n\n/** Open a JSON object in a JSON string.\n * @param dest Pointer to the end of JSON under construction.\n * @param name Pointer to null-terminated string or null for unnamed.\n * @return Pointer to the new end of JSON under construction. */\nchar* json_objOpen(char* dest, const char* name);\n\n/** Close a JSON object in a JSON string.\n * @param dest Pointer to the end of JSON under construction.\n * @return Pointer to the new end of JSON under construction. */\nchar* json_objClose(char* dest);\n\n/** Used to finish the root JSON object. After call json_objClose().\n * @param dest Pointer to the end of JSON under construction.\n * @return Pointer to the new end of JSON under construction. */\nchar* json_end(char* dest);\n\n/** Open an array in a JSON string.\n * @param dest Pointer to the end of JSON under construction.\n * @param name Pointer to null-terminated string or null for unnamed.\n * @return Pointer to the new end of JSON under construction. */\nchar* json_arrOpen(char* dest, const char* name);\n\n/** Close an array in a JSON string.\n * @param dest Pointer to the end of JSON under construction.\n * @return Pointer to the new end of JSON under construction. */\nchar* json_arrClose(char* dest);\n\n/** Add a text property in a JSON string.\n * @param dest Pointer to the end of JSON under construction.\n * @param name Pointer to null-terminated string or null for unnamed.\n * @param value A valid null-terminated string with the value.\n * Backslash escapes will be added for special characters.\n * @return Pointer to the new end of JSON under construction. */\nchar* json_str(char* dest, const char* name, const char* value);\n\n/** Add a boolean property in a JSON string.\n * @param dest Pointer to the end of JSON under construction.\n * @param name Pointer to null-terminated string or null for unnamed.\n * @param value Zero for false. Non zero for true.\n * @return Pointer to the new end of JSON under construction. */\nchar* json_bool(char* dest, const char* name, int value);\n\n/** Add a null property in a JSON string.\n * @param dest Pointer to the end of JSON under construction.\n * @param name Pointer to null-terminated string or null for unnamed.\n * @return Pointer to the new end of JSON under construction. */\nchar* json_null(char* dest, const char* name);\n\n/** Add an integer property in a JSON string.\n * @param dest Pointer to the end of JSON under construction.\n * @param name Pointer to null-terminated string or null for unnamed.\n * @param value Value of the property.\n * @return Pointer to the new end of JSON under construction. */\nchar* json_int(char* dest, const char* name, int value);\n\n/** Add an unsigned integer property in a JSON string.\n * @param dest Pointer to the end of JSON under construction.\n * @param name Pointer to null-terminated string or null for unnamed.\n * @param value Value of the property.\n * @return Pointer to the new end of JSON under construction. */\nchar* json_uint(char* dest, const char* name, unsigned int value);\n\n/** Add a long integer property in a JSON string.\n * @param dest Pointer to the end of JSON under construction.\n * @param name Pointer to null-terminated string or null for unnamed.\n * @param value Value of the property.\n * @return Pointer to the new end of JSON under construction. */\nchar* json_long(char* dest, const char* name, long int value);\n\n/** Add an unsigned long integer property in a JSON string.\n * @param dest Pointer to the end of JSON under construction.\n * @param name Pointer to null-terminated string or null for unnamed.\n * @param value Value of the property.\n * @return Pointer to the new end of JSON under construction. */\nchar* json_ulong(char* dest, const char* name, unsigned long int value);\n\n/** Add a long long integer property in a JSON string.\n * @param dest Pointer to the end of JSON under construction.\n * @param name Pointer to null-terminated string or null for unnamed.\n * @param value Value of the property.\n * @return Pointer to the new end of JSON under construction. */\nchar* json_verylong(char* dest, const char* name, long long int value);\n\n/** Add a double precision number property in a JSON string.\n * @param dest Pointer to the end of JSON under construction.\n * @param name Pointer to null-terminated string or null for unnamed.\n * @param value Value of the property.\n * @return Pointer to the new end of JSON under construction. */\nchar* json_double(char* dest, const char* name, double value);\n\n/** @ } */\n\n#ifdef __cplusplus\n}\n#endif\n\n#endif\t/* _TINY_JSON_H_ */\n" }, { "path": "FEXCore/CMakeLists.txt", "content": "cmake_minimum_required(VERSION 3.14)\nset(PROJECT_NAME FEXCore)\nproject(${PROJECT_NAME}\n VERSION 0.01\n LANGUAGES CXX)\n\nif (CMAKE_SYSTEM_PROCESSOR MATCHES \"x86_64\")\n set(ARCHITECTURE_x86_64 1)\n set(CMAKE_CXX_FLAGS \"${CMAKE_CXX_FLAGS} -mcx16\")\nendif()\n\nif (CMAKE_SYSTEM_PROCESSOR MATCHES \"^aarch64|^arm64|^armv8\\.*\")\n set(ARCHITECTURE_arm64 1)\nendif()\n\nset(CMAKE_POSITION_INDEPENDENT_CODE ON)\ncmake_policy(SET CMP0083 NEW) # Follow new PIE policy\ninclude(CheckPIESupported)\ncheck_pie_supported()\n\nset(CMAKE_INCLUDE_CURRENT_DIR ON)\n\ninclude(CheckCXXCompilerFlag)\ninclude(CheckIncludeFileCXX)\ninclude(CheckCXXSourceCompiles)\n\nset(CMAKE_CXX_STANDARD 20)\nset(CMAKE_EXPORT_COMPILE_COMMANDS ON)\n\nconfigure_file(${CMAKE_CURRENT_SOURCE_DIR}/include/git_version.h.in\n ${CMAKE_BINARY_DIR}/generated/git_version.h)\n\ninclude_directories(${CMAKE_BINARY_DIR}/generated)\n\n# Disable strict aliasing for all build modes\n# See discussion in https://github.com/FEX-Emu/FEX/pull/4494#issuecomment-2800608944\n# for background context.\nadd_compile_options($<$:-fno-strict-aliasing> $<$:-fno-exceptions>)\n\nadd_subdirectory(Source/)\n\nif (NOT BUILD_STEAM_SUPPORT)\n install (DIRECTORY include/FEXCore ${CMAKE_BINARY_DIR}/include/FEXCore\n DESTINATION include\n COMPONENT Development)\nendif()\n\nif (BUILD_TESTING)\n add_subdirectory(unittests/)\nendif()\n" }, { "path": "FEXCore/LICENSE", "content": "MIT License\n\nCopyright (c) 2019 Ryan Houdek \n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n" }, { "path": "FEXCore/Readme.md", "content": "# FEXCore - Fast x86 Core emulation library\nThis is the core emulation library that is used for the FEX emulator project.\nThis project aims to provide a fast and functional x86-64 emulation library that can meet and surpass other x86-64 emulation libraries.\n### Goals\n* Be as fast as possible, beating and exceeding current options for x86-64 emulation\n * 25% - 50% lower performance than native code would be desired target\n * Use an IR to efficiently translate x86-64 to our host architecture\n * Support a tiered recompiler to allow for fast runtime performance\n * Support offline compilation and offline tooling for inspection and performance analysis\n * Support threaded emulation. Including emulating x86-64's strong memory model on weak memory model architectures\n* Support a significant portion of the x86-64 instruction space.\n * Including MMX, SSE, SSE2, SSE3, SSSE3, and SSE4*\n* Support fallback routines for uncommonly used x86-64 instructions\n * Including x87 and 3DNow!\n* Only support userspace emulation.\n * All x86-64 instructions run as if they are under CPL-3(userland) security layer\n* Minimal Linux Syscall emulation for testing purposes\n* Portable library implementation in order to support easy integration in to applications\n### Target Host Architecture\nThe target host architecture for this library is AArch64. Specifically the ARMv8.1 version or newer.\nThe CPU IR is designed with AArch64 in mind but should allow for other architectures as well.\nx86-64 host support is available for ease of development, but is not a priority.\n### Not desired\n* Kernel space emulation\n* CPL0-2 emulation\n* Real Mode, Protected Mode, Virtual-8086 Mode, System Management Mode\n* IRQs\n* SVM\n* \"Cycle Accurate\" emulation\n" }, { "path": "FEXCore/Scripts/config_generator.py", "content": "import datetime\nimport json\nimport sys\n\ndef print_header():\n header = '''#ifndef OPT_BASE\n#define OPT_BASE(type, group, enum, json, default)\n#endif\n#ifndef OPT_BOOL\n#define OPT_BOOL(group, enum, json, default) OPT_BASE(bool, group, enum, json, default)\n#endif\n#ifndef OPT_UINT8\n#define OPT_UINT8(group, enum, json, default) OPT_BASE(uint8_t, group, enum, json, default)\n#endif\n#ifndef OPT_INT32\n#define OPT_INT32(group, enum, json, default) OPT_BASE(int32_t, group, enum, json, default)\n#endif\n#ifndef OPT_UINT32\n#define OPT_UINT32(group, enum, json, default) OPT_BASE(uint32_t, group, enum, json, default)\n#endif\n#ifndef OPT_UINT64\n#define OPT_UINT64(group, enum, json, default) OPT_BASE(uint64_t, group, enum, json, default)\n#endif\n#ifndef OPT_STR\n#define OPT_STR(group, enum, json, default) OPT_BASE(fextl::string, group, enum, json, default)\n#endif\n#ifndef OPT_STRARRAY\n#define OPT_STRARRAY(group, enum, json, default) OPT_BASE(fextl::string, group, enum, json, default)\n#endif\n#ifndef OPT_STRENUM\n#define OPT_STRENUM(group, enum, json, default) OPT_BASE(uint64_t, group, enum, json, default)\n#endif\n'''\n output_file.write(header)\n\ndef print_tail():\n tail = '''#undef OPT_BASE\n#undef OPT_BOOL\n#undef OPT_UINT8\n#undef OPT_INT32\n#undef OPT_UINT32\n#undef OPT_UINT64\n#undef OPT_STR\n#undef OPT_STRARRAY\n#undef OPT_STRENUM\n'''\n output_file.write(tail)\n\ndef print_config(type, group_name, json_name, default_value):\n output_file.write(\"OPT_{0} ({1}, {2}, {3}, {4})\\n\".format(type.upper(), group_name.upper(), json_name.upper(), json_name, default_value))\n\ndef print_options(options):\n for op_group, group_vals in options.items():\n for op_key, op_vals in group_vals.items():\n default = op_vals[\"Default\"]\n if (op_vals[\"Type\"] == \"str\" or op_vals[\"Type\"] == \"strarray\"):\n # Wrap the string argument in quotes\n default = \"\\\"\" + default + \"\\\"\"\n\n print_config(\n op_vals[\"Type\"],\n op_group,\n op_key,\n default)\n\n output_file.write(\"\\n\")\n\ndef print_unnamed_options(options):\n output_file.write(\"// Unnamed configuration options\\n\")\n for op_group, group_vals in options.items():\n for op_key, op_vals in group_vals.items():\n default = op_vals[\"Default\"]\n if (op_vals[\"Type\"] == \"str\" or op_vals[\"Type\"] == \"strarray\"):\n # Wrap the string argument in quotes\n default = \"\\\"\" + default + \"\\\"\"\n\n print_config(\n op_vals[\"Type\"],\n op_group,\n op_key.upper(), # KEY is the enum here, there is no json configuration for these\n default)\n\n output_file.write(\"\\n\")\n\ndef print_man_option(short, long, desc, default):\n if (short != None):\n output_man.write(\".It Fl {0} , \".format(short))\n else:\n output_man.write(\".It \")\n\n output_man.write(\"Fl Fl {0}=\".format(long))\n\n output_man.write(\"\\n\");\n\n # Print description\n for line in desc:\n output_man.write(\".Pp\\n\")\n output_man.write(\"{0}\\n\".format(line))\n\n output_man.write(\".Pp\\n\")\n output_man.write(\"\\\\fBdefault:\\\\fR {0}\\n\".format(default))\n output_man.write(\".Pp\\n\\n\")\n\ndef print_man_env_option(name, desc, default, no_json_key):\n output_man.write(\"\\\\fBFEX_{0}\\\\fR\\n\".format(name.upper()))\n\n # Print description\n for line in desc:\n output_man.write(\".Pp\\n\")\n output_man.write(\"{0}\\n\".format(line))\n\n if (not no_json_key):\n output_man.write(\".Pp\\n\")\n output_man.write(\"\\\\fBJSON key:\\\\fR '{0}'\\n\".format(name))\n output_man.write(\".Pp\\n\\n\")\n\n output_man.write(\".Pp\\n\")\n output_man.write(\"\\\\fBdefault:\\\\fR {0}\\n\".format(default))\n output_man.write(\".Pp\\n\\n\")\n\ndef print_man_environment(options):\n output_man.write(\".Sh ENVIRONMENT\\n\")\n output_man.write(\".Bl -tag -width -indent\\n\")\n for op_group, group_vals in options.items():\n for op_key, op_vals in group_vals.items():\n default = op_vals[\"Default\"]\n value_type = op_vals[\"Type\"]\n\n # Textual default rather than enum based\n if (\"TextDefault\" in op_vals):\n default = op_vals[\"TextDefault\"]\n\n if (value_type == \"str\" or value_type == \"strarray\" or value_type == \"strenum\"):\n # Wrap the string argument in quotes\n default = \"'\" + default + \"'\"\n print_man_env_option(\n op_key,\n op_vals[\"Desc\"],\n default,\n False\n )\n\n if (value_type == \"strenum\"):\n Enums = op_vals[\"Enums\"]\n output_man.write(\"\\\\fBAvailable Options:\\\\fR\\n\")\n output_man.write(\", \".join(f\"{enum_op_val}\" for [_, enum_op_val] in Enums.items()))\n output_man.write(\"\\n.sp\\n\")\n\n print_man_environment_tail()\n output_man.write(\".El\\n\")\n\ndef print_man_environment_tail():\n\n # Additional environment variables that live outside of the normal loop\n print_man_env_option(\n \"APP_CONFIG_LOCATION\",\n [\n \"Allows the user to override where FEX looks for configuration files\",\n \"By default FEX will look in ${XDG_CONFIG_HOME, $HOME/.config}/fex-emu/\",\n \"This will override the full path\",\n \"If FEX_PORTABLE is declared then relative paths are also supported\",\n \"For FEX: Relative to the FEX binary\",\n \"For WINE: Relative to %LOCALAPPDATA%\"\n ],\n \"''\", True)\n\n print_man_env_option(\n \"APP_CONFIG\",\n [\n \"Allows the user to override where FEX looks for only the application config file\",\n \"By default FEX will look in ${XDG_CONFIG_HOME, $HOME/.config}/fex-emu/Config.json\",\n \"This will override this file location\",\n \"One must be careful with this option as it will override any applications that load with execve as well\"\n \"If you need to support applications that execve then use FEX_APP_CONFIG_LOCATION instead\"\n \"If FEX_PORTABLE is declared then relative paths are also supported\",\n \"For FEX: Relative to the FEX binary\",\n \"For WINE: Relative to %LOCALAPPDATA%\"\n ],\n \"''\", True)\n\n print_man_env_option(\n \"APP_DATA_LOCATION\",\n [\n \"Allows the user to override where FEX looks for data files\",\n \"By default FEX will look in {$XDG_DATA_HOME, $HOME/.local/share}/fex-emu/\",\n \"This will override the full path\",\n \"This is the folder where FEX stores generated files like IR cache\"\n ],\n \"''\", True)\n\n print_man_env_option(\n \"PORTABLE\",\n [\n \"Allows FEX to run without installation. Global locations for configuration and binfmt_misc are ignored.\",\n \"For FEX on Linux:\",\n \"These files are instead read from /fex-emu/ by default.\",\n \"For Arm64ec/Wow64 WINE builds:\",\n \"These files are instead read from $LOCALAPPDATA/fex-emu/ by default.\",\n \"For further customization, see FEX_APP_CONFIG_LOCATION and FEX_APP_DATA_LOCATION.\"\n ],\n \"''\", True)\n\n print_man_env_option(\n \"APP_CACHE_LOCATION\",\n [\n \"Allows the user to override where FEX stores and loads cache files\",\n \"By default FEX will look in ${XDG_CACHE_HOME, $HOME/.cache}/fex-emu/\",\n \"This will override the full path, trailing forward-slash is expected to exist\",\n ],\n \"''\", True)\n\ndef print_man_header():\n header ='''.Dd {0}\n.Dt FEX\n.Os Linux\n.Sh NAME\n.Nm FEX\n.Nm FEXBash\n.Nd Fast x86-64 and x86 emulation.\n.Sh SYNOPSIS\n.Nm\n.Ar ...\n.Pp\n.Nm FEXBash\n.Ar ...\n.Sh DESCRIPTION\nFEX allows you to run x86 and x86-64 binaries on an AArch64 host, similar to qemu-user and box86.\nIt has native support for a rootfs overlay, so you don't need to chroot, as well as some thunklibs so it can forward things like GL to the host.\nFEX presents a Linux 5.0 interface to the guest, and supports both AArch64 and x86-64 as hosts.\nFEX is very much work in progress, so expect things to change.\n'''\n output_man.write(header.format(datetime.datetime.now().strftime(\"%d-%m-%Y\")))\n\ndef print_man_tail():\n tail ='''.Sh FILES\n.Bl -tag -width \"$prefix/share/fex-emu/GuestThunks\" -compact\n.It Pa $XDG_CONFIG_DIR/fex-emu\nDefault FEX user configuration directory\n.It Pa $prefix/share/fex-emu/AppConfig\nSystem level application configuration files\n.It Pa $prefix/share/fex-emu/GuestThunks\nguest-side thunk data libraries\n.It Pa $prefix/lib/fex-emu/HostThunks\nhost-side thunks for guest communication\n.El\n'''\n output_man.write(tail)\n\ndef print_config_option(type, group_name, json_name, default_value, short, choices, desc):\n if (type == \"bool\"):\n # Bool gets some special handling to add an inverted case\n output_argloader.write(\"{0}Group\".format(group_name))\n\n options = \"\"\n AddedArg = False\n if (short != None):\n AddedArg = True\n options += \"\\\"-{0}\\\"\".format(short)\n\n if (AddedArg):\n options += \", \"\n options += \"\\\"--{0}\\\"\".format(json_name.lower())\n\n output_argloader.write(\".add_option({0})\".format(options))\n\n output_argloader.write(\"\\n\")\n\n output_argloader.write(\"\\t.action(\\\"store_true\\\")\\n\")\n\n output_argloader.write(\"\\t.dest(\\\"{0}\\\")\\n\".format(json_name));\n\n # help\n output_argloader.write(\"\\t.help(\\n\")\n desc_line_ender = \"\"\n if (len(desc) > 1):\n desc_line_ender = \"\\\\n\"\n\n for line in desc:\n output_argloader.write(\"\\t\\t\\\"{0}{1}\\\"\\n\".format(line, desc_line_ender))\n output_argloader.write(\"\\t)\\n\")\n\n output_argloader.write(\"\\t.set_default({0});\\n\\n\".format(default_value));\n\n output_argloader.write(\"{0}Group\".format(group_name))\n output_argloader.write(\".add_option(\\\"--no-{0}\\\")\\n\".format(json_name.lower()))\n\n # Inverted case sets the bool to false\n output_argloader.write(\"\\t.action(\\\"store_false\\\")\\n\")\n\n output_argloader.write(\"\\t.dest(\\\"{0}\\\");\\n\".format(json_name));\n else:\n output_argloader.write(\"{0}Group\".format(group_name))\n options = \"\"\n AddedArg = False\n if (short != None):\n AddedArg = True\n options += \"\\\"-{0}\\\"\".format(short)\n\n if (AddedArg):\n options += \", \"\n options += \"\\\"--{0}\\\"\".format(json_name.lower())\n\n output_argloader.write(\".add_option({0})\".format(options))\n\n output_argloader.write(\"\\n\")\n\n output_argloader.write(\"\\t.dest(\\\"{0}\\\")\\n\".format(json_name));\n\n if (choices != None):\n output_argloader.write(\"\\t.choices({\\n\")\n for choice in choices:\n output_argloader.write(\"\\t\\t\\\"{0}\\\",\\n\".format(choice))\n output_argloader.write(\"\\t})\\n\")\n\n\n # help\n output_argloader.write(\"\\t.help(\\n\")\n desc_line_ender = \"\"\n if (len(desc) > 1):\n desc_line_ender = \"\\\\n\"\n\n for line in desc:\n output_argloader.write(\"\\t\\t\\\"{0}{1}\\\"\\n\".format(line, desc_line_ender))\n output_argloader.write(\"\\t)\\n\")\n\n output_argloader.write(\"\\t.set_default({0});\\n\".format(default_value));\n\n output_argloader.write(\"\\n\");\n\ndef print_parse_envloader_options(options):\n output_argloader.write(\"#ifdef ENVLOADER\\n\")\n output_argloader.write(\"#undef ENVLOADER\\n\")\n output_argloader.write(\"if (false) {}\\n\")\n\n for op_group, group_vals in options.items():\n for op_key, op_vals in group_vals.items():\n value_type = op_vals[\"Type\"]\n if (value_type == \"strenum\"):\n output_argloader.write(\"else if (Key == \\\"FEX_{0}\\\") {{\\n\".format(op_key.upper()))\n output_argloader.write(\"\\tValue = FEXCore::Config::EnumParser(FEXCore::Config::{}_EnumPairs, Value_View);\\n\".format(op_key, op_key))\n output_argloader.write(\"}\\n\")\n\n if (\"ArgumentHandler\" in op_vals):\n conversion_func = \"FEXCore::Config::Handler::{0}\".format(op_vals[\"ArgumentHandler\"])\n output_argloader.write(\"else if (Key == \\\"FEX_{0}\\\") {{\\n\".format(op_key.upper()))\n output_argloader.write(\"\\tValue = {0}(Value_View);\\n\".format(conversion_func))\n output_argloader.write(\"}\\n\")\n output_argloader.write(\"#endif\\n\")\n\ndef print_parse_jsonloader_options(options):\n output_argloader.write(\"#ifdef JSONLOADER\\n\")\n output_argloader.write(\"#undef JSONLOADER\\n\")\n output_argloader.write(\"if (false) {}\\n\")\n op_key = None\n for op_group, group_vals in options.items():\n for op_key, op_vals in group_vals.items():\n value_type = op_vals[\"Type\"]\n if (value_type == \"strenum\"):\n output_argloader.write(\"else if (KeyName == \\\"{0}\\\") {{\\n\".format(op_key))\n output_argloader.write(\"\\tSet(KeyOption, FEXCore::Config::EnumParser(FEXCore::Config::{}_EnumPairs, Value_View));\\n\".format(op_key, op_key))\n output_argloader.write(\"}\\n\")\n elif (value_type == \"strarray\"):\n output_argloader.write(\"else if (KeyName == \\\"{0}\\\") {{\\n\".format(op_key))\n output_argloader.write(\"\\tAppendStrArrayValue(KeyOption, ConfigString);\\n\")\n output_argloader.write(\"}\\n\")\n assert op_key is not None, \"No options found in JSONLOADER\"\n output_argloader.write(\"else {\\n\")\n output_argloader.write(\"\\tSet(KeyOption, ConfigString);\\n\")\n output_argloader.write(\"}\\n\")\n\n output_argloader.write(\"#endif\\n\")\n\ndef print_parse_enum_options(options):\n output_argloader.write(\"#ifdef ENUMDEFINES\\n\")\n output_argloader.write(\"#undef ENUMDEFINES\\n\")\n for op_group, group_vals in options.items():\n for op_key, op_vals in group_vals.items():\n if (op_vals[\"Type\"] == \"strenum\"):\n output_argloader.write(\"enum class {} : uint64_t {{\\n\".format(op_key))\n Enums = op_vals[\"Enums\"]\n i = 0\n # Always have an OFF.\n output_argloader.write(\"\\tOFF = 0,\\n\")\n for enum_op_key, enum_op_vals in Enums.items():\n output_argloader.write(\"\\t{} = 1ULL << {},\\n\".format(enum_op_key.upper(), i))\n i += 1\n\n output_argloader.write(\"};\\n\")\n output_argloader.write(\"FEX_DEF_NUM_OPS({})\\n\".format(op_key))\n\n\n for op_group, group_vals in options.items():\n for op_key, op_vals in group_vals.items():\n if (op_vals[\"Type\"] == \"strenum\"):\n Enums = op_vals[\"Enums\"]\n\n output_argloader.write(\"using {}ConfigPair = std::pair;\\n\".format(op_key, op_key))\n output_argloader.write(\"constexpr static std::array<{}ConfigPair, {}> {}_EnumPairs = {{{{\\n\".format(op_key, len(Enums) + 1, op_key))\n i = 0\n # Always have an OFF.\n output_argloader.write(\"\\t{{ \\\"off\\\", FEXCore::Config::{}::OFF }},\\n\".format(op_key))\n for enum_op_key, enum_op_vals in Enums.items():\n output_argloader.write(\"\\t{{ \\\"{}\\\", FEXCore::Config::{}::{} }},\\n\".format(enum_op_vals, op_key, enum_op_key.upper()))\n i += 1\n\n output_argloader.write(\"}};\\n\")\n\n output_argloader.write(\"#endif\\n\")\n\nif (len(sys.argv) < 5):\n sys.exit()\n\noutput_filename = sys.argv[2]\noutput_man_page = sys.argv[3]\noutput_argumentloader_filename = sys.argv[4]\n\njson_file = open(sys.argv[1], \"r\")\njson_text = json_file.read()\njson_file.close()\n\njson_object = json.loads(json_text)\n\noptions = json_object[\"Options\"]\nunnamed_options = json_object[\"UnnamedOptions\"]\n\n# Generate config include file\noutput_file = open(output_filename, \"w\")\nprint_header()\nprint_options(options)\nprint_unnamed_options(unnamed_options)\nprint_tail()\noutput_file.close()\n\n# Generate man file\noutput_man = open(output_man_page, \"w\")\nprint_man_header()\nprint_man_environment(options)\nprint_man_tail()\n\noutput_man.close()\n\n# Generate argument loader code\noutput_argloader = open(output_argumentloader_filename, \"w\")\n\n# Generate environment loader code\nprint_parse_envloader_options(options);\n\n# Generate json loader code\nprint_parse_jsonloader_options(options);\n\n# Generate enum variable options\nprint_parse_enum_options(options);\n\noutput_argloader.close()\n" }, { "path": "FEXCore/Scripts/json_ir_doc_generator.py", "content": "import collections\nimport json\nimport sys\n\nOpClasses = collections.OrderedDict()\n\ndef get_ir_classes(ops, defines):\n global OpClasses\n\n for op_class, opslist in ops.items():\n if not (op_class in OpClasses):\n OpClasses[op_class] = []\n\n for op, op_val in opslist.items():\n OpClasses[op_class].append([op, op_val])\n\n # Sort the dictionary after we are done parsing it\n OpClasses = collections.OrderedDict(sorted(OpClasses.items()))\n\ndef print_ir_op_index():\n output_file.write(\"# Index\\n\")\n output_file.write(\"## Op Classes\\n\")\n for class_key, class_value in OpClasses.items():\n output_file.write(\"- [%s](#%s)\\n\\n\" % (class_key, class_key))\n\n output_file.write(\"## Definitions\\n\")\n output_file.write(\"- [Defines](#Defines)\\n\\n\")\n\ndef print_ir_ops():\n for class_key, class_value in OpClasses.items():\n output_file.write(\"# %s\\n\\n\" % (class_key))\n for op in class_value:\n op_key = op[0]\n op_vals = op[1]\n output_file.write(\"## %s\\n\" % (op_key))\n\n output_file.write(\">\")\n output_file.write(op_key)\n\n output_file.write(\"\\n\\n\")\n\n if (\"Desc\" in op_vals):\n desc = op_vals[\"Desc\"]\n if (isinstance(desc, list)):\n for line in desc:\n output_file.write(\"%s\\n\\n\" % line)\n else:\n output_file.write(\"%s\\n\" % op_vals[\"Desc\"])\n else:\n output_file.write(\"XXX: Missing op desc!\\n\")\n\ndef print_ir_defines(defines):\n output_file.write(\"## Defines\\n\")\n output_file.write(\"```cpp\\n\")\n for define in defines:\n output_file.write(\"%s\\n\" % (define))\n output_file.write(\"```\\n\")\n\nif (len(sys.argv) < 3):\n sys.exit()\n\noutput_filename = sys.argv[2]\njson_file = open(sys.argv[1], \"r\")\njson_text = json_file.read()\njson_file.close()\n\njson_object = json.loads(json_text)\njson_object = {k.upper(): v for k, v in json_object.items()}\n\nops = json_object[\"OPS\"]\ndefines = json_object[\"DEFINES\"]\n\nget_ir_classes(ops, defines)\n\noutput_file = open(output_filename, \"w\")\n\nprint_ir_op_index()\n\noutput_file.write(\"# IR documentation\\n\\n\")\n\nprint_ir_ops()\n\nprint_ir_defines(defines)\n\noutput_file.close()\n" }, { "path": "FEXCore/Scripts/json_ir_generator.py", "content": "#!/bin/python3\nimport json\nimport sys\nfrom dataclasses import dataclass, field\nimport textwrap\n\ndef ExitError(msg):\n print(msg)\n sys.exit(-1)\n\n@dataclass\nclass IRType:\n IRName: str\n CXXName: str\n def __init__(self, IRName, CXXName):\n self.IRName = IRName\n self.CXXName = CXXName\n\n@dataclass\nclass OpArgument:\n Type: str\n IsSSA: bool\n Temporary: bool\n Name: str\n NameWithPrefix: str\n DefaultInitializer: str\n\n def __init__(self):\n self.Type = None\n self.IsSSA = False\n self.Temporary = False\n self.Name = None\n self.NameWithPrefix = None\n self.DefaultInitializer = None\n return\n\n def print(self):\n attrs = vars(self)\n print(\", \".join(\"%s: %s\" % item for item in attrs.items()))\n\n@dataclass\nclass OpDefinition:\n Name: str\n HasDest: bool\n DestType: str\n DestSize: str\n ElementSize: str\n OpClass: str\n HasSideEffects: bool\n ImplicitFlagClobber: bool\n RAOverride: int\n SwitchGen: bool\n ArgPrinter: bool\n SSAArgNum: int\n NonSSAArgNum: int\n DynamicDispatch: bool\n LoweredX87: bool\n JITDispatch: bool\n JITDispatchOverride: str\n TiedSource: int\n Inline: list[str]\n Arguments: list[OpArgument]\n EmitValidation: list[str]\n Desc: list[str]\n\n def __init__(self):\n self.Name = None\n self.HasDest = False\n self.DestType = None\n self.DestSize = None\n self.ElementSize = None\n self.OpClass = None\n self.OpSize = 0\n self.HasSideEffects = False\n self.ImplicitFlagClobber = False\n self.RAOverride = -1\n self.SwitchGen = True\n self.ArgPrinter = True\n self.SSAArgNum = 0\n self.NonSSAArgNum = 0\n self.DynamicDispatch = False\n self.LoweredX87 = False\n self.JITDispatch = True\n self.JITDispatchOverride = None\n self.TiedSource = -1\n self.Arguments = []\n self.EmitValidation = []\n self.Desc = []\n return\n\n def print(self):\n attrs = vars(self)\n print(\", \".join(\"%s: %s\" % item for item in attrs.items()))\n\nIRTypesToCXX: dict[str, IRType] = {}\nCXXTypeToIR: dict[str, IRType] = {}\nIROps: list[OpDefinition] = []\n\nIROpNameSet: set[str] = set()\n\ndef is_ssa_type(op_type: str):\n return op_type in {\"SSA\", \"GPR\", \"GPRPair\", \"FPR\"}\n\ndef parse_irtypes(irtypes):\n for op_key, op_val in irtypes.items():\n IRTypesToCXX[op_key] = IRType(op_key, op_val)\n CXXTypeToIR[op_val] = IRType(op_key, op_val)\n\ndef parse_ops(ops):\n for op_class, opslist in ops.items():\n for op, op_val in opslist.items():\n if \"Ignore\" in op_val:\n # Skip these\n continue\n\n OpDef = OpDefinition()\n\n # Check if we have a destination\n # Only happens if the IR name contains `=`\n EqualSplit = op.split(\"=\", 1)\n\n RHS = EqualSplit[0].strip()\n if len(EqualSplit) > 1:\n LHS = EqualSplit[0].strip()\n RHS = EqualSplit[1].strip()\n\n if \":\" in LHS:\n # Named destinations. This is a hack, but so is the entire\n # multi-destination support bolten onto the old IR...\n #\n # Named destinations require side effects because they break\n # SSA hard. Validate that.\n assert(\"HasSideEffects\" in op_val and op_val[\"HasSideEffects\"])\n\n for Dest in LHS.split(\",\"):\n Dest = Dest.strip()\n DType, Name = Dest.split(\":$\")\n\n # If the destination appears also as a source, it is\n # read-modify-write.\n if Dest in RHS:\n # Turn RMW into an in/out source\n RHS = RHS.replace(Dest.strip(), f\"{DType}:$Inout{Name}\")\n else:\n # Turn named destinations into an out source.\n RHS += f\", {DType}:$Out{Name}\"\n else:\n # Single anonymous destination\n if LHS not in [\"SSA\", \"GPR\", \"GPRPair\", \"FPR\"]:\n ExitError(f\"Unknown destination class type {LHS}. Needs to be one of SSA, GPR, GPRPair, FPR\")\n\n OpDef.HasDest = True\n OpDef.DestType = LHS\n\n # IR Op needs to start with a name\n RHS = RHS.split(\" \", 1)\n\n if len(RHS) < 1:\n ExitError(\"Missing IR op name. Needs to be a string\")\n\n # Set the op name\n OpDef.Name = RHS[0]\n\n # Parse the arguments\n if len(RHS) > 1:\n Arguments = RHS[1].strip().split(\",\")\n for Argument in Arguments:\n Argument = Argument.strip()\n OpArg = OpArgument()\n\n Split = Argument.split(\":\", 1)\n if len(Split) != 2:\n ExitError(\"Error parsing argument. Missing Type and name colon split\")\n\n # Type is the first argument\n OpArg.Type = Split[0]\n\n # Validate typing is in our type map\n if not OpArg.Type in IRTypesToCXX:\n ExitError(\"IR type {} isn't in IR type map. From IR op {}, argument {}\".format(OpArg.Type, OpDef.Name, Argument))\n\n # Style is the first byte of the name\n if Split[1][0] == \"#\":\n OpArg.Temporary = True\n OpArg.IsSSA = False\n elif Split[1][0] == \"$\":\n OpArg.Temporary = False\n OpArg.IsSSA = is_ssa_type(OpArg.Type)\n if OpArg.IsSSA:\n OpDef.SSAArgNum = OpDef.SSAArgNum + 1\n else:\n OpDef.NonSSAArgNum = OpDef.NonSSAArgNum + 1\n else:\n ExitError(\"IR Op {} missing value argument style specifier. Needs to be one of {{#, $}}\".format(OpDef.Name))\n\n Prefix = Split[1][0]\n ArgName = Split[1][1:]\n NameWithPrefix = Prefix + ArgName\n\n if len(ArgName) == 0:\n ExitError(\"Argument is missing variable name\")\n\n DefaultInit = ArgName.split(\"{\", 1)\n if len(DefaultInit) > 1:\n # We have a default initializer, need to do some more work\n # First argument will still be the argument name\n ArgName = DefaultInit[0].strip()\n NameWithPrefix = Prefix + ArgName\n # Second argument will be the default initializer\n # Since we stripped the opening curly brace then it'll end with a closing brace\n if DefaultInit[1][-1] != \"}\":\n ExitError(\"IR op {} Argument {} is missing closing curly brace in default initializer?\".format(OpDef.Name, ArgName))\n\n OpArg.DefaultInitializer = DefaultInit[1][:-1]\n\n # If SSA type then we can generate validation for this op\n if OpArg.IsSSA and OpArg.Type in {\"GPR\", \"GPRPair\", \"FPR\"}:\n OpDef.EmitValidation.append(f\"GetOpRegClass({ArgName}) == RegClass::Invalid || WalkFindRegClass({ArgName}) == RegClass::{OpArg.Type}\")\n\n OpArg.Name = ArgName\n OpArg.NameWithPrefix = NameWithPrefix\n OpDef.Arguments.append(OpArg)\n\n # Additional metadata\n if \"DestSize\" in op_val:\n OpDef.DestSize = op_val[\"DestSize\"]\n\n if \"ElementSize\" in op_val:\n OpDef.ElementSize = op_val[\"ElementSize\"]\n\n if len(op_class):\n OpDef.OpClass = op_class\n\n if \"HasSideEffects\" in op_val:\n OpDef.HasSideEffects = bool(op_val[\"HasSideEffects\"])\n\n if \"ImplicitFlagClobber\" in op_val:\n OpDef.ImplicitFlagClobber = bool(op_val[\"ImplicitFlagClobber\"])\n\n if \"ArgPrinter\" in op_val:\n OpDef.ArgPrinter = bool(op_val[\"ArgPrinter\"])\n\n if \"RAOverride\" in op_val:\n OpDef.RAOverride = int(op_val[\"RAOverride\"])\n\n if \"SwitchGen\" in op_val:\n OpDef.SwitchGen = op_val[\"SwitchGen\"]\n\n if \"EmitValidation\" in op_val:\n OpDef.EmitValidation.extend(op_val[\"EmitValidation\"])\n\n if \"Desc\" in op_val:\n OpDef.Desc = op_val[\"Desc\"]\n\n if \"DynamicDispatch\" in op_val:\n OpDef.DynamicDispatch = bool(op_val[\"DynamicDispatch\"])\n\n if \"JITDispatch\" in op_val:\n OpDef.JITDispatch = bool(op_val[\"JITDispatch\"])\n\n if \"JITDispatchOverride\" in op_val:\n OpDef.JITDispatchOverride = op_val[\"JITDispatchOverride\"]\n\n if \"X87\" in op_val:\n OpDef.LoweredX87 = op_val[\"X87\"]\n\n # X87 implies !JITDispatch\n assert(\"JITDispatch\" not in op_val)\n OpDef.JITDispatch = False\n\n if \"TiedSource\" in op_val:\n OpDef.TiedSource = op_val[\"TiedSource\"]\n\n # Pad Inline out to the argument count\n OpDef.Inline = [''] * len(OpDef.Arguments)\n if \"Inline\" in op_val:\n Value = op_val[\"Inline\"]\n OpDef.Inline[0:len(Value)] = Value\n\n # Do some fixups of the data here\n if len(OpDef.EmitValidation) != 0:\n for i in range(len(OpDef.EmitValidation)):\n # Patch up all the argument names\n for Arg in OpDef.Arguments:\n # Temporary ops just replace all instances no prefix variant\n OpDef.EmitValidation[i] = OpDef.EmitValidation[i].replace(Arg.NameWithPrefix, Arg.Name)\n\n #OpDef.print()\n\n # Error on duplicate op\n if OpDef.Name in IROpNameSet:\n ExitError(\"Duplicate Op defined! {}\".format(OpDef.Name))\n\n IROps.append(OpDef)\n IROpNameSet.add(OpDef.Name)\n\n# Print out enum values\ndef print_enums(enums):\n output_file.write(\"#ifdef IROP_ENUM\\n\")\n output_file.write(\"enum IROps : uint16_t {\\n\")\n for op in IROps:\n output_file.write(\"\\tOP_{},\\n\" .format(op.Name.upper()))\n output_file.write(\"};\\n\")\n\n for name, members in enums.items():\n output_file.write(f\"enum {name} {{\\n\")\n for member in members:\n if member:\n output_file.write(f\"\\t{member}\\n\")\n else:\n output_file.write(\"\\n\")\n output_file.write(\"};\\n\\n\")\n\n output_file.write(\"#undef IROP_ENUM\\n\")\n output_file.write(\"#endif\\n\\n\")\n\ndef print_ir_structs(defines):\n output_file.write(\"#ifdef IROP_STRUCTS\\n\")\n\n # Print out defines here\n for op_val in defines:\n if op_val:\n output_file.write(\"\\t%s;\\n\" % op_val)\n else:\n output_file.write(\"\\n\")\n\n # Emit the default struct first\n output_file.write(\"// Default structs\\n\")\n output_file.write(\"struct __attribute__((packed)) IROp_Header {\\n\")\n output_file.write(\"\\tvoid* Data[0];\\n\")\n output_file.write(\"\\tIROps Op;\\n\\n\")\n output_file.write(\"\\tIR::OpSize Size;\\n\")\n output_file.write(\"\\tIR::OpSize ElementSize;\\n\")\n\n output_file.write(\"\\ttemplate\\n\")\n output_file.write(\"\\tT const* C() const { return reinterpret_cast(Data); }\\n\")\n output_file.write(\"\\ttemplate\\n\")\n output_file.write(\"\\tT* CW() { return reinterpret_cast(Data); }\\n\")\n\n output_file.write(\"\\tOrderedNodeWrapper Args[0];\\n\")\n\n output_file.write(\"};\\n\\n\");\n output_file.write(\"static_assert(sizeof(IROp_Header) == sizeof(uint32_t), \\\"IROp_Header should be 32-bits in size\\\");\\n\\n\");\n\n # Now the user defined types\n output_file.write(\"// User defined IR Op structs\\n\")\n for op in IROps:\n output_file.write(\"struct __attribute__((packed)) IROp_{} {{\\n\".format(op.Name))\n output_file.write(\"\\tIROp_Header Header;\\n\")\n\n # SSA arguments have a hard requirement to appear after the header\n if op.SSAArgNum > 0:\n output_file.write(\"\\t// SSA arguments\\n\")\n\n # Walk the SSA arguments and place them in order of declaration\n for arg in op.Arguments:\n if arg.IsSSA:\n output_file.write(\"\\tOrderedNodeWrapper {};\\n\".format(arg.Name));\n\n # Non-SSA arguments are also placed in order of declaration, after SSA though\n if op.NonSSAArgNum > 0:\n output_file.write(\"\\t// Non-SSA arguments\\n\")\n for arg in op.Arguments:\n if not arg.Temporary and not arg.IsSSA:\n CType = IRTypesToCXX[arg.Type].CXXName\n output_file.write(\"\\t{} {};\\n\".format(CType, arg.Name));\n\n output_file.write(\"\\tstatic constexpr IROps OPCODE = OP_{};\\n\".format(op.Name.upper()))\n\n\n if op.SSAArgNum > 0:\n output_file.write(\"\\t// Get index of argument by name\\n\")\n SSAArg = 0\n for arg in op.Arguments:\n if arg.IsSSA:\n output_file.write(\"\\tstatic constexpr size_t {}_Index = {};\\n\".format(arg.Name, SSAArg))\n SSAArg = SSAArg + 1\n\n\n output_file.write(\"};\\n\")\n\n # Add a static assert that the IR ops must be pod\n output_file.write(\"static_assert(std::is_trivially_copyable_v);\\n\".format(op.Name))\n output_file.write(\"static_assert(std::is_standard_layout_v);\\n\\n\".format(op.Name))\n\n output_file.write(\"#undef IROP_STRUCTS\\n\")\n output_file.write(\"#endif\\n\\n\")\n\n# Print out const expression to calculate IR Op sizes\ndef print_ir_sizes():\n output_file.write(\"#ifdef IROP_SIZES\\n\")\n\n output_file.write(\"constexpr std::array IRSizes = {\\n\")\n for op in IROps:\n if op.Name == \"Last\":\n output_file.write(\"\\t-1ULL,\\n\")\n else:\n output_file.write(f\"\\tsizeof(IROp_{op.Name}),\\n\")\n\n output_file.write(textwrap.dedent(\"\"\"\n };\n\n // Make sure our array maps directly to the IROps enum\n static_assert(IRSizes[IROps::OP_LAST] == -1ULL);\n\n [[nodiscard]] inline size_t GetSize(IROps Op) { return IRSizes[Op]; }\n [[nodiscard, gnu::const]] std::string_view const& GetName(IROps Op);\n [[nodiscard, gnu::const]] uint8_t GetArgs(IROps Op);\n [[nodiscard, gnu::const]] uint8_t GetRAArgs(IROps Op);\n [[nodiscard, gnu::const]] FEXCore::IR::RegClass GetRegClass(IROps Op);\n [[nodiscard, gnu::const]] bool HasSideEffects(IROps Op);\n [[nodiscard, gnu::const]] bool ImplicitFlagClobber(IROps Op);\n [[nodiscard, gnu::const]] bool GetHasDest(IROps Op);\n [[nodiscard, gnu::const]] bool LoweredX87(IROps Op);\n [[nodiscard, gnu::const]] int8_t TiedSource(IROps Op);\n\n #undef IROP_SIZES\n #endif\n \"\"\"))\n\ndef print_ir_reg_classes():\n output_file.write(\"#ifdef IROP_REG_CLASSES_IMPL\\n\")\n\n output_file.write(\"constexpr std::array IRRegClasses = {\\n\")\n for op in IROps:\n if op.Name == \"Last\":\n output_file.write(\"\\tRegClass::Invalid,\\n\")\n else:\n if op.HasDest and op.DestType is None:\n ExitError(\"IR op {} has destination with no destination class\".format(op.Name))\n\n if op.HasDest and op.DestType == \"SSA\": # Special case SSA type\n output_file.write(\"\\tRegClass::Complex,\\n\")\n elif op.HasDest:\n output_file.write(\"\\tRegClass::{},\\n\".format(op.DestType))\n else:\n # No destination so it has an invalid destination class\n output_file.write(\"\\tRegClass::Invalid, // No destination\\n\")\n\n\n output_file.write(\"};\\n\\n\")\n\n output_file.write(\"// Make sure our array maps directly to the IROps enum\\n\")\n output_file.write(\"static_assert(IRRegClasses[IROps::OP_LAST] == RegClass::Invalid);\\n\\n\")\n\n output_file.write(\"FEXCore::IR::RegClass GetRegClass(IROps Op) { return IRRegClasses[Op]; }\\n\\n\")\n\n output_file.write(\"#undef IROP_REG_CLASSES_IMPL\\n\")\n output_file.write(\"#endif\\n\\n\")\n\n# Print out the name printer implementation\ndef print_ir_getname():\n output_file.write(\"#ifdef IROP_GETNAME_IMPL\\n\")\n output_file.write(\"constexpr std::array IRNames = {\\n\")\n for op in IROps:\n output_file.write(\"\\t\\\"{}\\\",\\n\".format(op.Name))\n\n output_file.write(\"};\\n\\n\")\n\n output_file.write(\"static_assert(IRNames[OP_LAST] == \\\"Last\\\");\\n\\n\")\n\n output_file.write(\"std::string_view const& GetName(IROps Op) {\\n\")\n output_file.write(\" return IRNames[Op];\\n\")\n output_file.write(\"}\\n\")\n\n output_file.write(\"#undef IROP_GETNAME_IMPL\\n\")\n output_file.write(\"#endif\\n\\n\")\n\n# Print out the number of SSA args that need to be RA'd\ndef print_ir_getraargs():\n output_file.write(\"#ifdef IROP_GETRAARGS_IMPL\\n\")\n\n output_file.write(\"constexpr std::array IRRAArgs = {\\n\")\n for op in IROps:\n SSAArgs = op.SSAArgNum\n\n if op.RAOverride != -1:\n if op.RAOverride > op.SSAArgNum:\n ExitError(\"Op {} has RA override of {} which is more than total SSA values {}. This doesn't work\".format(op.Name, op.RAOverride, op.SSAArgNum))\n SSAArgs = op.RAOverride\n\n output_file.write(\"\\t{},\\n\".format(SSAArgs))\n\n output_file.write(\"};\\n\\n\")\n\n\n output_file.write(\"constexpr std::array IRArgs = {\\n\")\n for op in IROps:\n SSAArgs = op.SSAArgNum\n output_file.write(\"\\t{},\\n\".format(SSAArgs))\n\n output_file.write(\"};\\n\\n\")\n\n output_file.write(\"uint8_t GetRAArgs(IROps Op) {\\n\")\n output_file.write(\" return IRRAArgs[Op];\\n\")\n output_file.write(\"}\\n\")\n\n output_file.write(\"uint8_t GetArgs(IROps Op) {\\n\")\n output_file.write(\" return IRArgs[Op];\\n\")\n output_file.write(\"}\\n\")\n\n output_file.write(\"#undef IROP_GETRAARGS_IMPL\\n\")\n output_file.write(\"#endif\\n\\n\")\n\ndef print_ir_hassideeffects():\n output_file.write(\"#ifdef IROP_HASSIDEEFFECTS_IMPL\\n\")\n\n for prop, T in [\n (\"HasSideEffects\", \"bool\"),\n (\"ImplicitFlagClobber\", \"bool\"),\n (\"LoweredX87\", \"bool\"),\n (\"TiedSource\", \"int8_t\"),\n ]:\n output_file.write(\n f\"constexpr std::array<{'uint8_t' if T == 'bool' else T}, OP_LAST + 1> {prop}_ = {{\\n\"\n )\n for op in IROps:\n if T == \"bool\":\n output_file.write(\n \"\\t{},\\n\".format((\"true\" if getattr(op, prop) else \"false\"))\n )\n else:\n output_file.write(f\"\\t{getattr(op, prop)},\\n\")\n\n output_file.write(\"};\\n\\n\")\n\n output_file.write(f\"{T} {prop}(IROps Op) {{\\n\")\n output_file.write(f\" return {prop}_[Op];\\n\")\n output_file.write(\"}\\n\")\n\n output_file.write(\"#undef IROP_HASSIDEEFFECTS_IMPL\\n\")\n output_file.write(\"#endif\\n\\n\")\n\ndef print_ir_gethasdest():\n output_file.write(\"#ifdef IROP_GETHASDEST_IMPL\\n\")\n\n output_file.write(\"constexpr std::array IRDest = {\\n\")\n for op in IROps:\n if op.HasDest:\n output_file.write(\"\\ttrue,\\n\")\n else:\n output_file.write(\"\\tfalse,\\n\")\n\n output_file.write(\"};\\n\\n\")\n\n output_file.write(\"bool GetHasDest(IROps Op) {\\n\")\n output_file.write(\" return IRDest[Op];\\n\")\n output_file.write(\"}\\n\")\n\n output_file.write(\"#undef IROP_GETHASDEST_IMPL\\n\")\n output_file.write(\"#endif\\n\\n\")\n\n# Print out IR argument printing\ndef print_ir_arg_printer():\n output_file.write(\"#ifdef IROP_ARGPRINTER_HELPER\\n\")\n output_file.write(\"switch (IROp->Op) {\\n\")\n for op in IROps:\n if not op.ArgPrinter:\n continue\n\n output_file.write(\"case IROps::OP_{}: {{\\n\".format(op.Name.upper()))\n\n if len(op.Arguments) != 0:\n output_file.write(\"\\t[[maybe_unused]] auto Op = IROp->C();\\n\".format(op.Name))\n output_file.write(\"\\t*out << \\\" \\\";\\n\")\n\n SSAArgNum = 0\n FirstArg = True\n for arg in op.Arguments:\n # No point printing temporaries that we can't recover\n if arg.Temporary:\n continue\n\n if FirstArg:\n FirstArg = False\n else:\n output_file.write('\\t*out << \", \";\\n')\n\n if arg.IsSSA:\n # SSA value\n output_file.write(\"\\tPrintArg(out, IR, Op->Header.Args[{}]);\\n\".format(SSAArgNum))\n SSAArgNum = SSAArgNum + 1\n else:\n # User defined op that is stored\n output_file.write(\"\\tPrintArg(out, IR, Op->{});\\n\".format(arg.Name))\n\n output_file.write(\"break;\\n\")\n output_file.write(\"}\\n\")\n\n output_file.write(\"#undef IROP_ARGPRINTER_HELPER\\n\")\n output_file.write(\"#endif\\n\")\n\ndef print_validation(op):\n if len(op.EmitValidation) != 0:\n output_file.write(\"#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED\\n\")\n\n for Validation in op.EmitValidation:\n Sanitized = Validation.replace(\"\\\"\", \"\\\\\\\"\")\n output_file.write(\"\\t\\tLOGMAN_THROW_A_FMT({}, \\\"{}\\\");\\n\".format(Validation, Sanitized))\n output_file.write(\"#endif\\n\")\n\n# Print out IR allocator helpers\ndef print_ir_allocator_helpers():\n output_file.write(\"#ifdef IROP_ALLOCATE_HELPERS\\n\")\n\n output_file.write(\"\\ttemplate \\n\")\n output_file.write(\"\\tstruct Wrapper final {\\n\")\n output_file.write(\"\\t\\tT *first;\\n\")\n output_file.write(\"\\t\\tOrderedNode *Node; ///< Actual offset of this IR in ths list\\n\")\n output_file.write(\"\\n\")\n output_file.write(\"\\t\\toperator Wrapper() const { return Wrapper {reinterpret_cast(first), Node}; }\\n\")\n output_file.write(\"\\t\\toperator OrderedNode *() { return Node; }\\n\")\n output_file.write(\"\\t\\toperator const OrderedNode *() const { return Node; }\\n\")\n output_file.write(\"\\t\\toperator OpNodeWrapper () const { return Node->Header.Value; }\\n\")\n output_file.write(\"\\t};\\n\")\n\n output_file.write(\"\\ttemplate \\n\")\n output_file.write(\"\\tusing IRPair = Wrapper;\\n\\n\")\n\n output_file.write(\"\\tIRPair AllocateRawOp(size_t HeaderSize) {\\n\")\n output_file.write(\"\\t\\tauto Op = reinterpret_cast(DualListData.DataAllocate(HeaderSize));\\n\")\n output_file.write(\"\\t\\tmemset(Op, 0, HeaderSize);\\n\")\n output_file.write(\"\\t\\tOp->Op = IROps::OP_DUMMY;\\n\")\n output_file.write(\"\\t\\treturn IRPair{Op, CreateNode(Op)};\\n\")\n output_file.write(\"\\t}\\n\\n\")\n\n output_file.write(\"\\ttemplate\\n\")\n output_file.write(\"\\tT *AllocateOrphanOp() {\\n\")\n output_file.write(\"\\t\\tsize_t Size = FEXCore::IR::GetSize(T2);\\n\")\n output_file.write(\"\\t\\tauto Op = reinterpret_cast(DualListData.DataAllocate(Size));\\n\")\n output_file.write(\"\\t\\tmemset(Op, 0, Size);\\n\")\n output_file.write(\"\\t\\tOp->Header.Op = T2;\\n\")\n output_file.write(\"\\t\\treturn Op;\\n\")\n output_file.write(\"\\t}\\n\\n\")\n\n output_file.write(\"\\ttemplate\\n\")\n output_file.write(\"\\tIRPair AllocateOp() {\\n\")\n output_file.write(\"\\t\\tsize_t Size = FEXCore::IR::GetSize(T2);\\n\")\n output_file.write(\"\\t\\tauto Op = reinterpret_cast(DualListData.DataAllocate(Size));\\n\")\n output_file.write(\"\\t\\tmemset(Op, 0, Size);\\n\")\n output_file.write(\"\\t\\tOp->Header.Op = T2;\\n\")\n output_file.write(\"\\t\\treturn IRPair{Op, CreateNode(&Op->Header)};\\n\")\n output_file.write(\"\\t}\\n\\n\")\n\n output_file.write(\"\\tIR::OpSize GetOpSize(const OrderedNode *Op) const {\\n\")\n output_file.write(\"\\t\\tauto HeaderOp = Op->Header.Value.GetNode(DualListData.DataBegin());\\n\")\n output_file.write(\"\\t\\treturn HeaderOp->Size;\\n\")\n output_file.write(\"\\t}\\n\\n\")\n\n output_file.write(\"\\tIR::OpSize GetOpElementSize(const OrderedNode *Op) const {\\n\")\n output_file.write(\"\\t\\tauto HeaderOp = Op->Header.Value.GetNode(DualListData.DataBegin());\\n\")\n output_file.write(\"\\t\\treturn HeaderOp->ElementSize;\\n\")\n output_file.write(\"\\t}\\n\\n\")\n\n output_file.write(\"\\tuint8_t GetOpElements(const OrderedNode *Op) const {\\n\")\n output_file.write(\"\\t\\tLOGMAN_THROW_A_FMT(OpHasDest(Op), \\\"Op {} has no dest\\\\n\\\", GetOpName(Op));\\n\")\n output_file.write(\"\\t\\treturn IR::OpSizeToSize(GetOpSize(Op)) / IR::OpSizeToSize(GetOpElementSize(Op));\\n\")\n output_file.write(\"\\t}\\n\\n\")\n\n output_file.write(\"\\tbool OpHasDest(const OrderedNode *Op) const {\\n\")\n output_file.write(\"\\t\\tauto HeaderOp = Op->Header.Value.GetNode(DualListData.DataBegin());\\n\")\n output_file.write(\"\\t\\treturn GetHasDest(HeaderOp->Op);\\n\")\n output_file.write(\"\\t}\\n\\n\")\n\n output_file.write(\"\\tIROps GetOpType(const OrderedNode *Op) const {\\n\")\n output_file.write(\"\\t\\tauto HeaderOp = Op->Header.Value.GetNode(DualListData.DataBegin());\\n\")\n output_file.write(\"\\t\\treturn HeaderOp->Op;\\n\")\n output_file.write(\"\\t}\\n\\n\")\n\n output_file.write(\"\\tFEXCore::IR::RegClass GetOpRegClass(const OrderedNode *Op) const {\\n\")\n output_file.write(\"\\t\\treturn GetRegClass(GetOpType(Op));\\n\")\n output_file.write(\"\\t}\\n\\n\")\n\n output_file.write(\"\\tstd::string_view const& GetOpName(const OrderedNode *Op) const {\\n\")\n output_file.write(\"\\t\\treturn IR::GetName(GetOpType(Op));\\n\")\n output_file.write(\"\\t}\\n\\n\")\n\n # Generate helpers with operands\n for op in IROps:\n if op.Name != \"Last\":\n output_file.write(\"\\t///\\n\".join([\"\\t/// {}\\n\" .format(comment) for comment in op.Desc]))\n output_file.write(\"\\tIRPair _{}(\" .format(op.Name, op.Name))\n\n # Output SSA args first\n for i, arg in enumerate(op.Arguments):\n LastArg = i == len(op.Arguments) - 1\n\n if arg.Temporary:\n CType = IRTypesToCXX[arg.Type].CXXName\n output_file.write(\"{} {}\".format(CType, arg.Name))\n elif arg.IsSSA:\n # SSA value\n output_file.write(\"OrderedNodeWrapper {}\".format(arg.Name))\n else:\n # User defined op that is stored\n CType = IRTypesToCXX[arg.Type].CXXName\n output_file.write(\"{} {}\".format(CType, arg.Name))\n\n if arg.DefaultInitializer:\n output_file.write(\" = {}\".format(arg.DefaultInitializer))\n\n if not LastArg:\n output_file.write(\", \")\n\n output_file.write(\") {\\n\")\n\n # Save NZCV if needed before clobbering NZCV\n if op.ImplicitFlagClobber:\n output_file.write(\"\\t\\tSaveNZCV(IROps::OP_{});\".format(op.Name.upper()))\n\n # We gather the \"has x87?\" flag as we go. This saves the user from\n # having to keep track of whether they emitted any x87.\n # Also changes the mmx state to X87.\n if op.LoweredX87:\n output_file.write(\"\\t\\tRecordX87Use();\\n\")\n output_file.write(\n \"\\t\\tif(MMXState == MMXState_MMX) ChgStateMMX_X87();\\n\"\n )\n\n output_file.write(\"\\t\\tauto _Op = AllocateOp();\\n\".format(op.Name, op.Name.upper()))\n\n if op.SSAArgNum != 0:\n for arg in op.Arguments:\n if arg.IsSSA:\n output_file.write(\"\\t\\t_Op.first->{} = {};\\n\".format(arg.Name, arg.Name))\n\n if len(op.Arguments) != 0:\n for arg in op.Arguments:\n if not arg.Temporary and not arg.IsSSA:\n output_file.write(\"\\t\\t_Op.first->{} = {};\\n\".format(arg.Name, arg.Name))\n\n assert not (op.HasDest and op.DestSize is None)\n\n # Some ops without a destination still need an operating size\n # Effectively reusing the destination size value for operation size\n if op.DestSize != None:\n output_file.write(\"\\t\\t_Op.first->Header.Size = {};\\n\".format(op.DestSize))\n\n if op.ElementSize == None:\n output_file.write(\"\\t\\t_Op.first->Header.ElementSize = _Op.first->Header.Size;\\n\")\n else:\n output_file.write(\"\\t\\t_Op.first->Header.ElementSize = {};\\n\".format(op.ElementSize))\n\n\n # Only validate here if there's no OrderedNode * version. Else\n # validation is in that version, see the comment below.\n if op.SSAArgNum == 0:\n print_validation(op)\n\n output_file.write(\"\\t\\treturn _Op;\\n\")\n output_file.write(\"\\t}\\n\\n\")\n\n # Now do the OrderedNode * version if necessary\n if op.SSAArgNum:\n output_file.write(\"\\t///\\n\".join([\"\\t/// {}\\n\" .format(comment) for comment in op.Desc]))\n output_file.write(\"\\tIRPair _{}(\" .format(op.Name, op.Name))\n\n for i, arg in enumerate(op.Arguments):\n LastArg = i == len(op.Arguments) - 1\n\n if arg.Temporary:\n CType = IRTypesToCXX[arg.Type].CXXName\n output_file.write(\"{} {}\".format(CType, arg.Name))\n elif arg.IsSSA:\n output_file.write(\"OrderedNode *{}\".format(arg.Name))\n else:\n CType = IRTypesToCXX[arg.Type].CXXName\n output_file.write(\"{} {}\".format(CType, arg.Name))\n\n if arg.DefaultInitializer:\n output_file.write(\" = {}\".format(arg.DefaultInitializer))\n\n if not LastArg:\n output_file.write(\", \")\n\n output_file.write(\") {\\n\")\n output_file.write(\"\\t\\tauto ListDataBegin = DualListData.ListBegin();\\n\")\n\n idx = 0\n for arg in op.Arguments:\n if arg.IsSSA:\n # Inline an immediate if we can\n inline = op.Inline[idx]\n idx += 1\n\n if inline != '':\n Sized = \"Size\" in [x.Name for x in op.Arguments]\n P = [\"Size\" if Sized else \"OpSize::i64Bit\", arg.Name]\n\n # A few cases need extra info plumbed.\n if inline == \"SubtractZero\":\n P += [\"Src2\"]\n elif inline == \"Mem\":\n P += [\"OffsetType\", \"OffsetScale\"]\n elif inline == \"Memtso\":\n P += [\"OffsetType\", \"OffsetScale\", \"true /* TSO */\"]\n inline = \"Mem\"\n\n output_file.write(f\"\\t\\t{arg.Name} = Inline{inline}({', '.join(P)});\\n\")\n\n output_file.write(f\"\\t\\t{arg.Name}->AddUse();\\n\")\n\n # Insert validation here. This is skipped for the\n # OrderedNodeWrapper version because validation can depend on\n # the OrderedNode, but that's ok in practice. Everything pre-RA\n # uses the OrderedNode version, and anything RA-onwards is\n # dubious to validate.\n print_validation(op)\n\n output_file.write(f\"\\t\\treturn _{op.Name}(\")\n for i, arg in enumerate(op.Arguments):\n LastArg = i == len(op.Arguments) - 1\n output_file.write(arg.Name)\n if arg.IsSSA:\n output_file.write(\"->Wrapped(ListDataBegin)\")\n if not LastArg:\n output_file.write(\", \")\n output_file.write(\");\\n\")\n output_file.write(\"\\t}\\n\\n\")\n\n output_file.write(\"#undef IROP_ALLOCATE_HELPERS\\n\")\n output_file.write(\"#endif\\n\")\n\ndef print_ir_dispatcher_defs():\n output_dispatch_file.write(\"#ifdef IROP_DISPATCH_DEFS\\n\")\n for op in IROps:\n if op.Name != \"Last\" and op.SwitchGen and op.JITDispatch and op.JITDispatchOverride == None:\n output_dispatch_file.write(\"DEF_OP({});\\n\".format(op.Name))\n\n output_dispatch_file.write(\"#undef IROP_DISPATCH_DEFS\\n\")\n output_dispatch_file.write(\"#endif\\n\")\n\ndef print_ir_dispatcher_dispatch():\n output_dispatch_file.write(\"#ifdef IROP_DISPATCH_DISPATCH\\n\")\n for op in IROps:\n if op.Name != \"Last\" and op.JITDispatch:\n DispatchName = op.Name\n if op.JITDispatchOverride != None:\n DispatchName = op.JITDispatchOverride\n\n if (op.DynamicDispatch):\n output_dispatch_file.write(\"REGISTER_OP_RT({}, {});\\n\".format(op.Name.upper(), DispatchName))\n else:\n output_dispatch_file.write(\"REGISTER_OP({}, {});\\n\".format(op.Name.upper(), DispatchName))\n\n output_dispatch_file.write(\"#undef IROP_DISPATCH_DISPATCH\\n\")\n output_dispatch_file.write(\"#endif\\n\")\n\n\nif len(sys.argv) < 4:\n ExitError(\"Insufficient parameters passed to script\")\n\noutput_filename = sys.argv[2]\noutput_dispatcher_filename = sys.argv[3]\n\njson_file = open(sys.argv[1], \"r\")\njson_text = json_file.read()\njson_file.close()\n\njson_object = json.loads(json_text)\njson_object = {k.upper(): v for k, v in json_object.items()}\n\nenums = json_object[\"ENUMS\"]\nops = json_object[\"OPS\"]\nirtypes = json_object[\"IRTYPES\"]\ndefines = json_object[\"DEFINES\"]\n\nparse_irtypes(irtypes)\nparse_ops(ops)\n\noutput_file = open(output_filename, \"w\")\n\nprint_enums(enums)\nprint_ir_structs(defines)\nprint_ir_sizes()\nprint_ir_reg_classes()\nprint_ir_getname()\nprint_ir_getraargs()\nprint_ir_hassideeffects()\nprint_ir_gethasdest()\nprint_ir_arg_printer()\nprint_ir_allocator_helpers()\n\noutput_file.close()\n\noutput_dispatch_file = open(output_dispatcher_filename, \"w\")\nprint_ir_dispatcher_defs()\nprint_ir_dispatcher_dispatch()\n\noutput_dispatch_file.close()\n" }, { "path": "FEXCore/Source/CMakeLists.txt", "content": "set(MAN_DIR share/man CACHE PATH \"MAN_DIR\")\n\nset(FEXCORE_BASE_SRCS\n Interface/Config/Config.cpp\n Utils/Allocator.cpp\n Utils/FileLoading.cpp\n Utils/ForcedAssert.cpp\n Utils/LogManager.cpp\n Utils/SpinWaitLock.cpp)\n\nif (NOT MINGW)\n list(APPEND FEXCORE_BASE_SRCS\n Utils/Allocator/64BitAllocator.cpp)\nendif()\n\nset(SRCS\n Common/JitSymbols.cpp\n Interface/Context/Context.cpp\n Interface/Core/LookupCache.cpp\n Interface/Core/CodeCache.cpp\n Interface/Core/Core.cpp\n Interface/Core/CPUBackend.cpp\n Interface/Core/Addressing.cpp\n Interface/Core/CPUID.cpp\n Interface/Core/Frontend.cpp\n Interface/Core/OpcodeDispatcher/AVX_128.cpp\n Interface/Core/OpcodeDispatcher/Crypto.cpp\n Interface/Core/OpcodeDispatcher/Flags.cpp\n Interface/Core/OpcodeDispatcher/Vector.cpp\n Interface/Core/OpcodeDispatcher/X87.cpp\n Interface/Core/OpcodeDispatcher/X87F64.cpp\n Interface/Core/OpcodeDispatcher.cpp\n Interface/Core/ArchHelpers/Arm64Emitter.cpp\n Interface/Core/Dispatcher/Dispatcher.cpp\n Interface/Core/Interpreter/Fallbacks/InterpreterFallbacks.cpp\n Interface/Core/Interpreter/Fallbacks/StringCompareFallbacks.cpp\n Interface/Core/JIT/JIT.cpp\n Interface/Core/JIT/ALUOps.cpp\n Interface/Core/JIT/AtomicOps.cpp\n Interface/Core/JIT/BranchOps.cpp\n Interface/Core/JIT/ConversionOps.cpp\n Interface/Core/JIT/EncryptionOps.cpp\n Interface/Core/JIT/MemoryOps.cpp\n Interface/Core/JIT/MiscOps.cpp\n Interface/Core/JIT/MoveOps.cpp\n Interface/Core/JIT/VectorOps.cpp\n Interface/Core/JIT/Arm64Relocations.cpp\n Interface/Core/X86Tables/BaseTables.cpp\n Interface/Core/X86Tables/DDDTables.cpp\n Interface/Core/X86Tables/H0F38Tables.cpp\n Interface/Core/X86Tables/H0F3ATables.cpp\n Interface/Core/X86Tables/PrimaryGroupTables.cpp\n Interface/Core/X86Tables/SecondaryGroupTables.cpp\n Interface/Core/X86Tables/SecondaryModRMTables.cpp\n Interface/Core/X86Tables/SecondaryTables.cpp\n Interface/Core/X86Tables/VEXTables.cpp\n Interface/Core/X86Tables/X87Tables.cpp\n Interface/GDBJIT/GDBJIT.cpp\n Interface/IR/IRDumper.cpp\n Interface/IR/IREmitter.cpp\n Interface/IR/PassManager.cpp\n Interface/IR/Passes/IRDumperPass.cpp\n Interface/IR/Passes/IRValidation.cpp\n Interface/IR/Passes/RedundantFlagCalculationElimination.cpp\n Interface/IR/Passes/RegisterAllocationPass.cpp\n Interface/IR/Passes/x87StackOptimizationPass.cpp\n Utils/LongJump.cpp\n Utils/Telemetry.cpp\n Utils/Threads.cpp\n Utils/Profiler.cpp)\n\nif (ARCHITECTURE_arm64)\n list(APPEND SRCS Utils/ArchHelpers/Arm64.cpp)\nelse()\n list(APPEND SRCS Utils/ArchHelpers/Arm64_stubs.cpp)\nendif()\n\nif (ENABLE_GLIBC_ALLOCATOR_HOOK_FAULT)\n list(APPEND FEXCORE_BASE_SRCS\n Utils/AllocatorOverride.cpp)\nendif()\n\nset(DEFINES -DJIT_ARM64)\n\nif (ARCHITECTURE_x86_64)\n list(APPEND DEFINES -DARCHITECTURE_x86_64=1)\nendif()\n\nif (ARCHITECTURE_arm64)\n list(APPEND DEFINES -DARCHITECTURE_arm64=1)\nendif()\n\nif (ENABLE_VIXL_DISASSEMBLER)\n list(APPEND DEFINES -DVIXL_DISASSEMBLER=1)\nendif()\n\nif (ENABLE_ZYDIS)\n list(APPEND DEFINES -DZYDIS_DISASSEMBLER=1)\nendif()\n\nif (ARCHITECTURE_arm64 AND HAS_CLANG_PRESERVE_ALL)\n list(APPEND DEFINES \"-DFEXCORE_PRESERVE_ALL_ATTR=__attribute__((preserve_all));-DFEXCORE_HAS_PRESERVE_ALL_ATTR=1\")\nelse()\n list(APPEND DEFINES \"-DFEXCORE_PRESERVE_ALL_ATTR=;-DFEXCORE_HAS_PRESERVE_ALL_ATTR=0\")\nendif()\n\nset(LIBS fmt::fmt xxHash::xxhash FEXHeaderUtils CodeEmitter cephes_128bit)\n\nif (ENABLE_VIXL_DISASSEMBLER OR ENABLE_VIXL_SIMULATOR)\n list(APPEND LIBS vixl::vixl)\nendif()\n\nif (ENABLE_ZYDIS)\n list(APPEND LIBS Zydis::Zydis)\nendif()\n\nif (NOT MINGW)\n list(APPEND LIBS dl)\nelse()\n list(APPEND LIBS synchronization)\n if (ARCHITECTURE_arm64ec)\n list(APPEND LIBS mincore)\n endif()\nendif()\n\nif (CMAKE_CXX_COMPILER_ID STREQUAL \"GNU\")\n # GCC requires libatomic to use 128-bit atomics\n list(APPEND LIBS atomic)\nendif()\n\n# Generate config\nconfigure_file(${CMAKE_CURRENT_SOURCE_DIR}/Interface/Config/Config.json.in\n ${CMAKE_BINARY_DIR}/generated/Config/Config.json)\n\n# Generate IR include file\nset(OUTPUT_IR_FOLDER \"${CMAKE_BINARY_DIR}/include/FEXCore/IR\")\nset(OUTPUT_NAME \"${OUTPUT_IR_FOLDER}/IRDefines.inc\")\nset(OUTPUT_DISPATCHER_NAME \"${OUTPUT_IR_FOLDER}/IRDefines_Dispatch.inc\")\nset(INPUT_NAME \"${CMAKE_CURRENT_SOURCE_DIR}/Interface/IR/IR.json\")\n\nfile(MAKE_DIRECTORY \"${OUTPUT_IR_FOLDER}\")\n\nadd_custom_command(\n OUTPUT \"${OUTPUT_NAME}\" \"${OUTPUT_DISPATCHER_NAME}\"\n DEPENDS \"${INPUT_NAME}\"\n DEPENDS \"${CMAKE_CURRENT_SOURCE_DIR}/../Scripts/json_ir_generator.py\"\n COMMAND \"python3\" \"${CMAKE_CURRENT_SOURCE_DIR}/../Scripts/json_ir_generator.py\"\n \"${INPUT_NAME}\" \"${OUTPUT_NAME}\" \"${OUTPUT_DISPATCHER_NAME}\")\n\nset_source_files_properties(${OUTPUT_NAME} PROPERTIES GENERATED TRUE)\n\n# Generate IR documentation\nset(OUTPUT_IR_DOC \"${CMAKE_BINARY_DIR}/IR.md\")\n\nadd_custom_command(\n OUTPUT \"${OUTPUT_IR_DOC}\"\n DEPENDS \"${INPUT_NAME}\"\n DEPENDS \"${CMAKE_CURRENT_SOURCE_DIR}/../Scripts/json_ir_doc_generator.py\"\n COMMAND \"python3\" \"${CMAKE_CURRENT_SOURCE_DIR}/../Scripts/json_ir_doc_generator.py\"\n \"${INPUT_NAME}\" \"${OUTPUT_IR_DOC}\")\n\nset_source_files_properties(${OUTPUT_IR_NAME} PROPERTIES GENERATED TRUE)\n\n# Create the target\nadd_custom_target(IR_INC\n DEPENDS \"${OUTPUT_NAME}\"\n DEPENDS \"${OUTPUT_IR_DOC}\")\n\n# Generate the configuration include file\nset(OUTPUT_CONFIG_FOLDER \"${CMAKE_BINARY_DIR}/include/FEXCore/Config\")\nset(OUTPUT_CONFIG_NAME \"${OUTPUT_CONFIG_FOLDER}/ConfigValues.inl\")\nset(OUTPUT_CONFIG_OPTION_NAME \"${OUTPUT_CONFIG_FOLDER}/ConfigOptions.inl\")\nset(INPUT_CONFIG_NAME \"${CMAKE_BINARY_DIR}/generated/Config/Config.json\")\nset(OUTPUT_MAN_NAME \"${CMAKE_BINARY_DIR}/generated/FEX.1\")\nset(OUTPUT_MAN_NAME_COMPRESS \"${CMAKE_BINARY_DIR}/generated/FEX.1.gz\")\n\nfile(MAKE_DIRECTORY \"${OUTPUT_CONFIG_FOLDER}\")\n\nadd_custom_command(\n OUTPUT \"${OUTPUT_CONFIG_NAME}\"\n OUTPUT \"${OUTPUT_CONFIG_OPTION_NAME}\"\n OUTPUT \"${OUTPUT_MAN_NAME}\"\n DEPENDS \"${INPUT_CONFIG_NAME}\"\n DEPENDS \"${CMAKE_CURRENT_SOURCE_DIR}/../Scripts/config_generator.py\"\n COMMAND \"python3\" \"${CMAKE_CURRENT_SOURCE_DIR}/../Scripts/config_generator.py\" \"${INPUT_CONFIG_NAME}\" \"${OUTPUT_CONFIG_NAME}\" \"${OUTPUT_MAN_NAME}\"\n \"${OUTPUT_CONFIG_OPTION_NAME}\")\n\nadd_custom_command(\n OUTPUT \"${OUTPUT_MAN_NAME_COMPRESS}\"\n DEPENDS \"${OUTPUT_MAN_NAME}\"\n COMMAND \"gzip\" \"-kf9n\" \"${OUTPUT_MAN_NAME}\")\n\nset_source_files_properties(${OUTPUT_CONFIG_NAME} PROPERTIES\n GENERATED TRUE)\nset_source_files_properties(${OUTPUT_CONFIG_OPTION_NAME} PROPERTIES\n GENERATED TRUE)\n\nset_source_files_properties(${OUTPUT_MAN_NAME} PROPERTIES\n GENERATED TRUE)\nset_source_files_properties(${OUTPUT_MAN_NAME_COMPRESS} PROPERTIES\n GENERATED TRUE)\n\n# Create the target\nadd_custom_target(CONFIG_INC\n DEPENDS \"${OUTPUT_CONFIG_NAME}\"\n DEPENDS \"${OUTPUT_CONFIG_OPTION_NAME}\"\n DEPENDS \"${OUTPUT_MAN_NAME}\"\n DEPENDS \"${OUTPUT_MAN_NAME_COMPRESS}\")\n\nif (NOT BUILD_STEAM_SUPPORT)\n # Install the compressed man page\n install(FILES ${OUTPUT_MAN_NAME_COMPRESS} COMPONENT Runtime DESTINATION ${MAN_DIR}/man1)\nendif()\n\n# Add in diagnostic colours if the option is available.\n# Ninja code generator will kill colours if this isn't here\ncheck_cxx_compiler_flag(-fdiagnostics-color=always GCC_COLOR)\ncheck_cxx_compiler_flag(-fcolor-diagnostics CLANG_COLOR)\n\nfunction(AddDefaultOptionsToTarget Name)\n set_target_properties(${Name} PROPERTIES C_VISIBILITY_PRESET hidden)\n set_target_properties(${Name} PROPERTIES CXX_VISIBILITY_PRESET hidden)\n set_target_properties(${Name} PROPERTIES VISIBILITY_INLINES_HIDDEN TRUE)\n target_include_directories(${Name} PUBLIC \"${CMAKE_CURRENT_BINARY_DIR}\")\n\n target_include_directories(${Name} PRIVATE IncludePrivate/)\n target_include_directories(${Name} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/)\n\n target_include_directories(${Name} PUBLIC \"${PROJECT_SOURCE_DIR}/include/\")\n target_include_directories(${Name} PUBLIC \"${CMAKE_BINARY_DIR}/include/\")\n\n target_compile_definitions(${Name} PRIVATE ${DEFINES})\n add_dependencies(${Name} CONFIG_INC IR_INC)\n\n target_compile_options(${Name} PRIVATE\n -Wall\n -Werror=cast-qual\n -Werror=ignored-qualifiers\n -Werror=implicit-fallthrough\n\n -Wno-trigraphs\n -ffunction-sections\n -fwrapv)\n\n if (GCC_COLOR)\n target_compile_options(${Name} PRIVATE \"-fdiagnostics-color=always\")\n endif()\n\n if (CLANG_COLOR)\n target_compile_options(${Name} PRIVATE \"-fcolor-diagnostics\")\n endif()\n\n LinkerGC(${Name})\n target_link_libraries(${Name} PUBLIC unordered_dense::unordered_dense)\nendfunction()\n\n# Build FEXCore_Base static library\nadd_library(FEXCore_Base STATIC ${FEXCORE_BASE_SRCS})\ntarget_link_libraries(FEXCore_Base PUBLIC ${LIBS})\nAddDefaultOptionsToTarget(FEXCore_Base)\n\nif (ENABLE_FEXCORE_PROFILER AND FEXCORE_PROFILER_BACKEND STREQUAL \"TRACY\")\n target_link_libraries(FEXCore_Base PUBLIC TracyClient)\nendif()\n\nfunction(AddObject Name)\n add_library(${Name} OBJECT ${SRCS})\n\n target_link_libraries(${Name} PRIVATE FEXCore_Base)\n target_compile_options(${Name} PRIVATE ${FEX_TUNE_COMPILE_FLAGS})\n AddDefaultOptionsToTarget(${Name})\nendfunction()\n\nfunction(AddLibrary Name Type)\n add_library(${Name} ${Type} $)\n set_target_properties(${Name} PROPERTIES OUTPUT_NAME FEXCore)\n\n # During generation of the import library (dll.a), MinGW needs some extra symbols from libraries\n # such as fmt, which are propagated by FEXCore_Base. Wonderful.\n if (MINGW)\n target_link_libraries(${Name} PRIVATE FEXCore_Base)\n endif()\n AddDefaultOptionsToTarget(${Name})\nendfunction()\n\nAddObject(${PROJECT_NAME}_object)\nAddLibrary(${PROJECT_NAME} STATIC)\nAddLibrary(${PROJECT_NAME}_shared SHARED)\n\nif (NOT MINGW AND NOT BUILD_STEAM_SUPPORT)\n install(TARGETS ${PROJECT_NAME}_shared LIBRARY\n DESTINATION ${CMAKE_INSTALL_LIBDIR}\n COMPONENT Libraries)\nendif()\n\n# Meta-library to link jemalloc libraries enabled in the build configuration.\n# Only needed for targets that run emulation. For others, use JemallocDummy.\nadd_library(JemallocLibs STATIC Utils/AllocatorHooks.cpp)\nif (ENABLE_FEX_ALLOCATOR)\n target_compile_definitions(JemallocLibs PRIVATE ENABLE_FEX_ALLOCATOR=1)\n target_link_libraries(JemallocLibs PUBLIC rpmalloc)\nendif()\nif (ENABLE_JEMALLOC_GLIBC_ALLOC)\n set_source_files_properties(Interface/HLE/Thunks/Thunks.cpp PROPERTIES COMPILE_DEFINITIONS ENABLE_JEMALLOC_GLIBC=1)\n target_link_libraries(JemallocLibs INTERFACE FEX_jemalloc_glibc)\nendif()\n\nif (NOT MINGW)\n # Dummy project to use for host tools.\n # This overrides use of jemalloc in FEXCore with the normal glibc allocator.\n add_library(JemallocDummy STATIC Utils/AllocatorHooks.cpp)\n target_include_directories(JemallocDummy PRIVATE \"${PROJECT_SOURCE_DIR}/include/\")\nendif()\n\n# The shared library should always link enabled jemalloc libraries\ntarget_link_libraries(${PROJECT_NAME}_shared PRIVATE JemallocLibs)\n" }, { "path": "FEXCore/Source/Common/BitSet.h", "content": "// SPDX-License-Identifier: MIT\n#pragma once\n\n#include \n#include \n#include \n\n#include \n#include \n#include \n#include \n\nnamespace FEXCore {\n\ntemplate\nstruct BitSet final {\n using ElementType = T;\n constexpr static size_t MinimumSize = sizeof(ElementType);\n constexpr static size_t MinimumSizeBits = sizeof(ElementType) * 8;\n\n ElementType* Memory;\n void Allocate(size_t Elements) {\n size_t AllocateSize = ToBytes(Elements);\n LOGMAN_THROW_A_FMT((AllocateSize * MinimumSize) >= Elements, \"Fail\");\n Memory = static_cast(FEXCore::Allocator::malloc(AllocateSize));\n }\n void Realloc(size_t Elements) {\n size_t AllocateSize = ToBytes(Elements);\n LOGMAN_THROW_A_FMT((AllocateSize * MinimumSize) >= Elements, \"Fail\");\n Memory = static_cast(FEXCore::Allocator::realloc(Memory, AllocateSize));\n }\n void Free() {\n FEXCore::Allocator::free(Memory);\n Memory = nullptr;\n }\n bool Get(T Element) {\n return (Memory[Element / MinimumSizeBits] & (1ULL << (Element % MinimumSizeBits))) != 0;\n }\n void Set(T Element) {\n Memory[Element / MinimumSizeBits] |= (1ULL << (Element % MinimumSizeBits));\n }\n void Clear(T Element) {\n Memory[Element / MinimumSizeBits] &= (1ULL << (Element % MinimumSizeBits));\n }\n void MemClear(size_t Elements) {\n memset(Memory, 0, ToBytes(Elements));\n }\n void MemSet(size_t Elements) {\n memset(Memory, 0xFF, ToBytes(Elements));\n }\n uint32_t ToBytes(size_t Elements) {\n return AlignUp(Elements, MinimumSizeBits) / MinimumSize;\n }\n\n // This very explicitly doesn't let you take an address\n // Is only a getter\n bool operator[](T Element) {\n return Get(Element);\n }\n};\n\ntemplate\nstruct BitSetView final {\n using ElementType = T;\n constexpr static size_t MinimumSize = sizeof(ElementType);\n constexpr static size_t MinimumSizeBits = sizeof(ElementType) * 8;\n\n ElementType* Memory;\n\n void GetView(BitSet& Set, uint64_t ElementOffset) {\n LOGMAN_THROW_A_FMT((ElementOffset % MinimumSize) == 0, \"Bitset view offset needs to be aligned to size of backing element\");\n Memory = &Set.Memory[ElementOffset / MinimumSizeBits];\n }\n\n bool Get(T Element) {\n return (Memory[Element / MinimumSizeBits] & (1ULL << (Element % MinimumSizeBits))) != 0;\n }\n void Set(T Element) {\n Memory[Element / MinimumSizeBits] |= (1ULL << (Element % MinimumSizeBits));\n }\n void Clear(T Element) {\n Memory[Element / MinimumSizeBits] &= (1ULL << (Element % MinimumSizeBits));\n }\n void MemClear(size_t Elements) {\n memset(Memory, 0, AlignUp(Elements / MinimumSizeBits, MinimumSizeBits));\n }\n void MemSet(size_t Elements) {\n memset(Memory, 0xFF, AlignUp(Elements / MinimumSizeBits, MinimumSizeBits));\n }\n\n // This very explicitly doesn't let you take an address\n // Is only a getter\n bool operator[](T Element) {\n return Get(Element);\n }\n};\n\nstatic_assert(sizeof(BitSet) == sizeof(uintptr_t), \"Needs to just be a pointer\");\nstatic_assert(std::is_trivially_copyable_v>, \"Needs to trivially copyable\");\n\nstatic_assert(sizeof(BitSetView) == sizeof(uintptr_t), \"Needs to just be a pointer\");\nstatic_assert(std::is_trivially_copyable_v>, \"Needs to trivially copyable\");\n\n} // namespace FEXCore\n" }, { "path": "FEXCore/Source/Common/JitSymbols.cpp", "content": "// SPDX-License-Identifier: MIT\n#include \n\n#include \"Common/JitSymbols.h\"\n\n#include \n#include \n\nnamespace FEXCore {\nJITSymbols::JITSymbols() {}\n\nJITSymbols::~JITSymbols() {\n if (fd != -1) {\n close(fd);\n }\n}\n\nvoid JITSymbols::InitFile() {\n // We can't use FILE here since we must be robust against forking processes closing our FD from under us.\n#ifdef __ANDROID__\n // Android simpleperf looks in /data/local/tmp instead of /tmp\n const auto PerfMap = fextl::fmt::format(\"/data/local/tmp/perf-{}.map\", getpid());\n#else\n const auto PerfMap = fextl::fmt::format(\"/tmp/perf-{}.map\", getpid());\n#endif\n fd = open(PerfMap.c_str(), O_CREAT | O_TRUNC | O_WRONLY | O_APPEND, 0644);\n}\n\nvoid JITSymbols::RegisterNamedRegion(const void* HostAddr, uint32_t CodeSize, std::string_view Name) {\n if (fd == -1) {\n return;\n }\n\n // Linux perf format is very straightforward\n // ` \\n`\n const auto Buffer = fextl::fmt::format(\"{} {:x} {}\\n\", HostAddr, CodeSize, Name);\n auto Result = write(fd, Buffer.c_str(), Buffer.size());\n if (Result == -1 && errno == EBADF) {\n fd = -1;\n }\n}\n\nvoid JITSymbols::RegisterJITSpace(const void* HostAddr, uint32_t CodeSize) {\n if (fd == -1) {\n return;\n }\n\n // Linux perf format is very straightforward\n // ` \\n`\n const auto Buffer = fextl::fmt::format(\"{} {:x} FEXJIT\\n\", HostAddr, CodeSize);\n auto Result = write(fd, Buffer.c_str(), Buffer.size());\n if (Result == -1 && errno == EBADF) {\n fd = -1;\n }\n}\n\n// Buffered JIT symbols.\nvoid JITSymbols::Register(FEXCore::JITSymbolBuffer* Buffer, const void* HostAddr, uint64_t GuestAddr, uint32_t CodeSize) {\n if (fd == -1) {\n return;\n }\n\n // Calculate remaining sizes.\n const auto RemainingSize = Buffer->BUFFER_SIZE - Buffer->Offset;\n const auto CurrentBufferOffset = &Buffer->Buffer[Buffer->Offset];\n\n // Linux perf format is very straightforward\n // ` \\n`\n const auto FMTResult = fmt::format_to_n(CurrentBufferOffset, RemainingSize, \"{} {:x} JIT_0x{:x}_{}\\n\", HostAddr, CodeSize, GuestAddr, HostAddr);\n if (FMTResult.out >= &Buffer->Buffer[Buffer->BUFFER_SIZE]) {\n // Couldn't fit, need to force a write.\n WriteBuffer(Buffer, true);\n // Rerun\n Register(Buffer, HostAddr, GuestAddr, CodeSize);\n return;\n }\n\n Buffer->Offset += FMTResult.size;\n WriteBuffer(Buffer);\n}\n\nvoid JITSymbols::Register(FEXCore::JITSymbolBuffer* Buffer, const void* HostAddr, uint32_t CodeSize, std::string_view Name, uintptr_t Offset) {\n if (fd == -1) {\n return;\n }\n\n // Calculate remaining sizes.\n const auto RemainingSize = Buffer->BUFFER_SIZE - Buffer->Offset;\n const auto CurrentBufferOffset = &Buffer->Buffer[Buffer->Offset];\n\n // Linux perf format is very straightforward\n // ` \\n`\n const auto FMTResult =\n fmt::format_to_n(CurrentBufferOffset, RemainingSize, \"{} {:x} {}+0x{:x} ({})\\n\", HostAddr, CodeSize, Name, Offset, HostAddr);\n if (FMTResult.out >= &Buffer->Buffer[Buffer->BUFFER_SIZE]) {\n // Couldn't fit, need to force a write.\n WriteBuffer(Buffer, true);\n // Rerun\n Register(Buffer, HostAddr, CodeSize, Name, Offset);\n return;\n }\n\n Buffer->Offset += FMTResult.size;\n WriteBuffer(Buffer);\n}\n\nvoid JITSymbols::RegisterNamedRegion(FEXCore::JITSymbolBuffer* Buffer, const void* HostAddr, uint32_t CodeSize, std::string_view Name) {\n if (fd == -1) {\n return;\n }\n\n // Calculate remaining sizes.\n const auto RemainingSize = Buffer->BUFFER_SIZE - Buffer->Offset;\n const auto CurrentBufferOffset = &Buffer->Buffer[Buffer->Offset];\n\n // Linux perf format is very straightforward\n // ` \\n`\n const auto FMTResult = fmt::format_to_n(CurrentBufferOffset, RemainingSize, \"{} {:x} {}\\n\", HostAddr, CodeSize, Name);\n if (FMTResult.out >= &Buffer->Buffer[Buffer->BUFFER_SIZE]) {\n // Couldn't fit, need to force a write.\n WriteBuffer(Buffer, true);\n // Rerun\n RegisterNamedRegion(Buffer, HostAddr, CodeSize, Name);\n return;\n }\n\n Buffer->Offset += FMTResult.size;\n WriteBuffer(Buffer);\n}\n\nvoid JITSymbols::WriteBuffer(FEXCore::JITSymbolBuffer* Buffer, bool ForceWrite) {\n auto Now = std::chrono::steady_clock::now();\n if (!ForceWrite) {\n if (((Buffer->LastWrite - Now) < Buffer->MAXIMUM_THRESHOLD) && Buffer->Offset < Buffer->NEEDS_WRITE_DISTANCE) {\n // Still buffering, no need to write.\n return;\n }\n }\n\n Buffer->LastWrite = Now;\n auto Result = write(fd, Buffer->Buffer, Buffer->Offset);\n if (Result == -1 && errno == EBADF) {\n fd = -1;\n }\n\n Buffer->Offset = 0;\n}\n} // namespace FEXCore\n" }, { "path": "FEXCore/Source/Common/JitSymbols.h", "content": "// SPDX-License-Identifier: MIT\n#pragma once\n#include \n\n#include \n\n#include \n#include \n#include \n#include \n\nnamespace FEXCore {\n// Buffered JIT symbol tracking.\nstruct JITSymbolBuffer {\n // Maximum buffer size to ensure we are a page in size.\n constexpr static size_t BUFFER_SIZE = FEXCore::Utils::FEX_PAGE_SIZE - (8 * 2);\n // Maximum distance until the end of the buffer to do a write.\n constexpr static size_t NEEDS_WRITE_DISTANCE = BUFFER_SIZE - 64;\n // Maximum time threshhold to wait before a buffer write occurs.\n constexpr static std::chrono::milliseconds MAXIMUM_THRESHOLD {100};\n\n JITSymbolBuffer()\n : LastWrite {std::chrono::steady_clock::now()} {}\n // stead_clock to ensure a monotonic increasing clock.\n // In highly stressed situations this can still cause >2% CPU time in vdso_clock_gettime.\n // If we need lower CPU time when JIT symbols are enabled then FEX can read the cycle counter directly.\n std::chrono::steady_clock::time_point LastWrite {};\n size_t Offset {};\n char Buffer[BUFFER_SIZE] {};\n};\nstatic_assert(sizeof(JITSymbolBuffer) == FEXCore::Utils::FEX_PAGE_SIZE, \"Ensure this is one page in size\");\n\nclass JITSymbols final {\npublic:\n JITSymbols();\n ~JITSymbols();\n\n void InitFile();\n void RegisterNamedRegion(const void* HostAddr, uint32_t CodeSize, std::string_view Name);\n void RegisterJITSpace(const void* HostAddr, uint32_t CodeSize);\n\n // Allocate JIT buffer.\n static fextl::unique_ptr AllocateBuffer() {\n return fextl::make_unique();\n }\n\n void Register(FEXCore::JITSymbolBuffer* Buffer, const void* HostAddr, uint64_t GuestAddr, uint32_t CodeSize);\n void Register(FEXCore::JITSymbolBuffer* Buffer, const void* HostAddr, uint32_t CodeSize, std::string_view Name, uintptr_t Offset);\n void RegisterNamedRegion(FEXCore::JITSymbolBuffer* Buffer, const void* HostAddr, uint32_t CodeSize, std::string_view Name);\n\nprivate:\n int fd {-1};\n void WriteBuffer(FEXCore::JITSymbolBuffer* Buffer, bool ForceWrite = false);\n};\n} // namespace FEXCore\n" }, { "path": "FEXCore/Source/Common/SoftFloat.h", "content": "// SPDX-License-Identifier: MIT\n#pragma once\n\n#include \n#include \n#include \n#include \"cephes_128bit.h\"\n\n#include \n#include \n#include \n#include \n\n#include \"Common/VectorRegType.h\"\n\nextern \"C\" {\n#include \"SoftFloat-3e/platform.h\"\n#include \"SoftFloat-3e/softfloat.h\"\n}\n\nstruct FEX_PACKED X80SoftFloat {\n#ifdef ARCHITECTURE_x86_64\n// Define this to push some operations to x87\n// Only useful to see if precision loss is killing something\n// #define DEBUG_X86_FLOAT\n#ifdef DEBUG_X86_FLOAT\n#define BIGFLOAT long double\n#define BIGFLOATSIZE 10\n#else\n#define BIGFLOAT float128_t\n#define BIGFLOATSIZE 16\n#endif\n#elif defined(ARCHITECTURE_arm64)\n#define BIGFLOAT float128_t\n#define BIGFLOATSIZE 16\n#else\n#error No 128bit float for this target!\n#endif\n\n uint64_t Significand;\n union {\n uint16_t Raw;\n struct {\n uint16_t Exponent : 15;\n uint16_t Sign : 1;\n };\n } Top;\n\n X80SoftFloat() {\n memset(this, 0, sizeof(*this));\n }\n X80SoftFloat(uint16_t _Sign, uint16_t _Exponent, uint64_t _Significand)\n : Significand {_Significand}\n , Top {.Raw = static_cast((_Exponent & 0x7FFF) | (_Sign << 15))} {}\n\n fextl::string str() const {\n fextl::ostringstream string;\n string << std::hex << Top.Sign;\n string << \"_\" << Top.Exponent;\n string << \"_\" << (Significand >> 63);\n string << \"_\" << (Significand & ((1ULL << 63) - 1));\n return string.str();\n }\n\n // Ops\n FEXCORE_PRESERVE_ALL_ATTR static X80SoftFloat FADD(softfloat_state* state, const X80SoftFloat& lhs, const X80SoftFloat& rhs) {\n#ifdef DEBUG_X86_FLOAT\n BIGFLOAT Result;\n asm(R\"(\n fninit;\n fldt %[rhs]; # st1\n fldt %[lhs]; # st0\n faddp;\n fstpt %[result];\n )\"\n : [result] \"=m\"(Result)\n : [lhs] \"m\"(lhs), [rhs] \"m\"(rhs)\n : \"st\", \"st(1)\");\n\n return Result;\n#else\n return extF80_add(state, lhs, rhs);\n#endif\n }\n\n FEXCORE_PRESERVE_ALL_ATTR static X80SoftFloat FSUB(softfloat_state* state, const X80SoftFloat& lhs, const X80SoftFloat& rhs) {\n#ifdef DEBUG_X86_FLOAT\n BIGFLOAT Result;\n asm(R\"(\n fninit;\n fldt %[rhs]; # st1\n fldt %[lhs]; # st0\n fsubp;\n fstpt %[result];\n )\"\n : [result] \"=m\"(Result)\n : [lhs] \"m\"(lhs), [rhs] \"m\"(rhs)\n : \"st\", \"st(1)\");\n\n return Result;\n#else\n return extF80_sub(state, lhs, rhs);\n#endif\n }\n\n FEXCORE_PRESERVE_ALL_ATTR static X80SoftFloat FMUL(softfloat_state* state, const X80SoftFloat& lhs, const X80SoftFloat& rhs) {\n#ifdef DEBUG_X86_FLOAT\n BIGFLOAT Result;\n asm(R\"(\n fninit;\n fldt %[rhs]; # st1\n fldt %[lhs]; # st0\n fmulp;\n fstpt %[result];\n )\"\n : [result] \"=m\"(Result)\n : [lhs] \"m\"(lhs), [rhs] \"m\"(rhs)\n : \"st\", \"st(1)\");\n\n return Result;\n#else\n return extF80_mul(state, lhs, rhs);\n#endif\n }\n\n FEXCORE_PRESERVE_ALL_ATTR static X80SoftFloat FDIV(softfloat_state* state, const X80SoftFloat& lhs, const X80SoftFloat& rhs) {\n#ifdef DEBUG_X86_FLOAT\n BIGFLOAT Result;\n asm(R\"(\n fninit;\n fldt %[rhs]; # st1\n fldt %[lhs]; # st0\n fdivp;\n fstpt %[result];\n )\"\n : [result] \"=m\"(Result)\n : [lhs] \"m\"(lhs), [rhs] \"m\"(rhs)\n : \"st\", \"st(1)\");\n\n return Result;\n#else\n return extF80_div(state, lhs, rhs);\n#endif\n }\n\n FEXCORE_PRESERVE_ALL_ATTR static X80SoftFloat FREM(softfloat_state* state, const X80SoftFloat& lhs, const X80SoftFloat& rhs) {\n#if defined(DEBUG_X86_FLOAT)\n BIGFLOAT Result;\n asm(R\"(\n fninit;\n fldt %[rhs]; # st1\n fldt %[lhs]; # st0\n fprem;\n fstpt %[result];\n ffreep %%st(0);\n )\"\n : [result] \"=m\"(Result)\n : [lhs] \"m\"(lhs), [rhs] \"m\"(rhs)\n : \"st\", \"st(1)\");\n\n return Result;\n#else\n /*\n * Check for invalid operation cases first - Intel FPREM sets Invalid Operation\n * for several cases including infinity dividend and zero divisor.\n */\n X80SoftFloat result = 0;\n if (HandleInfinityOp(state, lhs, result)) {\n return result;\n } else if (lhs.Top.Exponent == 0x7FFF && (lhs.Significand & 0x7FFFFFFFFFFFFFFFULL)) { // NaN\n // propagate NaN\n state->exceptionFlags |= softfloat_flag_invalid;\n return lhs;\n }\n\n // Check for zero divisor - fprem(x, 0) is invalid operation\n if (rhs.Top.Exponent == 0 && rhs.Significand == 0) {\n state->exceptionFlags |= softfloat_flag_invalid;\n // Return QNaN\n result.Top.Sign = 0;\n result.Top.Exponent = 0x7FFF;\n result.Significand = 0xC000000000000000ULL;\n return result;\n }\n\n /*\n * FPREM is not an IEEE-754 remainder. From the Intel spec:\n *\n * Computes the remainder obtained from dividing the value in the ST(0)\n * register (the dividend) by the value in the ST(1) register (the divisor\n * or modulus), and stores the result in ST(0). The remainder represents the\n * following value:\n *\n * Remainder := ST(0) − (Q * ST(1))\n *\n * Here, Q is an integer value that is obtained by truncating the\n * floating-point number quotient of [ST(0) / ST(1)] toward zero.\n *\n * We implement this sequence literally. softfloat_round_minMag means\n * \"truncate towards zero\".\n */\n extFloat80_t quotient = extF80_div(state, lhs, rhs);\n extFloat80_t Q = extF80_roundToInt(state, quotient, softfloat_round_minMag, true);\n bool Q_zero = Q.signif == 0 && (Q.signExp & ~(1 << 15)) == 0;\n\n if (Q_zero) {\n return lhs;\n } else {\n return extF80_sub(state, lhs, extF80_mul(state, Q, rhs));\n }\n#endif\n }\n\n FEXCORE_PRESERVE_ALL_ATTR static X80SoftFloat FREM1(softfloat_state* state, const X80SoftFloat& lhs, const X80SoftFloat& rhs) {\n#if defined(DEBUG_X86_FLOAT)\n BIGFLOAT Result;\n asm(R\"(\n fninit;\n fldt %[rhs]; # st1\n fldt %[lhs]; # st0\n fprem1;\n fstpt %[result];\n ffreep %%st(0);\n )\"\n : [result] \"=m\"(Result)\n : [lhs] \"m\"(lhs), [rhs] \"m\"(rhs)\n : \"st\", \"st(1)\");\n\n return Result;\n#else\n return extF80_rem(state, lhs, rhs);\n#endif\n }\n\n FEXCORE_PRESERVE_ALL_ATTR static X80SoftFloat FRNDINT(softfloat_state* state, const X80SoftFloat& lhs) {\n return extF80_roundToInt(state, lhs, state->roundingMode, false);\n }\n\n FEXCORE_PRESERVE_ALL_ATTR static X80SoftFloat FRNDINT(softfloat_state* state, const X80SoftFloat& lhs, uint_fast8_t RoundMode) {\n return extF80_roundToInt(state, lhs, RoundMode, false);\n }\n\n FEXCORE_PRESERVE_ALL_ATTR static X80SoftFloat FXTRACT_SIG(const X80SoftFloat& lhs) {\n#if defined(DEBUG_X86_FLOAT)\n BIGFLOAT Result;\n asm(R\"(\n fninit;\n fldt %[lhs]; # st0\n fxtract;\n fstpt %[result];\n ffreep %%st(0);\n )\"\n : [result] \"=m\"(Result)\n : [lhs] \"m\"(lhs)\n : \"st\", \"st(1)\");\n\n return Result;\n#else\n // Zero is a special case, the significand for +/- 0 is +/- zero.\n if (lhs.Top.Exponent == 0x0 && lhs.Significand == 0x0) {\n return lhs;\n }\n X80SoftFloat Tmp = lhs;\n Tmp.Top.Exponent = 0x3FFF;\n Tmp.Top.Sign = lhs.Top.Sign;\n return Tmp;\n#endif\n }\n\n FEXCORE_PRESERVE_ALL_ATTR static X80SoftFloat FXTRACT_EXP(const X80SoftFloat& lhs) {\n#if defined(DEBUG_X86_FLOAT)\n BIGFLOAT Result;\n asm(R\"(\n fninit;\n fldt %[lhs]; # st0\n fxtract;\n ffreep %%st(0);\n fstpt %[result];\n )\"\n : [result] \"=m\"(Result)\n : [lhs] \"m\"(lhs)\n : \"st\", \"st(1)\");\n\n return Result;\n#else\n // Zero is a special case, the exponent is always -inf\n if (lhs.Top.Exponent == 0x0 && lhs.Significand == 0x0) {\n X80SoftFloat Result(1, 0x7FFFUL, 0x8000'0000'0000'0000UL);\n return Result;\n }\n\n int32_t TrueExp = lhs.Top.Exponent - ExponentBias;\n return i32_to_extF80(TrueExp);\n#endif\n }\n\n FEXCORE_PRESERVE_ALL_ATTR static void\n FCMP(softfloat_state* state, const X80SoftFloat& lhs, const X80SoftFloat& rhs, bool* eq, bool* lt, bool* nan) {\n *eq = extF80_eq(state, lhs, rhs);\n *lt = extF80_lt(state, lhs, rhs);\n\n // Use IEEE 754 semantics: unordered if neither <, =, nor > is true\n // This is more reliable than custom NaN detection\n bool gt = !(*eq) && !(*lt) && extF80_le(state, rhs, lhs);\n *nan = !(*eq) && !(*lt) && !gt;\n }\n\n FEXCORE_PRESERVE_ALL_ATTR static X80SoftFloat FSCALE(softfloat_state* state, const X80SoftFloat& lhs, const X80SoftFloat& rhs) {\n#ifdef DEBUG_X86_FLOAT\n BIGFLOAT Result;\n asm(R\"(\n fninit;\n fldt %[rhs]; # st1\n fldt %[lhs]; # st0\n fscale; # st0 = st0 * 2^(rdint(st1))\n fstpt %[result];\n ffreep %%st(0);\n )\"\n : [result] \"=m\"(Result)\n : [lhs] \"m\"(lhs), [rhs] \"m\"(rhs)\n : \"st\", \"st(1)\");\n\n return Result;\n#else\n extFloat80_t Zero {0, 0};\n if (extF80_eq(state, lhs, Zero)) {\n return lhs;\n }\n X80SoftFloat Int = FRNDINT(state, rhs, softfloat_round_minMag);\n BIGFLOAT Src2_d = Int.ToFMax(state);\n Src2_d = FEXCore::cephes_128bit::exp2l(Src2_d);\n X80SoftFloat Src2_X80(state, Src2_d);\n X80SoftFloat Result = extF80_mul(state, lhs, Src2_X80);\n return Result;\n#endif\n }\n\n FEXCORE_PRESERVE_ALL_ATTR static X80SoftFloat F2XM1(softfloat_state* state, const X80SoftFloat& lhs) {\n#ifdef DEBUG_X86_FLOAT\n BIGFLOAT Result;\n asm(R\"(\n fninit;\n fldt %[lhs]; # st0\n f2xm1; # st0 = 2^st(0) - 1\n fstpt %[result];\n )\"\n : [result] \"=m\"(Result)\n : [lhs] \"m\"(lhs)\n : \"st\");\n\n return Result;\n#else\n auto Src1_d = lhs.ToFMax(state);\n auto Result = FEXCore::cephes_128bit::exp2l(Src1_d);\n\n static const float128_t one {0x0ULL, 0x3fff000000000000ULL};\n return X80SoftFloat(state, f128_sub(state, Result, one));\n#endif\n }\n\n FEXCORE_PRESERVE_ALL_ATTR static X80SoftFloat FYL2X(softfloat_state* state, const X80SoftFloat& lhs, const X80SoftFloat& rhs) {\n#ifdef DEBUG_X86_FLOAT\n BIGFLOAT Result;\n asm(R\"(\n fninit;\n fldt %[rhs]; # st(1)\n fldt %[lhs]; # st(0)\n fyl2x; # st(1) * log2l(st(0))\n fstpt %[result];\n )\"\n : [result] \"=m\"(Result)\n : [lhs] \"m\"(lhs), [rhs] \"m\"(rhs)\n : \"st\", \"st(1)\");\n\n return Result;\n#else\n auto Src1_d = lhs.ToFMax(state);\n auto Src2_d = rhs.ToFMax(state);\n\n auto Tmp = f128_mul(state, Src2_d, FEXCore::cephes_128bit::log2l(Src1_d));\n return X80SoftFloat(state, Tmp);\n#endif\n }\n\n FEXCORE_PRESERVE_ALL_ATTR static X80SoftFloat FATAN(softfloat_state* state, const X80SoftFloat& lhs, const X80SoftFloat& rhs) {\n#ifdef DEBUG_X86_FLOAT\n BIGFLOAT Result;\n asm(R\"(\n fninit;\n fldt %[lhs];\n fldt %[rhs];\n fpatan;\n fstpt %[result];\n )\"\n : [result] \"=m\"(Result)\n : [lhs] \"m\"(lhs), [rhs] \"m\"(rhs)\n : \"st\", \"st(1)\");\n\n return Result;\n#else\n BIGFLOAT Src1_d = lhs.ToFMax(state);\n BIGFLOAT Src2_d = rhs.ToFMax(state);\n BIGFLOAT Tmp = FEXCore::cephes_128bit::atan2l(Src1_d, Src2_d);\n return X80SoftFloat(state, Tmp);\n#endif\n }\n\n FEXCORE_PRESERVE_ALL_ATTR static X80SoftFloat FTAN(softfloat_state* state, const X80SoftFloat& lhs) {\n#ifdef DEBUG_X86_FLOAT\n BIGFLOAT Result;\n asm(R\"(\n fninit;\n fldt %[lhs]; # st0\n fptan;\n ffreep %%st(0);\n fstpt %[result];\n )\"\n : [result] \"=m\"(Result)\n : [lhs] \"m\"(lhs)\n : \"st\");\n\n return Result;\n#else\n X80SoftFloat result;\n if (HandleInfinityOp(state, lhs, result)) {\n return result;\n }\n\n BIGFLOAT Src_d = lhs.ToFMax(state);\n Src_d = FEXCore::cephes_128bit::tanl(Src_d);\n return X80SoftFloat(state, Src_d);\n#endif\n }\n\n FEXCORE_PRESERVE_ALL_ATTR static X80SoftFloat FSIN(softfloat_state* state, const X80SoftFloat& lhs) {\n#ifdef DEBUG_X86_FLOAT\n BIGFLOAT Result;\n asm(R\"(\n fninit;\n fldt %[lhs]; # st0\n fsin;\n fstpt %[result];\n )\"\n : [result] \"=m\"(Result)\n : [lhs] \"m\"(lhs)\n : \"st\");\n\n return Result;\n#else\n X80SoftFloat result;\n if (HandleInfinityOp(state, lhs, result)) {\n return result;\n }\n\n BIGFLOAT Src_d = lhs.ToFMax(state);\n Src_d = FEXCore::cephes_128bit::sinl(Src_d);\n return X80SoftFloat(state, Src_d);\n#endif\n }\n\n FEXCORE_PRESERVE_ALL_ATTR static X80SoftFloat FCOS(softfloat_state* state, const X80SoftFloat& lhs) {\n#ifdef DEBUG_X86_FLOAT\n BIGFLOAT Result;\n asm(R\"(\n fninit;\n fldt %[lhs]; # st0\n fcos;\n fstpt %[result];\n )\"\n : [result] \"=m\"(Result)\n : [lhs] \"m\"(lhs)\n : \"st\");\n\n return Result;\n#else\n X80SoftFloat result;\n if (HandleInfinityOp(state, lhs, result)) {\n return result;\n }\n\n BIGFLOAT Src_d = lhs.ToFMax(state);\n Src_d = FEXCore::cephes_128bit::cosl(Src_d);\n return X80SoftFloat(state, Src_d);\n#endif\n }\n\n FEXCORE_PRESERVE_ALL_ATTR static X80SoftFloat FSQRT(softfloat_state* state, const X80SoftFloat& lhs) {\n#ifdef DEBUG_X86_FLOAT\n BIGFLOAT Result;\n asm(R\"(\n fninit;\n fldt %[lhs]; # st0\n fsqrt;\n fstpt %[result];\n )\"\n : [result] \"=m\"(Result)\n : [lhs] \"m\"(lhs)\n : \"st\");\n\n return Result;\n#else\n return extF80_sqrt(state, lhs);\n#endif\n }\n\n float ToF32(softfloat_state* state) const {\n const float32_t Result = extF80_to_f32(state, *this);\n return std::bit_cast(Result);\n }\n\n double ToF64(softfloat_state* state) const {\n const float64_t Result = extF80_to_f64(state, *this);\n return std::bit_cast(Result);\n }\n\n FEXCore::VectorRegType ToVector() const {\n FEXCore::VectorRegType Ret {};\n memcpy(&Ret, this, sizeof(*this));\n return Ret;\n }\n\n BIGFLOAT ToFMax(softfloat_state* state) const {\n#if BIGFLOATSIZE == 16\n const float128_t Result = extF80_to_f128(state, *this);\n return std::bit_cast(Result);\n#else\n BIGFLOAT result {};\n memcpy(&result, this, sizeof(result));\n return result;\n#endif\n }\n\n int16_t ToI16(softfloat_state* state) const {\n auto rv = extF80_to_i32(state, *this, state->roundingMode, false);\n if (rv > INT16_MAX || rv < INT16_MIN) {\n ///< Indefinite value for 16-bit conversions.\n return INT16_MIN;\n } else {\n return rv;\n }\n }\n\n int32_t ToI32(softfloat_state* state) const {\n return extF80_to_i32(state, *this, state->roundingMode, false);\n }\n\n int64_t ToI64(softfloat_state* state) const {\n return extF80_to_i64(state, *this, state->roundingMode, false);\n }\n\n uint64_t ToUI64(softfloat_state* state) const {\n return extF80_to_ui64(state, *this, state->roundingMode, false);\n }\n\n void operator=(const int16_t rhs) {\n *this = i32_to_extF80(rhs);\n }\n\n void operator=(const int32_t rhs) {\n *this = i32_to_extF80(rhs);\n }\n\n void operator=(const uint64_t rhs) {\n *this = ui64_to_extF80(rhs);\n }\n\n#if BIGFLOATSIZE == 10\n void operator=(const long double rhs) {\n memcpy(this, &rhs, sizeof(rhs));\n }\n#endif\n\n operator void*() {\n return reinterpret_cast(this);\n }\n\n X80SoftFloat(extFloat80_t rhs) {\n Significand = rhs.signif;\n Top.Raw = rhs.signExp;\n }\n\n X80SoftFloat(softfloat_state* state, const float rhs) {\n *this = f32_to_extF80(state, std::bit_cast(rhs));\n }\n\n X80SoftFloat(softfloat_state* state, const double rhs) {\n *this = f64_to_extF80(state, std::bit_cast(rhs));\n }\n\n X80SoftFloat(softfloat_state* state, BIGFLOAT rhs) {\n#if BIGFLOATSIZE == 16\n *this = f128_to_extF80(state, std::bit_cast(rhs));\n#else\n *this = std::bit_cast(rhs);\n#endif\n }\n\n X80SoftFloat(const int16_t rhs) {\n *this = i32_to_extF80(rhs);\n }\n\n X80SoftFloat(const int32_t rhs) {\n *this = i32_to_extF80(rhs);\n }\n\n X80SoftFloat(const FEXCore::VectorRegType rhs) {\n memcpy(this, &rhs, sizeof(*this));\n }\n\n void operator=(extFloat80_t rhs) {\n Significand = rhs.signif;\n Top.Raw = rhs.signExp;\n }\n\n operator FEXCore::VectorRegType() const {\n return ToVector();\n }\n\n operator extFloat80_t() const {\n extFloat80_t Result {};\n Result.signif = Significand;\n Result.signExp = Top.Raw;\n return Result;\n }\n\n static bool IsNan(const X80SoftFloat& lhs) {\n return (lhs.Top.Exponent == 0x7FFF) && (lhs.Significand & IntegerBit) && (lhs.Significand & Bottom62Significand);\n }\n\n static bool SignBit(const X80SoftFloat& lhs) {\n return lhs.Top.Sign;\n }\n\nprivate:\n static constexpr uint64_t IntegerBit = (1ULL << 63);\n static constexpr uint64_t Bottom62Significand = ((1ULL << 62) - 1);\n static constexpr uint32_t ExponentBias = 16383;\n\n // Helper function to check for infinity and set invalid operation flag.\n // Returns true if infinity is dealt with, false otherwise.\n FEXCORE_PRESERVE_ALL_ATTR static bool HandleInfinityOp(softfloat_state* state, const X80SoftFloat& arg, X80SoftFloat& result) {\n if (arg.Top.Exponent == 0x7FFF && arg.Significand == 0x8000000000000000ULL) {\n state->exceptionFlags |= softfloat_flag_invalid;\n // Return QNaN.\n result.Top.Sign = 0;\n result.Top.Exponent = 0x7FFF;\n result.Significand = 0xC000000000000000ULL;\n return true;\n }\n return false;\n }\n};\n\nstatic_assert(sizeof(X80SoftFloat) == 10, \"tword must be 10bytes in size\");\n" }, { "path": "FEXCore/Source/Common/StringConv.h", "content": "// SPDX-License-Identifier: MIT\n#pragma once\n#include \n\n#include \n#include \n\nnamespace FEXCore::StrConv {\ntemplate\nbool Conv(std::string_view Value, T* Result) {\n if constexpr (std::is_signed_v) {\n *Result = static_cast(std::strtoll(Value.data(), nullptr, 0));\n } else {\n *Result = static_cast(std::strtoull(Value.data(), nullptr, 0));\n }\n return true;\n}\n\ntemplate, T>>\nbool Conv(std::string_view Value, T* Result) {\n *Result = static_cast(std::strtoull(Value.data(), nullptr, 0));\n return true;\n}\n\ninline bool Conv(std::string_view Value, fextl::string* Result) {\n *Result = Value;\n return true;\n}\n} // namespace FEXCore::StrConv\n" }, { "path": "FEXCore/Source/Common/VectorRegType.h", "content": "// SPDX-License-Identifier: MIT\n#pragma once\n\n#ifdef ARCHITECTURE_x86_64\n#include \n#include \n#else\n#include \n#endif\n\nnamespace FEXCore {\nstruct VectorScalarF64Pair {\n double val[2];\n};\n\n#ifdef ARCHITECTURE_arm64\n// Can't use uint8x16_t directly from arm_neon.h here.\n// Overrides softfloat-3e's defines which causes problems.\n#ifdef __clang__\nusing VectorRegType = __attribute__((neon_vector_type(16))) uint8_t;\n#else\nusing VectorRegType = __attribute__((vector_size(16))) uint8_t;\n#endif\nstruct VectorRegPairType {\n VectorRegType val[2];\n};\n\nstatic inline VectorRegPairType MakeVectorRegPair(VectorRegType low, VectorRegType high) {\n return VectorRegPairType {low, high};\n}\n\n#elif defined(ARCHITECTURE_x86_64)\nusing VectorRegType = __m128i;\nusing VectorRegPairType = __m256i;\n\nstatic inline VectorRegPairType MakeVectorRegPair(VectorRegType low, VectorRegType high) {\n return _mm256_set_m128i(high, low);\n}\n#endif\n} // namespace FEXCore\n" }, { "path": "FEXCore/Source/Interface/Config/Config.cpp", "content": "// SPDX-License-Identifier: MIT\n#include \"Common/StringConv.h\"\n#include \"FEXCore/Utils/EnumUtils.h\"\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\nnamespace FEXCore::Context {\nclass Context;\n}\n\nnamespace FEXCore::Config {\nnamespace detail {\n#define P(x) x\n#define OPT_BASE(type, group, enum, json, default) const P(type) P(enum) = P(default);\n#define OPT_STR(group, enum, json, default) const std::string_view P(enum) = P(default);\n#define OPT_STRARRAY(group, enum, json, default) OPT_STR(group, enum, json, default)\n#define OPT_STRENUM(group, enum, json, default) const uint64_t P(enum) = FEXCore::ToUnderlying(P(default));\n#include \n} // namespace detail\n\nenum Paths {\n PATH_DATA_DIR_LOCAL = 0,\n PATH_DATA_DIR_GLOBAL,\n PATH_CONFIG_DIR_LOCAL,\n PATH_CONFIG_DIR_GLOBAL,\n PATH_CONFIG_FILE_LOCAL,\n PATH_CONFIG_FILE_GLOBAL,\n PATH_CONFIG_TELEMETRY_FOLDER,\n PATH_LAST,\n};\nstatic std::array Paths;\n\nvoid SetDataDirectory(const std::string_view Path, bool Global) {\n Paths[PATH_DATA_DIR_LOCAL + Global] = Path;\n}\n\nvoid SetConfigDirectory(const std::string_view Path, bool Global) {\n Paths[PATH_CONFIG_DIR_LOCAL + Global] = Path;\n}\n\nvoid SetConfigFileLocation(const std::string_view Path, bool Global) {\n Paths[PATH_CONFIG_FILE_LOCAL + Global] = Path;\n}\n\nconst fextl::string& GetTelemetryDirectory() {\n auto& Path = Paths[PATH_CONFIG_TELEMETRY_FOLDER];\n if (Path.empty()) {\n FEX_CONFIG_OPT(TelemetryDirectory, TELEMETRYDIRECTORY);\n if (!TelemetryDirectory().empty()) {\n Path = TelemetryDirectory;\n Path += \"/\";\n } else {\n Path = Config::GetDataDirectory(false) + \"Telemetry/\";\n }\n }\n\n return Path;\n}\n\nconst fextl::string& GetDataDirectory(bool Global) {\n return Paths[PATH_DATA_DIR_LOCAL + Global];\n}\n\nconst fextl::string& GetConfigDirectory(bool Global) {\n return Paths[PATH_CONFIG_DIR_LOCAL + Global];\n}\n\nconst fextl::string& GetConfigFileLocation(bool Global) {\n return Paths[PATH_CONFIG_FILE_LOCAL + Global];\n}\n\nfextl::string GetApplicationConfig(const std::string_view Program, bool Global) {\n fextl::string ConfigFile = GetConfigDirectory(Global);\n\n if (!Global && !FHU::Filesystem::Exists(ConfigFile) && !FHU::Filesystem::CreateDirectories(ConfigFile)) {\n LogMan::Msg::DFmt(\"Couldn't create config directory: '{}'\", ConfigFile);\n // Let's go local in this case\n return fextl::fmt::format(\"./{}.json\", Program);\n }\n\n ConfigFile += \"AppConfig/\";\n\n // Attempt to create the local folder if it doesn't exist\n if (!Global && !FHU::Filesystem::Exists(ConfigFile) && !FHU::Filesystem::CreateDirectories(ConfigFile)) {\n // Let's go local in this case\n return fextl::fmt::format(\"./{}.json\", Program);\n }\n\n return fextl::fmt::format(\"{}{}.json\", ConfigFile, Program);\n}\n\nstatic fextl::map> ConfigLayers;\nclass MetaLayer;\nstatic FEXCore::Config::MetaLayer* Meta {};\n\nconstexpr std::array LoadOrder = {\n FEXCore::Config::LayerType::LAYER_GLOBAL_MAIN, FEXCore::Config::LayerType::LAYER_MAIN,\n FEXCore::Config::LayerType::LAYER_GLOBAL_STEAM_APP, FEXCore::Config::LayerType::LAYER_GLOBAL_APP,\n FEXCore::Config::LayerType::LAYER_LOCAL_STEAM_APP, FEXCore::Config::LayerType::LAYER_LOCAL_APP,\n FEXCore::Config::LayerType::LAYER_ARGUMENTS, FEXCore::Config::LayerType::LAYER_USER_OVERRIDE,\n FEXCore::Config::LayerType::LAYER_ENVIRONMENT, FEXCore::Config::LayerType::LAYER_TOP};\n\nLayer::Layer(const LayerType _Type)\n : Type {_Type} {}\n\nLayer::~Layer() {}\n\nclass MetaLayer final : public FEXCore::Config::Layer {\npublic:\n MetaLayer(const LayerType _Type)\n : FEXCore::Config::Layer(_Type) {}\n ~MetaLayer() {}\n void Load();\n\n template\n requires (!std::is_same_v && !std::is_same_v)\n std::optional GetConv(ConfigOption Option) {\n const auto it = OptionMap.find(Option);\n if (it == OptionMap.end()) {\n return std::nullopt;\n }\n\n const auto& Value = it->second;\n LOGMAN_THROW_A_FMT(!std::holds_alternative(Value), \"Tried to get config of invalid type!\");\n\n if (std::holds_alternative(Value)) [[likely]] {\n return std::get(Value);\n }\n\n T ConvertedValue;\n if (std::holds_alternative(Value)) {\n const auto& StrVal = std::get(Value);\n if (FEXCore::StrConv::Conv(StrVal, &ConvertedValue)) {\n // Convert the value.\n OptionMap[Option].emplace(ConvertedValue);\n return ConvertedValue;\n } else {\n LOGMAN_MSG_A_FMT(\"Couldn't Convert {} to specified type!\", StrVal);\n }\n }\n\n FEX_UNREACHABLE;\n }\n\nprivate:\n void MergeConfigMap(const LayerOptions& Options);\n void MergeEnvironmentVariables(const ConfigOption& Option, const StringArrayType& Value);\n};\n\nvoid MetaLayer::Load() {\n OptionMap.clear();\n\n for (auto CurrentLayer = LoadOrder.begin(); CurrentLayer != LoadOrder.end(); ++CurrentLayer) {\n auto it = ConfigLayers.find(*CurrentLayer);\n if (it != ConfigLayers.end() && *CurrentLayer != Type) {\n // Merge this layer's options to this layer\n MergeConfigMap(it->second->GetOptionMap());\n }\n }\n}\n\n\nvoid MetaLayer::MergeEnvironmentVariables(const ConfigOption& Option, const StringArrayType& Value) {\n // Environment variables need a bit of additional work\n // We want to merge the arrays rather than overwrite entirely\n auto MetaEnvironment = OptionMap.find(Option);\n if (MetaEnvironment == OptionMap.end()) {\n // Doesn't exist, just insert\n OptionMap.insert_or_assign(Option, Value);\n return;\n }\n\n // If an environment variable exists in both current meta and in the incoming layer then the meta layer value is overwritten\n fextl::unordered_map LookupMap;\n const auto AddToMap = [&LookupMap](const StringArrayType& Value) {\n for (const auto& EnvVar : Value) {\n const auto ItEq = EnvVar.find_first_of('=');\n if (ItEq == fextl::string::npos) {\n // Broken environment variable\n // Skip\n continue;\n }\n auto Key = fextl::string(EnvVar.begin(), EnvVar.begin() + ItEq);\n auto Value = fextl::string(EnvVar.begin() + ItEq + 1, EnvVar.end());\n\n // Add the key to the map, overwriting whatever previous value was there\n LookupMap.insert_or_assign(std::move(Key), std::move(Value));\n }\n };\n\n AddToMap(std::get(MetaEnvironment->second));\n AddToMap(Value);\n\n // Now with the two layers merged in the map\n // Add all the values to the option\n Erase(Option);\n for (auto& Val : LookupMap) {\n // Set will emplace multiple options in to its list\n AppendStrArrayValue(Option, Val.first + \"=\" + Val.second);\n }\n}\n\nvoid MetaLayer::MergeConfigMap(const LayerOptions& Options) {\n // Insert this layer's options, overlaying previous options that exist here\n for (auto& it : Options) {\n if (it.first == FEXCore::Config::ConfigOption::CONFIG_ENV || it.first == FEXCore::Config::ConfigOption::CONFIG_HOSTENV) {\n LOGMAN_THROW_A_FMT(std::holds_alternative(it.second), \"Tried to get config of invalid type!\");\n MergeEnvironmentVariables(it.first, std::get(it.second));\n } else {\n OptionMap.insert_or_assign(it.first, it.second);\n }\n }\n}\n\nvoid Initialize() {\n AddLayer(fextl::make_unique(FEXCore::Config::LayerType::LAYER_TOP));\n Meta = dynamic_cast(ConfigLayers.begin()->second.get());\n}\n\nvoid Shutdown() {\n ConfigLayers.clear();\n Meta = nullptr;\n}\n\nvoid Load() {\n for (auto CurrentLayer = LoadOrder.begin(); CurrentLayer != LoadOrder.end(); ++CurrentLayer) {\n auto it = ConfigLayers.find(*CurrentLayer);\n if (it != ConfigLayers.end()) {\n it->second->Load();\n }\n }\n}\n\nfextl::string ExpandPath(const fextl::string& ContainerPrefix, const fextl::string& PathName) {\n if (PathName.empty()) {\n return {};\n }\n\n // Expand home if it exists\n if (FHU::Filesystem::IsRelative(PathName)) {\n fextl::string Home = getenv(\"HOME\") ?: \"\";\n // Home expansion only works if it is the first character\n // This matches bash behaviour\n if (PathName.starts_with(\"~/\")) {\n Home.append(PathName.begin() + 1, PathName.end());\n return Home;\n }\n\n // Expand relative path to absolute\n char ExistsTempPath[PATH_MAX];\n char* RealPath = FHU::Filesystem::Absolute(PathName.c_str(), ExistsTempPath);\n if (RealPath && FHU::Filesystem::Exists(RealPath)) {\n return RealPath;\n }\n\n // Only return if it exists\n if (FHU::Filesystem::Exists(PathName)) {\n return PathName;\n }\n } else {\n // If the containerprefix and pathname isn't empty\n // Then we check if the pathname exists in our current namespace\n // If the path DOESN'T exist but DOES exist with the prefix applied\n // then redirect to the prefix\n //\n // This might not be expected behaviour for some edge cases but since\n // all paths aren't mounted inside the container, then it'll be fine\n //\n // Main catch case for this is the default thunk install folders\n // HostThunks: $CMAKE_INSTALL_PREFIX/lib/fex-emu/HostThunks/\n // GuestThunks: $CMAKE_INSTALL_PREFIX/share/fex-emu/GuestThunks/\n if (!ContainerPrefix.empty() && !PathName.empty()) {\n if (!FHU::Filesystem::Exists(PathName)) {\n auto ContainerPath = ContainerPrefix + PathName;\n if (FHU::Filesystem::Exists(ContainerPath)) {\n return ContainerPath;\n }\n }\n }\n }\n return {};\n}\n\nconstexpr char ContainerManager[] = \"/run/host/container-manager\";\n\nfextl::string FindContainer() {\n // We only support pressure-vessel at the moment\n if (FHU::Filesystem::Exists(ContainerManager)) {\n fextl::string Manager {};\n if (FEXCore::FileLoading::LoadFile(Manager, ContainerManager)) {\n // Trim the whitespace, may contain a newline\n return FEXCore::StringUtils::Trim(Manager);\n }\n }\n return {};\n}\n\nfextl::string FindContainerPrefix() {\n // We only support pressure-vessel at the moment\n if (FHU::Filesystem::Exists(ContainerManager)) {\n fextl::string Manager {};\n if (FEXCore::FileLoading::LoadFile(Manager, ContainerManager)) {\n // Trim the whitespace, may contain a newline\n if (FEXCore::StringUtils::Trim(Manager) == \"pressure-vessel\") {\n // We are running inside of pressure vessel\n // Our $CMAKE_INSTALL_PREFIX paths are now inside of /run/host/$CMAKE_INSTALL_PREFIX\n return \"/run/host/\";\n }\n }\n }\n return {};\n}\n\nvoid ReloadMetaLayer() {\n Meta->Load();\n\n const fextl::string ContainerPrefix {FindContainerPrefix()};\n auto ExpandPathIfExists = [&ContainerPrefix](FEXCore::Config::ConfigOption Config, const fextl::string& PathName) {\n const auto NewPath = ExpandPath(ContainerPrefix, PathName);\n if (!NewPath.empty()) {\n FEXCore::Config::Set(Config, NewPath);\n }\n };\n\n if (FEXCore::Config::Exists(FEXCore::Config::CONFIG_ROOTFS)) {\n const auto PathName = *Meta->Get(FEXCore::Config::CONFIG_ROOTFS);\n const auto ExpandedString = ExpandPath(ContainerPrefix, *PathName);\n if (!ExpandedString.empty()) {\n // Adjust the path if it ended up being relative\n FEXCore::Config::Set(FEXCore::Config::CONFIG_ROOTFS, ExpandedString);\n } else if (!PathName->empty()) {\n // If the filesystem doesn't exist then let's see if it exists in the fex-emu folder\n const auto PathNameCopy = *PathName;\n for (auto Global : {true, false}) {\n for (auto DirectoryFetchers : {GetDataDirectory, GetConfigDirectory}) {\n fextl::string NamedRootFS = DirectoryFetchers(Global) + \"RootFS/\" + PathNameCopy;\n if (FHU::Filesystem::Exists(NamedRootFS)) {\n FEXCore::Config::Set(FEXCore::Config::CONFIG_ROOTFS, NamedRootFS);\n }\n }\n }\n }\n }\n if (FEXCore::Config::Exists(FEXCore::Config::CONFIG_THUNKHOSTLIBS)) {\n const auto PathName = *Meta->Get(FEXCore::Config::CONFIG_THUNKHOSTLIBS);\n ExpandPathIfExists(FEXCore::Config::CONFIG_THUNKHOSTLIBS, *PathName);\n }\n if (FEXCore::Config::Exists(FEXCore::Config::CONFIG_THUNKGUESTLIBS)) {\n const auto PathName = *Meta->Get(FEXCore::Config::CONFIG_THUNKGUESTLIBS);\n ExpandPathIfExists(FEXCore::Config::CONFIG_THUNKGUESTLIBS, *PathName);\n }\n if (FEXCore::Config::Exists(FEXCore::Config::CONFIG_THUNKCONFIG)) {\n const auto PathName = *Meta->Get(FEXCore::Config::CONFIG_THUNKCONFIG);\n const auto ExpandedString = ExpandPath(ContainerPrefix, *PathName);\n if (!ExpandedString.empty()) {\n // Adjust the path if it ended up being relative\n FEXCore::Config::Set(FEXCore::Config::CONFIG_THUNKCONFIG, ExpandedString);\n } else if (!PathName->empty()) {\n // If the filesystem doesn't exist then let's see if it exists in the fex-emu folder\n const auto PathNameCopy = *PathName;\n for (auto Global : {true, false}) {\n for (auto DirectoryFetchers : {GetDataDirectory, GetConfigDirectory}) {\n fextl::string NamedConfig = DirectoryFetchers(Global) + \"ThunkConfigs/\" + PathNameCopy;\n if (FHU::Filesystem::Exists(NamedConfig)) {\n FEXCore::Config::Set(FEXCore::Config::CONFIG_THUNKCONFIG, NamedConfig);\n }\n }\n }\n }\n }\n if (FEXCore::Config::Exists(FEXCore::Config::CONFIG_OUTPUTLOG)) {\n const auto PathName = *Meta->Get(FEXCore::Config::CONFIG_OUTPUTLOG);\n if (*PathName != \"stdout\" && *PathName != \"stderr\" && *PathName != \"server\") {\n ExpandPathIfExists(FEXCore::Config::CONFIG_OUTPUTLOG, *PathName);\n }\n }\n\n if (FEXCore::Config::Exists(FEXCore::Config::CONFIG_DUMPIR) && !FEXCore::Config::Exists(FEXCore::Config::CONFIG_PASSMANAGERDUMPIR)) {\n // If DumpIR is set but no PassManagerDumpIR configuration is set, then default to `afteropt`\n const auto PathName = *Meta->Get(FEXCore::Config::CONFIG_DUMPIR);\n if (*PathName != \"no\") {\n Set(FEXCore::Config::ConfigOption::CONFIG_PASSMANAGERDUMPIR,\n fextl::fmt::format(\"{}\", static_cast(FEXCore::Config::PassManagerDumpIR::AFTEROPT)));\n }\n }\n\n if (FEXCore::Config::Exists(FEXCore::Config::CONFIG_SINGLESTEP) && Meta->GetConv(FEXCore::Config::CONFIG_SINGLESTEP).value_or(false)) {\n // Single stepping also enforces single instruction size blocks\n Set(FEXCore::Config::ConfigOption::CONFIG_MAXINST, \"1\");\n }\n}\n\nvoid AddLayer(fextl::unique_ptr _Layer) {\n ConfigLayers.emplace(_Layer->GetLayerType(), std::move(_Layer));\n}\n\nbool Exists(ConfigOption Option) {\n return Meta->OptionExists(Option);\n}\n\nstd::optional All(ConfigOption Option) {\n return Meta->All(Option);\n}\n\nstd::optional Get(ConfigOption Option) {\n return Meta->Get(Option);\n}\n\ntemplate\nstd::optional GetConv(ConfigOption Option) {\n return Meta->GetConv(Option);\n}\n\ntemplate std::optional GetConv(ConfigOption Option);\ntemplate std::optional GetConv(ConfigOption Option);\ntemplate std::optional GetConv(ConfigOption Option);\ntemplate std::optional GetConv(ConfigOption Option);\ntemplate std::optional GetConv(ConfigOption Option);\n\nvoid Set(ConfigOption Option, std::string_view Data) {\n Meta->Set(Option, Data);\n}\n\nvoid Erase(ConfigOption Option) {\n Meta->Erase(Option);\n}\n\ntemplate\nT Value::GetIfExists(FEXCore::Config::ConfigOption Option, T Default) {\n auto Value = FEXCore::Config::GetConv(Option);\n if (Value) {\n return *Value;\n }\n\n return Default;\n}\n\ntemplate<>\nfextl::string Value::GetIfExists(FEXCore::Config::ConfigOption Option, fextl::string Default) {\n auto Value = FEXCore::Config::Get(Option);\n if (Value) {\n return **Value;\n } else {\n return Default;\n }\n}\n\ntemplate<>\nfextl::string Value::GetIfExists(FEXCore::Config::ConfigOption Option, std::string_view Default) {\n auto Value = FEXCore::Config::Get(Option);\n if (Value) {\n return **Value;\n } else {\n return fextl::string(Default);\n }\n}\n\ntemplate bool Value::GetIfExists(FEXCore::Config::ConfigOption Option, bool Default);\ntemplate int8_t Value::GetIfExists(FEXCore::Config::ConfigOption Option, int8_t Default);\ntemplate uint8_t Value::GetIfExists(FEXCore::Config::ConfigOption Option, uint8_t Default);\ntemplate int16_t Value::GetIfExists(FEXCore::Config::ConfigOption Option, int16_t Default);\ntemplate uint16_t Value::GetIfExists(FEXCore::Config::ConfigOption Option, uint16_t Default);\ntemplate int32_t Value::GetIfExists(FEXCore::Config::ConfigOption Option, int32_t Default);\ntemplate uint32_t Value::GetIfExists(FEXCore::Config::ConfigOption Option, uint32_t Default);\ntemplate int64_t Value::GetIfExists(FEXCore::Config::ConfigOption Option, int64_t Default);\ntemplate uint64_t Value::GetIfExists(FEXCore::Config::ConfigOption Option, uint64_t Default);\n\n// Constructor\ntemplate Value::Value(FEXCore::Config::ConfigOption _Option, fextl::string Default);\ntemplate Value::Value(FEXCore::Config::ConfigOption _Option, bool Default);\ntemplate Value::Value(FEXCore::Config::ConfigOption _Option, uint8_t Default);\ntemplate Value::Value(FEXCore::Config::ConfigOption _Option, uint64_t Default);\n\ntemplate\nvoid Value::GetListIfExists(FEXCore::Config::ConfigOption Option, StringArrayType* List) {\n auto Value = FEXCore::Config::All(Option);\n List->clear();\n if (Value) {\n *List = **Value;\n }\n}\ntemplate void Value::GetListIfExists(FEXCore::Config::ConfigOption Option, StringArrayType* List);\n} // namespace FEXCore::Config\n" }, { "path": "FEXCore/Source/Interface/Config/Config.json.in", "content": "{\n \"Options\": {\n \"CPU\": {\n \"Multiblock\": {\n \"Type\": \"bool\",\n \"Default\": \"true\",\n \"Desc\": [\n \"Controls multiblock code compilation\",\n \"Can cause long JIT compilation times and stutter\"\n ]\n },\n \"MaxInst\": {\n \"Type\": \"int32\",\n \"Default\": \"5000\",\n \"Desc\": [\n \"Maximum number of instruction to store in a block\"\n ]\n },\n \"EnableCodeCachingWIP\": {\n \"Type\": \"bool\",\n \"Default\": \"false\",\n \"Desc\": [\n \"Enable the code caching subsystem\"\n ]\n },\n \"EnableCodeCacheValidation\": {\n \"Type\": \"bool\",\n \"Default\": \"false\",\n \"Desc\": [\n \"Enable expensive validation when loading code caches\"\n ]\n },\n \"HostFeatures\": {\n \"Type\": \"strenum\",\n \"Default\": \"FEXCore::Config::HostFeatures::OFF\",\n \"Enums\": {\n \"ENABLESVE\": \"enablesve\",\n \"DISABLESVE\": \"disablesve\",\n \"ENABLEAVX\": \"enableavx\",\n \"DISABLEAVX\": \"disableavx\",\n \"ENABLEAFP\": \"enableafp\",\n \"DISABLEAFP\": \"disableafp\",\n \"ENABLELRCPC\": \"enablelrcpc\",\n \"DISABLELRCPC\": \"disablelrcpc\",\n \"ENABLELRCPC2\": \"enablelrcpc2\",\n \"DISABLELRCPC2\": \"disablelrcpc2\",\n \"ENABLECSSC\": \"enablecssc\",\n \"DISABLECSSC\": \"disablecssc\",\n \"ENABLEPMULL128\": \"enablepmull128\",\n \"DISABLEPMULL128\": \"disablepmull128\",\n \"ENABLERNG\": \"enablerng\",\n \"DISABLERNG\": \"disablerng\",\n \"ENABLECLZERO\": \"enableclzero\",\n \"DISABLECLZERO\": \"disableclzero\",\n \"ENABLEATOMICS\": \"enableatomics\",\n \"DISABLEATOMICS\": \"disableatomics\",\n \"ENABLEFCMA\": \"enablefcma\",\n \"DISABLEFCMA\": \"disablefcma\",\n \"ENABLEFLAGM\": \"enableflagm\",\n \"DISABLEFLAGM\": \"disableflagm\",\n \"ENABLEFLAGM2\": \"enableflagm2\",\n \"DISABLEFLAGM2\": \"disableflagm2\",\n \"ENABLEFRINTTS\": \"enablefrintts\",\n \"DISABLEFRINTTS\": \"disablefrintts\",\n \"ENABLECRYPTO\": \"enablecrypto\",\n \"DISABLECRYPTO\": \"disablecrypto\",\n \"ENABLERPRES\": \"enablerpres\",\n \"DISABLERPRES\": \"disablerpres\",\n \"ENABLESVEBITPERM\": \"enablesvebitperm\",\n \"DISABLESVEBITPERM\": \"disablesvebitperm\",\n \"ENABLEPRESERVEALLABI\": \"enablepreserveallabi\",\n \"DISABLEPRESERVEALLABI\": \"disablepreserveallabi\",\n \"ENABLEWFXT\": \"enablewfxt\",\n \"DISABLEWFXT\": \"disablewfxt\",\n \"ENABLE3DNOW\": \"enable3dnow\",\n \"DISABLE3DNOW\": \"disable3dnow\",\n \"ENABLESSE4A\": \"enablesse4a\",\n \"DISABLESSE4A\": \"disablesse4a\",\n \"ENABLEMOPS\": \"enablemops\",\n \"DISABLEMOPS\": \"disablemops\"\n },\n \"Desc\": [\n \"Allows controlling of the CPU features in the JIT.\",\n \"\\toff: Default CPU features queried from CPU features\",\n \"\\t{enable,disable}sve: Will force enable or disable sve even if the host doesn't support it\",\n \"\\t{enable,disable}avx: Will force enable or disable avx even if the host doesn't support it\",\n \"\\t{enable,disable}afp: Will force enable or disable afp even if the host doesn't support it\",\n \"\\t{enable,disable}lrcpc: Will force enable or disable lrcpc even if the host doesn't support it\",\n \"\\t{enable,disable}lrcpc2: Will force enable or disable lrcpc2 even if the host doesn't support it\",\n \"\\t{enable,disable}cssc: Will force enable or disable cssc even if the host doesn't support it\",\n \"\\t{enable,disable}pmull128: Will force enable or disable pmull128 even if the host doesn't support it\",\n \"\\t{enable,disable}rng: Will force enable or disable rng even if the host doesn't support it\",\n \"\\t{enable,disable}clzero: Will force enable or disable clzero even if the host doesn't support it\",\n \"\\t{enable,disable}atomics: Will force enable or disable ARMv8.1 LSE atomics even if the host doesn't support it\",\n \"\\t{enable,disable}fcma: Will force enable or disable fcma even if the host doesn't support it\",\n \"\\t{enable,disable}flagm: Will force enable or disable flagm even if the host doesn't support it\",\n \"\\t{enable,disable}flagm2: Will force enable or disable flagm2 even if the host doesn't support it\",\n \"\\t{enable,disable}crypto: Will force enable or disable crypto extensions even if the host doesn't support it\",\n \"\\t{enable,disable}rpres: Will force enable or disable rpres even if the host doesn't support it\",\n \"\\t{enable,disable}svebitperm: Will force enable or disable svebitperm even if the host doesn't support it\",\n \"\\t{enable,disable}preserveallabi: Will force enable or disable preserve_all abi even if the host doesn't support it\",\n \"\\t{enable,disable}wfxt: Will force enable or disable wfxt even if the host doesn't support it\",\n \"\\t{enable,disable}3dnow: Will force enable or disable 3DNow! even if the host doesn't support it\",\n \"\\t{enable,disable}sse4a: Will force enable or disable SSE4a even if the host doesn't support it\",\n \"\\t{enable,disable}mops: Will force enable or disable FEAT_MOPS even if the host doesn't support it\"\n ]\n },\n \"SmallTSCScale\": {\n \"Type\": \"bool\",\n \"Default\": \"true\",\n \"Desc\": [\n \"Scales the cycle counter on systems that have low frequencies.\"\n ]\n },\n \"HideHybrid\": {\n \"Type\": \"bool\",\n \"Default\": \"true\",\n \"Desc\": [\n \"Hides hybrid CPU core arrangement.\"\n ]\n },\n \"CPUFeatureRegisters\": {\n \"Type\": \"str\",\n \"Default\": \"\",\n \"Desc\": [\n \"Allows overriding cpu feature flags for manual testing\"\n ]\n }\n },\n \"Emulation\": {\n \"RootFS\": {\n \"Type\": \"str\",\n \"Default\": \"\",\n \"Desc\": [\n \"Which Root filesystem prefix to use\",\n \"This can be a filesystem path\",\n \"\\teg: ~/RootFS/Debian_x86_64\",\n \"Or this can be a name of a rootfs\",\n \"If the named rootfs exists in the FEX data folder then it will use that one\",\n \"\\teg: $XDG_DATA_HOME/fex-emu/RootFS//\",\n \"If XDG_DATA_HOME is unset, ~/.local/share will be used in its place.\",\n \"\\teg: $HOME/.local/share/fex-emu/RootFS//\"\n ]\n },\n \"ThunkHostLibs\": {\n \"Type\": \"str\",\n \"Default\": \"@CMAKE_INSTALL_FULL_LIBDIR@/fex-emu/HostThunks\",\n \"Desc\": [\n \"Folder to find the host-side thunking libraries.\"\n ]\n },\n \"ThunkGuestLibs\": {\n \"Type\": \"str\",\n \"Default\": \"@CMAKE_INSTALL_PREFIX@/share/fex-emu/GuestThunks\",\n \"Desc\": [\n \"Folder to find the guest-side thunking libraries.\"\n ]\n },\n \"ThunkConfig\": {\n \"Type\": \"str\",\n \"Default\": \"\",\n \"Desc\": [\n \"A json file specifying where to overlay the thunks.\",\n \"This can be a filesystem path\",\n \"\\teg: ~/MyThunkConfig.json\",\n \"Or this can be a named of a Thunk config file\",\n \"If the named config file exists in the FEX data folder folder the it will use that one\",\n \"\\teg: $XDG_DATA_HOME/fex-emu/ThunkConfigs/\",\n \"If XDG_DATA_HOME is unset, ~/.local/share will be used in its place.\",\n \"\\teg: $HOME/.local/share/fex-emu/ThunkConfigs/\"\n ]\n },\n \"Env\": {\n \"Type\": \"strarray\",\n \"Default\": \"\",\n \"Desc\": [\n \"Adds an environment variable to the emulated environment.\"\n ]\n },\n \"HostEnv\": {\n \"Type\": \"strarray\",\n \"Default\": \"\",\n \"Desc\": [\n \"Adds an environment variable to the host environment.\",\n \"This can be useful for setting environment variables that thunks can pick up.\",\n \"Typically isn't necessary since the guest libc isn't thunked. But is possible.\"\n ]\n },\n \"AdditionalArguments\": {\n \"Type\": \"strarray\",\n \"Default\": \"\",\n \"Desc\": [\n \"Allows the user to pass additional arguments to the application\"\n ]\n },\n \"DisableL2Cache\": {\n \"Type\": \"bool\",\n \"Default\": \"true\",\n \"Desc\": [\n \"Disables FEXCore's JIT L2 cache lookup. Saving memory.\",\n \"Can potentially introduce more stutters.\"\n ]\n },\n \"DynamicL1Cache\": {\n \"Type\": \"bool\",\n \"Default\": \"true\",\n \"Desc\": [\n \"Switches FEXCore's JIT L1 cache to be dynamically sized. Saving memory.\",\n \"Can potentially introduce more stutters.\"\n ]\n },\n \"DynamicL1CacheIncreaseCountHeuristic\": {\n \"Type\": \"uint64\",\n \"Default\": \"250\",\n \"Desc\": [\n \"Threshold of lookups per second that the L1 dynamic cache should increase its size.\",\n \"Lower numbers means more aggressive scaling upward to the maximum size.\",\n \"Higher numbers means more conservative scaling, using less memory.\",\n \"Can potentially introduce stutters, more likely the higher the number.\",\n \"Don't have this number smaller than the decrease count!\"\n ]\n },\n \"DynamicL1CacheDecreaseCountHeuristic\": {\n \"Type\": \"uint64\",\n \"Default\": \"50\",\n \"Desc\": [\n \"Threshold of lookups per second that the L1 dynamic cache should decrease its size.\",\n \"The higher the number, the more aggressively it reduces the L1 cache size.\",\n \"Lower numbers means more conservative memory savings.\",\n \"Can potentially introduce more stutters, more likely the higher the number.\",\n \"Don't have this number larger than the increase count!\"\n ]\n }\n },\n \"Debug\": {\n \"SingleStep\": {\n \"Type\": \"bool\",\n \"Default\": \"false\",\n \"Desc\": [\n \"Single stepping configuration.\"\n ]\n },\n \"GdbServer\": {\n \"Type\": \"bool\",\n \"Default\": \"false\",\n \"Desc\": [\n \"Enables the GDB server.\"\n ]\n },\n \"DumpIR\": {\n \"Type\": \"str\",\n \"Default\": \"no\",\n \"Desc\": [\n \"Folder to dump the IR in to.\",\n \"[no, stdout, stderr, server, ]\"\n ]\n },\n \"PassManagerDumpIR\": {\n \"Type\": \"strenum\",\n \"Default\": \"FEXCore::Config::PassManagerDumpIR::OFF\",\n \"Enums\": {\n \"BEFOREOPT\": \"beforeopt\",\n \"AFTEROPT\": \"afteropt\",\n \"BEFOREPASS\": \"beforepass\",\n \"AFTERPASS\": \"afterpass\"\n },\n \"Desc\": [\n \"Allows controlling when FEX dumps its IR.\",\n \"\\toff: IR dumping will be disabled\",\n \"\\tbeforeopt: Dump IR before any optimizations\",\n \"\\tafteropt: Dump IR after all optimizations\",\n \"\\tbeforepass: Dump IR before every optimization pass\",\n \"\\tafterpass: Dump IR after every optimization pass\"\n ]\n },\n \"DumpGPRs\": {\n \"Type\": \"bool\",\n \"Default\": \"false\",\n \"Desc\": [\n \"When the test harness ends, print the GPR state.\"\n ]\n },\n \"O0\": {\n \"Type\": \"bool\",\n \"Default\": \"false\",\n \"Desc\": [\n \"Disables optimizations passes for debugging.\"\n ]\n },\n \"GlobalJITNaming\": {\n \"Type\": \"bool\",\n \"Default\": \"false\",\n \"Desc\": [\n \"Uses JITSymbols to name all JIT state as one symbol\",\n \"Useful for querying how much time is spent inside of the JIT\",\n \"Profiling tools will show JIT time as FEXJIT\"\n ]\n },\n \"LibraryJITNaming\": {\n \"Type\": \"bool\",\n \"Default\": \"false\",\n \"Desc\": [\n \"Uses JITSymbols to name JIT symbols grouped by library\",\n \"Useful for querying how much time is spent in each guest library\",\n \"Can be used to help guide thunk generation\"\n ]\n },\n \"BlockJITNaming\": {\n \"Type\": \"bool\",\n \"Default\": \"false\",\n \"Desc\": [\n \"Uses JITSymbols to name JIT symbols\",\n \"Useful for determining hot blocks of code\",\n \"Has some file writing overhead per JIT block\"\n ]\n },\n \"GDBSymbols\": {\n \"Type\": \"bool\",\n \"Default\": \"false\",\n \"Desc\": [\n \"Integrates with GDB using the JIT interface.\",\n \"Needs the fex jit loader in GDB, which can be loaded via `jit-reader-load libFEXGDBReader.so.`\",\n \"Also needs x86_64-linux-gnu-objdump in PATH.\",\n \"Can be very slow.\"\n ]\n },\n \"InjectLibSegFault\": {\n \"Type\": \"bool\",\n \"Default\": \"false\",\n \"Desc\": [\n \"Sets the environment variable LD_PRELOAD=libSegFault.so\",\n \"This allows the user to very easily enable libSegFault without dealing with environment variables\",\n \"Very useful for applications that have launch scripts that set the variable to nothing at launch\",\n \"Set this in an application configuration for injecting in to only specific applications.\",\n \"\\tNote: If x86/x86_64 libSegFault.so isn't installed then this option won't work.\"\n ]\n },\n \"Disassemble\": {\n \"Type\": \"strenum\",\n \"Default\": \"FEXCore::Config::Disassemble::OFF\",\n \"Enums\": {\n \"DISPATCHER\": \"dispatcher\",\n \"BLOCKS\": \"blocks\",\n \"STATS\": \"stats\"\n },\n \"Desc\": [\n \"Allows controlling of the vixl disassembler for generated ARM code.\",\n \"\\toff: No disassembly will be output\",\n \"\\tdispatcher: Will enable disassembly of the JIT dispatcher loop\",\n \"\\tblocks: Will enable disassembly of the translated instruction code blocks\",\n \"\\tstats: Will print stats when disassembling the code\"\n ]\n },\n \"X86Disassemble\": {\n \"Type\": \"bool\",\n \"Default\": \"false\",\n \"Desc\": [\n \"Enables x86/x86-64 guest disassembly output for compiled blocks.\",\n \"Requires FEX to be built with -DENABLE_ZYDIS=TRUE\"\n ]\n },\n \"ForceSVEWidth\": {\n \"Type\": \"uint32\",\n \"Default\": \"0\",\n \"Desc\": [\n \"Allows overriding the SVE width in the vixl simulator.\",\n \"Useful as a debugging feature.\"\n ]\n },\n \"DisableTelemetry\": {\n \"Type\": \"bool\",\n \"Default\": \"false\",\n \"Desc\": [\n \"Disables telemetry at runtime.\",\n \"Useful for CI instcountCI mostly\"\n ]\n }\n },\n \"Logging\": {\n \"SilentLog\": {\n \"Type\": \"bool\",\n \"Default\": \"true\",\n \"Desc\": [\n \"Disables logging\"\n ]\n },\n \"OutputLog\": {\n \"Type\": \"str\",\n \"Default\": \"server\",\n \"Desc\": [\n \"File to write FEX output to.\",\n \"[stderr, server, ]\"\n ]\n },\n \"TelemetryDirectory\": {\n \"Type\": \"str\",\n \"Default\": \"\",\n \"Desc\": [\n \"Redirects the telemetry folder that FEX usually writes to.\",\n \"By default telemetry data is stored in {$FEX_APP_DATA_LOCATION,{$XDG_DATA_HOME,$HOME}/fex-emu/Telemetry/}\"\n ]\n },\n \"ProfileStats\": {\n \"Type\": \"bool\",\n \"Default\": \"false\",\n \"Desc\": [\n \"Enables FEX's low-overhead sampling profile statistics.\",\n \"Requires a supported version of Mangohud to see the results\"\n ]\n },\n \"EnableGpuvisProfiling\": {\n \"Type\": \"bool\",\n \"Default\": \"false\",\n \"Desc\": [\n \"Enables profiling when FEX was built with the gpuvis profiler backend.\"\n ]\n }\n },\n \"Hacks\": {\n \"SMCChecks\": {\n \"Type\": \"uint8\",\n \"Default\": \"FEXCore::Config::CONFIG_SMC_MTRACK\",\n \"TextDefault\": \"mtrack\",\n \"ArgumentHandler\": \"SMCCheckHandler\",\n \"Desc\": [\n \"Checks code for modification before execution.\",\n \"\\tnone: No checks\",\n \"\\tmtrack: Page tracking based invalidation (default)\",\n \"\\tfull: Validate code before every run (slow)\"\n ]\n },\n \"TSOEnabled\": {\n \"Type\": \"bool\",\n \"Default\": \"true\",\n \"Desc\": [\n \"Controls TSO IR ops.\",\n \"Highly likely to break any multithreaded application if disabled.\"\n ]\n },\n \"VectorTSOEnabled\": {\n \"Type\": \"bool\",\n \"Default\": \"false\",\n \"Desc\": [\n \"When TSO emulation is enabled, controls if vector loadstores should also be atomic.\"\n ]\n },\n \"MemcpySetTSOEnabled\": {\n \"Type\": \"bool\",\n \"Default\": \"false\",\n \"Desc\": [\n \"When TSO emulation is enabled, controls if memcpy and memset should also be atomic.\",\n \"Only affects REP MOVS and REP STOS instructions\"\n ]\n },\n \"HalfBarrierTSOEnabled\": {\n \"Type\": \"bool\",\n \"Default\": \"true\",\n \"Desc\": [\n \"When TSO emulation is enabled, controls if unaligned loads and stores should be backpatched to half-barrier atomics.\",\n \"Can be dangerous due to aligned loadstores through the same code now become non-atomic.\"\n ]\n },\n \"StrictInProcessSplitLocks\": {\n \"Type\": \"bool\",\n \"Default\": \"false\",\n \"Desc\": [\n \"Strict global lock when handling an unaligned atomic that crosses a 16-byte or cacheline granularity\",\n \"This is required to ensure a split-lock doesn't tear inside the process\"\n ]\n },\n \"KernelUnalignedAtomicBackpatching\": {\n \"Type\": \"bool\",\n \"Default\": \"true\",\n \"Desc\": [\n \"When the kernel unaligned atomic handler is enabled, use backpatching to reduce kernel context switches.\"\n ]\n },\n \"VolatileMetadata\": {\n \"Type\": \"bool\",\n \"Default\": \"true\",\n \"Desc\": [\n \"Use volatile metadata in PE files to inform TSO instructions when available.\",\n \"When metadata is unavailable falls back to the currently enabled TSO options.\"\n ]\n },\n \"X87ReducedPrecision\": {\n \"Type\": \"bool\",\n \"Default\": \"false\",\n \"Desc\": [\n \"Emulates X87 floating point using 64-bit precision. This reduces emulation accuracy and may result in rendering bugs.\"\n ]\n },\n \"StallProcess\": {\n \"Type\": \"bool\",\n \"Default\": \"false\",\n \"Desc\": [\n \"Forces a process to stall out on initialization\",\n \"Useful for a process that keeps restarting and doesn't work\"\n ]\n },\n \"HideHypervisorBit\": {\n \"Type\": \"bool\",\n \"Default\": \"false\",\n \"Desc\": [\n \"Hides the hypervisor CPUID bit when set.\",\n \"Should only be used for applications that have issues with this set.\"\n ]\n },\n \"StartupSleep\": {\n \"Type\": \"uint32\",\n \"Default\": \"0\",\n \"Desc\": [\n \"Sleeps the process at startup for a duration of seconds.\",\n \"Useful if an application crashes too quickly to attach a debugger.\"\n ]\n },\n \"StartupSleepProcName\": {\n \"Type\": \"str\",\n \"Default\": \"\",\n \"Desc\": [\n \"Contrains the startup sleep to only apply to processes that match this name.\"\n ]\n },\n \"MonoHacks\": {\n \"Type\": \"bool\",\n \"Default\": \"true\",\n \"Desc\": [\n \"Permits a hook-based SMC approach and smaller JIT blocks when mono is detected.\"\n ]\n }\n },\n \"Misc\": {\n \"ServerSocketPath\": {\n \"Type\": \"str\",\n \"Default\": \"\",\n \"Desc\": [\n \"Override for a FEXServer socket path. Only useful for chroots.\"\n ]\n },\n \"NeedsSeccomp\": {\n \"Type\": \"bool\",\n \"Default\": \"false\",\n \"Desc\": [\n \"Disables inline syscalls in order to support seccomp handling\"\n ]\n },\n \"ExtendedVolatileMetadata\": {\n \"Type\": \"str\",\n \"Default\": \"\",\n \"Desc\": [\n \"Configuration provided volatile metadata. Only implemented for WoW64/arm64ec.\",\n \"Limited in its use but can be handy.\",\n \"Extends on top of what Microsoft has for volatile metadata, but also supported for WoW64.\",\n \"Colon delimited modules, then semi-colon delimited instructions, then comma delimited ranges\",\n \"Default disables TSO in the module, unless instructions overlap the range\",\n \";-,...;,...:\",\n \"examples:\",\n \" * Disable TSO for a full module: Just provide the module name:\",\n \" `hl2_linux`\",\n \" * Disable TSO for a part of the module:\",\n \" `hl2_linux;-`\",\n \" * Disable TSO for a part of the module, but enable TSO for some instructions within the module\",\n \" `hl2_linux;-;,`\",\n \" * Disable TSO for multiple modules\",\n \" `hl2_linux:libsdl2.so`\"\n ]\n }\n }\n },\n \"UnnamedOptions\": {\n \"Misc\": {\n \"INTERPRETER_INSTALLED\": {\n \"Type\": \"bool\",\n \"Default\": \"false\"\n },\n \"APP_FILENAME\": {\n \"Type\": \"str\",\n \"Default\": \"\"\n },\n \"APP_CONFIG_NAME\": {\n \"Type\": \"str\",\n \"Default\": \"\",\n \"Desc\": [\n \"This is the application config name that has been loaded.\",\n \"This differs from APP_FILENAME in two ways\",\n \"Where APP_FILENAME always points to the executable path that FEX-Emu is executing.\",\n \"This matches what is used to load the AppLayer configuration name.\",\n \"When running through a compatibility layer like wine, this will only be the exe name, instead of wine full path.\"\n ]\n },\n \"IS64BIT_MODE\": {\n \"Type\": \"bool\",\n \"Default\": \"false\"\n },\n \"DISABLE_VIXL_INDIRECT_RUNTIME_CALLS\": {\n \"Type\": \"bool\",\n \"Default\": \"true\",\n \"Desc\": [\n \"This option is used for the InstructionCountCI so it can generate the same codegen between Arm64 hosts and vixl simulator hosts.\",\n \"Vixl simulator indirect runtime calls are a special hlt instruction with metadata after it. Effectively making a custom call instruction.\",\n \"With visual simulator calls disabled, the code generation would be the same as on a native Arm64 host, but running the code is broken.\"\n ]\n }\n }\n }\n}\n\n" }, { "path": "FEXCore/Source/Interface/Context/Context.cpp", "content": "// SPDX-License-Identifier: MIT\n#include \"Interface/Context/Context.h\"\n#include \"Interface/Core/OpcodeDispatcher.h\"\n#include \"Interface/Core/Dispatcher/Dispatcher.h\"\n#include \"Interface/Core/X86Tables/X86Tables.h\"\n\n#include \n#include \n#include \n#include \n#include \n#include \n\n#include \n#include \"FEXCore/Debug/InternalThreadState.h\"\n\nnamespace FEXCore::Context {\nfextl::unique_ptr FEXCore::Context::Context::CreateNewContext(const FEXCore::HostFeatures& Features) {\n return fextl::make_unique(Features);\n}\n\nvoid FEXCore::Context::ContextImpl::CompileRIP(FEXCore::Core::InternalThreadState* Thread, uint64_t GuestRIP) {\n CompileBlock(Thread->CurrentFrame, GuestRIP);\n}\n\nvoid FEXCore::Context::ContextImpl::CompileRIPCount(FEXCore::Core::InternalThreadState* Thread, uint64_t GuestRIP, uint64_t MaxInst) {\n CompileBlock(Thread->CurrentFrame, GuestRIP, MaxInst);\n}\n\nvoid FEXCore::Context::ContextImpl::SetSignalDelegator(FEXCore::SignalDelegator* _SignalDelegation) {\n SignalDelegation = _SignalDelegation;\n}\n\nvoid FEXCore::Context::ContextImpl::SetSyscallHandler(FEXCore::HLE::SyscallHandler* Handler) {\n SyscallHandler = Handler;\n SourcecodeResolver = Handler->GetSourcecodeResolver();\n}\n\nvoid FEXCore::Context::ContextImpl::SetThunkHandler(FEXCore::ThunkHandler* Handler) {\n ThunkHandler = Handler;\n}\n\nFEXCore::CPUID::FunctionResults FEXCore::Context::ContextImpl::RunCPUIDFunction(uint32_t Function, uint32_t Leaf) {\n return CPUID.RunFunction(Function, Leaf);\n}\n\nFEXCore::CPUID::XCRResults FEXCore::Context::ContextImpl::RunXCRFunction(uint32_t Function) {\n return CPUID.RunXCRFunction(Function);\n}\n\nFEXCore::CPUID::FunctionResults FEXCore::Context::ContextImpl::RunCPUIDFunctionName(uint32_t Function, uint32_t Leaf, uint32_t CPU) {\n return CPUID.RunFunctionName(Function, Leaf, CPU);\n}\n\nbool FEXCore::Context::ContextImpl::IsAddressInCodeBuffer(FEXCore::Core::InternalThreadState* Thread, uintptr_t Address) const {\n return Thread->CPUBackend->IsAddressInCodeBuffer(Address);\n}\n} // namespace FEXCore::Context\n" }, { "path": "FEXCore/Source/Interface/Context/Context.h", "content": "// SPDX-License-Identifier: MIT\n#pragma once\n\n#include \"Common/JitSymbols.h\"\n#include \"Interface/Core/CPUBackend.h\"\n#include \"Interface/Core/CPUID.h\"\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n#include \n#include \n#include \n#include \n#include \n#include \n\nnamespace FEXCore {\nclass SignalDelegator;\nclass ThunkHandler;\nstruct LookupCacheWriteLockToken;\n\nnamespace Core {\n struct DebugData;\n struct InternalThreadState;\n} // namespace Core\n\nnamespace CPU {\n class Dispatcher;\n} // namespace CPU\n\nnamespace HLE {\n class SourcecodeResolver;\n class SyscallHandler;\n} // namespace HLE\n} // namespace FEXCore\n\nnamespace FEXCore::Context {\nstruct FEX_PACKED ExitFunctionLinkData {\n uint64_t HostCode;\n uint64_t GuestRIP;\n int64_t CallerOffset;\n};\n\nstruct CustomIRResult {\n void* Creator;\n void* Data;\n\n CustomIRResult(void* Creator, void* Data)\n : Creator(Creator)\n , Data(Data) {}\n};\n\nusing BlockDelinkerFunc = void (*)(FEXCore::Context::ExitFunctionLinkData* Record);\nconstexpr uint32_t TSC_SCALE_MAXIMUM = 1'000'000'000; ///< 1Ghz\n\nclass CodeCache : public AbstractCodeCache {\npublic:\n CodeCache(ContextImpl&);\n ~CodeCache();\n\n ContextImpl& CTX;\n fextl::unique_ptr ValidationCTX;\n fextl::unique_ptr ValidationThread;\n FEXCore::Core::CPUState::gdt_segment ValidationGDT[32] {};\n bool IsGeneratingCache = false;\n\n FEX_CONFIG_OPT(EnableCodeCaching, ENABLECODECACHINGWIP);\n FEX_CONFIG_OPT(EnableCodeCacheValidation, ENABLECODECACHEVALIDATION);\n\n uint64_t ComputeCodeMapId(std::string_view Filename, int FD) override;\n bool SaveData(Core::InternalThreadState&, int TargetFD, const ExecutableFileSectionInfo&, uint64_t SerializedBaseAddress) override;\n bool LoadData(Core::InternalThreadState*, std::byte* MappedCacheFile, const ExecutableFileSectionInfo&) override;\n\n /**\n * Performs expensive extra validation on the loaded code cache data.\n *\n * This kicks off an in-process recompile of all cached blocks and compares\n * them with the cached data. Differences will be reported as fatal errors,\n * which can uncover bugs like for example:\n * - mismatches of the JIT configuration used during cache generation\n * - hidden position dependencies due to missing FEX relocations\n * - incorrect instruction padding\n */\n void Validate(const ExecutableFileSectionInfo&, fextl::set GuestBlocks, const fextl::set& HostBlocks,\n std::span CachedCode);\n\n void InitiateCacheGeneration() override {\n IsGeneratingCache = true;\n }\n\n /**\n * Applies a set of FEX relocations to the given code section.\n *\n * FEX relocations describe runtime-dependencies of FEX-generated code.\n * When loading a code cache, they are used to move cached code to the\n * dynamically chosen base address of the guest binary.\n *\n * Conversely, relocations are applied in reverse when writing code caches\n * to ensure consistency across generation runs.\n *\n * Note that FEX relocations are unrelated to ELF/PE relocations.\n *\n * @param GuestDelta Guest address offset to apply to RIP-relative data\n * @param ForStorage True for serializing data (producing deterministic output); false for de-serializing it (resolving dynamic symbols)\n *\n * @return Returns true on success\n */\n [[nodiscard]]\n bool ApplyCodeRelocations(uint64_t GuestDelta, std::span Code, std::span Relocations, bool ForStorage);\n};\n\nclass ContextImpl final : public FEXCore::Context::Context, public CPU::CodeBufferManager {\npublic:\n // Context base class implementation.\n bool InitCore() override;\n\n void ExecuteThread(FEXCore::Core::InternalThreadState* Thread) override;\n\n bool CheckIfBlockIsCacheable(FEXCore::Core::InternalThreadState&, uint64_t GuestRIP, uint64_t MaxInst) override;\n void CompileRIP(FEXCore::Core::InternalThreadState* Thread, uint64_t GuestRIP) override;\n void CompileRIPCount(FEXCore::Core::InternalThreadState* Thread, uint64_t GuestRIP, uint64_t MaxInst) override;\n\n void HandleCallback(FEXCore::Core::InternalThreadState* Thread, uint64_t RIP) override;\n\n bool IsAddressInCurrentBlock(FEXCore::Core::InternalThreadState* Thread, uint64_t Address, uint64_t Size) override;\n bool IsCurrentBlockSingleInst(FEXCore::Core::InternalThreadState* Thread) override;\n uint64_t GetGuestBlockEntry(FEXCore::Core::InternalThreadState* Thread) override;\n\n uint64_t RestoreRIPFromHostPC(FEXCore::Core::InternalThreadState* Thread, uint64_t HostPC) override;\n uint32_t ReconstructCompactedEFLAGS(FEXCore::Core::InternalThreadState* Thread, bool WasInJIT, const uint64_t* HostGPRs, uint64_t PSTATE) override;\n void SetFlagsFromCompactedEFLAGS(FEXCore::Core::InternalThreadState* Thread, uint32_t EFLAGS) override;\n\n void ReconstructXMMRegisters(const FEXCore::Core::InternalThreadState* Thread, __uint128_t* XMM_Low, __uint128_t* YMM_High) override;\n void SetXMMRegistersFromState(FEXCore::Core::InternalThreadState* Thread, const __uint128_t* XMM_Low, const __uint128_t* YMM_High) override;\n\n /**\n * @brief Used to create FEX thread objects in preparation for creating a true OS thread. Does set a TID or PID.\n *\n * @param InitialRIP The starting RIP of this thread\n * @param StackPointer The starting RSP of this thread\n * @param NewThreadState The initial thread state to setup for our state, if inheriting.\n *\n * @return The InternalThreadState object that tracks all of the emulated thread's state\n *\n * Usecases:\n * Parent thread Creation:\n * - Thread = CreateThread(InitialRIP, InitialStack, nullptr, 0);\n * - CTX->ExecuteThread(Thread);\n * OS thread Creation:\n * - Thread = CreateThread(0, 0, NewState, PPID);\n * - Thread->ExecutionThread = FEXCore::Threads::Thread::Create(ThreadHandler, Arg);\n * - ThreadHandler calls `CTX->ExecuteThread(Thread)`\n * OS fork (New thread created with a clone of thread state):\n * - clone{2, 3}\n * - Thread = CreateThread(0, 0, CopyOfThreadState, PPID);\n * - ExecuteThread(Thread); // Starts executing without creating another host thread\n * Thunk callback executing guest code from native host thread\n * - Thread = CreateThread(0, 0, NewState, PPID);\n * - HandleCallback(Thread, RIP);\n */\n\n FEXCore::Core::InternalThreadState* CreateThread(uint64_t InitialRIP, uint64_t StackPointer, const FEXCore::Core::CPUState* NewThreadState) override;\n\n /**\n * @brief Destroys this FEX thread object and stops tracking it internally\n *\n * @param Thread The internal FEX thread state object\n */\n void DestroyThread(FEXCore::Core::InternalThreadState* Thread) override;\n\n#ifndef _WIN32\n void LockBeforeFork(FEXCore::Core::InternalThreadState* Thread) override;\n void UnlockAfterFork(FEXCore::Core::InternalThreadState* Thread, bool Child) override;\n#endif\n void SetSignalDelegator(FEXCore::SignalDelegator* SignalDelegation) override;\n void SetSyscallHandler(FEXCore::HLE::SyscallHandler* Handler) override;\n void SetThunkHandler(FEXCore::ThunkHandler* Handler) override;\n\n FEXCore::CPUID::FunctionResults RunCPUIDFunction(uint32_t Function, uint32_t Leaf) override;\n FEXCore::CPUID::XCRResults RunXCRFunction(uint32_t Function) override;\n FEXCore::CPUID::FunctionResults RunCPUIDFunctionName(uint32_t Function, uint32_t Leaf, uint32_t CPU) override;\n\n CodeCache& GetCodeCache() override {\n return CodeCache;\n }\n\n void SetCodeMapWriter(fextl::unique_ptr Writer) override {\n CodeMapWriter = std::move(Writer);\n }\n\n void FlushAndCloseCodeMap() override {\n if (CodeMapWriter) {\n CodeMapWriter.reset();\n }\n }\n\n void OnCodeBufferAllocated(const std::shared_ptr&) override;\n void ClearCodeCache(FEXCore::Core::InternalThreadState* Thread, bool NewCodeBuffer = true) override;\n void InvalidateCodeBuffersCodeRange(uint64_t Start, uint64_t Length) override;\n void InvalidateThreadCachedCodeRange(FEXCore::Core::InternalThreadState* Thread, uint64_t Start, uint64_t Length) override;\n FEXCore::ForkableSharedMutex& GetCodeInvalidationMutex() override {\n return CodeInvalidationMutex;\n }\n\n void ConfigureAOTGen(FEXCore::Core::InternalThreadState* Thread, fextl::set* ExternalBranches, uint64_t SectionMaxAddress) override;\n\n bool IsAddressInCodeBuffer(FEXCore::Core::InternalThreadState* Thread, uintptr_t Address) const override;\n\n // returns false if a handler was already registered\n std::optional\n AddCustomIREntrypoint(uintptr_t Entrypoint, CustomIREntrypointHandler Handler, void* Creator = nullptr, void* Data = nullptr);\n\n void AddThunkTrampolineIRHandler(uintptr_t Entrypoint, uintptr_t GuestThunkEntrypoint) override;\n\n void AddForceTSOInformation(const IntervalList& ValidRanges, fextl::set&& Instructions) override;\n\n void RemoveForceTSOInformation(uint64_t Address, uint64_t Size) override;\n\n void MarkMonoDetected() override {\n MonoDetected = true;\n }\n\n void MarkMonoBackpatcherBlock(uint64_t BlockEntry) override;\n\npublic:\n struct {\n uint64_t VirtualMemSize {1ULL << 36};\n uint64_t TSCScale = 0;\n\n // Used if the JIT needs to have its interrupt fault code emitted.\n bool NeedsPendingInterruptFaultCheck {false};\n\n FEX_CONFIG_OPT(Multiblock, MULTIBLOCK);\n FEX_CONFIG_OPT(SingleStepConfig, SINGLESTEP);\n FEX_CONFIG_OPT(GdbServer, GDBSERVER);\n FEX_CONFIG_OPT(Is64BitMode, IS64BIT_MODE);\n FEX_CONFIG_OPT(TSOEnabled, TSOENABLED);\n FEX_CONFIG_OPT(VectorTSOEnabled, VECTORTSOENABLED);\n FEX_CONFIG_OPT(MemcpySetTSOEnabled, MEMCPYSETTSOENABLED);\n FEX_CONFIG_OPT(SMCChecks, SMCCHECKS);\n FEX_CONFIG_OPT(MaxInstPerBlock, MAXINST);\n FEX_CONFIG_OPT(RootFSPath, ROOTFS);\n FEX_CONFIG_OPT(GlobalJITNaming, GLOBALJITNAMING);\n FEX_CONFIG_OPT(LibraryJITNaming, LIBRARYJITNAMING);\n FEX_CONFIG_OPT(BlockJITNaming, BLOCKJITNAMING);\n FEX_CONFIG_OPT(GDBSymbols, GDBSYMBOLS);\n FEX_CONFIG_OPT(x87ReducedPrecision, X87REDUCEDPRECISION);\n FEX_CONFIG_OPT(DisableTelemetry, DISABLETELEMETRY);\n FEX_CONFIG_OPT(DisableVixlIndirectCalls, DISABLE_VIXL_INDIRECT_RUNTIME_CALLS);\n FEX_CONFIG_OPT(SmallTSCScale, SMALLTSCSCALE);\n FEX_CONFIG_OPT(StrictInProcessSplitLocks, STRICTINPROCESSSPLITLOCKS);\n FEX_CONFIG_OPT(MonoHacks, MONOHACKS);\n } Config;\n\n FEXCore::ForkableSharedMutex CodeInvalidationMutex;\n\n uint32_t StrictSplitLockMutex {};\n\n FEXCore::HostFeatures HostFeatures;\n // CPUID depends on HostFeatures so needs to be initialized after that.\n FEXCore::CPUIDEmu CPUID;\n FEXCore::HLE::SyscallHandler* SyscallHandler {};\n FEXCore::HLE::SourcecodeResolver* SourcecodeResolver {};\n FEXCore::ThunkHandler* ThunkHandler {};\n fextl::unique_ptr Dispatcher;\n CodeCache CodeCache;\n fextl::unique_ptr CodeMapWriter;\n\n SignalDelegator* SignalDelegation {};\n\n ContextImpl(const FEXCore::HostFeatures& Features);\n\n static void ThreadRemoveCodeEntryFromJit(FEXCore::Core::CpuStateFrame* Frame, uint64_t GuestRIP);\n\n // This is used as a replacement for the SMC writes in the mono callsite backpatcher that avoids atomic operations\n // (safe as the invalidation mutex is locked) and manually invalidates the modified range. Allowing SMC to be detected\n // even if faulting is disabled.\n static void MonoBackpatcherWrite(FEXCore::Core::CpuStateFrame* Frame, uint8_t Size, uint64_t Address, uint64_t Value);\n\n void RemoveCustomIREntrypoint(FEXCore::Core::InternalThreadState* Thread, uintptr_t Entrypoint);\n\n struct GenerateIRResult {\n std::optional IRView;\n uint64_t TotalInstructions;\n uint64_t TotalInstructionsLength;\n uint64_t StartAddr;\n uint64_t Length;\n bool NeedsAddGuestCodeRanges;\n };\n [[nodiscard]]\n GenerateIRResult GenerateIR(FEXCore::Core::InternalThreadState* Thread, uint64_t GuestRIP, bool ExtendedDebugInfo, uint64_t MaxInst);\n\n struct CompileCodeResult {\n CPU::CPUBackend::CompiledCode CompiledCode;\n fextl::unique_ptr DebugData;\n uint64_t StartAddr;\n uint64_t Length;\n bool NeedsAddGuestCodeRanges;\n };\n [[nodiscard]]\n CompileCodeResult CompileCode(FEXCore::Core::InternalThreadState* Thread, uint64_t GuestRIP, uint64_t MaxInst = 0);\n uintptr_t CompileBlock(FEXCore::Core::CpuStateFrame* Frame, uint64_t GuestRIP, uint64_t MaxInst = 0);\n uintptr_t CompileSingleStep(FEXCore::Core::CpuStateFrame* Frame, uint64_t GuestRIP);\n\n FEXCore::JITSymbols Symbols;\n\n FEXCore::Utils::PooledAllocatorVirtual OpDispatcherAllocator {\"FEXMem_OpDispatcher\"};\n FEXCore::Utils::PooledAllocatorVirtual FrontendAllocator {\"FEXMem_Frontend\"};\n FEXCore::Utils::PooledAllocatorVirtualWithGuard CPUBackendAllocator {\"FEXMem_CPUBackend\"};\n\n // If Atomic-based TSO emulation is enabled or not.\n bool IsAtomicTSOEnabled() const {\n return AtomicTSOEmulationEnabled;\n }\n\n // If atomic-based TSO emulation is enabled for vector operations.\n bool IsVectorAtomicTSOEnabled() const {\n return VectorAtomicTSOEmulationEnabled;\n }\n\n // If atomic-based TSO emulation is enabled for memcpy operations.\n bool IsMemcpyAtomicTSOEnabled() const {\n return MemcpyAtomicTSOEmulationEnabled;\n }\n\n void SetHardwareTSOSupport(bool HardwareTSOSupported) override {\n SupportsHardwareTSO = HardwareTSOSupported;\n UpdateAtomicTSOEmulationConfig();\n }\n\n void EnableExitOnHLT() override {\n ExitOnHLT = true;\n }\n\n bool ExitOnHLTEnabled() const {\n return ExitOnHLT;\n }\n\n bool AreMonoHacksActive() const {\n return Config.MonoHacks && MonoDetected;\n }\n\nprotected:\n void UpdateAtomicTSOEmulationConfig() {\n if (SupportsHardwareTSO) {\n // If the hardware supports TSO then we don't need to emulate it through atomics.\n AtomicTSOEmulationEnabled = false;\n VectorAtomicTSOEmulationEnabled = false;\n MemcpyAtomicTSOEmulationEnabled = false;\n } else {\n AtomicTSOEmulationEnabled = Config.TSOEnabled;\n VectorAtomicTSOEmulationEnabled = Config.TSOEnabled && Config.VectorTSOEnabled;\n MemcpyAtomicTSOEmulationEnabled = Config.TSOEnabled && Config.MemcpySetTSOEnabled;\n }\n }\n\nprivate:\n /**\n * @brief Initializes the JIT compilers for the thread\n *\n * @param State The internal FEX thread state object\n *\n * InitializeCompiler is called inside of CreateThread, so you likely don't need this\n */\n void InitializeCompiler(FEXCore::Core::InternalThreadState* Thread);\n\n bool SupportsHardwareTSO = false;\n bool AtomicTSOEmulationEnabled = true;\n bool VectorAtomicTSOEmulationEnabled = false;\n bool MemcpyAtomicTSOEmulationEnabled = false;\n\n bool ExitOnHLT = false;\n FEX_CONFIG_OPT(AppFilename, APP_FILENAME);\n\n std::shared_mutex CustomIRMutex;\n std::atomic HasCustomIRHandlers {};\n struct CustomIRHandlerEntry final {\n CustomIREntrypointHandler Handler;\n void* Creator;\n void* Data;\n };\n fextl::unordered_map CustomIRHandlers;\n IntervalList ForceTSOValidRanges; // The ranges for which ForceTSOInstructions has populated data\n fextl::set ForceTSOInstructions;\n\n bool MonoDetected = false;\n std::atomic MonoBackpatcherBlock;\n\n std::mutex CodeBufferListLock;\n fextl::vector> CodeBufferList;\n};\n} // namespace FEXCore::Context\n" }, { "path": "FEXCore/Source/Interface/Core/Addressing.cpp", "content": "// SPDX-License-Identifier: MIT\n#include \"Interface/Core/Addressing.h\"\n\n#include \"Interface/IR/IREmitter.h\"\n#include \"FEXCore/Utils/MathUtils.h\"\n#include \"Interface/IR/IR.h\"\n\nnamespace FEXCore::IR {\n\nRef LoadEffectiveAddress(IREmitter* IREmit, const AddressMode& A, IR::OpSize GPRSize, bool AddSegmentBase, bool AllowUpperGarbage) {\n Ref Tmp = A.Base;\n\n if (A.Offset) {\n Tmp = Tmp ? IREmit->Add(GPRSize, Tmp, A.Offset) : IREmit->Constant(A.Offset);\n }\n\n if (A.Index) {\n if (A.IndexScale != 1) {\n uint32_t Log2 = FEXCore::ilog2(A.IndexScale);\n\n if (Tmp) {\n Tmp = IREmit->_AddShift(GPRSize, Tmp, A.Index, ShiftType::LSL, Log2);\n } else {\n Tmp = IREmit->_Lshl(GPRSize, A.Index, IREmit->Constant(Log2));\n }\n } else {\n Tmp = Tmp ? IREmit->Add(GPRSize, Tmp, A.Index) : A.Index;\n }\n }\n\n // For 64-bit AddrSize can be 32-bit or 64-bit\n // For 32-bit AddrSize can be 32-bit or 16-bit\n //\n // If the AddrSize is not the GPRSize then we need to clear the upper bits.\n if ((A.AddrSize < GPRSize) && !AllowUpperGarbage && Tmp) {\n uint32_t Bits = IR::OpSizeAsBits(A.AddrSize);\n\n if (A.Base || A.Index) {\n Tmp = IREmit->_Bfe(GPRSize, Bits, 0, Tmp);\n } else if (A.Offset) {\n uint64_t X = A.Offset;\n X &= (1ull << Bits) - 1;\n Tmp = IREmit->Constant(X);\n }\n }\n\n if (A.Segment && AddSegmentBase) {\n Tmp = Tmp ? IREmit->Add(GPRSize, Tmp, A.Segment) : A.Segment;\n }\n\n return Tmp ?: IREmit->Constant(0);\n}\n\nAddressMode SelectAddressMode(IREmitter* IREmit, const AddressMode& A, IR::OpSize GPRSize, bool HostSupportsTSOImm9, bool AtomicTSO,\n bool Vector, IR::OpSize AccessSize) {\n const auto Is32Bit = GPRSize == OpSize::i32Bit;\n const auto GPRSizeMatchesAddrSize = A.AddrSize == GPRSize;\n const auto OffsetIndexToLargeFor32Bit = Is32Bit && (A.Offset <= -16384 || A.Offset >= 16384);\n if (!GPRSizeMatchesAddrSize || OffsetIndexToLargeFor32Bit) {\n // If address size doesn't match GPR size then no optimizations can occur.\n return {\n .Base = LoadEffectiveAddress(IREmit, A, GPRSize, true),\n .Index = IREmit->Invalid(),\n };\n }\n\n // Loadstore rules:\n // Non-TSO GPR:\n // * LDR/STR: [Reg]\n // * LDR/STR: [Reg + Reg, {Shift }]\n // * Can't use with 32-bit\n // * LDR/STR: [Reg + [0,4095] * ]\n // * Imm must be smaller than 16k with 32-bit\n // * LDUR/STUR: [Reg + [-256, 255]]\n //\n // TSO GPR:\n // * ARMv8.0:\n // LDAR/STLR: [Reg]\n // * FEAT_LRCPC:\n // LDAPR: [Reg]\n // * FEAT_LRCPC2:\n // LDAPUR/STLUR: [Reg + [-256, 255]]\n //\n // Non-TSO Vector:\n // * LDR/STR: [Reg + [0,4095] * ]\n // * LDUR/STUR: [Reg + [-256,255]]\n //\n // TSO Vector:\n // * ARMv8.0:\n // Just DMB + previous\n // * FEAT_LRCPC3 (Unsupported by FEXCore currently):\n // LDAPUR/STLUR: [Reg + [-256,255]]\n\n const auto AccessSizeAsImm = OpSizeToSize(AccessSize);\n const bool OffsetIsSIMM9 = A.Offset && A.Offset >= -256 && A.Offset <= 255;\n const bool OffsetIsUnsignedScaled = A.Offset > 0 && (A.Offset & (AccessSizeAsImm - 1)) == 0 && (A.Offset / AccessSizeAsImm) <= 4095;\n\n if ((AtomicTSO && !Vector && HostSupportsTSOImm9 && OffsetIsSIMM9) || (!AtomicTSO && (OffsetIsSIMM9 || OffsetIsUnsignedScaled))) {\n // Peel off the offset\n AddressMode B = A;\n B.Offset = 0;\n\n return {\n .Base = LoadEffectiveAddress(IREmit, B, GPRSize, true /* AddSegmentBase */, false),\n .Index = IREmit->Constant(A.Offset),\n .IndexType = MemOffsetType::SXTX,\n .IndexScale = 1,\n };\n }\n\n if (AtomicTSO) {\n // TODO: LRCPC3 support for vector Imm9.\n } else if (!Is32Bit && A.Base && (A.Index || A.Segment) && !A.Offset && (A.IndexScale == 1 || A.IndexScale == AccessSizeAsImm)) {\n AddressMode B = A;\n\n // ScaledRegisterLoadstore\n if (B.Index && B.Segment) {\n B.Base = IREmit->Add(GPRSize, B.Base, B.Segment);\n } else if (B.Segment) {\n B.Index = B.Segment;\n B.IndexScale = 1;\n }\n\n return B;\n }\n\n if (Vector || !AtomicTSO) {\n if ((A.Base || A.Segment) && A.Offset) {\n const bool Const_16K = A.Offset > -16384 && A.Offset < 16384 && GPRSizeMatchesAddrSize && Is32Bit;\n\n if (!Is32Bit || Const_16K) {\n // Peel off the offset\n AddressMode B = A;\n B.Offset = 0;\n\n return {\n .Base = LoadEffectiveAddress(IREmit, B, GPRSize, true /* AddSegmentBase */, false),\n .Index = IREmit->Constant(A.Offset),\n .IndexType = MemOffsetType::SXTX,\n .IndexScale = 1,\n };\n }\n }\n }\n\n // Fallback on software address calculation\n return {\n .Base = LoadEffectiveAddress(IREmit, A, GPRSize, true),\n .Index = IREmit->Invalid(),\n };\n}\n\n\n}; // namespace FEXCore::IR\n" }, { "path": "FEXCore/Source/Interface/Core/Addressing.h", "content": "// SPDX-License-Identifier: MIT\n#pragma once\n\n#include \"Interface/IR/IR.h\"\n#include \n\nnamespace FEXCore::IR {\nclass IREmitter;\n\nstruct AddressMode {\n Ref Segment {nullptr};\n Ref Base {nullptr};\n Ref Index {nullptr};\n int64_t Offset = 0;\n\n MemOffsetType IndexType = MemOffsetType::SXTX;\n uint8_t IndexScale = 1;\n\n // Size in bytes for the address calculation. 8 for an arm64 hardware mode.\n IR::OpSize AddrSize;\n bool NonTSO;\n};\n\nRef LoadEffectiveAddress(IREmitter* IREmit, const AddressMode& A, IR::OpSize GPRSize, bool AddSegmentBase, bool AllowUpperGarbage = false);\nAddressMode SelectAddressMode(IREmitter* IREmit, const AddressMode& A, IR::OpSize GPRSize, bool HostSupportsTSOImm9, bool AtomicTSO,\n bool Vector, IR::OpSize AccessSize);\n\n} // namespace FEXCore::IR" }, { "path": "FEXCore/Source/Interface/Core/ArchHelpers/Arm64Emitter.cpp", "content": "// SPDX-License-Identifier: MIT\n#include \"Interface/Core/ArchHelpers/Arm64Emitter.h\"\n#include \"Interface/Core/Dispatcher/Dispatcher.h\"\n#include \"Interface/Context/Context.h\"\n\n#include \n#include \n#include \n#include \n\n#include \n#include \n#include \n\n#ifdef VIXL_DISASSEMBLER\n#include \n#include \n#include \n#include \n#endif\n\n#include \n#include \n#include \n\nnamespace FEXCore::CPU {\n\n// LLVM's preserve_all doc, this is used throughout this file and reproduced\n// here for reference:\n//\n// the callee preserve all general purpose registers,\n// except X0-X8 and X16-X18. Furthermore it also preserves lower 128 bits of\n// V8-V31 SIMD - floating point registers.\n//\n// Note that the call necessarily also clobbers x30, the link register (LR)\n// which is not considered general purpose.\n//\n// Meanwhile, for non-preserve_all, the AAPCS64 ABI says:\n//\n// A subroutine invocation must preserve the contents of the registers\n// r19-r29 and SP.\n\nnamespace x64 {\n#ifndef ARCHITECTURE_arm64ec\n // All but x19 and x29 are caller saved\n // Note that rax/rdx are rearranged here so we can coalesce cmpxchg.\n constexpr std::array SRA = {\n ARMEmitter::Reg::r4,\n ARMEmitter::Reg::r7,\n ARMEmitter::Reg::r5,\n ARMEmitter::Reg::r6,\n ARMEmitter::Reg::r8,\n ARMEmitter::Reg::r9,\n ARMEmitter::Reg::r10,\n ARMEmitter::Reg::r11,\n ARMEmitter::Reg::r12,\n ARMEmitter::Reg::r13,\n ARMEmitter::Reg::r14,\n ARMEmitter::Reg::r15,\n ARMEmitter::Reg::r16,\n ARMEmitter::Reg::r17,\n ARMEmitter::Reg::r19,\n ARMEmitter::Reg::r29,\n // PF/AF must be last.\n REG_PF,\n REG_AF,\n };\n\n // I wish this could get constexpr generated from SRA's definition but impossible until libstdc++12, libc++15.\n // SRA GPRs that need to be spilled when calling a function with `preserve_all` ABI.\n constexpr std::array PreserveAll_SRA = {\n ARMEmitter::Reg::r4, ARMEmitter::Reg::r5, ARMEmitter::Reg::r6, ARMEmitter::Reg::r7,\n ARMEmitter::Reg::r8, ARMEmitter::Reg::r16, ARMEmitter::Reg::r17,\n };\n\n constexpr std::array RA = {\n // All these callee saved\n ARMEmitter::Reg::r20, ARMEmitter::Reg::r21, ARMEmitter::Reg::r22, ARMEmitter::Reg::r23,\n ARMEmitter::Reg::r24, ARMEmitter::Reg::r30, ARMEmitter::Reg::r18,\n };\n\n constexpr unsigned RAPairs = 4;\n\n // Dynamic GPRs\n constexpr std::array PreserveAll_Dynamic = {\n ARMEmitter::Reg::r18,\n ARMEmitter::Reg::r30,\n };\n\n constexpr std::array NotPreserved_Dynamic = PreserveAll_Dynamic;\n\n // All are caller saved\n constexpr std::array SRAFPR = {\n ARMEmitter::VReg::v16, ARMEmitter::VReg::v17, ARMEmitter::VReg::v18, ARMEmitter::VReg::v19,\n ARMEmitter::VReg::v20, ARMEmitter::VReg::v21, ARMEmitter::VReg::v22, ARMEmitter::VReg::v23,\n ARMEmitter::VReg::v24, ARMEmitter::VReg::v25, ARMEmitter::VReg::v26, ARMEmitter::VReg::v27,\n ARMEmitter::VReg::v28, ARMEmitter::VReg::v29, ARMEmitter::VReg::v30, ARMEmitter::VReg::v31};\n\n // SRA FPRs that need to be spilled when calling a function with `preserve_all` ABI.\n constexpr std::array PreserveAll_SRAFPR = {\n // None.\n };\n\n // v8..v15 = (lower 64bits) Callee saved\n constexpr std::array RAFPR = {\n // v0 ~ v1 are used as temps.\n // ARMEmitter::VReg::v0, ARMEmitter::VReg::v1,\n\n ARMEmitter::VReg::v2, ARMEmitter::VReg::v3, ARMEmitter::VReg::v4, ARMEmitter::VReg::v5, ARMEmitter::VReg::v6,\n ARMEmitter::VReg::v7, ARMEmitter::VReg::v8, ARMEmitter::VReg::v9, ARMEmitter::VReg::v10, ARMEmitter::VReg::v11,\n ARMEmitter::VReg::v12, ARMEmitter::VReg::v13, ARMEmitter::VReg::v14, ARMEmitter::VReg::v15,\n };\n\n constexpr std::array PreserveAll_DynamicFPR = {\n ARMEmitter::VReg::v2, ARMEmitter::VReg::v3, ARMEmitter::VReg::v4, ARMEmitter::VReg::v5, ARMEmitter::VReg::v6, ARMEmitter::VReg::v7,\n };\n#else\n constexpr std::array SRA = {\n ARMEmitter::Reg::r8,\n ARMEmitter::Reg::r0,\n ARMEmitter::Reg::r1,\n ARMEmitter::Reg::r27,\n // SP's register location isn't specified by the ARM64EC ABI, we choose to use r23\n ARMEmitter::Reg::r23,\n ARMEmitter::Reg::r29,\n ARMEmitter::Reg::r25,\n ARMEmitter::Reg::r26,\n ARMEmitter::Reg::r2,\n ARMEmitter::Reg::r3,\n ARMEmitter::Reg::r4,\n ARMEmitter::Reg::r5,\n ARMEmitter::Reg::r19,\n ARMEmitter::Reg::r20,\n ARMEmitter::Reg::r21,\n ARMEmitter::Reg::r22,\n // PF/AF must be last.\n REG_PF,\n REG_AF,\n };\n\n constexpr std::array PreserveAll_SRA = {\n ARMEmitter::Reg::r0, ARMEmitter::Reg::r1, ARMEmitter::Reg::r2, ARMEmitter::Reg::r3,\n ARMEmitter::Reg::r4, ARMEmitter::Reg::r5, ARMEmitter::Reg::r8,\n };\n\n constexpr std::array RA = {\n ARMEmitter::Reg::r6, ARMEmitter::Reg::r7, ARMEmitter::Reg::r14, ARMEmitter::Reg::r15, ARMEmitter::Reg::r16, ARMEmitter::Reg::r30,\n };\n\n constexpr std::array PreserveAll_Dynamic = {ARMEmitter::Reg::r6, ARMEmitter::Reg::r7, ARMEmitter::Reg::r16,\n ARMEmitter::Reg::r17, ARMEmitter::Reg::r30};\n\n constexpr std::array NotPreserved_Dynamic = {ARMEmitter::Reg::r6, ARMEmitter::Reg::r7, ARMEmitter::Reg::r14,\n ARMEmitter::Reg::r15, ARMEmitter::Reg::r16, ARMEmitter::Reg::r17,\n ARMEmitter::Reg::r30};\n\n constexpr unsigned RAPairs = 4;\n\n constexpr std::array SRAFPR = {\n ARMEmitter::VReg::v0, ARMEmitter::VReg::v1, ARMEmitter::VReg::v2, ARMEmitter::VReg::v3,\n ARMEmitter::VReg::v4, ARMEmitter::VReg::v5, ARMEmitter::VReg::v6, ARMEmitter::VReg::v7,\n ARMEmitter::VReg::v8, ARMEmitter::VReg::v9, ARMEmitter::VReg::v10, ARMEmitter::VReg::v11,\n ARMEmitter::VReg::v12, ARMEmitter::VReg::v13, ARMEmitter::VReg::v14, ARMEmitter::VReg::v15,\n };\n\n constexpr std::array PreserveAll_SRAFPR = {\n ARMEmitter::VReg::v0, ARMEmitter::VReg::v1, ARMEmitter::VReg::v2, ARMEmitter::VReg::v3,\n ARMEmitter::VReg::v4, ARMEmitter::VReg::v5, ARMEmitter::VReg::v6, ARMEmitter::VReg::v7,\n };\n\n constexpr std::array RAFPR = {\n ARMEmitter::VReg::v18, ARMEmitter::VReg::v19, ARMEmitter::VReg::v20, ARMEmitter::VReg::v21, ARMEmitter::VReg::v22,\n ARMEmitter::VReg::v23, ARMEmitter::VReg::v24, ARMEmitter::VReg::v25, ARMEmitter::VReg::v26, ARMEmitter::VReg::v27,\n ARMEmitter::VReg::v28, ARMEmitter::VReg::v29, ARMEmitter::VReg::v30, ARMEmitter::VReg::v31};\n\n constexpr std::array PreserveAll_DynamicFPR = {\n // None\n };\n#endif\n\n constexpr uint32_t PreserveAll_SRAMask = {[]() -> uint32_t {\n uint32_t Mask {};\n for (auto Reg : PreserveAll_SRA) {\n switch (Reg.Idx()) {\n case 0:\n case 1:\n case 2:\n case 3:\n case 4:\n case 5:\n case 6:\n case 7:\n case 8:\n case 16:\n case 17: Mask |= (1U << Reg.Idx()); break;\n default: break;\n }\n }\n\n return Mask;\n }()};\n\n constexpr uint32_t PreserveAll_SRAFPRMask = {[]() -> uint32_t {\n uint32_t Mask {};\n for (auto Reg : PreserveAll_SRAFPR) {\n Mask |= (1U << Reg.Idx());\n }\n return Mask;\n }()};\n\n // SRA FPRs that need to be spilled when the host supports SVE-256bit with `preserve_all` ABI.\n // This is /all/ of the SRA registers\n constexpr std::array PreserveAll_SRAFPRSVE = SRAFPR;\n\n constexpr uint32_t PreserveAll_SRAFPRSVEMask = {[]() -> uint32_t {\n uint32_t Mask {};\n for (auto Reg : PreserveAll_SRAFPRSVE) {\n Mask |= (1U << Reg.Idx());\n }\n return Mask;\n }()};\n\n // Dynamic FPRs when the host supports SVE-256bit.\n constexpr std::array PreserveAll_DynamicFPRSVE = {\n // v0 ~ v1 are used as temps.\n ARMEmitter::VReg::v2, ARMEmitter::VReg::v3, ARMEmitter::VReg::v4, ARMEmitter::VReg::v5, ARMEmitter::VReg::v6,\n ARMEmitter::VReg::v7, ARMEmitter::VReg::v8, ARMEmitter::VReg::v9, ARMEmitter::VReg::v10, ARMEmitter::VReg::v11,\n ARMEmitter::VReg::v12, ARMEmitter::VReg::v13, ARMEmitter::VReg::v14, ARMEmitter::VReg::v15,\n };\n} // namespace x64\n\nnamespace x32 {\n // All but x19 and x29 are caller saved. eax/edx rearranged for cmpxchg.\n constexpr std::array SRA = {\n ARMEmitter::Reg::r4,\n ARMEmitter::Reg::r7,\n ARMEmitter::Reg::r5,\n ARMEmitter::Reg::r6,\n ARMEmitter::Reg::r8,\n ARMEmitter::Reg::r9,\n ARMEmitter::Reg::r10,\n ARMEmitter::Reg::r11,\n // PF/AF must be last.\n REG_PF,\n REG_AF,\n };\n\n constexpr std::array RA = {\n // All these callee saved\n ARMEmitter::Reg::r20,\n ARMEmitter::Reg::r21,\n ARMEmitter::Reg::r22,\n ARMEmitter::Reg::r23,\n\n // Registers only available on 32-bit\n // All these are caller saved (except for r19).\n ARMEmitter::Reg::r12,\n ARMEmitter::Reg::r13,\n ARMEmitter::Reg::r14,\n ARMEmitter::Reg::r15,\n ARMEmitter::Reg::r16,\n ARMEmitter::Reg::r17,\n ARMEmitter::Reg::r29,\n ARMEmitter::Reg::r30,\n\n ARMEmitter::Reg::r24,\n ARMEmitter::Reg::r19,\n };\n\n constexpr std::array NotPreserved_Dynamic = {\n ARMEmitter::Reg::r12, ARMEmitter::Reg::r13, ARMEmitter::Reg::r14, ARMEmitter::Reg::r15,\n ARMEmitter::Reg::r16, ARMEmitter::Reg::r17, ARMEmitter::Reg::r30,\n };\n\n constexpr unsigned RAPairs = 10;\n\n // All are caller saved\n constexpr std::array SRAFPR = {\n ARMEmitter::VReg::v16, ARMEmitter::VReg::v17, ARMEmitter::VReg::v18, ARMEmitter::VReg::v19,\n ARMEmitter::VReg::v20, ARMEmitter::VReg::v21, ARMEmitter::VReg::v22, ARMEmitter::VReg::v23,\n };\n\n // v8..v15 = (lower 64bits) Callee saved\n constexpr std::array RAFPR = {\n // v0 ~ v1 are used as temps.\n // ARMEmitter::VReg::v0, ARMEmitter::VReg::v1,\n\n ARMEmitter::VReg::v2, ARMEmitter::VReg::v3, ARMEmitter::VReg::v4, ARMEmitter::VReg::v5, ARMEmitter::VReg::v6,\n ARMEmitter::VReg::v7, ARMEmitter::VReg::v8, ARMEmitter::VReg::v9, ARMEmitter::VReg::v10, ARMEmitter::VReg::v11,\n ARMEmitter::VReg::v12, ARMEmitter::VReg::v13, ARMEmitter::VReg::v14, ARMEmitter::VReg::v15,\n\n ARMEmitter::VReg::v24, ARMEmitter::VReg::v25, ARMEmitter::VReg::v26, ARMEmitter::VReg::v27, ARMEmitter::VReg::v28,\n ARMEmitter::VReg::v29, ARMEmitter::VReg::v30, ARMEmitter::VReg::v31};\n\n // I wish this could get constexpr generated from SRA's definition but impossible until libstdc++12, libc++15.\n // SRA GPRs that need to be spilled when calling a function with `preserve_all` ABI.\n constexpr std::array PreserveAll_SRA = {\n ARMEmitter::Reg::r4, ARMEmitter::Reg::r5, ARMEmitter::Reg::r6, ARMEmitter::Reg::r7, ARMEmitter::Reg::r8,\n };\n\n constexpr uint32_t PreserveAll_SRAMask = {[]() -> uint32_t {\n uint32_t Mask {};\n for (auto Reg : PreserveAll_SRA) {\n switch (Reg.Idx()) {\n case 0:\n case 1:\n case 2:\n case 3:\n case 4:\n case 5:\n case 6:\n case 7:\n case 8:\n case 16:\n case 17: Mask |= (1U << Reg.Idx()); break;\n default: break;\n }\n }\n\n return Mask;\n }()};\n\n // Dynamic GPRs\n constexpr std::array PreserveAll_Dynamic = {ARMEmitter::Reg::r16, ARMEmitter::Reg::r17, ARMEmitter::Reg::r30};\n\n // SRA FPRs that need to be spilled when calling a function with `preserve_all` ABI.\n constexpr uint32_t PreserveAll_SRAFPRMask = 0;\n\n // Dynamic FPRs\n // - v0-v7\n constexpr std::array PreserveAll_DynamicFPR = {\n // v0 ~ v1 are temps\n ARMEmitter::VReg::v2, ARMEmitter::VReg::v3, ARMEmitter::VReg::v4, ARMEmitter::VReg::v5, ARMEmitter::VReg::v6, ARMEmitter::VReg::v7,\n };\n\n // SRA FPRs that need to be spilled when the host supports SVE-256bit with `preserve_all` ABI.\n // This is /all/ of the SRA registers\n constexpr std::array PreserveAll_SRAFPRSVE = SRAFPR;\n\n constexpr uint32_t PreserveAll_SRAFPRSVEMask = {[]() -> uint32_t {\n uint32_t Mask {};\n for (auto Reg : PreserveAll_SRAFPRSVE) {\n Mask |= (1U << Reg.Idx());\n }\n return Mask;\n }()};\n\n // Dynamic FPRs when the host supports SVE-256bit.\n constexpr std::array PreserveAll_DynamicFPRSVE = {\n // v0 ~ v1 are used as temps.\n ARMEmitter::VReg::v2, ARMEmitter::VReg::v3, ARMEmitter::VReg::v4, ARMEmitter::VReg::v5, ARMEmitter::VReg::v6,\n ARMEmitter::VReg::v7, ARMEmitter::VReg::v8, ARMEmitter::VReg::v9, ARMEmitter::VReg::v10, ARMEmitter::VReg::v11,\n ARMEmitter::VReg::v12, ARMEmitter::VReg::v13, ARMEmitter::VReg::v14, ARMEmitter::VReg::v15,\n\n ARMEmitter::VReg::v24, ARMEmitter::VReg::v25, ARMEmitter::VReg::v26, ARMEmitter::VReg::v27, ARMEmitter::VReg::v28,\n ARMEmitter::VReg::v29, ARMEmitter::VReg::v30, ARMEmitter::VReg::v31};\n} // namespace x32\n\n// We want vixl to not allocate a default buffer. Jit and dispatcher will manually create one.\nArm64Emitter::Arm64Emitter(FEXCore::Context::ContextImpl* ctx, void* EmissionPtr, size_t size)\n : Emitter(static_cast(EmissionPtr), size)\n , EmitterCTX {ctx}\n#ifdef VIXL_SIMULATOR\n , Simulator {&SimDecoder, stdout, vixl::aarch64::SimStack(SimulatorStackSize).Allocate()}\n#endif\n{\n#ifdef VIXL_SIMULATOR\n FEX_CONFIG_OPT(ForceSVEWidth, FORCESVEWIDTH);\n // Hardcode a 256-bit vector width if we are running in the simulator.\n // Allow the user to override this.\n Simulator.SetVectorLengthInBits(ForceSVEWidth() ? ForceSVEWidth() : 256);\n // FEX doesn't support GCS.\n Simulator.DisableGCSCheck();\n#endif\n#ifdef VIXL_DISASSEMBLER\n // Only setup the disassembler if enabled.\n // vixl's decoder is expensive to setup.\n if (Disassemble()) {\n DisasmBuffer.resize(DISASM_BUFFER_SIZE);\n Disasm = fextl::make_unique(DisasmBuffer.data(), DISASM_BUFFER_SIZE);\n DisasmDecoder = fextl::make_unique();\n DisasmDecoder->AppendVisitor(Disasm.get());\n }\n#endif\n\n // Number of register available is dependent on what operating mode the proccess is in.\n if (EmitterCTX->Config.Is64BitMode()) {\n StaticRegisters = x64::SRA;\n GeneralRegisters = x64::RA;\n GeneralRegistersNotPreserved = x64::NotPreserved_Dynamic;\n StaticFPRegisters = x64::SRAFPR;\n GeneralFPRegisters = x64::RAFPR;\n PairRegisters = x64::RAPairs;\n } else {\n PairRegisters = x32::RAPairs;\n\n StaticRegisters = x32::SRA;\n GeneralRegisters = x32::RA;\n GeneralRegistersNotPreserved = x32::NotPreserved_Dynamic;\n\n StaticFPRegisters = x32::SRAFPR;\n GeneralFPRegisters = x32::RAFPR;\n }\n}\n\nFEXCore::X86State::X86Reg Arm64Emitter::GetX86RegRelationToARMReg(ARMEmitter::Register Reg) {\n for (size_t i = 0; i < StaticRegisters.size(); ++i) {\n const auto& RegI = StaticRegisters[i];\n if (RegI == Reg) {\n // X86 Registers are mapped linerally from the StaticRegisters span.\n // Directly correlating Enum index to span index.\n return static_cast(FEXCore::ToUnderlying(FEXCore::X86State::X86Reg::REG_RAX) + i);\n }\n }\n\n // Unmapped register.\n return FEXCore::X86State::X86Reg::REG_INVALID;\n}\n\nvoid Arm64Emitter::LoadConstant(ARMEmitter::Size s, ARMEmitter::Register Reg, uint64_t Constant, PadType Pad, int MaxBytes) {\n bool NOPPad = false;\n if (Pad == PadType::DOPAD) {\n NOPPad = true;\n } else if (Pad == PadType::NOPAD) {\n NOPPad = false;\n } else if (Pad == PadType::AUTOPAD) {\n // Force NOP padding to ensure relocated constants always have enough encoding space available\n NOPPad = EnableCodeCaching;\n }\n\n bool Is64Bit = s == ARMEmitter::Size::i64Bit;\n const auto UpperBound = Is64Bit ? 4 : 2;\n int Segments = MaxBytes ? (MaxBytes / 2) : UpperBound;\n\n LOGMAN_THROW_A_FMT(MaxBytes >= 0 && MaxBytes <= (UpperBound * 2) && (MaxBytes & 1) == 0,\n \"MaxBytes must be bounded in the range of [0, {}] and 16-bit aligned\", UpperBound);\n // If MaxBytes specified then make sure to sanity check incoming data.\n LOGMAN_THROW_A_FMT(MaxBytes == 0 || (Constant >> (MaxBytes * 8)) == 0, \"MaxBytes provided but data can't fit within provided range.\");\n\n if (Is64Bit && ((~Constant) >> 16) == 0) {\n if (NOPPad) {\n nop();\n nop();\n nop();\n }\n\n movn(s, Reg, (~Constant) & 0xFFFF);\n return;\n }\n\n if ((Constant >> 32) == 0 && !NOPPad) {\n // If the upper 32-bits is all zero, we can now switch to a 32-bit move.\n // NOTE: The NOP padding code does not appropriately adjust to this yet,\n // so we skip this optimization in that case\n s = ARMEmitter::Size::i32Bit;\n Is64Bit = false;\n Segments = std::min(Segments, 2);\n }\n\n if (!Is64Bit && ((~Constant) & 0xFFFF0000) == 0) {\n if (NOPPad) {\n nop();\n nop();\n nop();\n }\n\n movn(s, Reg.W(), (~Constant) & 0xFFFF);\n return;\n }\n\n int RequiredMoveSegments {};\n\n // Count the number of move segments\n // We only want to use ADRP+ADD if we have more than 1 segment\n for (size_t i = 0; i < Segments; ++i) {\n uint16_t Part = (Constant >> (i * 16)) & 0xFFFF;\n if (Part != 0) {\n ++RequiredMoveSegments;\n }\n }\n\n // If this can be loaded with a mov bitmask.\n if (RequiredMoveSegments > 1) {\n // Only try to use this path if the number of segments is > 1.\n // `movz` is better than `orr` since hardware will rename or merge if possible when `movz` is used.\n const auto IsImm = ARMEmitter::Emitter::IsImmLogical(Constant, RegSizeInBits(s));\n if (IsImm) {\n if (NOPPad) {\n nop();\n nop();\n nop();\n }\n orr(s, Reg, ARMEmitter::Reg::zr, Constant);\n return;\n }\n }\n\n // If we can't handle negatives with the orr, try with movn+movk\n if (Is64Bit && ((~Constant) >> 32) == 0) {\n if (NOPPad) {\n nop();\n nop();\n }\n movn(s, Reg, (~Constant) & 0xFFFF);\n movk(s, Reg, (Constant >> 16) & 0xFFFF, 16);\n return;\n }\n\n // ADRP+ADD is specifically optimized in hardware\n // Check if we can use this\n auto PC = GetCursorAddress();\n\n // PC aligned to page\n uint64_t AlignedPC = PC & ~0xFFFULL;\n\n // Offset from aligned PC\n auto AlignedOffset = std::bit_cast(Constant - AlignedPC);\n\n int NumMoves = 0;\n\n // If the aligned offset is within the 4GB window then we can use ADRP+ADD\n // and the number of move segments more than 1\n // NOTE: JIT output is moved to a different buffer after compilation, so the\n // current cursor address doesn't match the runtime instruction address.\n // Hence this optimization is disabled until we enable code relocation patches.\n if (RequiredMoveSegments > 1 && ARMEmitter::Emitter::IsInt32(AlignedOffset) && false) {\n // If this is 4k page aligned then we only need ADRP\n if ((AlignedOffset & 0xFFF) == 0) {\n adrp(Reg, AlignedOffset >> 12);\n } else {\n // If the constant is within 1MB of PC then we can still use ADR to load in a single instruction\n // 21-bit signed integer here\n auto SmallOffset = std::bit_cast(Constant - PC);\n if (ARMEmitter::Emitter::IsInt21(SmallOffset)) {\n adr(Reg, SmallOffset);\n } else {\n // Need to use ADRP + ADD\n adrp(Reg, AlignedOffset >> 12);\n add(s, Reg, Reg, Constant & 0xFFF);\n NumMoves = 2;\n }\n }\n } else {\n int CurrentSegment = 0;\n for (; CurrentSegment < Segments; ++CurrentSegment) {\n uint16_t Part = (Constant >> (CurrentSegment * 16)) & 0xFFFF;\n if (Part) {\n movz(s, Reg, Part, CurrentSegment * 16);\n ++CurrentSegment;\n ++NumMoves;\n break;\n }\n }\n\n for (; CurrentSegment < Segments; ++CurrentSegment) {\n uint16_t Part = (Constant >> (CurrentSegment * 16)) & 0xFFFF;\n if (Part) {\n movk(s, Reg, Part, CurrentSegment * 16);\n ++NumMoves;\n }\n }\n\n if (NumMoves == 0) {\n // If we didn't move anything that means this is a zero move. Special case this.\n movz(s, Reg, 0);\n ++NumMoves;\n }\n }\n\n if (NOPPad) {\n for (int i = NumMoves; i < Segments; ++i) {\n nop();\n }\n }\n}\n\nvoid Arm64Emitter::PushCalleeSavedRegisters() {\n // We need to save pairs of registers\n // We save r19-r30\n constexpr static std::array, 6> CalleeSaved = {{\n {ARMEmitter::XReg::x19, ARMEmitter::XReg::x20},\n {ARMEmitter::XReg::x21, ARMEmitter::XReg::x22},\n {ARMEmitter::XReg::x23, ARMEmitter::XReg::x24},\n {ARMEmitter::XReg::x25, ARMEmitter::XReg::x26},\n {ARMEmitter::XReg::x27, ARMEmitter::XReg::x28},\n {ARMEmitter::XReg::x29, ARMEmitter::XReg::x30},\n }};\n\n for (auto& RegPair : CalleeSaved) {\n stp(RegPair.first, RegPair.second, ARMEmitter::Reg::rsp, -16);\n }\n\n // Additionally we need to store the lower 64bits of v8-v15\n // Here's a fun thing, we can use two ST4 instructions to store everything\n // We just need a single sub to sp before that\n constexpr static std::array< std::tuple, 2> FPRs = {{\n {ARMEmitter::DReg::d8, ARMEmitter::DReg::d9, ARMEmitter::DReg::d10, ARMEmitter::DReg::d11},\n {ARMEmitter::DReg::d12, ARMEmitter::DReg::d13, ARMEmitter::DReg::d14, ARMEmitter::DReg::d15},\n }};\n\n uint32_t VectorSaveSize = sizeof(uint64_t) * 8;\n sub(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::rsp, ARMEmitter::Reg::rsp, VectorSaveSize);\n // SP supporting move\n // We just saved x19 so it is safe\n add(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r19, ARMEmitter::Reg::rsp, 0);\n\n for (auto& RegQuad : FPRs) {\n st4(ARMEmitter::SubRegSize::i64Bit, std::get<0>(RegQuad), std::get<1>(RegQuad), std::get<2>(RegQuad), std::get<3>(RegQuad), 0,\n ARMEmitter::Reg::r19, 32);\n }\n}\n\nvoid Arm64Emitter::PopCalleeSavedRegisters() {\n constexpr static std::array< std::tuple, 2> FPRs = {{\n {ARMEmitter::DReg::d8, ARMEmitter::DReg::d9, ARMEmitter::DReg::d10, ARMEmitter::DReg::d11},\n {ARMEmitter::DReg::d12, ARMEmitter::DReg::d13, ARMEmitter::DReg::d14, ARMEmitter::DReg::d15},\n }};\n\n for (auto& RegQuad : FPRs) {\n ld4(ARMEmitter::SubRegSize::i64Bit, std::get<0>(RegQuad), std::get<1>(RegQuad), std::get<2>(RegQuad), std::get<3>(RegQuad), 0,\n ARMEmitter::Reg::rsp, 32);\n }\n\n constexpr static std::array, 6> CalleeSaved = {{\n {ARMEmitter::XReg::x29, ARMEmitter::XReg::x30},\n {ARMEmitter::XReg::x27, ARMEmitter::XReg::x28},\n {ARMEmitter::XReg::x25, ARMEmitter::XReg::x26},\n {ARMEmitter::XReg::x23, ARMEmitter::XReg::x24},\n {ARMEmitter::XReg::x21, ARMEmitter::XReg::x22},\n {ARMEmitter::XReg::x19, ARMEmitter::XReg::x20},\n }};\n\n for (auto& RegPair : CalleeSaved) {\n ldp(RegPair.first, RegPair.second, ARMEmitter::Reg::rsp, 16);\n }\n}\n\nvoid Arm64Emitter::FillSpecialRegs(ARMEmitter::Register TmpReg, ARMEmitter::Register TmpReg2, bool SetFIZ, bool SetPredRegs) {\n#ifndef VIXL_SIMULATOR\n if (EmitterCTX->HostFeatures.SupportsAFP) {\n // Enable AFP features when filling JIT state.\n mrs(TmpReg, ARMEmitter::SystemRegister::FPCR);\n\n // Enable FPCR.NEP and FPCR.AH\n // NEP(2): Changes ASIMD scalar instructions to insert in to the lower bits of the destination.\n // AH(1): Changes NaN behaviour in some instructions. Specifically fmin, fmax.\n //\n // Additional interesting AFP bits:\n // FIZ(0): Flush Inputs to Zero\n orr(ARMEmitter::Size::i64Bit, TmpReg, TmpReg,\n (1U << 2) | // NEP\n (1U << 1)); // AH\n\n if (SetFIZ) {\n // Insert MXCSR.DAZ in to FIZ\n ldr(TmpReg2.W(), STATE.R(), offsetof(FEXCore::Core::CPUState, mxcsr));\n bfxil(ARMEmitter::Size::i64Bit, TmpReg, TmpReg2, 6, 1);\n }\n\n msr(ARMEmitter::SystemRegister::FPCR, TmpReg);\n }\n#endif\n\n if (SetPredRegs && (EmitterCTX->HostFeatures.SupportsSVE256 || EmitterCTX->HostFeatures.SupportsSVE128)) {\n // Set up predicate registers.\n // We don't bother spilling these in SpillStaticRegs,\n // since all that matters is we restore them on a fill.\n // It's not a concern if they get trounced by something else.\n if (EmitterCTX->HostFeatures.SupportsSVE256) {\n ptrue(ARMEmitter::SubRegSize::i8Bit, PRED_TMP_32B, ARMEmitter::PredicatePattern::SVE_VL32);\n }\n\n if (EmitterCTX->HostFeatures.SupportsSVE128) {\n ptrue(ARMEmitter::SubRegSize::i8Bit, PRED_TMP_16B, ARMEmitter::PredicatePattern::SVE_VL16);\n }\n\n // Fill in the predicate register for the x87 ldst SVE optimization.\n ptrue(ARMEmitter::SubRegSize::i16Bit, PRED_X87_SVEOPT, ARMEmitter::PredicatePattern::SVE_VL5);\n }\n}\n\nvoid Arm64Emitter::SpillStaticRegs(ARMEmitter::Register TmpReg, SpillStaticRegOptions Options) {\n#ifndef VIXL_SIMULATOR\n if (EmitterCTX->HostFeatures.SupportsAFP) {\n // Disable AFP features when spilling registers.\n //\n // Disable FPCR.NEP and FPCR.AH and FPCR.FIZ\n // NEP(2): Changes ASIMD scalar instructions to insert in to the lower bits of the destination.\n // AH(1): Changes NaN behaviour in some instructions. Specifically fmin, fmax.\n // Also interacts with RPRES to change reciprocal/rsqrt precision from 8-bit mantissa to 12-bit.\n //\n // Additional interesting AFP bits:\n // FIZ(0): Flush Inputs to Zero\n mrs(TmpReg, ARMEmitter::SystemRegister::FPCR);\n bic(ARMEmitter::Size::i64Bit, TmpReg, TmpReg,\n (1U << 2) | // NEP\n (1U << 1) | // AH\n (1U << 0)); // FIZ\n msr(ARMEmitter::SystemRegister::FPCR, TmpReg);\n }\n#endif\n\n if (Options.NZCV) {\n // Regardless of what GPRs/FPRs we're spilling, we need to spill NZCV since it\n // is always static and almost certainly clobbered by the subsequent code.\n //\n // TODO: Can we prove that NZCV is not used across a call in some cases and\n // omit this? Might help x87 perf? Future idea.\n mrs(TmpReg, ARMEmitter::SystemRegister::NZCV);\n str(TmpReg.W(), STATE.R(), offsetof(FEXCore::Core::CpuStateFrame, State.flags[24]));\n }\n\n // PF/AF are special, remove them from the mask\n uint32_t PFAFMask = ((1u << REG_PF.Idx()) | ((1u << REG_AF.Idx())));\n unsigned PFAFSpillMask = Options.GPRSpillMask & PFAFMask;\n Options.GPRSpillMask &= ~PFAFSpillMask;\n\n str(REG_CALLRET_SP, STATE.R(), offsetof(FEXCore::Core::CpuStateFrame, State.callret_sp));\n\n for (size_t i = 0; i < StaticRegisters.size(); i += 2) {\n auto Reg1 = StaticRegisters[i];\n auto Reg2 = StaticRegisters[i + 1];\n if (((1U << Reg1.Idx()) & Options.GPRSpillMask) && ((1U << Reg2.Idx()) & Options.GPRSpillMask)) {\n stp(Reg1.X(), Reg2.X(), STATE.R(), ARRAY_OFFSETOF(FEXCore::Core::CpuStateFrame, State.gregs, i));\n } else if (((1U << Reg1.Idx()) & Options.GPRSpillMask)) {\n str(Reg1.X(), STATE.R(), ARRAY_OFFSETOF(FEXCore::Core::CpuStateFrame, State.gregs, i));\n } else if (((1U << Reg2.Idx()) & Options.GPRSpillMask)) {\n str(Reg2.X(), STATE.R(), ARRAY_OFFSETOF(FEXCore::Core::CpuStateFrame, State.gregs, i + 1));\n }\n }\n\n // Now handle PF/AF\n if (Options.NZCV && PFAFSpillMask) {\n auto PFOffset = offsetof(FEXCore::Core::CpuStateFrame, State.pf_raw);\n auto AFOffset = offsetof(FEXCore::Core::CpuStateFrame, State.af_raw);\n LOGMAN_THROW_A_FMT(PFAFSpillMask == PFAFMask, \"PF/AF not spilled together\");\n LOGMAN_THROW_A_FMT(AFOffset == PFOffset + 4, \"PF/AF are together\");\n\n stp(REG_PF.W(), REG_AF.W(), STATE.R(), PFOffset);\n }\n\n if (Options.FPRs) {\n if (EmitterCTX->HostFeatures.SupportsAVX && EmitterCTX->HostFeatures.SupportsSVE256) {\n for (size_t i = 0; i < StaticFPRegisters.size(); i++) {\n const auto Reg = StaticFPRegisters[i];\n\n if (((1U << Reg.Idx()) & Options.FPRSpillMask) != 0) {\n mov(ARMEmitter::Size::i64Bit, TmpReg, ARRAY_OFFSETOF(Core::CpuStateFrame, State.xmm.avx.data, i));\n st1b(Reg.Z(), PRED_TMP_32B, STATE.R(), TmpReg);\n }\n }\n } else {\n if (Options.GPRSpillMask && Options.FPRSpillMask == ~0U) {\n // Optimize the common case where we can spill four registers per instruction\n // Load the sse offset in to the temporary register\n add(ARMEmitter::Size::i64Bit, TmpReg, STATE.R(), offsetof(FEXCore::Core::CpuStateFrame, State.xmm.sse.data));\n for (size_t i = 0; i < StaticFPRegisters.size(); i += 4) {\n const auto Reg1 = StaticFPRegisters[i];\n const auto Reg2 = StaticFPRegisters[i + 1];\n const auto Reg3 = StaticFPRegisters[i + 2];\n const auto Reg4 = StaticFPRegisters[i + 3];\n st1(Reg1.Q(), Reg2.Q(), Reg3.Q(), Reg4.Q(), TmpReg, 64);\n }\n } else {\n for (size_t i = 0; i < StaticFPRegisters.size(); i += 2) {\n const auto Reg1 = StaticFPRegisters[i];\n const auto Reg2 = StaticFPRegisters[i + 1];\n\n if (((1U << Reg1.Idx()) & Options.FPRSpillMask) && ((1U << Reg2.Idx()) & Options.FPRSpillMask)) {\n stp(Reg1.Q(), Reg2.Q(), STATE.R(), ARRAY_OFFSETOF(FEXCore::Core::CpuStateFrame, State.xmm.sse.data, i));\n } else if (((1U << Reg1.Idx()) & Options.FPRSpillMask)) {\n str(Reg1.Q(), STATE.R(), ARRAY_OFFSETOF(FEXCore::Core::CpuStateFrame, State.xmm.sse.data, i));\n } else if (((1U << Reg2.Idx()) & Options.FPRSpillMask)) {\n str(Reg2.Q(), STATE.R(), ARRAY_OFFSETOF(FEXCore::Core::CpuStateFrame, State.xmm.sse.data, i + 1));\n }\n }\n }\n }\n }\n}\n\nvoid Arm64Emitter::FillStaticRegs(FillStaticRegOptions Options) {\n auto FindTempReg = [this](uint32_t* GPRFillMask) -> std::optional {\n for (auto Reg : StaticRegisters) {\n if (((1U << Reg.Idx()) & *GPRFillMask)) {\n *GPRFillMask &= ~(1U << Reg.Idx());\n return std::make_optional(Reg);\n }\n }\n return std::nullopt;\n };\n\n LOGMAN_THROW_A_FMT(Options.GPRFillMask != 0, \"Must fill at least 2 GPRs for a temp\");\n uint32_t TempGPRFillMask = Options.GPRFillMask;\n if (!Options.OptionalReg.has_value()) {\n Options.OptionalReg = FindTempReg(&TempGPRFillMask);\n }\n\n if (!Options.OptionalReg2.has_value()) {\n Options.OptionalReg2 = FindTempReg(&TempGPRFillMask);\n }\n LOGMAN_THROW_A_FMT(Options.OptionalReg.has_value() && Options.OptionalReg2.has_value(), \"Didn't have an SRA register to use as a \"\n \"temporary while \"\n \"spilling!\");\n\n auto TmpReg = *Options.OptionalReg;\n auto TmpReg2 = *Options.OptionalReg2;\n\n#ifdef ARCHITECTURE_arm64ec\n // Load STATE in from the CPU area as x28 is not callee saved in the ARM64EC ABI.\n ldr(TmpReg.X(), ARMEmitter::Reg::r18, TEB_CPU_AREA_OFFSET);\n ldr(STATE, TmpReg, CPU_AREA_EMULATOR_DATA_OFFSET);\n#endif\n\n ldr(REG_CALLRET_SP, STATE.R(), offsetof(FEXCore::Core::CpuStateFrame, State.callret_sp));\n\n if (Options.NZCV) {\n // Regardless of what GPRs/FPRs we're filling, we need to fill NZCV since it\n // is always static and was almost certainly clobbered.\n //\n // TODO: Can we prove that NZCV is not used across a call in some cases and\n // omit this? Might help x87 perf? Future idea.\n ldr(TmpReg.W(), STATE.R(), offsetof(FEXCore::Core::CpuStateFrame, State.flags[24]));\n msr(ARMEmitter::SystemRegister::NZCV, TmpReg);\n }\n\n FillSpecialRegs(TmpReg, TmpReg2, true, Options.FPRs);\n\n if (Options.FPRs) {\n if (EmitterCTX->HostFeatures.SupportsAVX && EmitterCTX->HostFeatures.SupportsSVE256) {\n for (size_t i = 0; i < StaticFPRegisters.size(); i++) {\n const auto Reg = StaticFPRegisters[i];\n if (((1U << Reg.Idx()) & Options.FPRFillMask) != 0) {\n mov(ARMEmitter::Size::i64Bit, TmpReg, ARRAY_OFFSETOF(Core::CpuStateFrame, State.xmm.avx.data, i));\n ld1b(Reg.Z(), PRED_TMP_32B.Zeroing(), STATE.R(), TmpReg);\n }\n }\n } else {\n if (Options.GPRFillMask && Options.FPRFillMask == ~0U) {\n // Optimize the common case where we can fill four registers per instruction.\n // Use one of the filling static registers before we fill it.\n // Load the sse offset in to the temporary register\n add(ARMEmitter::Size::i64Bit, TmpReg, STATE.R(), offsetof(FEXCore::Core::CpuStateFrame, State.xmm.sse.data));\n for (size_t i = 0; i < StaticFPRegisters.size(); i += 4) {\n const auto Reg1 = StaticFPRegisters[i];\n const auto Reg2 = StaticFPRegisters[i + 1];\n const auto Reg3 = StaticFPRegisters[i + 2];\n const auto Reg4 = StaticFPRegisters[i + 3];\n ld1(Reg1.Q(), Reg2.Q(), Reg3.Q(), Reg4.Q(), TmpReg, 64);\n }\n } else {\n for (size_t i = 0; i < StaticFPRegisters.size(); i += 2) {\n const auto Reg1 = StaticFPRegisters[i];\n const auto Reg2 = StaticFPRegisters[i + 1];\n\n if (((1U << Reg1.Idx()) & Options.FPRFillMask) && ((1U << Reg2.Idx()) & Options.FPRFillMask)) {\n ldp(Reg1.Q(), Reg2.Q(), STATE.R(), ARRAY_OFFSETOF(FEXCore::Core::CpuStateFrame, State.xmm.sse.data, i));\n } else if (((1U << Reg1.Idx()) & Options.FPRFillMask)) {\n ldr(Reg1.Q(), STATE.R(), ARRAY_OFFSETOF(FEXCore::Core::CpuStateFrame, State.xmm.sse.data, i));\n } else if (((1U << Reg2.Idx()) & Options.FPRFillMask)) {\n ldr(Reg2.Q(), STATE.R(), ARRAY_OFFSETOF(FEXCore::Core::CpuStateFrame, State.xmm.sse.data, i + 1));\n }\n }\n }\n }\n }\n\n // PF/AF are special, remove them from the mask\n uint32_t PFAFMask = ((1u << REG_PF.Idx()) | ((1u << REG_AF.Idx())));\n uint32_t PFAFFillMask = Options.GPRFillMask & PFAFMask;\n Options.GPRFillMask &= ~PFAFMask;\n\n for (size_t i = 0; i < StaticRegisters.size(); i += 2) {\n auto Reg1 = StaticRegisters[i];\n auto Reg2 = StaticRegisters[i + 1];\n if (((1U << Reg1.Idx()) & Options.GPRFillMask) && ((1U << Reg2.Idx()) & Options.GPRFillMask)) {\n ldp(Reg1.X(), Reg2.X(), STATE.R(), ARRAY_OFFSETOF(FEXCore::Core::CpuStateFrame, State.gregs, i));\n } else if ((1U << Reg1.Idx()) & Options.GPRFillMask) {\n ldr(Reg1.X(), STATE.R(), ARRAY_OFFSETOF(FEXCore::Core::CpuStateFrame, State.gregs, i));\n } else if ((1U << Reg2.Idx()) & Options.GPRFillMask) {\n ldr(Reg2.X(), STATE.R(), ARRAY_OFFSETOF(FEXCore::Core::CpuStateFrame, State.gregs, i + 1));\n }\n }\n\n // Now handle PF/AF\n if (Options.NZCV && PFAFFillMask) {\n LOGMAN_THROW_A_FMT(PFAFFillMask == PFAFMask, \"PF/AF not filled together\");\n\n ldp(REG_PF.W(), REG_AF.W(), STATE.R(), offsetof(FEXCore::Core::CpuStateFrame, State.pf_raw));\n }\n}\n\nvoid Arm64Emitter::PushVectorRegisters(ARMEmitter::Register TmpReg, bool SVE256Regs, std::span VRegs) {\n if (SVE256Regs) {\n size_t i = 0;\n\n for (; i < (VRegs.size() % 4); i += 2) {\n const auto Reg1 = VRegs[i];\n const auto Reg2 = VRegs[i + 1];\n st2b(Reg1.Z(), Reg2.Z(), PRED_TMP_32B, TmpReg, 0);\n add(ARMEmitter::Size::i64Bit, TmpReg, TmpReg, 32 * 2);\n }\n\n for (; i < VRegs.size(); i += 4) {\n const auto Reg1 = VRegs[i];\n const auto Reg2 = VRegs[i + 1];\n const auto Reg3 = VRegs[i + 2];\n const auto Reg4 = VRegs[i + 3];\n st4b(Reg1.Z(), Reg2.Z(), Reg3.Z(), Reg4.Z(), PRED_TMP_32B, TmpReg, 0);\n add(ARMEmitter::Size::i64Bit, TmpReg, TmpReg, 32 * 4);\n }\n } else {\n size_t i = 0;\n for (; i < (VRegs.size() % 4); i += 2) {\n const auto Reg1 = VRegs[i];\n const auto Reg2 = VRegs[i + 1];\n st1(Reg1.Q(), Reg2.Q(), TmpReg, 32);\n }\n\n for (; i < VRegs.size(); i += 4) {\n const auto Reg1 = VRegs[i];\n const auto Reg2 = VRegs[i + 1];\n const auto Reg3 = VRegs[i + 2];\n const auto Reg4 = VRegs[i + 3];\n st1(Reg1.Q(), Reg2.Q(), Reg3.Q(), Reg4.Q(), TmpReg, 64);\n }\n }\n}\n\nvoid Arm64Emitter::PushGeneralRegisters(ARMEmitter::Register TmpReg, std::span Regs) {\n size_t i = 0;\n for (; i < (Regs.size() % 2); ++i) {\n const auto Reg1 = Regs[i];\n str(Reg1.X(), TmpReg, 16);\n }\n\n for (; i < Regs.size(); i += 2) {\n const auto Reg1 = Regs[i];\n const auto Reg2 = Regs[i + 1];\n stp(Reg1.X(), Reg2.X(), TmpReg, 16);\n }\n}\n\nvoid Arm64Emitter::PopVectorRegisters(bool SVE256Regs, std::span VRegs) {\n if (SVE256Regs) {\n size_t i = 0;\n for (; i < (VRegs.size() % 4); i += 2) {\n const auto Reg1 = VRegs[i];\n const auto Reg2 = VRegs[i + 1];\n ld2b(Reg1.Z(), Reg2.Z(), PRED_TMP_32B.Zeroing(), ARMEmitter::Reg::rsp);\n add(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::rsp, ARMEmitter::Reg::rsp, 32 * 2);\n }\n\n for (; i < VRegs.size(); i += 4) {\n const auto Reg1 = VRegs[i];\n const auto Reg2 = VRegs[i + 1];\n const auto Reg3 = VRegs[i + 2];\n const auto Reg4 = VRegs[i + 3];\n ld4b(Reg1.Z(), Reg2.Z(), Reg3.Z(), Reg4.Z(), PRED_TMP_32B.Zeroing(), ARMEmitter::Reg::rsp);\n add(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::rsp, ARMEmitter::Reg::rsp, 32 * 4);\n }\n } else {\n size_t i = 0;\n for (; i < (VRegs.size() % 4); i += 2) {\n const auto Reg1 = VRegs[i];\n const auto Reg2 = VRegs[i + 1];\n ld1(Reg1.Q(), Reg2.Q(), ARMEmitter::Reg::rsp, 32);\n }\n\n for (; i < VRegs.size(); i += 4) {\n const auto Reg1 = VRegs[i];\n const auto Reg2 = VRegs[i + 1];\n const auto Reg3 = VRegs[i + 2];\n const auto Reg4 = VRegs[i + 3];\n ld1(Reg1.Q(), Reg2.Q(), Reg3.Q(), Reg4.Q(), ARMEmitter::Reg::rsp, 64);\n }\n }\n}\n\nvoid Arm64Emitter::PopGeneralRegisters(std::span Regs) {\n size_t i = 0;\n for (; i < (Regs.size() % 2); ++i) {\n const auto Reg1 = Regs[i];\n ldr(Reg1.X(), ARMEmitter::Reg::rsp, 16);\n }\n for (; i < Regs.size(); i += 2) {\n const auto Reg1 = Regs[i];\n const auto Reg2 = Regs[i + 1];\n ldp(Reg1.X(), Reg2.X(), ARMEmitter::Reg::rsp, 16);\n }\n}\n\nsize_t Arm64Emitter::PushDynamicRegs(ARMEmitter::Register TmpReg) {\n const auto CanUseSVE256 = EmitterCTX->HostFeatures.SupportsSVE256;\n const auto GPRSize = GeneralRegistersNotPreserved.size() * Core::CPUState::GPR_REG_SIZE;\n const auto FPRRegSize = CanUseSVE256 ? 32 : 16;\n const auto FPRSize = GeneralFPRegisters.size() * FPRRegSize;\n const uint64_t SPOffset = AlignUp(GPRSize + FPRSize, 16);\n\n sub(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::rsp, ARMEmitter::Reg::rsp, SPOffset);\n\n // rsp capable move\n add(ARMEmitter::Size::i64Bit, TmpReg, ARMEmitter::Reg::rsp, 0);\n\n LOGMAN_THROW_A_FMT(GeneralFPRegisters.size() % 2 == 0, \"Needs to have multiple of 2 FPRs for RA\");\n\n // Push the vector registers\n PushVectorRegisters(TmpReg, CanUseSVE256, GeneralFPRegisters);\n\n // Push the general registers.\n PushGeneralRegisters(TmpReg, GeneralRegistersNotPreserved);\n\n return SPOffset;\n}\n\nvoid Arm64Emitter::PopDynamicRegs() {\n const auto CanUseSVE256 = EmitterCTX->HostFeatures.SupportsSVE256;\n\n // Pop vectors first\n PopVectorRegisters(CanUseSVE256, GeneralFPRegisters);\n\n // Pop GPRs second\n PopGeneralRegisters(GeneralRegistersNotPreserved);\n}\n\nsize_t Arm64Emitter::SpillForPreserveAllABICall(ARMEmitter::Register TmpReg, bool FPRs) {\n const auto CanUseSVE256 = EmitterCTX->HostFeatures.SupportsSVE256;\n const auto FPRRegSize = CanUseSVE256 ? 32 : 16;\n\n std::span DynamicGPRs {};\n std::span DynamicFPRs {};\n uint32_t PreserveSRAMask {};\n uint32_t PreserveSRAFPRMask {};\n if (EmitterCTX->Config.Is64BitMode()) {\n DynamicGPRs = x64::PreserveAll_Dynamic;\n DynamicFPRs = x64::PreserveAll_DynamicFPR;\n PreserveSRAMask = x64::PreserveAll_SRAMask;\n PreserveSRAFPRMask = x64::PreserveAll_SRAFPRMask;\n\n if (CanUseSVE256) {\n DynamicFPRs = x64::PreserveAll_DynamicFPRSVE;\n PreserveSRAFPRMask = x64::PreserveAll_SRAFPRSVEMask;\n }\n } else {\n DynamicGPRs = x32::PreserveAll_Dynamic;\n DynamicFPRs = x32::PreserveAll_DynamicFPR;\n PreserveSRAMask = x32::PreserveAll_SRAMask;\n PreserveSRAFPRMask = x32::PreserveAll_SRAFPRMask;\n\n if (CanUseSVE256) {\n DynamicFPRs = x32::PreserveAll_DynamicFPRSVE;\n PreserveSRAFPRMask = x32::PreserveAll_SRAFPRSVEMask;\n }\n }\n\n const auto GPRSize = AlignUp(DynamicGPRs.size(), 2) * Core::CPUState::GPR_REG_SIZE;\n const auto FPRSize = DynamicFPRs.size() * FPRRegSize;\n const uint64_t SPOffset = AlignUp(GPRSize + FPRSize, 16);\n\n // Spill the static registers.\n SpillStaticRegs(TmpReg, {\n .GPRSpillMask = PreserveSRAMask,\n .FPRSpillMask = PreserveSRAFPRMask,\n });\n\n sub(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::rsp, ARMEmitter::Reg::rsp, SPOffset);\n\n // rsp capable move\n add(ARMEmitter::Size::i64Bit, TmpReg, ARMEmitter::Reg::rsp, 0);\n\n // Push the vector registers.\n PushVectorRegisters(TmpReg, CanUseSVE256, DynamicFPRs);\n\n // Push the general registers.\n PushGeneralRegisters(TmpReg, DynamicGPRs);\n\n return SPOffset;\n}\n\nvoid Arm64Emitter::FillForPreserveAllABICall(bool FPRs) {\n const auto CanUseSVE256 = EmitterCTX->HostFeatures.SupportsSVE256;\n\n std::span DynamicGPRs {};\n std::span DynamicFPRs {};\n uint32_t PreserveSRAMask {};\n uint32_t PreserveSRAFPRMask {};\n\n if (EmitterCTX->Config.Is64BitMode()) {\n DynamicGPRs = x64::PreserveAll_Dynamic;\n DynamicFPRs = x64::PreserveAll_DynamicFPR;\n PreserveSRAMask = x64::PreserveAll_SRAMask;\n PreserveSRAFPRMask = x64::PreserveAll_SRAFPRMask;\n\n if (CanUseSVE256) {\n DynamicFPRs = x64::PreserveAll_DynamicFPRSVE;\n PreserveSRAFPRMask = x64::PreserveAll_SRAFPRSVEMask;\n }\n } else {\n DynamicGPRs = x32::PreserveAll_Dynamic;\n DynamicFPRs = x32::PreserveAll_DynamicFPR;\n PreserveSRAMask = x32::PreserveAll_SRAMask;\n PreserveSRAFPRMask = x32::PreserveAll_SRAFPRMask;\n\n if (CanUseSVE256) {\n DynamicFPRs = x32::PreserveAll_DynamicFPRSVE;\n PreserveSRAFPRMask = x32::PreserveAll_SRAFPRSVEMask;\n }\n }\n\n // Fill the static registers.\n FillStaticRegs({\n .GPRFillMask = PreserveSRAMask,\n .FPRFillMask = PreserveSRAFPRMask,\n .FPRs = FPRs,\n });\n\n // Pop the vector registers.\n PopVectorRegisters(CanUseSVE256, DynamicFPRs);\n\n // Pop the general registers.\n PopGeneralRegisters(DynamicGPRs);\n}\n\nvoid Arm64Emitter::Align16B() {\n uint64_t CurrentOffset = GetCursorAddress();\n for (uint64_t i = (-CurrentOffset & 0xF); i != 0; i -= 4) {\n nop();\n }\n}\n\n} // namespace FEXCore::CPU\n" }, { "path": "FEXCore/Source/Interface/Core/ArchHelpers/Arm64Emitter.h", "content": "// SPDX-License-Identifier: MIT\n#pragma once\n\n#include \n\n#ifdef VIXL_DISASSEMBLER\n#include \n#include \n#include \n#endif\n#ifdef VIXL_SIMULATOR\n#include \n#include \n#endif\n\n#include \n#include \n\n#include \n#include \n#include \n#include \n\nnamespace FEXCore::Context {\nclass ContextImpl;\n}\nnamespace FEXCore::X86State {\nenum X86Reg : uint32_t;\n}\n\nnamespace FEXCore::CPU {\n// Contains the address to the currently available CPU state\nconstexpr auto STATE = ARMEmitter::XReg::x28;\n\n#ifndef ARCHITECTURE_arm64ec\n// GPR temporaries. Only x3 can be used across spill boundaries\n// so if these ever need to change, be very careful about that.\nconstexpr auto TMP1 = ARMEmitter::XReg::x0;\nconstexpr auto TMP2 = ARMEmitter::XReg::x1;\nconstexpr auto TMP3 = ARMEmitter::XReg::x2;\nconstexpr auto TMP4 = ARMEmitter::XReg::x3;\nconstexpr bool TMP_ABIARGS = true;\n\n// We pin r26/r27 as PF/AF respectively, this is internal FEX ABI.\nconstexpr auto REG_PF = ARMEmitter::Reg::r26;\nconstexpr auto REG_AF = ARMEmitter::Reg::r27;\n\nconstexpr auto REG_CALLRET_SP = ARMEmitter::XReg::x25;\n\n// Vector temporaries\nconstexpr auto VTMP1 = ARMEmitter::VReg::v0;\nconstexpr auto VTMP2 = ARMEmitter::VReg::v1;\n\n// Predicate register for X87 SVE Optimization\nconstexpr auto SVE_OPT_PRED = ARMEmitter::PReg::p2;\n\n#else\nconstexpr auto TMP1 = ARMEmitter::XReg::x10;\nconstexpr auto TMP2 = ARMEmitter::XReg::x11;\nconstexpr auto TMP3 = ARMEmitter::XReg::x12;\nconstexpr auto TMP4 = ARMEmitter::XReg::x13;\nconstexpr bool TMP_ABIARGS = false;\n\n// We pin r11/r12 as PF/AF respectively for arm64ec, as r26/r27 are used for SRA.\nconstexpr auto REG_PF = ARMEmitter::Reg::r9;\nconstexpr auto REG_AF = ARMEmitter::Reg::r24;\n\nconstexpr auto REG_CALLRET_SP = ARMEmitter::XReg::x17;\n\n// Vector temporaries\nconstexpr auto VTMP1 = ARMEmitter::VReg::v16;\nconstexpr auto VTMP2 = ARMEmitter::VReg::v17;\n\n// Entry/Exit ABI\nconstexpr auto EC_CALL_CHECKER_PC_REG = ARMEmitter::XReg::x9;\nconstexpr auto EC_ENTRY_CPUAREA_REG = ARMEmitter::XReg::x17;\n\n// Predicate register for X87 SVE Optimization\nconstexpr auto SVE_OPT_PRED = ARMEmitter::PReg::p2;\n\n// These structures are not included in the standard Windows headers, define the offsets of members we care about for EC here.\nconstexpr size_t TEB_CPU_AREA_OFFSET = 0x1788;\nconstexpr size_t TEB_PEB_OFFSET = 0x60;\nconstexpr size_t PEB_EC_CODE_BITMAP_OFFSET = 0x368;\nconstexpr size_t CPU_AREA_IN_SYSCALL_CALLBACK_OFFSET = 0x1;\nconstexpr size_t CPU_AREA_EMULATOR_STACK_BASE_OFFSET = 0x8;\nconstexpr size_t CPU_AREA_EMULATOR_DATA_OFFSET = 0x30;\n\nconstexpr uint64_t EC_CODE_BITMAP_MAX_ADDRESS = 1ULL << 47;\n#endif\n\n// Will force one single instruction block to be generated first if set when entering the JIT filling SRA.\n// FillStaticRegs must preserve this\nconstexpr auto ENTRY_FILL_SRA_SINGLE_INST_REG = TMP2;\n\n// Predicate to use in the X87 SVE optimization\nconstexpr ARMEmitter::PRegister PRED_X87_SVEOPT = ARMEmitter::PReg::p2;\n\n// Predicate register temporaries (used when AVX support is enabled)\n// PRED_TMP_16B indicates a predicate register that indicates the first 16 bytes set to 1.\n// PRED_TMP_32B indicates a predicate register that indicates the first 32 bytes set to 1.\nconstexpr ARMEmitter::PRegister PRED_TMP_16B = ARMEmitter::PReg::p6;\nconstexpr ARMEmitter::PRegister PRED_TMP_32B = ARMEmitter::PReg::p7;\n\n\n// This class contains common emitter utility functions that can\n// be used by both Arm64 JIT and ARM64 Dispatcher\nclass Arm64Emitter : public ARMEmitter::Emitter {\npublic:\n Arm64Emitter(FEXCore::Context::ContextImpl* ctx, void* EmissionPtr = nullptr, size_t size = 0);\n\n enum class PadType {\n // Explicitly does not need padding, even if code-caching is enabled.\n NOPAD,\n // Explicitly needs padding, even if code-caching is disabled.\n DOPAD,\n // Choose to pad or not depending on if code-caching is enabled.\n AUTOPAD,\n };\n void LoadConstant(ARMEmitter::Size s, ARMEmitter::Register Reg, uint64_t Constant, PadType Pad = PadType::NOPAD, int MaxBytes = 0);\n\nprotected:\n FEXCore::Context::ContextImpl* EmitterCTX;\n\n std::span StaticRegisters {};\n std::span GeneralRegisters {};\n std::span GeneralRegistersNotPreserved {};\n std::span StaticFPRegisters {};\n std::span GeneralFPRegisters {};\n uint32_t PairRegisters = 0;\n\n void FillSpecialRegs(ARMEmitter::Register TmpReg, ARMEmitter::Register TmpReg2, bool SetFIZ, bool SetPredRegs);\n\n // Correlate an ARM register back to an x86 register index.\n // Returning REG_INVALID if there was no mapping.\n FEXCore::X86State::X86Reg GetX86RegRelationToARMReg(ARMEmitter::Register Reg);\n\n struct SpillStaticRegOptions final {\n uint32_t GPRSpillMask {~0U};\n uint32_t FPRSpillMask {~0U};\n bool FPRs {true};\n bool NZCV {true};\n };\n\n struct FillStaticRegOptions final {\n std::optional OptionalReg {std::nullopt};\n std::optional OptionalReg2 {std::nullopt};\n uint32_t GPRFillMask {~0U};\n uint32_t FPRFillMask {~0U};\n bool FPRs {true};\n bool NZCV {true};\n };\n\n void SpillStaticRegs(ARMEmitter::Register TmpReg, SpillStaticRegOptions Options);\n void FillStaticRegs(FillStaticRegOptions Options);\n\n\n void SpillStaticRegs(ARMEmitter::Register TmpReg) {\n // Work around a clang bug: https://bugs.llvm.org/show_bug.cgi?id=36684\n SpillStaticRegs(TmpReg, {});\n }\n\n void FillStaticRegs() {\n // Work around a clang bug: https://bugs.llvm.org/show_bug.cgi?id=36684\n FillStaticRegs({});\n }\n\n // Register 0-18 + 29 + 30 are caller saved\n static constexpr uint32_t CALLER_GPR_MASK = 0b0110'0000'0000'0111'1111'1111'1111'1111U;\n\n // This isn't technically true because the lower 64-bits of v8..v15 are callee saved\n // We can't guarantee only the lower 64bits are used so flush everything\n static constexpr uint32_t CALLER_FPR_MASK = ~0U;\n\n // Generic push and pop vector registers.\n void PushVectorRegisters(ARMEmitter::Register TmpReg, bool SVERegs, std::span VRegs);\n void PushGeneralRegisters(ARMEmitter::Register TmpReg, std::span Regs);\n\n void PopVectorRegisters(bool SVERegs, std::span VRegs);\n void PopGeneralRegisters(std::span Regs);\n\n // Returns stack size consumed for pushing dynamic registers.\n size_t PushDynamicRegs(ARMEmitter::Register TmpReg);\n void PopDynamicRegs();\n\n void PushCalleeSavedRegisters();\n void PopCalleeSavedRegisters();\n\n // Spills and fills SRA/Dynamic registers that are required for Arm64 `preserve_all` ABI.\n // This ABI changes most registers to be callee saved.\n // Caller Saved:\n // - X0-X8, X16-X18, X30.\n // - v0-v7\n // - For 256-bit SVE hosts: top 128-bits of v8-v31\n //\n // Callee Saved:\n // - X9-X15, X19-X29, X31\n // - Low 128-bits of v8-v31\n size_t SpillForPreserveAllABICall(ARMEmitter::Register TmpReg, bool FPRs = true);\n void FillForPreserveAllABICall(bool FPRs = true);\n\n size_t SpillForABICall(bool SupportsPreserveAllABI, ARMEmitter::Register TmpReg, bool FPRs = true) {\n if (SupportsPreserveAllABI) {\n return SpillForPreserveAllABICall(TmpReg, FPRs);\n } else {\n SpillStaticRegs(TmpReg, {\n .FPRs = FPRs,\n });\n return PushDynamicRegs(TmpReg);\n }\n }\n\n void FillForABICall(bool SupportsPreserveAllABI, bool FPRs = true) {\n if (SupportsPreserveAllABI) {\n FillForPreserveAllABICall(FPRs);\n } else {\n PopDynamicRegs();\n FillStaticRegs({.FPRs = FPRs});\n }\n }\n\n void Align16B();\n\n#ifdef VIXL_SIMULATOR\n // Generates a vixl simulator runtime call.\n //\n // This matches behaviour of vixl's macro assembler, but we need to reimplement it since we aren't using the macro assembler.\n // This isn't too complex with how vixl emits this.\n //\n // Emit:\n // 1) hlt(kRuntimeCallOpcode)\n // 2) Simulator wrapper handler\n // 3) Function to call\n // 4) Style of the function call (Call versus tail-call)\n\n template\n void GenerateRuntimeCall(R (*Function)(P...)) {\n uintptr_t SimulatorWrapperAddress = reinterpret_cast(&(vixl::aarch64::Simulator::RuntimeCallStructHelper::Wrapper));\n\n uintptr_t FunctionAddress = reinterpret_cast(Function);\n\n hlt(vixl::aarch64::kRuntimeCallOpcode);\n\n // Simulator wrapper address pointer.\n dc64(SimulatorWrapperAddress);\n\n // Runtime function address to call\n dc64(FunctionAddress);\n\n // Call type\n dc32(vixl::aarch64::kCallRuntime);\n }\n\n template\n void GenerateIndirectRuntimeCall(ARMEmitter::Register Reg) {\n uintptr_t SimulatorWrapperAddress = reinterpret_cast(&(vixl::aarch64::Simulator::RuntimeCallStructHelper::Wrapper));\n\n hlt(vixl::aarch64::kIndirectRuntimeCallOpcode);\n\n // Simulator wrapper address pointer.\n dc64(SimulatorWrapperAddress);\n\n // Register that contains the function to call\n dc32(Reg.Idx());\n\n // Call type\n dc32(vixl::aarch64::kCallRuntime);\n }\n\n template<>\n void GenerateIndirectRuntimeCall(ARMEmitter::Register Reg) {\n uintptr_t SimulatorWrapperAddress =\n reinterpret_cast(&(vixl::aarch64::Simulator::RuntimeCallStructHelper::Wrapper));\n\n hlt(vixl::aarch64::kIndirectRuntimeCallOpcode);\n\n // Simulator wrapper address pointer.\n dc64(SimulatorWrapperAddress);\n\n // Register that contains the function to call\n dc32(Reg.Idx());\n\n // Call type\n dc32(vixl::aarch64::kCallRuntime);\n }\n#else\n template\n void GenerateRuntimeCall(R (*Function)(P...)) {\n // Explicitly doing nothing.\n }\n template\n void GenerateIndirectRuntimeCall(ARMEmitter::Register Reg) {\n // Explicitly doing nothing.\n }\n#endif\n\n#ifdef VIXL_SIMULATOR\n vixl::aarch64::Decoder SimDecoder;\n vixl::aarch64::Simulator Simulator;\n constexpr static size_t SimulatorStackSize = 8 * 1024 * 1024;\n#endif\n\n#ifdef VIXL_DISASSEMBLER\n fextl::vector DisasmBuffer;\n constexpr static int DISASM_BUFFER_SIZE {256};\n fextl::unique_ptr Disasm;\n fextl::unique_ptr DisasmDecoder;\n\n FEX_CONFIG_OPT(Disassemble, DISASSEMBLE);\n#endif\n\n FEX_CONFIG_OPT(EnableCodeCaching, ENABLECODECACHINGWIP);\n};\n\n} // namespace FEXCore::CPU\n" }, { "path": "FEXCore/Source/Interface/Core/CPUBackend.cpp", "content": "// SPDX-License-Identifier: MIT\n#include \"FEXCore/Config/Config.h\"\n#include \"Interface/Context/Context.h\"\n#include \"Interface/Core/CPUBackend.h\"\n#include \"Interface/Core/LookupCache.h\"\n#include \"Interface/Core/Dispatcher/Dispatcher.h\"\n\n#include \n#include \n#include \n\n#include \n\n#ifndef _WIN32\n#include \n#include \n#endif\n\nnamespace FEXCore {\nnamespace CPU {\n\n static constexpr size_t INITIAL_CODE_SIZE = 1024 * 1024 * 16;\n // We don't want to move above 128MB atm because that means we will have to encode longer jumps\n static constexpr size_t MAX_CODE_SIZE = 1024 * 1024 * 128;\n\n constexpr static uint64_t NamedVectorConstants[FEXCore::IR::NamedVectorConstant::NAMED_VECTOR_CONST_POOL_MAX][2] = {\n {0x0003'0002'0001'0000ULL, 0x0007'0006'0005'0004ULL}, // NAMED_VECTOR_INCREMENTAL_U16_INDEX\n {0x000B'000A'0009'0008ULL, 0x000F'000E'000D'000CULL}, // NAMED_VECTOR_INCREMENTAL_U16_INDEX_UPPER\n {0x0000'0000'8000'0000ULL, 0x0000'0000'8000'0000ULL}, // NAMED_VECTOR_PADDSUBPS_INVERT\n {0x0000'0000'8000'0000ULL, 0x0000'0000'8000'0000ULL}, // NAMED_VECTOR_PADDSUBPS_INVERT_UPPER\n {0x8000'0000'0000'0000ULL, 0x0000'0000'0000'0000ULL}, // NAMED_VECTOR_PADDSUBPD_INVERT\n {0x8000'0000'0000'0000ULL, 0x0000'0000'0000'0000ULL}, // NAMED_VECTOR_PADDSUBPD_INVERT_UPPER\n {0x8000'0000'0000'0000ULL, 0x8000'0000'0000'0000ULL}, // NAMED_VECTOR_PSUBADDPS_INVERT\n {0x8000'0000'0000'0000ULL, 0x8000'0000'0000'0000ULL}, // NAMED_VECTOR_PSUBADDPS_INVERT_UPPER\n {0x0000'0000'0000'0000ULL, 0x8000'0000'0000'0000ULL}, // NAMED_VECTOR_PSUBADDPD_INVERT\n {0x0000'0000'0000'0000ULL, 0x8000'0000'0000'0000ULL}, // NAMED_VECTOR_PSUBADDPD_INVERT_UPPER\n {0x0000'0001'0000'0000ULL, 0x0000'0003'0000'0002ULL}, // NAMED_VECTOR_MOVMSKPS_SHIFT\n {0x040B'0E01'0B0E'0104ULL, 0x0C03'0609'0306'090CULL}, // NAMED_VECTOR_AESKEYGENASSIST_SWIZZLE\n {0x0706'0504'FFFF'FFFFULL, 0xFFFF'FFFF'0B0A'0908ULL}, // NAMED_VECTOR_BLENDPS_0110B\n {0x0706'0504'0302'0100ULL, 0xFFFF'FFFF'0B0A'0908ULL}, // NAMED_VECTOR_BLENDPS_0111B\n {0xFFFF'FFFF'0302'0100ULL, 0x0F0E'0D0C'FFFF'FFFFULL}, // NAMED_VECTOR_BLENDPS_1001B\n {0x0706'0504'0302'0100ULL, 0x0F0E'0D0C'FFFF'FFFFULL}, // NAMED_VECTOR_BLENDPS_1011B\n {0xFFFF'FFFF'0302'0100ULL, 0x0F0E'0D0C'0B0A'0908ULL}, // NAMED_VECTOR_BLENDPS_1101B\n {0x0706'0504'FFFF'FFFFULL, 0x0F0E'0D0C'0B0A'0908ULL}, // NAMED_VECTOR_BLENDPS_1110B\n {0x8040'2010'0804'0201ULL, 0x8040'2010'0804'0201ULL}, // NAMED_VECTOR_MOVMASKB\n {0x8040'2010'0804'0201ULL, 0x8040'2010'0804'0201ULL}, // NAMED_VECTOR_MOVMASKB_UPPER\n {0x8000'0000'0000'0000ULL, 0x0000'0000'0000'3FFFULL}, // NAMED_VECTOR_X87_ONE\n {0xD49A'784B'CD1B'8AFEULL, 0x0000'0000'0000'4000ULL}, // NAMED_VECTOR_X87_LOG2_10\n {0xB8AA'3B29'5C17'F0BCULL, 0x0000'0000'0000'3FFFULL}, // NAMED_VECTOR_X87_LOG2_E\n {0xC90F'DAA2'2168'C235ULL, 0x0000'0000'0000'4000ULL}, // NAMED_VECTOR_X87_PI\n {0x9A20'9A84'FBCF'F799ULL, 0x0000'0000'0000'3FFDULL}, // NAMED_VECTOR_X87_LOG10_2\n {0xB172'17F7'D1CF'79ACULL, 0x0000'0000'0000'3FFEULL}, // NAMED_VECTOR_X87_LOG_2\n {0x4F00'0000'4F00'0000ULL, 0x4F00'0000'4F00'0000ULL}, // NAMED_VECTOR_CVTMAX_F32_I32\n {0x4F00'0000'4F00'0000ULL, 0x4F00'0000'4F00'0000ULL}, // NAMED_VECTOR_CVTMAX_F32_I32_UPPER\n {0x5F00'0000'5F00'0000ULL, 0x5F00'0000'5F00'0000ULL}, // NAMED_VECTOR_CVTMAX_F32_I64\n {0x41E0'0000'0000'0000ULL, 0x41E0'0000'0000'0000ULL}, // NAMED_VECTOR_CVTMAX_F64_I32\n {0x41E0'0000'0000'0000ULL, 0x41E0'0000'0000'0000ULL}, // NAMED_VECTOR_CVTMAX_F64_I32_UPPER\n {0x43E0'0000'0000'0000ULL, 0x43E0'0000'0000'0000ULL}, // NAMED_VECTOR_CVTMAX_F64_I64\n {0x8000'0000'8000'0000ULL, 0x8000'0000'8000'0000ULL}, // NAMED_VECTOR_CVTMAX_I32\n {0x8000'0000'0000'0000ULL, 0x8000'0000'0000'0000ULL}, // NAMED_VECTOR_CVTMAX_I64\n {0x0000'0000'0000'0000ULL, 0x0000'0000'0000'8000ULL}, // NAMED_VECTOR_F80_SIGN_MASK\n {0x5A82'7999'5A82'7999ULL, 0x5A82'7999'5A82'7999ULL}, // NAMED_VECTOR_SHA1RNDS_K0\n {0x6ED9'EBA1'6ED9'EBA1ULL, 0x6ED9'EBA1'6ED9'EBA1ULL}, // NAMED_VECTOR_SHA1RNDS_K1\n {0x8F1B'BCDC'8F1B'BCDCULL, 0x8F1B'BCDC'8F1B'BCDCULL}, // NAMED_VECTOR_SHA1RNDS_K2\n {0xCA62'C1D6'CA62'C1D6ULL, 0xCA62'C1D6'CA62'C1D6ULL}, // NAMED_VECTOR_SHA1RNDS_K3\n };\n\n constexpr static auto PSHUFLW_LUT {[]() consteval {\n struct LUTType {\n uint64_t Val[2];\n };\n // Expectation for this LUT is to simulate PSHUFLW with ARM's TBL (single register) instruction\n // PSHUFLW behaviour:\n // 16-bit words in [63:48], [47:32], [31:16], [15:0] are selected using the 8-bit Index.\n // For 128-bit PSHUFLW, bits [127:64] are identity copied.\n constexpr uint64_t IdentityCopyUpper = 0x0f'0e'0d'0c'0b'0a'09'08;\n std::array TotalLUT {};\n uint64_t WordSelection[4] = {\n 0x01'00,\n 0x03'02,\n 0x05'04,\n 0x07'06,\n };\n for (size_t i = 0; i < 256; ++i) {\n auto& LUT = TotalLUT[i];\n const auto Word0 = (i >> 0) & 0b11;\n const auto Word1 = (i >> 2) & 0b11;\n const auto Word2 = (i >> 4) & 0b11;\n const auto Word3 = (i >> 6) & 0b11;\n\n LUT.Val[0] = (WordSelection[Word0] << 0) | (WordSelection[Word1] << 16) | (WordSelection[Word2] << 32) | (WordSelection[Word3] << 48);\n\n LUT.Val[1] = IdentityCopyUpper;\n }\n return TotalLUT;\n }()};\n\n constexpr static auto PSHUFHW_LUT {[]() consteval {\n struct LUTType {\n uint64_t Val[2];\n };\n // Expectation for this LUT is to simulate PSHUFHW with ARM's TBL (single register) instruction\n // PSHUFHW behaviour:\n // 16-bit words in [127:112], [111:96], [95:80], [79:64] are selected using the 8-bit Index.\n // Incoming words come from bits [127:64] of the source.\n // Bits [63:0] are identity copied.\n constexpr uint64_t IdentityCopyLower = 0x07'06'05'04'03'02'01'00;\n std::array TotalLUT {};\n uint64_t WordSelection[4] = {\n 0x09'08,\n 0x0b'0a,\n 0x0d'0c,\n 0x0f'0e,\n };\n for (size_t i = 0; i < 256; ++i) {\n auto& LUT = TotalLUT[i];\n const auto Word0 = (i >> 0) & 0b11;\n const auto Word1 = (i >> 2) & 0b11;\n const auto Word2 = (i >> 4) & 0b11;\n const auto Word3 = (i >> 6) & 0b11;\n\n LUT.Val[0] = IdentityCopyLower;\n\n LUT.Val[1] = (WordSelection[Word0] << 0) | (WordSelection[Word1] << 16) | (WordSelection[Word2] << 32) | (WordSelection[Word3] << 48);\n }\n return TotalLUT;\n }()};\n\n constexpr static auto PSHUFD_LUT {[]() consteval {\n struct LUTType {\n uint64_t Val[2];\n };\n // Expectation for this LUT is to simulate PSHUFD with ARM's TBL (single register) instruction\n // PSHUFD behaviour:\n // 32-bit words in [127:96], [95:64], [63:32], [31:0] are selected using the 8-bit Index.\n std::array TotalLUT {};\n uint64_t WordSelection[4] = {\n 0x03'02'01'00,\n 0x07'06'05'04,\n 0x0b'0a'09'08,\n 0x0f'0e'0d'0c,\n };\n for (size_t i = 0; i < 256; ++i) {\n auto& LUT = TotalLUT[i];\n const auto Word0 = (i >> 0) & 0b11;\n const auto Word1 = (i >> 2) & 0b11;\n const auto Word2 = (i >> 4) & 0b11;\n const auto Word3 = (i >> 6) & 0b11;\n\n LUT.Val[0] = (WordSelection[Word0] << 0) | (WordSelection[Word1] << 32);\n\n LUT.Val[1] = (WordSelection[Word2] << 0) | (WordSelection[Word3] << 32);\n }\n return TotalLUT;\n }()};\n\n constexpr static auto SHUFPS_LUT {[]() consteval {\n struct LUTType {\n uint64_t Val[2];\n };\n // 32-bit words in [127:96], [95:64], [63:32], [31:0] are selected using the 8-bit Index.\n // Expectation for this LUT is to simulate SHUFPS with ARM's TBL (two register) instruction.\n // SHUFPS behaviour:\n // Two 32-bits words from each source are selected from each source in the lower and upper halves of the 128-bit destination.\n // Dest[31:0] = Src1[]\n // Dest[63:32] = Src1[]\n // Dest[95:64] = Src2[]\n // Dest[127:96] = Src2[]\n\n std::array TotalLUT {};\n const uint64_t WordSelectionSrc1[4] = {\n 0x03'02'01'00,\n 0x07'06'05'04,\n 0x0b'0a'09'08,\n 0x0f'0e'0d'0c,\n };\n\n // Src2 needs to offset each byte index by 16-bytes to pull from the second source.\n const uint64_t WordSelectionSrc2[4] = {\n 0x03'02'01'00 + (0x10101010),\n 0x07'06'05'04 + (0x10101010),\n 0x0b'0a'09'08 + (0x10101010),\n 0x0f'0e'0d'0c + (0x10101010),\n };\n\n for (size_t i = 0; i < 256; ++i) {\n auto& LUT = TotalLUT[i];\n const auto Word0 = (i >> 0) & 0b11;\n const auto Word1 = (i >> 2) & 0b11;\n const auto Word2 = (i >> 4) & 0b11;\n const auto Word3 = (i >> 6) & 0b11;\n\n LUT.Val[0] = (WordSelectionSrc1[Word0] << 0) | (WordSelectionSrc1[Word1] << 32);\n\n LUT.Val[1] = (WordSelectionSrc2[Word2] << 0) | (WordSelectionSrc2[Word3] << 32);\n }\n return TotalLUT;\n }()};\n\n constexpr static auto DPPS_MASK {[]() consteval {\n struct LUTType {\n uint32_t Val[4];\n };\n\n std::array TotalLUT {};\n for (size_t i = 0; i < TotalLUT.size(); ++i) {\n auto& LUT = TotalLUT[i];\n constexpr auto GetLUT = [](size_t i, size_t Index) {\n if (i & (1U << Index)) {\n return -1U;\n }\n return 0U;\n };\n\n LUT.Val[0] = GetLUT(i, 0);\n LUT.Val[1] = GetLUT(i, 1);\n LUT.Val[2] = GetLUT(i, 2);\n LUT.Val[3] = GetLUT(i, 3);\n }\n return TotalLUT;\n }()};\n\n constexpr static auto DPPD_MASK {[]() consteval {\n struct LUTType {\n uint64_t Val[2];\n };\n\n std::array TotalLUT {};\n for (size_t i = 0; i < TotalLUT.size(); ++i) {\n auto& LUT = TotalLUT[i];\n constexpr auto GetLUT = [](size_t i, size_t Index) {\n if (i & (1U << Index)) {\n return -1ULL;\n }\n return 0ULL;\n };\n\n LUT.Val[0] = GetLUT(i, 0);\n LUT.Val[1] = GetLUT(i, 1);\n }\n return TotalLUT;\n }()};\n\n constexpr static auto PBLENDW_LUT {[]() consteval {\n struct LUTType {\n uint16_t Val[8];\n };\n // 16-bit words in [127:112], [111:96], [95:80], [79:64], [63:48], [47:32], [31:16], [15:0] are selected using 8-bit swizzle.\n // Expectation for this LUT is to simulate PBLENDW with ARM's TBX (one register) instruction.\n // PBLENDW behaviour:\n // 16-bit words from the source is moved in to the destination based on the bit in the swizzle.\n // Dest[15:0] = Swizzle[0] ? Src[15:0] : Dest[15:0]\n // Dest[31:16] = Swizzle[1] ? Src[31:16] : Dest[31:16]\n // Dest[47:32] = Swizzle[2] ? Src[47:32] : Dest[47:32]\n // Dest[63:48] = Swizzle[3] ? Src[63:48] : Dest[63:48]\n // Dest[79:64] = Swizzle[4] ? Src[79:64] : Dest[79:64]\n // Dest[95:80] = Swizzle[5] ? Src[95:80] : Dest[95:80]\n // Dest[111:96] = Swizzle[6] ? Src[111:96] : Dest[111:96]\n // Dest[127:112] = Swizzle[7] ? Src[127:112] : Dest[127:112]\n\n std::array TotalLUT {};\n const uint16_t WordSelectionSrc[8] = {\n 0x01'00, 0x03'02, 0x05'04, 0x07'06, 0x09'08, 0x0B'0A, 0x0D'0C, 0x0F'0E,\n };\n\n constexpr uint16_t OriginalDest = 0xFF'FF;\n\n for (size_t i = 0; i < 256; ++i) {\n auto& LUT = TotalLUT[i];\n for (size_t j = 0; j < 8; ++j) {\n LUT.Val[j] = ((i >> j) & 1) ? WordSelectionSrc[j] : OriginalDest;\n }\n }\n return TotalLUT;\n }()};\n\n CPUBackend::CPUBackend(CodeBufferManager& CodeBuffers, FEXCore::Core::InternalThreadState* ThreadState)\n : ThreadState(ThreadState)\n , CodeBuffers(CodeBuffers) {\n\n auto& Ptrs = ThreadState->CurrentFrame->Pointers;\n\n // Initialize named vector constants.\n for (size_t i = 0; i < FEXCore::IR::NamedVectorConstant::NAMED_VECTOR_CONST_POOL_MAX; ++i) {\n Ptrs.NamedVectorConstantPointers[i] = reinterpret_cast(NamedVectorConstants[i]);\n }\n\n // Copy named vector constants.\n memcpy(Ptrs.NamedVectorConstants, NamedVectorConstants, sizeof(NamedVectorConstants));\n\n // Initialize Indexed named vector constants.\n Ptrs.IndexedNamedVectorConstantPointers[FEXCore::IR::IndexNamedVectorConstant::INDEXED_NAMED_VECTOR_PSHUFLW] =\n reinterpret_cast(PSHUFLW_LUT.data());\n Ptrs.IndexedNamedVectorConstantPointers[FEXCore::IR::IndexNamedVectorConstant::INDEXED_NAMED_VECTOR_PSHUFHW] =\n reinterpret_cast(PSHUFHW_LUT.data());\n Ptrs.IndexedNamedVectorConstantPointers[FEXCore::IR::IndexNamedVectorConstant::INDEXED_NAMED_VECTOR_PSHUFD] =\n reinterpret_cast(PSHUFD_LUT.data());\n Ptrs.IndexedNamedVectorConstantPointers[FEXCore::IR::IndexNamedVectorConstant::INDEXED_NAMED_VECTOR_SHUFPS] =\n reinterpret_cast(SHUFPS_LUT.data());\n Ptrs.IndexedNamedVectorConstantPointers[FEXCore::IR::IndexNamedVectorConstant::INDEXED_NAMED_VECTOR_DPPS_MASK] =\n reinterpret_cast(DPPS_MASK.data());\n Ptrs.IndexedNamedVectorConstantPointers[FEXCore::IR::IndexNamedVectorConstant::INDEXED_NAMED_VECTOR_DPPD_MASK] =\n reinterpret_cast(DPPD_MASK.data());\n Ptrs.IndexedNamedVectorConstantPointers[FEXCore::IR::IndexNamedVectorConstant::INDEXED_NAMED_VECTOR_PBLENDW] =\n reinterpret_cast(PBLENDW_LUT.data());\n\n#ifndef FEX_DISABLE_TELEMETRY\n // Fill in telemetry values\n for (size_t i = 0; i < FEXCore::Telemetry::TYPE_LAST; ++i) {\n auto& Telem = FEXCore::Telemetry::GetTelemetryValue(static_cast(i));\n Ptrs.TelemetryValueAddresses[i] = reinterpret_cast(&Telem);\n }\n#endif\n }\n\n CPUBackend::~CPUBackend() = default;\n\n auto CPUBackend::GetEmptyCodeBuffer() -> CodeBuffer* {\n auto PrevCodeBuffer = CurrentCodeBuffer;\n\n // Resize the code buffer and reallocate our code size\n CurrentCodeBuffer = CodeBuffers.StartLargerCodeBuffer();\n\n RegisterForSignalHandler(std::move(PrevCodeBuffer));\n return CurrentCodeBuffer.get();\n }\n\n void CPUBackend::RegisterForSignalHandler(fextl::shared_ptr CodeBuffer) {\n if (ThreadState->CurrentFrame->SignalHandlerRefCounter != 0) {\n // We have signal handlers that have generated code\n // This means that we can not safely clear the code at this point in time\n // Keep a reference to the old code buffer to delay deallocation\n SignalHandlerCodeBuffers.push_back(std::move(CodeBuffer));\n } else {\n SignalHandlerCodeBuffers.clear();\n }\n }\n\n fextl::shared_ptr CPUBackend::CheckCodeBufferUpdate() {\n auto NewCodeBuffer = CodeBuffers.GetLatest();\n if (CurrentCodeBuffer != NewCodeBuffer) {\n RegisterForSignalHandler(CurrentCodeBuffer);\n return std::exchange(CurrentCodeBuffer, NewCodeBuffer);\n }\n return nullptr;\n }\n\n GuestToHostMap& GetLookupCache(const CodeBuffer& Buffer) {\n return *Buffer.LookupCache;\n }\n\n CodeBuffer::CodeBuffer(size_t Size)\n : AllocatedSize(Size) {\n Ptr = static_cast(FEXCore::Allocator::VirtualAlloc(Size, true));\n LOGMAN_THROW_A_FMT(!!Ptr, \"Couldn't allocate code buffer\");\n\n // Protect the last page of the allocated buffer to trigger SIGSEGV on write access\n uintptr_t LastPageAddr = AlignDown(reinterpret_cast(Ptr) + Size - 1, FEXCore::Utils::FEX_PAGE_SIZE);\n if (!FEXCore::Allocator::VirtualProtect(reinterpret_cast(LastPageAddr), FEXCore::Utils::FEX_PAGE_SIZE,\n FEXCore::Allocator::ProtectOptions::None)) {\n LogMan::Msg::EFmt(\"Failed to mprotect last page of code buffer.\");\n }\n\n FEXCore::Allocator::VirtualName(\"FEXMemJIT\", reinterpret_cast(Ptr), Size);\n\n // Huge-pages reduce the amount of iTLB misses dramatically when it works.\n FEXCore::Allocator::VirtualTHPControl(reinterpret_cast(Ptr), Size, FEXCore::Allocator::THPControl::Enable);\n\n LookupCache = fextl::make_unique();\n }\n\n CodeBuffer::~CodeBuffer() {\n FEXCore::Allocator::VirtualFree(Ptr, AllocatedSize);\n }\n\n auto CodeBufferManager::AllocateNew(size_t Size) -> fextl::shared_ptr {\n#ifndef _WIN32\n// MDWE (Memory-Deny-Write-Execute) is a new Linux 6.3 feature.\n// It's equivalent to systemd's `MemoryDenyWriteExecute` but implemented entirely in the kernel.\n//\n// MDWE prevents applications from creating RWX memory mappings.\n// This prevents FEX from doing anything JIT related, as FEX uses RWX for JIT memory mappings.\n//\n// A potential workaround to make FEX work with MDWE is to call mprotect every time we need to write or modify code.\n// Alternatively, FEX could use a memory mirror where one half is mapped as RW and the other is RX.\n//\n// Once MDWE is enabled with the prctl, the feature is sealed and it can /NOT/ be turned off.\n//\n// Status of MDWE is queried through prctl using `PR_GET_MDWE`:\n// -1: The kernel doesn't support MDWE\n// 0: MDWE is supported but disabled\n// >0: MDWE is enabled, hence prohibiting RWX mappings\n#ifndef PR_GET_MDWE\n#define PR_GET_MDWE 66\n#endif\n int MDWE = ::prctl(PR_GET_MDWE, 0, 0, 0, 0);\n if (MDWE != -1 && MDWE != 0) {\n LogMan::Msg::EFmt(\"MDWE was set to 0x{:x} which means FEX can't allocate executable memory\", MDWE);\n }\n#endif\n\n auto Buffer = fextl::make_shared(Size);\n\n Latest = Buffer;\n LatestOffset = 0;\n\n OnCodeBufferAllocated(Buffer);\n\n return Buffer;\n }\n\n fextl::shared_ptr CodeBufferManager::GetLatest() {\n if (!Latest) {\n if (FEXCore::Config::Get_ENABLECODECACHINGWIP()) {\n // Start with a larger code buffer to avoid resizes that would discard\n // code loaded from caches\n AllocateNew(MAX_CODE_SIZE);\n } else {\n AllocateNew(INITIAL_CODE_SIZE);\n }\n }\n return Latest;\n }\n\n fextl::shared_ptr CodeBufferManager::StartLargerCodeBuffer() {\n if (!Latest) {\n // Allocate initial CodeBuffer and return it\n return GetLatest();\n }\n\n auto NewCodeBufferSize = GetLatest()->AllocatedSize;\n NewCodeBufferSize = std::min(NewCodeBufferSize * 2, MAX_CODE_SIZE);\n return AllocateNew(NewCodeBufferSize);\n }\n\n\n bool CPUBackend::IsAddressInCodeBuffer(uintptr_t Address) const {\n auto CheckCodeBuffer = [](CodeBuffer& Buffer, uintptr_t Address) {\n // The last page of the code buffer is protected, so we need to exclude it from the valid range\n // when checking if the address is in the code buffer.\n uintptr_t LastPageAddr = AlignDown(reinterpret_cast(Buffer.Ptr) + Buffer.AllocatedSize - 1, FEXCore::Utils::FEX_PAGE_SIZE);\n return (Address >= reinterpret_cast(Buffer.Ptr) && Address < LastPageAddr);\n };\n\n if (CheckCodeBuffer(*CurrentCodeBuffer, Address)) {\n return true;\n }\n for (auto& Buffer : SignalHandlerCodeBuffers) {\n if (CheckCodeBuffer(*Buffer, Address)) {\n return true;\n }\n }\n return false;\n }\n\n} // namespace CPU\n} // namespace FEXCore\n" }, { "path": "FEXCore/Source/Interface/Core/CPUBackend.h", "content": "// SPDX-License-Identifier: MIT\n/*\n$info$\ncategory: backend ~ IR to host code generation\ntags: backend|shared\n$end_info$\n*/\n\n#pragma once\n\n#include \n#include \n#include \n#include \n#include \n#include \n\n#include \n\nnamespace FEXCore::CPU {\nunion Relocation;\n}\n\nnamespace FEXCore {\n\nnamespace IR {\n class IRListView;\n} // namespace IR\n\nnamespace Core {\n struct DebugData;\n struct ThreadState;\n struct CpuStateFrame;\n struct InternalThreadState;\n} // namespace Core\n\nnamespace CodeSerialize {\n struct CodeObjectFileSection;\n}\n\nstruct GuestToHostMap;\n\nnamespace CPU {\n struct CodeBuffer {\n uint8_t* Ptr;\n size_t AllocatedSize; // including guard page; see UsableSize()\n\n fextl::unique_ptr LookupCache;\n\n CodeBuffer(size_t Size);\n CodeBuffer(const CodeBuffer&) = delete;\n CodeBuffer& operator=(const CodeBuffer&) = delete;\n CodeBuffer(CodeBuffer&& oth) = delete;\n CodeBuffer& operator=(CodeBuffer&&) = delete;\n\n ~CodeBuffer();\n\n /// Returns the number of bytes available for storing code\n size_t UsableSize() const {\n return AllocatedSize - FEXCore::Utils::FEX_PAGE_SIZE;\n }\n };\n\n /**\n * A manager that coordinates access to the CodeBuffer used for compiling new code across threads.\n *\n * The CodeBuffer is managed as a partially persistent data structure:\n * - Exactly one CodeBuffer is now designated as \"active\", which means data can be appended to it\n * - Lossy modifications to the active CodeBuffer will not invalidate any data in use by other threads (which is what enables save CodeBuffer sharing across threads)\n * - Instead, such lossy modifications trigger a new \"version\" of the data in the modifying thread. Old versions of the CodeBuffer persist as read-only data for use by the other threads.\n * - The other threads can update their version of the CodeBuffer. This will decrease the reference count and eventually trigger deallocation of the old version\n */\n class CodeBufferManager {\n public:\n // Get the CodeBuffer that was most recently allocated.\n // This is the only CodeBuffer that data may be written to.\n fextl::shared_ptr GetLatest();\n\n // Allocate a new CodeBuffer with geometric growth up to an internal maximum.\n // Subsequent calls to GetLatest will point to the returned buffer.\n fextl::shared_ptr StartLargerCodeBuffer();\n\n // Write offset into the latest CodeBuffer\n std::size_t LatestOffset {};\n\n // Protects writes to the latest CodeBuffer and changes to LatestOffset\n FEXCore::ForkableUniqueMutex CodeBufferWriteMutex;\n\n virtual void OnCodeBufferAllocated(const std::shared_ptr&) {};\n\n private:\n fextl::shared_ptr Latest;\n\n fextl::shared_ptr AllocateNew(size_t Size);\n };\n\n class CPUBackend {\n public:\n\n CPUBackend(CodeBufferManager&, FEXCore::Core::InternalThreadState*);\n\n virtual ~CPUBackend();\n\n struct CompiledCode {\n // Where this code block begins.\n uint8_t* BlockBegin;\n fextl::map EntryPoints;\n // The total size of the codeblock from [BlockBegin, BlockBegin+Size).\n size_t Size;\n };\n\n // Header that can live at the start of a JIT block.\n // We want the header to be quite small, with most data living in the tail object.\n struct JITCodeHeader {\n // Offset from the start of this header to where the tail lives.\n // Only 32-bit since the tail block won't ever be more than 4GB away.\n uint32_t OffsetToBlockTail;\n };\n\n // Header that can live at the end of the JIT block.\n // For any state reconstruction or other data, this is where it should live.\n // Any data that is explicitly tied to the JIT code and needs to be cached with it\n // should end up in this data structure.\n struct JITCodeTail {\n // The total size of the codeblock from [BlockBegin, BlockBegin+Size).\n size_t Size;\n\n // RIP that the block's entry comes from.\n uint64_t RIP;\n\n // The length of the guest code for this block.\n size_t GuestSize;\n\n // Number of RIP entries for this JIT Code section.\n uint32_t NumberOfRIPEntries;\n\n // Offset after this block to the start of the RIP entries.\n uint32_t OffsetToRIPEntries;\n\n // Shared-code modification spin-loop futex.\n uint32_t SpinLockFutex;\n\n // If this block represents a single guest instruction.\n bool SingleInst;\n\n uint8_t _Pad[3];\n };\n\n /**\n * @brief Tells this CPUBackend to compile code for the provided IR and DebugData\n *\n * The returned pointer needs to be long lived and be executable in the host environment\n * FEXCore's frontend will store this pointer in to a cache for the current RIP when this was executed\n *\n * This is a thread specific compilation unit since there is one CPUBackend per guest thread\n *\n * @param Size - The byte size of the guest code for this block\n * @param SingleInst - If this block represents a single guest instruction\n * @param IR - IR that maps to the IR for this RIP\n * @param DebugData - Debug data that is available for this IR indirectly\n * @param CheckTF - If EFLAGS.TF checks should be emitted at the start of the block\n *\n * @return Information about the compiled code block.\n */\n [[nodiscard]]\n virtual CompiledCode CompileCode(uint64_t Entry, uint64_t Size, bool SingleInst, const FEXCore::IR::IRListView* IR,\n FEXCore::Core::DebugData* DebugData, bool CheckTF) = 0;\n\n virtual fextl::vector TakeRelocations(uint64_t GuestBaseAddress) = 0;\n\n virtual void ClearCache() {}\n\n /**\n * @brief Clear any relocations after JIT compiling\n */\n virtual void ClearRelocations() {}\n\n bool IsAddressInCodeBuffer(uintptr_t Address) const;\n\n // Updates the CodeBuffer if needed and returns a reference to the old one.\n // The returned reference should be kept alive carefully to avoid early deletion of resources.\n [[nodiscard]]\n fextl::shared_ptr CheckCodeBufferUpdate();\n\n protected:\n // Max spill slot size in bytes. We need at most 32 bytes\n // to be able to handle a 256-bit vector store to a slot.\n constexpr static uint32_t MaxSpillSlotSize = 32;\n\n FEXCore::Core::InternalThreadState* ThreadState;\n\n [[nodiscard]]\n CodeBuffer* GetEmptyCodeBuffer();\n\n // This is the code buffer containing the main code under execution by this thread.\n // CheckCodeBufferUpdate must be used before compiling new code.\n fextl::shared_ptr CurrentCodeBuffer;\n\n // Old CodeBuffer generations required to be valid until returning from signal handlers\n fextl::vector> SignalHandlerCodeBuffers;\n\n CodeBufferManager& CodeBuffers;\n\n private:\n void RegisterForSignalHandler(fextl::shared_ptr);\n };\n\n} // namespace CPU\n} // namespace FEXCore\n" }, { "path": "FEXCore/Source/Interface/Core/CPUID.cpp", "content": "// SPDX-License-Identifier: MIT\n/*\n$info$\ntags: opcodes|cpuid\ndesc: Handles presented capability bits for guest cpu\n$end_info$\n*/\n\n#include \"Common/StringConv.h\"\n#include \"Interface/Context/Context.h\"\n#include \"Interface/Core/CPUID.h\"\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n#include \"git_version.h\"\n\n#include \n\nnamespace FEXCore {\nnamespace ProductNames {\n#ifdef ARCHITECTURE_arm64\n static const char ARM_UNKNOWN[] = \"Unknown ARM CPU\";\n static const char ARM_A57[] = \"Cortex-A57\";\n static const char ARM_A72[] = \"Cortex-A72\";\n static const char ARM_A73[] = \"Cortex-A73\";\n static const char ARM_A75[] = \"Cortex-A75\";\n static const char ARM_A76[] = \"Cortex-A76\";\n static const char ARM_A76AE[] = \"Cortex-A76AE\";\n static const char ARM_V1[] = \"Neoverse V1\";\n static const char ARM_V2[] = \"Neoverse V2\";\n static const char ARM_V3[] = \"Neoverse V3\";\n static const char ARM_V3AE[] = \"Neoverse V3AE\";\n static const char ARM_A77[] = \"Cortex-A77\";\n static const char ARM_A78[] = \"Cortex-A78\";\n static const char ARM_A78AE[] = \"Cortex-A78AE\";\n static const char ARM_A78C[] = \"Cortex-A78C\";\n static const char ARM_A710[] = \"Cortex-A710\";\n static const char ARM_A715[] = \"Cortex-A715\";\n static const char ARM_A720[] = \"Cortex-A720\";\n static const char ARM_A725[] = \"Cortex-A725\";\n static const char ARM_C1Pro[] = \"C1-Pro\";\n static const char ARM_C1Premium[] = \"C1-Premium\";\n static const char ARM_X1[] = \"Cortex-X1\";\n static const char ARM_X1C[] = \"Cortex-X1C\";\n static const char ARM_X2[] = \"Cortex-X2\";\n static const char ARM_X3[] = \"Cortex-X3\";\n static const char ARM_X4[] = \"Cortex-X4\";\n static const char ARM_X925[] = \"Cortex-X925\";\n static const char ARM_C1Ultra[] = \"C1-Ultra\";\n static const char ARM_N1[] = \"Neoverse N1\";\n static const char ARM_N2[] = \"Neoverse N2\";\n static const char ARM_N3[] = \"Neoverse N3\";\n static const char ARM_E1[] = \"Neoverse E1\";\n static const char ARM_A35[] = \"Cortex-A35\";\n static const char ARM_A53[] = \"Cortex-A53\";\n static const char ARM_A55[] = \"Cortex-A55\";\n static const char ARM_A65[] = \"Cortex-A65\";\n static const char ARM_A510[] = \"Cortex-A510\";\n static const char ARM_A520[] = \"Cortex-A520\";\n static const char ARM_C1Nano[] = \"C1-Nano\";\n\n static const char ARM_Kryo200[] = \"Kryo 2xx\";\n static const char ARM_Kryo300[] = \"Kryo 3xx\";\n static const char ARM_Kryo400[] = \"Kryo 4xx/5xx\";\n\n static const char ARM_Kryo200S[] = \"Kryo 2xx S\";\n static const char ARM_Kryo300S[] = \"Kryo 3xx S\";\n static const char ARM_Kryo400S[] = \"Kryo 4xx/5xx S\";\n\n static const char ARM_Denver[] = \"Nvidia Denver\";\n static const char ARM_Carmel[] = \"Nvidia Carmel\";\n static const char ARM_Olympus[] = \"Nvidia Olympus\";\n\n static const char ARM_Firestorm_M1[] = \"Apple Firestorm (M1)\";\n static const char ARM_Icestorm_M1[] = \"Apple Icestorm (M1)\";\n static const char ARM_Firestorm_M1Pro[] = \"Apple Firestorm (M1 Pro)\";\n static const char ARM_Icestorm_M1Pro[] = \"Apple Icestorm (M1 Pro)\";\n static const char ARM_Firestorm_M1Max[] = \"Apple Firestorm (M1 Max)\";\n static const char ARM_Icestorm_M1Max[] = \"Apple Icestorm (M1 Max)\";\n static const char ARM_Avalanche_M2[] = \"Apple Avalanche (M2)\";\n static const char ARM_Blizzard_M2[] = \"Apple Blizzard (M2)\";\n static const char ARM_Avalanche_M2Pro[] = \"Apple Avalanche (M2 Pro)\";\n static const char ARM_Blizzard_M2Pro[] = \"Apple Blizzard (M2 Pro)\";\n static const char ARM_Avalanche_M2Max[] = \"Apple Avalanche (M2 Max)\";\n static const char ARM_Blizzard_M2Max[] = \"Apple Blizzard (M2 Max)\";\n static const char ARM_AppleSilicon[] = \"Apple Silicon\";\n\n static const char ARM_ORYON_1[] = \"Oryon-1\";\n static const char ARM_Ampere_1[] = \"AmpereOne\";\n static const char ARM_Ampere_1A[] = \"AmpereOneA\";\n static const char ARM_Ampere_1B[] = \"AmpereOneB\";\n static const char ARM_Ampere_1C[] = \"AmpereOneC\";\n#endif\n} // namespace ProductNames\n\nuint32_t GetCPUID_Syscall() {\n uint32_t CPU {};\n FHU::Syscalls::getcpu(&CPU, nullptr);\n return CPU;\n}\n\nstruct CPUFamily {\n uint32_t Stepping : 4;\n uint32_t Model : 4;\n uint32_t ExtendedModel : 4;\n uint32_t FamilyID : 4;\n uint32_t ExtendedFamilyID : 8;\n uint32_t ProcessorType : 4;\n};\n\nconstexpr static uint32_t GenerateFamily(const CPUFamily Family) {\n return Family.Stepping | (Family.Model << 4) | (Family.FamilyID << 8) | (Family.ProcessorType << 12) | (Family.ExtendedModel << 16) |\n (Family.ExtendedFamilyID << 20);\n}\n\n#ifdef CPUID_AMD\nconstexpr uint32_t FAMILY_IDENTIFIER = GenerateFamily(CPUFamily {\n .Stepping = 0,\n .Model = 0xA,\n .ExtendedModel = 0,\n .FamilyID = 0xF,\n .ExtendedFamilyID = 1,\n .ProcessorType = 0,\n});\n\n#else\nconstexpr uint32_t FAMILY_IDENTIFIER = GenerateFamily(CPUFamily {\n .Stepping = 1,\n .Model = 6,\n .ExtendedModel = 0xA,\n .FamilyID = 6,\n .ExtendedFamilyID = 0,\n .ProcessorType = 0,\n});\n#endif\n\n#ifdef ARCHITECTURE_arm64\nuint64_t GetCycleCounterFrequency() {\n uint64_t Result {};\n __asm(\"mrs %[Res], CNTFRQ_EL0\" : [Res] \"=r\"(Result));\n return Result;\n}\n\nuint32_t GetCPUID_TPIDRRO() {\n uint64_t Result {};\n __asm(\"mrs %[Res], TPIDRRO_EL0\" : [Res] \"=r\"(Result));\n return Result;\n}\n\nvoid CPUIDEmu::SetupHostHybridFlag() {\n FEX_CONFIG_OPT(HideHybrid, HIDEHYBRID);\n PerCPUData.resize(Cores);\n\n uint64_t MIDR {};\n for (size_t i = 0; i < Cores; ++i) {\n auto NewMIDR = CTX->HostFeatures.CPUMIDRs[i];\n if (MIDR != 0 && MIDR != NewMIDR) {\n // CPU mismatch, claim hybrid\n Hybrid = true;\n }\n\n // Truncate to 32-bits, top 32-bits are all reserved in MIDR\n PerCPUData[i].ProductName = ProductNames::ARM_UNKNOWN;\n PerCPUData[i].MIDR = NewMIDR;\n MIDR = NewMIDR;\n }\n\n if (HideHybrid()) {\n // Hide the hybrid flag.\n Hybrid = false;\n }\n\n struct CPUMIDR {\n uint8_t Implementer;\n uint16_t Part;\n bool DefaultBig; // Defaults to a big core\n const char* ProductName {};\n };\n\n // CPU priority order\n // This is mostly arbitrary but will sort by some sort of CPU priority by performance\n // Relative list so things they will commonly end up in big.little configurations sort of relate\n static constexpr std::array CPUMIDRs = {{\n // Typically big CPU cores\n {0x51, 0x001, 1, ProductNames::ARM_ORYON_1}, // Qualcomm Oryon-1\n\n {0x61, 0x039, 1, ProductNames::ARM_Avalanche_M2Max}, // Apple Avalanche (M2 Max)\n {0x61, 0x035, 1, ProductNames::ARM_Avalanche_M2Pro}, // Apple Avalanche (M2 Pro)\n {0x61, 0x033, 1, ProductNames::ARM_Avalanche_M2}, // Apple Avalanche (M2)\n {0x61, 0x029, 1, ProductNames::ARM_Firestorm_M1Max}, // Apple Firestorm (M1 Max)\n {0x61, 0x025, 1, ProductNames::ARM_Firestorm_M1Pro}, // Apple Firestorm (M1 Pro)\n {0x61, 0x023, 1, ProductNames::ARM_Firestorm_M1}, // Apple Firestorm (M1)\n {0x61, 0, 1, ProductNames::ARM_AppleSilicon}, // QEmu Apple Silicon\n\n {0x41, 0xd8c, 1, ProductNames::ARM_C1Ultra}, // C1-Ultra\n {0x41, 0xd90, 1, ProductNames::ARM_C1Premium}, // C1-Premium\n {0x41, 0xd8b, 1, ProductNames::ARM_C1Pro}, // C1-Pro\n {0x41, 0xd85, 1, ProductNames::ARM_X925}, // X925\n {0x41, 0xd87, 1, ProductNames::ARM_A725}, // A725\n {0x41, 0xd84, 1, ProductNames::ARM_V3}, // V3\n {0x41, 0xd83, 1, ProductNames::ARM_V3AE}, // V3AE\n {0x41, 0xd8e, 1, ProductNames::ARM_N3}, // N3\n {0x41, 0xd82, 1, ProductNames::ARM_X4}, // X4\n {0x41, 0xd81, 1, ProductNames::ARM_A720}, // A720\n {0x41, 0xd4e, 1, ProductNames::ARM_X3}, // X3\n {0x41, 0xd4d, 1, ProductNames::ARM_A715}, // A715\n {0x41, 0xd4f, 1, ProductNames::ARM_V2}, // V2\n {0x41, 0xd4b, 1, ProductNames::ARM_A78C}, // A78C\n {0x41, 0xd4a, 1, ProductNames::ARM_E1}, // E1\n {0x41, 0xd49, 1, ProductNames::ARM_N2}, // N2\n {0x41, 0xd48, 1, ProductNames::ARM_X2}, // X2\n {0x41, 0xd47, 1, ProductNames::ARM_A710}, // A710\n {0x41, 0xd4C, 1, ProductNames::ARM_X1C}, // X1C\n {0x41, 0xd44, 1, ProductNames::ARM_X1}, // X1\n {0x41, 0xd42, 1, ProductNames::ARM_A78AE}, // A78AE\n {0x41, 0xd41, 1, ProductNames::ARM_A78}, // A78\n {0x41, 0xd40, 1, ProductNames::ARM_V1}, // V1\n {0x41, 0xd0e, 1, ProductNames::ARM_A76AE}, // A76AE\n {0x41, 0xd0d, 1, ProductNames::ARM_A77}, // A77\n {0x41, 0xd0c, 1, ProductNames::ARM_N1}, // N1\n {0x41, 0xd0b, 1, ProductNames::ARM_A76}, // A76\n {0x51, 0x804, 1, ProductNames::ARM_Kryo400}, // Kryo 4xx Gold (A76 based)\n {0x41, 0xd0a, 1, ProductNames::ARM_A75}, // A75\n {0x51, 0x802, 1, ProductNames::ARM_Kryo300}, // Kryo 3xx Gold (A75 based)\n {0x41, 0xd09, 1, ProductNames::ARM_A73}, // A73\n {0x51, 0x800, 1, ProductNames::ARM_Kryo200}, // Kryo 2xx Gold (A73 based)\n {0x41, 0xd08, 1, ProductNames::ARM_A72}, // A72\n\n {0xc0, 0xac3, 1, ProductNames::ARM_Ampere_1}, // AmpereOne\n {0xc0, 0xac4, 1, ProductNames::ARM_Ampere_1A}, // AmpereOneA\n {0xc0, 0xac5, 1, ProductNames::ARM_Ampere_1B}, // AmpereOneB\n {0xc0, 0xac7, 1, ProductNames::ARM_Ampere_1C}, // AmpereOneC\n\n {0x4e, 0x010, 1, ProductNames::ARM_Olympus}, // Olympus\n {0x4e, 0x004, 1, ProductNames::ARM_Carmel}, // Carmel\n\n // Denver rated above A57 to match TX2 weirdness\n {0x4e, 0x003, 1, ProductNames::ARM_Denver}, // Denver\n\n {0x41, 0xd07, 1, ProductNames::ARM_A57}, // A57\n\n // Typically Little CPU cores\n {0x61, 0x038, 0, ProductNames::ARM_Blizzard_M2Max}, // Apple Blizzard (M2 Max)\n {0x61, 0x034, 0, ProductNames::ARM_Blizzard_M2Pro}, // Apple Blizzard (M2 Pro)\n {0x61, 0x032, 0, ProductNames::ARM_Blizzard_M2}, // Apple Blizzard (M2)\n {0x61, 0x028, 0, ProductNames::ARM_Icestorm_M1Max}, // Apple Icestorm (M1 Max)\n {0x61, 0x024, 0, ProductNames::ARM_Icestorm_M1Pro}, // Apple Icestorm (M1 Pro)\n {0x61, 0x022, 0, ProductNames::ARM_Icestorm_M1}, // Apple Icestorm (M1)\n\n {0x41, 0xd8a, 1, ProductNames::ARM_C1Nano}, // C1-Nano\n {0x41, 0xd80, 0, ProductNames::ARM_A520}, // A520\n {0x41, 0xd46, 0, ProductNames::ARM_A510}, // A510\n {0x41, 0xd06, 0, ProductNames::ARM_A65}, // A65\n {0x41, 0xd05, 0, ProductNames::ARM_A55}, // A55\n {0x51, 0x805, 0, ProductNames::ARM_Kryo400S}, // Kryo 4xx/5xx Silver (A55 based)\n {0x51, 0x803, 0, ProductNames::ARM_Kryo300S}, // Kryo 3xx Silver (A55 based)\n {0x41, 0xd03, 0, ProductNames::ARM_A53}, // A53\n {0x51, 0x801, 0, ProductNames::ARM_Kryo200S}, // Kryo 2xx Silver (A53 based)\n {0x41, 0xd04, 0, ProductNames::ARM_A35}, // A35\n\n {0x41, 0, 0, ProductNames::ARM_UNKNOWN}, // Invalid CPU or Apple CPU inside Parallels VM\n {0x0, 0, 0, ProductNames::ARM_UNKNOWN}, // Invalid starting point is lowest ranked\n }};\n\n auto FindDefinedMIDR = [](uint32_t MIDR) -> const CPUMIDR* {\n uint8_t Implementer = MIDR >> 24;\n uint16_t Part = (MIDR >> 4) & 0xFFF;\n\n for (auto& MIDROption : CPUMIDRs) {\n if (MIDROption.Implementer == Implementer && MIDROption.Part == Part) {\n return &MIDROption;\n }\n }\n\n return nullptr;\n };\n\n if (Hybrid) {\n // Walk the MIDRs and calculate big little designs\n fextl::vector BigCores;\n fextl::vector LittleCores;\n\n // Separate CPU cores out to big or little selected\n for (size_t i = 0; i < Cores; ++i) {\n uint32_t MIDR = PerCPUData[i].MIDR;\n auto MIDROption = FindDefinedMIDR(MIDR);\n if (MIDROption) {\n // Found one\n if (MIDROption->DefaultBig) {\n BigCores.emplace_back(MIDROption);\n } else {\n LittleCores.emplace_back(MIDROption);\n }\n } else {\n // If we didn't insert this MIDR then claim it is a little core.\n LittleCores.emplace_back(&CPUMIDRs.back());\n }\n }\n\n if (LittleCores.empty()) {\n // If we only ended up with big cores then we need to move some to be little cores\n uint32_t LowestMIDR = ~0U;\n uint32_t LowestMIDRIdx = 0;\n // Walk all the big cores\n for (size_t i = 0; i < BigCores.size(); ++i) {\n uint8_t Implementer = BigCores[i]->Implementer;\n uint16_t Part = BigCores[i]->Part;\n\n // Walk our list of CPUMIDRs to find the most little core\n for (size_t j = LowestMIDRIdx; j < CPUMIDRs.size(); ++j) {\n auto& MIDROption = CPUMIDRs[i];\n if ((MIDROption.Implementer == Implementer && MIDROption.Part == Part) || (MIDROption.Implementer == 0 && MIDROption.Part == 0)) {\n\n LowestMIDRIdx = j;\n LowestMIDR = MIDR;\n break;\n }\n }\n }\n\n // Now we WILL have found a big core to demote to little status\n // Demote them\n std::erase_if(BigCores, [&LittleCores, LowestMIDR](auto* Entry) {\n // Demote by erase copy to little array\n uint8_t Implementer = LowestMIDR >> 24;\n uint16_t Part = (LowestMIDR >> 4) & 0xFFF;\n\n if (Entry->Implementer == Implementer && Entry->Part == Part) {\n // Add it to the BigCore list\n LittleCores.emplace_back(Entry);\n return true;\n }\n return false;\n });\n }\n\n if (BigCores.empty()) {\n // We never found a CPU core we understand\n // Grab the first core, consider it as little, move everything else to Big\n uint32_t LittleMIDR = PerCPUData[0].MIDR;\n // Now walk the little cores and move them to Big if they don't match\n std::erase_if(LittleCores, [&BigCores, LittleMIDR](auto* Entry) {\n // You're promoted now\n uint8_t Implementer = LittleMIDR >> 24;\n uint16_t Part = (LittleMIDR >> 4) & 0xFFF;\n\n if (Entry->Implementer != Implementer || Entry->Part != Part) {\n // Add it to the BigCore list\n BigCores.emplace_back(Entry);\n return true;\n }\n return false;\n });\n }\n\n // Now walk the per CPU data one more time and set if it is big or little\n for (auto& Data : PerCPUData) {\n uint8_t Implementer = Data.MIDR >> 24;\n uint16_t Part = (Data.MIDR >> 4) & 0xFFF;\n\n bool FoundBig {};\n const CPUMIDR* MIDR {};\n for (auto Big : BigCores) {\n if (Big->Implementer == Implementer && Big->Part == Part) {\n FoundBig = true;\n MIDR = Big;\n break;\n }\n }\n\n if (!FoundBig) {\n for (auto Little : LittleCores) {\n if (Little->Implementer == Implementer && Little->Part == Part) {\n MIDR = Little;\n break;\n }\n }\n }\n\n Data.IsBig = FoundBig;\n if (MIDR) {\n Data.ProductName = MIDR->ProductName ?: ProductNames::ARM_UNKNOWN;\n } else {\n Data.ProductName = ProductNames::ARM_UNKNOWN;\n }\n }\n } else {\n // If we aren't hybrid then just claim everything is big\n for (size_t i = 0; i < Cores; ++i) {\n const auto MIDRIndex = HideHybrid() ? 0 : i;\n uint32_t MIDR = PerCPUData[MIDRIndex].MIDR;\n auto MIDROption = FindDefinedMIDR(MIDR);\n\n PerCPUData[i].IsBig = true;\n if (MIDROption) {\n PerCPUData[i].ProductName = MIDROption->ProductName ?: ProductNames::ARM_UNKNOWN;\n } else {\n PerCPUData[i].ProductName = ProductNames::ARM_UNKNOWN;\n }\n }\n }\n}\n\n#else\nuint64_t GetCycleCounterFrequency() {\n return 0;\n}\n\nvoid CPUIDEmu::SetupHostHybridFlag() {}\n\n#endif\n\n\nvoid CPUIDEmu::SetupFeatures() {\n if (CTX->HostFeatures.SupportsAVX) {\n XCR0 |= XCR0_AVX;\n }\n\n Features.SHA = CTX->HostFeatures.SupportsSHA;\n}\n\nFEXCore::CPUID::FunctionResults CPUIDEmu::Function_0h(uint32_t Leaf) const {\n FEXCore::CPUID::FunctionResults Res {};\n\n // EBX, EDX, ECX become the manufacturer id string\n#ifdef CPUID_AMD\n Res.eax = 0x0D; // Let's say we are a Zen+\n Res.ebx = CPUID_VENDOR_AMD1;\n Res.edx = CPUID_VENDOR_AMD2;\n Res.ecx = CPUID_VENDOR_AMD3;\n#else\n Res.eax = 0x16; // Let's say we are a Skylake\n Res.ebx = CPUID_VENDOR_INTEL1;\n Res.edx = CPUID_VENDOR_INTEL2;\n Res.ecx = CPUID_VENDOR_INTEL3;\n#endif\n return Res;\n}\n\n// Processor Info and Features bits\nFEXCore::CPUID::FunctionResults CPUIDEmu::Function_01h(uint32_t Leaf) const {\n FEXCore::CPUID::FunctionResults Res {};\n\n // Hypervisor bit is normally set but some applications have issues with it.\n uint32_t Hypervisor = HideHypervisorBit() ? 0 : 1;\n\n Res.eax = FAMILY_IDENTIFIER;\n\n Res.ebx = 0 | // Brand index\n (8 << 8) | // Cache line size in bytes\n (Cores << 16) | // Number of addressable IDs for the logical cores in the physical CPU\n (GetCPUID() << 24); // Local APIC ID\n\n Res.ecx = (1 << 0) | // SSE3\n (CTX->HostFeatures.SupportsPMULL_128Bit << 1) | // PCLMULQDQ\n (1 << 2) | // DS area supports 64bit layout\n (1 << 3) | // MWait\n (0 << 4) | // DS-CPL\n (0 << 5) | // VMX\n (0 << 6) | // SMX\n (0 << 7) | // Intel SpeedStep\n (1 << 8) | // Thermal Monitor 2\n (1 << 9) | // SSSE3\n (0 << 10) | // L1 context ID\n (0 << 11) | // Silicon debug\n (SupportsAVX() << 12) | // FMA3\n (1 << 13) | // CMPXCHG16B\n (0 << 14) | // xTPR update control\n (0 << 15) | // Perfmon and debug capability\n (0 << 16) | // Reserved\n (0 << 17) | // Process-context identifiers\n (0 << 18) | // Prefetching from memory mapped device\n (1 << 19) | // SSE4.1\n (CTX->HostFeatures.SupportsCRC << 20) | // SSE4.2\n (0 << 21) | // X2APIC\n (1 << 22) | // MOVBE\n (1 << 23) | // POPCNT\n (0 << 24) | // APIC TSC-Deadline\n (CTX->HostFeatures.SupportsAES << 25) | // AES\n (SupportsAVX() << 26) | // XSAVE\n (SupportsAVX() << 27) | // OSXSAVE\n (SupportsAVX() << 28) | // AVX\n (SupportsAVX() << 29) | // F16C\n (CTX->HostFeatures.SupportsRAND << 30) | // RDRAND\n (Hypervisor << 31);\n\n Res.edx = (1 << 0) | // FPU\n (1 << 1) | // Virtual 8086 mode enhancements\n (0 << 2) | // Debugging extensions\n (0 << 3) | // Page size extension\n (1 << 4) | // RDTSC supported\n (1 << 5) | // MSR supported\n (1 << 6) | // PAE\n (1 << 7) | // Machine Check exception\n (1 << 8) | // CMPXCHG8B\n (1 << 9) | // APIC on-chip\n (0 << 10) | // Reserved\n (1 << 11) | // SYSENTER/SYSEXIT\n (1 << 12) | // Memory Type Range registers, MTRRs are supported\n (1 << 13) | // Page Global bit\n (1 << 14) | // Machine Check architecture\n (1 << 15) | // CMOV\n (1 << 16) | // Page Attribute Table\n (1 << 17) | // 36bit page size extension\n (0 << 18) | // Processor serial number\n (1 << 19) | // CLFLUSH\n (0 << 20) | // Reserved\n (0 << 21) | // Debug store\n (0 << 22) | // Thermal monitor and software controled clock\n (1 << 23) | // MMX\n (1 << 24) | // FXSAVE/FXRSTOR\n (1 << 25) | // SSE\n (1 << 26) | // SSE2\n (0 << 27) | // Self Snoop\n (0 << 28) | // (HTT) Max APIC IDs reserved field is valid\n (1 << 29) | // Thermal monitor\n (0 << 30) | // Reserved\n (0 << 31); // Pending break enable\n return Res;\n}\n\n// 2: Cache and TLB information\nFEXCore::CPUID::FunctionResults CPUIDEmu::Function_02h(uint32_t Leaf) const {\n FEXCore::CPUID::FunctionResults Res {};\n\n // returns default values from i7 model 1Ah\n Res.eax = 0x1 | // Number of iterations needed for all descriptors\n (0x5A << 8) | (0x03 << 16) | (0x55 << 24);\n\n Res.ebx = 0xE4 | (0xB2 << 8) | (0xF0 << 16) | (0 << 24);\n\n Res.ecx = 0; // null descriptors\n\n Res.edx = 0x2C | (0x21 << 8) | (0xCA << 16) | (0x09 << 24);\n\n return Res;\n}\n\n// 4: Deterministic cache parameters for each level\nFEXCore::CPUID::FunctionResults CPUIDEmu::Function_04h(uint32_t Leaf) const {\n FEXCore::CPUID::FunctionResults Res {};\n constexpr uint32_t CacheType_Data = 1;\n constexpr uint32_t CacheType_Instruction = 2;\n constexpr uint32_t CacheType_Unified = 3;\n\n if (Leaf == 0) {\n // Report L1D\n uint32_t CoreCount = Cores - 1;\n\n Res.eax = CacheType_Data | // Cache type\n (0b001 << 5) | // Cache level\n (1 << 8) | // Self initializing cache level\n (0 << 9) | // Fully associative\n (0 << 14) | // Maximum number of addressable IDs for logical processors sharing this cache (With SMT this would be 1)\n (CoreCount << 26); // Maximum number of addressable IDs for processor cores in the physical package\n\n Res.ebx = (63 << 0) | // Line Size - 1 : Claiming 64 byte\n (0 << 12) | // Physical Line partitions\n (7 << 22); // Associativity - 1 : Claiming 8 way\n\n // 32KB\n Res.ecx = 63; // Number of sets - 1 : Claiming 64 sets\n\n Res.edx = (0 << 0) | // Write-back invalidate\n (0 << 1) | // Cache inclusiveness - Includes lower caches\n (0 << 2); // Complex cache indexing - 0: Direct, 1: Complex\n } else if (Leaf == 1) {\n // Report L1I\n uint32_t CoreCount = Cores - 1;\n\n Res.eax = CacheType_Instruction | // Cache type\n (0b001 << 5) | // Cache level\n (1 << 8) | // Self initializing cache level\n (0 << 9) | // Fully associative\n (0 << 14) | // Maximum number of addressable IDs for logical processors sharing this cache (With SMT this would be 1)\n (CoreCount << 26); // Maximum number of addressable IDs for processor cores in the physical package\n\n Res.ebx = (63 << 0) | // Line Size - 1 : Claiming 64 byte\n (0 << 12) | // Physical Line partitions\n (7 << 22); // Associativity - 1 : Claiming 8 way\n\n // 32KB\n Res.ecx = 63; // Number of sets - 1 : Claiming 64 sets\n\n Res.edx = (0 << 0) | // Write-back invalidate\n (0 << 1) | // Cache inclusiveness - Includes lower caches\n (0 << 2); // Complex cache indexing - 0: Direct, 1: Complex\n } else if (Leaf == 2) {\n // Report L2\n uint32_t CoreCount = Cores - 1;\n\n Res.eax = CacheType_Unified | // Cache type\n (0b010 << 5) | // Cache level\n (1 << 8) | // Self initializing cache level\n (0 << 9) | // Fully associative\n (0 << 14) | // Maximum number of addressable IDs for logical processors sharing this cache\n (CoreCount << 26); // Maximum number of addressable IDs for processor cores in the physical package\n\n Res.ebx = (63 << 0) | // Line Size - 1 : Claiming 64 byte\n (0 << 12) | // Physical Line partitions\n (7 << 22); // Associativity - 1 : Claiming 8 way\n\n // 512KB\n Res.ecx = 0x3FF; // Number of sets - 1 : Claiming 1024 sets\n\n Res.edx = (0 << 0) | // Write-back invalidate\n (0 << 1) | // Cache inclusiveness - Includes lower caches\n (0 << 2); // Complex cache indexing - 0: Direct, 1: Complex\n } else if (Leaf == 3) {\n // Report L3\n uint32_t CoreCount = Cores - 1;\n\n Res.eax = CacheType_Unified | // Cache type\n (0b011 << 5) | // Cache level\n (1 << 8) | // Self initializing cache level\n (0 << 9) | // Fully associative\n (CoreCount << 14) | // Maximum number of addressable IDs for logical processors sharing this cache\n (CoreCount << 26); // Maximum number of addressable IDs for processor cores in the physical package\n\n Res.ebx = (63 << 0) | // Line Size - 1 : Claiming 64 byte\n (0 << 12) | // Physical Line partitions\n (7 << 22); // Associativity - 1 : Claiming 8 way\n\n // 8MB\n Res.ecx = 0x4000; // Number of sets - 1 : Claiming 16384 sets\n\n Res.edx = (0 << 0) | // Write-back invalidate\n (0 << 1) | // Cache inclusiveness - Includes lower caches\n (1 << 2); // Complex cache indexing - 0: Direct, 1: Complex\n }\n\n return Res;\n}\n\nFEXCore::CPUID::FunctionResults CPUIDEmu::Function_06h(uint32_t Leaf) const {\n FEXCore::CPUID::FunctionResults Res {};\n Res.eax = (1 << 2); // Always running APIC\n Res.ecx = (0 << 3); // Intel performance energy bias preference (EPB)\n return Res;\n}\n\nFEXCore::CPUID::FunctionResults CPUIDEmu::Function_07h(uint32_t Leaf) const {\n FEXCore::CPUID::FunctionResults Res {};\n if (Leaf == 0) {\n // Disable Enhanced REP MOVS when TSO is enabled.\n // vcruntime140 memmove will use `rep movsb` in this case which completely destroys perf in Hades(appId 1145360)\n // This is due to LRCPC performance on Cortex being abysmal.\n // Only enable EnhancedREPMOVS if atomic memcpy tso emulation isn't enabled.\n const uint32_t SupportsEnhancedREPMOVS = CTX->IsMemcpyAtomicTSOEnabled() == false;\n const uint32_t SupportsVPCLMULQDQ = CTX->HostFeatures.SupportsPMULL_128Bit && SupportsAVX();\n const uint32_t SupportsWFXT = CTX->HostFeatures.SupportsWFXT;\n\n // Number of subfunctions\n Res.eax = 0x0;\n Res.ebx = (1 << 0) | // FS/GS support\n (0 << 1) | // TSC adjust MSR\n (0 << 2) | // SGX\n (SupportsAVX() << 3) | // BMI1\n (0 << 4) | // Intel Hardware Lock Elison\n (SupportsAVX() << 5) | // AVX2 support\n (1 << 6) | // FPU data pointer updated only on exception\n (1 << 7) | // SMEP support\n (SupportsAVX() << 8) | // BMI2\n (SupportsEnhancedREPMOVS << 9) | // Enhanced REP MOVSB/STOSB\n (1 << 10) | // INVPCID for system software control of process-context\n (0 << 11) | // Restricted transactional memory\n (0 << 12) | // Intel resource directory technology Monitoring\n (1 << 13) | // Deprecates FPU CS and DS\n (0 << 14) | // Intel MPX\n (0 << 15) | // Intel Resource Directory Technology Allocation\n (0 << 16) | // AVX512-F\n (0 << 17) | // AVX512-DQ\n (CTX->HostFeatures.SupportsRAND << 18) | // RDSEED\n (1 << 19) | // ADCX and ADOX instructions\n (0 << 20) | // SMAP Supervisor mode access prevention and CLAC/STAC instructions\n (0 << 21) | // AVX512-IFMA\n (0 << 22) | // PCOMMIT (deprecated?)\n (1 << 23) | // CLFLUSHOPT instruction\n (1 << 24) | // CLWB instruction\n (0 << 25) | // Intel processor trace\n (0 << 26) | // AVX512-PF\n (0 << 27) | // AVX512-ER\n (0 << 28) | // AVX512-CD\n (Features.SHA << 29) | // SHA instructions\n (0 << 30) | // AVX512-BW\n (0 << 31); // AVX512-VL\n\n Res.ecx = (1 << 0) | // PREFETCHWT1\n (0 << 1) | // AVX512VBMI\n (0 << 2) | // Usermode instruction prevention\n (0 << 3) | // Protection keys for user mode pages\n (0 << 4) | // OS protection keys\n (SupportsWFXT << 5) | // waitpkg\n (0 << 6) | // AVX512-VBMI2\n (0 << 7) | // CET shadow stack\n (0 << 8) | // GFNI\n (CTX->HostFeatures.SupportsAES256 << 9) | // VAES\n (SupportsVPCLMULQDQ << 10) | // VPCLMULQDQ\n (0 << 11) | // AVX512-VNNI\n (0 << 12) | // AVX512-BITALG\n (0 << 13) | // Intel Total Memory Encryption\n (0 << 14) | // AVX512-VPOPCNTDQ\n (0 << 15) | // FZM (TDX)\n (0 << 16) | // 5 Level page tables\n (0 << 17) | // MPX MAWAU\n (0 << 18) | // MPX MAWAU\n (0 << 19) | // MPX MAWAU\n (0 << 20) | // MPX MAWAU\n (0 << 21) | // MPX MAWAU\n (1 << 22) | // RDPID Read Processor ID\n (0 << 23) | // AES Key Locker\n (1 << 24) | // bus-lock-detect\n (0 << 25) | // CLDEMOTE\n (0 << 26) | // MPRR (TDX)\n (0 << 27) | // MOVDIRI\n (0 << 28) | // MOVDIR64B\n (0 << 29) | // ENQCMD\n (0 << 30) | // SGX Launch configuration\n (0 << 31); // PKS\n\n Res.edx = (0 << 0) | // SGX-TEM (TDX)\n (0 << 1) | // SGX-KEYS\n (0 << 2) | // AVX512-4VNNIW\n (0 << 3) | // AVX512-4FMAPS\n (1 << 4) | // Fast Short Rep Mov\n (0 << 5) | // UINTR\n (0 << 6) | // Reserved\n (0 << 7) | // Reserved\n (0 << 8) | // AVX512-VP2INTERSECT\n (0 << 9) | // SRBDS_CTRL (Special Register Buffer Data Sampling Mitigations)\n (0 << 10) | // VERW clears CPU buffers\n (0 << 11) | // rtm-always-abort\n (0 << 12) | // Reserved\n (0 << 13) | // TSX Force Abort (TSX will force abort if attempted)\n (0 << 14) | // SERIALIZE instruction\n ((Hybrid ? 1U : 0U) << 15) | // Hybrid\n (0 << 16) | // TSXLDTRK (TSX Suspend load address tracking) - Allows untracked memory loads inside TSX region\n (0 << 17) | // Reserved\n (0 << 18) | // Intel PCONFIG\n (0 << 19) | // Intel Architectural LBR\n (0 << 20) | // Intel CET\n (0 << 21) | // Reserved\n (0 << 22) | // AMX-BF16 - Tile computation on bfloat16\n (0 << 23) | // AVX512-FP16 - FP16 AVX512 instructions\n (0 << 24) | // AMX-tile - If AMX is implemented\n (0 << 25) | // AMX-int8 - AMX on 8-bit integers\n (0 << 26) | // IBRS_IBPB - Speculation control\n (0 << 27) | // STIBP - Single Thread Indirect Branch Predictor, Part of IBC\n (0 << 28) | // L1D Flush\n (0 << 29) | // Arch capabilities - Speculative side channel mitigations\n (0 << 30) | // Arch capabilities - MSR module specific\n (0 << 31); // SSBD - Speculative Store Bypass Disable\n }\n\n return Res;\n}\n\nFEXCore::CPUID::FunctionResults CPUIDEmu::Function_0Dh(uint32_t Leaf) const {\n // Leaf 0\n FEXCore::CPUID::FunctionResults Res {};\n\n uint32_t XFeatureSupportedSizeMax = SupportsAVX() ? 0x0000'0340 : 0x0000'0240; // XFeatureEnabledSizeMax: Legacy Header + FPU/SSE + AVX\n if (Leaf == 0) {\n // XFeatureSupportedMask[31:0]\n Res.eax = (1 << 0) | // X87 support\n (1 << 1) | // 128-bit SSE support\n (SupportsAVX() << 2) | // 256-bit AVX support\n (0b00 << 3) | // MPX State\n (0b000 << 5) | // AVX-512 state\n (0 << 8) | // \"Used for IA32_XSS\" ... Used for what?\n (0 << 9); // PKRU state\n\n // EBX and ECX doesn't need to match if a feature is supported but not enabled\n Res.ebx = XFeatureSupportedSizeMax;\n Res.ecx = XFeatureSupportedSizeMax; // XFeatureSupportedSizeMax: Size in bytes of XSAVE/XRSTOR area\n\n // XFeatureSupportedMask[63:32]\n Res.edx = 0; // Upper 32-bits of XFeatureSupportedMask\n } else if (Leaf == 1) {\n Res.eax = (1 << 0) | // XSAVEOPT\n (0 << 1) | // XSAVEC (and XRSTOR)\n (0 << 2) | // XGETBV - XGETBV with ECX=1 supported\n (0 << 3); // XSAVES - XSAVES, XRSTORS, and IA32_XSS supported\n\n // Same information as Leaf 0 for ebx\n Res.ebx = XFeatureSupportedSizeMax;\n\n // Lower supported 32bits of IA32_XSS MSR. IA32_XSS[n] can only be set to 1 if ECX[n] is 1\n Res.ecx = (0b0000'0000 << 0) | // Used for XCR0\n (0 << 8) | // PT state\n (0 << 9); // Used for XCR0\n\n // Upper supported 32bits of IA32_XSS MSR. IA32_XSS[n+32] can only be set to 1 if EDX[n] is 1\n // Entirely reserved atm\n Res.edx = 0;\n } else if (Leaf == 2) {\n Res.eax = SupportsAVX() ? 0x0000'0100 : 0; // YmmSaveStateSize\n Res.ebx = SupportsAVX() ? 0x0000'0240 : 0; // YmmSaveStateOffset\n\n // Reserved\n Res.ecx = 0;\n Res.edx = 0;\n }\n return Res;\n}\n\nFEXCore::CPUID::FunctionResults CPUIDEmu::Function_15h(uint32_t Leaf) const {\n FEXCore::CPUID::FunctionResults Res {};\n // TSC frequency = ECX * EBX / EAX\n uint64_t FrequencyHz = GetCycleCounterFrequency();\n if (FrequencyHz) {\n Res.eax = 1;\n Res.ebx = 1U << CTX->Config.TSCScale;\n Res.ecx = FrequencyHz;\n }\n return Res;\n}\n\nFEXCore::CPUID::FunctionResults CPUIDEmu::Function_1Ah(uint32_t Leaf) const {\n FEXCore::CPUID::FunctionResults Res {};\n if (Hybrid) {\n uint32_t CPU = GetCPUID();\n auto& Data = PerCPUData[CPU];\n // 0x40 is a big CPU\n // 0x20 is a little CPU\n Res.eax |= (Data.IsBig ? 0x40 : 0x20) << 24;\n }\n return Res;\n}\n\n// Hypervisor CPUID information leaf\nFEXCore::CPUID::FunctionResults CPUIDEmu::Function_4000_0000h(uint32_t Leaf) const {\n FEXCore::CPUID::FunctionResults Res {};\n // Maximum supported hypervisor leafs\n // We only expose the information leaf\n //\n // Common courtesy to follow VMWare's \"Hypervisor CPUID Interface proposal\"\n // 4000_0000h - Information leaf. Advertising to the software which hypervisor this is\n // 4000_0001h - 4000_000Fh - Hypervisor specific leafs. FEX can use these for anything\n // 4000_0010h - 4000_00FFh - \"Generic Leafs\" - Try not to overwrite, other hypervisors might expect information in these\n //\n // CPUID documentation information:\n // 4000_0000h - 4FFF_FFFFh - No existing or future CPU will return information in this range\n // Reserved entirely for VMs to do whatever they want.\n Res.eax = 0x40000001;\n\n // EBX, EDX, ECX become the hypervisor ID signature\n constexpr static char HypervisorID[12] = \"FEXIFEXIEMU\";\n memcpy(&Res.ebx, HypervisorID, sizeof(HypervisorID));\n return Res;\n}\n\nconstexpr std::array::length(GIT_DESCRIBE_STRING) + 1> GitString = {GIT_DESCRIBE_STRING};\nstatic_assert(GitString.size() < 32);\n\n// Hypervisor CPUID information leaf\nFEXCore::CPUID::FunctionResults CPUIDEmu::Function_4000_0001h(uint32_t Leaf) const {\n FEXCore::CPUID::FunctionResults Res {};\n constexpr uint32_t MaximumSubLeafNumber = 2;\n if (Leaf == 0) {\n // EAX[3:0] Is the host architecture that FEX is running under\n#ifdef ARCHITECTURE_x86_64\n // EAX[3:0] = 1 = x86_64 host architecture\n Res.eax |= 0b0001;\n#elif defined(ARCHITECTURE_arm64)\n // EAX[3:0] = 2 = AArch64 host architecture\n Res.eax |= 0b0010;\n#else\n // EAX[3:0] = 0 = Unknown architecture\n#endif\n\n // EAX[15:4] = Reserved\n\n // EAX[31:16] = Maximum sub-leaf value.\n Res.eax |= MaximumSubLeafNumber << 16;\n } else if (Leaf == 1) {\n memcpy(&Res, GitString.data(), std::min(GitString.size(), sizeof(FEXCore::CPUID::FunctionResults)));\n } else if (Leaf == 2) {\n memcpy(&Res, GitString.data() + 16, std::min(std::max(0, GitString.size() - 16), sizeof(FEXCore::CPUID::FunctionResults)));\n }\n\n return Res;\n}\n\n// Highest extended function implemented\nFEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0000h(uint32_t Leaf) const {\n FEXCore::CPUID::FunctionResults Res {};\n Res.eax = 0x8000001F;\n\n // EBX, EDX, ECX become the manufacturer id string\n // Just like cpuid function 0\n#ifdef CPUID_AMD\n Res.ebx = CPUID_VENDOR_AMD1;\n Res.edx = CPUID_VENDOR_AMD2;\n Res.ecx = CPUID_VENDOR_AMD3;\n#else\n Res.ebx = CPUID_VENDOR_INTEL1;\n Res.edx = CPUID_VENDOR_INTEL2;\n Res.ecx = CPUID_VENDOR_INTEL3;\n#endif\n return Res;\n}\n\n// Extended processor and feature bits\nFEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0001h(uint32_t Leaf) const {\n\n#ifndef _WIN32\n constexpr uint32_t SUPPORTS_RDTSCP = 1;\n#else\n // RDTSCP under WIN32 is only supported if CPUIndex is available in TPIDRRO.\n const uint32_t SUPPORTS_RDTSCP = SupportsCPUIndexInTPIDRRO;\n#endif\n FEXCore::CPUID::FunctionResults Res {};\n\n Res.eax = FAMILY_IDENTIFIER;\n\n Res.ecx = (1 << 0) | // LAHF/SAHF\n (1 << 1) | // 0 = Single core product, 1 = multi core product\n (0 << 2) | // SVM\n (1 << 3) | // Extended APIC register space\n (0 << 4) | // LOCK MOV CR0 means MOV CR8\n (1 << 5) | // ABM instructions\n (CTX->HostFeatures.SupportsSSE4a << 6) | // SSE4a\n (0 << 7) | // Misaligned SSE mode\n (1 << 8) | // PREFETCHW\n (0 << 9) | // OS visible workaround support\n (0 << 10) | // Instruction based sampling support\n (0 << 11) | // XOP\n (0 << 12) | // SKINIT\n (0 << 13) | // Watchdog timer support\n (0 << 14) | // Reserved\n (0 << 15) | // Lightweight profiling support\n (0 << 16) | // FMA4\n (1 << 17) | // Translation cache extension\n (0 << 18) | // Reserved\n (0 << 19) | // Reserved\n (0 << 20) | // Reserved\n (0 << 21) | // XOP-TBM\n (0 << 22) | // Topology extensions support\n (0 << 23) | // Core performance counter extensions\n (0 << 24) | // NB performance counter extensions\n (0 << 25) | // Reserved\n (0 << 26) | // Data breakpoints extensions\n (0 << 27) | // Performance TSC\n (0 << 28) | // L2 perf counter extensions\n (0 << 29) | // MONITORX\n (0 << 30) | // Reserved\n (0 << 31); // Reserved\n\n Res.edx = (1 << 0) | // FPU\n (1 << 1) | // Virtual mode extensions\n (1 << 2) | // Debugging extensions\n (1 << 3) | // Page size extensions\n (1 << 4) | // TSC\n (1 << 5) | // MSR support\n (1 << 6) | // PAE\n (1 << 7) | // Machine Check Exception\n (1 << 8) | // CMPXCHG8B\n (1 << 9) | // APIC\n (0 << 10) | // Reserved\n (1 << 11) | // SYSCALL/SYSRET\n (1 << 12) | // MTRR\n (1 << 13) | // Page global extension\n (1 << 14) | // Machine Check architecture\n (1 << 15) | // CMOV\n (1 << 16) | // Page attribute table\n (1 << 17) | // Page-size extensions\n (0 << 18) | // Reserved\n (0 << 19) | // Reserved\n (1 << 20) | // NX\n (0 << 21) | // Reserved\n (1 << 22) | // MMXExt\n (1 << 23) | // MMX\n (1 << 24) | // FXSAVE/FXRSTOR\n (1 << 25) | // FXSAVE/FXRSTOR Optimizations\n (0 << 26) | // 1 gigabit pages\n (SUPPORTS_RDTSCP << 27) | // RDTSCP\n (0 << 28) | // Reserved\n (1 << 29) | // Long Mode\n (CTX->HostFeatures.Supports3DNow << 30) | // 3DNow! Extensions\n (CTX->HostFeatures.Supports3DNow << 31); // 3DNow!\n return Res;\n}\n\n// Processor brand string\nFEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0002h(uint32_t Leaf) const {\n return Function_8000_0002h(Leaf, GetCPUID());\n}\n\nFEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0003h(uint32_t Leaf) const {\n return Function_8000_0003h(Leaf, GetCPUID());\n}\n\nFEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0004h(uint32_t Leaf) const {\n return Function_8000_0004h(Leaf, GetCPUID());\n}\n\nFEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0002h(uint32_t Leaf, uint32_t CPU) const {\n FEXCore::CPUID::FunctionResults Res {};\n auto& Data = PerCPUData[CPU];\n memcpy(&Res, Data.ProductName, std::min(strlen(Data.ProductName), sizeof(FEXCore::CPUID::FunctionResults)));\n return Res;\n}\n\nFEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0003h(uint32_t Leaf, uint32_t CPU) const {\n FEXCore::CPUID::FunctionResults Res {};\n auto& Data = PerCPUData[CPU];\n const auto RemainingStringSize = std::max(0, strlen(Data.ProductName) - 16);\n memcpy(&Res, Data.ProductName + 16, std::min(RemainingStringSize, sizeof(FEXCore::CPUID::FunctionResults)));\n return Res;\n}\n\nFEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0004h(uint32_t Leaf, uint32_t CPU) const {\n FEXCore::CPUID::FunctionResults Res {};\n auto& Data = PerCPUData[CPU];\n const auto RemainingStringSize = std::max(0, strlen(Data.ProductName) - 32);\n memcpy(&Res, Data.ProductName + 32, std::min(RemainingStringSize, sizeof(FEXCore::CPUID::FunctionResults)));\n return Res;\n}\n\n// L1 Cache and TLB identifiers\nFEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0005h(uint32_t Leaf) const {\n FEXCore::CPUID::FunctionResults Res {};\n\n // L1 TLB Information for 2MB and 4MB pages\n Res.eax = (64 << 0) | // Number of TLB instruction entries\n (255 << 8) | // instruction TLB associativity type (full)\n (64 << 16) | // Number of TLB data entries\n (255 << 24); // data TLB associativity type (full)\n\n // L1 TLB Information for 4KB pages\n Res.ebx = (64 << 0) | // Number of TLB instruction entries\n (255 << 8) | // instruction TLB associativity type (full)\n (64 << 16) | // Number of TLB data entries\n (255 << 24); // data TLB associativity type (full)\n\n // L1 data cache identifiers\n Res.ecx = (64 << 0) | // L1 data cache size line in bytes\n (1 << 8) | // L1 data cachelines per tag\n (8 << 16) | // L1 data cache associativity\n (32 << 24); // L1 data cache size in KB\n\n // L1 instruction cache identifiers\n Res.edx = (64 << 0) | // L1 instruction cache line size in bytes\n (1 << 8) | // L1 instruction cachelines per tag\n (4 << 16) | // L1 instruction cache associativity\n (64 << 24); // L1 instruction cache size in KB\n\n return Res;\n}\n\n// L2 Cache identifiers\nFEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0006h(uint32_t Leaf) const {\n FEXCore::CPUID::FunctionResults Res {};\n\n // L2 TLB Information for 2MB and 4MB pages\n Res.eax = (1024 << 0) | // Number of TLB instruction entries\n (6 << 12) | // instruction TLB associativity type\n (1536 << 16) | // Number of TLB data entries\n (3 << 28); // data TLB associativity type\n\n // L2 TLB Information for 4KB pages\n Res.ebx = (1024 << 0) | // Number of TLB instruction entries\n (6 << 12) | // instruction TLB associativity type\n (1536 << 16) | // Number of TLB data entries\n (5 << 28); // data TLB associativity type\n\n // L2 cache identifiers\n Res.ecx = (64 << 0) | // cacheline size\n (1 << 8) | // cachelines per tag\n (6 << 12) | // cache associativity\n (512 << 16); // L2 cache size in KB\n\n // L3 cache identifiers\n Res.edx = (64 << 0) | // cacheline size\n (1 << 8) | // cachelines per tag\n (6 << 12) | // cache associativity\n (16 << 18); // L2 cache size in KB\n return Res;\n}\n\n// Advanced power management\nFEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0007h(uint32_t Leaf) const {\n FEXCore::CPUID::FunctionResults Res {};\n Res.eax = (1 << 2); // APIC timer not affected by p-state\n Res.edx = (1 << 8); // Invariant TSC\n return Res;\n}\n\n// Virtual and physical address sizes\nFEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0008h(uint32_t Leaf) const {\n FEXCore::CPUID::FunctionResults Res {};\n Res.eax = (48 << 0) | // PhysAddrSize = 48-bit\n (48 << 8) | // LinAddrSize = 48-bit\n (0 << 16); // GuestPhysAddrSize == PhysAddrSize\n\n Res.ebx = (0 << 2) | // XSaveErPtr: Saving and restoring error pointers\n (0 << 1) | // IRPerf: Instructions retired count support\n (CTX->HostFeatures.SupportsCLZERO << 0); // CLZERO support\n\n uint32_t CoreCount = Cores - 1;\n Res.ecx = (0 << 16) | // PerfTscSize: Performance timestamp count size\n (std::bit_ceil(Cores) << 12) | // ApicIdSize: Number of bits in ApicID\n (CoreCount << 0); // Count count subtract one\n\n return Res;\n}\n\n// TLB 1GB page identifiers\nFEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0019h(uint32_t Leaf) const {\n FEXCore::CPUID::FunctionResults Res {};\n Res.eax = (0xF << 28) | // L1 DTLB associativity for 1GB pages\n (64 << 16) | // L1 DTLB entry count for 1GB pages\n (0xF << 12) | // L1 ITLB associativity for 1GB pages\n (64 << 0); // L1 ITLB entry count for 1GB pages\n\n Res.ebx = (0 << 28) | // L2 DTLB associativity for 1GB pages\n (0 << 16) | // L2 DTLB entry count for 1GB pages\n (0 << 12) | // L2 ITLB associativity for 1GB pages\n (0 << 0); // L2 ITLB entry count for 1GB pages\n return Res;\n}\n\n// Deterministic cache parameters for each level\nFEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_001Dh(uint32_t Leaf) const {\n // This is nearly a copy of CPUID function 4h\n // There are some minor changes though\n\n FEXCore::CPUID::FunctionResults Res {};\n constexpr uint32_t CacheType_Data = 1;\n constexpr uint32_t CacheType_Instruction = 2;\n constexpr uint32_t CacheType_Unified = 3;\n\n if (Leaf == 0) {\n // Report L1D\n Res.eax = CacheType_Data | // Cache type\n (0b001 << 5) | // Cache level\n (1 << 8) | // Self initializing cache level\n (0 << 9) | // Fully associative\n (0 << 14); // Maximum number of addressable IDs for logical processors sharing this cache (With SMT this would be 1)\n\n Res.ebx = (63 << 0) | // Line Size - 1 : Claiming 64 byte\n (0 << 12) | // Physical Line partitions\n (7 << 22); // Associativity - 1 : Claiming 8 way\n\n // 32KB\n Res.ecx = 63; // Number of sets - 1 : Claiming 64 sets\n\n Res.edx = (0 << 0) | // Write-back invalidate\n (0 << 1); // Cache inclusiveness - Includes lower caches\n } else if (Leaf == 1) {\n // Report L1I\n Res.eax = CacheType_Instruction | // Cache type\n (0b001 << 5) | // Cache level\n (1 << 8) | // Self initializing cache level\n (0 << 9) | // Fully associative\n (0 << 14); // Maximum number of addressable IDs for logical processors sharing this cache (With SMT this would be 1)\n\n Res.ebx = (63 << 0) | // Line Size - 1 : Claiming 64 byte\n (0 << 12) | // Physical Line partitions\n (7 << 22); // Associativity - 1 : Claiming 8 way\n\n // 32KB\n Res.ecx = 63; // Number of sets - 1 : Claiming 64 sets\n\n Res.edx = (0 << 0) | // Write-back invalidate\n (0 << 1); // Cache inclusiveness - Includes lower caches\n } else if (Leaf == 2) {\n // Report L2\n Res.eax = CacheType_Unified | // Cache type\n (0b010 << 5) | // Cache level\n (1 << 8) | // Self initializing cache level\n (0 << 9) | // Fully associative\n (0 << 14); // Maximum number of addressable IDs for logical processors sharing this cache\n\n Res.ebx = (63 << 0) | // Line Size - 1 : Claiming 64 byte\n (0 << 12) | // Physical Line partitions\n (7 << 22); // Associativity - 1 : Claiming 8 way\n\n // 512KB\n Res.ecx = 0x3FF; // Number of sets - 1 : Claiming 1024 sets\n\n Res.edx = (0 << 0) | // Write-back invalidate\n (0 << 1); // Cache inclusiveness - Includes lower caches\n } else if (Leaf == 3) {\n // Report L3\n uint32_t CoreCount = Cores - 1;\n\n Res.eax = CacheType_Unified | // Cache type\n (0b011 << 5) | // Cache level\n (1 << 8) | // Self initializing cache level\n (0 << 9) | // Fully associative\n (CoreCount << 14); // Maximum number of addressable IDs for logical processors sharing this cache\n\n Res.ebx = (63 << 0) | // Line Size - 1 : Claiming 64 byte\n (0 << 12) | // Physical Line partitions\n (7 << 22); // Associativity - 1 : Claiming 8 way\n\n // 8MB\n Res.ecx = 0x4000; // Number of sets - 1 : Claiming 16384 sets\n\n Res.edx = (0 << 0) | // Write-back invalidate\n (0 << 1); // Cache inclusiveness - Includes lower caches\n }\n\n return Res;\n}\n\nFEXCore::CPUID::FunctionResults CPUIDEmu::Function_Reserved(uint32_t Leaf) const {\n FEXCore::CPUID::FunctionResults Res {};\n return Res;\n}\n\nFEXCore::CPUID::XCRResults CPUIDEmu::XCRFunction_0h() const {\n // This just returns XCR0\n FEXCore::CPUID::XCRResults Res {\n .eax = static_cast(XCR0),\n .edx = static_cast(XCR0 >> 32),\n };\n\n return Res;\n}\n\nCPUIDEmu::CPUIDEmu(const FEXCore::Context::ContextImpl* ctx)\n : CTX {ctx}\n , SupportsCPUIndexInTPIDRRO {CTX->HostFeatures.SupportsCPUIndexInTPIDRRO}\n , GetCPUID {GetCPUID_Syscall} {\n Cores = CTX->HostFeatures.CPUMIDRs.size();\n\n // Setup some state tracking\n SetupHostHybridFlag();\n\n SetupFeatures();\n\n#ifdef ARCHITECTURE_arm64\n if (SupportsCPUIndexInTPIDRRO) {\n GetCPUID = GetCPUID_TPIDRRO;\n }\n#endif\n}\n} // namespace FEXCore\n" }, { "path": "FEXCore/Source/Interface/Core/CPUID.h", "content": "// SPDX-License-Identifier: MIT\n#pragma once\n\n#include \n#include \n#include \n\n#include \n#include \n#include \n\nnamespace FEXCore {\nnamespace Context {\n class ContextImpl;\n}\n\nuint64_t GetCycleCounterFrequency();\n\n// Debugging define to switch what family of CPU we execute as.\n// Might be useful if an application makes an assumption about a CPU.\n// #define CPUID_AMD\nclass CPUIDEmu final {\nprivate:\n constexpr static uint32_t CPUID_VENDOR_INTEL1 = 0x756E6547; // \"Genu\"\n constexpr static uint32_t CPUID_VENDOR_INTEL2 = 0x49656E69; // \"ineI\"\n constexpr static uint32_t CPUID_VENDOR_INTEL3 = 0x6C65746E; // \"ntel\"\n\n constexpr static uint32_t CPUID_VENDOR_AMD1 = 0x68747541; // \"Auth\"\n constexpr static uint32_t CPUID_VENDOR_AMD2 = 0x69746E65; // \"enti\"\n constexpr static uint32_t CPUID_VENDOR_AMD3 = 0x444D4163; // \"cAMD\"\n\npublic:\n CPUIDEmu(const FEXCore::Context::ContextImpl* ctx);\n\n // X86 cacheline size effectively has to be hardcoded to 64\n // if we report anything differently then applications are likely to break\n constexpr static uint64_t CACHELINE_SIZE = 64;\n\n FEXCore::CPUID::FunctionResults RunFunction(uint32_t Function, uint32_t Leaf) const {\n if (Function < Primary.size()) {\n const auto Handler = Primary[Function];\n return (this->*Handler)(Leaf);\n }\n\n constexpr uint32_t HypervisorBase = 0x4000'0000;\n if (Function >= HypervisorBase && Function < (HypervisorBase + Hypervisor.size())) {\n const auto Handler = Hypervisor[Function - HypervisorBase];\n return (this->*Handler)(Leaf);\n }\n\n constexpr uint32_t ExtendedBase = 0x8000'0000;\n if (Function >= ExtendedBase && Function < (ExtendedBase + Extended.size())) {\n const auto Handler = Extended[Function - ExtendedBase];\n return (this->*Handler)(Leaf);\n }\n\n return Function_Reserved(Leaf);\n }\n\n FEXCore::CPUID::FunctionResults RunFunctionName(uint32_t Function, uint32_t Leaf, uint32_t CPU) const {\n if (Function == 0x8000'0002U) {\n return Function_8000_0002h(Leaf, CPU % PerCPUData.size());\n } else if (Function == 0x8000'0003U) {\n return Function_8000_0003h(Leaf, CPU % PerCPUData.size());\n } else {\n return Function_8000_0004h(Leaf, CPU % PerCPUData.size());\n }\n }\n\n FEXCore::CPUID::XCRResults RunXCRFunction(uint32_t Function) const {\n if (Function >= 1) {\n // XCR function 1 is not yet supported.\n return {};\n }\n\n return XCRFunction_0h();\n }\n\n bool DoesXCRFunctionReportConstantData(uint32_t Function) const {\n // Every function currently returns constant data.\n return true;\n }\n\n enum class SupportsConstant {\n CONSTANT,\n NONCONSTANT,\n };\n enum class NeedsLeafConstant {\n NEEDSLEAFCONSTANT,\n NOLEAFCONSTANT,\n };\n struct FunctionConstant {\n SupportsConstant SupportsConstantFunction;\n NeedsLeafConstant NeedsLeaf;\n };\n\n static constexpr FunctionConstant DoesFunctionReportConstantData(uint32_t Function) {\n if (Function < Primary.size()) {\n return Primary_Constant[Function];\n }\n\n constexpr uint32_t HypervisorBase = 0x4000'0000;\n if (Function >= HypervisorBase && Function < (HypervisorBase + Hypervisor.size())) {\n return Hypervisor_Constant[Function - HypervisorBase];\n }\n\n constexpr uint32_t ExtendedBase = 0x8000'0000;\n if (Function >= ExtendedBase && Function < (ExtendedBase + Extended.size())) {\n return Extended_Constant[Function - ExtendedBase];\n }\n\n // Anything unsupported is known constant return of reserved data.\n return {SupportsConstant::CONSTANT, NeedsLeafConstant::NOLEAFCONSTANT};\n }\n\nprivate:\n const FEXCore::Context::ContextImpl* CTX;\n [[maybe_unused]] bool SupportsCPUIndexInTPIDRRO {};\n bool Hybrid {};\n uint32_t Cores {};\n FEX_CONFIG_OPT(HideHypervisorBit, HIDEHYPERVISORBIT);\n\n // XFEATURE_ENABLED_MASK\n // Mask that configures what features are enabled on the CPU.\n // Affects XSAVE and XRSTOR when modified.\n // Bit layout is as follows.\n // [0] - x87 enabled\n // [1] - SSE enabled\n // [2] - YMM enabled (256-bit SSE)\n // [8:3] - Reserved. MBZ.\n // [9] - MPK\n // [10] - Reserved. MBZ.\n // [11] - CET_U\n // [12] - CET_S\n // [61:13] - Reserved. MBZ.\n // [62] - LWP (Lightweight profiling)\n // [63] - Reserved for XCR bit vector expansion. MBZ.\n // Always enable x87 and SSE by default.\n constexpr static uint64_t XCR0_X87 = 1ULL << 0;\n constexpr static uint64_t XCR0_SSE = 1ULL << 1;\n constexpr static uint64_t XCR0_AVX = 1ULL << 2;\n\n struct FeaturesConfig {\n uint64_t SHA : 1;\n uint64_t _pad : 63;\n };\n\n FeaturesConfig Features {\n .SHA = 1,\n };\n\n uint64_t XCR0 {XCR0_X87 | XCR0_SSE};\n\n uint32_t SupportsAVX() const {\n return (XCR0 & XCR0_AVX) ? 1 : 0;\n }\n\n using FunctionHandler = FEXCore::CPUID::FunctionResults (CPUIDEmu::*)(uint32_t Leaf) const;\n\n struct CPUData {\n const char* ProductName {};\n#ifdef ARCHITECTURE_arm64\n uint32_t MIDR {};\n#endif\n bool IsBig {};\n };\n fextl::vector PerCPUData {};\n\n // Functions\n FEXCore::CPUID::FunctionResults Function_0h(uint32_t Leaf) const;\n FEXCore::CPUID::FunctionResults Function_01h(uint32_t Leaf) const;\n FEXCore::CPUID::FunctionResults Function_02h(uint32_t Leaf) const;\n FEXCore::CPUID::FunctionResults Function_04h(uint32_t Leaf) const;\n FEXCore::CPUID::FunctionResults Function_06h(uint32_t Leaf) const;\n FEXCore::CPUID::FunctionResults Function_07h(uint32_t Leaf) const;\n FEXCore::CPUID::FunctionResults Function_0Dh(uint32_t Leaf) const;\n FEXCore::CPUID::FunctionResults Function_15h(uint32_t Leaf) const;\n FEXCore::CPUID::FunctionResults Function_1Ah(uint32_t Leaf) const;\n FEXCore::CPUID::FunctionResults Function_4000_0000h(uint32_t Leaf) const;\n FEXCore::CPUID::FunctionResults Function_4000_0001h(uint32_t Leaf) const;\n FEXCore::CPUID::FunctionResults Function_8000_0000h(uint32_t Leaf) const;\n FEXCore::CPUID::FunctionResults Function_8000_0001h(uint32_t Leaf) const;\n FEXCore::CPUID::FunctionResults Function_8000_0002h(uint32_t Leaf) const;\n FEXCore::CPUID::FunctionResults Function_8000_0003h(uint32_t Leaf) const;\n FEXCore::CPUID::FunctionResults Function_8000_0004h(uint32_t Leaf) const;\n\n FEXCore::CPUID::FunctionResults Function_8000_0002h(uint32_t Leaf, uint32_t CPU) const;\n FEXCore::CPUID::FunctionResults Function_8000_0003h(uint32_t Leaf, uint32_t CPU) const;\n FEXCore::CPUID::FunctionResults Function_8000_0004h(uint32_t Leaf, uint32_t CPU) const;\n\n FEXCore::CPUID::FunctionResults Function_8000_0005h(uint32_t Leaf) const;\n FEXCore::CPUID::FunctionResults Function_8000_0006h(uint32_t Leaf) const;\n FEXCore::CPUID::FunctionResults Function_8000_0007h(uint32_t Leaf) const;\n FEXCore::CPUID::FunctionResults Function_8000_0008h(uint32_t Leaf) const;\n FEXCore::CPUID::FunctionResults Function_8000_0019h(uint32_t Leaf) const;\n FEXCore::CPUID::FunctionResults Function_8000_001Dh(uint32_t Leaf) const;\n FEXCore::CPUID::FunctionResults Function_Reserved(uint32_t Leaf) const;\n\n FEXCore::CPUID::XCRResults XCRFunction_0h() const;\n\n void SetupHostHybridFlag();\n void SetupFeatures();\n static constexpr size_t PRIMARY_FUNCTION_COUNT = 27;\n static constexpr size_t HYPERVISOR_FUNCTION_COUNT = 2;\n static constexpr size_t EXTENDED_FUNCTION_COUNT = 32;\n static constexpr std::array Primary = {\n // 0: Highest function parameter and ID\n &CPUIDEmu::Function_0h,\n // 1: Processor info\n &CPUIDEmu::Function_01h,\n // 2: Cache and TLB info\n &CPUIDEmu::Function_02h,\n // 3: Serial Number(previously), now reserved\n &CPUIDEmu::Function_Reserved,\n#ifndef CPUID_AMD\n // 4: Deterministic cache parameters for each level\n &CPUIDEmu::Function_04h,\n#else\n &CPUIDEmu::Function_Reserved,\n#endif\n // 5: Monitor/mwait\n &CPUIDEmu::Function_Reserved,\n // 6: Thermal and power management\n &CPUIDEmu::Function_06h,\n // 7: Extended feature flags\n &CPUIDEmu::Function_07h,\n // 0x08: Reserved?\n &CPUIDEmu::Function_Reserved,\n // 9: Direct Cache Access information\n &CPUIDEmu::Function_Reserved,\n // 0x0A: Architectural performance monitoring\n &CPUIDEmu::Function_Reserved,\n // 0x0B: Extended topology enumeration\n &CPUIDEmu::Function_Reserved,\n // 0x0C: Reserved?\n &CPUIDEmu::Function_Reserved,\n // 0x0D: Processor extended state enumeration\n &CPUIDEmu::Function_0Dh,\n // 0x0E: Reserved?\n &CPUIDEmu::Function_Reserved,\n // 0x0F: Intel RDT monitoring\n &CPUIDEmu::Function_Reserved,\n // 0x10: Intel RDT allocation enumeration\n &CPUIDEmu::Function_Reserved,\n // 0x12: Reserved?\n &CPUIDEmu::Function_Reserved,\n // 0x12: Intel SGX capability enumeration\n &CPUIDEmu::Function_Reserved,\n // 0x13: Reserved\n &CPUIDEmu::Function_Reserved,\n // 0x14: Intel Processor trace\n &CPUIDEmu::Function_Reserved,\n#ifndef CPUID_AMD\n // Timestamp counter information\n // Doesn't exist on AMD hardware\n &CPUIDEmu::Function_15h,\n#else\n &CPUIDEmu::Function_Reserved,\n#endif\n // 0x16: Processor frequency information\n &CPUIDEmu::Function_Reserved,\n // 0x17: SoC vendor attribute enumeration\n &CPUIDEmu::Function_Reserved,\n // 0x18: Reserved?\n &CPUIDEmu::Function_Reserved,\n // 0x19: Reserved?\n &CPUIDEmu::Function_Reserved,\n#ifndef CPUID_AMD\n // 0x1A: Hybrid Information Sub-leaf\n &CPUIDEmu::Function_1Ah,\n#else\n &CPUIDEmu::Function_Reserved,\n#endif\n };\n\n static constexpr std::array Primary_Constant = {{\n // 0: Highest function parameter and ID\n {SupportsConstant::CONSTANT, NeedsLeafConstant::NOLEAFCONSTANT},\n // 1: Processor info\n {SupportsConstant::NONCONSTANT, NeedsLeafConstant::NOLEAFCONSTANT},\n // 2: Cache and TLB info\n {SupportsConstant::CONSTANT, NeedsLeafConstant::NOLEAFCONSTANT},\n // 3: Serial Number(previously), now reserved\n {SupportsConstant::CONSTANT, NeedsLeafConstant::NOLEAFCONSTANT},\n#ifndef CPUID_AMD\n // 4: Deterministic cache parameters for each level\n {SupportsConstant::CONSTANT, NeedsLeafConstant::NEEDSLEAFCONSTANT},\n#else\n // 4: Reserved\n {SupportsConstant::CONSTANT, NeedsLeafConstant::NOLEAFCONSTANT},\n#endif\n // 5: Monitor/mwait\n {SupportsConstant::CONSTANT, NeedsLeafConstant::NOLEAFCONSTANT},\n // 6: Thermal and power management\n {SupportsConstant::CONSTANT, NeedsLeafConstant::NOLEAFCONSTANT},\n // 7: Extended feature flags\n {SupportsConstant::CONSTANT, NeedsLeafConstant::NEEDSLEAFCONSTANT},\n // 0x08: Reserved?\n {SupportsConstant::CONSTANT, NeedsLeafConstant::NOLEAFCONSTANT},\n // 9: Direct Cache Access information\n {SupportsConstant::CONSTANT, NeedsLeafConstant::NOLEAFCONSTANT},\n // 0x0A: Architectural performance monitoring\n {SupportsConstant::CONSTANT, NeedsLeafConstant::NOLEAFCONSTANT},\n // 0x0B: Extended topology enumeration\n {SupportsConstant::CONSTANT, NeedsLeafConstant::NOLEAFCONSTANT},\n // 0x0C: Reserved?\n {SupportsConstant::CONSTANT, NeedsLeafConstant::NOLEAFCONSTANT},\n // 0x0D: Processor extended state enumeration\n {SupportsConstant::CONSTANT, NeedsLeafConstant::NEEDSLEAFCONSTANT},\n // 0x0E: Reserved?\n {SupportsConstant::CONSTANT, NeedsLeafConstant::NOLEAFCONSTANT},\n // 0x0F: Intel RDT monitoring\n {SupportsConstant::CONSTANT, NeedsLeafConstant::NOLEAFCONSTANT},\n // 0x10: Intel RDT allocation enumeration\n {SupportsConstant::CONSTANT, NeedsLeafConstant::NOLEAFCONSTANT},\n // 0x12: Reserved?\n {SupportsConstant::CONSTANT, NeedsLeafConstant::NOLEAFCONSTANT},\n // 0x12: Intel SGX capability enumeration\n {SupportsConstant::CONSTANT, NeedsLeafConstant::NOLEAFCONSTANT},\n // 0x13: Reserved\n {SupportsConstant::CONSTANT, NeedsLeafConstant::NOLEAFCONSTANT},\n // 0x14: Intel Processor trace\n {SupportsConstant::CONSTANT, NeedsLeafConstant::NOLEAFCONSTANT},\n#ifndef CPUID_AMD\n // 0x15: Timestamp counter information\n // Doesn't exist on AMD hardware\n {SupportsConstant::CONSTANT, NeedsLeafConstant::NOLEAFCONSTANT},\n#else\n // 0x15: Reserved\n {SupportsConstant::CONSTANT, NeedsLeafConstant::NOLEAFCONSTANT},\n#endif\n // 0x16: Processor frequency information\n {SupportsConstant::CONSTANT, NeedsLeafConstant::NOLEAFCONSTANT},\n // 0x17: SoC vendor attribute enumeration\n {SupportsConstant::CONSTANT, NeedsLeafConstant::NOLEAFCONSTANT},\n // 0x18: Reserved?\n {SupportsConstant::CONSTANT, NeedsLeafConstant::NOLEAFCONSTANT},\n // 0x19: Reserved?\n {SupportsConstant::CONSTANT, NeedsLeafConstant::NOLEAFCONSTANT},\n#ifndef CPUID_AMD\n // 0x1A: Hybrid Information Sub-leaf\n {SupportsConstant::NONCONSTANT, NeedsLeafConstant::NOLEAFCONSTANT},\n#else\n // 0x1A: Reserved\n {SupportsConstant::CONSTANT, NeedsLeafConstant::NOLEAFCONSTANT},\n#endif\n }};\n\n static constexpr std::array Hypervisor = {\n // Hypervisor CPUID information leaf\n &CPUIDEmu::Function_4000_0000h,\n // FEX-Emu specific leaf\n &CPUIDEmu::Function_4000_0001h,\n };\n\n static constexpr std::array Hypervisor_Constant = {{\n // Hypervisor CPUID information leaf\n {SupportsConstant::CONSTANT, NeedsLeafConstant::NOLEAFCONSTANT},\n // FEX-Emu specific leaf\n {SupportsConstant::CONSTANT, NeedsLeafConstant::NEEDSLEAFCONSTANT},\n }};\n\n static constexpr std::array Extended = {\n // Largest extended function number\n &CPUIDEmu::Function_8000_0000h,\n // Processor vendor\n &CPUIDEmu::Function_8000_0001h,\n // Processor brand string\n &CPUIDEmu::Function_8000_0002h,\n // Processor brand string continued\n &CPUIDEmu::Function_8000_0003h,\n // Processor brand string continued\n &CPUIDEmu::Function_8000_0004h,\n#ifdef CPUID_AMD\n // 0x8000'0005: L1 Cache and TLB identifiers\n &CPUIDEmu::Function_8000_0005h,\n#else\n &CPUIDEmu::Function_Reserved,\n#endif\n // 0x8000'0006: L2 Cache identifiers\n &CPUIDEmu::Function_8000_0006h,\n // 0x8000'0007: Advanced power management information\n &CPUIDEmu::Function_8000_0007h,\n // 0x8000'0008: Virtual and physical address sizes\n &CPUIDEmu::Function_8000_0008h,\n // 0x8000'0009: Reserved?\n &CPUIDEmu::Function_Reserved,\n // 0x8000'000A: SVM Revision\n &CPUIDEmu::Function_Reserved,\n // 0x8000'000B: Reserved?\n &CPUIDEmu::Function_Reserved,\n // 0x8000'000C: Reserved?\n &CPUIDEmu::Function_Reserved,\n // 0x8000'000D: Reserved?\n &CPUIDEmu::Function_Reserved,\n // 0x8000'000E: Reserved?\n &CPUIDEmu::Function_Reserved,\n // 0x8000'000F: Reserved?\n &CPUIDEmu::Function_Reserved,\n // 0x8000'0010: Reserved?\n &CPUIDEmu::Function_Reserved,\n // 0x8000'0011: Reserved?\n &CPUIDEmu::Function_Reserved,\n // 0x8000'0012: Reserved?\n &CPUIDEmu::Function_Reserved,\n // 0x8000'0013: Reserved?\n &CPUIDEmu::Function_Reserved,\n // 0x8000'0014: Reserved?\n &CPUIDEmu::Function_Reserved,\n // 0x8000'0015: Reserved?\n &CPUIDEmu::Function_Reserved,\n // 0x8000'0016: Reserved?\n &CPUIDEmu::Function_Reserved,\n // 0x8000'0017: Reserved?\n &CPUIDEmu::Function_Reserved,\n // 0x8000'0018: Reserved?\n &CPUIDEmu::Function_Reserved,\n // 0x8000'0019: TLB 1GB page identifiers\n &CPUIDEmu::Function_8000_0019h,\n // 0x8000'001A: Performance optimization identifiers\n &CPUIDEmu::Function_Reserved,\n // 0x8000'001B: Instruction based sampling identifiers\n &CPUIDEmu::Function_Reserved,\n // 0x8000'001C: Lightweight profiling capabilities\n &CPUIDEmu::Function_Reserved,\n#ifdef CPUID_AMD\n // 0x8000'001D: Cache properties\n &CPUIDEmu::Function_8000_001Dh,\n#else\n &CPUIDEmu::Function_Reserved,\n#endif\n // 0x8000'001E: Extended APIC ID\n &CPUIDEmu::Function_Reserved,\n // 0x8000'001F: AMD Secure Encryption\n &CPUIDEmu::Function_Reserved,\n };\n\n static constexpr std::array Extended_Constant = {{\n // Largest extended function number\n {SupportsConstant::CONSTANT, NeedsLeafConstant::NOLEAFCONSTANT},\n // Processor vendor\n {SupportsConstant::CONSTANT, NeedsLeafConstant::NOLEAFCONSTANT},\n // Processor brand string\n {SupportsConstant::NONCONSTANT, NeedsLeafConstant::NOLEAFCONSTANT},\n // Processor brand string continued\n {SupportsConstant::NONCONSTANT, NeedsLeafConstant::NOLEAFCONSTANT},\n // Processor brand string continued\n {SupportsConstant::NONCONSTANT, NeedsLeafConstant::NOLEAFCONSTANT},\n#ifdef CPUID_AMD\n // 0x8000'0005: L1 Cache and TLB identifiers\n {SupportsConstant::CONSTANT, NeedsLeafConstant::NOLEAFCONSTANT},\n#else\n // 0x8000'0005: Reserved\n {SupportsConstant::CONSTANT, NeedsLeafConstant::NOLEAFCONSTANT},\n#endif\n // 0x8000'0006: L2 Cache identifiers\n {SupportsConstant::CONSTANT, NeedsLeafConstant::NOLEAFCONSTANT},\n // 0x8000'0007: Advanced power management information\n {SupportsConstant::CONSTANT, NeedsLeafConstant::NOLEAFCONSTANT},\n // 0x8000'0008: Virtual and physical address sizes\n {SupportsConstant::CONSTANT, NeedsLeafConstant::NOLEAFCONSTANT},\n // 0x8000'0009: Reserved?\n {SupportsConstant::CONSTANT, NeedsLeafConstant::NOLEAFCONSTANT},\n // 0x8000'000A: SVM Revision\n {SupportsConstant::CONSTANT, NeedsLeafConstant::NOLEAFCONSTANT},\n // 0x8000'000B: Reserved?\n {SupportsConstant::CONSTANT, NeedsLeafConstant::NOLEAFCONSTANT},\n // 0x8000'000C: Reserved?\n {SupportsConstant::CONSTANT, NeedsLeafConstant::NOLEAFCONSTANT},\n // 0x8000'000D: Reserved?\n {SupportsConstant::CONSTANT, NeedsLeafConstant::NOLEAFCONSTANT},\n // 0x8000'000E: Reserved?\n {SupportsConstant::CONSTANT, NeedsLeafConstant::NOLEAFCONSTANT},\n // 0x8000'000F: Reserved?\n {SupportsConstant::CONSTANT, NeedsLeafConstant::NOLEAFCONSTANT},\n // 0x8000'0010: Reserved?\n {SupportsConstant::CONSTANT, NeedsLeafConstant::NOLEAFCONSTANT},\n // 0x8000'0011: Reserved?\n {SupportsConstant::CONSTANT, NeedsLeafConstant::NOLEAFCONSTANT},\n // 0x8000'0012: Reserved?\n {SupportsConstant::CONSTANT, NeedsLeafConstant::NOLEAFCONSTANT},\n // 0x8000'0013: Reserved?\n {SupportsConstant::CONSTANT, NeedsLeafConstant::NOLEAFCONSTANT},\n // 0x8000'0014: Reserved?\n {SupportsConstant::CONSTANT, NeedsLeafConstant::NOLEAFCONSTANT},\n // 0x8000'0015: Reserved?\n {SupportsConstant::CONSTANT, NeedsLeafConstant::NOLEAFCONSTANT},\n // 0x8000'0016: Reserved?\n {SupportsConstant::CONSTANT, NeedsLeafConstant::NOLEAFCONSTANT},\n // 0x8000'0017: Reserved?\n {SupportsConstant::CONSTANT, NeedsLeafConstant::NOLEAFCONSTANT},\n // 0x8000'0018: Reserved?\n {SupportsConstant::CONSTANT, NeedsLeafConstant::NOLEAFCONSTANT},\n // 0x8000'0019: TLB 1GB page identifiers\n {SupportsConstant::CONSTANT, NeedsLeafConstant::NOLEAFCONSTANT},\n // 0x8000'001A: Performance optimization identifiers\n {SupportsConstant::CONSTANT, NeedsLeafConstant::NOLEAFCONSTANT},\n // 0x8000'001B: Instruction based sampling identifiers\n {SupportsConstant::CONSTANT, NeedsLeafConstant::NOLEAFCONSTANT},\n // 0x8000'001C: Lightweight profiling capabilities\n {SupportsConstant::CONSTANT, NeedsLeafConstant::NOLEAFCONSTANT},\n#ifdef CPUID_AMD\n // 0x8000'001D: Cache properties\n {SupportsConstant::CONSTANT, NeedsLeafConstant::NEEDSLEAFCONSTANT},\n#else\n // 0x8000'001D: Reserved\n {SupportsConstant::CONSTANT, NeedsLeafConstant::NOLEAFCONSTANT},\n#endif\n // 0x8000'001E: Extended APIC ID\n {SupportsConstant::CONSTANT, NeedsLeafConstant::NOLEAFCONSTANT},\n // 0x8000'001F: AMD Secure Encryption\n {SupportsConstant::CONSTANT, NeedsLeafConstant::NOLEAFCONSTANT},\n }};\n\n using GetCPUIDPtr = uint32_t (*)();\n GetCPUIDPtr GetCPUID;\n};\n} // namespace FEXCore\n" }, { "path": "FEXCore/Source/Interface/Core/CodeCache.cpp", "content": "// SPDX-License-Identifier: MIT\n#include \"Utils/SpinWaitLock.h\"\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n#include \n#include \n#include \n\n#include \n\n#include \n\n#include \n\n#include \n\nnamespace FEXCore {\n\n#if __clang_major__ < 16\nExecutableFileInfo::ExecutableFileInfo(fextl::unique_ptr Map, uint64_t FileId, fextl::string Filename)\n : SourcecodeMap(std::move(Map))\n , FileId(FileId)\n , Filename(Filename) {}\n#endif\nExecutableFileInfo::~ExecutableFileInfo() = default;\n\nfextl::string CodeMap::GetBaseFilename(const ExecutableFileInfo& MainExecutable, bool AddNombSuffix) {\n auto FileId = MainExecutable.FileId;\n\n std::string_view base_filename = FHU::Filesystem::GetFilename(std::string_view {MainExecutable.Filename});\n if (FileId != 0xffff'ffff'ffff'ffff) {\n return fextl::fmt::format(\"{}-{:016x}{}\", base_filename, MainExecutable.FileId, AddNombSuffix ? \"-nomb\" : \"\");\n }\n\n return \"\";\n}\n\nfextl::map CodeMap::ParseCodeMap(std::ifstream& File) {\n fextl::map Ret;\n while (true) {\n Entry Entry;\n File.read(reinterpret_cast(&Entry), sizeof(Entry));\n if (!File) {\n break;\n }\n\n if (Entry.FileId == LoadExternalLibrary.FileId && Entry.BlockOffset == LoadExternalLibrary.BlockOffset) {\n ExternalLibraryInfo Info;\n File.read(reinterpret_cast(&Info), sizeof(Info));\n\n fextl::string Filename;\n std::getline(File, Filename, '\\0');\n\n // Align to 4-byte boundary\n char Null[4];\n File.read(Null, AlignUp(Filename.size() + 1, 4) - Filename.size() - 1);\n if (!File) {\n break;\n }\n Ret[Info.ExternalFileId].Filename = std::move(Filename);\n } else if (Entry.FileId == SetExecutableFileId {}.Marker.FileId && Entry.BlockOffset == SetExecutableFileId {}.Marker.BlockOffset) {\n CodeMapFileId ExecutableFileId;\n File.read(reinterpret_cast(&ExecutableFileId), sizeof(ExecutableFileId));\n if (!File) {\n break;\n }\n Ret[ExecutableFileId].IsExecutable = true;\n } else {\n if (!Ret.contains(Entry.FileId)) {\n LogMan::Msg::EFmt(\"Code map referenced unknown file id {:016x}\", Entry.FileId);\n } else {\n Ret[Entry.FileId].Blocks.insert(Entry.BlockOffset);\n }\n }\n\n if (!File) {\n break;\n }\n }\n return Ret;\n}\n\nCodeMapWriter::CodeMapWriter(CodeMapOpener& Opener, bool OpenEagerly)\n : Buffer(4096)\n , FileOpener(Opener) {\n if (OpenEagerly) {\n CodeMapFD = FileOpener.OpenCodeMapFile();\n }\n}\n\nCodeMapWriter::~CodeMapWriter() {\n if (CodeMapFD.value_or(-1) != -1) {\n Flush(BufferOffset);\n close(*CodeMapFD);\n }\n}\n\nbool CodeMapWriter::IsWriteEnabled(const ExecutableFileSectionInfo& Section) {\n if (CodeMapFD == -1) {\n return false;\n }\n\n // PV libraries can't yet be read by FEXServer, so skip dumping them\n if (Section.FileInfo.Filename.starts_with(\"/run/pressure-vessel\")) {\n return false;\n }\n\n if (CodeMapFD) {\n return true;\n }\n\n // Acquire mutex and re-check CodeMapFD to avoid race conditions\n auto lk = std::unique_lock {Mutex};\n if (!CodeMapFD) {\n CodeMapFD = FileOpener.OpenCodeMapFile();\n }\n\n return CodeMapFD != -1;\n}\n\nvoid CodeMapWriter::Flush(size_t Offset) {\n // Acquire exclusive lock and flush circular buffer\n std::unique_lock Lock {Mutex};\n Flush(Offset, Lock);\n}\n\nvoid CodeMapWriter::Flush(size_t Offset, std::unique_lock&) {\n write(*CodeMapFD, Buffer.data(), Offset);\n BufferOffset = 0;\n}\n\nvoid CodeMapWriter::AppendBlock(const FEXCore::ExecutableFileSectionInfo& SectionInfo, uint64_t BlockEntry) {\n if (!IsWriteEnabled(SectionInfo)) {\n return;\n }\n\n BlockEntry -= SectionInfo.FileStartVA;\n if (BlockEntry > std::numeric_limits::max()) {\n ERROR_AND_DIE_FMT(\"Cannot write code map\");\n }\n\n // Register new library if not already known\n bool NewLibraryLoad = false;\n {\n // Check prior registration with shared lock\n std::shared_lock Lock {Mutex};\n NewLibraryLoad = !KnownFileIds.contains(SectionInfo.FileInfo.FileId);\n }\n if (NewLibraryLoad) {\n // Register to map with exclusive lock\n std::unique_lock Lock {Mutex};\n NewLibraryLoad &= KnownFileIds.insert(SectionInfo.FileInfo.FileId).second;\n }\n if (NewLibraryLoad) {\n // Add entry to code map\n AppendLibraryLoad(SectionInfo.FileInfo);\n }\n\n // Register the actual code block\n CodeMap::Entry DataEntry {SectionInfo.FileInfo.FileId, static_cast(BlockEntry)};\n AppendData(std::as_bytes(std::span {&DataEntry, 1}));\n}\n\nvoid CodeMapWriter::AppendLibraryLoad(const FEXCore::ExecutableFileInfo& FileInfo) {\n // See CodeMap::ExternalLibraryInfo\n auto ExternalFileId = FileInfo.FileId;\n auto TotalSize = AlignUp(sizeof(CodeMap::LoadExternalLibrary) + sizeof(ExternalFileId) + FileInfo.Filename.size() + 1, 4);\n const auto Data = reinterpret_cast(alloca(TotalSize));\n auto WritePtr = std::copy_n(reinterpret_cast(&CodeMap::LoadExternalLibrary), sizeof(CodeMap::LoadExternalLibrary), Data);\n WritePtr = std::copy_n(reinterpret_cast(&ExternalFileId), sizeof(ExternalFileId), WritePtr);\n WritePtr = std::copy(FileInfo.Filename.begin(), FileInfo.Filename.end(), WritePtr);\n std::fill(WritePtr, Data + TotalSize, 0);\n AppendData(std::as_bytes(std::span {Data, TotalSize}));\n}\n\nvoid CodeMapWriter::AppendSetMainExecutable(const FEXCore::ExecutableFileInfo& FileInfo) {\n CodeMap::SetExecutableFileId Data {.ExecutableFileId = FileInfo.FileId};\n AppendData(std::span {reinterpret_cast(&Data), sizeof(Data)});\n}\n\nvoid CodeMapWriter::AppendData(std::span Data) {\n std::shared_lock Lock {Mutex};\n auto Offset = BufferOffset.fetch_add(Data.size_bytes());\n if (Offset + Data.size_bytes() > Buffer.size()) {\n // Acquire exclusive lock and flush the buffer.\n // Under heavy pressure, multiple threads may observe an exhausted buffer simultaneously.\n // The thread with the last in-bounds Offset is responsible for flushing the buffer.\n Lock.unlock();\n bool IsResponsibleForFlush = false;\n {\n std::unique_lock ExclusiveLock {Mutex};\n IsResponsibleForFlush = (Offset <= Buffer.size());\n if (IsResponsibleForFlush) {\n Flush(Offset, ExclusiveLock);\n }\n }\n if (!IsResponsibleForFlush) {\n // Wait for the buffer to be flushed on the responsible thread\n Utils::SpinWaitLock::WaitPred, size_t>(reinterpret_cast(&BufferOffset), Buffer.size());\n }\n AppendData(Data);\n return;\n }\n\n memcpy(&Buffer.at(Offset), Data.data(), Data.size_bytes());\n}\n\n} // namespace FEXCore\n\nnamespace FEXCore::Context {\n\nCodeCache::CodeCache(ContextImpl& CTX_)\n : CTX(CTX_) {}\nCodeCache::~CodeCache() = default;\n\nuint64_t CodeCache::ComputeCodeMapId(std::string_view Filename, int FD) {\n if (Filename.empty()) {\n return 0xffff'ffff'ffff'ffff;\n }\n\n // For now, we just use the file path as an identifier.\n // TODO: Ensure the hash is unique enough to distinguish executables while remaining independent of the installation location\n return XXH3_64bits(Filename.data(), Filename.size());\n}\n\nstruct CodeCacheHeader {\n std::array Magic = ExpectedMagic;\n uint32_t FormatVersion = 1;\n uint8_t FEXVersion[20] = {};\n uint32_t NumBlocks;\n uint32_t NumCodePages;\n uint32_t CodeBufferSize;\n uint32_t NumRelocations;\n uint32_t padding;\n uint64_t SerializedBaseAddress;\n // TODO: Consider including information from LookupCache.BlockLinks\n\n static constexpr std::array ExpectedMagic = {'F', 'X', 'C', 'C'};\n};\n\ntemplate\nconcept OrderedContainer = requires { typename T::key_compare; };\n\nbool CodeCache::SaveData(Core::InternalThreadState& Thread, int fd, const ExecutableFileSectionInfo& SourceBinary, uint64_t SerializedBaseAddress) {\n auto CodeBuffer = CTX.GetLatest();\n auto& LookupCache = *Thread.LookupCache->Shared;\n auto Relocations = Thread.CPUBackend->TakeRelocations(SourceBinary.FileStartVA);\n\n // Write file header\n CodeCacheHeader header {};\n static_assert(GIT_HASH.size() == sizeof(header.FEXVersion));\n std::ranges::copy(GIT_HASH, header.FEXVersion);\n header.NumBlocks = LookupCache.BlockList.size();\n header.NumCodePages = LookupCache.CodePages.size();\n header.CodeBufferSize = CTX.LatestOffset;\n header.NumRelocations = Relocations.size();\n header.SerializedBaseAddress = SerializedBaseAddress;\n ::write(fd, &header, sizeof(header));\n\n // Dump guest<->host block mappings\n {\n // Cache contents must be deterministic, so copy the unordered block list and then sort by key\n static_assert(!OrderedContainer, \"Already deterministic; drop temporary container\");\n fextl::vector> BlockList;\n BlockList.reserve(LookupCache.BlockList.size());\n for (auto& [Guest, BlockEntry] : LookupCache.BlockList) {\n static_assert(sizeof(Guest) == 8, \"Breaking change in code cache data layout\");\n BlockList.emplace_back(Guest, &BlockEntry);\n }\n std::ranges::sort(BlockList);\n\n for (auto [Guest, Host] : BlockList) {\n static_assert(sizeof(Host->HostCode) == 8, \"Breaking change in code cache data layout\");\n static_assert(sizeof(Host->CodePages[0]) == 8, \"Breaking change in code cache data layout\");\n\n Guest -= SourceBinary.FileStartVA;\n ::write(fd, &Guest, sizeof(Guest));\n uint64_t HostCode = Host->HostCode - reinterpret_cast(CodeBuffer->Ptr);\n ::write(fd, &HostCode, sizeof(HostCode));\n uint64_t NumCodePages = Host->CodePages.size();\n ::write(fd, &NumCodePages, sizeof(NumCodePages));\n LOGMAN_THROW_A_FMT(std::ranges::is_sorted(Host->CodePages), \"Code pages aren't sorted\");\n for (auto CodePage : Host->CodePages) {\n CodePage -= SourceBinary.FileStartVA;\n ::write(fd, &CodePage, sizeof(CodePage));\n }\n }\n }\n\n // Dump relocations\n static_assert(sizeof(Relocations[0]) == 48, \"Breaking change in code cache data layout\");\n ::write(fd, Relocations.data(), Relocations.size() * sizeof(Relocations[0]));\n\n // Pad to next page in file so that the CodeBuffer can be mmap'ed into process on load\n char Zero[64] {};\n auto Off = lseek(fd, 0, SEEK_CUR);\n while (Off != AlignUp(Off, Utils::FEX_PAGE_SIZE)) {\n auto BytesToWrite = std::min(AlignUp(Off, Utils::FEX_PAGE_SIZE) - Off, sizeof(Zero));\n ::write(fd, Zero, BytesToWrite);\n Off += BytesToWrite;\n }\n\n // Dump the host code (relocated for position-independent serialization)\n std::span CodeBufferData(reinterpret_cast(CodeBuffer->Ptr), reinterpret_cast(CodeBuffer->Ptr) + CTX.LatestOffset);\n if (!ApplyCodeRelocations(SerializedBaseAddress, CodeBufferData, Relocations, true)) {\n LOGMAN_THROW_A_FMT(false, \"Failed to apply code relocations\");\n return false;\n }\n ::write(fd, CodeBufferData.data(), CodeBufferData.size());\n\n // Dump code pages\n static_assert(OrderedContainer, \"Non-deterministic data source\");\n for (const auto& [PageIndex, Entrypoints] : LookupCache.CodePages) {\n uint64_t PageAddr = (PageIndex << 12) - SourceBinary.FileStartVA;\n ::write(fd, &PageAddr, sizeof(PageAddr));\n uint64_t NumEntrypoints = Entrypoints.size();\n ::write(fd, &NumEntrypoints, sizeof(NumEntrypoints));\n for (uint64_t Entrypoint : Entrypoints) {\n Entrypoint -= SourceBinary.FileStartVA;\n ::write(fd, &Entrypoint, sizeof(Entrypoint));\n }\n }\n\n return true;\n}\n\nbool CodeCache::LoadData(Core::InternalThreadState* Thread, std::byte* MappedCacheFile, const ExecutableFileSectionInfo& BinarySection) {\n if (!EnableCodeCaching) {\n return true;\n }\n\n namespace ranges = std::ranges;\n\n // Read file header\n CodeCacheHeader header {};\n ::memcpy(&header, MappedCacheFile, sizeof(header));\n MappedCacheFile += sizeof(header);\n\n LogMan::Msg::IFmt(\"Cache load: {:5} blocks; base={:#14x}; off={:#9x}-{:#09x}; {:016x} {}\", header.NumBlocks, BinarySection.FileStartVA,\n BinarySection.BeginVA - BinarySection.FileStartVA, BinarySection.EndVA - BinarySection.FileStartVA,\n BinarySection.FileInfo.FileId, BinarySection.FileInfo.Filename);\n\n if (!ranges::equal(header.Magic, header.ExpectedMagic)) {\n LogMan::Msg::EFmt(\"Invalid cache file header\");\n return false;\n }\n\n if (!ranges::equal(header.FEXVersion, GIT_HASH)) {\n LogMan::Msg::IFmt(\"Cache generated from old FEX version {:02x}, current is {:02x}; skipping\", fmt::join(header.FEXVersion, \"\"),\n fmt::join(GIT_HASH, \"\"));\n return false;\n }\n\n if (header.NumBlocks == 0) {\n // Valid caches are never empty\n LogMan::Msg::IFmt(\"Code cache empty, aborting\");\n return false;\n }\n\n // Read guest<->host block mappings\n using BlockListEntry = decltype(GuestToHostMap::BlockList)::value_type;\n fextl::vector BlockList(header.NumBlocks);\n {\n for (auto& BlockPtr : BlockList) {\n ::memcpy(&BlockPtr.first, MappedCacheFile, sizeof(BlockPtr.first));\n MappedCacheFile += sizeof(BlockPtr.first);\n ::memcpy(&BlockPtr.second.HostCode, MappedCacheFile, sizeof(BlockPtr.second.HostCode));\n MappedCacheFile += sizeof(BlockPtr.second.HostCode);\n uint64_t NumGuestPages;\n ::memcpy(&NumGuestPages, MappedCacheFile, sizeof(NumGuestPages));\n MappedCacheFile += sizeof(NumGuestPages);\n\n BlockPtr.second.CodePages.resize(NumGuestPages);\n ::memcpy(BlockPtr.second.CodePages.data(), MappedCacheFile, std::span {BlockPtr.second.CodePages}.size_bytes());\n MappedCacheFile += std::span {BlockPtr.second.CodePages}.size_bytes();\n }\n\n // Consistency check: VMA regions at the top and end should belong to the same file\n auto [min_val, max_val] = ranges::minmax_element(BlockList, std::less {}, &decltype(BlockList)::value_type::first);\n auto MinBound = CTX.SyscallHandler->LookupExecutableFileSection(Thread, min_val->first + BinarySection.FileStartVA);\n auto MaxBound = CTX.SyscallHandler->LookupExecutableFileSection(Thread, max_val->first + BinarySection.FileStartVA);\n if (&MinBound->FileInfo != &BinarySection.FileInfo || &MaxBound->FileInfo != &BinarySection.FileInfo) {\n ERROR_AND_DIE_FMT(\"Cached blocks offsets {:#x}-{:#x} out of bounds for guest library {} ({:016x} @ {:#x}) while trying to load \"\n \"section {:#x}-{:#x}!\",\n min_val->first, max_val->first, BinarySection.FileInfo.Filename, BinarySection.FileInfo.FileId,\n BinarySection.FileStartVA, BinarySection.BeginVA, BinarySection.EndVA);\n }\n\n // Constrain BlockList to the given ExecutableFileSectionInfo\n LOGMAN_THROW_A_FMT(ranges::is_sorted(BlockList, [](auto& a, auto& b) { return a.first < b.first; }), \"Expected sorted block list\");\n auto begin = ranges::lower_bound(BlockList, BinarySection.BeginVA - BinarySection.FileStartVA, std::less {}, &BlockListEntry::first);\n auto end =\n ranges::upper_bound(begin, BlockList.end(), BinarySection.EndVA - BinarySection.FileStartVA - 1, std::less {}, &BlockListEntry::first);\n BlockList.erase(end, BlockList.end());\n BlockList.erase(BlockList.begin(), begin);\n if (BlockList.empty()) {\n // Not an error since there is just no data to load\n LogMan::Msg::IFmt(\"No blocks cached in this range, aborting\");\n return true;\n }\n }\n\n // Read relocations\n fextl::vector Relocations(header.NumRelocations, FEXCore::CPU::Relocation::Default());\n ::memcpy(Relocations.data(), MappedCacheFile, Relocations.size() * sizeof(Relocations[0]));\n MappedCacheFile += Relocations.size() * sizeof(Relocations[0]);\n\n // Pad to next page in file, which contains CodeBuffer data\n MappedCacheFile = reinterpret_cast(AlignUp(reinterpret_cast(MappedCacheFile), Utils::FEX_PAGE_SIZE));\n\n // Prepare CodeBuffer: Page aligned and big enough to hold all cached data\n auto Lock = std::unique_lock {CTX.CodeBufferWriteMutex};\n if (Thread) {\n if (auto Prev = Thread->CPUBackend->CheckCodeBufferUpdate()) {\n Allocator::VirtualDontNeed(Thread->CallRetStackBase, FEXCore::Core::InternalThreadState::CALLRET_STACK_SIZE);\n auto lk = Thread->LookupCache->AcquireWriteLock();\n Thread->LookupCache->ChangeGuestToHostMapping(*Prev, *CTX.GetLatest()->LookupCache, lk);\n }\n }\n\n auto CodeBuffer = CTX.GetLatest();\n LOGMAN_THROW_A_FMT(reinterpret_cast(CodeBuffer->Ptr) % 0x1000 == 0, \"Expected CodeBuffer base to be page-aligned\");\n const auto Delta = AlignUp(CTX.LatestOffset, 0x1000) - CTX.LatestOffset;\n CTX.LatestOffset += Delta;\n\n while (CTX.LatestOffset + header.CodeBufferSize > CodeBuffer->UsableSize()) {\n if (Thread) {\n CTX.ClearCodeCache(Thread);\n CodeBuffer = CTX.GetLatest();\n LogMan::Msg::IFmt(\"Increased code buffer size to {} MiB for cache load\", CodeBuffer->AllocatedSize / 1024 / 1024);\n } else {\n ERROR_AND_DIE_FMT(\"Cannot extend codebuffer without thread!\");\n }\n }\n\n // Read CodeBuffer data from file. Make sure the destination is page-aligned.\n // TODO: Only load the data needed for the selected section\n auto CodeBufferRange =\n std::as_writable_bytes(std::span {CodeBuffer->Ptr, CodeBuffer->UsableSize()}).subspan(CTX.LatestOffset, header.CodeBufferSize);\n ::memcpy(CodeBufferRange.data(), MappedCacheFile, header.CodeBufferSize);\n MappedCacheFile += header.CodeBufferSize;\n CTX.LatestOffset += header.CodeBufferSize;\n\n // Apply FEX relocations\n auto Ret = ApplyCodeRelocations(BinarySection.FileStartVA, CodeBufferRange, Relocations, false);\n LOGMAN_THROW_A_FMT(Ret == true, \"Failed to apply code cache relocations\");\n\n {\n auto& LookupCache = *CodeBuffer->LookupCache;\n auto WriteLock = LookupCache.AcquireWriteLock();\n\n // Register blocks to LookupCache\n for (auto& [Guest, Host] : BlockList) {\n for (auto& CodePage : Host.CodePages) {\n CodePage += BinarySection.FileStartVA;\n }\n auto HostCode = reinterpret_cast(Host.HostCode + reinterpret_cast(CodeBufferRange.data()));\n LookupCache.AddBlockMapping(Guest + BinarySection.FileStartVA, std::move(Host.CodePages), HostCode, WriteLock);\n }\n\n // Register loaded code ranges\n fextl::vector Entrypoints;\n for (uint32_t i = 0; i < header.NumCodePages; ++i) {\n uint64_t CodePage;\n memcpy(&CodePage, MappedCacheFile, sizeof(CodePage));\n CodePage += BinarySection.FileStartVA;\n MappedCacheFile += sizeof(CodePage);\n\n uint64_t NumEntrypoints;\n memcpy(&NumEntrypoints, MappedCacheFile, sizeof(NumEntrypoints));\n MappedCacheFile += sizeof(NumEntrypoints);\n\n Entrypoints.resize(NumEntrypoints);\n memcpy(Entrypoints.data(), MappedCacheFile, NumEntrypoints * sizeof(Entrypoints[0]));\n MappedCacheFile += NumEntrypoints * sizeof(Entrypoints[0]);\n for (auto& Entrypoint : Entrypoints) {\n Entrypoint += BinarySection.FileStartVA;\n }\n\n if (LookupCache.AddBlockExecutableRange(Entrypoints, CodePage, FEXCore::Utils::FEX_PAGE_SIZE, WriteLock)) {\n CTX.SyscallHandler->MarkGuestExecutableRange(Thread, CodePage, FEXCore::Utils::FEX_PAGE_SIZE);\n }\n }\n }\n\n if (EnableCodeCacheValidation) {\n fextl::set GuestBlocks, HostBlocks;\n for (auto& [Guest, Host] : BlockList) {\n GuestBlocks.insert(Guest + BinarySection.FileStartVA);\n HostBlocks.insert(Host.HostCode);\n }\n\n Validate(BinarySection, std::move(GuestBlocks), HostBlocks, CodeBufferRange);\n }\n\n return true;\n}\n\nvoid CodeCache::Validate(const ExecutableFileSectionInfo& Section, fextl::set GuestBlocks, const fextl::set& HostBlocks,\n std::span CachedCode) {\n LOGMAN_THROW_A_FMT(!HostBlocks.empty(), \"Tried to validate without any host blocks\");\n // Skip any cached data before the first host block\n CachedCode = CachedCode.subspan(*HostBlocks.begin() - sizeof(CPU::CPUBackend::JITCodeHeader));\n\n if (!ValidationCTX) {\n ValidationCTX.reset(static_cast(FEXCore::Context::Context::CreateNewContext(CTX.HostFeatures).release()));\n ValidationCTX->SetSignalDelegator(CTX.SignalDelegation);\n ValidationCTX->SetSyscallHandler(CTX.SyscallHandler);\n ValidationCTX->SetThunkHandler(CTX.ThunkHandler);\n if (!ValidationCTX->InitCore()) {\n ERROR_AND_DIE_FMT(\"Failed to create cache load validation context\");\n }\n\n ValidationThread.reset(ValidationCTX->CreateThread(0, 0, nullptr));\n\n auto Frame = ValidationThread->CurrentFrame;\n Frame->State.segment_arrays[FEXCore::Core::CPUState::SEGMENT_ARRAY_INDEX_GDT] = &ValidationGDT[0];\n Frame->State.segment_arrays[FEXCore::Core::CPUState::SEGMENT_ARRAY_INDEX_LDT] = &ValidationGDT[0];\n Frame->State.cs_idx = 0;\n Frame->State.cs_cached = 0;\n\n if (ValidationCTX->Config.Is64BitMode()) {\n ValidationGDT[0].L = 1; // L = Long Mode = 64-bit\n ValidationGDT[0].D = 0; // D = Default Operand Size = Reserved\n } else {\n ValidationGDT[0].L = 0; // L = Long Mode = 32-bit\n ValidationGDT[0].D = 1; // D = Default Operand Size = 32-bit\n }\n }\n\n auto NewCodeBuffer = ValidationCTX->GetLatest();\n while (CachedCode.size_bytes() > NewCodeBuffer->UsableSize()) {\n ValidationCTX->ClearCodeCache(ValidationThread.get());\n NewCodeBuffer = ValidationCTX->GetLatest();\n LogMan::Msg::IFmt(\"Increased cache validation code buffer size to {} MiB\", NewCodeBuffer->AllocatedSize / 1024 / 1024);\n }\n\n std::span CodeBufferRangeRef =\n std::as_writable_bytes(std::span {NewCodeBuffer->Ptr, NewCodeBuffer->Ptr + NewCodeBuffer->UsableSize()}).subspan(0, CachedCode.size_bytes());\n\n while (!GuestBlocks.empty()) {\n auto [CompiledBlocks, _, _2, _3, _4] = ValidationCTX->CompileCode(ValidationThread.get(), *GuestBlocks.begin(), 0 /* TODO: Set MaxInst? */);\n for (auto& Entry : CompiledBlocks.EntryPoints) {\n GuestBlocks.erase(Entry.first);\n }\n }\n\n // Patch FEX-internal function addresses with values from the main Context to ensure the code blocks are comparable\n auto NewRelocations = ValidationThread->CPUBackend->TakeRelocations(Section.FileStartVA);\n NewRelocations.erase(std::remove_if(NewRelocations.begin(), NewRelocations.end(), [](const CPU::Relocation& Reloc) {\n return Reloc.Header.Type != CPU::RelocationTypes::RELOC_NAMED_SYMBOL_LITERAL && Reloc.Header.Type != CPU::RelocationTypes::RELOC_NAMED_THUNK_MOVE;\n }));\n (void)ApplyCodeRelocations(Section.FileStartVA, CodeBufferRangeRef, NewRelocations, false);\n\n if (ValidationCTX->LatestOffset <= CodeBufferRangeRef.size()) {\n // Reference compilation produced fewer bytes than our cache, so validation is going to fail.\n // Make sure we don't output any garbage bytes though.\n CodeBufferRangeRef = CodeBufferRangeRef.subspan(0, ValidationCTX->LatestOffset);\n }\n\n auto [Mismatch, _] = std::mismatch(CodeBufferRangeRef.begin(), CodeBufferRangeRef.end(), CachedCode.begin());\n if (Mismatch != CodeBufferRangeRef.end()) {\n // Align down to instruction size\n auto Idx = AlignDown(std::distance(CodeBufferRangeRef.begin(), Mismatch), 4);\n\n auto BlockIt = std::prev(HostBlocks.lower_bound(*HostBlocks.begin() + Idx + 1));\n std::optional GuestBlockAddr;\n std::optional GuestBlockAddrRef;\n if (BlockIt != HostBlocks.end()) {\n for (int i : {0, 1}) {\n std::span Buffer = (i == 0 ? CachedCode : CodeBufferRangeRef);\n\n // Second instruction is always a constant load for relative offset to the (multi)block start\n int32_t addr = (*reinterpret_cast(&Buffer[*BlockIt - *HostBlocks.begin() + 4]) & 0x3ff'ffe0) << 11;\n addr >>= 14;\n auto header = reinterpret_cast(&Buffer[*BlockIt - *HostBlocks.begin() + 4 + addr]);\n auto tail = reinterpret_cast(reinterpret_cast(header) + header->OffsetToBlockTail);\n (i == 0 ? GuestBlockAddr : GuestBlockAddrRef) = tail->RIP - Section.FileStartVA;\n LogMan::Msg::EFmt(\"Recorded rip {}: {:#x} (offset {:#x})\", i, tail->RIP, tail->RIP - Section.FileStartVA);\n\n if (i == 1) {\n if (tail->RIP >= Section.BeginVA && tail->RIP < Section.EndVA) {\n auto [IRView, TotalInstructions, TotalInstructionsLength, StartAddr, Length, _] =\n ValidationCTX->GenerateIR(ValidationThread.get(), tail->RIP, false, FEXCore::Config::Get_MAXINST());\n fextl::stringstream ss;\n FEXCore::IR::Dump(&ss, &*IRView);\n LogMan::Msg::EFmt(\"IR:\\n{}\", ss.str());\n } else {\n LogMan::Msg::EFmt(\"Can't dump IR for out-of-range RIP {:#x}\", tail->RIP);\n }\n }\n }\n }\n\n fextl::string GuestBlockInfo = \"UNKNOWN\";\n if (GuestBlockAddr) {\n GuestBlockInfo = fextl::fmt::format(\"{:#x}\", GuestBlockAddr.value());\n }\n if (GuestBlockAddr != GuestBlockAddrRef) {\n GuestBlockInfo += \" (MISMATCH)\";\n }\n ERROR_AND_DIE_FMT(\"Cache validation failed at offset {:#x}: {:02x} <-> {:02x} (at {} <-> {}, guest block {})\", Idx,\n fmt::join(CachedCode.subspan(Idx, 4), \"\"), fmt::join(CodeBufferRangeRef.subspan(Idx, 4), \"\"),\n fmt::ptr(CachedCode.data()), fmt::ptr(CodeBufferRangeRef.data()), GuestBlockInfo);\n }\n\n // Reset Context state for next validation\n ValidationThread->LookupCache->ClearCache(ValidationThread->LookupCache->AcquireWriteLock());\n ValidationCTX->LatestOffset = 0;\n\n LogMan::Msg::IFmt(\"\\tSuccessfully validated cache\");\n}\n\nbool CodeCache::ApplyCodeRelocations(uint64_t GuestEntry, std::span Code,\n std::span EntryRelocations, bool ForStorage) {\n CPU::Arm64Emitter Emitter(&CTX, Code.data(), Code.size_bytes());\n for (size_t j = 0; j < EntryRelocations.size(); ++j) {\n const FEXCore::CPU::Relocation& Reloc = EntryRelocations[j];\n Emitter.SetCursorOffset(Reloc.Header.Offset);\n\n switch (Reloc.Header.Type) {\n case FEXCore::CPU::RelocationTypes::RELOC_NAMED_SYMBOL_LITERAL: {\n // Generate a literal so we can place it\n uint64_t Pointer = ForStorage ? 0 : GetNamedSymbolLiteral(CTX, Reloc.NamedSymbolLiteral.Symbol);\n Emitter.dc64(Pointer);\n break;\n }\n case FEXCore::CPU::RelocationTypes::RELOC_NAMED_THUNK_MOVE: {\n uint64_t Pointer = ForStorage ? 0 : reinterpret_cast(CTX.ThunkHandler->LookupThunk(Reloc.NamedThunkMove.Symbol));\n if (Pointer == ~0ULL) {\n return false;\n }\n // TODO: Pointers are required to fit within 48-bit VA space.\n // But forcing 6-byte broke relocations.\n Emitter.LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Register(Reloc.NamedThunkMove.RegisterIndex), Pointer,\n CPU::Arm64Emitter::PadType::DOPAD);\n break;\n }\n case FEXCore::CPU::RelocationTypes::RELOC_GUEST_RIP_LITERAL: {\n Emitter.dc64(GuestEntry + Reloc.GuestRIP.GuestRIP);\n break;\n }\n case FEXCore::CPU::RelocationTypes::RELOC_GUEST_RIP_MOVE: {\n uint64_t Pointer = Reloc.GuestRIP.GuestRIP + GuestEntry;\n // TODO: Pointers are required to fit within 48-bit VA space.\n // But forcing 6-byte broke relocations.\n Emitter.LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Register(Reloc.GuestRIP.RegisterIndex), Pointer, CPU::Arm64Emitter::PadType::DOPAD);\n break;\n }\n\n default: ERROR_AND_DIE_FMT(\"Unknown relocation type {}\", ToUnderlying(Reloc.Header.Type));\n }\n }\n\n return true;\n}\n\n} // namespace FEXCore::Context\n" }, { "path": "FEXCore/Source/Interface/Core/Core.cpp", "content": "// SPDX-License-Identifier: MIT\n/*\n$info$\ncategory: glue ~ Logic that binds various parts together\nmeta: glue|driver ~ Emulation mainloop related glue logic\ntags: glue|driver\ndesc: Glues Frontend, OpDispatcher and IR Opts & Compilation, LookupCache, Dispatcher and provides the Execution loop entrypoint\n$end_info$\n*/\n\n#include \n#ifdef ZYDIS_DISASSEMBLER\n#include \n#endif\n#include \"Interface/Core/ArchHelpers/Arm64Emitter.h\"\n#include \"Interface/Core/LookupCache.h\"\n#include \"Interface/Core/CPUBackend.h\"\n#include \"Interface/Core/CPUID.h\"\n#include \"Interface/Core/Frontend.h\"\n#include \"Interface/Core/OpcodeDispatcher.h\"\n#include \"Interface/Core/JIT/JITClass.h\"\n#include \"Interface/Core/Dispatcher/Dispatcher.h\"\n#include \"Interface/Core/X86Tables/X86Tables.h\"\n#include \n#include \"Interface/IR/IR.h\"\n#include \"Interface/IR/IREmitter.h\"\n#include \"Interface/IR/Passes/RegisterAllocationPass.h\"\n#include \"Interface/IR/Passes.h\"\n#include \"Interface/IR/PassManager.h\"\n#include \"Interface/IR/RegisterAllocationData.h\"\n#include \"Utils/Allocator.h\"\n#include \"Utils/Allocator/HostAllocator.h\"\n#include \"Utils/SpinWaitLock.h\"\n#include \"Utils/variable_length_integer.h\"\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \"FEXCore/Utils/SignalScopeGuards.h\"\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\nnamespace FEXCore::Context {\nContextImpl::ContextImpl(const FEXCore::HostFeatures& Features)\n : HostFeatures {Features}\n , CPUID {this}\n , CodeCache {*this} {\n if (!Config.Is64BitMode()) {\n // When operating in 32-bit mode, the virtual memory we care about is only the lower 32-bits.\n Config.VirtualMemSize = 1ULL << 32;\n }\n\n if (Config.BlockJITNaming() || Config.GlobalJITNaming() || Config.LibraryJITNaming()) {\n // Only initialize symbols file if enabled. Ensures we don't pollute /tmp with empty files.\n Symbols.InitFile();\n }\n\n uint64_t FrequencyCounter = FEXCore::GetCycleCounterFrequency();\n if (FrequencyCounter && FrequencyCounter < FEXCore::Context::TSC_SCALE_MAXIMUM && Config.SmallTSCScale()) {\n // Scale TSC until it is at the minimum required.\n while (FrequencyCounter < FEXCore::Context::TSC_SCALE_MAXIMUM) {\n FrequencyCounter <<= 1;\n ++Config.TSCScale;\n }\n }\n\n // Track atomic TSO emulation configuration.\n UpdateAtomicTSOEmulationConfig();\n}\n\nstruct GetFrameBlockInfoResult {\n const CPU::CPUBackend::JITCodeHeader* InlineHeader;\n const CPU::CPUBackend::JITCodeTail* InlineTail;\n};\nstatic GetFrameBlockInfoResult GetFrameBlockInfo(FEXCore::Core::CpuStateFrame* Frame) {\n const uint64_t BlockBegin = Frame->State.InlineJITBlockHeader;\n auto InlineHeader = reinterpret_cast(BlockBegin);\n\n if (InlineHeader) {\n auto InlineTail = reinterpret_cast(Frame->State.InlineJITBlockHeader + InlineHeader->OffsetToBlockTail);\n return {InlineHeader, InlineTail};\n }\n\n return {InlineHeader, nullptr};\n}\n\nbool ContextImpl::IsAddressInCurrentBlock(FEXCore::Core::InternalThreadState* Thread, uint64_t Address, uint64_t Size) {\n auto [_, InlineTail] = GetFrameBlockInfo(Thread->CurrentFrame);\n return InlineTail && (Address + Size > InlineTail->RIP && Address < InlineTail->RIP + InlineTail->GuestSize);\n}\n\nbool ContextImpl::IsCurrentBlockSingleInst(FEXCore::Core::InternalThreadState* Thread) {\n auto [_, InlineTail] = GetFrameBlockInfo(Thread->CurrentFrame);\n return InlineTail && InlineTail->SingleInst;\n}\n\nuint64_t ContextImpl::GetGuestBlockEntry(FEXCore::Core::InternalThreadState* Thread) {\n auto [_, InlineTail] = GetFrameBlockInfo(Thread->CurrentFrame);\n return InlineTail ? InlineTail->RIP : 0;\n}\n\nuint64_t ContextImpl::RestoreRIPFromHostPC(FEXCore::Core::InternalThreadState* Thread, uint64_t HostPC) {\n const auto Frame = Thread->CurrentFrame;\n const uint64_t BlockBegin = Frame->State.InlineJITBlockHeader;\n auto [InlineHeader, InlineTail] = GetFrameBlockInfo(Thread->CurrentFrame);\n\n if (InlineHeader) {\n // Check if the host PC is currently within a code block.\n // If it is then RIP can be reconstructed from the beginning of the code block.\n // This is currently as close as FEX can get RIP reconstructions.\n if (HostPC >= reinterpret_cast(BlockBegin) && HostPC < reinterpret_cast(BlockBegin + InlineTail->Size)) {\n\n auto RIPEntry =\n reinterpret_cast(Frame->State.InlineJITBlockHeader + InlineHeader->OffsetToBlockTail + InlineTail->OffsetToRIPEntries);\n\n // Reconstruct RIP from JIT entries for this block.\n uint64_t StartingHostPC = BlockBegin;\n uint64_t StartingGuestRIP = InlineTail->RIP;\n\n for (uint32_t i = 0; i < InlineTail->NumberOfRIPEntries; ++i) {\n auto Offset = FEXCore::Utils::vl64pair::Decode(RIPEntry);\n RIPEntry += Offset.Size;\n if (HostPC >= (StartingHostPC + Offset.IntegerARMPC)) {\n // We are beyond this entry, keep going forward.\n StartingHostPC += Offset.IntegerARMPC;\n StartingGuestRIP += Offset.IntegerX86RIP;\n } else {\n // Passed where the Host PC is at. Break now.\n break;\n }\n }\n return StartingGuestRIP;\n }\n }\n\n // Fallback to what is stored in the RIP currently.\n return Frame->State.rip;\n}\n\nuint32_t ContextImpl::ReconstructCompactedEFLAGS(FEXCore::Core::InternalThreadState* Thread, bool WasInJIT, const uint64_t* HostGPRs,\n uint64_t PSTATE) {\n const auto Frame = Thread->CurrentFrame;\n uint32_t EFLAGS {};\n\n // Currently these flags just map 1:1 inside of the resulting value.\n for (size_t i = 0; i < FEXCore::Core::CPUState::NUM_EFLAG_BITS; ++i) {\n switch (i) {\n case X86State::RFLAG_CF_RAW_LOC:\n case X86State::RFLAG_PF_RAW_LOC:\n case X86State::RFLAG_AF_RAW_LOC:\n case X86State::RFLAG_TF_RAW_LOC:\n case X86State::RFLAG_ZF_RAW_LOC:\n case X86State::RFLAG_SF_RAW_LOC:\n case X86State::RFLAG_OF_RAW_LOC:\n case X86State::RFLAG_DF_RAW_LOC:\n // Intentionally do nothing.\n // These contain multiple bits which can corrupt other members when compacted.\n break;\n default: EFLAGS |= uint32_t {Frame->State.flags[i]} << i; break;\n }\n }\n\n uint32_t Packed_NZCV {};\n if (WasInJIT) {\n // If we were in the JIT then NZCV is in the CPU's PSTATE object.\n // Packed in to the same bit locations as RFLAG_NZCV_LOC.\n Packed_NZCV = PSTATE;\n\n // If we were in the JIT then PF and AF are in registers.\n // Move them to the CPUState frame now.\n Frame->State.pf_raw = HostGPRs[CPU::REG_PF.Idx()];\n Frame->State.af_raw = HostGPRs[CPU::REG_AF.Idx()];\n } else {\n // If we were not in the JIT then the NZCV state is stored in the CPUState RFLAG_NZCV_LOC.\n // SF/ZF/CF/OF are packed in a 32-bit value in RFLAG_NZCV_LOC.\n memcpy(&Packed_NZCV, &Frame->State.flags[X86State::RFLAG_NZCV_LOC], sizeof(Packed_NZCV));\n }\n\n uint32_t OF = (Packed_NZCV >> IR::OpDispatchBuilder::IndexNZCV(X86State::RFLAG_OF_RAW_LOC)) & 1;\n uint32_t CF = (Packed_NZCV >> IR::OpDispatchBuilder::IndexNZCV(X86State::RFLAG_CF_RAW_LOC)) & 1;\n uint32_t ZF = (Packed_NZCV >> IR::OpDispatchBuilder::IndexNZCV(X86State::RFLAG_ZF_RAW_LOC)) & 1;\n uint32_t SF = (Packed_NZCV >> IR::OpDispatchBuilder::IndexNZCV(X86State::RFLAG_SF_RAW_LOC)) & 1;\n\n // CF is inverted in our representation, undo the invert here.\n CF ^= 1;\n\n // Pack in to EFLAGS\n EFLAGS |= OF << X86State::RFLAG_OF_RAW_LOC;\n EFLAGS |= CF << X86State::RFLAG_CF_RAW_LOC;\n EFLAGS |= ZF << X86State::RFLAG_ZF_RAW_LOC;\n EFLAGS |= SF << X86State::RFLAG_SF_RAW_LOC;\n\n // PF calculation is deferred, calculate it now.\n // Popcount the 8-bit flag and then extract the lower bit.\n uint32_t PFByte = Frame->State.pf_raw & 0xff;\n uint32_t PF = std::popcount(PFByte ^ 1) & 1;\n EFLAGS |= PF << X86State::RFLAG_PF_RAW_LOC;\n\n // AF calculation is deferred, calculate it now.\n // XOR with PF byte and extract bit 4.\n uint32_t AF = ((Frame->State.af_raw ^ PFByte) & (1 << 4)) ? 1 : 0;\n EFLAGS |= AF << X86State::RFLAG_AF_RAW_LOC;\n\n uint8_t TFByte = Frame->State.flags[X86State::RFLAG_TF_RAW_LOC];\n EFLAGS |= (TFByte & 1) << X86State::RFLAG_TF_RAW_LOC;\n\n // DF is pretransformed, undo the transform from 1/-1 back to 0/1\n uint8_t DFByte = Frame->State.flags[X86State::RFLAG_DF_RAW_LOC];\n if (DFByte & 0x80) {\n EFLAGS |= 1 << X86State::RFLAG_DF_RAW_LOC;\n }\n\n return EFLAGS;\n}\n\nvoid ContextImpl::ReconstructXMMRegisters(const FEXCore::Core::InternalThreadState* Thread, __uint128_t* XMM_Low, __uint128_t* YMM_High) {\n const size_t MaximumRegisters = Config.Is64BitMode ? FEXCore::Core::CPUState::NUM_XMMS : 8;\n\n if (YMM_High != nullptr && HostFeatures.SupportsAVX) {\n const bool SupportsConvergedRegisters = HostFeatures.SupportsSVE256;\n\n if (SupportsConvergedRegisters) {\n ///< Output wants to de-interleave\n for (size_t i = 0; i < MaximumRegisters; ++i) {\n memcpy(&XMM_Low[i], &Thread->CurrentFrame->State.xmm.avx.data[i][0], sizeof(__uint128_t));\n memcpy(&YMM_High[i], &Thread->CurrentFrame->State.xmm.avx.data[i][2], sizeof(__uint128_t));\n }\n } else {\n ///< Matches what FEX wants with non-converged registers\n for (size_t i = 0; i < MaximumRegisters; ++i) {\n memcpy(&XMM_Low[i], &Thread->CurrentFrame->State.xmm.sse.data[i][0], sizeof(__uint128_t));\n memcpy(&YMM_High[i], &Thread->CurrentFrame->State.avx_high[i][0], sizeof(__uint128_t));\n }\n }\n } else {\n // Only support SSE, no AVX here, even if requested.\n memcpy(XMM_Low, Thread->CurrentFrame->State.xmm.sse.data, MaximumRegisters * sizeof(__uint128_t));\n }\n}\n\nvoid ContextImpl::SetXMMRegistersFromState(FEXCore::Core::InternalThreadState* Thread, const __uint128_t* XMM_Low, const __uint128_t* YMM_High) {\n const size_t MaximumRegisters = Config.Is64BitMode ? FEXCore::Core::CPUState::NUM_XMMS : 8;\n if (YMM_High != nullptr && HostFeatures.SupportsAVX) {\n const bool SupportsConvergedRegisters = HostFeatures.SupportsSVE256;\n\n if (SupportsConvergedRegisters) {\n ///< Output wants to de-interleave\n for (size_t i = 0; i < MaximumRegisters; ++i) {\n memcpy(&Thread->CurrentFrame->State.xmm.avx.data[i][0], &XMM_Low[i], sizeof(__uint128_t));\n memcpy(&Thread->CurrentFrame->State.xmm.avx.data[i][2], &YMM_High[i], sizeof(__uint128_t));\n }\n } else {\n ///< Matches what FEX wants with non-converged registers\n for (size_t i = 0; i < MaximumRegisters; ++i) {\n memcpy(&Thread->CurrentFrame->State.xmm.sse.data[i][0], &XMM_Low[i], sizeof(__uint128_t));\n memcpy(&Thread->CurrentFrame->State.avx_high[i][0], &YMM_High[i], sizeof(__uint128_t));\n }\n }\n } else {\n // Only support SSE, no AVX here, even if requested.\n memcpy(Thread->CurrentFrame->State.xmm.sse.data, XMM_Low, MaximumRegisters * sizeof(__uint128_t));\n }\n}\n\nvoid ContextImpl::SetFlagsFromCompactedEFLAGS(FEXCore::Core::InternalThreadState* Thread, uint32_t EFLAGS) {\n const auto Frame = Thread->CurrentFrame;\n for (size_t i = 0; i < FEXCore::Core::CPUState::NUM_EFLAG_BITS; ++i) {\n switch (i) {\n case X86State::RFLAG_OF_RAW_LOC:\n case X86State::RFLAG_CF_RAW_LOC:\n case X86State::RFLAG_ZF_RAW_LOC:\n case X86State::RFLAG_SF_RAW_LOC:\n // Intentionally do nothing.\n break;\n case X86State::RFLAG_AF_RAW_LOC:\n // AF stored in bit 4 in our internal representation. It is also\n // XORed with byte 4 of the PF byte, but we write that as zero here so\n // we don't need any special handling for that.\n Frame->State.af_raw = (EFLAGS & (1U << i)) ? (1 << 4) : 0;\n break;\n case X86State::RFLAG_PF_RAW_LOC:\n // PF is inverted in our internal representation.\n Frame->State.pf_raw = (EFLAGS & (1U << i)) ? 0 : 1;\n break;\n case X86State::RFLAG_DF_RAW_LOC:\n // DF is encoded as 1/-1\n Frame->State.flags[i] = (EFLAGS & (1U << i)) ? 0xff : 1;\n break;\n default: Frame->State.flags[i] = (EFLAGS & (1U << i)) ? 1 : 0; break;\n }\n }\n\n // Calculate packed NZCV. Note CF is inverted.\n uint32_t Packed_NZCV {};\n Packed_NZCV |= (EFLAGS & (1U << X86State::RFLAG_OF_RAW_LOC)) ? 1U << IR::OpDispatchBuilder::IndexNZCV(X86State::RFLAG_OF_RAW_LOC) : 0;\n Packed_NZCV |= (EFLAGS & (1U << X86State::RFLAG_CF_RAW_LOC)) ? 0 : 1U << IR::OpDispatchBuilder::IndexNZCV(X86State::RFLAG_CF_RAW_LOC);\n Packed_NZCV |= (EFLAGS & (1U << X86State::RFLAG_ZF_RAW_LOC)) ? 1U << IR::OpDispatchBuilder::IndexNZCV(X86State::RFLAG_ZF_RAW_LOC) : 0;\n Packed_NZCV |= (EFLAGS & (1U << X86State::RFLAG_SF_RAW_LOC)) ? 1U << IR::OpDispatchBuilder::IndexNZCV(X86State::RFLAG_SF_RAW_LOC) : 0;\n memcpy(&Frame->State.flags[X86State::RFLAG_NZCV_LOC], &Packed_NZCV, sizeof(Packed_NZCV));\n\n // Reserved, Read-As-1, Write-as-1\n Frame->State.flags[X86State::RFLAG_RESERVED_LOC] = 1;\n // Interrupt Flag. Can't be written by CPL-3 userland.\n Frame->State.flags[X86State::RFLAG_IF_LOC] = 1;\n}\n\nbool ContextImpl::InitCore() {\n // Initialize the CPU core signal handlers & DispatcherConfig\n Dispatcher = FEXCore::CPU::Dispatcher::Create(this);\n\n // Set up the SignalDelegator config since core is initialized.\n SignalDelegation->SetConfig(Dispatcher->MakeSignalDelegatorConfig());\n\n#if defined(_WIN32) && !defined(ARCHITECTURE_arm64ec)\n // WOW64 always needs the interrupt fault check to be enabled.\n Config.NeedsPendingInterruptFaultCheck = true;\n#endif\n\n if (Config.GdbServer) {\n // If gdbserver is enabled then this needs to be enabled.\n Config.NeedsPendingInterruptFaultCheck = true;\n }\n\n return true;\n}\n\nvoid ContextImpl::HandleCallback(FEXCore::Core::InternalThreadState* Thread, uint64_t RIP) {\n static_cast(Thread->CTX)->Dispatcher->ExecuteJITCallback(Thread->CurrentFrame, RIP);\n}\n\nvoid ContextImpl::ExecuteThread(FEXCore::Core::InternalThreadState* Thread) {\n // Update the thread pointer for Thunk return to the latest.\n Thread->CurrentFrame->Pointers.ThunkCallbackRet = SignalDelegation->GetThunkCallbackRET();\n\n Dispatcher->ExecuteDispatch(Thread->CurrentFrame);\n\n // If it is the parent thread that died then just leave\n // TODO: This doesn't make sense when the parent thread doesn't outlive its children\n}\n\nvoid ContextImpl::InitializeCompiler(FEXCore::Core::InternalThreadState* Thread) {\n Thread->OpDispatcher = fextl::make_unique(this);\n Thread->OpDispatcher->SetMultiblock(Config.Multiblock);\n Thread->LookupCache = fextl::make_unique(this);\n Thread->FrontendDecoder = fextl::make_unique(Thread);\n Thread->PassManager = fextl::make_unique();\n\n Thread->CurrentFrame->State.L1Pointer = Thread->LookupCache->GetL1Pointer();\n Thread->CurrentFrame->State.L1Mask = Thread->LookupCache->GetScaledL1PointerMask();\n\n Thread->CurrentFrame->Pointers.L2Pointer = Thread->LookupCache->GetPagePointer();\n\n Dispatcher->InitThreadPointers(Thread);\n\n Thread->PassManager->AddDefaultPasses(this);\n Thread->PassManager->AddDefaultValidationPasses();\n\n Thread->PassManager->RegisterSyscallHandler(SyscallHandler);\n\n // Create CPU backend\n Thread->PassManager->InsertRegisterAllocationPass(this);\n Thread->CPUBackend = FEXCore::CPU::CreateArm64JITCore(this, Thread);\n\n Thread->PassManager->Finalize();\n}\n\nFEXCore::Core::InternalThreadState*\nContextImpl::CreateThread(uint64_t InitialRIP, uint64_t StackPointer, const FEXCore::Core::CPUState* NewThreadState) {\n FEXCore::Core::InternalThreadState* Thread = new FEXCore::Core::InternalThreadState {\n .CTX = this,\n };\n FEXCore::Allocator::VirtualName(\"FEXMem_ThreadState\", Thread, sizeof(*Thread));\n\n Thread->CurrentFrame->State.gregs[X86State::REG_RSP] = StackPointer;\n Thread->CurrentFrame->State.rip = InitialRIP;\n\n // Copy over the new thread state to the new object\n if (NewThreadState) {\n memcpy(&Thread->CurrentFrame->State, NewThreadState, sizeof(FEXCore::Core::CPUState));\n }\n\n // Set up the thread manager state\n Thread->CurrentFrame->Thread = Thread;\n\n InitializeCompiler(Thread);\n\n Thread->CurrentFrame->State.DeferredSignalRefCount.Store(0);\n\n if (Config.BlockJITNaming() || Config.GlobalJITNaming() || Config.LibraryJITNaming()) {\n // Allocate a JIT symbol buffer only if enabled.\n Thread->SymbolBuffer = JITSymbols::AllocateBuffer();\n }\n\n return Thread;\n}\n\nvoid ContextImpl::DestroyThread(FEXCore::Core::InternalThreadState* Thread) {\n FEXCore::Allocator::VirtualProtect(&Thread->InterruptFaultPage, sizeof(Thread->InterruptFaultPage),\n Allocator::ProtectOptions::Read | Allocator::ProtectOptions::Write);\n delete Thread;\n}\n\n#ifndef _WIN32\nvoid ContextImpl::UnlockAfterFork(FEXCore::Core::InternalThreadState* LiveThread, bool Child) {\n Allocator::UnlockAfterFork(LiveThread, Child);\n\n Profiler::PostForkAction(Child);\n if (Child) {\n if (CodeMapWriter) {\n CodeMapWriter->ResetAfterFork();\n }\n\n CodeInvalidationMutex.StealAndDropActiveLocks();\n if (Config.StrictInProcessSplitLocks) {\n StrictSplitLockMutex = 0;\n }\n } else {\n CodeInvalidationMutex.unlock();\n if (Config.StrictInProcessSplitLocks) {\n FEXCore::Utils::SpinWaitLock::unlock(&StrictSplitLockMutex);\n }\n return;\n }\n}\n\nvoid ContextImpl::LockBeforeFork(FEXCore::Core::InternalThreadState* Thread) {\n CodeInvalidationMutex.lock();\n Allocator::LockBeforeFork(Thread);\n if (Config.StrictInProcessSplitLocks) {\n FEXCore::Utils::SpinWaitLock::lock(&StrictSplitLockMutex);\n }\n}\n#endif\n\nvoid ContextImpl::OnCodeBufferAllocated(const fextl::shared_ptr& Buffer) {\n if (Config.GlobalJITNaming()) {\n Symbols.RegisterJITSpace(Buffer->Ptr, Buffer->AllocatedSize);\n }\n\n {\n std::scoped_lock lk {CodeBufferListLock};\n CodeBufferList.emplace_back(Buffer);\n }\n}\n\nvoid ContextImpl::ClearCodeCache(FEXCore::Core::InternalThreadState* Thread, bool NewCodeBuffer) {\n FEXCORE_PROFILE_INSTANT(\"ClearCodeCache\");\n\n if (NewCodeBuffer) {\n // Allocate new CodeBuffer + L3 LookupCache and clear L1+L2 caches\n Thread->CPUBackend->ClearCache();\n } else {\n // Clear L1+L2 cache of this thread, and clear L3 cache across any threads using it\n auto lk = Thread->LookupCache->AcquireWriteLock();\n Thread->LookupCache->ClearCache(lk);\n }\n Allocator::VirtualDontNeed(Thread->CallRetStackBase, FEXCore::Core::InternalThreadState::CALLRET_STACK_SIZE);\n}\n\nstatic void IRDumper(FEXCore::Core::InternalThreadState* Thread, IR::IREmitter* IREmitter, uint64_t GuestRIP) {\n FEXCore::File::File FD = FEXCore::File::File::GetStdERR();\n fextl::stringstream out;\n auto NewIR = IREmitter->ViewIR();\n FEXCore::IR::Dump(&out, &NewIR);\n fextl::fmt::print(FD, \"IR-ShouldDump-{} 0x{:x}:\\n{}\\n@@@@@\\n\", NewIR.PostRA() ? \"post\" : \"pre\", GuestRIP, out.str());\n};\n\nbool ContextImpl::CheckIfBlockIsCacheable(FEXCore::Core::InternalThreadState& Thread, uint64_t GuestRIP, uint64_t MaxInst) {\n return Thread.FrontendDecoder->CheckIfCacheable(Thread, reinterpret_cast(GuestRIP), GuestRIP, MaxInst);\n}\n\nContextImpl::GenerateIRResult\nContextImpl::GenerateIR(FEXCore::Core::InternalThreadState* Thread, uint64_t GuestRIP, bool ExtendedDebugInfo, uint64_t MaxInst) {\n FEXCORE_PROFILE_SCOPED(\"GenerateIR\");\n\n Thread->OpDispatcher->ResetWorkingList();\n\n uint64_t TotalInstructions {0};\n uint64_t TotalInstructionsLength {0};\n\n bool HasCustomIR {};\n\n if (HasCustomIRHandlers.load(std::memory_order_relaxed)) {\n std::shared_lock lk(CustomIRMutex);\n auto Handler = CustomIRHandlers.find(GuestRIP);\n if (Handler != CustomIRHandlers.end()) {\n TotalInstructions = 1;\n TotalInstructionsLength = 1;\n Handler->second.Handler(GuestRIP, Thread->OpDispatcher.get());\n HasCustomIR = true;\n }\n }\n\n if (!HasCustomIR) {\n const uint8_t* GuestCode {};\n GuestCode = reinterpret_cast(GuestRIP);\n\n bool HadDispatchError {false};\n bool HadInvalidInst {false};\n\n Thread->FrontendDecoder->DecodeInstructionsAtEntry(Thread, GuestCode, GuestRIP, MaxInst);\n\n auto BlockInfo = Thread->FrontendDecoder->GetDecodedBlockInfo();\n auto CodeBlocks = &BlockInfo->Blocks;\n\n Thread->OpDispatcher->BeginFunction(GuestRIP, CodeBlocks, BlockInfo->TotalInstructionCount, BlockInfo->Is64BitMode,\n AreMonoHacksActive() && MonoBackpatcherBlock.load(std::memory_order_relaxed) == GuestRIP);\n\n const auto GPRSize = Thread->OpDispatcher->GetGPROpSize();\n\n#ifdef ZYDIS_DISASSEMBLER\n const auto ZydisMachineMode = Config.Is64BitMode ? ZYDIS_MACHINE_MODE_LONG_64 : ZYDIS_MACHINE_MODE_LEGACY_32;\n if (FEXCore::Config::Get_X86DISASSEMBLE()) {\n const uint64_t DecodedMin = Thread->FrontendDecoder->DecodedMinAddress;\n const uint64_t DecodedMax = Thread->FrontendDecoder->DecodedMaxAddress;\n LogMan::Msg::IFmt(\"Guest x86 Begin (RIP={:#x}, {:#x}-{:#x})\", GuestRIP, DecodedMin, DecodedMax);\n }\n#endif\n\n for (size_t j = 0; j < CodeBlocks->size(); ++j) {\n const FEXCore::Frontend::Decoder::DecodedBlocks& Block = CodeBlocks->at(j);\n\n#ifdef ZYDIS_DISASSEMBLER\n if (FEXCore::Config::Get_X86DISASSEMBLE() && CodeBlocks->size() > 1) {\n LogMan::Msg::IFmt(\" Block {} Entry={:#x} NumInsts={}\", j, Block.Entry, Block.NumInstructions);\n }\n#endif\n\n bool BlockInForceTSOValidRange = false;\n auto InstForceTSOIt = ForceTSOInstructions.end();\n if (ForceTSOValidRanges.Contains({Block.Entry, Block.Entry + Block.Size})) {\n if (auto It = ForceTSOInstructions.lower_bound(Block.Entry); *It < Block.Entry + Block.Size) {\n InstForceTSOIt = It;\n BlockInForceTSOValidRange = true;\n }\n }\n\n // Set the block entry point\n Thread->OpDispatcher->SetNewBlockIfChanged(Block.Entry);\n\n uint64_t BlockInstructionsLength {};\n\n // Reset any block-specific state\n Thread->OpDispatcher->StartNewBlock();\n\n uint64_t InstsInBlock = Block.NumInstructions;\n\n if (InstsInBlock == 0) {\n // Special case for an empty instruction block.\n Thread->OpDispatcher->ExitFunction(Thread->OpDispatcher->_InlineEntrypointOffset(GPRSize, Block.Entry - GuestRIP));\n }\n\n for (size_t i = 0; i < InstsInBlock; ++i) {\n uint64_t InstAddress = Block.Entry + BlockInstructionsLength;\n const FEXCore::X86Tables::X86InstInfo* TableInfo {nullptr};\n const FEXCore::X86Tables::DecodedInst* DecodedInfo {nullptr};\n\n TableInfo = Block.DecodedInstructions[i].TableInfo;\n DecodedInfo = &Block.DecodedInstructions[i];\n\n#ifdef ZYDIS_DISASSEMBLER\n if (FEXCore::Config::Get_X86DISASSEMBLE()) {\n const uint8_t* InstBytes = reinterpret_cast(InstAddress);\n ZydisDisassembledInstruction ZydisInst;\n if (ZYAN_SUCCESS(ZydisDisassembleIntel(ZydisMachineMode, InstAddress, InstBytes, DecodedInfo->InstSize, &ZydisInst))) {\n LogMan::Msg::IFmt(\" {:#x}: {}\", InstAddress, ZydisInst.text);\n } else {\n LogMan::Msg::IFmt(\" {:#x}: (decode failed, {} bytes)\", InstAddress, DecodedInfo->InstSize);\n }\n }\n#endif\n\n bool IsLocked = DecodedInfo->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_LOCK;\n\n // Do a partial register cache flush before every instruction. This\n // prevents cross-instruction static register caching, while allowing\n // context load/stores to be optimized within a block. Theoretically,\n // this flush is not required for correctness, all mandatory flushes are\n // included in instruction-specific handlers. Instead, this is a blunt\n // heuristic to make the register cache less aggressive, as the current\n // RA generates bad code in common cases with tied registers otherwise.\n //\n // However, it makes our exception handling behaviour more predictable.\n // It is potentially correctness bearing in that sense, but that is a\n // side effect here and (if that behaviour is required) we should handle\n // that more explicitly later.\n Thread->OpDispatcher->FlushRegisterCache(true);\n\n if (ExtendedDebugInfo || Thread->OpDispatcher->CanHaveSideEffects(TableInfo, DecodedInfo)) {\n Thread->OpDispatcher->_GuestOpcode(InstAddress - GuestRIP);\n }\n\n if (Config.SMCChecks == FEXCore::Config::CONFIG_SMC_FULL || Block.ForceFullSMCDetection) {\n auto ExistingCodePtr = reinterpret_cast(Block.Entry + BlockInstructionsLength);\n auto InstAddressReg = Thread->OpDispatcher->_EntrypointOffset(GPRSize, InstAddress - GuestRIP);\n std::array CodeOriginal;\n memcpy(CodeOriginal.data(), ExistingCodePtr, DecodedInfo->InstSize);\n auto CodeChanged = Thread->OpDispatcher->_ValidateCode(CodeOriginal, InstAddressReg, DecodedInfo->InstSize);\n\n auto InvalidateCodeCond = Thread->OpDispatcher->CondJump(CodeChanged);\n\n auto CurrentBlock = Thread->OpDispatcher->GetCurrentBlock();\n auto CodeWasChangedBlock = Thread->OpDispatcher->CreateNewCodeBlockAtEnd();\n Thread->OpDispatcher->SetTrueJumpTarget(InvalidateCodeCond, CodeWasChangedBlock);\n\n Thread->OpDispatcher->SetCurrentCodeBlock(CodeWasChangedBlock);\n Thread->OpDispatcher->_ThreadRemoveCodeEntry();\n Thread->OpDispatcher->ExitFunction(Thread->OpDispatcher->_InlineEntrypointOffset(GPRSize, InstAddress - GuestRIP));\n\n auto NextOpBlock = Thread->OpDispatcher->CreateNewCodeBlockAfter(CurrentBlock);\n\n Thread->OpDispatcher->SetFalseJumpTarget(InvalidateCodeCond, NextOpBlock);\n Thread->OpDispatcher->SetCurrentCodeBlock(NextOpBlock);\n }\n\n if (TableInfo && TableInfo->OpcodeDispatcher.OpDispatch) {\n auto Fn = TableInfo->OpcodeDispatcher.OpDispatch;\n Thread->OpDispatcher->ResetHandledLock();\n Thread->OpDispatcher->ResetDecodeFailure();\n IR::ForceTSOMode ForceTSO = IR::ForceTSOMode::NoOverride;\n if (BlockInForceTSOValidRange) {\n if (InstForceTSOIt != ForceTSOInstructions.end() && *InstForceTSOIt == InstAddress) {\n ForceTSO = IR::ForceTSOMode::ForceEnabled;\n } else {\n ForceTSO = IR::ForceTSOMode::ForceDisabled;\n }\n } else if (DecodedInfo->Flags & X86Tables::DecodeFlags::FLAG_FORCE_TSO) {\n ForceTSO = IR::ForceTSOMode::ForceEnabled;\n }\n\n Thread->OpDispatcher->SetForceTSO(ForceTSO);\n std::invoke(Fn, Thread->OpDispatcher, DecodedInfo);\n if (Thread->OpDispatcher->HadDecodeFailure()) {\n HadDispatchError = true;\n } else {\n if (Thread->OpDispatcher->HasHandledLock() != IsLocked) {\n HadDispatchError = true;\n LogMan::Msg::EFmt(\"Missing LOCK HANDLER at 0x{:x}{{'{}'}}\", InstAddress, TableInfo->Name ?: \"UND\");\n }\n BlockInstructionsLength += DecodedInfo->InstSize;\n TotalInstructionsLength += DecodedInfo->InstSize;\n ++TotalInstructions;\n\n // Walk InstForceTSOIt forward past the handled instruction\n InstForceTSOIt =\n std::find_if(InstForceTSOIt, ForceTSOInstructions.end(), [&](auto Val) { return Val >= Block.Entry + BlockInstructionsLength; });\n }\n } else {\n // Invalid instruction\n if (!BlockInstructionsLength) {\n // SMC can modify block contents and patch invalid instructions to valid ones inline.\n // End blocks upon encountering them and only emit an invalid opcode exception if there are no prior instructions in the block (that could have modified it to be valid).\n\n if (TableInfo) {\n LogMan::Msg::EFmt(\"Invalid or Unknown instruction: {} 0x{:x}\", TableInfo->Name ?: \"UND\", Block.Entry - GuestRIP);\n }\n\n if (Block.BlockStatus == Frontend::Decoder::DecodedBlockStatus::INVALID_INST ||\n Block.BlockStatus == Frontend::Decoder::DecodedBlockStatus::BAD_RELOCATION) {\n Thread->OpDispatcher->InvalidOp(DecodedInfo);\n } else {\n Thread->OpDispatcher->NoExecOp(DecodedInfo);\n }\n }\n\n HadInvalidInst = true;\n }\n\n const bool NeedsBlockEnd = (HadDispatchError && TotalInstructions > 0) ||\n (Thread->OpDispatcher->NeedsBlockEnder() && i + 1 == InstsInBlock) || HadInvalidInst;\n\n // If we had a dispatch error then leave early\n if (HadDispatchError && TotalInstructions == 0) {\n // Couldn't handle any instruction in op dispatcher\n Thread->OpDispatcher->DelayedDisownBuffer();\n return {std::nullopt, 0, 0, 0, 0};\n }\n\n if (NeedsBlockEnd) {\n // We had some instructions. Early exit\n Thread->OpDispatcher->ExitFunction(\n Thread->OpDispatcher->_InlineEntrypointOffset(GPRSize, Block.Entry + BlockInstructionsLength - GuestRIP));\n break;\n }\n\n\n if (Thread->OpDispatcher->FinishOp(DecodedInfo->PC + DecodedInfo->InstSize, i + 1 == InstsInBlock)) {\n break;\n }\n }\n }\n\n#ifdef ZYDIS_DISASSEMBLER\n if (FEXCore::Config::Get_X86DISASSEMBLE()) {\n LogMan::Msg::IFmt(\"Guest x86 End\");\n }\n#endif\n\n Thread->OpDispatcher->Finalize();\n\n Thread->FrontendDecoder->DelayedDisownBuffer();\n }\n\n IR::IREmitter* IREmitter = Thread->OpDispatcher.get();\n\n auto ShouldDump = Thread->OpDispatcher->ShouldDumpIR();\n // Debug\n if (ShouldDump) {\n IRDumper(Thread, IREmitter, GuestRIP);\n }\n\n // Run the passmanager over the IR from the dispatcher\n Thread->PassManager->Run(IREmitter);\n\n // Debug\n if (ShouldDump) {\n IRDumper(Thread, IREmitter, GuestRIP);\n }\n\n return {\n .IRView = IREmitter->ViewIR(),\n .TotalInstructions = TotalInstructions,\n .TotalInstructionsLength = TotalInstructionsLength,\n .StartAddr = Thread->FrontendDecoder->DecodedMinAddress,\n .Length = Thread->FrontendDecoder->DecodedMaxAddress - Thread->FrontendDecoder->DecodedMinAddress,\n .NeedsAddGuestCodeRanges = !HasCustomIR,\n };\n}\n\nContextImpl::CompileCodeResult ContextImpl::CompileCode(FEXCore::Core::InternalThreadState* Thread, uint64_t GuestRIP, uint64_t MaxInst) {\n if (SourcecodeResolver && Config.GDBSymbols()) {\n auto MappedSection = SyscallHandler->LookupExecutableFileSection(Thread, GuestRIP);\n if (MappedSection) {\n MappedSection->FileInfo.SourcecodeMap =\n SourcecodeResolver->GenerateMap(MappedSection->FileInfo.Filename, CodeMap::GetBaseFilename(MappedSection->FileInfo, false));\n }\n }\n\n // Generate IR + Meta Info\n auto [IRView, TotalInstructions, TotalInstructionsLength, StartAddr, Length, NeedsAddGuestCodeRanges] =\n GenerateIR(Thread, GuestRIP, Config.GDBSymbols(), MaxInst);\n if (!IRView) {\n // OpDispatcher IR already released in this case.\n return {{}, nullptr, 0, 0, false};\n }\n\n // Attempt to get the CPU backend to compile this code\n // Re-check if another thread raced us in compiling this block.\n // We could lock CodeBufferWriteMutex earlier to prevent this from happening,\n // but this would increase lock contention. Redundant frontend runs aren't\n // as expensive and are easily reverted.\n if (MaxInst != 1) {\n if (auto Block = Thread->LookupCache->FindBlock(Thread, GuestRIP)) {\n // Raced to compile, release the OpDispatcher IR.\n Thread->OpDispatcher->DelayedDisownBuffer();\n return {.CompiledCode = {.BlockBegin = reinterpret_cast(Block), .EntryPoints = {{GuestRIP, reinterpret_cast(Block)}}},\n .DebugData = nullptr,\n .StartAddr = 0,\n .Length = 0,\n .NeedsAddGuestCodeRanges = false};\n }\n }\n\n auto DebugData = fextl::make_unique();\n\n // If the trap flag is set we generate single instruction blocks that each check to generate a single step exception.\n bool TFSet = Thread->CurrentFrame->State.flags[X86State::RFLAG_TF_RAW_LOC];\n\n auto CompiledCode = Thread->CPUBackend->CompileCode(GuestRIP, Length, TotalInstructions == 1, &*IRView, DebugData.get(), TFSet);\n\n // Release the IR\n Thread->OpDispatcher->DelayedDisownBuffer();\n\n return {\n .CompiledCode = std::move(CompiledCode),\n .DebugData = std::move(DebugData),\n .StartAddr = StartAddr,\n .Length = Length,\n .NeedsAddGuestCodeRanges = NeedsAddGuestCodeRanges,\n };\n}\n\nuintptr_t ContextImpl::CompileBlock(FEXCore::Core::CpuStateFrame* Frame, uint64_t GuestRIP, uint64_t MaxInst) {\n auto Thread = Frame->Thread;\n FEXCORE_PROFILE_SCOPED(\"CompileBlock\");\n FEXCORE_PROFILE_ACCUMULATION(Thread, AccumulatedJITTime);\n\n static_cast(Thread->CTX)->SyscallHandler->PreCompile();\n\n // Invalidate might take a unique lock on this, to guarantee that during invalidation no code gets compiled\n auto lk = GuardSignalDeferringSection(CodeInvalidationMutex, Thread);\n\n // Is the code in the cache?\n // The backends only check L1 and L2, not L3\n if (auto HostCode = Thread->LookupCache->FindBlock(Thread, GuestRIP)) {\n return HostCode;\n }\n\n // Accumulate a JIT count now, as even if another thread raced us, it should count as a compile.\n FEXCORE_PROFILE_INSTANT_INCREMENT(Thread, AccumulatedJITCount, 1);\n\n auto [CompiledCode, DebugData, StartAddr, Length, NeedsAddGuestCodeRanges] = CompileCode(Thread, GuestRIP, MaxInst);\n auto CodePtr = CompiledCode.EntryPoints[GuestRIP];\n if (CodePtr == nullptr) {\n return 0;\n } else if (!DebugData) {\n // DebugData wasn't populated, indicating another thread raced us for compiling this block\n return reinterpret_cast(CodePtr);\n }\n\n // The core managed to compile the code.\n if (Config.BlockJITNaming()) {\n auto FragmentBasePtr = CompiledCode.BlockBegin;\n\n auto GuestRIPLookup = SyscallHandler->LookupExecutableFileSection(Thread, GuestRIP);\n\n if (DebugData->Subblocks.size()) {\n for (auto& Subblock : DebugData->Subblocks) {\n auto BlockBasePtr = FragmentBasePtr + Subblock.HostCodeOffset;\n if (GuestRIPLookup) {\n Symbols.Register(Thread->SymbolBuffer.get(), BlockBasePtr, CompiledCode.Size, GuestRIPLookup->FileInfo.Filename,\n GuestRIP - GuestRIPLookup->FileStartVA);\n } else {\n Symbols.Register(Thread->SymbolBuffer.get(), BlockBasePtr, GuestRIP, Subblock.HostCodeSize);\n }\n }\n } else {\n if (GuestRIPLookup) {\n Symbols.Register(Thread->SymbolBuffer.get(), FragmentBasePtr, CompiledCode.Size, GuestRIPLookup->FileInfo.Filename,\n GuestRIP - GuestRIPLookup->FileStartVA);\n } else {\n Symbols.Register(Thread->SymbolBuffer.get(), FragmentBasePtr, GuestRIP, CompiledCode.Size);\n }\n }\n }\n\n if (Config.LibraryJITNaming() || Config.GDBSymbols()) {\n auto MappedSection = SyscallHandler->LookupExecutableFileSection(Thread, GuestRIP);\n if (MappedSection) {\n if (Config.LibraryJITNaming()) {\n Symbols.RegisterNamedRegion(Thread->SymbolBuffer.get(), CodePtr, DebugData->HostCodeSize, MappedSection->FileInfo.Filename);\n }\n\n if (Config.GDBSymbols()) {\n GDBJITRegister(MappedSection->FileInfo, MappedSection->FileStartVA, GuestRIP, (uintptr_t)CodePtr, *DebugData);\n }\n }\n }\n\n // Clear any relocations that might have been generated\n if (!CodeCache.IsGeneratingCache) {\n Thread->CPUBackend->ClearRelocations();\n }\n\n fextl::vector CodePages;\n\n if (NeedsAddGuestCodeRanges) {\n // Track in the guest to host map all entrypoints for all pages the compiled block touches, if any page didn't previously\n // contain code, inform the frontend so it can setup SMC detection.\n auto BlockInfo = Thread->FrontendDecoder->GetDecodedBlockInfo();\n CodePages.reserve(BlockInfo->CodePages.size());\n CodePages.insert(CodePages.end(), BlockInfo->CodePages.begin(), BlockInfo->CodePages.end());\n for (auto CodePage : BlockInfo->CodePages) {\n if (Thread->LookupCache->AddBlockExecutableRange(Thread, BlockInfo->EntryPoints, CodePage, FEXCore::Utils::FEX_PAGE_SIZE)) {\n SyscallHandler->MarkGuestExecutableRange(Thread, CodePage, FEXCore::Utils::FEX_PAGE_SIZE);\n }\n }\n }\n\n // Insert to lookup cache\n\n for (auto [GuestAddr, HostAddr] : CompiledCode.EntryPoints) {\n Thread->LookupCache->AddBlockMapping(Thread, GuestAddr, CodePages, HostAddr);\n }\n\n if (CodeMapWriter) {\n auto Region = SyscallHandler->LookupExecutableFileSection(Thread, GuestRIP);\n if (Region && Region->FileStartVA != 0) {\n CodeMapWriter->AppendBlock(*Region, GuestRIP);\n }\n }\n\n return (uintptr_t)CodePtr;\n}\n\nuintptr_t ContextImpl::CompileSingleStep(FEXCore::Core::CpuStateFrame* Frame, uint64_t GuestRIP) {\n FEXCORE_PROFILE_SCOPED(\"CompileSingleStep\");\n auto Thread = Frame->Thread;\n\n static_cast(Thread->CTX)->SyscallHandler->PreCompile();\n\n // Invalidate might take a unique lock on this, to guarantee that during invalidation no code gets compiled\n auto lk = GuardSignalDeferringSection(CodeInvalidationMutex, Thread);\n\n auto [CompiledCode, DebugData, StartAddr, Length, _] = CompileCode(Thread, GuestRIP, 1);\n auto CodePtr = CompiledCode.EntryPoints[GuestRIP];\n if (CodePtr == nullptr) {\n return 0;\n }\n\n // Clear any relocations that might have been generated\n Thread->CPUBackend->ClearRelocations();\n\n return (uintptr_t)CodePtr;\n}\n\nvoid ContextImpl::InvalidateCodeBuffersCodeRange(uint64_t Start, uint64_t Length) {\n FEXCORE_PROFILE_SCOPED(\"InvalidateCodeBuffersCodeRange\");\n\n LOGMAN_THROW_A_FMT(CodeInvalidationMutex.try_lock() == false, \"CodeInvalidationMutex needs to be unique_locked here\");\n std::scoped_lock lk {CodeBufferListLock};\n auto it = CodeBufferList.begin();\n while (it != CodeBufferList.end()) {\n if (auto Strong = it->lock()) {\n Strong->LookupCache->InvalidateRange(Start, Length);\n it++;\n } else {\n it = CodeBufferList.erase(it);\n }\n }\n}\n\nvoid ContextImpl::InvalidateThreadCachedCodeRange(FEXCore::Core::InternalThreadState* Thread, uint64_t Start, uint64_t Length) {\n LOGMAN_THROW_A_FMT(CodeInvalidationMutex.try_lock() == false, \"CodeInvalidationMutex needs to be unique_locked here\");\n\n // Ensures now-modified mappings aren't cached as being in their previous non-executable state.\n // Accessing FrontendDecoder is safe as the thread's code invalidation mutex must be locked here.\n Thread->FrontendDecoder->ResetExecutableRangeCache();\n\n if (Thread->LookupCache->InvalidateCacheRange(Start, Length)) {\n FEXCORE_PROFILE_SCOPED(\"InvalidateCallRet\");\n\n // This may cause access violations in the thread on Windows as zeroing is not atomic, this is handled by the frontend\n Allocator::VirtualDontNeed(Thread->CallRetStackBase, FEXCore::Core::InternalThreadState::CALLRET_STACK_SIZE);\n }\n}\n\nvoid ContextImpl::ThreadRemoveCodeEntryFromJit(FEXCore::Core::CpuStateFrame* Frame, uint64_t GuestRIP) {\n static_cast(Frame->Thread->CTX)->SyscallHandler->InvalidateGuestCodeRange(Frame->Thread, GuestRIP, 1);\n}\n\nstd::optional\nContextImpl::AddCustomIREntrypoint(uintptr_t Entrypoint, CustomIREntrypointHandler Handler, void* Creator, void* Data) {\n LOGMAN_THROW_A_FMT(Config.Is64BitMode || !(Entrypoint >> 32), \"64-bit Entrypoint in 32-bit mode {:x}\", Entrypoint);\n\n std::unique_lock lk(CustomIRMutex);\n\n auto InsertedIterator = CustomIRHandlers.emplace(Entrypoint, CustomIRHandlerEntry {Handler, Creator, Data});\n HasCustomIRHandlers = true;\n\n if (!InsertedIterator.second) {\n const auto& [fn, Creator, Data] = InsertedIterator.first->second;\n return CustomIRResult(Creator, Data);\n }\n\n return std::nullopt;\n}\n\nvoid ContextImpl::AddThunkTrampolineIRHandler(uintptr_t Entrypoint, uintptr_t GuestThunkEntrypoint) {\n LOGMAN_THROW_A_FMT(Entrypoint, \"Tried to link null pointer address to guest function\");\n LOGMAN_THROW_A_FMT(GuestThunkEntrypoint, \"Tried to link address to null pointer guest function\");\n if (!Config.Is64BitMode) {\n LOGMAN_THROW_A_FMT((Entrypoint >> 32) == 0, \"Tried to link 64-bit address in 32-bit mode\");\n LOGMAN_THROW_A_FMT((GuestThunkEntrypoint >> 32) == 0, \"Tried to link 64-bit address in 32-bit mode\");\n }\n\n LogMan::Msg::DFmt(\"Thunks: Adding guest trampoline from address {:#x} to guest function {:#x}\", Entrypoint, GuestThunkEntrypoint);\n\n auto Result = AddCustomIREntrypoint(\n Entrypoint,\n [this, GuestThunkEntrypoint](uintptr_t Entrypoint, FEXCore::IR::IREmitter* emit) {\n auto IRHeader = emit->_IRHeader(emit->Invalid(), Entrypoint, 0, 0, 0, 0);\n auto Block = emit->CreateCodeNode(true, 0);\n IRHeader.first->Blocks = emit->WrapNode(Block);\n emit->SetCurrentCodeBlock(Block);\n\n const auto GPRSize = this->Config.Is64BitMode ? IR::OpSize::i64Bit : IR::OpSize::i32Bit;\n\n // Thunk entry-points don't get cached, don't need to be padded.\n if (GPRSize == IR::OpSize::i64Bit) {\n IR::Ref R = emit->_StoreRegister(emit->Constant(Entrypoint), GPRSize);\n R->Reg = IR::PhysicalRegister(IR::RegClass::GPRFixed, X86State::REG_R11).Raw;\n } else {\n emit->_StoreContextFPR(GPRSize, emit->_VCastFromGPR(IR::OpSize::i64Bit, IR::OpSize::i64Bit, emit->Constant(Entrypoint)),\n offsetof(Core::CPUState, mm[0][0]));\n }\n emit->_ExitFunction(IR::OpSize::i64Bit, emit->Constant(GuestThunkEntrypoint), IR::BranchHint::None, emit->Invalid(), emit->Invalid());\n },\n ThunkHandler, (void*)GuestThunkEntrypoint);\n\n if (Result.has_value()) {\n if (Result->Creator != ThunkHandler) {\n ERROR_AND_DIE_FMT(\"Input address for AddThunkTrampoline is already linked by another module\");\n }\n if (Result->Data != (void*)GuestThunkEntrypoint) {\n // NOTE: This may happen in Vulkan thunks if the Vulkan driver resolves two different symbols\n // to the same function (e.g. vkGetPhysicalDeviceFeatures2/vkGetPhysicalDeviceFeatures2KHR)\n LogMan::Msg::EFmt(\"Input address for AddThunkTrampoline is already linked elsewhere\");\n }\n }\n}\n\nvoid ContextImpl::AddForceTSOInformation(const IntervalList& ValidRanges, fextl::set&& Instructions) {\n LogMan::Throw::AFmt(CodeInvalidationMutex.try_lock() == false, \"CodeInvalidationMutex needs to be unique_locked here\");\n ForceTSOValidRanges.Insert(ValidRanges);\n ForceTSOInstructions.merge(std::move(Instructions));\n}\n\nvoid ContextImpl::RemoveForceTSOInformation(uint64_t Address, uint64_t Size) {\n LogMan::Throw::AFmt(CodeInvalidationMutex.try_lock() == false, \"CodeInvalidationMutex needs to be unique_locked here\");\n\n ForceTSOValidRanges.Remove({Address, Address + Size});\n ForceTSOInstructions.erase(ForceTSOInstructions.lower_bound(Address), ForceTSOInstructions.upper_bound(Address + Size));\n}\n\nvoid ContextImpl::MarkMonoBackpatcherBlock(uint64_t BlockEntry) {\n MonoBackpatcherBlock.store(BlockEntry, std::memory_order_relaxed);\n}\n\nvoid ContextImpl::RemoveCustomIREntrypoint(FEXCore::Core::InternalThreadState* Thread, uintptr_t Entrypoint) {\n LOGMAN_THROW_A_FMT(Config.Is64BitMode || !(Entrypoint >> 32), \"64-bit Entrypoint in 32-bit mode {:x}\", Entrypoint);\n\n std::scoped_lock lk(CustomIRMutex);\n\n CustomIRHandlers.erase(Entrypoint);\n HasCustomIRHandlers = !CustomIRHandlers.empty();\n SyscallHandler->InvalidateGuestCodeRange(Thread, Entrypoint, 1);\n}\n\nvoid ContextImpl::MonoBackpatcherWrite(FEXCore::Core::CpuStateFrame* Frame, uint8_t Size, uint64_t Address, uint64_t Value) {\n auto Thread = Frame->Thread;\n auto CTX = static_cast(Thread->CTX);\n {\n auto lk = GuardSignalDeferringSection(CTX->CodeInvalidationMutex, Thread);\n\n if (Size == 8) {\n *reinterpret_cast(Address) = Value;\n } else if (Size == 4) {\n *reinterpret_cast(Address) = Value;\n } else {\n ERROR_AND_DIE_FMT(\"Unexpected write size for backpatcher: {}\", Size);\n }\n }\n\n CTX->SyscallHandler->InvalidateGuestCodeRange(Thread, Address, Size);\n}\n\nvoid ContextImpl::ConfigureAOTGen(FEXCore::Core::InternalThreadState* Thread, fextl::set* ExternalBranches, uint64_t SectionMaxAddress) {\n Thread->FrontendDecoder->SetExternalBranches(ExternalBranches);\n}\n} // namespace FEXCore::Context\n" }, { "path": "FEXCore/Source/Interface/Core/Dispatcher/Dispatcher.cpp", "content": "// SPDX-License-Identifier: MIT\n\n#include \"Common/VectorRegType.h\"\n#include \"Interface/Context/Context.h\"\n#include \"Interface/Core/CPUBackend.h\"\n#include \"Interface/Core/Dispatcher/Dispatcher.h\"\n#include \"Interface/Core/LookupCache.h\"\n#include \"Utils/MemberFunctionToPointer.h\"\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n#include \n\n#ifdef VIXL_SIMULATOR\n#include \n#endif\n\n#include \n#include \n#include \n\nnamespace FEXCore::CPU {\n\nstatic void SleepThread(FEXCore::Context::ContextImpl* CTX, FEXCore::Core::CpuStateFrame* Frame) {\n CTX->SyscallHandler->SleepThread(CTX, Frame);\n}\n\nconstexpr size_t MAX_DISPATCHER_CODE_SIZE = FEXCore::Utils::FEX_PAGE_SIZE * 4;\n\nDispatcher::Dispatcher(FEXCore::Context::ContextImpl* ctx)\n : Arm64Emitter(ctx, FEXCore::Allocator::VirtualAlloc(MAX_DISPATCHER_CODE_SIZE, true), MAX_DISPATCHER_CODE_SIZE)\n , CTX {ctx} {\n EmitDispatcher();\n\n FEXCore::Allocator::VirtualName(\"FEXMem_Misc\", reinterpret_cast(GetBufferBase()), MAX_DISPATCHER_CODE_SIZE);\n}\n\nDispatcher::~Dispatcher() {\n auto BufferSize = GetBufferSize();\n if (BufferSize) {\n FEXCore::Allocator::VirtualFree(GetBufferBase(), BufferSize);\n }\n}\n\nvoid Dispatcher::EmitDispatcher() {\n // Don't modify TMP3 since it contains our RIP once the block doesn't exist\n auto RipReg = TMP3;\n#ifdef VIXL_DISASSEMBLER\n const auto DisasmBegin = GetCursorAddress();\n#endif\n\n DispatchPtr = GetCursorAddress();\n\n // while (true) {\n // Ptr = FindBlock(RIP)\n // if (!Ptr)\n // Ptr = CTX->CompileBlock(RIP);\n //\n // Ptr();\n // }\n\n ARMEmitter::ForwardLabel l_CTX;\n ARMEmitter::ForwardLabel l_Sleep;\n ARMEmitter::ForwardLabel l_CompileBlock;\n ARMEmitter::ForwardLabel l_CompileSingleStep;\n\n // Push all the register we need to save\n PushCalleeSavedRegisters();\n\n // Push our memory base to the correct register\n // Move our thread pointer to the correct register\n // This is passed in to parameter 0 (x0)\n mov(STATE, ARMEmitter::XReg::x0);\n\n // Save this stack pointer so we can cleanly shutdown the emulation with a long jump\n // regardless of where we were in the stack\n add(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, ARMEmitter::Reg::rsp, 0);\n str(ARMEmitter::XReg::x0, STATE_PTR(CpuStateFrame, ReturningStackLocation));\n\n ARMEmitter::ForwardLabel CompileSingleStep;\n AbsoluteLoopTopAddressFillSRA = GetCursorAddress();\n\n FillStaticRegs();\n ldr(RipReg, STATE_PTR(CpuStateFrame, State.rip));\n (void)cbnz(ARMEmitter::Size::i32Bit, ENTRY_FILL_SRA_SINGLE_INST_REG, &CompileSingleStep);\n\n ARMEmitter::BiDirectionalLabel LoopTop {};\n\n#ifdef ARCHITECTURE_arm64ec\n (void)b(&LoopTop);\n\n AbsoluteLoopTopAddressEnterECFillSRA = GetCursorAddress();\n ldr(STATE, EC_ENTRY_CPUAREA_REG, CPU_AREA_EMULATOR_DATA_OFFSET);\n FillStaticRegs();\n\n ldr(RipReg, STATE_PTR(CpuStateFrame, State.rip));\n // Force a single instruction block if ENTRY_FILL_SRA_SINGLE_INST_REG is nonzero entering the JIT, used for inline SMC handling.\n (void)cbnz(ARMEmitter::Size::i32Bit, ENTRY_FILL_SRA_SINGLE_INST_REG, &CompileSingleStep);\n\n // Enter JIT\n (void)b(&LoopTop);\n\n AbsoluteLoopTopAddressEnterEC = GetCursorAddress();\n // Load ThreadState and write the target PC there\n ldr(STATE, EC_ENTRY_CPUAREA_REG, CPU_AREA_EMULATOR_DATA_OFFSET);\n str(EC_CALL_CHECKER_PC_REG, STATE_PTR(CpuStateFrame, State.rip));\n\n // Swap stacks to the emulator stack\n ldr(TMP1, EC_ENTRY_CPUAREA_REG, CPU_AREA_EMULATOR_STACK_BASE_OFFSET);\n add(ARMEmitter::Size::i64Bit, StaticRegisters[X86State::REG_RSP], ARMEmitter::Reg::rsp, 0);\n add(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::rsp, TMP1, 0);\n\n ldr(REG_CALLRET_SP, STATE_PTR(CpuStateFrame, State.callret_sp));\n\n FillSpecialRegs(TMP1, TMP2, false, true);\n\n // As ARM64EC uses this as an entrypoint for both guest calls and host returns, opportunistically try to return\n // using the call-ret stack to avoid unbalancing it.\n ldp(TMP1, TMP2, REG_CALLRET_SP);\n // EC_CALL_CHECKER_PC_REG is REG_PF which isn't touched by any of the above\n sub(ARMEmitter::Size::i64Bit, TMP1, EC_CALL_CHECKER_PC_REG, TMP1);\n (void)cbnz(ARMEmitter::Size::i64Bit, TMP1, &LoopTop);\n\n // If the entry at the TOS is for the target address, pop it and return to the JIT code\n add(ARMEmitter::Size::i64Bit, REG_CALLRET_SP, REG_CALLRET_SP, 0x10);\n ret(TMP2);\n\n // Enter JIT\n#endif\n\n // We want to ensure that we are 16 byte aligned at the top of this loop\n Align16B();\n\n (void)Bind(&LoopTop);\n AbsoluteLoopTopAddress = GetCursorAddress();\n\n // Load in our RIP\n ldr(RipReg, STATE_PTR(CpuStateFrame, State.rip));\n\n#ifdef ARCHITECTURE_arm64ec\n // Clobbers TMP1/2\n // Check the EC code bitmap incase we need to exit the JIT to call into native code.\n ARMEmitter::ForwardLabel l_NotECCode;\n ldr(TMP1, ARMEmitter::XReg::x18, TEB_PEB_OFFSET);\n ldr(TMP1, TMP1, PEB_EC_CODE_BITMAP_OFFSET);\n\n lsr(ARMEmitter::Size::i64Bit, TMP2, RipReg, 15);\n and_(ARMEmitter::Size::i64Bit, TMP2, TMP2, 0x1fffffffffff8);\n ldr(TMP1, TMP1, TMP2, ARMEmitter::ExtendedType::LSL_64, 0);\n lsr(ARMEmitter::Size::i64Bit, TMP2, RipReg, 12);\n lsrv(ARMEmitter::Size::i64Bit, TMP1, TMP1, TMP2);\n (void)tbz(TMP1, 0, &l_NotECCode);\n\n str(REG_CALLRET_SP, STATE_PTR(CpuStateFrame, State.callret_sp));\n\n add(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::rsp, StaticRegisters[X86State::REG_RSP], 0);\n mov(EC_CALL_CHECKER_PC_REG, RipReg);\n ldr(TMP2, STATE_PTR(CpuStateFrame, Pointers.ExitFunctionEC));\n br(TMP2);\n\n (void)Bind(&l_NotECCode);\n#endif\n\n ldrb(TMP1, STATE_PTR(CpuStateFrame, State.flags[X86State::RFLAG_TF_RAW_LOC]));\n (void)cbnz(ARMEmitter::Size::i32Bit, TMP1, &CompileSingleStep);\n\n ARMEmitter::ForwardLabel NoBlock;\n\n if (DisableL2Cache()) {\n (void)b(&NoBlock);\n } else {\n // This is the block cache lookup routine\n // It matches what is going on it LookupCache.h::FindBlock\n ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.L2Pointer));\n\n // Mask the address by the virtual address size so we can check for aliases\n uint64_t VirtualMemorySize = CTX->Config.VirtualMemSize;\n if (std::popcount(VirtualMemorySize) == 1) {\n and_(ARMEmitter::Size::i64Bit, TMP4, RipReg.R(), VirtualMemorySize - 1);\n } else {\n LoadConstant(ARMEmitter::Size::i64Bit, TMP4, VirtualMemorySize);\n and_(ARMEmitter::Size::i64Bit, TMP4, RipReg.R(), TMP4);\n }\n\n {\n // Offset the address and add to our page pointer\n lsr(ARMEmitter::Size::i64Bit, TMP2, TMP4, 12);\n\n // Load the pointer from the offset\n ldr(TMP1, TMP1, TMP2, ARMEmitter::ExtendedType::LSL_64, 3);\n\n // If page pointer is zero then we have no block\n (void)cbz(ARMEmitter::Size::i64Bit, TMP1, &NoBlock);\n\n // Steal the page offset\n and_(ARMEmitter::Size::i64Bit, TMP2, TMP4, 0x0FFF);\n\n // Shift the offset by the size of the block cache entry\n add(TMP1, TMP1, TMP2, ARMEmitter::ShiftType::LSL, FEXCore::ilog2(sizeof(LookupCache::LookupCacheEntry)));\n\n // The the full LookupCacheEntry with a single LDP.\n // Check the guest address first to ensure it maps to the address we are currently at.\n // This fixes aliasing problems\n ldp(TMP4, TMP2, TMP1, 0);\n\n // If the guest address doesn't match, Compile the block.\n sub(TMP2, TMP2, RipReg);\n (void)cbnz(ARMEmitter::Size::i64Bit, TMP2, &NoBlock);\n\n // Check the host address to see if it matches, else compile the block.\n (void)cbz(ARMEmitter::Size::i64Bit, TMP4, &NoBlock);\n\n // If we've made it here then we have a real compiled block\n {\n // update L1 cache\n ldp(TMP1, TMP2, STATE, offsetof(FEXCore::Core::CpuStateFrame, State.L1Pointer));\n\n // Calculate (tmp1 + ((ripreg & L1_ENTRIES_MASK) << 4)) for the address\n // L1Mask is pre-shifted.\n and_(ARMEmitter::Size::i64Bit, TMP2, TMP2, RipReg.R(), ARMEmitter::ShiftType::LSL, FEXCore::ilog2(sizeof(LookupCache::LookupCacheEntry)));\n add(TMP1, TMP1, TMP2);\n\n stp(TMP4, RipReg, TMP1);\n\n // Jump to the block\n br(TMP4);\n }\n }\n }\n\n {\n ThreadStopHandlerAddressSpillSRA = GetCursorAddress();\n SpillStaticRegs(TMP1);\n\n ThreadStopHandlerAddress = GetCursorAddress();\n\n PopCalleeSavedRegisters();\n\n // Return from the function\n // LR is set to the correct return location now\n ret();\n }\n\n // Clobbers TMP1/2\n auto EmitSignalGuardedRegion = [&](auto Body) {\n#ifndef _WIN32\n ldr(TMP2, STATE, offsetof(FEXCore::Core::CPUState, DeferredSignalRefCount));\n add(ARMEmitter::Size::i64Bit, TMP2, TMP2, 1);\n str(TMP2, STATE, offsetof(FEXCore::Core::CPUState, DeferredSignalRefCount));\n#endif\n\n#ifdef ARCHITECTURE_arm64ec\n ldr(TMP2, ARMEmitter::XReg::x18, TEB_CPU_AREA_OFFSET);\n LoadConstant(ARMEmitter::Size::i32Bit, TMP1, 1);\n strb(TMP1.W(), TMP2, CPU_AREA_IN_SYSCALL_CALLBACK_OFFSET);\n#endif\n\n Body();\n\n#ifdef ARCHITECTURE_arm64ec\n ldr(TMP2, ARMEmitter::XReg::x18, TEB_CPU_AREA_OFFSET);\n strb(ARMEmitter::WReg::zr, TMP2, CPU_AREA_IN_SYSCALL_CALLBACK_OFFSET);\n#endif\n\n#ifndef _WIN32\n ldr(TMP2, STATE, offsetof(FEXCore::Core::CPUState, DeferredSignalRefCount));\n sub(ARMEmitter::Size::i64Bit, TMP2, TMP2, 1);\n str(TMP2, STATE, offsetof(FEXCore::Core::CPUState, DeferredSignalRefCount));\n\n // Trigger segfault if any deferred signals are pending\n strb(ARMEmitter::XReg::zr, STATE,\n offsetof(FEXCore::Core::InternalThreadState, InterruptFaultPage) - offsetof(FEXCore::Core::InternalThreadState, BaseFrameState));\n#endif\n };\n\n {\n ExitFunctionLinkerAddress = GetCursorAddress();\n EmitSignalGuardedRegion([&]() {\n SpillStaticRegs(TMP1);\n\n mov(ARMEmitter::XReg::x0, STATE);\n mov(ARMEmitter::XReg::x1, ARMEmitter::XReg::lr);\n\n ldr(ARMEmitter::XReg::x2, STATE_PTR(CpuStateFrame, Pointers.ExitFunctionLink));\n if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] {\n GenerateIndirectRuntimeCall(ARMEmitter::Reg::r2);\n } else {\n blr(ARMEmitter::Reg::r2);\n }\n\n if (!TMP_ABIARGS) {\n mov(TMP1, ARMEmitter::XReg::x0);\n }\n\n FillStaticRegs();\n });\n\n br(TMP1);\n }\n\n // Need to create the block\n {\n (void)Bind(&NoBlock);\n\n EmitSignalGuardedRegion([&]() {\n SpillStaticRegs(TMP1);\n\n if (!TMP_ABIARGS) {\n mov(ARMEmitter::XReg::x2, RipReg);\n }\n\n ldr(ARMEmitter::XReg::x0, &l_CTX);\n mov(ARMEmitter::XReg::x1, STATE);\n // x2 contains guest RIP\n mov(ARMEmitter::XReg::x3, 0);\n ldr(ARMEmitter::XReg::x4, &l_CompileBlock);\n\n if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] {\n GenerateIndirectRuntimeCall(ARMEmitter::Reg::r4);\n } else {\n blr(ARMEmitter::Reg::r4); // { CTX, Frame, RIP, MaxInst }\n }\n\n // Result is now in x0\n if (!TMP_ABIARGS) {\n mov(TMP1, ARMEmitter::XReg::x0);\n }\n\n FillStaticRegs();\n });\n\n // Jump to the compiled block\n br(TMP1);\n }\n\n {\n (void)Bind(&CompileSingleStep);\n\n EmitSignalGuardedRegion([&]() {\n SpillStaticRegs(TMP1);\n\n if (!TMP_ABIARGS) {\n mov(ARMEmitter::XReg::x2, RipReg);\n }\n\n ldr(ARMEmitter::XReg::x0, &l_CTX);\n mov(ARMEmitter::XReg::x1, STATE);\n // x2 contains guest RIP\n ldr(ARMEmitter::XReg::x4, &l_CompileSingleStep);\n\n if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] {\n GenerateIndirectRuntimeCall(ARMEmitter::Reg::r4);\n } else {\n blr(ARMEmitter::Reg::r4); // { CTX, Frame, RIP }\n }\n\n // Result is now in x0\n if (!TMP_ABIARGS) {\n mov(TMP1, ARMEmitter::XReg::x0);\n }\n\n FillStaticRegs();\n });\n\n // Jump to the compiled block\n br(TMP1);\n }\n\n {\n SignalHandlerReturnAddress = GetCursorAddress();\n\n // Now to get back to our old location we need to do a fault dance\n // We can't use SIGTRAP here since gdb catches it and never gives it to the application!\n hlt(0);\n }\n\n {\n SignalHandlerReturnAddressRT = GetCursorAddress();\n\n // Now to get back to our old location we need to do a fault dance\n // We can't use SIGTRAP here since gdb catches it and never gives it to the application!\n hlt(0);\n }\n\n {\n // Guest SIGILL handler\n // Needs to be distinct from the SignalHandlerReturnAddress\n GuestSignal_SIGILL = GetCursorAddress();\n\n SpillStaticRegs(TMP1);\n\n hlt(0);\n }\n\n {\n // Guest SIGTRAP handler\n // Needs to be distinct from the SignalHandlerReturnAddress\n GuestSignal_SIGTRAP = GetCursorAddress();\n\n SpillStaticRegs(TMP1);\n\n brk(0);\n }\n\n {\n // Guest Overflow handler\n // Needs to be distinct from the SignalHandlerReturnAddress\n GuestSignal_SIGSEGV = GetCursorAddress();\n\n SpillStaticRegs(TMP1);\n\n // hlt/udf = SIGILL\n // brk = SIGTRAP\n // ??? = SIGSEGV\n // Force a SIGSEGV by loading zero\n if (CTX->ExitOnHLTEnabled()) {\n ldr(ARMEmitter::XReg::x0, STATE_PTR(CpuStateFrame, ReturningStackLocation));\n add(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::rsp, ARMEmitter::Reg::r0, 0);\n PopCalleeSavedRegisters();\n ret();\n } else {\n LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r1, 0);\n ldr(ARMEmitter::XReg::x1, ARMEmitter::Reg::r1);\n }\n }\n\n {\n ThreadPauseHandlerAddressSpillSRA = GetCursorAddress();\n SpillStaticRegs(TMP1);\n\n ThreadPauseHandlerAddress = GetCursorAddress();\n // We are pausing, this means the frontend should be waiting for this thread to idle\n // We will have faulted and jumped to this location at this point\n\n // Call our sleep handler\n ldr(ARMEmitter::XReg::x0, &l_CTX);\n mov(ARMEmitter::XReg::x1, STATE);\n ldr(ARMEmitter::XReg::x2, &l_Sleep);\n if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] {\n GenerateIndirectRuntimeCall(ARMEmitter::Reg::r2);\n } else {\n blr(ARMEmitter::Reg::r2);\n }\n\n PauseReturnInstruction = GetCursorAddress();\n // Fault to start running again\n hlt(0);\n }\n\n {\n // The expectation here is that a thunked function needs to call back in to the JIT in a reentrant safe way\n // To do this safely we need to do some state tracking and register saving\n //\n // eg:\n // JIT Call->\n // Thunk->\n // Thunk callback->\n //\n // The thunk callback needs to execute JIT code and when it returns, it needs to safely return to the thunk rather than JIT space\n // This is handled by pushing a return address trampoline to the stack so when the guest address returns it hits our custom thunk return\n // - This will safely return us to the thunk\n //\n // On return to the thunk, the thunk can get whatever its return value is from the thread context depending on ABI handling on its end\n // When the thunk itself returns, it'll do its regular return logic there\n // void ReentrantCallback(FEXCore::Core::InternalThreadState *Thread, uint64_t RIP);\n CallbackPtr = GetCursorAddress();\n\n // We expect the thunk to have previously pushed the registers it was using\n PushCalleeSavedRegisters();\n\n // First thing we need to move the thread state pointer back in to our register\n mov(STATE, ARMEmitter::XReg::x0);\n\n // Make sure to adjust the refcounter so we don't clear the cache now\n ldr(ARMEmitter::WReg::w2, STATE_PTR(CpuStateFrame, SignalHandlerRefCounter));\n add(ARMEmitter::Size::i32Bit, ARMEmitter::Reg::r2, ARMEmitter::Reg::r2, 1);\n str(ARMEmitter::WReg::w2, STATE_PTR(CpuStateFrame, SignalHandlerRefCounter));\n\n // Now push the callback return trampoline to the guest stack\n // Guest will be misaligned because calling a thunk won't correct the guest's stack once we call the callback from the host\n ldr(ARMEmitter::XReg::x0, STATE_PTR(CpuStateFrame, Pointers.ThunkCallbackRet));\n\n ldr(ARMEmitter::XReg::x2, STATE_PTR(CpuStateFrame, State.gregs[X86State::REG_RSP]));\n sub(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r2, ARMEmitter::Reg::r2, CTX->Config.Is64BitMode ? 16 : 12);\n str(ARMEmitter::XReg::x2, STATE_PTR(CpuStateFrame, State.gregs[X86State::REG_RSP]));\n\n // Store the trampoline to the guest stack\n // Guest stack is now correctly misaligned after a regular call instruction\n str(ARMEmitter::XReg::x0, ARMEmitter::Reg::r2, 0);\n\n // Store RIP to the context state\n str(ARMEmitter::XReg::x1, STATE_PTR(CpuStateFrame, State.rip));\n\n // load static regs\n FillStaticRegs();\n stp(ARMEmitter::XReg::zr, ARMEmitter::XReg::zr, REG_CALLRET_SP, -0x10);\n\n // Now go back to the regular dispatcher loop\n (void)b(&LoopTop);\n }\n\n auto EmitLongALUOpHandler = [&](auto R, auto Offset) {\n auto Address = GetCursorAddress();\n\n PushDynamicRegs(TMP4);\n SpillStaticRegs(TMP4);\n\n if (!TMP_ABIARGS) {\n mov(ARMEmitter::XReg::x0, TMP1);\n mov(ARMEmitter::XReg::x1, TMP2);\n mov(ARMEmitter::XReg::x2, TMP3);\n }\n\n ldr(ARMEmitter::XReg::x3, R, Offset);\n if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] {\n GenerateIndirectRuntimeCall<__uint128_t, uint64_t, uint64_t, uint64_t>(ARMEmitter::Reg::r3);\n } else {\n blr(ARMEmitter::Reg::r3);\n }\n\n // Result is now in x0, x1\n if (!TMP_ABIARGS) {\n mov(TMP1, ARMEmitter::XReg::x0);\n mov(TMP2, ARMEmitter::XReg::x1);\n }\n\n FillStaticRegs();\n\n // Fix the stack and any values that were stepped on\n PopDynamicRegs();\n\n // Go back to our code block\n ret();\n return Address;\n };\n\n LUDIVHandlerAddress = EmitLongALUOpHandler(STATE_PTR(CpuStateFrame, Pointers.LUDIV));\n LDIVHandlerAddress = EmitLongALUOpHandler(STATE_PTR(CpuStateFrame, Pointers.LDIV));\n\n EmitF64Sin();\n EmitF64Cos();\n EmitF64Tan();\n\n // Interpreter fallbacks\n {\n constexpr static std::array ABIS {{\n FABI_F80_I16_F32_PTR,\n FABI_F80_I16_F64_PTR,\n FABI_F80_I16_I16_PTR,\n FABI_F80_I16_I32_PTR,\n FABI_F32_I16_F80_PTR,\n FABI_F64_I16_F80_PTR,\n FABI_F64_F64_PTR,\n FABI_F64_F64_F64_PTR,\n FABI_I16_I16_F80_PTR,\n FABI_I32_I16_F80_PTR,\n FABI_I64_I16_F80_PTR,\n FABI_I64_I16_F80_F80_PTR,\n FABI_F80_I16_F80_PTR,\n FABI_F80_I16_F80_F80_PTR,\n FABI_F80x2_I16_F80_PTR,\n FABI_F64x2_F64_PTR,\n FABI_I32_I64_I64_V128_V128_I16,\n FABI_I32_V128_V128_I16,\n }};\n\n for (auto ABI : ABIS) {\n ABIPointers[ABI] = GenerateABICall(ABI);\n }\n }\n\n (void)Bind(&l_CTX);\n dc64(reinterpret_cast(CTX));\n (void)Bind(&l_Sleep);\n dc64(reinterpret_cast(SleepThread));\n (void)Bind(&l_CompileBlock);\n FEXCore::Utils::MemberFunctionToPointerCast PMFCompileBlock(&FEXCore::Context::ContextImpl::CompileBlock);\n dc64(PMFCompileBlock.GetConvertedPointer());\n (void)Bind(&l_CompileSingleStep);\n\n FEXCore::Utils::MemberFunctionToPointerCast PMFCompileSingleStep(&FEXCore::Context::ContextImpl::CompileSingleStep);\n dc64(PMFCompileSingleStep.GetConvertedPointer());\n\n Start = reinterpret_cast(DispatchPtr);\n End = GetCursorAddress();\n ClearICache(reinterpret_cast(DispatchPtr), End - reinterpret_cast(DispatchPtr));\n\n if (CTX->Config.BlockJITNaming()) {\n fextl::string Name = fextl::fmt::format(\"Dispatch_{}\", FHU::Syscalls::gettid());\n CTX->Symbols.RegisterNamedRegion(reinterpret_cast(DispatchPtr), End - reinterpret_cast(DispatchPtr), Name);\n }\n if (CTX->Config.GlobalJITNaming()) {\n CTX->Symbols.RegisterJITSpace(reinterpret_cast(DispatchPtr), End - reinterpret_cast(DispatchPtr));\n }\n\n#ifdef VIXL_DISASSEMBLER\n if (Disassemble() & FEXCore::Config::Disassemble::DISPATCHER) {\n const auto DisasmEnd = GetCursorAddress();\n for (auto PCToDecode = DisasmBegin; PCToDecode < DisasmEnd; PCToDecode += 4) {\n DisasmDecoder->Decode(PCToDecode);\n auto Output = Disasm->GetOutput();\n LogMan::Msg::IFmt(\"{}\", Output);\n }\n }\n#endif\n}\n\n#ifdef VIXL_SIMULATOR\nvoid Dispatcher::ExecuteDispatch(FEXCore::Core::CpuStateFrame* Frame) {\n Simulator.WriteXRegister(0, reinterpret_cast(Frame));\n Simulator.WriteXRegister(1, 0);\n Simulator.RunFrom(reinterpret_cast< const vixl::aarch64::Instruction*>(DispatchPtr));\n}\n\nvoid Dispatcher::ExecuteJITCallback(FEXCore::Core::CpuStateFrame* Frame, uint64_t RIP) {\n Simulator.WriteXRegister(0, reinterpret_cast(Frame));\n Simulator.WriteXRegister(1, RIP);\n Simulator.RunFrom(reinterpret_cast< const vixl::aarch64::Instruction*>(CallbackPtr));\n}\n\n#endif\n\nvoid Dispatcher::EmitI32ToExtF80() {\n ARMEmitter::ForwardLabel ZeroCase;\n ARMEmitter::ForwardLabel Done;\n\n (void)cbz(ARMEmitter::Size::i32Bit, TMP2, &ZeroCase);\n\n lsr(ARMEmitter::Size::i32Bit, TMP4, TMP2, 31);\n tst(ARMEmitter::Size::i32Bit, TMP2, TMP2);\n neg(ARMEmitter::Size::i32Bit, TMP3, TMP2);\n csel(ARMEmitter::Size::i32Bit, TMP3, TMP3, TMP2, ARMEmitter::Condition::CC_MI);\n\n clz(ARMEmitter::Size::i32Bit, TMP1, TMP3);\n\n mov(ARMEmitter::Size::i32Bit, TMP2, 0x401E);\n sub(ARMEmitter::Size::i32Bit, TMP2, TMP2, TMP1);\n orr(ARMEmitter::Size::i32Bit, TMP2, TMP2, TMP4, ARMEmitter::ShiftType::LSL, 15);\n\n lslv(ARMEmitter::Size::i32Bit, TMP3, TMP3, TMP1);\n\n lsl(ARMEmitter::Size::i64Bit, TMP3, TMP3, 32);\n\n fmov(ARMEmitter::Size::i64Bit, VTMP1.D(), TMP3);\n ins(ARMEmitter::SubRegSize::i16Bit, VTMP1, 4, TMP2);\n\n (void)b(&Done);\n\n (void)Bind(&ZeroCase);\n movi(ARMEmitter::SubRegSize::i64Bit, VTMP1.Q(), 0);\n\n (void)Bind(&Done);\n}\n\nvoid Dispatcher::EmitI16ToExtF80() {\n sxth(ARMEmitter::Size::i32Bit, TMP2, TMP2);\n\n ARMEmitter::ForwardLabel ZeroCase;\n ARMEmitter::ForwardLabel Done;\n\n (void)cbz(ARMEmitter::Size::i32Bit, TMP2, &ZeroCase);\n\n lsr(ARMEmitter::Size::i32Bit, TMP4, TMP2, 31);\n tst(ARMEmitter::Size::i32Bit, TMP2, TMP2);\n neg(ARMEmitter::Size::i32Bit, TMP3, TMP2);\n csel(ARMEmitter::Size::i32Bit, TMP3, TMP3, TMP2, ARMEmitter::Condition::CC_MI);\n\n clz(ARMEmitter::Size::i32Bit, TMP1, TMP3);\n\n mov(ARMEmitter::Size::i32Bit, TMP2, 0x401E);\n sub(ARMEmitter::Size::i32Bit, TMP2, TMP2, TMP1);\n orr(ARMEmitter::Size::i32Bit, TMP2, TMP2, TMP4, ARMEmitter::ShiftType::LSL, 15);\n\n lslv(ARMEmitter::Size::i32Bit, TMP3, TMP3, TMP1);\n\n lsl(ARMEmitter::Size::i64Bit, TMP3, TMP3, 32);\n\n fmov(ARMEmitter::Size::i64Bit, VTMP1.D(), TMP3);\n ins(ARMEmitter::SubRegSize::i16Bit, VTMP1, 4, TMP2);\n\n (void)b(&Done);\n\n (void)Bind(&ZeroCase);\n movi(ARMEmitter::SubRegSize::i64Bit, VTMP1.Q(), 0);\n\n (void)Bind(&Done);\n}\n\nvoid Dispatcher::EmitF32ToExtF80() {\n ARMEmitter::ForwardLabel InfNaN;\n ARMEmitter::ForwardLabel ZeroDenormal;\n ARMEmitter::ForwardLabel Denormal;\n ARMEmitter::ForwardLabel NaN;\n ARMEmitter::ForwardLabel Done;\n ARMEmitter::BiDirectionalLabel NormalPath;\n ARMEmitter::ForwardLabel ZeroResult;\n\n fmov(ARMEmitter::Size::i32Bit, TMP1, VTMP1.S());\n\n ubfx(ARMEmitter::Size::i32Bit, TMP2, TMP1, 23, 8);\n and_(ARMEmitter::Size::i32Bit, TMP3, TMP1, 0x007FFFFF);\n lsr(ARMEmitter::Size::i32Bit, TMP4, TMP1, 31);\n\n cmp(ARMEmitter::Size::i32Bit, TMP2, 0xFF);\n (void)b(ARMEmitter::Condition::CC_EQ, &InfNaN);\n\n (void)cbz(ARMEmitter::Size::i32Bit, TMP2, &ZeroDenormal);\n\n (void)Bind(&NormalPath);\n // Exponent bias adjustment, where bias is 0x3F80\n LoadConstant(ARMEmitter::Size::i32Bit, TMP1, 0x3F80);\n add(ARMEmitter::Size::i32Bit, TMP2, TMP2, TMP1);\n orr(ARMEmitter::Size::i32Bit, TMP2, TMP2, TMP4, ARMEmitter::ShiftType::LSL, 15);\n\n // Set implicit bit and shift fraction to extF80 position\n LoadConstant(ARMEmitter::Size::i64Bit, TMP1, 0x00800000ULL);\n orr(ARMEmitter::Size::i64Bit, TMP3, TMP3, TMP1);\n lsl(ARMEmitter::Size::i64Bit, TMP3, TMP3, 40);\n\n fmov(ARMEmitter::Size::i64Bit, VTMP1.D(), TMP3);\n ins(ARMEmitter::SubRegSize::i16Bit, VTMP1, 4, TMP2);\n (void)b(&Done);\n\n (void)Bind(&ZeroDenormal);\n (void)cbz(ARMEmitter::Size::i32Bit, TMP3, &ZeroResult);\n\n (void)Bind(&Denormal);\n clz(ARMEmitter::Size::i32Bit, TMP1, TMP3);\n sub(ARMEmitter::Size::i32Bit, TMP1, TMP1, 8);\n mov(ARMEmitter::Size::i32Bit, TMP2, 1);\n sub(ARMEmitter::Size::i32Bit, TMP2, TMP2, TMP1);\n lslv(ARMEmitter::Size::i32Bit, TMP3, TMP3, TMP1);\n (void)b(&NormalPath);\n\n (void)Bind(&ZeroResult);\n lsl(ARMEmitter::Size::i32Bit, TMP2, TMP4, 15);\n movi(ARMEmitter::SubRegSize::i64Bit, VTMP1.Q(), 0);\n ins(ARMEmitter::SubRegSize::i16Bit, VTMP1, 4, TMP2);\n (void)b(&Done);\n\n (void)Bind(&InfNaN);\n (void)cbnz(ARMEmitter::Size::i32Bit, TMP3, &NaN);\n\n lsl(ARMEmitter::Size::i32Bit, TMP2, TMP4, 15);\n orr(ARMEmitter::Size::i32Bit, TMP2, TMP2, 0x7FFF);\n\n LoadConstant(ARMEmitter::Size::i64Bit, TMP3, 0x8000000000000000ULL);\n fmov(ARMEmitter::Size::i64Bit, VTMP1.D(), TMP3);\n ins(ARMEmitter::SubRegSize::i16Bit, VTMP1, 4, TMP2);\n (void)b(&Done);\n\n (void)Bind(&NaN);\n lsl(ARMEmitter::Size::i32Bit, TMP2, TMP4, 15);\n orr(ARMEmitter::Size::i32Bit, TMP2, TMP2, 0x7FFF);\n\n lsl(ARMEmitter::Size::i64Bit, TMP3, TMP3, 40);\n LoadConstant(ARMEmitter::Size::i64Bit, TMP1, 0xC000000000000000ULL);\n orr(ARMEmitter::Size::i64Bit, TMP3, TMP3, TMP1);\n\n fmov(ARMEmitter::Size::i64Bit, VTMP1.D(), TMP3);\n ins(ARMEmitter::SubRegSize::i16Bit, VTMP1, 4, TMP2);\n\n (void)Bind(&Done);\n}\n\nvoid Dispatcher::EmitF64ToExtF80() {\n ARMEmitter::ForwardLabel InfNaN;\n ARMEmitter::ForwardLabel ZeroDenormal;\n ARMEmitter::ForwardLabel Denormal;\n ARMEmitter::ForwardLabel NaN;\n ARMEmitter::ForwardLabel Done;\n ARMEmitter::BiDirectionalLabel NormalPath;\n ARMEmitter::ForwardLabel ZeroResult;\n\n fmov(ARMEmitter::Size::i64Bit, TMP1, VTMP1.D());\n\n lsr(ARMEmitter::Size::i64Bit, TMP4, TMP1, 63);\n ubfx(ARMEmitter::Size::i64Bit, TMP2, TMP1, 52, 11);\n LoadConstant(ARMEmitter::Size::i64Bit, TMP3, 0x000FFFFFFFFFFFFFULL);\n and_(ARMEmitter::Size::i64Bit, TMP3, TMP1, TMP3);\n\n cmp(ARMEmitter::Size::i64Bit, TMP2, 0x7FF);\n (void)b(ARMEmitter::Condition::CC_EQ, &InfNaN);\n\n (void)cbz(ARMEmitter::Size::i64Bit, TMP2, &ZeroDenormal);\n\n (void)Bind(&NormalPath);\n // Exponent bias adjustment where bias difference is 0x3C00\n add(ARMEmitter::Size::i64Bit, TMP2, TMP2, 0x3000);\n add(ARMEmitter::Size::i64Bit, TMP2, TMP2, 0xC00);\n orr(ARMEmitter::Size::i64Bit, TMP2, TMP2, TMP4, ARMEmitter::ShiftType::LSL, 15);\n\n LoadConstant(ARMEmitter::Size::i64Bit, TMP1, 0x0010000000000000ULL);\n orr(ARMEmitter::Size::i64Bit, TMP3, TMP3, TMP1);\n lsl(ARMEmitter::Size::i64Bit, TMP3, TMP3, 11);\n\n fmov(ARMEmitter::Size::i64Bit, VTMP1.D(), TMP3);\n ins(ARMEmitter::SubRegSize::i16Bit, VTMP1, 4, TMP2);\n (void)b(&Done);\n\n (void)Bind(&ZeroDenormal);\n (void)cbz(ARMEmitter::Size::i64Bit, TMP3, &ZeroResult);\n\n (void)Bind(&Denormal);\n clz(ARMEmitter::Size::i64Bit, TMP1, TMP3);\n sub(ARMEmitter::Size::i64Bit, TMP1, TMP1, 11);\n mov(ARMEmitter::Size::i64Bit, TMP2, 1);\n sub(ARMEmitter::Size::i64Bit, TMP2, TMP2, TMP1);\n lslv(ARMEmitter::Size::i64Bit, TMP3, TMP3, TMP1);\n (void)b(&NormalPath);\n\n (void)Bind(&ZeroResult);\n lsl(ARMEmitter::Size::i64Bit, TMP2, TMP4, 15);\n movi(ARMEmitter::SubRegSize::i64Bit, VTMP1.Q(), 0);\n ins(ARMEmitter::SubRegSize::i16Bit, VTMP1, 4, TMP2);\n (void)b(&Done);\n\n (void)Bind(&InfNaN);\n (void)cbnz(ARMEmitter::Size::i64Bit, TMP3, &NaN);\n\n lsl(ARMEmitter::Size::i64Bit, TMP2, TMP4, 15);\n orr(ARMEmitter::Size::i64Bit, TMP2, TMP2, 0x7FFF);\n\n LoadConstant(ARMEmitter::Size::i64Bit, TMP3, 0x8000000000000000ULL);\n fmov(ARMEmitter::Size::i64Bit, VTMP1.D(), TMP3);\n ins(ARMEmitter::SubRegSize::i16Bit, VTMP1, 4, TMP2);\n (void)b(&Done);\n\n (void)Bind(&NaN);\n lsl(ARMEmitter::Size::i64Bit, TMP2, TMP4, 15);\n orr(ARMEmitter::Size::i64Bit, TMP2, TMP2, 0x7FFF);\n\n lsl(ARMEmitter::Size::i64Bit, TMP3, TMP3, 11);\n LoadConstant(ARMEmitter::Size::i64Bit, TMP1, 0xC000000000000000ULL);\n orr(ARMEmitter::Size::i64Bit, TMP3, TMP3, TMP1);\n\n fmov(ARMEmitter::Size::i64Bit, VTMP1.D(), TMP3);\n ins(ARMEmitter::SubRegSize::i16Bit, VTMP1, 4, TMP2);\n\n (void)Bind(&Done);\n}\n\nvoid Dispatcher::EmitF64Sin() {\n F64SinHandlerAddress = GetCursorAddress();\n\n constexpr auto V2 = ARMEmitter::VReg::v2;\n constexpr auto V3 = ARMEmitter::VReg::v3;\n constexpr auto V4 = ARMEmitter::VReg::v4;\n constexpr auto V5 = ARMEmitter::VReg::v5;\n\n ARMEmitter::ForwardLabel Fallback, NonZero;\n ARMEmitter::ForwardLabel InvPiPi1Label, Pi23Label;\n ARMEmitter::ForwardLabel C0Label, C1Label, C2Label, C3Label, C4Label, C5Label, C6Label;\n ARMEmitter::ForwardLabel RangeLabel;\n\n // sin(+/-0) = +/-0\n fmov(ARMEmitter::Size::i64Bit, TMP1, VTMP1.D());\n lsl(ARMEmitter::Size::i64Bit, TMP1, TMP1, 1);\n (void)cbnz(ARMEmitter::Size::i64Bit, TMP1, &NonZero);\n ret();\n (void)Bind(&NonZero);\n\n // Save q2-q5.\n stp(ARMEmitter::QReg::q2, ARMEmitter::QReg::q3, ARMEmitter::Reg::rsp, -64);\n stp(ARMEmitter::QReg::q4, ARMEmitter::QReg::q5, ARMEmitter::Reg::rsp, 32);\n\n // save nzcv\n mrs(TMP1, ARMEmitter::SystemRegister::NZCV);\n str(TMP1.W(), STATE.R(), offsetof(FEXCore::Core::CpuStateFrame, State.flags[24]));\n\n // Range check: fall back for |x| >= 2^23, NaN, and inf.\n fabs(VTMP2.D(), VTMP1.D());\n ldr(V2.D(), &RangeLabel);\n fcmp(VTMP2.D(), V2.D());\n (void)b(ARMEmitter::Condition::CC_HS, &Fallback);\n\n // n = rint(x/pi).\n ldr(V2.Q(), &InvPiPi1Label); // q2 = {inv_pi, pi_1}\n fmul(VTMP2.D(), VTMP1.D(), V2.D());\n frinta(VTMP2.D(), VTMP2.D());\n\n // odd = (int(n) & 1) << 63.\n fcvtzs(ARMEmitter::Size::i64Bit, TMP1, VTMP2.D());\n lsl(ARMEmitter::Size::i64Bit, TMP1, TMP1, 63);\n\n // r = x - n*pi (range reduction) via .2D lane-indexed FMLS.\n ldr(V3.Q(), &Pi23Label); // q3 = {pi_2, pi_3}\n fmov(V4.D(), VTMP1.D()); // r = x\n fmls(ARMEmitter::SubRegSize::i64Bit, V4.Q(), VTMP2.Q(), V2.Q(), 1); // r -= n * pi_1\n fmls(ARMEmitter::SubRegSize::i64Bit, V4.Q(), VTMP2.Q(), V3.Q(), 0); // r -= n * pi_2\n fmls(ARMEmitter::SubRegSize::i64Bit, V4.Q(), VTMP2.Q(), V3.Q(), 1); // r -= n * pi_3\n\n // r^2, r^4.\n fmul(V5.D(), V4.D(), V4.D());\n fmov(ARMEmitter::Size::i64Bit, TMP2, V4.D());\n fmul(V3.D(), V5.D(), V5.D());\n\n // Estrin polynomial: p = c0 + r2*c1 + r4*(c2 + r2*c3) + r8*(c4 + r2*c5 + r4*c6).\n // Level 1 (independent FMAs).\n ldr(VTMP1.D(), &C0Label);\n ldr(VTMP2.D(), &C1Label);\n fmadd(VTMP1.D(), V5.D(), VTMP2.D(), VTMP1.D()); // p01 = c0 + r2*c1\n\n ldr(VTMP2.D(), &C2Label);\n ldr(V2.D(), &C3Label);\n fmadd(VTMP2.D(), V5.D(), V2.D(), VTMP2.D()); // p23 = c2 + r2*c3\n\n ldr(V2.D(), &C4Label);\n ldr(V4.D(), &C5Label);\n fmadd(V2.D(), V5.D(), V4.D(), V2.D()); // p45 = c4 + r2*c5\n\n // Level 2 (serial).\n ldr(V4.D(), &C6Label);\n fmadd(V2.D(), V3.D(), V4.D(), V2.D()); // p46 = p45 + r4*c6\n fmadd(VTMP2.D(), V3.D(), V2.D(), VTMP2.D()); // p26 = p23 + r4*p46\n fmadd(VTMP1.D(), V3.D(), VTMP2.D(), VTMP1.D()); // p06 = p01 + r4*p26\n\n // y = r + r^3 * p06.\n fmov(ARMEmitter::Size::i64Bit, V4.D(), TMP2);\n fmul(V5.D(), V5.D(), V4.D());\n fmadd(VTMP1.D(), V5.D(), VTMP1.D(), V4.D());\n\n // result = y XOR odd.\n fmov(ARMEmitter::Size::i64Bit, TMP2, VTMP1.D());\n eor(ARMEmitter::Size::i64Bit, TMP2, TMP2, TMP1);\n fmov(ARMEmitter::Size::i64Bit, VTMP1.D(), TMP2);\n\n // restore nzcv\n ldr(TMP1.W(), STATE.R(), offsetof(FEXCore::Core::CpuStateFrame, State.flags[24]));\n msr(ARMEmitter::SystemRegister::NZCV, TMP1);\n\n // Restore q2-q5 and return.\n ldp(ARMEmitter::QReg::q4, ARMEmitter::QReg::q5, ARMEmitter::Reg::rsp, 32);\n ldp(ARMEmitter::QReg::q2, ARMEmitter::QReg::q3, ARMEmitter::Reg::rsp, 64);\n ret();\n\n // Fallback path.\n (void)Bind(&Fallback);\n ldp(ARMEmitter::QReg::q4, ARMEmitter::QReg::q5, ARMEmitter::Reg::rsp, 32);\n ldp(ARMEmitter::QReg::q2, ARMEmitter::QReg::q3, ARMEmitter::Reg::rsp, 64);\n str(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, -16);\n ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.FallbackHandlerPointers[FEXCore::Core::OPINDEX_F64SIN].ABIHandler));\n ldr(TMP4, STATE_PTR(CpuStateFrame, Pointers.FallbackHandlerPointers[FEXCore::Core::OPINDEX_F64SIN].Func));\n blr(TMP1);\n ldr(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, 16);\n ret();\n\n // Constant pool.\n Align(16);\n (void)Bind(&InvPiPi1Label);\n dc64(0x3FD4'5F30'6DC9'C883ULL); // inv_pi\n dc64(0x4009'21FB'5444'2D18ULL); // pi_1\n (void)Bind(&Pi23Label);\n dc64(0x3CA1'A626'3314'5C06ULL); // pi_2\n dc64(0x395C'1CD1'2902'4E09ULL); // pi_3\n (void)Bind(&C0Label);\n dc64(0xBFC5'5555'5555'547BULL); // c0\n (void)Bind(&C1Label);\n dc64(0x3F81'1111'1110'8A4DULL); // c1\n (void)Bind(&C2Label);\n dc64(0xBF2A'01A0'1993'6F27ULL); // c2\n (void)Bind(&C3Label);\n dc64(0x3EC7'1DE3'7A97'D93EULL); // c3\n (void)Bind(&C4Label);\n dc64(0xBE5A'E633'9199'87C6ULL); // c4\n (void)Bind(&C5Label);\n dc64(0x3DE6'0E27'7AE0'7CECULL); // c5\n (void)Bind(&C6Label);\n dc64(0xBD69'E954'0300'A100ULL); // c6\n (void)Bind(&RangeLabel);\n dc64(0x4160'0000'0000'0000ULL); // 2^23\n}\n\nvoid Dispatcher::EmitF64Cos() {\n F64CosHandlerAddress = GetCursorAddress();\n\n constexpr auto Accum = ARMEmitter::VReg::v2;\n\n ARMEmitter::ForwardLabel Fallback;\n ARMEmitter::ForwardLabel RangeLabel, InvPiLabel;\n ARMEmitter::ForwardLabel Pi1Label, Pi2Label, Pi3Label;\n ARMEmitter::ForwardLabel C0Label, C1Label, C2Label, C3Label, C4Label, C5Label, C6Label;\n\n // Save q2 for use as accumulator\n str(ARMEmitter::QReg::q2, ARMEmitter::Reg::rsp, -16);\n\n // save nzcv\n mrs(TMP1, ARMEmitter::SystemRegister::NZCV);\n str(TMP1.W(), STATE.R(), offsetof(FEXCore::Core::CpuStateFrame, State.flags[24]));\n\n // Range check: fall back for |x| >= 2^23, NaN, and inf.\n fabs(VTMP2.D(), VTMP1.D());\n ldr(Accum.D(), &RangeLabel);\n fcmp(VTMP2.D(), Accum.D());\n (void)b(ARMEmitter::Condition::CC_HS, &Fallback);\n\n // n = rint(x * (1/pi) + 0.5).\n ldr(Accum.D(), &InvPiLabel);\n fmov(ARMEmitter::ScalarRegSize::i64Bit, VTMP2, 0.5f);\n fmadd(VTMP2.D(), VTMP1.D(), Accum.D(), VTMP2.D());\n frinta(VTMP2.D(), VTMP2.D());\n\n // odd = (int(n) & 1) << 63.\n fcvtzs(ARMEmitter::Size::i64Bit, TMP1, VTMP2.D());\n lsl(ARMEmitter::Size::i64Bit, TMP1, TMP1, 63);\n\n // Save input to Accum before overwriting VTMP1.\n fmov(Accum.D(), VTMP1.D());\n\n // n = n - 0.5.\n fmov(ARMEmitter::ScalarRegSize::i64Bit, VTMP1, 0.5f);\n fsub(VTMP2.D(), VTMP2.D(), VTMP1.D());\n\n // r = x - n*pi (range reduction), in extended precision.\n ldr(VTMP1.D(), &Pi1Label);\n fmsub(Accum.D(), VTMP2.D(), VTMP1.D(), Accum.D());\n ldr(VTMP1.D(), &Pi2Label);\n fmsub(Accum.D(), VTMP2.D(), VTMP1.D(), Accum.D());\n ldr(VTMP1.D(), &Pi3Label);\n fmsub(Accum.D(), VTMP2.D(), VTMP1.D(), Accum.D());\n\n // sin(r) poly approx.\n fmul(VTMP1.D(), Accum.D(), Accum.D());\n fmov(ARMEmitter::Size::i64Bit, TMP2, Accum.D());\n\n // Horner: p = c6 + r2*(c5 + r2*(... + r2*c0)).\n ldr(VTMP2.D(), &C6Label);\n ldr(Accum.D(), &C5Label);\n fmadd(VTMP2.D(), VTMP1.D(), VTMP2.D(), Accum.D());\n ldr(Accum.D(), &C4Label);\n fmadd(VTMP2.D(), VTMP1.D(), VTMP2.D(), Accum.D());\n ldr(Accum.D(), &C3Label);\n fmadd(VTMP2.D(), VTMP1.D(), VTMP2.D(), Accum.D());\n ldr(Accum.D(), &C2Label);\n fmadd(VTMP2.D(), VTMP1.D(), VTMP2.D(), Accum.D());\n ldr(Accum.D(), &C1Label);\n fmadd(VTMP2.D(), VTMP1.D(), VTMP2.D(), Accum.D());\n ldr(Accum.D(), &C0Label);\n fmadd(VTMP2.D(), VTMP1.D(), VTMP2.D(), Accum.D());\n\n // y = r + r^3 * p.\n fmov(ARMEmitter::Size::i64Bit, Accum.D(), TMP2);\n fmul(VTMP1.D(), VTMP1.D(), Accum.D());\n fmadd(Accum.D(), VTMP1.D(), VTMP2.D(), Accum.D());\n\n // result = y XOR odd.\n fmov(ARMEmitter::Size::i64Bit, TMP2, Accum.D());\n eor(ARMEmitter::Size::i64Bit, TMP2, TMP2, TMP1);\n fmov(ARMEmitter::Size::i64Bit, VTMP1.D(), TMP2);\n\n // restore nzcv\n ldr(TMP1.W(), STATE.R(), offsetof(FEXCore::Core::CpuStateFrame, State.flags[24]));\n msr(ARMEmitter::SystemRegister::NZCV, TMP1);\n\n // Restore q2 and return.\n ldr(ARMEmitter::QReg::q2, ARMEmitter::Reg::rsp, 16);\n ret();\n\n // Fallback path.\n (void)Bind(&Fallback);\n ldr(ARMEmitter::QReg::q2, ARMEmitter::Reg::rsp, 16);\n str(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, -16);\n ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.FallbackHandlerPointers[FEXCore::Core::OPINDEX_F64COS].ABIHandler));\n ldr(TMP4, STATE_PTR(CpuStateFrame, Pointers.FallbackHandlerPointers[FEXCore::Core::OPINDEX_F64COS].Func));\n blr(TMP1);\n ldr(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, 16);\n ret();\n\n // Constant pool.\n Align(16);\n (void)Bind(&InvPiLabel);\n dc64(0x3FD4'5F30'6DC9'C883ULL); // inv_pi\n (void)Bind(&Pi1Label);\n dc64(0x4009'21FB'5444'2D18ULL); // pi_1\n (void)Bind(&Pi2Label);\n dc64(0x3CA1'A626'3314'5C06ULL); // pi_2\n (void)Bind(&Pi3Label);\n dc64(0x395C'1CD1'2902'4E09ULL); // pi_3\n (void)Bind(&C0Label);\n dc64(0xBFC5'5555'5555'547BULL); // c0\n (void)Bind(&C1Label);\n dc64(0x3F81'1111'1110'8A4DULL); // c1\n (void)Bind(&C2Label);\n dc64(0xBF2A'01A0'1993'6F27ULL); // c2\n (void)Bind(&C3Label);\n dc64(0x3EC7'1DE3'7A97'D93EULL); // c3\n (void)Bind(&C4Label);\n dc64(0xBE5A'E633'9199'87C6ULL); // c4\n (void)Bind(&C5Label);\n dc64(0x3DE6'0E27'7AE0'7CECULL); // c5\n (void)Bind(&C6Label);\n dc64(0xBD69'E954'0300'A100ULL); // c6\n (void)Bind(&RangeLabel);\n dc64(0x4160'0000'0000'0000ULL); // 2^23\n}\n\nvoid Dispatcher::EmitF64Tan() {\n F64TanHandlerAddress = GetCursorAddress();\n\n constexpr auto Accum = ARMEmitter::VReg::v2;\n\n ARMEmitter::ForwardLabel Fallback, NonZero;\n ARMEmitter::ForwardLabel RangeLabel, TwoOverPiLabel;\n ARMEmitter::ForwardLabel HalfPi0Label, HalfPi1Label;\n ARMEmitter::ForwardLabel C0Label, C1Label, C2Label, C3Label, C4Label, C5Label, C6Label, C7Label, C8Label;\n\n // tan(+/-0) = +/-0\n fmov(ARMEmitter::Size::i64Bit, TMP1, VTMP1.D());\n lsl(ARMEmitter::Size::i64Bit, TMP1, TMP1, 1);\n (void)cbnz(ARMEmitter::Size::i64Bit, TMP1, &NonZero);\n ret();\n (void)Bind(&NonZero);\n\n // Save q2 for use as accumulator\n str(ARMEmitter::QReg::q2, ARMEmitter::Reg::rsp, -16);\n\n // save nzcv\n mrs(TMP1, ARMEmitter::SystemRegister::NZCV);\n str(TMP1.W(), STATE.R(), offsetof(FEXCore::Core::CpuStateFrame, State.flags[24]));\n\n // Range check: fall back for |x| >= 2^23, NaN, and inf.\n fabs(VTMP2.D(), VTMP1.D());\n ldr(Accum.D(), &RangeLabel);\n fcmp(VTMP2.D(), Accum.D());\n (void)b(ARMEmitter::Condition::CC_HS, &Fallback);\n\n // q = nearest integer to 2 * x / pi.\n ldr(VTMP2.D(), &TwoOverPiLabel);\n fmul(VTMP2.D(), VTMP1.D(), VTMP2.D());\n frinta(VTMP2.D(), VTMP2.D());\n\n // qi = int(q).\n fcvtzs(ARMEmitter::Size::i64Bit, TMP1, VTMP2.D());\n\n // r = x - q * pi/2 (range reduction), in extended precision.\n fmov(Accum.D(), VTMP1.D());\n ldr(VTMP1.D(), &HalfPi0Label);\n fmsub(Accum.D(), VTMP2.D(), VTMP1.D(), Accum.D());\n ldr(VTMP1.D(), &HalfPi1Label);\n fmsub(Accum.D(), VTMP2.D(), VTMP1.D(), Accum.D());\n\n // Further reduce r to [-pi/8, pi/8].\n fmov(ARMEmitter::ScalarRegSize::i64Bit, VTMP1, 0.5f);\n fmul(Accum.D(), Accum.D(), VTMP1.D());\n\n // Approximate tan(r) using order 8 polynomial.\n fmul(VTMP1.D(), Accum.D(), Accum.D());\n fmov(ARMEmitter::Size::i64Bit, TMP2, Accum.D());\n\n // Horner: p = C8 + r2*(C7 + r2*(... + r2*C0)).\n ldr(VTMP2.D(), &C8Label);\n ldr(Accum.D(), &C7Label);\n fmadd(VTMP2.D(), VTMP1.D(), VTMP2.D(), Accum.D());\n ldr(Accum.D(), &C6Label);\n fmadd(VTMP2.D(), VTMP1.D(), VTMP2.D(), Accum.D());\n ldr(Accum.D(), &C5Label);\n fmadd(VTMP2.D(), VTMP1.D(), VTMP2.D(), Accum.D());\n ldr(Accum.D(), &C4Label);\n fmadd(VTMP2.D(), VTMP1.D(), VTMP2.D(), Accum.D());\n ldr(Accum.D(), &C3Label);\n fmadd(VTMP2.D(), VTMP1.D(), VTMP2.D(), Accum.D());\n ldr(Accum.D(), &C2Label);\n fmadd(VTMP2.D(), VTMP1.D(), VTMP2.D(), Accum.D());\n ldr(Accum.D(), &C1Label);\n fmadd(VTMP2.D(), VTMP1.D(), VTMP2.D(), Accum.D());\n ldr(Accum.D(), &C0Label);\n fmadd(VTMP2.D(), VTMP1.D(), VTMP2.D(), Accum.D());\n\n // p = r + r^3 * p.\n fmov(ARMEmitter::Size::i64Bit, Accum.D(), TMP2);\n fmul(VTMP1.D(), VTMP1.D(), Accum.D());\n fmadd(Accum.D(), VTMP1.D(), VTMP2.D(), Accum.D());\n\n // Double-angle reconstruction: tan(2x) = 2*tan(x) / (1 - tan^2(x)).\n fadd(VTMP1.D(), Accum.D(), Accum.D());\n fmul(VTMP2.D(), Accum.D(), Accum.D());\n fmov(ARMEmitter::ScalarRegSize::i64Bit, Accum, 1.0f);\n fsub(VTMP2.D(), VTMP2.D(), Accum.D());\n\n ARMEmitter::ForwardLabel SkipSwap;\n (void)tbnz(TMP1, 0, &SkipSwap);\n\n fneg(Accum.D(), VTMP1.D());\n fmov(VTMP1.D(), VTMP2.D());\n fmov(VTMP2.D(), Accum.D());\n\n (void)Bind(&SkipSwap);\n\n // result = numerator / denominator -> VTMP1.\n fdiv(VTMP1.D(), VTMP2.D(), VTMP1.D());\n\n // restore nzcv\n ldr(TMP1.W(), STATE.R(), offsetof(FEXCore::Core::CpuStateFrame, State.flags[24]));\n msr(ARMEmitter::SystemRegister::NZCV, TMP1);\n\n // Restore q2 and return.\n ldr(ARMEmitter::QReg::q2, ARMEmitter::Reg::rsp, 16);\n ret();\n\n // Fallback path.\n (void)Bind(&Fallback);\n ldr(ARMEmitter::QReg::q2, ARMEmitter::Reg::rsp, 16);\n str(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, -16);\n ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.FallbackHandlerPointers[FEXCore::Core::OPINDEX_F64TAN].ABIHandler));\n ldr(TMP4, STATE_PTR(CpuStateFrame, Pointers.FallbackHandlerPointers[FEXCore::Core::OPINDEX_F64TAN].Func));\n blr(TMP1);\n ldr(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, 16);\n ret();\n\n // Constant pool.\n Align(16);\n (void)Bind(&TwoOverPiLabel);\n dc64(0x3FE4'5F30'6DC9'C883ULL); // two_over_pi\n (void)Bind(&HalfPi0Label);\n dc64(0x3FF9'21FB'5444'2D18ULL); // half_pi[0]\n (void)Bind(&HalfPi1Label);\n dc64(0x3C91'A626'3314'5C07ULL); // half_pi[1]\n (void)Bind(&C0Label);\n dc64(0x3FD5'5555'5555'5556ULL); // C0\n (void)Bind(&C1Label);\n dc64(0x3FC1'1111'1111'0A63ULL); // C1\n (void)Bind(&C2Label);\n dc64(0x3FAB'A1BA'1BB4'6414ULL); // C2\n (void)Bind(&C3Label);\n dc64(0x3F96'64F4'7E5B'5445ULL); // C3\n (void)Bind(&C4Label);\n dc64(0x3F82'26E5'E5EC'DFA3ULL); // C4\n (void)Bind(&C5Label);\n dc64(0x3F6D'6C7D'DBF8'7047ULL); // C5\n (void)Bind(&C6Label);\n dc64(0x3F57'EA75'D05B'583EULL); // C6\n (void)Bind(&C7Label);\n dc64(0x3F42'89F2'2964'A03CULL); // C7\n (void)Bind(&C8Label);\n dc64(0x3F34'E4FD'1414'7622ULL); // C8\n (void)Bind(&RangeLabel);\n dc64(0x4160'0000'0000'0000ULL); // 2^23\n}\n\nuint64_t Dispatcher::GenerateABICall(FallbackABI ABI) {\n auto Address = GetCursorAddress();\n constexpr static auto FallbackPointerReg = TMP4;\n constexpr static auto ABI1 = ARMEmitter::XReg::x0;\n constexpr static auto ABI2 = ARMEmitter::XReg::x1;\n constexpr static auto ABI3 = ARMEmitter::XReg::x2;\n\n constexpr static auto VABI1 = ARMEmitter::VReg::v0;\n constexpr static auto VABI2 = ARMEmitter::VReg::v1;\n\n auto FillF80x2Result = [&]() {\n if (!TMP_ABIARGS) {\n mov(VTMP1.Q(), VABI1.Q());\n mov(VTMP2.Q(), VABI2.Q());\n }\n FillForABICall(CTX->HostFeatures.SupportsPreserveAllABI, true);\n };\n\n auto FillF64x2Result = [&]() {\n if (!TMP_ABIARGS) {\n fmov(VTMP1.D(), VABI1.D());\n fmov(VTMP2.D(), VABI2.D());\n }\n FillForABICall(CTX->HostFeatures.SupportsPreserveAllABI, true);\n };\n\n auto FillF80Result = [&]() {\n if (VTMP1 != VABI1) {\n mov(VTMP1.Q(), VABI1.Q());\n }\n FillForABICall(CTX->HostFeatures.SupportsPreserveAllABI, true);\n };\n\n auto FillF64Result = [&]() {\n if (!TMP_ABIARGS) {\n fmov(VTMP1.D(), VABI1.D());\n }\n FillForABICall(CTX->HostFeatures.SupportsPreserveAllABI, true);\n };\n\n auto FillF32Result = [&]() {\n if (!TMP_ABIARGS) {\n fmov(VTMP1.S(), VABI1.S());\n }\n FillForABICall(CTX->HostFeatures.SupportsPreserveAllABI, true);\n };\n\n auto FillI64Result = [&]() {\n if (!TMP_ABIARGS) {\n mov(TMP1, ARMEmitter::XReg::x0);\n }\n FillForABICall(CTX->HostFeatures.SupportsPreserveAllABI, true);\n };\n\n auto FillI32Result = [&]() {\n if (!TMP_ABIARGS) {\n mov(TMP1.W(), ARMEmitter::WReg::w0);\n }\n FillForABICall(CTX->HostFeatures.SupportsPreserveAllABI, true);\n };\n\n auto FillI16Result = [&]() {\n if (!TMP_ABIARGS) {\n mov(TMP1, ARMEmitter::XReg::x0);\n }\n FillForABICall(CTX->HostFeatures.SupportsPreserveAllABI, true);\n };\n\n switch (ABI) {\n case FABI_F80_I16_F32_PTR: {\n // Save NZCV - it's a static register (guest x86 flags) and the inline code clobbers it\n mrs(TMP1, ARMEmitter::SystemRegister::NZCV);\n str(TMP1.W(), STATE.R(), offsetof(FEXCore::Core::CpuStateFrame, State.flags[24]));\n EmitF32ToExtF80();\n ldr(TMP1.W(), STATE.R(), offsetof(FEXCore::Core::CpuStateFrame, State.flags[24]));\n msr(ARMEmitter::SystemRegister::NZCV, TMP1);\n } break;\n case FABI_F80_I16_F64_PTR: {\n mrs(TMP1, ARMEmitter::SystemRegister::NZCV);\n str(TMP1.W(), STATE.R(), offsetof(FEXCore::Core::CpuStateFrame, State.flags[24]));\n EmitF64ToExtF80();\n ldr(TMP1.W(), STATE.R(), offsetof(FEXCore::Core::CpuStateFrame, State.flags[24]));\n msr(ARMEmitter::SystemRegister::NZCV, TMP1);\n } break;\n case FABI_F80_I16_I16_PTR: {\n mrs(TMP1, ARMEmitter::SystemRegister::NZCV);\n str(TMP1.W(), STATE.R(), offsetof(FEXCore::Core::CpuStateFrame, State.flags[24]));\n EmitI16ToExtF80();\n ldr(TMP1.W(), STATE.R(), offsetof(FEXCore::Core::CpuStateFrame, State.flags[24]));\n msr(ARMEmitter::SystemRegister::NZCV, TMP1);\n } break;\n case FABI_F80_I16_I32_PTR: {\n mrs(TMP1, ARMEmitter::SystemRegister::NZCV);\n str(TMP1.W(), STATE.R(), offsetof(FEXCore::Core::CpuStateFrame, State.flags[24]));\n EmitI32ToExtF80();\n ldr(TMP1.W(), STATE.R(), offsetof(FEXCore::Core::CpuStateFrame, State.flags[24]));\n msr(ARMEmitter::SystemRegister::NZCV, TMP1);\n } break;\n case FABI_F32_I16_F80_PTR: {\n // Linux Reg/Win32 Reg:\n // tmp4 (x4/x13): FallbackHandler\n // x30: return\n // vtmp1 (v0/v16): source\n SpillForABICall(CTX->HostFeatures.SupportsPreserveAllABI, TMP3, true);\n\n ldrh(ARMEmitter::WReg::w0, STATE, offsetof(FEXCore::Core::CPUState, FCW));\n if (!TMP_ABIARGS) {\n mov(VABI1.Q(), VTMP1.Q());\n }\n\n mov(ARMEmitter::XReg::x1, STATE);\n if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] {\n GenerateIndirectRuntimeCall(FallbackPointerReg);\n } else {\n blr(FallbackPointerReg);\n }\n\n FillF32Result();\n } break;\n case FABI_F64_I16_F80_PTR: {\n // Linux Reg/Win32 Reg:\n // tmp4 (x4/x13): FallbackHandler\n // x30: return\n // vtmp1 (v0/v16): source\n SpillForABICall(CTX->HostFeatures.SupportsPreserveAllABI, TMP3, true);\n\n ldrh(ARMEmitter::WReg::w0, STATE, offsetof(FEXCore::Core::CPUState, FCW));\n if (!TMP_ABIARGS) {\n mov(VABI1.Q(), VTMP1.Q());\n }\n mov(ARMEmitter::XReg::x1, STATE);\n\n if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] {\n GenerateIndirectRuntimeCall(FallbackPointerReg);\n } else {\n blr(FallbackPointerReg);\n }\n\n FillF64Result();\n } break;\n case FABI_F64_F64_PTR: {\n // Linux Reg/Win32 Reg:\n // tmp4 (x4/x13): FallbackHandler\n // x30: return\n // vtmp1 (v0/v16): vector source\n SpillForABICall(CTX->HostFeatures.SupportsPreserveAllABI, TMP3, true);\n\n if (!TMP_ABIARGS) {\n fmov(VABI1.D(), VTMP1.D());\n }\n mov(ARMEmitter::XReg::x0, STATE);\n\n if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] {\n GenerateIndirectRuntimeCall(FallbackPointerReg);\n } else {\n blr(FallbackPointerReg);\n }\n\n FillF64Result();\n } break;\n case FABI_F64_F64_F64_PTR: {\n // Linux Reg/Win32 Reg:\n // tmp4 (x4/x13): FallbackHandler\n // x30: return\n // vtmp1 (v0/v16): vector source 1\n // vtmp2 (v1/v17): vector source 2\n SpillForABICall(CTX->HostFeatures.SupportsPreserveAllABI, TMP3, true);\n\n if (!TMP_ABIARGS) {\n fmov(VABI1.D(), VTMP1.D());\n fmov(VABI2.D(), VTMP2.D());\n }\n\n mov(ARMEmitter::XReg::x0, STATE);\n if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] {\n GenerateIndirectRuntimeCall(FallbackPointerReg);\n } else {\n blr(FallbackPointerReg);\n }\n\n FillF64Result();\n } break;\n case FABI_I16_I16_F80_PTR: {\n // Linux Reg/Win32 Reg:\n // tmp4 (x4/x13): FallbackHandler\n // x30: return\n // vtmp1 (v0/v16): source\n SpillForABICall(CTX->HostFeatures.SupportsPreserveAllABI, TMP3, true);\n\n ldrh(ARMEmitter::WReg::w0, STATE, offsetof(FEXCore::Core::CPUState, FCW));\n if (!TMP_ABIARGS) {\n mov(VABI1.Q(), VTMP1.Q());\n }\n mov(ARMEmitter::XReg::x1, STATE);\n\n if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] {\n GenerateIndirectRuntimeCall(FallbackPointerReg);\n } else {\n blr(FallbackPointerReg);\n }\n\n FillI16Result();\n } break;\n case FABI_I32_I16_F80_PTR: {\n // Linux Reg/Win32 Reg:\n // tmp4 (x4/x13): FallbackHandler\n // x30: return\n // vtmp1 (v0/v16): source\n SpillForABICall(CTX->HostFeatures.SupportsPreserveAllABI, TMP3, true);\n\n ldrh(ARMEmitter::WReg::w0, STATE, offsetof(FEXCore::Core::CPUState, FCW));\n if (!TMP_ABIARGS) {\n mov(VABI1.Q(), VTMP1.Q());\n }\n mov(ARMEmitter::XReg::x1, STATE);\n\n if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] {\n GenerateIndirectRuntimeCall(FallbackPointerReg);\n } else {\n blr(FallbackPointerReg);\n }\n\n FillI32Result();\n } break;\n case FABI_I64_I16_F80_PTR: {\n // Linux Reg/Win32 Reg:\n // tmp4 (x4/x13): FallbackHandler\n // x30: return\n // vtmp1 (v0/v16): source\n SpillForABICall(CTX->HostFeatures.SupportsPreserveAllABI, TMP3, true);\n\n ldrh(ARMEmitter::WReg::w0, STATE, offsetof(FEXCore::Core::CPUState, FCW));\n if (!TMP_ABIARGS) {\n mov(VABI1.Q(), VTMP1.Q());\n }\n mov(ARMEmitter::XReg::x1, STATE);\n\n if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] {\n GenerateIndirectRuntimeCall(FallbackPointerReg);\n } else {\n blr(FallbackPointerReg);\n }\n\n FillI64Result();\n } break;\n case FABI_I64_I16_F80_F80_PTR: {\n // Linux Reg/Win32 Reg:\n // tmp4 (x4/x13): FallbackHandler\n // x30: return\n // vtmp1 (v0/v16): vector source 1\n // vtmp2 (v1/v17): vector source 2\n SpillForABICall(CTX->HostFeatures.SupportsPreserveAllABI, TMP3, true);\n\n ldrh(ARMEmitter::WReg::w0, STATE, offsetof(FEXCore::Core::CPUState, FCW));\n if (!TMP_ABIARGS) {\n mov(VABI1.Q(), VTMP1.Q());\n mov(VABI2.Q(), VTMP2.Q());\n }\n mov(ARMEmitter::XReg::x1, STATE);\n\n if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] {\n GenerateIndirectRuntimeCall(FallbackPointerReg);\n } else {\n blr(FallbackPointerReg);\n }\n\n FillI64Result();\n } break;\n case FABI_F80_I16_F80_PTR: {\n // Linux Reg/Win32 Reg:\n // tmp4 (x4/x13): FallbackHandler\n // x30: return\n // vtmp1 (v0/v16): vector source 1\n SpillForABICall(CTX->HostFeatures.SupportsPreserveAllABI, TMP3, true);\n\n ldrh(ARMEmitter::WReg::w0, STATE, offsetof(FEXCore::Core::CPUState, FCW));\n mov(ARMEmitter::XReg::x1, STATE);\n if (!TMP_ABIARGS) {\n mov(VABI1.Q(), VTMP1.Q());\n }\n\n if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] {\n GenerateIndirectRuntimeCall(FallbackPointerReg);\n } else {\n blr(FallbackPointerReg);\n }\n\n FillF80Result();\n } break;\n case FABI_F80_I16_F80_F80_PTR: {\n // Linux Reg/Win32 Reg:\n // tmp4 (x4/x13): FallbackHandler\n // x30: return\n // vtmp1 (v0/v16): vector source 1\n // vtmp2 (v1/v17): vector source 2\n SpillForABICall(CTX->HostFeatures.SupportsPreserveAllABI, TMP3, true);\n\n ldrh(ARMEmitter::WReg::w0, STATE, offsetof(FEXCore::Core::CPUState, FCW));\n if (!TMP_ABIARGS) {\n mov(VABI1.Q(), VTMP1.Q());\n mov(VABI2.Q(), VTMP2.Q());\n }\n mov(ARMEmitter::XReg::x1, STATE);\n\n if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] {\n GenerateIndirectRuntimeCall(FallbackPointerReg);\n } else {\n blr(FallbackPointerReg);\n }\n\n FillF80Result();\n } break;\n case FABI_F80x2_I16_F80_PTR: {\n // Linux Reg/Win32 Reg:\n // tmp4 (x4/x13): FallbackHandler\n // x30: return\n // vtmp1 (v0/v16): vector source 1\n // vtmp2 (v1/v16): vector source 2\n\n SpillForABICall(CTX->HostFeatures.SupportsPreserveAllABI, TMP3, true);\n\n ldrh(ARMEmitter::WReg::w0, STATE, offsetof(FEXCore::Core::CPUState, FCW));\n mov(ARMEmitter::XReg::x1, STATE);\n if (!TMP_ABIARGS) {\n mov(VABI1.Q(), VTMP1.Q());\n }\n\n if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] {\n // GenerateIndirectRuntimeCall(FallbackPointerReg);\n } else {\n blr(FallbackPointerReg);\n }\n\n FillF80x2Result();\n } break;\n case FABI_F64x2_F64_PTR: {\n // Linux Reg/Win32 Reg:\n // tmp4 (x4/x13): FallbackHandler\n // x30: return\n // vtmp1 (v0/v16): vector source 1\n // vtmp2 (v1/v16): vector source 2\n\n SpillForABICall(CTX->HostFeatures.SupportsPreserveAllABI, TMP3, true);\n\n mov(ARMEmitter::XReg::x0, STATE);\n if (!TMP_ABIARGS) {\n fmov(VABI1.D(), VTMP1.D());\n }\n\n if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] {\n // GenerateIndirectRuntimeCall(FallbackPointerReg);\n } else {\n blr(FallbackPointerReg);\n }\n\n FillF64x2Result();\n } break;\n case FABI_I32_I64_I64_V128_V128_I16: {\n // Linux Reg/Win32 Reg:\n // stack: FallbackHandler\n // x30: return\n // vtmp1 (v0/v16): vector source 1\n // vtmp2 (v1/v17): vector source 2\n // tmp1 (x0/x10): source 1\n // tmp2 (x1/x11): source 2\n // tmp3 (x2/x12): source 3\n\n const size_t OriginalSPOffset = SpillForABICall(CTX->HostFeatures.SupportsPreserveAllABI, TMP4, true);\n\n // Load the Fallback handler pointer from the stack.\n ldr(FallbackPointerReg, ARMEmitter::XReg::rsp, OriginalSPOffset);\n\n if (!TMP_ABIARGS) {\n mov(ABI1, TMP1);\n mov(ABI2, TMP2);\n mov(ABI3, TMP3);\n mov(VABI1.Q(), VTMP1.Q());\n mov(VABI2.Q(), VTMP2.Q());\n }\n\n if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] {\n GenerateIndirectRuntimeCall(FallbackPointerReg);\n } else {\n blr(FallbackPointerReg);\n }\n\n FillI32Result();\n } break;\n case FABI_I32_V128_V128_I16: {\n // Linux Reg/Win32 Reg:\n // tmp4 (x4/x13): FallbackHandler\n // x30: return\n // vtmp1 (v0/v16): vector source 1\n // vtmp2 (v1/v17): vector source 2\n // tmp1 (x0/x10): source 1\n SpillForABICall(CTX->HostFeatures.SupportsPreserveAllABI, TMP3, true);\n\n if (!TMP_ABIARGS) {\n mov(VABI1.Q(), VTMP1.Q());\n mov(VABI2.Q(), VTMP2.Q());\n mov(ABI1, TMP1);\n }\n\n if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] {\n GenerateIndirectRuntimeCall(FallbackPointerReg);\n } else {\n blr(FallbackPointerReg);\n }\n\n FillI32Result();\n } break;\n case FABI_UNKNOWN:\n default:\n#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED\n LOGMAN_MSG_A_FMT(\"Unhandled IR Fallback ABI: {}\", ToUnderlying(ABI));\n#endif\n break;\n }\n\n // Return to JIT\n ret();\n\n return Address;\n}\n\nvoid Dispatcher::InitThreadPointers(FEXCore::Core::InternalThreadState* Thread) {\n // Setup dispatcher specific pointers that need to be accessed from JIT code\n {\n auto& Ptrs = Thread->CurrentFrame->Pointers;\n\n Ptrs.DispatcherLoopTop = AbsoluteLoopTopAddress;\n Ptrs.DispatcherLoopTopFillSRA = AbsoluteLoopTopAddressFillSRA;\n Ptrs.DispatcherLoopTopEnterEC = AbsoluteLoopTopAddressEnterEC;\n Ptrs.DispatcherLoopTopEnterECFillSRA = AbsoluteLoopTopAddressEnterECFillSRA;\n Ptrs.ExitFunctionLinker = ExitFunctionLinkerAddress;\n Ptrs.ThreadStopHandlerSpillSRA = ThreadStopHandlerAddressSpillSRA;\n Ptrs.ThreadPauseHandlerSpillSRA = ThreadPauseHandlerAddressSpillSRA;\n Ptrs.GuestSignal_SIGILL = GuestSignal_SIGILL;\n Ptrs.GuestSignal_SIGTRAP = GuestSignal_SIGTRAP;\n Ptrs.GuestSignal_SIGSEGV = GuestSignal_SIGSEGV;\n Ptrs.SignalReturnHandler = SignalHandlerReturnAddress;\n Ptrs.SignalReturnHandlerRT = SignalHandlerReturnAddressRT;\n Ptrs.LUDIVHandler = LUDIVHandlerAddress;\n Ptrs.LDIVHandler = LDIVHandlerAddress;\n Ptrs.F64SinHandler = F64SinHandlerAddress;\n Ptrs.F64CosHandler = F64CosHandlerAddress;\n Ptrs.F64TanHandler = F64TanHandlerAddress;\n\n // Fill in the fallback handlers\n InterpreterOps::FillFallbackIndexPointers(Ptrs.FallbackHandlerPointers, &ABIPointers[0]);\n }\n}\n\nSignalDelegatorConfig Dispatcher::MakeSignalDelegatorConfig() const {\n // PF/AF are the final two SRA registers. We only want GPRs\n const auto GPRCount = uint16_t(StaticRegisters.size() - 2);\n const auto FPRCount = uint16_t(StaticFPRegisters.size());\n\n const auto GetSRAGPRMapping = [GPRCount, this] {\n SignalDelegatorConfig::SRAIndexMapping Mapping {};\n for (size_t i = 0; i < GPRCount; ++i) {\n Mapping[i] = StaticRegisters[i].Idx();\n }\n return Mapping;\n };\n\n const auto GetSRAFPRMapping = [FPRCount, this] {\n SignalDelegatorConfig::SRAIndexMapping Mapping {};\n for (size_t i = 0; i < FPRCount; ++i) {\n Mapping[i] = StaticFPRegisters[i].Idx();\n }\n return Mapping;\n };\n\n return FEXCore::SignalDelegatorConfig {\n .DispatcherBegin = Start,\n .DispatcherEnd = End,\n\n .AbsoluteLoopTopAddress = AbsoluteLoopTopAddress,\n .AbsoluteLoopTopAddressFillSRA = AbsoluteLoopTopAddressFillSRA,\n .SignalHandlerReturnAddress = SignalHandlerReturnAddress,\n .SignalHandlerReturnAddressRT = SignalHandlerReturnAddressRT,\n\n .PauseReturnInstruction = PauseReturnInstruction,\n .ThreadPauseHandlerAddressSpillSRA = ThreadPauseHandlerAddressSpillSRA,\n .ThreadPauseHandlerAddress = ThreadPauseHandlerAddress,\n\n // Stop handlers.\n .ThreadStopHandlerAddressSpillSRA = ThreadStopHandlerAddressSpillSRA,\n .ThreadStopHandlerAddress = ThreadStopHandlerAddress,\n\n // SRA information.\n .SRAGPRCount = GPRCount,\n .SRAFPRCount = FPRCount,\n\n .SRAGPRMapping = GetSRAGPRMapping(),\n .SRAFPRMapping = GetSRAFPRMapping(),\n };\n}\n\nfextl::unique_ptr Dispatcher::Create(FEXCore::Context::ContextImpl* CTX) {\n return fextl::make_unique(CTX);\n}\n\n} // namespace FEXCore::CPU\n" }, { "path": "FEXCore/Source/Interface/Core/Dispatcher/Dispatcher.h", "content": "// SPDX-License-Identifier: MIT\n#pragma once\n\n#include \"Interface/Core/ArchHelpers/Arm64Emitter.h\"\n#include \"Interface/Core/Interpreter/InterpreterOps.h\"\n\n#include \n#include \n\n#include \n#include \n#include \n\nnamespace FEXCore {\nstruct GuestSigAction;\nstruct SignalDelegatorConfig;\n} // namespace FEXCore\n\nnamespace FEXCore::Core {\nstruct CpuStateFrame;\nstruct InternalThreadState;\n} // namespace FEXCore::Core\n\nnamespace FEXCore::Context {\nclass ContextImpl;\n}\n\nnamespace FEXCore::CPU {\n\n#define STATE_PTR(STATE_TYPE, FIELD) STATE.R(), offsetof(FEXCore::Core::STATE_TYPE, FIELD)\n#define STATE_PTR_IDX(STATE_TYPE, FIELD, INDEX) STATE.R(), ARRAY_OFFSETOF(FEXCore::Core::STATE_TYPE, FIELD, INDEX)\n#define FALLBACK_HANDLER_OFFSET(INDEX, FIELD) \\\n STATE.R(), \\\n (ARRAY_OFFSETOF(FEXCore::Core::CpuStateFrame, Pointers.FallbackHandlerPointers, INDEX) + offsetof(FEXCore::Core::FallbackABIInfo, FIELD))\n\nclass Dispatcher final : public Arm64Emitter {\npublic:\n static fextl::unique_ptr Create(FEXCore::Context::ContextImpl* CTX);\n\n Dispatcher(FEXCore::Context::ContextImpl* ctx);\n ~Dispatcher();\n\n void InitThreadPointers(FEXCore::Core::InternalThreadState* Thread);\n\n#ifdef VIXL_SIMULATOR\n void ExecuteDispatch(FEXCore::Core::CpuStateFrame* Frame);\n void ExecuteJITCallback(FEXCore::Core::CpuStateFrame* Frame, uint64_t RIP);\n#else\n void ExecuteDispatch(FEXCore::Core::CpuStateFrame* Frame) {\n DispatchPtr(Frame, false);\n }\n\n void ExecuteJITCallback(FEXCore::Core::CpuStateFrame* Frame, uint64_t RIP) {\n CallbackPtr(Frame, RIP);\n }\n#endif\n\n uint64_t GetExitFunctionLinkerAddress() const {\n return ExitFunctionLinkerAddress;\n }\n\n SignalDelegatorConfig MakeSignalDelegatorConfig() const;\n\nprotected:\n FEXCore::Context::ContextImpl* CTX;\n\n using AsmDispatch = void (*)(FEXCore::Core::CpuStateFrame* Frame, bool SingleInst);\n using JITCallback = void (*)(FEXCore::Core::CpuStateFrame* Frame, uint64_t RIP);\n\n AsmDispatch DispatchPtr;\n JITCallback CallbackPtr;\nprivate:\n /**\n * @name Dispatch Helper functions\n * @{ */\n uint64_t ThreadStopHandlerAddress {};\n uint64_t ThreadStopHandlerAddressSpillSRA {};\n uint64_t AbsoluteLoopTopAddress {};\n uint64_t AbsoluteLoopTopAddressFillSRA {};\n uint64_t AbsoluteLoopTopAddressEnterEC {};\n uint64_t AbsoluteLoopTopAddressEnterECFillSRA {};\n uint64_t ThreadPauseHandlerAddress {};\n uint64_t ThreadPauseHandlerAddressSpillSRA {};\n uint64_t ExitFunctionLinkerAddress {};\n uint64_t SignalHandlerReturnAddress {};\n uint64_t SignalHandlerReturnAddressRT {};\n uint64_t GuestSignal_SIGILL {};\n uint64_t GuestSignal_SIGTRAP {};\n uint64_t GuestSignal_SIGSEGV {};\n\n uint64_t PauseReturnInstruction {};\n std::array ABIPointers {};\n /** @} */\n\n uint64_t Start {};\n uint64_t End {};\n\n // Long division helpers\n uint64_t LUDIVHandlerAddress {};\n uint64_t LDIVHandlerAddress {};\n\n // F64 trig shared handlers\n uint64_t F64SinHandlerAddress {};\n uint64_t F64CosHandlerAddress {};\n uint64_t F64TanHandlerAddress {};\n\n void EmitDispatcher();\n uint64_t GenerateABICall(FallbackABI ABI);\n\n // Inline softfloat conversion emitters - avoid FPCR save/restore overhead\n // These emit ARM64 code that performs the conversion using only integer ops\n void EmitI16ToExtF80();\n void EmitI32ToExtF80();\n void EmitF32ToExtF80();\n void EmitF64ToExtF80();\n\n void EmitF64Sin();\n void EmitF64Cos();\n void EmitF64Tan();\n\n FEX_CONFIG_OPT(DisableL2Cache, DISABLEL2CACHE);\n};\n\n} // namespace FEXCore::CPU\n" }, { "path": "FEXCore/Source/Interface/Core/Frontend.cpp", "content": "// SPDX-License-Identifier: MIT\n/*\n$info$\ntags: frontend|x86-meta-blocks\ndesc: Extracts instruction & block meta info, frontend multiblock logic\n$end_info$\n*/\n\n#include \"Interface/Context/Context.h\"\n#include \"Interface/Core/Frontend.h\"\n#include \"Interface/Core/X86Tables/X86Tables.h\"\n#include \"Interface/Core/LookupCache.h\"\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\nnamespace FEXCore::Frontend {\n#include \"Interface/Core/VSyscall/VSyscall.inc\"\n\nusing namespace FEXCore::X86Tables;\n\nstatic uint32_t MapModRMToReg(uint8_t REX, uint8_t bits, bool HighBits, bool HasREX, bool HasXMM, bool HasMM, uint8_t InvalidOffset = 16) {\n using GPRArray = std::array;\n\n static constexpr GPRArray GPR8BitHighIndexes = {\n // Classical ordering?\n FEXCore::X86State::REG_RAX, FEXCore::X86State::REG_RCX, FEXCore::X86State::REG_RDX, FEXCore::X86State::REG_RBX,\n FEXCore::X86State::REG_RAX, FEXCore::X86State::REG_RCX, FEXCore::X86State::REG_RDX, FEXCore::X86State::REG_RBX,\n FEXCore::X86State::REG_R8, FEXCore::X86State::REG_R9, FEXCore::X86State::REG_R10, FEXCore::X86State::REG_R11,\n FEXCore::X86State::REG_R12, FEXCore::X86State::REG_R13, FEXCore::X86State::REG_R14, FEXCore::X86State::REG_R15,\n };\n\n uint8_t Offset = (REX << 3) | bits;\n\n if (Offset == InvalidOffset) {\n return FEXCore::X86State::REG_INVALID;\n }\n\n if (HasXMM) {\n return FEXCore::X86State::REG_XMM_0 + Offset;\n } else if (HasMM) {\n return FEXCore::X86State::REG_MM_0 + bits; // Ignore REX extension for MMX registers\n } else if (!(HighBits && !HasREX)) {\n return FEXCore::X86State::REG_RAX + Offset;\n }\n\n return GPR8BitHighIndexes[Offset];\n}\n\nstatic uint32_t MapVEXToReg(uint8_t vvvv, bool HasXMM) {\n if (HasXMM) {\n return FEXCore::X86State::REG_XMM_0 + vvvv;\n } else {\n return FEXCore::X86State::REG_RAX + vvvv;\n }\n}\n\nDecoder::Decoder(FEXCore::Core::InternalThreadState* Thread)\n : Thread {Thread}\n , CTX {static_cast(Thread->CTX)}\n , OSABI {CTX->SyscallHandler ? CTX->SyscallHandler->GetOSABI() : FEXCore::HLE::SyscallOSABI::OS_UNKNOWN}\n , PoolObject {CTX->FrontendAllocator, sizeof(FEXCore::X86Tables::DecodedInst) * DefaultDecodedBufferSize} {\n\n FEX_CONFIG_OPT(ReducedPrecision, X87REDUCEDPRECISION);\n if (ReducedPrecision) {\n X87Table = &FEXCore::X86Tables::X87F64Ops;\n } else {\n X87Table = &FEXCore::X86Tables::X87F80Ops;\n }\n\n if (CTX->HostFeatures.SupportsAVX && CTX->HostFeatures.SupportsSVE256) {\n VEXTable = &FEXCore::X86Tables::VEXTableOps;\n VEXTableGroup = &FEXCore::X86Tables::VEXTableGroupOps;\n } else if (CTX->HostFeatures.SupportsAVX) {\n VEXTable = &FEXCore::X86Tables::VEXTableOps_AVX128;\n VEXTableGroup = &FEXCore::X86Tables::VEXTableGroupOps_AVX128;\n }\n}\n\nbool Decoder::CheckRangeExecutable(uint64_t Address, uint64_t Size) {\n while (Address < ExecutableRangeBase || Address + Size > ExecutableRangeEnd) {\n auto RangeInfo = CTX->SyscallHandler->QueryGuestExecutableRange(Thread, Address);\n ExecutableRangeBase = RangeInfo.Base;\n ExecutableRangeEnd = RangeInfo.Base + RangeInfo.Size;\n ExecutableRangeWritable = RangeInfo.Writable;\n\n if (RangeInfo.Size == 0) {\n return false;\n }\n\n uint64_t RangeRemainingSize = ExecutableRangeEnd - Address;\n if (Size > RangeRemainingSize) {\n Size -= RangeRemainingSize;\n Address += RangeRemainingSize;\n }\n }\n\n return true;\n}\n\nuint8_t Decoder::ReadByte() {\n LOGMAN_THROW_A_FMT(InstructionSize < MAX_INST_SIZE, \"Max instruction size exceeded!\");\n std::optional Byte = PeekByte(0);\n if (!Byte) {\n HitNonExecutableRange = true;\n // Pretend we read 0, the main decode loop will see HitNonExecutableRange and rollback the instruction.\n return 0;\n }\n\n Instruction[InstructionSize] = *Byte;\n InstructionSize++;\n return *Byte;\n}\n\nstd::optional Decoder::PeekByte(uint8_t Offset) {\n uint64_t ByteAddress = reinterpret_cast(InstStream + InstructionSize + Offset);\n if (CheckRangeExecutable(ByteAddress, 1)) {\n return InstStream[InstructionSize + Offset];\n } else {\n return std::nullopt;\n }\n}\n\nstd::pair Decoder::ReadData(uint8_t Size) {\n LOGMAN_THROW_A_FMT(Size != 0 && Size <= sizeof(uint64_t), \"Unknown data size to read\");\n\n uint64_t Res = 0;\n uint64_t Address = reinterpret_cast(InstStream + InstructionSize);\n if (CheckRangeExecutable(Address, Size)) {\n std::memcpy(&Res, &InstStream[InstructionSize], Size);\n } else {\n HitNonExecutableRange = true;\n // See PeekByte, this specific case may cause some executable memory to read as 0 but it doesn't matter as the entire instruction will be rolled back anyway.\n Res = 0;\n }\n\n\n#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED\n for (size_t i = 0; i < Size; ++i) {\n ReadByte();\n }\n#else\n SkipBytes(Size);\n#endif\n\n if (Relocations) {\n uint32_t SectionOffset = static_cast(Address - SectionMinAddress);\n if (auto It = Relocations->find(SectionOffset); It != Relocations->end()) {\n if (It->second == GuestRelocationType::Rel32 && Size == 4) {\n return {static_cast(static_cast(Res) - static_cast(EntryPoint)), true};\n } else if (It->second == GuestRelocationType::Rel64 && Size == 8) {\n return {static_cast(Res) - static_cast(EntryPoint), true};\n } else {\n HitBadRelocation = true;\n Res = 0;\n }\n }\n }\n\n return {Res, false};\n}\n\nvoid Decoder::DecodeModRM_16(X86Tables::DecodedOperand* Operand, X86Tables::ModRMDecoded ModRM) {\n // 16bit modrm behaves similar to SIB but encoded directly in modrm\n // mod != 0b11 case\n // RM | Result\n // ===============\n // 0b000 | [BX + SI]\n // 0b001 | [BX + DI]\n // 0b010 | [BP + SI]\n // 0b011 | [BP + DI]\n // 0b100 | [SI]\n // 0b101 | [DI]\n // 0b110 | {[BP], disp16}\n // 0b111 | [BX]\n // if mod = 0b00\n // 0b110 = disp16\n // if mod = 0b01\n // All encodings gain 8bit displacement\n // 0b110 = [BP] + disp8\n // if mod = 0b10\n // All encodings gain 16bit displacement\n // 0b110 = [BP] + disp16\n uint32_t Literal {};\n uint8_t DisplacementSize {};\n if ((ModRM.mod == 0 && ModRM.rm == 0b110) || ModRM.mod == 0b10) {\n DisplacementSize = 2;\n } else if (ModRM.mod == 0b01) {\n DisplacementSize = 1;\n }\n if (DisplacementSize) {\n bool IsRelocation = false;\n std::tie(Literal, IsRelocation) = ReadData(DisplacementSize);\n LOGMAN_THROW_A_FMT(!IsRelocation, \"1/2 byte relocations unsupported\");\n if (DisplacementSize == 1) {\n Literal = static_cast(Literal);\n }\n }\n\n Operand->Type = DecodedOperand::OpType::SIB;\n Operand->Data.SIB.Scale = 1;\n Operand->Data.SIB.Offset = Literal;\n\n // Only called when ModRM.mod != 0b11\n struct Encodings {\n uint8_t Base;\n uint8_t Index;\n };\n constexpr static std::array Lookup = {{\n // Mod = 0b00\n {FEXCore::X86State::REG_RBX, FEXCore::X86State::REG_RSI},\n {FEXCore::X86State::REG_RBX, FEXCore::X86State::REG_RDI},\n {FEXCore::X86State::REG_RBP, FEXCore::X86State::REG_RSI},\n {FEXCore::X86State::REG_RBP, FEXCore::X86State::REG_RDI},\n {FEXCore::X86State::REG_RSI, FEXCore::X86State::REG_INVALID},\n {FEXCore::X86State::REG_RDI, FEXCore::X86State::REG_INVALID},\n {FEXCore::X86State::REG_INVALID, FEXCore::X86State::REG_INVALID},\n {FEXCore::X86State::REG_RBX, FEXCore::X86State::REG_INVALID},\n // Mod = 0b01\n {FEXCore::X86State::REG_RBX, FEXCore::X86State::REG_RSI},\n {FEXCore::X86State::REG_RBX, FEXCore::X86State::REG_RDI},\n {FEXCore::X86State::REG_RBP, FEXCore::X86State::REG_RSI},\n {FEXCore::X86State::REG_RBP, FEXCore::X86State::REG_RDI},\n {FEXCore::X86State::REG_RSI, FEXCore::X86State::REG_INVALID},\n {FEXCore::X86State::REG_RDI, FEXCore::X86State::REG_INVALID},\n {FEXCore::X86State::REG_RBP, FEXCore::X86State::REG_INVALID},\n {FEXCore::X86State::REG_RBX, FEXCore::X86State::REG_INVALID},\n // Mod = 0b10\n {FEXCore::X86State::REG_RBX, FEXCore::X86State::REG_RSI},\n {FEXCore::X86State::REG_RBX, FEXCore::X86State::REG_RDI},\n {FEXCore::X86State::REG_RBP, FEXCore::X86State::REG_RSI},\n {FEXCore::X86State::REG_RBP, FEXCore::X86State::REG_RDI},\n {FEXCore::X86State::REG_RSI, FEXCore::X86State::REG_INVALID},\n {FEXCore::X86State::REG_RDI, FEXCore::X86State::REG_INVALID},\n {FEXCore::X86State::REG_RBP, FEXCore::X86State::REG_INVALID},\n {FEXCore::X86State::REG_RBX, FEXCore::X86State::REG_INVALID},\n }};\n\n uint8_t LookupIndex = ModRM.mod << 3 | ModRM.rm;\n auto it = Lookup[LookupIndex];\n Operand->Data.SIB.Base = it.Base;\n Operand->Data.SIB.Index = it.Index;\n}\n\nvoid Decoder::DecodeModRM_64(X86Tables::DecodedOperand* Operand, X86Tables::ModRMDecoded ModRM) {\n uint8_t Displacement {};\n // Do we have an offset?\n if (ModRM.mod == 0b01) {\n Displacement = 1;\n } else if (ModRM.mod == 0b10) {\n Displacement = 4;\n } else if (ModRM.mod == 0 && ModRM.rm == 0b101) {\n Displacement = 4;\n }\n\n // Calculate SIB\n bool HasSIB = ((ModRM.mod != 0b11) && (ModRM.rm == 0b100));\n\n if (HasSIB) {\n FEXCore::X86Tables::SIBDecoded SIB;\n if (DecodeInst->Flags & DecodeFlags::FLAG_DECODED_SIB) {\n SIB.Hex = DecodeInst->SIB;\n } else {\n // Haven't yet grabbed SIB, pull it now\n DecodeInst->SIB = ReadByte();\n SIB.Hex = DecodeInst->SIB;\n DecodeInst->Flags |= DecodeFlags::FLAG_DECODED_SIB;\n }\n\n // If the SIB base is 0b101, aka BP or R13 then we have a 32bit displacement\n if (ModRM.mod == 0b00 && ModRM.rm == 0b100 && SIB.base == 0b101) {\n Displacement = 4;\n }\n\n // SIB\n Operand->Type = DecodedOperand::OpType::SIB;\n Operand->Data.SIB.Scale = 1 << SIB.scale;\n\n // The invalid encoding types are described at Table 1-12. \"promoted nsigned is always non-zero\"\n {\n // If we have a VSIB byte (as opposed to SIB), then the index register is a vector.\n // DecodeInst->TableInfo may be null in the case of 3DNow! ModRM decoding.\n const bool IsIndexVector = DecodeInst->TableInfo && (DecodeInst->TableInfo->Flags & InstFlags::FLAGS_VEX_VSIB) != 0;\n uint8_t InvalidSIBIndex = 0b100; ///< SIB Index where there is no register encoding.\n if (IsIndexVector) {\n DecodeInst->Flags |= X86Tables::DecodeFlags::FLAG_VSIB_BYTE;\n InvalidSIBIndex = ~0; ///< No Invalid SIB Index with Index Vectors.\n }\n\n const uint8_t IndexREX = (DecodeInst->Flags & DecodeFlags::FLAG_REX_XGPR_X) != 0 ? 1 : 0;\n const uint8_t BaseREX = (DecodeInst->Flags & DecodeFlags::FLAG_REX_XGPR_B) != 0 ? 1 : 0;\n\n Operand->Data.SIB.Index = MapModRMToReg(IndexREX, SIB.index, false, false, IsIndexVector, false, InvalidSIBIndex);\n Operand->Data.SIB.Base = MapModRMToReg(BaseREX, SIB.base, false, false, false, false, ModRM.mod == 0 ? 0b101 : 16);\n }\n\n LOGMAN_THROW_A_FMT(Displacement <= 4, \"Number of bytes should be <= 4 for literal src\");\n\n if (Displacement) {\n auto [Literal, IsRelocation] = ReadData(Displacement);\n if (IsRelocation) {\n Operand->Type = DecodedOperand::OpType::SIBRelocation;\n }\n if (Displacement == 1) {\n Literal = static_cast(Literal);\n }\n Operand->Data.SIB.Offset = Literal;\n }\n } else if (ModRM.mod == 0) {\n // Explained in Table 1-14. \"Operand Addressing Using ModRM and SIB Bytes\"\n if (ModRM.rm == 0b101) {\n // 32bit Displacement\n auto [Literal, IsRelocation] = ReadData(4);\n Operand->Type = IsRelocation ? DecodedOperand::OpType::RIPRelativeRelocation : DecodedOperand::OpType::RIPRelative;\n Operand->Data.RIPLiteral.Value = Literal;\n } else {\n // Register-direct addressing\n Operand->Type = DecodedOperand::OpType::GPRDirect;\n Operand->Data.GPR.GPR = MapModRMToReg(DecodeInst->Flags & DecodeFlags::FLAG_REX_XGPR_B ? 1 : 0, ModRM.rm, false, false, false, false);\n }\n } else {\n uint8_t DisplacementSize = ModRM.mod == 1 ? 1 : 4;\n auto [Literal, IsRelocation] = ReadData(DisplacementSize);\n if (DisplacementSize == 1) {\n Literal = static_cast(Literal);\n }\n\n Operand->Type = IsRelocation ? DecodedOperand::OpType::GPRIndirectRelocation : DecodedOperand::OpType::GPRIndirect;\n Operand->Data.GPRIndirect.GPR = MapModRMToReg(DecodeInst->Flags & DecodeFlags::FLAG_REX_XGPR_B ? 1 : 0, ModRM.rm, false, false, false, false);\n Operand->Data.GPRIndirect.Displacement = Literal;\n }\n}\n\nbool Decoder::NormalOp(const FEXCore::X86Tables::X86InstInfo* Info, uint16_t Op, DecodedHeader Options) {\n if (Info->Type == FEXCore::X86Tables::TYPE_ARCH_DISPATCHER) [[unlikely]] {\n // Dispatcher Op.\n // TODO: Move this in to `NormalOpHeader`, Dispatch tables have a bug currently where some subtables don't inherit flags correctly.\n // Can be seen by running FEX asm tests if this is removed.\n return NormalOp(&Info->OpcodeDispatcher.Indirect[BlockInfo.Is64BitMode ? 1 : 0], Op);\n }\n\n DecodeInst->OP = Op;\n DecodeInst->TableInfo = Info;\n\n if (Info->Type == FEXCore::X86Tables::TYPE_UNKNOWN) {\n return false;\n }\n\n if (Info->Type == FEXCore::X86Tables::TYPE_INVALID) {\n return false;\n }\n\n LOGMAN_THROW_A_FMT(!(Info->Type >= FEXCore::X86Tables::TYPE_GROUP_1 && Info->Type <= FEXCore::X86Tables::TYPE_GROUP_P), \"Group Ops \"\n \"should have \"\n \"been decoded \"\n \"before this!\");\n\n uint8_t DestSize {};\n const bool HasWideningDisplacement =\n (FEXCore::X86Tables::DecodeFlags::GetOpAddr(DecodeInst->Flags, 0) & FEXCore::X86Tables::DecodeFlags::FLAG_WIDENING_SIZE_LAST) != 0 ||\n (Options.w && BlockInfo.Is64BitMode);\n const bool HasNarrowingDisplacement =\n (FEXCore::X86Tables::DecodeFlags::GetOpAddr(DecodeInst->Flags, 0) & FEXCore::X86Tables::DecodeFlags::FLAG_OPERAND_SIZE_LAST) != 0;\n\n const bool HasXMMFlags = (Info->Flags & InstFlags::FLAGS_XMM_FLAGS) != 0;\n bool HasXMMSrc =\n HasXMMFlags && !HAS_XMM_SUBFLAG(Info->Flags, InstFlags::FLAGS_SF_SRC_GPR) && !HAS_XMM_SUBFLAG(Info->Flags, InstFlags::FLAGS_SF_MMX_SRC);\n bool HasXMMDst =\n HasXMMFlags && !HAS_XMM_SUBFLAG(Info->Flags, InstFlags::FLAGS_SF_DST_GPR) && !HAS_XMM_SUBFLAG(Info->Flags, InstFlags::FLAGS_SF_MMX_DST);\n bool HasMMSrc =\n HasXMMFlags && !HAS_XMM_SUBFLAG(Info->Flags, InstFlags::FLAGS_SF_SRC_GPR) && HAS_XMM_SUBFLAG(Info->Flags, InstFlags::FLAGS_SF_MMX_SRC);\n bool HasMMDst =\n HasXMMFlags && !HAS_XMM_SUBFLAG(Info->Flags, InstFlags::FLAGS_SF_DST_GPR) && HAS_XMM_SUBFLAG(Info->Flags, InstFlags::FLAGS_SF_MMX_DST);\n\n // Is ModRM present via explicit instruction encoded or REX?\n const bool HasMODRM = !!(Info->Flags & FEXCore::X86Tables::InstFlags::FLAGS_MODRM);\n\n const bool HasREX = !!(DecodeInst->Flags & DecodeFlags::FLAG_REX_PREFIX);\n const bool Has16BitAddressing = !BlockInfo.Is64BitMode && DecodeInst->Flags & DecodeFlags::FLAG_ADDRESS_SIZE;\n\n if (Options.w && (Info->Flags & InstFlags::FLAGS_REX_W_0)) {\n return false;\n } else if (!Options.w && (Info->Flags & InstFlags::FLAGS_REX_W_1)) {\n return false;\n }\n\n if (Options.L && (Info->Flags & InstFlags::FLAGS_VEX_L_0)) {\n return false;\n } else if (!Options.L && (Info->Flags & InstFlags::FLAGS_VEX_L_1)) {\n return false;\n }\n\n const bool UseVEXL = Options.L && !(Info->Flags & InstFlags::FLAGS_VEX_L_IGNORE);\n\n // This is used for ModRM register modification\n // For both modrm.reg and modrm.rm(when mod == 0b11) when value is >= 0b100\n // then it changes from expected registers to the high 8bits of the lower registers\n // Bit annoying to support\n // In the case of no modrm (REX in byte situation) then it is unaffected\n bool Is8BitSrc {};\n bool Is8BitDest {};\n\n // If we require ModRM and haven't decoded it yet, do it now\n // Some instructions have to read modrm upfront, others do it later\n if (HasMODRM && !(DecodeInst->Flags & DecodeFlags::FLAG_DECODED_MODRM)) {\n DecodeInst->ModRM = ReadByte();\n DecodeInst->Flags |= DecodeFlags::FLAG_DECODED_MODRM;\n }\n\n // New instruction size decoding\n {\n // Decode destinations first\n const auto DstSizeFlag = FEXCore::X86Tables::InstFlags::GetSizeDstFlags(Info->Flags);\n const auto SrcSizeFlag = FEXCore::X86Tables::InstFlags::GetSizeSrcFlags(Info->Flags);\n\n if (DstSizeFlag == FEXCore::X86Tables::InstFlags::SIZE_8BIT) {\n DecodeInst->Flags |= DecodeFlags::GenSizeDstSize(DecodeFlags::SIZE_8BIT);\n DestSize = 1;\n Is8BitDest = true;\n } else if (DstSizeFlag == FEXCore::X86Tables::InstFlags::SIZE_16BIT) {\n DecodeInst->Flags |= DecodeFlags::GenSizeDstSize(DecodeFlags::SIZE_16BIT);\n DestSize = 2;\n } else if (DstSizeFlag == FEXCore::X86Tables::InstFlags::SIZE_128BIT) {\n if (UseVEXL) {\n DecodeInst->Flags |= DecodeFlags::GenSizeDstSize(DecodeFlags::SIZE_256BIT);\n DestSize = 32;\n } else {\n DecodeInst->Flags |= DecodeFlags::GenSizeDstSize(DecodeFlags::SIZE_128BIT);\n DestSize = 16;\n }\n } else if (DstSizeFlag == FEXCore::X86Tables::InstFlags::SIZE_256BIT) {\n DecodeInst->Flags |= DecodeFlags::GenSizeDstSize(DecodeFlags::SIZE_256BIT);\n DestSize = 32;\n } else if (HasNarrowingDisplacement &&\n (DstSizeFlag == FEXCore::X86Tables::InstFlags::SIZE_DEF || DstSizeFlag == FEXCore::X86Tables::InstFlags::SIZE_64BITDEF)) {\n // See table 1-2. Operand-Size Overrides for this decoding\n // If the default operating mode is 32bit and we have the operand size flag then the operating size drops to 16bit\n DecodeInst->Flags |= DecodeFlags::GenSizeDstSize(DecodeFlags::SIZE_16BIT);\n DestSize = 2;\n } else if ((HasXMMDst || HasMMDst || BlockInfo.Is64BitMode) && (HasWideningDisplacement || DstSizeFlag == FEXCore::X86Tables::InstFlags::SIZE_64BIT ||\n DstSizeFlag == FEXCore::X86Tables::InstFlags::SIZE_64BITDEF)) {\n DecodeInst->Flags |= DecodeFlags::GenSizeDstSize(DecodeFlags::SIZE_64BIT);\n DestSize = 8;\n } else {\n DecodeInst->Flags |= DecodeFlags::GenSizeDstSize(DecodeFlags::SIZE_32BIT);\n DestSize = 4;\n }\n\n // Decode sources\n if (SrcSizeFlag == FEXCore::X86Tables::InstFlags::SIZE_8BIT) {\n DecodeInst->Flags |= DecodeFlags::GenSizeSrcSize(DecodeFlags::SIZE_8BIT);\n Is8BitSrc = true;\n } else if (SrcSizeFlag == FEXCore::X86Tables::InstFlags::SIZE_16BIT) {\n DecodeInst->Flags |= DecodeFlags::GenSizeSrcSize(DecodeFlags::SIZE_16BIT);\n } else if (SrcSizeFlag == FEXCore::X86Tables::InstFlags::SIZE_128BIT) {\n if (UseVEXL) {\n DecodeInst->Flags |= DecodeFlags::GenSizeSrcSize(DecodeFlags::SIZE_256BIT);\n } else {\n DecodeInst->Flags |= DecodeFlags::GenSizeSrcSize(DecodeFlags::SIZE_128BIT);\n }\n } else if (SrcSizeFlag == FEXCore::X86Tables::InstFlags::SIZE_256BIT) {\n DecodeInst->Flags |= DecodeFlags::GenSizeSrcSize(DecodeFlags::SIZE_256BIT);\n } else if (HasNarrowingDisplacement &&\n (SrcSizeFlag == FEXCore::X86Tables::InstFlags::SIZE_DEF || SrcSizeFlag == FEXCore::X86Tables::InstFlags::SIZE_64BITDEF)) {\n // See table 1-2. Operand-Size Overrides for this decoding\n // If the default operating mode is 32bit and we have the operand size flag then the operating size drops to 16bit\n DecodeInst->Flags |= DecodeFlags::GenSizeSrcSize(DecodeFlags::SIZE_16BIT);\n } else if ((HasXMMSrc || HasMMSrc || BlockInfo.Is64BitMode) && (HasWideningDisplacement || SrcSizeFlag == FEXCore::X86Tables::InstFlags::SIZE_64BIT ||\n SrcSizeFlag == FEXCore::X86Tables::InstFlags::SIZE_64BITDEF)) {\n DecodeInst->Flags |= DecodeFlags::GenSizeSrcSize(DecodeFlags::SIZE_64BIT);\n } else {\n DecodeInst->Flags |= DecodeFlags::GenSizeSrcSize(DecodeFlags::SIZE_32BIT);\n }\n }\n\n auto* CurrentDest = &DecodeInst->Dest;\n\n if (HAS_NON_XMM_SUBFLAG(Info->Flags, FEXCore::X86Tables::InstFlags::FLAGS_SF_DST_RAX) ||\n HAS_NON_XMM_SUBFLAG(Info->Flags, FEXCore::X86Tables::InstFlags::FLAGS_SF_DST_RDX)) {\n // Some instructions hardcode their destination as RAX\n CurrentDest->Type = DecodedOperand::OpType::GPR;\n CurrentDest->Data.GPR.HighBits = false;\n CurrentDest->Data.GPR.GPR =\n HAS_NON_XMM_SUBFLAG(Info->Flags, FEXCore::X86Tables::InstFlags::FLAGS_SF_DST_RAX) ? FEXCore::X86State::REG_RAX : FEXCore::X86State::REG_RDX;\n CurrentDest = &DecodeInst->Src[0];\n } else if (HAS_NON_XMM_SUBFLAG(Info->Flags, FEXCore::X86Tables::InstFlags::FLAGS_SF_REX_IN_BYTE)) {\n LOGMAN_THROW_A_FMT(!HasMODRM, \"This instruction shouldn't have ModRM!\");\n\n // If the REX is in the byte that means the lower nibble of the OP contains the destination GPR\n // This also means that the destination is always a GPR on these ones\n // ADDITIONALLY:\n // If there is a REX prefix then that allows extended GPR usage\n CurrentDest->Type = DecodedOperand::OpType::GPR;\n DecodeInst->Dest.Data.GPR.HighBits = (Is8BitDest && !HasREX && (Op & 0b111) >= 0b100);\n CurrentDest->Data.GPR.GPR =\n MapModRMToReg(DecodeInst->Flags & DecodeFlags::FLAG_REX_XGPR_B ? 1 : 0, Op & 0b111, Is8BitDest, HasREX, false, false);\n\n if (CurrentDest->Data.GPR.GPR == FEXCore::X86State::REG_INVALID) {\n return false;\n }\n }\n\n uint8_t Bytes = Info->MoreBytes;\n\n if ((Info->Flags & FEXCore::X86Tables::InstFlags::FLAGS_DISPLACE_SIZE_MUL_2) && HasWideningDisplacement) {\n Bytes <<= 1;\n }\n if ((Info->Flags & FEXCore::X86Tables::InstFlags::FLAGS_DISPLACE_SIZE_DIV_2) && HasNarrowingDisplacement) {\n Bytes >>= 1;\n }\n\n if ((Info->Flags & FEXCore::X86Tables::InstFlags::FLAGS_MEM_OFFSET) && (DecodeInst->Flags & DecodeFlags::FLAG_ADDRESS_SIZE)) {\n // If we have a memory offset and have the address size override then divide it just like narrowing displacement\n Bytes >>= 1;\n }\n\n auto ModRMOperand = [&](FEXCore::X86Tables::DecodedOperand& GPR, FEXCore::X86Tables::DecodedOperand& NonGPR, bool HasXMMGPR,\n bool HasXMMNonGPR, bool HasMMGPR, bool HasMMNonGPR, bool GPR8Bit, bool NonGPR8Bit) {\n FEXCore::X86Tables::ModRMDecoded ModRM;\n ModRM.Hex = DecodeInst->ModRM;\n\n if (ModRM.reg != 0b000 && (Info->Flags & FEXCore::X86Tables::InstFlags::FLAGS_SF_MOD_ZERO_REG)) {\n return false;\n }\n\n if (ModRM.mod == 0b11 && (Info->Flags & FEXCore::X86Tables::InstFlags::FLAGS_SF_MOD_MEM_ONLY)) {\n return false;\n }\n\n if (ModRM.mod != 0b11 && (Info->Flags & FEXCore::X86Tables::InstFlags::FLAGS_SF_MOD_REG_ONLY)) {\n return false;\n }\n\n // Decode the GPR source first\n GPR.Type = DecodedOperand::OpType::GPR;\n GPR.Data.GPR.HighBits = (GPR8Bit && ModRM.reg >= 0b100 && !HasREX);\n GPR.Data.GPR.GPR = MapModRMToReg(DecodeInst->Flags & DecodeFlags::FLAG_REX_XGPR_R ? 1 : 0, ModRM.reg, GPR8Bit, HasREX, HasXMMGPR, HasMMGPR);\n\n if (GPR.Data.GPR.GPR == FEXCore::X86State::REG_INVALID) {\n return false;\n }\n\n // ModRM.mod == 0b11 == Register\n // ModRM.Mod != 0b11 == Register-direct addressing\n if (ModRM.mod == 0b11) {\n NonGPR.Type = DecodedOperand::OpType::GPR;\n NonGPR.Data.GPR.HighBits = (NonGPR8Bit && ModRM.rm >= 0b100 && !HasREX);\n NonGPR.Data.GPR.GPR =\n MapModRMToReg(DecodeInst->Flags & DecodeFlags::FLAG_REX_XGPR_B ? 1 : 0, ModRM.rm, NonGPR8Bit, HasREX, HasXMMNonGPR, HasMMNonGPR);\n if (NonGPR.Data.GPR.GPR == FEXCore::X86State::REG_INVALID) {\n return false;\n }\n } else {\n // Only decode if we haven't pre-decoded\n if (NonGPR.IsNone()) {\n auto Disp = DecodeModRMs_Disp[Has16BitAddressing];\n (this->*Disp)(&NonGPR, ModRM);\n }\n }\n\n return true;\n };\n\n size_t CurrentSrc = 0;\n\n const auto VEXOperand = Info->Flags & FEXCore::X86Tables::InstFlags::FLAGS_VEX_SRC_MASK;\n if (VEXOperand == FEXCore::X86Tables::InstFlags::FLAGS_VEX_NO_OPERAND && Options.vvvv) {\n return false;\n }\n\n if (VEXOperand == FEXCore::X86Tables::InstFlags::FLAGS_VEX_1ST_SRC) {\n DecodeInst->Src[CurrentSrc].Type = DecodedOperand::OpType::GPR;\n DecodeInst->Src[CurrentSrc].Data.GPR.HighBits = false;\n\n // If we have XMM flags at all, then SRC 1 cannot be a GPR. The only case where\n // this is possible is with BMI1 and BMI2 instructions (which are all GPR-based\n // and don't use XMM flags)\n DecodeInst->Src[CurrentSrc].Data.GPR.GPR = MapVEXToReg(Options.vvvv, HasXMMFlags);\n\n ++CurrentSrc;\n }\n\n if (Info->Flags & FEXCore::X86Tables::InstFlags::FLAGS_MODRM) {\n if (Info->Flags & FEXCore::X86Tables::InstFlags::FLAGS_SF_MOD_DST) {\n if (!ModRMOperand(DecodeInst->Src[CurrentSrc], DecodeInst->Dest, HasXMMSrc, HasXMMDst, HasMMSrc, HasMMDst, Is8BitSrc, Is8BitDest)) {\n return false;\n }\n } else {\n if (!ModRMOperand(DecodeInst->Dest, DecodeInst->Src[CurrentSrc], HasXMMDst, HasXMMSrc, HasMMDst, HasMMSrc, Is8BitDest, Is8BitSrc)) {\n return false;\n }\n }\n ++CurrentSrc;\n }\n\n if (VEXOperand == FEXCore::X86Tables::InstFlags::FLAGS_VEX_2ND_SRC) {\n DecodeInst->Src[CurrentSrc].Type = DecodedOperand::OpType::GPR;\n DecodeInst->Src[CurrentSrc].Data.GPR.HighBits = false;\n DecodeInst->Src[CurrentSrc].Data.GPR.GPR = MapVEXToReg(Options.vvvv, HasXMMSrc);\n ++CurrentSrc;\n }\n\n if (HAS_NON_XMM_SUBFLAG(Info->Flags, FEXCore::X86Tables::InstFlags::FLAGS_SF_SRC_RAX)) {\n DecodeInst->Src[CurrentSrc].Type = DecodedOperand::OpType::GPR;\n DecodeInst->Src[CurrentSrc].Data.GPR.HighBits = false;\n DecodeInst->Src[CurrentSrc].Data.GPR.GPR = FEXCore::X86State::REG_RAX;\n ++CurrentSrc;\n } else if (HAS_NON_XMM_SUBFLAG(Info->Flags, FEXCore::X86Tables::InstFlags::FLAGS_SF_SRC_RCX)) {\n DecodeInst->Src[CurrentSrc].Type = DecodedOperand::OpType::GPR;\n DecodeInst->Src[CurrentSrc].Data.GPR.HighBits = false;\n DecodeInst->Src[CurrentSrc].Data.GPR.GPR = FEXCore::X86State::REG_RCX;\n ++CurrentSrc;\n }\n\n if (VEXOperand == FEXCore::X86Tables::InstFlags::FLAGS_VEX_DST) {\n CurrentDest->Type = DecodedOperand::OpType::GPR;\n CurrentDest->Data.GPR.HighBits = false;\n CurrentDest->Data.GPR.GPR = MapVEXToReg(Options.vvvv, HasXMMDst);\n }\n\n if (Bytes != 0) {\n LOGMAN_THROW_A_FMT(Bytes <= 8, \"Number of bytes should be <= 8 for literal src\");\n\n\n auto [Literal, IsRelocation] = ReadData(Bytes);\n if (IsRelocation) {\n DecodeInst->Src[CurrentSrc].Type = DecodedOperand::OpType::LiteralRelocation;\n DecodeInst->Src[CurrentSrc].Data.LiteralRelocation.EntrypointOffset = Literal;\n } else {\n DecodeInst->Src[CurrentSrc].Data.Literal.Size = Bytes;\n\n if ((Info->Flags & FEXCore::X86Tables::InstFlags::FLAGS_SRC_SEXT) ||\n (DecodeFlags::GetSizeDstFlags(DecodeInst->Flags) == DecodeFlags::SIZE_64BIT &&\n Info->Flags & FEXCore::X86Tables::InstFlags::FLAGS_SRC_SEXT64BIT)) {\n if (Bytes == 1) {\n Literal = static_cast(Literal);\n } else if (Bytes == 2) {\n Literal = static_cast(Literal);\n } else {\n Literal = static_cast(Literal);\n }\n DecodeInst->Src[CurrentSrc].Data.Literal.Size = DestSize;\n }\n\n DecodeInst->Src[CurrentSrc].Type = DecodedOperand::OpType::Literal;\n DecodeInst->Src[CurrentSrc].Data.Literal.Value = Literal;\n }\n\n Bytes = 0;\n }\n\n LOGMAN_THROW_A_FMT(Bytes == 0, \"Inst at 0x{:x}: 0x{:04x} '{}' Had an instruction of size {} with {} remaining\", DecodeInst->PC,\n DecodeInst->OP, DecodeInst->TableInfo->Name ?: \"UND\", InstructionSize, Bytes);\n DecodeInst->InstSize = InstructionSize;\n return true;\n}\n\nbool Decoder::NormalOpHeader(const FEXCore::X86Tables::X86InstInfo* Info, uint16_t Op) {\n DecodeInst->OPRaw = DecodeInst->OP = Op;\n DecodeInst->TableInfo = Info;\n\n if (Info->Type == FEXCore::X86Tables::TYPE_UNKNOWN) {\n return false;\n }\n\n if (Info->Type == FEXCore::X86Tables::TYPE_INVALID) {\n return false;\n }\n\n LOGMAN_THROW_A_FMT(Info->Type != FEXCore::X86Tables::TYPE_REX_PREFIX, \"REX PREFIX should have been decoded before this!\");\n\n // A normal instruction is the most likely.\n if (Info->Type == FEXCore::X86Tables::TYPE_INST) [[likely]] {\n return NormalOp(Info, Op);\n } else if (Info->Type == FEXCore::X86Tables::TYPE_ARCH_DISPATCHER) [[unlikely]] {\n // Dispatcher Op.\n return NormalOp(&Info->OpcodeDispatcher.Indirect[BlockInfo.Is64BitMode ? 1 : 0], Op);\n } else if (Info->Type >= FEXCore::X86Tables::TYPE_GROUP_1 && Info->Type <= FEXCore::X86Tables::TYPE_GROUP_11) {\n uint8_t ModRMByte = ReadByte();\n DecodeInst->ModRM = ModRMByte;\n DecodeInst->Flags |= DecodeFlags::FLAG_DECODED_MODRM;\n\n FEXCore::X86Tables::ModRMDecoded ModRM;\n ModRM.Hex = DecodeInst->ModRM;\n\n#define OPD(group, prefix, Reg) (((group - FEXCore::X86Tables::TYPE_GROUP_1) << 6) | (prefix) << 3 | (Reg))\n Op = OPD(Info->Type, Info->MoreBytes, ModRM.reg);\n return NormalOp(&PrimaryInstGroupOps[Op], Op);\n#undef OPD\n } else if (Info->Type >= FEXCore::X86Tables::TYPE_GROUP_6 && Info->Type <= FEXCore::X86Tables::TYPE_GROUP_P) {\n#define OPD(group, prefix, Reg) (((group - FEXCore::X86Tables::TYPE_GROUP_6) << 5) | (prefix) << 3 | (Reg))\n constexpr uint16_t PF_NONE = 0;\n constexpr uint16_t PF_F3 = 1;\n constexpr uint16_t PF_66 = 2;\n constexpr uint16_t PF_F2 = 3;\n\n uint16_t PrefixType = PF_NONE;\n if (LastEscapePrefix == 0xF3) {\n PrefixType = PF_F3;\n } else if (LastEscapePrefix == 0xF2) {\n PrefixType = PF_F2;\n } else if (LastEscapePrefix == 0x66) {\n PrefixType = PF_66;\n }\n\n // We have ModRM\n uint8_t ModRMByte = ReadByte();\n DecodeInst->ModRM = ModRMByte;\n DecodeInst->Flags |= DecodeFlags::FLAG_DECODED_MODRM;\n\n FEXCore::X86Tables::ModRMDecoded ModRM;\n ModRM.Hex = DecodeInst->ModRM;\n\n uint16_t LocalOp = OPD(Info->Type, PrefixType, ModRM.reg);\n const FEXCore::X86Tables::X86InstInfo* LocalInfo = &SecondInstGroupOps[LocalOp];\n#undef OPD\n if (LocalInfo->Type == FEXCore::X86Tables::TYPE_SECOND_GROUP_MODRM && ModRM.mod == 0b11) {\n // Everything in this group is privileged instructions aside from XGETBV\n constexpr std::array RegToField = {\n 255, 0, 1, 2, 255, 255, 255, 3,\n };\n uint8_t Field = RegToField[ModRM.reg];\n if (Field == 255) {\n return false;\n }\n\n LocalOp = (Field << 3) | ModRM.rm;\n return NormalOp(&SecondModRMTableOps[LocalOp], LocalOp);\n } else {\n return NormalOp(&SecondInstGroupOps[LocalOp], LocalOp);\n }\n } else if (Info->Type == FEXCore::X86Tables::TYPE_X87_TABLE_PREFIX) {\n // We have ModRM\n uint8_t ModRMByte = ReadByte();\n DecodeInst->ModRM = ModRMByte;\n DecodeInst->Flags |= DecodeFlags::FLAG_DECODED_MODRM;\n\n uint16_t X87Op = ((Op - 0xD8) << 8) | ModRMByte;\n return NormalOp(&(*X87Table)[X87Op], X87Op);\n } else if (Info->Type == FEXCore::X86Tables::TYPE_VEX_TABLE_PREFIX) {\n if (!VEXTable) {\n // AVX not enabled.\n return false;\n }\n\n uint16_t map_select = 1;\n uint16_t pp = 0;\n const uint8_t Byte1 = ReadByte();\n DecodedHeader options {};\n\n if ((Byte1 & 0b10000000) == 0) {\n if (!BlockInfo.Is64BitMode) {\n return false;\n }\n\n DecodeInst->Flags |= DecodeFlags::FLAG_REX_XGPR_R;\n }\n\n if (Op == 0xC5) { // Two byte VEX\n pp = Byte1 & 0b11;\n const uint8_t vvvv = ((Byte1 & 0b01111000) >> 3);\n if (!BlockInfo.Is64BitMode && vvvv <= 0b0111) {\n // Invalid on 32-bit, can't use the high registers.\n return false;\n }\n options.vvvv = 15 - vvvv;\n options.L = (Byte1 & 0b100) != 0;\n } else { // 0xC4 = Three byte VEX\n const uint8_t Byte2 = ReadByte();\n pp = Byte2 & 0b11;\n map_select = Byte1 & 0b11111;\n const uint8_t vvvv = ((Byte2 & 0b01111000) >> 3);\n if (!BlockInfo.Is64BitMode && vvvv <= 0b0111) {\n // Invalid on 32-bit, can't use the high registers.\n return false;\n }\n options.vvvv = 15 - vvvv;\n options.w = (Byte2 & 0b10000000) != 0;\n options.L = (Byte2 & 0b100) != 0;\n if ((Byte1 & 0b01000000) == 0) {\n if (!BlockInfo.Is64BitMode) {\n return false;\n }\n DecodeInst->Flags |= DecodeFlags::FLAG_REX_XGPR_X;\n }\n if (BlockInfo.Is64BitMode && (Byte1 & 0b00100000) == 0) {\n DecodeInst->Flags |= DecodeFlags::FLAG_REX_XGPR_B;\n }\n if (options.w) {\n DecodeInst->Flags |= DecodeFlags::FLAG_OPTION_AVX_W;\n }\n if (!(map_select >= 1 && map_select <= 3)) {\n return false;\n }\n }\n\n uint16_t VEXOp = ReadByte();\n#define OPD(map_select, pp, opcode) (((map_select - 1) << 10) | (pp << 8) | (opcode))\n Op = OPD(map_select, pp, VEXOp);\n#undef OPD\n\n const FEXCore::X86Tables::X86InstInfo* LocalInfo = &(*VEXTable)[Op];\n\n if (LocalInfo->Type >= FEXCore::X86Tables::TYPE_VEX_GROUP_12 && LocalInfo->Type <= FEXCore::X86Tables::TYPE_VEX_GROUP_17) {\n // We have ModRM\n uint8_t ModRMByte = ReadByte();\n DecodeInst->ModRM = ModRMByte;\n DecodeInst->Flags |= DecodeFlags::FLAG_DECODED_MODRM;\n\n FEXCore::X86Tables::ModRMDecoded ModRM;\n ModRM.Hex = DecodeInst->ModRM;\n\n#define OPD(group, pp, opcode) (((group - TYPE_VEX_GROUP_12) << 4) | (pp << 3) | (opcode))\n Op = OPD(LocalInfo->Type, pp, ModRM.reg);\n#undef OPD\n return NormalOp(&(*VEXTableGroup)[Op], Op, options);\n } else {\n return NormalOp(LocalInfo, Op, options);\n }\n } else if (Info->Type == FEXCore::X86Tables::TYPE_GROUP_EVEX) {\n FEXCORE_TELEMETRY_SET(TYPE_USES_EVEX_OPS, 1);\n // EVEX unsupported\n return false;\n }\n\n LOGMAN_MSG_A_FMT(\"Invalid instruction decoding type\");\n FEX_UNREACHABLE;\n}\n\nbool Decoder::DecodeInstructionImpl(uint64_t PC) {\n InstructionSize = 0;\n LastEscapePrefix = 0;\n Instruction.fill(0);\n\n DecodeInst = &DecodedBuffer[DecodedSize];\n memset(DecodeInst, 0, sizeof(DecodedInst));\n DecodeInst->PC = PC;\n\n for (;;) {\n if (InstructionSize >= MAX_INST_SIZE) {\n return false;\n }\n uint8_t Op = ReadByte();\n switch (Op) {\n case 0x0F: { // Escape Op\n uint8_t EscapeOp = ReadByte();\n switch (EscapeOp) {\n case 0x0F:\n [[unlikely]] { // 3DNow!\n DecodeREXIfValid(-2);\n // 3DNow! Instruction Encoding: 0F 0F [ModRM] [SIB] [Displacement] [Opcode]\n // Decode ModRM\n uint8_t ModRMByte = ReadByte();\n DecodeInst->ModRM = ModRMByte;\n DecodeInst->Flags |= DecodeFlags::FLAG_DECODED_MODRM;\n\n FEXCore::X86Tables::ModRMDecoded ModRM;\n ModRM.Hex = DecodeInst->ModRM;\n\n const bool Has16BitAddressing = !BlockInfo.Is64BitMode && DecodeInst->Flags & DecodeFlags::FLAG_ADDRESS_SIZE;\n\n // All 3DNow! instructions have the second argument as the rm handler\n // We need to decode it upfront to get the displacement out of the way\n if (ModRM.mod != 0b11) {\n auto Disp = DecodeModRMs_Disp[Has16BitAddressing];\n (this->*Disp)(&DecodeInst->Src[0], ModRM);\n }\n\n // Take a peek at the op just past the displacement\n uint8_t LocalOp = ReadByte();\n return NormalOpHeader(&FEXCore::X86Tables::DDDNowOps[LocalOp], LocalOp);\n break;\n }\n case 0x38: { // F38 Table!\n DecodeREXIfValid(-2);\n constexpr uint16_t PF_38_NONE = 0;\n constexpr uint16_t PF_38_66 = (1U << 0);\n constexpr uint16_t PF_38_F2 = (1U << 1);\n constexpr uint16_t PF_38_F3 = (1U << 2);\n\n uint16_t Prefix = PF_38_NONE;\n if (DecodeInst->Flags & DecodeFlags::FLAG_OPERAND_SIZE) {\n Prefix |= PF_38_66;\n }\n if (DecodeInst->Flags & DecodeFlags::FLAG_REPNE_PREFIX) {\n Prefix |= PF_38_F2;\n }\n if (DecodeInst->Flags & DecodeFlags::FLAG_REP_PREFIX) {\n Prefix |= PF_38_F3;\n }\n\n uint16_t LocalOp = (Prefix << 8) | ReadByte();\n\n bool NoOverlay66 = (FEXCore::X86Tables::H0F38TableOps[LocalOp].Flags & InstFlags::FLAGS_NO_OVERLAY66) != 0;\n if (LastEscapePrefix == 0x66 && NoOverlay66) { // Operand Size\n // Remove prefix so it doesn't effect calculations.\n // This is only an escape prefix rather than modifier now\n DecodeInst->Flags &= ~DecodeFlags::FLAG_OPERAND_SIZE;\n DecodeFlags::PopOpAddrIf(&DecodeInst->Flags, DecodeFlags::FLAG_OPERAND_SIZE_LAST);\n }\n return NormalOpHeader(&FEXCore::X86Tables::H0F38TableOps[LocalOp], LocalOp);\n break;\n }\n case 0x3A: { // F3A Table!\n DecodeREXIfValid(-2);\n constexpr uint16_t PF_3A_NONE = 0;\n constexpr uint16_t PF_3A_66 = (1 << 0);\n constexpr uint16_t PF_3A_REX = (1 << 1);\n\n uint16_t Prefix = PF_3A_NONE;\n if (LastEscapePrefix == 0x66) { // Operand Size\n Prefix = PF_3A_66;\n }\n\n if (DecodeInst->Flags & DecodeFlags::FLAG_REX_WIDENING) {\n Prefix |= PF_3A_REX;\n }\n\n uint16_t LocalOp = (Prefix << 8) | ReadByte();\n return NormalOpHeader(&FEXCore::X86Tables::H0F3ATableOps[LocalOp], LocalOp);\n break;\n }\n default:\n [[likely]] { // Two byte table!\n // x86-64 abuses three legacy prefixes to extend the table encodings\n // 0x66 - Operand Size prefix\n // 0xF2 - REPNE prefix\n // 0xF3 - REP prefix\n // If any of these three prefixes are used then it falls down the subtable\n // Additionally: If you hit repeat of differnt prefixes then only the LAST one before this one works for subtable selection\n\n bool NoOverlay = (FEXCore::X86Tables::SecondBaseOps[EscapeOp].Flags & InstFlags::FLAGS_NO_OVERLAY) != 0;\n bool NoOverlay66 = (FEXCore::X86Tables::SecondBaseOps[EscapeOp].Flags & InstFlags::FLAGS_NO_OVERLAY66) != 0;\n\n DecodeREXIfValid(-2);\n if (NoOverlay) { // This section of the table ignores prefix extention\n return NormalOpHeader(&FEXCore::X86Tables::SecondBaseOps[EscapeOp], EscapeOp);\n } else if (LastEscapePrefix == 0xF3) { // REP\n // Remove prefix so it doesn't effect calculations.\n // This is only an escape prefix rather tan modifier now\n DecodeInst->Flags &= ~DecodeFlags::FLAG_REP_PREFIX;\n return NormalOpHeader(&FEXCore::X86Tables::RepModOps[EscapeOp], EscapeOp);\n } else if (LastEscapePrefix == 0xF2) { // REPNE\n // Remove prefix so it doesn't effect calculations.\n // This is only an escape prefix rather tan modifier now\n DecodeInst->Flags &= ~DecodeFlags::FLAG_REPNE_PREFIX;\n return NormalOpHeader(&FEXCore::X86Tables::RepNEModOps[EscapeOp], EscapeOp);\n } else if (LastEscapePrefix == 0x66 && !NoOverlay66) { // Operand Size\n // Remove prefix so it doesn't effect calculations.\n // This is only an escape prefix rather tan modifier now\n DecodeInst->Flags &= ~DecodeFlags::FLAG_OPERAND_SIZE;\n DecodeFlags::PopOpAddrIf(&DecodeInst->Flags, DecodeFlags::FLAG_OPERAND_SIZE_LAST);\n return NormalOpHeader(&FEXCore::X86Tables::OpSizeModOps[EscapeOp], EscapeOp);\n } else {\n return NormalOpHeader(&FEXCore::X86Tables::SecondBaseOps[EscapeOp], EscapeOp);\n }\n break;\n }\n }\n break;\n }\n case 0x66: // Operand Size prefix\n DecodeInst->Flags |= DecodeFlags::FLAG_OPERAND_SIZE;\n LastEscapePrefix = Op;\n DecodeFlags::PushOpAddr(&DecodeInst->Flags, DecodeFlags::FLAG_OPERAND_SIZE_LAST);\n break;\n case 0x67: // Address Size override prefix\n DecodeInst->Flags |= DecodeFlags::FLAG_ADDRESS_SIZE;\n break;\n case 0x26: // ES legacy prefix\n if (!BlockInfo.Is64BitMode) {\n DecodeInst->Flags = (DecodeInst->Flags & ~FEXCore::X86Tables::DecodeFlags::FLAG_SEGMENTS) | DecodeFlags::FLAG_ES_PREFIX;\n }\n break;\n case 0x2E: // CS legacy prefix\n if (!BlockInfo.Is64BitMode) {\n DecodeInst->Flags = (DecodeInst->Flags & ~FEXCore::X86Tables::DecodeFlags::FLAG_SEGMENTS) | DecodeFlags::FLAG_CS_PREFIX;\n }\n break;\n case 0x36: // SS legacy prefix\n if (!BlockInfo.Is64BitMode) {\n DecodeInst->Flags = (DecodeInst->Flags & ~FEXCore::X86Tables::DecodeFlags::FLAG_SEGMENTS) | DecodeFlags::FLAG_SS_PREFIX;\n }\n break;\n case 0x3E: // DS legacy prefix\n if (!BlockInfo.Is64BitMode) {\n DecodeInst->Flags = (DecodeInst->Flags & ~FEXCore::X86Tables::DecodeFlags::FLAG_SEGMENTS) | DecodeFlags::FLAG_DS_PREFIX;\n }\n break;\n case 0xF0: // LOCK prefix\n DecodeInst->Flags |= DecodeFlags::FLAG_LOCK;\n break;\n case 0xF2: // REPNE prefix\n DecodeInst->Flags |= DecodeFlags::FLAG_REPNE_PREFIX;\n LastEscapePrefix = Op;\n break;\n case 0xF3: // REP prefix\n DecodeInst->Flags |= DecodeFlags::FLAG_REP_PREFIX;\n LastEscapePrefix = Op;\n break;\n case 0x64: // FS prefix\n DecodeInst->Flags = (DecodeInst->Flags & ~FEXCore::X86Tables::DecodeFlags::FLAG_SEGMENTS) | DecodeFlags::FLAG_FS_PREFIX;\n break;\n case 0x65: // GS prefix\n DecodeInst->Flags = (DecodeInst->Flags & ~FEXCore::X86Tables::DecodeFlags::FLAG_SEGMENTS) | DecodeFlags::FLAG_GS_PREFIX;\n break;\n default:\n [[likely]] { // Default base table\n const X86InstInfo* Info = &FEXCore::X86Tables::BaseOps[Op];\n if (Info->Type == FEXCore::X86Tables::TYPE_ARCH_DISPATCHER) {\n Info = &Info->OpcodeDispatcher.Indirect[BlockInfo.Is64BitMode ? 1 : 0];\n }\n\n if (Info->Type == FEXCore::X86Tables::TYPE_REX_PREFIX) {\n DecodeInst->REXIndex = InstructionSize;\n } else {\n DecodeREXIfValid();\n return NormalOpHeader(Info, Op);\n }\n\n break;\n }\n }\n }\n\n if (DecodeInst->Dest.IsGPR()) {\n return false;\n }\n\n return true;\n}\n\nvoid Decoder::DecodeREXIfValid(int8_t ExpectedOffset) {\n LOGMAN_THROW_A_FMT(ExpectedOffset < 0, \"Expecting an negative offset for the REX offset!\");\n const int8_t REXIndex = InstructionSize + ExpectedOffset;\n\n if (DecodeInst->REXIndex != 0 && DecodeInst->REXIndex == REXIndex) {\n const uint8_t Op = Instruction[REXIndex - 1];\n DecodeInst->Flags |= DecodeFlags::FLAG_REX_PREFIX;\n\n // Widening displacement\n if (Op & 0b1000) {\n DecodeInst->Flags |= DecodeFlags::FLAG_REX_WIDENING;\n DecodeFlags::PushOpAddr(&DecodeInst->Flags, DecodeFlags::FLAG_WIDENING_SIZE_LAST);\n }\n\n // XGPR_B bit set\n if (Op & 0b0001) {\n DecodeInst->Flags |= DecodeFlags::FLAG_REX_XGPR_B;\n }\n\n // XGPR_X bit set\n if (Op & 0b0010) {\n DecodeInst->Flags |= DecodeFlags::FLAG_REX_XGPR_X;\n }\n\n // XGPR_R bit set\n if (Op & 0b0100) {\n DecodeInst->Flags |= DecodeFlags::FLAG_REX_XGPR_R;\n }\n }\n}\n\nDecoder::DecodedBlockStatus Decoder::DecodeInstruction(uint64_t PC) {\n // Will be set if DecodeInstructionImpl tries to read non-executable memory\n HitNonExecutableRange = false;\n HitBadRelocation = false;\n bool ErrorDuringDecoding = !DecodeInstructionImpl(PC);\n\n if (ErrorDuringDecoding || HitNonExecutableRange || HitBadRelocation) [[unlikely]] {\n // Put an invalid instruction in the stream so the core can raise SIGILL if hit\n // Error while decoding instruction. We don't know the table or instruction size\n DecodeInst->TableInfo = nullptr;\n auto Result = ErrorDuringDecoding ? DecodedBlockStatus::INVALID_INST :\n DecodeInst->InstSize ? DecodedBlockStatus::PARTIAL_DECODE_INST :\n HitNonExecutableRange ? DecodedBlockStatus::NOEXEC_INST :\n DecodedBlockStatus::BAD_RELOCATION;\n DecodeInst->InstSize = 0;\n return Result;\n } else if (!DecodeInst->TableInfo || (DecodeInst->TableInfo->Type == TYPE_INST && !DecodeInst->TableInfo->OpcodeDispatcher.OpDispatch)) {\n // If there wasn't an error during decoding but we have no dispatcher for the instruction then claim invalid instruction.\n return DecodedBlockStatus::INVALID_INST;\n }\n\n if (CTX->AreMonoHacksActive()) {\n // Unity uses a standard SPSC ringbuffer with cached read/write pointers and thread waiting flags at the following\n // offsets, which are consistent between 32-bit and 64-bit Unity versions from 2015 onwards.\n auto IsKnownAtomicDisplacement = [](uint64_t Displacement) {\n return Displacement == 0x80 || Displacement == 0x84 || Displacement == 0xC0 || Displacement == 0xC4;\n };\n\n if (DecodeInst->OP == 0x8b && DecodeInst->Src[0].IsGPRIndirect() &&\n IsKnownAtomicDisplacement(DecodeInst->Src[0].Data.GPRIndirect.Displacement)) {\n DecodeInst->Flags |= X86Tables::DecodeFlags::FLAG_FORCE_TSO;\n }\n if (DecodeInst->OP == 0x89 && DecodeInst->Dest.IsGPRIndirect() && IsKnownAtomicDisplacement(DecodeInst->Dest.Data.GPRIndirect.Displacement)) {\n DecodeInst->Flags |= X86Tables::DecodeFlags::FLAG_FORCE_TSO;\n }\n }\n\n return DecodedBlockStatus::SUCCESS;\n}\n\nvoid Decoder::BranchTargetInMultiblockRange() {\n if (!CTX->Config.Multiblock) {\n return;\n }\n\n // If the RIP setting is conditional AND within our symbol range then it can be considered for multiblock\n uint64_t TargetRIP = 0;\n const auto GPRSize = GetGPROpSize();\n bool Conditional = true;\n const auto InstEnd = DecodeInst->PC + DecodeInst->InstSize;\n\n if (DecodeInst->TableInfo->Flags & FEXCore::X86Tables::InstFlags::FLAGS_CALL) {\n if (ExecutableRangeWritable && CTX->AreMonoHacksActive()) {\n // Mono generated code often contains noreturn calls with garbage following them, and calls are always backpatched\n // after CIL compilation leading to n recompiles for a multiblock with n calls. Choose to minimize stutters over\n // raw performance and disable tracking past calls for mono generated code.\n return;\n }\n\n AddBranchTarget(InstEnd);\n BlockInfo.EntryPoints.emplace(InstEnd);\n return;\n }\n\n // Calls are handled above\n switch (DecodeInst->OP) {\n case 0x70 ... 0x7F: // Conditional JUMP\n case 0x80 ... 0x8F: { // More conditional\n // Source is a literal\n // auto RIPOffset = LoadSource(Op, Op->Src[0], Op->Flags);\n // auto RIPTargetConst = Constant(Op->PC + Op->InstSize);\n // Target offset is PC + InstSize + Literal\n TargetRIP = InstEnd + DecodeInst->Src[0].Literal();\n break;\n }\n case 0xE9:\n case 0xEB: // Both are unconditional JMP instructions\n TargetRIP = InstEnd + DecodeInst->Src[0].Literal();\n Conditional = false;\n break;\n case 0xC2: // RET imm\n case 0xC3: // RET\n default: return; break;\n }\n\n if (GPRSize == IR::OpSize::i32Bit) {\n // If we are running a 32bit guest then wrap around addresses that go above 32bit\n TargetRIP &= 0xFFFFFFFFU;\n }\n\n if (Conditional) {\n // If we are conditional then a target can be the instruction past the conditional instruction\n AddBranchTarget(InstEnd);\n }\n\n // If the target RIP is x86 code within the symbol ranges then we are golden\n // Forbid distant branches to have the cost code better match the guest code layout, avoiding massive (range-wise) code\n // blocks in highly fragmented guest code. Such branches are often not-taken branches to garbage in obfuscated code.\n constexpr uint64_t MAX_FORWARD_BRANCH_DIST = FEXCore::Utils::FEX_PAGE_SIZE * 4;\n bool ValidMultiblockMember = TargetRIP >= EntryPoint && TargetRIP < std::min(InstEnd + MAX_FORWARD_BRANCH_DIST, SectionMaxAddress);\n\n#ifdef ARCHITECTURE_arm64ec\n ValidMultiblockMember = ValidMultiblockMember && !RtlIsEcCode(TargetRIP);\n#endif\n\n if (ValidMultiblockMember) {\n // Update our conditional branch ranges before we return\n if (Conditional) {\n MaxCondBranchForward = std::max(MaxCondBranchForward, TargetRIP);\n MaxCondBranchBackwards = std::min(MaxCondBranchBackwards, TargetRIP);\n }\n\n AddBranchTarget(TargetRIP);\n } else {\n if (ExternalBranches) {\n ExternalBranches->insert(TargetRIP);\n }\n }\n}\n\nbool Decoder::IsBranchMonoTailcall(uint64_t NumInstructions) const {\n // While the mono call backpatching block can easily be detected due it being the only one to contain SMC-faulting\n // atomics, that can't be said for the tailcall jump backpatcher which has changed several times across versions and\n // can be partially inlined. To work around this, instead detect the tailcall site itself and force full non-signal-based\n // SMC detection for that single block.\n if (!ExecutableRangeWritable) {\n // We only care about jitted code\n return false;\n }\n\n // See mini-{amd64,x86}.c in the mono codebase, specifically where METHOD_JUMP patches are emitted.\n if (GetGPROpSize() == IR::OpSize::i32Bit) {\n // Matches:\n // LEAVE\n // / NOP / MOV EAX, EAX / LEA EBP, [EBP+0]\n // JMP imm32\n if (DecodeInst->OP != 0xE9 || NumInstructions < 2) {\n return false;\n }\n\n auto PrevInst = std::prev(DecodeInst);\n if (PrevInst->OP == 0xC9) {\n return true;\n }\n\n if (NumInstructions < 3 || std::prev(PrevInst)->OP != 0xC9) {\n return false;\n }\n\n return PrevInst->OP == 0x90 || (PrevInst->OP == 0x8B && PrevInst->ModRM == 0xC0) ||\n (PrevInst->OP == 0x8D && PrevInst->ModRM == 0x6D && PrevInst->Src[1].IsLiteral() && PrevInst->Src[1].Literal() == 0);\n } else {\n FEXCore::X86Tables::ModRMDecoded ModRM;\n ModRM.Hex = DecodeInst->ModRM;\n if (DecodeInst->OPRaw == 0xFF && ModRM.reg == 4 && DecodeInst->Src[0].IsGPR()) {\n if (DecodeInst->Src[0].Data.GPR.GPR == FEXCore::X86State::REG_RAX) {\n // Found in versions of mono from 2024 onwards - matches:\n // REX.W JMP rax\n return (DecodeInst->Flags & (DecodeFlags::FLAG_REX_PREFIX | DecodeFlags::FLAG_REX_WIDENING | DecodeFlags::FLAG_REX_XGPR_B |\n DecodeFlags::FLAG_REX_XGPR_X | DecodeFlags::FLAG_REX_XGPR_R)) ==\n (DecodeFlags::FLAG_REX_PREFIX | DecodeFlags::FLAG_REX_WIDENING);\n } else if (NumInstructions > 1 && DecodeInst->Src[0].Data.GPR.GPR == FEXCore::X86State::REG_R11) {\n // Found in older versions of mono - match:\n // MOV r11, imm64\n // JMP r11\n auto PrevInst = std::prev(DecodeInst);\n return PrevInst->OP == 0xBB && PrevInst->Dest.IsGPR() && PrevInst->Dest.Data.GPR.GPR == FEXCore::X86State::REG_R11;\n }\n }\n }\n\n return false;\n}\n\nbool Decoder::InstCanContinue() const {\n if (DecodeInst->PC + DecodeInst->InstSize == NextBlockStartAddress) {\n return false;\n }\n\n if (!(DecodeInst->TableInfo->Flags & (FEXCore::X86Tables::InstFlags::FLAGS_BLOCK_END | FEXCore::X86Tables::InstFlags::FLAGS_SETS_RIP))) {\n return true;\n }\n\n uint64_t TargetRIP = 0;\n const auto GPRSize = GetGPROpSize();\n\n if (DecodeInst->OP == 0xE8) { // Call - immediate target\n const uint64_t NextRIP = DecodeInst->PC + DecodeInst->InstSize;\n TargetRIP = DecodeInst->PC + DecodeInst->InstSize + DecodeInst->Src[0].Literal();\n\n if (GPRSize == IR::OpSize::i32Bit) {\n // If we are running a 32bit guest then wrap around addresses that go above 32bit\n TargetRIP &= 0xFFFFFFFFU;\n }\n\n if (TargetRIP == NextRIP) {\n // Optimize the case that the instruction is jumping just after itself.\n // This is a GOT calculation which we can optimize out.\n // Optimization occurs inside of the OpDispatcher implementation\n return true;\n }\n }\n\n return false;\n}\n\nvoid Decoder::AddBranchTarget(uint64_t Target) {\n if (VisitedBlocks.contains(Target)) {\n return;\n }\n\n auto BlockSuccIt = std::lower_bound(BlockInfo.Blocks.begin(), BlockInfo.Blocks.end(), Target,\n [](const auto& a, uint64_t Address) { return a.Entry < Address; });\n\n LOGMAN_THROW_A_FMT(BlockSuccIt == BlockInfo.Blocks.end() || BlockSuccIt->Entry != Target, \"unexpected\");\n\n if (BlockSuccIt != BlockInfo.Blocks.begin()) {\n auto BlockIt = std::prev(BlockSuccIt);\n if (BlockIt->Entry + BlockIt->Size > Target) {\n uint64_t SplitIdx = 0;\n uint64_t SplitAddr = BlockIt->Entry;\n // Find the instruction boundary of the split\n for (; SplitIdx < BlockIt->NumInstructions && SplitAddr < Target; SplitIdx++) {\n SplitAddr += BlockIt->DecodedInstructions[SplitIdx].InstSize;\n }\n uint64_t SplitOffset = SplitAddr - BlockIt->Entry;\n\n LOGMAN_THROW_A_FMT(SplitIdx != 0, \"unexpected\");\n\n if (SplitAddr == Target) {\n // Split at the boundary\n DecodedBlocks SplitBlock {\n .Entry = SplitAddr,\n .Size = BlockIt->Size - SplitOffset,\n .NumInstructions = BlockIt->NumInstructions - SplitIdx,\n .DecodedInstructions = BlockIt->DecodedInstructions + SplitIdx,\n .BlockStatus = BlockIt->BlockStatus,\n };\n\n BlockIt->Size = SplitOffset;\n BlockIt->NumInstructions = SplitIdx;\n\n BlockInfo.Blocks.insert(BlockSuccIt, SplitBlock);\n } // else misaligned, leave as a branch out of the block\n\n // If we split a block then the target has already been visited as part of that, if it was\n // misaligned the jump will just leave the multiblock, mark it as visited to avoid running\n // this code path again and just bail out early.\n VisitedBlocks.insert(Target);\n return;\n }\n }\n\n CurrentBlockTargets.insert(Target);\n if (Target >= DecodeInst->PC + DecodeInst->InstSize && Target < NextBlockStartAddress) {\n NextBlockStartAddress = Target;\n }\n}\n\nconst uint8_t* Decoder::AdjustAddrForSpecialRegion(const uint8_t* _InstStream, uint64_t EntryPoint, uint64_t RIP) {\n constexpr uint64_t VSyscall_Base = 0xFFFF'FFFF'FF60'0000ULL;\n constexpr uint64_t VSyscall_End = VSyscall_Base + 0x1000;\n\n if (OSABI == FEXCore::HLE::SyscallOSABI::OS_LINUX64 && RIP >= VSyscall_Base && RIP < VSyscall_End) {\n // VSyscall\n // This doesn't exist on AArch64 and on x86_64 hosts this is emulated with faults to a region mapped with --xp permissions\n // Offset 0: vgettimeofday\n // Offset 0x400: vtime\n // Offset 0x800: vgetcpu\n uint64_t Offset = RIP - VSyscall_Base;\n return VSyscallData + Offset;\n }\n\n return _InstStream - EntryPoint + RIP;\n}\n\nbool Decoder::CheckIfCacheable(FEXCore::Core::InternalThreadState& Thread, const uint8_t* InstStream, uint64_t PC, uint64_t MaxInst) {\n DecodeInstructionsAtEntry(&Thread, InstStream, PC, MaxInst);\n bool Uncacheable = HitBadRelocation;\n DelayedDisownBuffer();\n return !Uncacheable;\n}\n\nvoid Decoder::DecodeInstructionsAtEntry(FEXCore::Core::InternalThreadState* Thread, const uint8_t* _InstStream, uint64_t PC, uint64_t MaxInst) {\n FEXCORE_PROFILE_SCOPED(\"DecodeInstructions\");\n BlockInfo.TotalInstructionCount = 0;\n BlockInfo.Blocks.clear();\n VisitedBlocks.clear();\n // Reset internal state management\n DecodedSize = 0;\n MaxCondBranchForward = 0;\n MaxCondBranchBackwards = ~0ULL;\n DecodedBuffer = PoolObject.ReownOrClaimBuffer();\n\n // Decode operating mode from thread's CS segment.\n const auto CSSegment = Core::CPUState::GetSegmentFromIndex(Thread->CurrentFrame->State, Thread->CurrentFrame->State.cs_idx);\n BlockInfo.Is64BitMode = CSSegment->L == 1;\n LOGMAN_THROW_A_FMT(BlockInfo.Is64BitMode == CTX->Config.Is64BitMode, \"Expected operating mode to not change at runtime!\");\n\n EntryPoint = PC;\n BlockInfo.EntryPoints = {PC};\n InstStream = _InstStream;\n\n uint64_t TotalInstructions {};\n\n SectionMinAddress = 0;\n SectionMaxAddress = ~0ULL;\n Relocations = nullptr;\n\n if (CTX->GetCodeCache().IsGeneratingCache || EnableCodeCacheValidation) {\n // If generating cache, attempt to load section bounds and relocations\n if (auto SectionInfo = CTX->SyscallHandler->LookupExecutableFileSection(Thread, EntryPoint)) {\n SectionMinAddress = SectionInfo->FileStartVA;\n SectionMaxAddress = SectionInfo->EndVA;\n Relocations = &SectionInfo->FileInfo.Relocations;\n }\n }\n\n DecodedMinAddress = EntryPoint;\n DecodedMaxAddress = EntryPoint;\n\n // Entry is a jump target\n BlocksToDecode = {PC};\n\n uint64_t CurrentCodePage = PC & FEXCore::Utils::FEX_PAGE_MASK;\n\n BlockInfo.CodePages = {CurrentCodePage};\n\n if (MaxInst == 0) {\n MaxInst = CTX->Config.MaxInstPerBlock;\n }\n\n bool EntryBlock {true};\n bool FinalInstruction {false};\n\n while (!FinalInstruction && !BlocksToDecode.empty()) {\n auto BlockDecodeIt = BlocksToDecode.begin();\n uint64_t RIPToDecode = *BlockDecodeIt;\n BlocksToDecode.erase(BlockDecodeIt);\n VisitedBlocks.emplace(RIPToDecode);\n\n auto BlockSuccIt = std::lower_bound(BlockInfo.Blocks.begin(), BlockInfo.Blocks.end(), RIPToDecode,\n [](const auto& a, uint64_t Address) { return a.Entry < Address; });\n\n LOGMAN_THROW_A_FMT(BlockSuccIt == BlockInfo.Blocks.end() || BlockSuccIt->Entry != RIPToDecode, \"unexpected\");\n\n NextBlockStartAddress = ~0ULL;\n if (!BlocksToDecode.empty()) {\n // We just erased the lowest, the front is then the second lowest\n NextBlockStartAddress = *BlocksToDecode.begin();\n }\n if (BlockSuccIt != BlockInfo.Blocks.end() && BlockSuccIt->Entry < NextBlockStartAddress) {\n NextBlockStartAddress = BlockSuccIt->Entry;\n }\n LOGMAN_THROW_A_FMT(NextBlockStartAddress > RIPToDecode, \"unexpected\");\n\n // Insert the block now so it can be looked up and split if necessary on a backward edge\n auto BlockIt = BlockInfo.Blocks.emplace(BlockSuccIt);\n\n BlockIt->Entry = RIPToDecode;\n BlockIt->Size = 0;\n BlockIt->IsEntryPoint = EntryBlock;\n\n uint64_t PCOffset = 0;\n uint64_t BlockStartOffset = DecodedSize;\n bool EraseBlock = true; // Unset once the block contains an instruction\n\n BlockIt->DecodedInstructions = &DecodedBuffer[BlockStartOffset];\n BlockIt->NumInstructions = 0;\n\n // Do a bit of pointer math to figure out where we are in code\n InstStream = AdjustAddrForSpecialRegion(_InstStream, EntryPoint, RIPToDecode);\n\n while (1) {\n InstructionSize = 0;\n\n // MAX_INST_SIZE assumes worst case\n auto OpAddress = RIPToDecode + PCOffset;\n auto OpMaxAddress = OpAddress + MAX_INST_SIZE;\n\n auto OpMinPage = OpAddress & FEXCore::Utils::FEX_PAGE_MASK;\n auto OpMaxPage = OpMaxAddress & FEXCore::Utils::FEX_PAGE_MASK;\n\n if (!EntryBlock && OpMinPage == OpMaxPage && PeekByte(0).value_or(0) == 0 && PeekByte(1).value_or(0) == 0) [[unlikely]] {\n // End the multiblock early if we hit 2 consecutive null bytes (add [rax], al) in the same page with the\n // assumption we are most likely trying to explore garbage code.\n break;\n }\n\n if (OpMinPage != CurrentCodePage) {\n CurrentCodePage = OpMinPage;\n BlockInfo.CodePages.insert(CurrentCodePage);\n }\n\n if (OpMaxPage != CurrentCodePage) {\n CurrentCodePage = OpMaxPage;\n BlockInfo.CodePages.insert(CurrentCodePage);\n }\n\n BlockIt->BlockStatus = DecodeInstruction(OpAddress);\n if (HitBadRelocation) {\n BlockInfo.TotalInstructionCount = 0;\n BlockInfo.Blocks = {*BlockIt};\n BlockInfo.EntryPoints.clear();\n BlockInfo.CodePages.clear();\n return;\n }\n uint64_t OpEndAddress = OpAddress + DecodeInst->InstSize;\n\n DecodedMinAddress = std::min(DecodedMinAddress, OpAddress);\n DecodedMaxAddress = std::max(DecodedMaxAddress, OpEndAddress);\n\n if (OpEndAddress > NextBlockStartAddress) {\n // This instruction would overlap with another so skip adding it to the multiblock\n break;\n }\n\n EraseBlock = false; // Block contains at least one valid instruction, so unset erase\n ++TotalInstructions;\n ++DecodedSize;\n ++BlockIt->NumInstructions;\n BlockIt->Size += DecodeInst->InstSize;\n\n // Can not continue this block at all on invalid instruction\n if (BlockIt->BlockStatus != DecodedBlockStatus::SUCCESS) [[unlikely]] {\n if (!EntryBlock && BlockIt->BlockStatus != DecodedBlockStatus::BAD_RELOCATION) {\n // In multiblock configurations, we can early terminate any non-entrypoint blocks with the expectation that this won't get hit.\n // Improves compile-times.\n // Just need to undo additions that this block decoding has caused.\n TotalInstructions -= BlockIt->NumInstructions;\n DecodedSize = BlockStartOffset;\n InstStream -= PCOffset;\n EraseBlock = true;\n } else {\n LogMan::Msg::EFmt(\"{} instruction in entry block: {:X}\",\n BlockIt->BlockStatus == DecodedBlockStatus::INVALID_INST ? \"Invalid\" :\n BlockIt->BlockStatus == DecodedBlockStatus::NOEXEC_INST ? \"NoExec\" :\n BlockIt->BlockStatus == DecodedBlockStatus::BAD_RELOCATION ? \"BadRelocation\" :\n \"PartialDecode\",\n OpAddress);\n }\n break;\n }\n\n // Check if we need to end the entire multiblock\n FinalInstruction = DecodedSize >= MaxInst || DecodedSize >= DefaultDecodedBufferSize || TotalInstructions >= MaxInst;\n if (FinalInstruction) {\n break;\n }\n\n if (!InstCanContinue()) {\n if (DecodeInst->TableInfo->Flags & FEXCore::X86Tables::InstFlags::FLAGS_SETS_RIP) {\n // If we have multiblock enabled\n // If the branch target is within our multiblock range then we can keep going on\n // We don't want to short circuit this since we want to calculate our ranges still\n // NOTE: This will invalidate BlockIt, this is fine as we immediately break from the loop and EraseBlock cannot be true\n BlockIt->ForceFullSMCDetection = CTX->AreMonoHacksActive() && IsBranchMonoTailcall(BlockIt->NumInstructions);\n BranchTargetInMultiblockRange();\n }\n\n break;\n }\n\n PCOffset += DecodeInst->InstSize;\n InstStream += DecodeInst->InstSize;\n }\n\n // NOTE: BlockIt is only valid here in the EraseBlock case\n if (EraseBlock) {\n BlockInfo.Blocks.erase(BlockIt);\n } else {\n BlocksToDecode.merge(CurrentBlockTargets);\n }\n\n CurrentBlockTargets.clear();\n EntryBlock = false;\n }\n\n BlockInfo.TotalInstructionCount = TotalInstructions;\n\n for (auto& Block : BlockInfo.Blocks) {\n Block.IsEntryPoint = BlockInfo.EntryPoints.contains(Block.Entry);\n }\n}\n\n} // namespace FEXCore::Frontend\n" }, { "path": "FEXCore/Source/Interface/Core/Frontend.h", "content": "// SPDX-License-Identifier: MIT\n#pragma once\n\n#include \"Interface/Core/X86Tables/X86Tables.h\"\n#include \"Interface/IR/IR.h\"\n\n#include \n#include \n#include \n#include \n#include \n#include \n\n#include \n#include \n#include \n#include \n\nnamespace FEXCore::Context {\nclass ContextImpl;\n}\nnamespace FEXCore::HLE {\nenum class SyscallOSABI;\n}\n\nnamespace FEXCore::Frontend {\nclass Decoder final {\npublic:\n enum class DecodedBlockStatus {\n SUCCESS,\n INVALID_INST,\n NOEXEC_INST,\n PARTIAL_DECODE_INST,\n BAD_RELOCATION,\n };\n\n // New Frontend decoding\n struct DecodedBlocks final {\n uint64_t Entry {};\n uint64_t Size {};\n uint64_t NumInstructions {};\n FEXCore::X86Tables::DecodedInst* DecodedInstructions;\n DecodedBlockStatus BlockStatus;\n bool IsEntryPoint {};\n bool ForceFullSMCDetection {};\n };\n\n struct DecodedBlockInformation final {\n uint64_t TotalInstructionCount;\n bool Is64BitMode {};\n fextl::vector Blocks;\n fextl::set EntryPoints;\n fextl::set CodePages; // Start addresses of all pages touching the block\n };\n\n Decoder(FEXCore::Core::InternalThreadState* Thread);\n bool CheckIfCacheable(FEXCore::Core::InternalThreadState&, const uint8_t* InstStream, uint64_t PC, uint64_t MaxInst);\n void DecodeInstructionsAtEntry(FEXCore::Core::InternalThreadState* Thread, const uint8_t* InstStream, uint64_t PC, uint64_t MaxInst);\n\n const DecodedBlockInformation* GetDecodedBlockInfo() const {\n return &BlockInfo;\n }\n\n uint64_t DecodedMinAddress {};\n uint64_t DecodedMaxAddress {~0ULL};\n\n void SetExternalBranches(fextl::set* v) {\n ExternalBranches = v;\n }\n\n void DelayedDisownBuffer() {\n PoolObject.DelayedDisownBuffer();\n }\n\n void ResetExecutableRangeCache() {\n ExecutableRangeBase = ExecutableRangeEnd = 0;\n }\n\nprivate:\n // To pass any information from instruction prefixes\n // down into the actual instruction handling machinery.\n struct DecodedHeader {\n uint8_t vvvv; // Encoded operand in a VEX prefix.\n bool w; // VEX.W bit.\n bool L; // VEX.L bit (if set then 256 bit operation, if unset then scalar or 128-bit operation)\n };\n\n FEXCore::Core::InternalThreadState* Thread;\n FEXCore::Context::ContextImpl* CTX;\n const FEXCore::HLE::SyscallOSABI OSABI {};\n\n FEX_CONFIG_OPT(EnableCodeCacheValidation, ENABLECODECACHEVALIDATION);\n\n bool DecodeInstructionImpl(uint64_t PC);\n DecodedBlockStatus DecodeInstruction(uint64_t PC);\n\n void BranchTargetInMultiblockRange();\n bool IsBranchMonoTailcall(uint64_t NumInstructions) const;\n bool InstCanContinue() const;\n\n void AddBranchTarget(uint64_t Target);\n\n bool CheckRangeExecutable(uint64_t Address, uint64_t Size);\n\n uint8_t ReadByte();\n std::optional PeekByte(uint8_t Offset);\n std::pair ReadData(uint8_t Size);\n\n void SkipBytes(uint8_t Size) {\n InstructionSize += Size;\n }\n\n bool NormalOp(const FEXCore::X86Tables::X86InstInfo* Info, uint16_t Op, DecodedHeader Options = {});\n bool NormalOpHeader(const FEXCore::X86Tables::X86InstInfo* Info, uint16_t Op);\n\n void DecodeREXIfValid(int8_t ExpectedOffset = -1);\n\n static constexpr size_t DefaultDecodedBufferSize = 0x10000;\n FEXCore::X86Tables::DecodedInst* DecodedBuffer {};\n Utils::PoolBufferWithTimedRetirement PoolObject;\n size_t DecodedSize {};\n\n uint64_t ExecutableRangeBase {};\n uint64_t ExecutableRangeEnd {};\n bool ExecutableRangeWritable {};\n bool HitNonExecutableRange {};\n bool HitBadRelocation {};\n\n const uint8_t* InstStream {};\n IR::OpSize GetGPROpSize() const {\n return BlockInfo.Is64BitMode ? IR::OpSize::i64Bit : IR::OpSize::i32Bit;\n }\n\n static constexpr size_t MAX_INST_SIZE = 15;\n uint8_t InstructionSize {};\n std::array Instruction;\n uint8_t LastEscapePrefix {};\n FEXCore::X86Tables::DecodedInst* DecodeInst;\n\n // This is for multiblock data tracking\n uint64_t EntryPoint {};\n uint64_t MaxCondBranchForward {};\n uint64_t MaxCondBranchBackwards {~0ULL};\n uint64_t SectionMaxAddress {~0ULL};\n uint64_t SectionMinAddress {};\n uint64_t NextBlockStartAddress {~0ULL};\n\n DecodedBlockInformation BlockInfo;\n fextl::set CurrentBlockTargets;\n fextl::set BlocksToDecode;\n fextl::set VisitedBlocks;\n fextl::set* ExternalBranches {nullptr};\n\n const fextl::robin_map* Relocations {nullptr};\n\n // ModRM rm decoding\n using DecodeModRMPtr = void (FEXCore::Frontend::Decoder::*)(X86Tables::DecodedOperand* Operand, X86Tables::ModRMDecoded ModRM);\n void DecodeModRM_16(X86Tables::DecodedOperand* Operand, X86Tables::ModRMDecoded ModRM);\n void DecodeModRM_64(X86Tables::DecodedOperand* Operand, X86Tables::ModRMDecoded ModRM);\n\n static constexpr std::array DecodeModRMs_Disp {\n &FEXCore::Frontend::Decoder::DecodeModRM_64,\n &FEXCore::Frontend::Decoder::DecodeModRM_16,\n };\n\n const std::array* X87Table;\n\n const std::array* VEXTable {};\n const std::array* VEXTableGroup {};\n\n const uint8_t* AdjustAddrForSpecialRegion(const uint8_t* _InstStream, uint64_t EntryPoint, uint64_t RIP);\n};\n} // namespace FEXCore::Frontend\n" }, { "path": "FEXCore/Source/Interface/Core/Interpreter/Fallbacks/F80Fallbacks.h", "content": "// SPDX-License-Identifier: MIT\n#pragma once\n#include \"Common/SoftFloat.h\"\n\n#include \"Interface/Core/Interpreter/Fallbacks/FallbackOpHandler.h\"\n#include \"Interface/IR/IR.h\"\n\n#include \n#include \n\nnamespace FEXCore::CPU {\nFEXCORE_PRESERVE_ALL_ATTR static softfloat_state SoftFloatStateFromFCW(uint16_t FCW, bool Force80BitPrecision = false) {\n softfloat_state State {};\n State.detectTininess = softfloat_tininess_afterRounding;\n State.exceptionFlags = 0;\n State.roundingPrecision = 80;\n\n if (!Force80BitPrecision) {\n auto PC = (FCW >> 8) & 3;\n switch (PC) {\n case 0: State.roundingPrecision = 32; break;\n case 2: State.roundingPrecision = 64; break;\n case 3: State.roundingPrecision = 80; break;\n case 1: LOGMAN_MSG_A_FMT(\"Invalid x87 precision mode, {}\", PC);\n }\n }\n\n auto RC = (FCW >> 10) & 3;\n switch (RC) {\n case 0: State.roundingMode = softfloat_round_near_even; break;\n case 1: State.roundingMode = softfloat_round_min; break;\n case 2: State.roundingMode = softfloat_round_max; break;\n case 3: State.roundingMode = softfloat_round_minMag; break;\n }\n\n return State;\n}\n\nFEXCORE_PRESERVE_ALL_ATTR static void HandleX87Exception(const softfloat_state& State, FEXCore::Core::CpuStateFrame* Frame) {\n // Check for Invalid Operation exception (bit 0 of X87 status word)\n if (State.exceptionFlags & softfloat_flag_invalid) {\n Frame->State.flags[FEXCore::X86State::X87FLAG_IE_LOC] = 1;\n }\n}\n\n// Wrapper for SoftFloat state to handle X87 exceptions\nclass ScopedSoftFloatState {\npublic:\n FEXCORE_PRESERVE_ALL_ATTR ScopedSoftFloatState(uint16_t FCW, FEXCore::Core::CpuStateFrame* Frame, bool Force80BitPrecision = false)\n : State(SoftFloatStateFromFCW(FCW, Force80BitPrecision))\n , Frame(Frame) {}\n\n FEXCORE_PRESERVE_ALL_ATTR ~ScopedSoftFloatState() {\n HandleX87Exception(State, Frame);\n }\n\n // Disable copy and move to ensure RAII semantics\n ScopedSoftFloatState(const ScopedSoftFloatState&) = delete;\n ScopedSoftFloatState& operator=(const ScopedSoftFloatState&) = delete;\n ScopedSoftFloatState(ScopedSoftFloatState&&) = delete;\n ScopedSoftFloatState& operator=(ScopedSoftFloatState&&) = delete;\n\n softfloat_state State;\n\nprivate:\n FEXCore::Core::CpuStateFrame* Frame;\n};\n\ntemplate<>\nstruct OpHandlers {\n FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle4(uint16_t FCW, float src, FEXCore::Core::CpuStateFrame* Frame) {\n FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);\n ScopedSoftFloatState State {FCW, Frame};\n return X80SoftFloat(&State.State, src);\n }\n\n FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle8(uint16_t FCW, double src, FEXCore::Core::CpuStateFrame* Frame) {\n FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);\n ScopedSoftFloatState State {FCW, Frame};\n return X80SoftFloat(&State.State, src);\n }\n};\n\ntemplate<>\nstruct OpHandlers {\n FEXCORE_PRESERVE_ALL_ATTR static uint64_t handle(uint16_t FCW, VectorRegType Src1, VectorRegType Src2, FEXCore::Core::CpuStateFrame* Frame) {\n FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);\n ScopedSoftFloatState State {FCW, Frame};\n\n bool eq, lt, nan;\n uint64_t ResultFlags = 0;\n\n X80SoftFloat::FCMP(&State.State, Src1, Src2, &eq, <, &nan);\n if (lt) {\n ResultFlags |= (1 << IR::FCMP_FLAG_LT);\n }\n if (nan) {\n ResultFlags |= (1 << IR::FCMP_FLAG_UNORDERED);\n }\n if (eq) {\n ResultFlags |= (1 << IR::FCMP_FLAG_EQ);\n }\n return ResultFlags;\n }\n};\n\ntemplate<>\nstruct OpHandlers {\n FEXCORE_PRESERVE_ALL_ATTR static float handle4(uint16_t FCW, VectorRegType src, FEXCore::Core::CpuStateFrame* Frame) {\n FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);\n ScopedSoftFloatState State {FCW, Frame};\n return X80SoftFloat(src).ToF32(&State.State);\n }\n\n FEXCORE_PRESERVE_ALL_ATTR static double handle8(uint16_t FCW, VectorRegType src, FEXCore::Core::CpuStateFrame* Frame) {\n FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);\n ScopedSoftFloatState State {FCW, Frame};\n return X80SoftFloat(src).ToF64(&State.State);\n }\n};\n\ntemplate<>\nstruct OpHandlers {\n FEXCORE_PRESERVE_ALL_ATTR static int16_t handle2(uint16_t FCW, VectorRegType src, FEXCore::Core::CpuStateFrame* Frame) {\n FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);\n ScopedSoftFloatState State {FCW, Frame};\n return X80SoftFloat(src).ToI16(&State.State);\n }\n\n FEXCORE_PRESERVE_ALL_ATTR static int32_t handle4(uint16_t FCW, VectorRegType src, FEXCore::Core::CpuStateFrame* Frame) {\n FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);\n ScopedSoftFloatState State {FCW, Frame};\n return X80SoftFloat(src).ToI32(&State.State);\n }\n\n FEXCORE_PRESERVE_ALL_ATTR static int64_t handle8(uint16_t FCW, VectorRegType src, FEXCore::Core::CpuStateFrame* Frame) {\n FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);\n ScopedSoftFloatState State {FCW, Frame};\n return X80SoftFloat(src).ToI64(&State.State);\n }\n\n FEXCORE_PRESERVE_ALL_ATTR static int16_t handle2t(uint16_t FCW, VectorRegType src, FEXCore::Core::CpuStateFrame* Frame) {\n FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);\n ScopedSoftFloatState State {FCW, Frame};\n auto rv = extF80_to_i32(&State.State, X80SoftFloat(src), softfloat_round_minMag, false);\n\n if (rv > INT16_MAX || rv < INT16_MIN) {\n ///< Indefinite value for 16-bit conversions.\n return INT16_MIN;\n } else {\n return rv;\n }\n }\n\n FEXCORE_PRESERVE_ALL_ATTR static int32_t handle4t(uint16_t FCW, VectorRegType src, FEXCore::Core::CpuStateFrame* Frame) {\n FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);\n ScopedSoftFloatState State {FCW, Frame};\n return extF80_to_i32(&State.State, X80SoftFloat(src), softfloat_round_minMag, false);\n }\n\n FEXCORE_PRESERVE_ALL_ATTR static int64_t handle8t(uint16_t FCW, VectorRegType src, FEXCore::Core::CpuStateFrame* Frame) {\n FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);\n ScopedSoftFloatState State {FCW, Frame};\n return extF80_to_i64(&State.State, X80SoftFloat(src), softfloat_round_minMag, false);\n }\n};\n\ntemplate<>\nstruct OpHandlers {\n FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle2(uint16_t FCW, int16_t src, FEXCore::Core::CpuStateFrame* Frame) {\n FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);\n return X80SoftFloat(src);\n }\n\n FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle4(uint16_t FCW, int32_t src, FEXCore::Core::CpuStateFrame* Frame) {\n FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);\n return X80SoftFloat(src);\n }\n};\n\ntemplate<>\nstruct OpHandlers {\n FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle(uint16_t FCW, VectorRegType Src1, FEXCore::Core::CpuStateFrame* Frame) {\n FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);\n ScopedSoftFloatState State {FCW, Frame, true};\n return X80SoftFloat::FRNDINT(&State.State, Src1);\n }\n};\n\ntemplate<>\nstruct OpHandlers {\n FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle(uint16_t FCW, VectorRegType Src1, FEXCore::Core::CpuStateFrame* Frame) {\n FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);\n ScopedSoftFloatState State {FCW, Frame, true};\n return X80SoftFloat::F2XM1(&State.State, Src1);\n }\n};\n\ntemplate<>\nstruct OpHandlers {\n FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle(uint16_t FCW, VectorRegType Src1, FEXCore::Core::CpuStateFrame* Frame) {\n FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);\n ScopedSoftFloatState State {FCW, Frame, true};\n return X80SoftFloat::FTAN(&State.State, Src1);\n }\n};\n\ntemplate<>\nstruct OpHandlers {\n FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle(uint16_t FCW, VectorRegType Src1, FEXCore::Core::CpuStateFrame* Frame) {\n FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);\n ScopedSoftFloatState State {FCW, Frame};\n return X80SoftFloat::FSQRT(&State.State, Src1);\n }\n};\n\ntemplate<>\nstruct OpHandlers {\n FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle(uint16_t FCW, VectorRegType Src1, FEXCore::Core::CpuStateFrame* Frame) {\n FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);\n ScopedSoftFloatState State {FCW, Frame, true};\n return X80SoftFloat::FSIN(&State.State, Src1);\n }\n};\n\ntemplate<>\nstruct OpHandlers {\n FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle(uint16_t FCW, VectorRegType Src1, FEXCore::Core::CpuStateFrame* Frame) {\n FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);\n ScopedSoftFloatState State {FCW, Frame, true};\n return X80SoftFloat::FCOS(&State.State, Src1);\n }\n};\n\ntemplate<>\nstruct OpHandlers {\n FEXCORE_PRESERVE_ALL_ATTR static VectorRegPairType handle(uint16_t FCW, VectorRegType Src1, FEXCore::Core::CpuStateFrame* Frame) {\n FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);\n ScopedSoftFloatState State {FCW, Frame, true};\n return FEXCore::MakeVectorRegPair(X80SoftFloat::FSIN(&State.State, Src1), X80SoftFloat::FCOS(&State.State, Src1));\n }\n};\n\ntemplate<>\nstruct OpHandlers {\n FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle(uint16_t FCW, VectorRegType Src1, FEXCore::Core::CpuStateFrame* Frame) {\n FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);\n return X80SoftFloat::FXTRACT_EXP(Src1);\n }\n};\n\ntemplate<>\nstruct OpHandlers {\n FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle(uint16_t FCW, VectorRegType Src1, FEXCore::Core::CpuStateFrame* Frame) {\n FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);\n return X80SoftFloat::FXTRACT_SIG(Src1);\n }\n};\n\ntemplate<>\nstruct OpHandlers {\n FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle(uint16_t FCW, VectorRegType Src1, VectorRegType Src2, FEXCore::Core::CpuStateFrame* Frame) {\n FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);\n ScopedSoftFloatState State {FCW, Frame};\n return X80SoftFloat::FADD(&State.State, Src1, Src2);\n }\n};\n\ntemplate<>\nstruct OpHandlers {\n FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle(uint16_t FCW, VectorRegType Src1, VectorRegType Src2, FEXCore::Core::CpuStateFrame* Frame) {\n FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);\n ScopedSoftFloatState State {FCW, Frame};\n return X80SoftFloat::FSUB(&State.State, Src1, Src2);\n }\n};\n\ntemplate<>\nstruct OpHandlers {\n FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle(uint16_t FCW, VectorRegType Src1, VectorRegType Src2, FEXCore::Core::CpuStateFrame* Frame) {\n FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);\n ScopedSoftFloatState State {FCW, Frame};\n return X80SoftFloat::FMUL(&State.State, Src1, Src2);\n }\n};\n\ntemplate<>\nstruct OpHandlers {\n FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle(uint16_t FCW, VectorRegType Src1, VectorRegType Src2, FEXCore::Core::CpuStateFrame* Frame) {\n FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);\n ScopedSoftFloatState State {FCW, Frame};\n return X80SoftFloat::FDIV(&State.State, Src1, Src2);\n }\n};\n\ntemplate<>\nstruct OpHandlers {\n FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle(uint16_t FCW, VectorRegType Src1, VectorRegType Src2, FEXCore::Core::CpuStateFrame* Frame) {\n FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);\n ScopedSoftFloatState State {FCW, Frame, true};\n return X80SoftFloat::FYL2X(&State.State, Src1, Src2);\n }\n};\n\ntemplate<>\nstruct OpHandlers {\n FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle(uint16_t FCW, VectorRegType Src1, VectorRegType Src2, FEXCore::Core::CpuStateFrame* Frame) {\n FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);\n ScopedSoftFloatState State {FCW, Frame, true};\n return X80SoftFloat::FATAN(&State.State, Src1, Src2);\n }\n};\n\ntemplate<>\nstruct OpHandlers {\n FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle(uint16_t FCW, VectorRegType Src1, VectorRegType Src2, FEXCore::Core::CpuStateFrame* Frame) {\n FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);\n ScopedSoftFloatState State {FCW, Frame, true};\n return X80SoftFloat::FREM1(&State.State, Src1, Src2);\n }\n};\n\ntemplate<>\nstruct OpHandlers {\n FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle(uint16_t FCW, VectorRegType Src1, VectorRegType Src2, FEXCore::Core::CpuStateFrame* Frame) {\n FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);\n ScopedSoftFloatState State {FCW, Frame, true};\n return X80SoftFloat::FREM(&State.State, Src1, Src2);\n }\n};\n\ntemplate<>\nstruct OpHandlers {\n FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle(uint16_t FCW, VectorRegType Src1, VectorRegType Src2, FEXCore::Core::CpuStateFrame* Frame) {\n FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);\n ScopedSoftFloatState State {FCW, Frame, true};\n return X80SoftFloat::FSCALE(&State.State, Src1, Src2);\n }\n};\n\ntemplate<>\nstruct OpHandlers {\n FEXCORE_PRESERVE_ALL_ATTR static double handle(double src, FEXCore::Core::CpuStateFrame* Frame) {\n FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);\n return sin(src);\n }\n};\n\ntemplate<>\nstruct OpHandlers {\n FEXCORE_PRESERVE_ALL_ATTR static double handle(double src, FEXCore::Core::CpuStateFrame* Frame) {\n FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);\n return cos(src);\n }\n};\n\ntemplate<>\nstruct OpHandlers {\n FEXCORE_PRESERVE_ALL_ATTR static VectorScalarF64Pair handle(double src, FEXCore::Core::CpuStateFrame* Frame) {\n FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);\n double sin, cos;\n#ifdef _WIN32\n sin = ::sin(src);\n cos = ::cos(src);\n#else\n sincos(src, &sin, &cos);\n#endif\n return VectorScalarF64Pair {sin, cos};\n }\n};\n\ntemplate<>\nstruct OpHandlers {\n FEXCORE_PRESERVE_ALL_ATTR static double handle(double src, FEXCore::Core::CpuStateFrame* Frame) {\n FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);\n return tan(src);\n }\n};\n\ntemplate<>\nstruct OpHandlers {\n FEXCORE_PRESERVE_ALL_ATTR static double handle(double src, FEXCore::Core::CpuStateFrame* Frame) {\n FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);\n return exp2(src) - 1.0;\n }\n};\n\ntemplate<>\nstruct OpHandlers {\n FEXCORE_PRESERVE_ALL_ATTR static double handle(double src1, double src2, FEXCore::Core::CpuStateFrame* Frame) {\n FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);\n return atan2(src1, src2);\n }\n};\n\ntemplate<>\nstruct OpHandlers {\n FEXCORE_PRESERVE_ALL_ATTR static double handle(double src1, double src2, FEXCore::Core::CpuStateFrame* Frame) {\n FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);\n return fmod(src1, src2);\n }\n};\n\ntemplate<>\nstruct OpHandlers {\n FEXCORE_PRESERVE_ALL_ATTR static double handle(double src1, double src2, FEXCore::Core::CpuStateFrame* Frame) {\n FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);\n return remainder(src1, src2);\n }\n};\n\ntemplate<>\nstruct OpHandlers {\n FEXCORE_PRESERVE_ALL_ATTR static double handle(double src1, double src2, FEXCore::Core::CpuStateFrame* Frame) {\n FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);\n return src2 * log2(src1);\n }\n};\n\ntemplate<>\nstruct OpHandlers {\n FEXCORE_PRESERVE_ALL_ATTR static double handle(double src1, double src2, FEXCore::Core::CpuStateFrame* Frame) {\n FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);\n if (src1 == 0.0) { // src1 might be +/- zero\n return src1; // this will return negative or positive zero if when appropriate\n }\n double trun = trunc(src2);\n return src1 * exp2(trun);\n }\n};\n\ntemplate<>\nstruct OpHandlers {\n FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle(uint16_t FCW, VectorRegType Src1q, FEXCore::Core::CpuStateFrame* Frame) {\n FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);\n X80SoftFloat Src1 = Src1q;\n ScopedSoftFloatState State {FCW, Frame};\n bool Negative = Src1.Top.Sign;\n\n Src1 = X80SoftFloat::FRNDINT(&State.State, Src1);\n\n // Clear the Sign bit\n Src1.Top.Sign = 0;\n\n uint64_t Tmp = Src1.ToI64(&State.State);\n X80SoftFloat Rv;\n uint8_t* BCD = reinterpret_cast(&Rv);\n\n for (size_t i = 0; i < 9; ++i) {\n if (Tmp == 0) {\n // Nothing left? Just leave\n break;\n }\n // Extract the lower 100 values\n uint8_t Digit = Tmp % 100;\n\n // Now divide it for the next iteration\n Tmp /= 100;\n\n uint8_t UpperNibble = Digit / 10;\n uint8_t LowerNibble = Digit % 10;\n\n // Now store the BCD\n BCD[i] = (UpperNibble << 4) | LowerNibble;\n }\n\n // Set negative flag once converted to x87\n BCD[9] = Negative ? 0x80 : 0;\n\n return Rv;\n }\n};\n\ntemplate<>\nstruct OpHandlers {\n FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle(uint16_t FCW, VectorRegType Src, FEXCore::Core::CpuStateFrame* Frame) {\n FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);\n uint8_t* Src1 = reinterpret_cast(&Src);\n uint64_t BCD {};\n // We walk through each uint8_t and pull out the BCD encoding\n // Each 4bit split is a digit\n // Only 0-9 is supported, A-F results in undefined data\n // | 4 bit | 4 bit |\n // | 10s place | 1s place |\n // EG 0x48 = 48\n // EG 0x4847 = 4847\n // This gives us an 18digit value encoded in BCD\n // The last byte lets us know if it negative or not\n for (size_t i = 0; i < 9; ++i) {\n uint8_t Digit = Src1[8 - i];\n // First shift our last value over\n BCD *= 100;\n\n // Add the tens place digit\n BCD += (Digit >> 4) * 10;\n\n // Add the ones place digit\n BCD += Digit & 0xF;\n }\n\n // Set negative flag once converted to x87\n bool Negative = Src1[9] & 0x80;\n X80SoftFloat Tmp;\n\n Tmp = BCD;\n Tmp.Top.Sign = Negative;\n return Tmp;\n }\n};\n\n} // namespace FEXCore::CPU\n" }, { "path": "FEXCore/Source/Interface/Core/Interpreter/Fallbacks/FallbackOpHandler.h", "content": "// SPDX-License-Identifier: MIT\n#pragma once\n\n#include \n\nnamespace FEXCore::IR {\nenum IROps : uint16_t;\n}\n\nnamespace FEXCore::CPU {\n\n// Base template for fallback handling.\n//\n// Registering and hooking up fallback is currently like so:\n//\n// 1. Go to InterpreterFallbacks.cpp and create a template specialization of\n// the GetFallbackInfo member function.\n//\n// This member function should reasonably define what the fallback you're\n// going to create will take as parameters and return as a result. For example:\n//\n// template<>\n// FallbackInfo GetFallbackInfo(X80SoftFloat(*fn)(double), Core::FallbackHandlerIndex Index) {\n// return {FABI_F80_F64, (void*)fn, Index};\n// }\n//\n// Defines info about a fallback that takes a double as an argument and\n// returns a X80SoftFloat instance.\n//\n// You will also want to define a new FallbackHandlerIndex enum member and use it\n// to set up the new info handler into the Info array in FillFallbackIndexPointers.\n//\n// 1.1. (potentially optional). Define a new ABI element in the FallbackAPI enum.\n// This ABI enum value will be used to tell the JITs how to handle the fallback\n// properly. These enum values specify the return type followed by its argument types.\n//\n// So, FABI_I64_F80_F80, for example indicates that the function will behave like a\n// function as if were defined as:\n//\n// uint64_t fn(X80SoftFloat, X80SoftFloat)\n//\n// 1.2. (potentially optional). If you needed to define a new enum ABI type like in 1.1, then\n// you need to add the handling for it in the JITs, which can be found in the respective\n// JIT's JIT.cpp file in a function called Op_Unhandled\n//\n// You need to add a new case to the ABI switch statement using the new ABI type\n// and do the necessary moving of data from register-allocated JIT parameters\n// into that platform's registers that respects the calling convention. After this is\n// done, most of the necessary background boilerplate is finished.\n//\n// 2. Now, make a specialization of this class with a member function named 'handle()'\n// that takes the same parameters as the ones described in the fallback info function\n// specialization.\n//\n// For example, if you have the fallback info from the example in step 1, it would be:\n//\n// template <>\n// struct OpHandlers {\n// static X80SoftFloat handle(double src) {\n// return ...;\n// }\n// };\n//\n// 3. Fill out the behavior of the OpHandler specialization to perform what you would like\n// the fallback to do.\n//\n// 4. Add an implementation of the IR op to the Interpreter that passes through to the\n// OpHandler implementation.\n//\n// 5. Done.\n//\ntemplate\nstruct OpHandlers {};\n\n} // namespace FEXCore::CPU\n" }, { "path": "FEXCore/Source/Interface/Core/Interpreter/Fallbacks/InterpreterFallbacks.cpp", "content": "// SPDX-License-Identifier: MIT\n#include \n\n#include \"Interface/Core/Interpreter/InterpreterOps.h\"\n#include \"Interface/Core/Interpreter/Fallbacks/F80Fallbacks.h\"\n#include \"Interface/Core/Interpreter/Fallbacks/VectorFallbacks.h\"\n\n#include \n#include \n\nnamespace FEXCore::CPU {\n\ntemplate\nstatic FallbackInfo GetFallbackInfo(R (*fn)(Args...), FEXCore::Core::FallbackHandlerIndex HandlerIndex) {\n return {FABI_UNKNOWN, HandlerIndex};\n}\n\nvoid InterpreterOps::FillFallbackIndexPointers(Core::FallbackABIInfo* Info, uint64_t* ABIHandlers) {\n Info[Core::OPINDEX_F80CVTTO_4] = {ABIHandlers[FABI_F80_I16_F32_PTR],\n reinterpret_cast(&FEXCore::CPU::OpHandlers::handle4)};\n Info[Core::OPINDEX_F80CVTTO_8] = {ABIHandlers[FABI_F80_I16_F64_PTR],\n reinterpret_cast(&FEXCore::CPU::OpHandlers::handle8)};\n Info[Core::OPINDEX_F80CVT_4] = {ABIHandlers[FABI_F32_I16_F80_PTR], reinterpret_cast(&FEXCore::CPU::OpHandlers::handle4)};\n Info[Core::OPINDEX_F80CVT_8] = {ABIHandlers[FABI_F64_I16_F80_PTR], reinterpret_cast(&FEXCore::CPU::OpHandlers::handle8)};\n Info[Core::OPINDEX_F80CVTINT_2] = {ABIHandlers[FABI_I16_I16_F80_PTR],\n reinterpret_cast(&FEXCore::CPU::OpHandlers::handle2)};\n Info[Core::OPINDEX_F80CVTINT_4] = {ABIHandlers[FABI_I32_I16_F80_PTR],\n reinterpret_cast(&FEXCore::CPU::OpHandlers::handle4)};\n Info[Core::OPINDEX_F80CVTINT_8] = {ABIHandlers[FABI_I64_I16_F80_PTR],\n reinterpret_cast(&FEXCore::CPU::OpHandlers::handle8)};\n Info[Core::OPINDEX_F80CVTINT_TRUNC2] = {ABIHandlers[FABI_I16_I16_F80_PTR],\n reinterpret_cast(&FEXCore::CPU::OpHandlers::handle2t)};\n Info[Core::OPINDEX_F80CVTINT_TRUNC4] = {ABIHandlers[FABI_I32_I16_F80_PTR],\n reinterpret_cast(&FEXCore::CPU::OpHandlers::handle4t)};\n Info[Core::OPINDEX_F80CVTINT_TRUNC8] = {ABIHandlers[FABI_I64_I16_F80_PTR],\n reinterpret_cast(&FEXCore::CPU::OpHandlers::handle8t)};\n Info[Core::OPINDEX_F80CMP] = {ABIHandlers[FABI_I64_I16_F80_F80_PTR],\n reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)};\n Info[Core::OPINDEX_F80CVTTOINT_2] = {ABIHandlers[FABI_F80_I16_I16_PTR],\n reinterpret_cast(&FEXCore::CPU::OpHandlers::handle2)};\n Info[Core::OPINDEX_F80CVTTOINT_4] = {ABIHandlers[FABI_F80_I16_I32_PTR],\n reinterpret_cast(&FEXCore::CPU::OpHandlers::handle4)};\n\n // Unary\n Info[Core::OPINDEX_F80ROUND] = {ABIHandlers[FABI_F80_I16_F80_PTR],\n reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)};\n Info[Core::OPINDEX_F80F2XM1] = {ABIHandlers[FABI_F80_I16_F80_PTR],\n reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)};\n Info[Core::OPINDEX_F80TAN] = {ABIHandlers[FABI_F80_I16_F80_PTR], reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)};\n Info[Core::OPINDEX_F80SQRT] = {ABIHandlers[FABI_F80_I16_F80_PTR], reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)};\n Info[Core::OPINDEX_F80SIN] = {ABIHandlers[FABI_F80_I16_F80_PTR], reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)};\n Info[Core::OPINDEX_F80COS] = {ABIHandlers[FABI_F80_I16_F80_PTR], reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)};\n Info[Core::OPINDEX_F80SINCOS] = {ABIHandlers[FABI_F80x2_I16_F80_PTR],\n reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)};\n Info[Core::OPINDEX_F80XTRACT_EXP] = {ABIHandlers[FABI_F80_I16_F80_PTR],\n reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)};\n Info[Core::OPINDEX_F80XTRACT_SIG] = {ABIHandlers[FABI_F80_I16_F80_PTR],\n reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)};\n Info[Core::OPINDEX_F80BCDSTORE] = {ABIHandlers[FABI_F80_I16_F80_PTR],\n reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)};\n Info[Core::OPINDEX_F80BCDLOAD] = {ABIHandlers[FABI_F80_I16_F80_PTR],\n reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)};\n\n // Binary\n Info[Core::OPINDEX_F80ADD] = {ABIHandlers[FABI_F80_I16_F80_F80_PTR],\n reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)};\n Info[Core::OPINDEX_F80SUB] = {ABIHandlers[FABI_F80_I16_F80_F80_PTR],\n reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)};\n Info[Core::OPINDEX_F80MUL] = {ABIHandlers[FABI_F80_I16_F80_F80_PTR],\n reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)};\n Info[Core::OPINDEX_F80DIV] = {ABIHandlers[FABI_F80_I16_F80_F80_PTR],\n reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)};\n Info[Core::OPINDEX_F80FYL2X] = {ABIHandlers[FABI_F80_I16_F80_F80_PTR],\n reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)};\n Info[Core::OPINDEX_F80ATAN] = {ABIHandlers[FABI_F80_I16_F80_F80_PTR],\n reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)};\n Info[Core::OPINDEX_F80FPREM1] = {ABIHandlers[FABI_F80_I16_F80_F80_PTR],\n reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)};\n Info[Core::OPINDEX_F80FPREM] = {ABIHandlers[FABI_F80_I16_F80_F80_PTR],\n reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)};\n Info[Core::OPINDEX_F80SCALE] = {ABIHandlers[FABI_F80_I16_F80_F80_PTR],\n reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)};\n\n // Double Precision Unary\n Info[Core::OPINDEX_F64SIN] = {ABIHandlers[FABI_F64_F64_PTR], reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)};\n Info[Core::OPINDEX_F64COS] = {ABIHandlers[FABI_F64_F64_PTR], reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)};\n Info[Core::OPINDEX_F64SINCOS] = {ABIHandlers[FABI_F64x2_F64_PTR],\n reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)};\n Info[Core::OPINDEX_F64TAN] = {ABIHandlers[FABI_F64_F64_PTR], reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)};\n Info[Core::OPINDEX_F64F2XM1] = {ABIHandlers[FABI_F64_F64_PTR], reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)};\n\n // Double Precision Binary\n Info[Core::OPINDEX_F64ATAN] = {ABIHandlers[FABI_F64_F64_F64_PTR], reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)};\n Info[Core::OPINDEX_F64FPREM] = {ABIHandlers[FABI_F64_F64_F64_PTR],\n reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)};\n Info[Core::OPINDEX_F64FPREM1] = {ABIHandlers[FABI_F64_F64_F64_PTR],\n reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)};\n Info[Core::OPINDEX_F64FYL2X] = {ABIHandlers[FABI_F64_F64_F64_PTR],\n reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)};\n Info[Core::OPINDEX_F64SCALE] = {ABIHandlers[FABI_F64_F64_F64_PTR],\n reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)};\n\n // SSE4.2 string instructions\n Info[Core::OPINDEX_VPCMPESTRX] = {ABIHandlers[FABI_I32_I64_I64_V128_V128_I16],\n reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)};\n Info[Core::OPINDEX_VPCMPISTRX] = {ABIHandlers[FABI_I32_V128_V128_I16],\n reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)};\n}\n\nbool InterpreterOps::GetFallbackHandler(const IR::IROp_Header* IROp, FallbackInfo* Info) {\n const auto OpSize = IROp->Size;\n switch (IROp->Op) {\n case IR::OP_F80CVTTO: {\n auto Op = IROp->C();\n\n switch (Op->SrcSize) {\n case IR::OpSize::i32Bit: {\n *Info = {FABI_F80_I16_F32_PTR, Core::OPINDEX_F80CVTTO_4};\n return true;\n }\n case IR::OpSize::i64Bit: {\n *Info = {FABI_F80_I16_F64_PTR, Core::OPINDEX_F80CVTTO_8};\n return true;\n }\n default: LogMan::Msg::DFmt(\"Unhandled size: {}\", OpSize);\n }\n break;\n }\n case IR::OP_F80CVT: {\n switch (OpSize) {\n case IR::OpSize::i32Bit: {\n *Info = {FABI_F32_I16_F80_PTR, Core::OPINDEX_F80CVT_4};\n return true;\n }\n case IR::OpSize::i64Bit: {\n *Info = {FABI_F64_I16_F80_PTR, Core::OPINDEX_F80CVT_8};\n return true;\n }\n default: LogMan::Msg::DFmt(\"Unhandled size: {}\", OpSize);\n }\n break;\n }\n case IR::OP_F80CVTINT: {\n auto Op = IROp->C();\n\n switch (OpSize) {\n case IR::OpSize::i16Bit: {\n if (Op->Truncate) {\n *Info = {FABI_I16_I16_F80_PTR, Core::OPINDEX_F80CVTINT_TRUNC2};\n } else {\n *Info = {FABI_I16_I16_F80_PTR, Core::OPINDEX_F80CVTINT_2};\n }\n return true;\n }\n case IR::OpSize::i32Bit: {\n if (Op->Truncate) {\n *Info = {FABI_I32_I16_F80_PTR, Core::OPINDEX_F80CVTINT_TRUNC4};\n } else {\n *Info = {FABI_I32_I16_F80_PTR, Core::OPINDEX_F80CVTINT_4};\n }\n return true;\n }\n case IR::OpSize::i64Bit: {\n if (Op->Truncate) {\n *Info = {FABI_I64_I16_F80_PTR, Core::OPINDEX_F80CVTINT_TRUNC8};\n } else {\n *Info = {FABI_I64_I16_F80_PTR, Core::OPINDEX_F80CVTINT_8};\n }\n return true;\n }\n default: LogMan::Msg::DFmt(\"Unhandled size: {}\", OpSize);\n }\n break;\n }\n case IR::OP_F80CMP: {\n *Info = {FABI_I64_I16_F80_F80_PTR, (Core::FallbackHandlerIndex)(Core::OPINDEX_F80CMP)};\n return true;\n }\n\n case IR::OP_F80CVTTOINT: {\n auto Op = IROp->C();\n\n switch (Op->SrcSize) {\n case IR::OpSize::i16Bit: {\n *Info = {FABI_F80_I16_I16_PTR, Core::OPINDEX_F80CVTTOINT_2};\n return true;\n }\n case IR::OpSize::i32Bit: {\n *Info = {FABI_F80_I16_I32_PTR, Core::OPINDEX_F80CVTTOINT_4};\n return true;\n }\n default: LogMan::Msg::DFmt(\"Unhandled size: {}\", OpSize);\n }\n break;\n }\n\n#define COMMON_UNARY_X87_OP(OP) \\\n case IR::OP_F80##OP: { \\\n *Info = {FABI_F80_I16_F80_PTR, Core::OPINDEX_F80##OP}; \\\n return true; \\\n }\n\n#define COMMON_UNARYPAIR_X87_OP(OP) \\\n case IR::OP_F80##OP: { \\\n *Info = {FABI_F80x2_I16_F80_PTR, Core::OPINDEX_F80##OP}; \\\n return true; \\\n }\n\n#define COMMON_BINARY_X87_OP(OP) \\\n case IR::OP_F80##OP: { \\\n *Info = {FABI_F80_I16_F80_F80_PTR, Core::OPINDEX_F80##OP}; \\\n return true; \\\n }\n\n#define COMMON_F64_OP(OP) \\\n case IR::OP_F64##OP: { \\\n *Info = GetFallbackInfo(&FEXCore::CPU::OpHandlers::handle, Core::OPINDEX_F64##OP); \\\n return true; \\\n }\n\n#define COMMON_UNARY_F64_OP(OP) \\\n case IR::OP_F64##OP: { \\\n *Info = {FABI_F64_F64_PTR, Core::OPINDEX_F64##OP}; \\\n return true; \\\n }\n#define COMMON_UNARYPAIR_F64_OP(OP) \\\n case IR::OP_F64##OP: { \\\n *Info = {FABI_F64x2_F64_PTR, Core::OPINDEX_F64##OP}; \\\n return true; \\\n }\n\n#define COMMON_BINARY_F64_OP(OP) \\\n case IR::OP_F64##OP: { \\\n *Info = {FABI_F64_F64_F64_PTR, Core::OPINDEX_F64##OP}; \\\n return true; \\\n }\n\n // Unary\n COMMON_UNARY_X87_OP(ROUND)\n COMMON_UNARY_X87_OP(F2XM1)\n COMMON_UNARY_X87_OP(TAN)\n COMMON_UNARY_X87_OP(SQRT)\n COMMON_UNARY_X87_OP(SIN)\n COMMON_UNARY_X87_OP(COS)\n COMMON_UNARYPAIR_X87_OP(SINCOS)\n COMMON_UNARY_X87_OP(XTRACT_EXP)\n COMMON_UNARY_X87_OP(XTRACT_SIG)\n COMMON_UNARY_X87_OP(BCDSTORE)\n COMMON_UNARY_X87_OP(BCDLOAD)\n\n // Binary\n COMMON_BINARY_X87_OP(ADD)\n COMMON_BINARY_X87_OP(SUB)\n COMMON_BINARY_X87_OP(MUL)\n COMMON_BINARY_X87_OP(DIV)\n COMMON_BINARY_X87_OP(FYL2X)\n COMMON_BINARY_X87_OP(ATAN)\n COMMON_BINARY_X87_OP(FPREM1)\n COMMON_BINARY_X87_OP(FPREM)\n COMMON_BINARY_X87_OP(SCALE)\n\n // Double Precision Unary\n COMMON_UNARY_F64_OP(F2XM1)\n COMMON_UNARY_F64_OP(TAN)\n COMMON_UNARY_F64_OP(SIN)\n COMMON_UNARY_F64_OP(COS)\n COMMON_UNARYPAIR_F64_OP(SINCOS)\n\n // Double Precision Binary\n COMMON_BINARY_F64_OP(FYL2X)\n COMMON_BINARY_F64_OP(ATAN)\n COMMON_BINARY_F64_OP(FPREM1)\n COMMON_BINARY_F64_OP(FPREM)\n COMMON_BINARY_F64_OP(SCALE)\n\n // SSE4.2 Fallbacks\n case IR::OP_VPCMPESTRX: *Info = {FABI_I32_I64_I64_V128_V128_I16, Core::OPINDEX_VPCMPESTRX}; return true;\n case IR::OP_VPCMPISTRX: *Info = {FABI_I32_V128_V128_I16, Core::OPINDEX_VPCMPISTRX}; return true;\n\n default: break;\n }\n\n return false;\n}\n\n\n} // namespace FEXCore::CPU\n" }, { "path": "FEXCore/Source/Interface/Core/Interpreter/Fallbacks/StringCompareFallbacks.cpp", "content": "// SPDX-License-Identifier: MIT\n#include \"Interface/Core/Interpreter/Fallbacks/VectorFallbacks.h\"\n#include \"Interface/IR/IR.h\"\n\n#ifdef ARCHITECTURE_arm64\n#include \n#endif\n\n#include \n\nnamespace FEXCore::CPU {\n#ifdef ARCHITECTURE_arm64\nFEXCORE_PRESERVE_ALL_ATTR static int32_t GetImplicitLength(FEXCore::VectorRegType data, uint16_t control) {\n const auto is_using_words = (control & 1) != 0;\n\n if (is_using_words) {\n uint16x8_t a = vreinterpretq_u16_u8(data);\n uint16x8_t VIndexes {};\n const uint16x8_t VIndex16 = vdupq_n_u16(8);\n uint16_t Indexes[8] = {\n 0, 1, 2, 3, 4, 5, 6, 7,\n };\n memcpy(&VIndexes, Indexes, sizeof(VIndexes));\n auto MaskResult = vceqzq_u16(a);\n auto SelectResult = vbslq_u16(MaskResult, VIndexes, VIndex16);\n return vminvq_u16(SelectResult);\n } else {\n uint8x16_t VIndexes {};\n const uint8x16_t VIndex16 = vdupq_n_u8(16);\n uint8_t Indexes[16] = {\n 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,\n };\n memcpy(&VIndexes, Indexes, sizeof(VIndexes));\n auto MaskResult = vceqzq_u8(data);\n auto SelectResult = vbslq_u8(MaskResult, VIndexes, VIndex16);\n return vminvq_u8(SelectResult);\n }\n}\n#else\nFEXCORE_PRESERVE_ALL_ATTR static int32_t GetImplicitLength(FEXCore::VectorRegType data, uint16_t control) {\n const auto* data_u8 = reinterpret_cast(&data);\n const auto is_using_words = (control & 1) != 0;\n\n int32_t length = 0;\n\n if (is_using_words) {\n const auto get_word = [data_u8](int32_t index) {\n const auto* src = data_u8 + (index * sizeof(uint16_t));\n\n uint16_t element {};\n std::memcpy(&element, src, sizeof(uint16_t));\n return element;\n };\n\n while (length < 8 && get_word(length) != 0) {\n length++;\n }\n } else {\n while (length < 16 && data_u8[length] != 0) {\n length++;\n }\n }\n\n return length;\n}\n#endif\n\n// Essentially the same in terms of behavior with VPCMPESTRX instructions,\n// with the only difference being that the length of the string is encoded\n// as part of the data vectors passed in.\n//\n// i.e. Length is determined by the presence of a NUL (all-zero) character\n// within the data.\n//\n// If no NUL character exists, then the length of the strings are assumed\n// to be the max length possible for the given character size specified\n// in the control flags (16 characters for 8-bit, and 8 characters for 16-bit).\n//\nFEXCORE_PRESERVE_ALL_ATTR uint32_t OpHandlers::handle(FEXCore::VectorRegType lhs, FEXCore::VectorRegType rhs, uint16_t control) {\n // Subtract by 1 in order to make validity limits 0-based\n const auto valid_lhs = GetImplicitLength(lhs, control) - 1;\n const auto valid_rhs = GetImplicitLength(rhs, control) - 1;\n __uint128_t lhs_i;\n memcpy(&lhs_i, &lhs, sizeof(lhs_i));\n __uint128_t rhs_i;\n memcpy(&rhs_i, &rhs, sizeof(rhs_i));\n\n return OpHandlers::MainBody(lhs_i, valid_lhs, rhs_i, valid_rhs, control);\n}\n\n} // namespace FEXCore::CPU\n" }, { "path": "FEXCore/Source/Interface/Core/Interpreter/Fallbacks/VectorFallbacks.h", "content": "// SPDX-License-Identifier: MIT\n#pragma once\n\n#include \n#include \n#include \n#include \n#include \n\n#include \"Interface/Core/Interpreter/Fallbacks/FallbackOpHandler.h\"\n#include \"Interface/IR/IR.h\"\n#include \"Common/VectorRegType.h\"\n\nnamespace FEXCore::CPU {\n\ntemplate<>\nstruct OpHandlers {\n enum class AggregationOp {\n EqualAny = 0b00,\n Ranges = 0b01,\n EqualEach = 0b10,\n EqualOrdered = 0b11,\n };\n\n enum class SourceData {\n U8,\n U16,\n S8,\n S16,\n };\n\n enum class Polarity {\n Positive,\n Negative,\n PositiveMasked,\n NegativeMasked,\n };\n\n FEXCORE_PRESERVE_ALL_ATTR static uint32_t handle(uint64_t RAX, uint64_t RDX, VectorRegType lhs_v, VectorRegType rhs_v, uint16_t control) {\n __uint128_t lhs;\n memcpy(&lhs, &lhs_v, sizeof(lhs));\n __uint128_t rhs;\n memcpy(&rhs, &rhs_v, sizeof(rhs));\n\n // Subtract by 1 in order to make validity limits 0-based\n const auto valid_lhs = GetExplicitLength(RAX, control) - 1;\n const auto valid_rhs = GetExplicitLength(RDX, control) - 1;\n\n return MainBody(lhs, valid_lhs, rhs, valid_rhs, control);\n }\n\n // Main PCMPXSTRX algorithm body. Allows for reuse with both implicit and explicit length variants.\n FEXCORE_PRESERVE_ALL_ATTR static uint32_t MainBody(const __uint128_t& lhs, int valid_lhs, const __uint128_t& rhs, int valid_rhs, uint16_t control) {\n const uint32_t aggregation = PerformAggregation(lhs, valid_lhs, rhs, valid_rhs, control);\n const int32_t upper_limit = (16 >> (control & 1)) - 1;\n\n // Bits are arranged as:\n // Bit #: 3 2 1 0\n // [SF | ZF | CF | OF]\n uint32_t flags = 0;\n flags |= (valid_rhs < upper_limit) ? 0b0100 : 0b0000;\n flags |= (valid_lhs < upper_limit) ? 0b1000 : 0b0000;\n\n const uint32_t result = HandlePolarity(aggregation, control, upper_limit, valid_rhs);\n if (result != 0) {\n flags |= 0b0010;\n }\n if ((result & 1) != 0) {\n flags |= 0b0001;\n }\n\n // We track the flags in the usual NZCV bit position so we can msr them\n // later. Avoids handling flags natively in JIT.\n return result | (flags << 28);\n }\n\n FEXCORE_PRESERVE_ALL_ATTR static int32_t GetExplicitLength(uint64_t reg, uint16_t control) {\n // Bit 8 controls whether or not the reg value is 64-bit or 32-bit.\n int64_t value = 0;\n if (((control >> 8) & 1) != 0) {\n value = static_cast(reg);\n } else {\n // We need a sign extend in this case.\n value = static_cast(reg);\n }\n\n // If control[0] is set, then we're dealing with words instead of bytes\n const int64_t limit = (control & 1) != 0 ? 8 : 16;\n\n // Length needs to saturate to 16 (if bytes) or 8 (if words)\n // when the length value is greater than 16 (if bytes)/8 (if words)\n // or if the length value is less than -16 (if bytes)/-8 (if words).\n if (value < -limit || value > limit) {\n return limit;\n }\n\n return std::abs(static_cast(value));\n }\n\n FEXCORE_PRESERVE_ALL_ATTR static int32_t GetElement(const __uint128_t& vec, int32_t index, uint16_t control) {\n const auto* vec_ptr = reinterpret_cast(&vec);\n\n // Control bits [1:0] define the data type being dealt with.\n switch (static_cast(control & 0b11)) {\n case SourceData::U8: return static_cast(vec_ptr[index]);\n case SourceData::U16: {\n uint16_t value {};\n std::memcpy(&value, vec_ptr + (sizeof(uint16_t) * static_cast(index)), sizeof(value));\n return value;\n }\n case SourceData::S8: return static_cast(vec_ptr[index]);\n case SourceData::S16:\n default: {\n int16_t value {};\n std::memcpy(&value, vec_ptr + (sizeof(int16_t) * static_cast(index)), sizeof(value));\n return value;\n }\n }\n }\n\n FEXCORE_PRESERVE_ALL_ATTR static uint32_t\n PerformAggregation(const __uint128_t& lhs, int32_t valid_lhs, const __uint128_t& rhs, int32_t valid_rhs, uint16_t control) {\n switch (static_cast((control >> 2) & 0b11)) {\n case AggregationOp::EqualAny: return HandleEqualAny(lhs, valid_lhs, rhs, valid_rhs, control);\n case AggregationOp::Ranges: return HandleRanges(lhs, valid_lhs, rhs, valid_rhs, control);\n case AggregationOp::EqualEach: return HandleEqualEach(lhs, valid_lhs, rhs, valid_rhs, control);\n case AggregationOp::EqualOrdered:\n default: return HandleEqualOrdered(lhs, valid_lhs, rhs, valid_rhs, control);\n }\n }\n\n FEXCORE_PRESERVE_ALL_ATTR static uint32_t HandlePolarity(uint32_t value, uint16_t control, int upper_limit, int valid_rhs) {\n switch (static_cast((control >> 4) & 0b11)) {\n case Polarity::Negative: return value ^ ((2U << upper_limit) - 1);\n case Polarity::NegativeMasked: return value ^ ((1U << (valid_rhs + 1)) - 1);\n case Polarity::Positive:\n case Polarity::PositiveMasked:\n default:\n // Both positive masking and positive polarity are documented\n // as both being equivalent to \"IntRes2 = IntRes1\", where IntRes1\n // is our 'value' parameter, so we don't need to do anything in\n // these cases except return the same value.\n return value;\n }\n }\n\n // Finds characters from an overall character set.\n //\n // Scans through RHS trying to find any characters contained in LHS.\n // Think of this as a sort of vectorized version of strspn (kind of).\n //\n // e.g. Assume operating on two character vectors as unsigned words\n //\n // 0 1 2 3 4 5 6 7\n // LHS -> [a, b, c, d, e, f, g, n]\n // RHS -> [z, k, v, c, d, o, p, n]\n //\n // With both explicit lengths for each string being 8 (the max length for words),\n // this would result in an intermediate result like:\n //\n // 0b1001'1000\n // │ │ │\n // 'n' match ───┘ │ │\n // │ │\n // 'd' match ──────┘ │\n // │\n // 'c' match ────────┘\n //\n FEXCORE_PRESERVE_ALL_ATTR static uint32_t\n HandleEqualAny(const __uint128_t& lhs, int32_t valid_lhs, const __uint128_t& rhs, int32_t valid_rhs, uint16_t control) {\n uint32_t result = 0;\n\n for (int j = valid_rhs; j >= 0; j--) {\n result <<= 1;\n\n const int rhs_value = GetElement(rhs, j, control);\n for (int i = valid_lhs; i >= 0; i--) {\n const int lhs_value = GetElement(lhs, i, control);\n result |= static_cast(rhs_value == lhs_value);\n }\n }\n\n return result;\n }\n\n // Determines if a character falls within a limited range\n //\n // Scans through rhs using a range denoted by two elements\n // in lhs and determines if the respective character in rhs\n // falls within its range.\n //\n // i.e.\n // lhs_upper_bound >= rhs_value && lhs_lower_bound <= rhs_value\n //\n // e.g. Assume operating on two character vectors as unsigned words\n //\n // 0 1 2 3 4 5 6 7\n // LHS -> [a, z, A, Z, 0, 0, 0, 0]\n // RHS -> [z, k, ., C, M, ;, \\, ']\n //\n // With LHS's length being 4 and RHS's lenth being 8,\n // this would result in an intermediate result like:\n //\n // 0b0001'1011\n // │ │ ││\n // 'z' >= 'M' && 'a' <= 'M' ─────┘ │ ││\n // │ ││\n // 'z' >= 'C' && 'a' <= 'C' ───────┘ ││\n // ││\n // 'Z' >= 'k' && 'A' <= 'k' ─────────┘│\n // │\n // 'Z' >= 'z' && 'A' <= 'z' ──────────┘\n //\n FEXCORE_PRESERVE_ALL_ATTR static uint32_t\n HandleRanges(const __uint128_t& lhs, int32_t valid_lhs, const __uint128_t& rhs, int32_t valid_rhs, uint16_t control) {\n uint32_t result = 0;\n\n for (int j = valid_rhs; j >= 0; j--) {\n result <<= 1;\n\n const int element = GetElement(rhs, j, control);\n for (int i = (valid_lhs - 1) | 1; i >= 0; i -= 2) {\n const int upper_bound = GetElement(lhs, i - 0, control);\n const int lower_bound = GetElement(lhs, i - 1, control);\n\n const bool ge = upper_bound >= element;\n const bool le = lower_bound <= element;\n\n result |= static_cast(ge && le);\n }\n }\n\n return result;\n }\n\n // Determines if each character is equal to one another (string compare)\n //\n // Essentially the PCMPXSTRX variant of memcmp/strcmp. Sets the bit of the\n // resulting mask if both elements are equal to one another. Otherwise\n // sets it to false.\n //\n // e.g. Assume operating on two character vectors as unsigned words\n //\n // 0 1 2 3 4 5 6 7\n // LHS -> [a, b, c, d, e, f, g, n]\n // RHS -> [a, b, c, d, e, f, e, x]\n //\n // With both explicit lengths for each string being 8 (the max length for words),\n // this would result in an intermediate result like:\n //\n // 0b0011'1111\n // ││ ││││\n // 'f' == 'f' ────┘│ ││││\n // │ ││││\n // 'e' == 'e' ─────┘ ││││\n // ││││\n // 'd' == 'd' ───────┘│││\n // │││\n // 'c' == 'c' ────────┘││\n // ││\n // 'b' == 'b' ─────────┘│\n // │\n // 'a' == 'a' ──────────┘\n //\n FEXCORE_PRESERVE_ALL_ATTR static uint32_t\n HandleEqualEach(const __uint128_t& lhs, int32_t valid_lhs, const __uint128_t& rhs, int32_t valid_rhs, uint16_t control) {\n const auto upper_limit = (16 >> (control & 1)) - 1;\n const auto max_valid = std::max(valid_lhs, valid_rhs);\n const auto min_valid = std::min(valid_lhs, valid_rhs);\n\n // All values past the end of string must be forced to true.\n // (See 4.1.6 Valid/Invalid Override of Comparisons in the Intel Software Development Manual)\n // So we can calculate this part of the mask ahead of time and set all those to-be bits to true\n // and then progressively shift them into place over the course of execution.\n uint32_t result = (1U << (upper_limit - max_valid)) - 1;\n result <<= (max_valid - min_valid);\n\n for (int i = min_valid; i >= 0; i--) {\n const int lhs_element = GetElement(lhs, i, control);\n const int rhs_element = GetElement(rhs, i, control);\n\n result <<= 1;\n result |= static_cast(lhs_element == rhs_element);\n }\n\n return result;\n }\n\n // Determines if a substring exists within an overall string\n //\n // Somewhat equivalent to the behavior of strstr.\n //\n // Sets the corresponding index in the result where a substring is found.\n //\n // e.g. Assume operating on two character vectors as unsigned words\n //\n // 0 1 2 3 4 5 6 7\n // LHS -> [b, a, x, z, y, v, o, m]\n // RHS -> [b, a, d, b, a, n, k, s]\n //\n // With the length of LHS being 2 and the length of RHS being 8, we have a composition like:\n //\n // Substring to look for\n // ┌──┴──┐\n // LHS -> [b, a, x, z, y, v, o, m]\n // RHS -> [b, a, d, b, a, n, k, s]\n // └───────────┬────────────┘\n // Entire string to search\n //\n // And we end up with a result like:\n //\n // 0b0000'1001\n // │ │\n // At index 3 ───────┘ │\n // │\n // At index 0 ──────────┘\n //\n FEXCORE_PRESERVE_ALL_ATTR static uint32_t\n HandleEqualOrdered(const __uint128_t& lhs, int32_t valid_lhs, const __uint128_t& rhs, int32_t valid_rhs, uint16_t control) {\n const auto upper_limit = (16 >> (control & 1)) - 1;\n\n // Edge case!\n // If we have *no* valid characters in our inner string, then\n // we need to return the intermediate result as\n // 0xFF (if operating on words) or 0xFFFF (if operating on bytes)\n if (valid_lhs == -1) {\n return (2U << upper_limit) - 1;\n }\n\n uint32_t result = 0;\n const int initial = valid_rhs == upper_limit ? valid_rhs : valid_rhs - valid_lhs;\n for (int j = initial; j >= 0; j--) {\n result <<= 1;\n\n uint32_t value = 1;\n const int start = std::min(valid_rhs - j, valid_lhs);\n for (int i = start; i >= 0; i--) {\n const int lhs_value = GetElement(lhs, i + 0, control);\n const int rhs_value = GetElement(rhs, i + j, control);\n\n value &= static_cast(lhs_value == rhs_value);\n }\n\n result |= value;\n }\n\n return result;\n }\n};\n\ntemplate<>\nstruct OpHandlers {\n FEXCORE_PRESERVE_ALL_ATTR static uint32_t handle(VectorRegType lhs, VectorRegType rhs, uint16_t control);\n};\n\n} // namespace FEXCore::CPU\n" }, { "path": "FEXCore/Source/Interface/Core/Interpreter/InterpreterOps.h", "content": "// SPDX-License-Identifier: MIT\n#pragma once\n\n#include \n\n#include \n#include \n\nnamespace FEXCore::IR {\nclass IRListView;\nstruct IROp_Header;\n} // namespace FEXCore::IR\n\nnamespace FEXCore::CPU {\nenum FallbackABI {\n FABI_F80_I16_F32_PTR,\n FABI_F80_I16_F64_PTR,\n FABI_F80_I16_I16_PTR,\n FABI_F80_I16_I32_PTR,\n FABI_F32_I16_F80_PTR,\n FABI_F64_I16_F80_PTR,\n FABI_F64_F64_PTR,\n FABI_F64_F64_F64_PTR,\n FABI_I16_I16_F80_PTR,\n FABI_I32_I16_F80_PTR,\n FABI_I64_I16_F80_PTR,\n FABI_I64_I16_F80_F80_PTR,\n FABI_F80_I16_F80_PTR,\n FABI_F80_I16_F80_F80_PTR,\n FABI_F80x2_I16_F80_PTR,\n FABI_F64x2_F64_PTR,\n FABI_I32_I64_I64_V128_V128_I16,\n FABI_I32_V128_V128_I16,\n FABI_UNKNOWN,\n};\nstruct FallbackInfo {\n FallbackABI ABI;\n FEXCore::Core::FallbackHandlerIndex HandlerIndex;\n};\n\nclass InterpreterOps {\npublic:\n static void FillFallbackIndexPointers(Core::FallbackABIInfo* Info, uint64_t* ABIHandlers);\n static bool GetFallbackHandler(const IR::IROp_Header* IROp, FallbackInfo* Info);\n};\n} // namespace FEXCore::CPU\n" }, { "path": "FEXCore/Source/Interface/Core/JIT/ALUOps.cpp", "content": "// SPDX-License-Identifier: MIT\n/*\n$info$\ntags: backend|arm64\n$end_info$\n*/\n\n#include \"CodeEmitter/Emitter.h\"\n#include \"FEXCore/IR/IR.h\"\n#include \"Interface/Context/Context.h\"\n#include \"Interface/Core/JIT/JITClass.h\"\n#include \"Interface/IR/Passes/RegisterAllocationPass.h\"\n\nnamespace FEXCore::CPU {\n\n#define GRD(Node) (IROp->Size <= 4 ? GetDst(Node) : GetDst(Node))\n#define GRS(Node) (IROp->Size <= 4 ? GetReg(Node) : GetReg(Node))\n\n#define DEF_BINOP_WITH_CONSTANT(FEXOp, VarOp, ConstOp) \\\n DEF_OP(FEXOp) { \\\n auto Op = IROp->C(); \\\n \\\n uint64_t Const; \\\n if (IsInlineConstant(Op->Src2, &Const)) { \\\n ConstOp(ConvertSize(IROp), GetReg(Node), GetReg(Op->Src1), Const); \\\n } else { \\\n VarOp(ConvertSize(IROp), GetReg(Node), GetZeroableReg(Op->Src1), GetReg(Op->Src2)); \\\n } \\\n }\n\nDEF_BINOP_WITH_CONSTANT(Add, add, add)\nDEF_BINOP_WITH_CONSTANT(Sub, sub, sub)\nDEF_BINOP_WITH_CONSTANT(AddWithFlags, adds, adds)\nDEF_BINOP_WITH_CONSTANT(SubWithFlags, subs, subs)\nDEF_BINOP_WITH_CONSTANT(Or, orr, orr)\nDEF_BINOP_WITH_CONSTANT(And, and_, and_)\nDEF_BINOP_WITH_CONSTANT(Andn, bic, bic)\nDEF_BINOP_WITH_CONSTANT(Xor, eor, eor)\nDEF_BINOP_WITH_CONSTANT(Lshl, lslv, lsl)\nDEF_BINOP_WITH_CONSTANT(Lshr, lsrv, lsr)\nDEF_BINOP_WITH_CONSTANT(Ror, rorv, ror)\n\nDEF_OP(Constant) {\n auto Op = IROp->C();\n auto Dst = GetReg(Node);\n\n const auto PadType = [Pad = Op->Pad]() {\n switch (Pad) {\n case IR::ConstPad::NoPad: return CPU::Arm64Emitter::PadType::NOPAD;\n case IR::ConstPad::DoPad: return CPU::Arm64Emitter::PadType::DOPAD;\n default: return CPU::Arm64Emitter::PadType::AUTOPAD;\n }\n }();\n LoadConstant(ARMEmitter::Size::i64Bit, Dst, Op->Constant, PadType, Op->MaxBytes);\n}\n\nDEF_OP(EntrypointOffset) {\n auto Op = IROp->C();\n\n auto Constant = Entry + Op->Offset;\n uint64_t Mask = ~0ULL;\n const auto OpSize = IROp->Size;\n if (OpSize == IR::OpSize::i32Bit) {\n Mask = 0xFFFF'FFFFULL;\n }\n\n InsertGuestRIPMove(GetReg(Node), Constant & Mask);\n}\n\nDEF_OP(InlineConstant) {\n // nop\n}\n\nDEF_OP(InlineEntrypointOffset) {\n // nop\n}\n\nDEF_OP(CycleCounter) {\n auto Op = IROp->C();\n if (CTX->HostFeatures.SupportsECV && Op->SelfSynchronizingLoads) {\n // CNTVCTSS_EL0 is \"self-synchronizing\", which means loads can't speculate past this instruction.\n // Stores still aren't synchronized although.\n mrs(GetReg(Node), ARMEmitter::SystemRegister::CNTVCTSS_EL0);\n } else {\n if (Op->SelfSynchronizingLoads) {\n // If ECV isn't supported then an ISB must be emitted to synchronize all instructions and loads before the cycle read.\n isb();\n }\n mrs(GetReg(Node), ARMEmitter::SystemRegister::CNTVCT_EL0);\n }\n}\n\nDEF_OP(AddShift) {\n auto Op = IROp->C();\n\n add(ConvertSize48(IROp), GetReg(Node), GetReg(Op->Src1), GetReg(Op->Src2), ConvertIRShiftType(Op->Shift), Op->ShiftAmount);\n}\n\nDEF_OP(AddNZCV) {\n auto Op = IROp->C();\n\n const auto EmitSize = ConvertSize(IROp);\n auto Src1 = GetReg(Op->Src1);\n\n uint64_t Const;\n if (IsInlineConstant(Op->Src2, &Const)) {\n LOGMAN_THROW_A_FMT(IROp->Size >= IR::OpSize::i32Bit, \"Constant not allowed here\");\n cmn(EmitSize, Src1, Const);\n } else if (IROp->Size < IR::OpSize::i32Bit) {\n unsigned Shift = 32 - IR::OpSizeAsBits(IROp->Size);\n\n lsl(ARMEmitter::Size::i32Bit, TMP1, Src1, Shift);\n cmn(EmitSize, TMP1, GetReg(Op->Src2), ARMEmitter::ShiftType::LSL, Shift);\n } else {\n cmn(EmitSize, Src1, GetReg(Op->Src2));\n }\n}\n\nDEF_OP(AdcNZCV) {\n auto Op = IROp->C();\n\n adcs(ConvertSize48(IROp), ARMEmitter::Reg::zr, GetReg(Op->Src1), GetReg(Op->Src2));\n}\n\nDEF_OP(AdcWithFlags) {\n auto Op = IROp->C();\n\n adcs(ConvertSize48(IROp), GetReg(Node), GetZeroableReg(Op->Src1), GetReg(Op->Src2));\n}\n\nDEF_OP(AdcZeroWithFlags) {\n auto Op = IROp->C();\n auto Size = ConvertSize48(IROp);\n\n cset(Size, TMP1, ARMEmitter::Condition::CC_CC);\n adds(Size, GetReg(Node), GetReg(Op->Src1), TMP1);\n}\n\nDEF_OP(AdcZero) {\n auto Op = IROp->C();\n auto Size = ConvertSize48(IROp);\n\n cinc(Size, GetReg(Node), GetReg(Op->Src1), ARMEmitter::Condition::CC_CC);\n}\n\nDEF_OP(Adc) {\n auto Op = IROp->C();\n\n adc(ConvertSize48(IROp), GetReg(Node), GetZeroableReg(Op->Src1), GetReg(Op->Src2));\n}\n\nDEF_OP(SbbWithFlags) {\n auto Op = IROp->C();\n\n sbcs(ConvertSize48(IROp), GetReg(Node), GetReg(Op->Src1), GetReg(Op->Src2));\n}\n\nDEF_OP(SbbNZCV) {\n auto Op = IROp->C();\n\n sbcs(ConvertSize48(IROp), ARMEmitter::Reg::zr, GetReg(Op->Src1), GetReg(Op->Src2));\n}\n\nDEF_OP(Sbb) {\n auto Op = IROp->C();\n\n sbc(ConvertSize48(IROp), GetReg(Node), GetZeroableReg(Op->Src1), GetReg(Op->Src2));\n}\n\nDEF_OP(TestNZ) {\n auto Op = IROp->C();\n const auto EmitSize = ConvertSize(IROp);\n\n uint64_t Const;\n auto Src1 = GetReg(Op->Src1);\n\n // Shift the sign bit into place, clearing out the garbage in upper bits.\n // Adding zero does an effective test, setting NZ according to the result and\n // zeroing CV.\n if (IROp->Size < IR::OpSize::i32Bit) {\n // Cheaper to and+cmn than to lsl+lsl+tst, so do the and ourselves if\n // needed.\n if (Op->Src1 != Op->Src2) {\n if (IsInlineConstant(Op->Src2, &Const)) {\n and_(EmitSize, TMP1, Src1, Const);\n } else {\n auto Src2 = GetReg(Op->Src2);\n and_(EmitSize, TMP1, Src1, Src2);\n }\n\n Src1 = TMP1;\n }\n\n unsigned Shift = 32 - IR::OpSizeAsBits(IROp->Size);\n cmn(EmitSize, ARMEmitter::Reg::zr, Src1, ARMEmitter::ShiftType::LSL, Shift);\n } else {\n if (IsInlineConstant(Op->Src2, &Const)) {\n tst(EmitSize, Src1, Const);\n } else {\n const auto Src2 = GetReg(Op->Src2);\n tst(EmitSize, Src1, Src2);\n }\n }\n}\n\nDEF_OP(TestZ) {\n auto Op = IROp->C();\n LOGMAN_THROW_A_FMT(IROp->Size < IR::OpSize::i32Bit, \"TestNZ used at higher sizes\");\n const auto EmitSize = ARMEmitter::Size::i32Bit;\n\n uint64_t Const;\n uint64_t Mask = IROp->Size == IR::OpSize::i64Bit ? ~0ULL : ((1ull << IR::OpSizeAsBits(IROp->Size)) - 1);\n auto Src1 = GetReg(Op->Src1);\n\n if (IsInlineConstant(Op->Src2, &Const)) {\n // We can promote 8/16-bit tests to 32-bit since the constant is masked.\n LOGMAN_THROW_A_FMT(!(Const & ~Mask), \"constant is already masked\");\n tst(EmitSize, Src1, Const);\n } else {\n const auto Src2 = GetReg(Op->Src2);\n if (Src1 == Src2) {\n tst(EmitSize, Src1 /* Src2 */, Mask);\n } else {\n and_(EmitSize, TMP1, Src1, Src2);\n tst(EmitSize, TMP1, Mask);\n }\n }\n}\n\nDEF_OP(SubShift) {\n auto Op = IROp->C();\n\n sub(ConvertSize48(IROp), GetReg(Node), GetReg(Op->Src1), GetReg(Op->Src2), ConvertIRShiftType(Op->Shift), Op->ShiftAmount);\n}\n\nDEF_OP(SubNZCV) {\n auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n const auto EmitSize = ConvertSize(IROp);\n\n uint64_t Const;\n if (IsInlineConstant(Op->Src2, &Const)) {\n LOGMAN_THROW_A_FMT(OpSize >= IR::OpSize::i32Bit, \"Constant not allowed here\");\n cmp(EmitSize, GetReg(Op->Src1), Const);\n } else {\n unsigned Shift = OpSize < IR::OpSize::i32Bit ? (32 - IR::OpSizeAsBits(OpSize)) : 0;\n ARMEmitter::Register ShiftedSrc1 = GetZeroableReg(Op->Src1);\n\n // Shift to fix flags for <32-bit ops.\n // Any shift of zero is still zero so optimize out silly zero shifts.\n if (OpSize < IR::OpSize::i32Bit && ShiftedSrc1 != ARMEmitter::Reg::zr) {\n lsl(ARMEmitter::Size::i32Bit, TMP1, ShiftedSrc1, Shift);\n ShiftedSrc1 = TMP1;\n }\n\n if (OpSize < IR::OpSize::i32Bit) {\n cmp(EmitSize, ShiftedSrc1, GetReg(Op->Src2), ARMEmitter::ShiftType::LSL, Shift);\n } else {\n cmp(EmitSize, ShiftedSrc1, GetReg(Op->Src2));\n }\n }\n}\n\nDEF_OP(CmpPairZ) {\n auto Op = IROp->C();\n const auto EmitSize = ConvertSize(IROp);\n\n // Save NZCV\n mrs(TMP1, ARMEmitter::SystemRegister::NZCV);\n\n // Compare, setting Z and clobbering NzCV\n cmp(EmitSize, GetReg(Op->Src1Lo), GetReg(Op->Src2Lo));\n ccmp(EmitSize, GetReg(Op->Src1Hi), GetReg(Op->Src2Hi), ARMEmitter::StatusFlags::None, ARMEmitter::Condition::CC_EQ);\n\n // Restore NzCV\n if (CTX->HostFeatures.SupportsFlagM) {\n rmif(TMP1, 0, 0xb /* NzCV */);\n } else {\n cset(ARMEmitter::Size::i32Bit, TMP2, ARMEmitter::Condition::CC_EQ);\n bfi(ARMEmitter::Size::i32Bit, TMP1, TMP2, 30 /* lsb: Z */, 1);\n msr(ARMEmitter::SystemRegister::NZCV, TMP1);\n }\n}\n\nDEF_OP(CarryInvert) {\n LOGMAN_THROW_A_FMT(CTX->HostFeatures.SupportsFlagM, \"Unsupported flagm op\");\n cfinv();\n}\n\nDEF_OP(RmifNZCV) {\n auto Op = IROp->C();\n LOGMAN_THROW_A_FMT(CTX->HostFeatures.SupportsFlagM, \"Unsupported flagm op\");\n\n rmif(GetZeroableReg(Op->Src).X(), Op->Rotate, Op->Mask);\n}\n\nDEF_OP(SetSmallNZV) {\n auto Op = IROp->C();\n LOGMAN_THROW_A_FMT(CTX->HostFeatures.SupportsFlagM, \"Unsupported flagm op\");\n\n const auto OpSize = IROp->Size;\n LOGMAN_THROW_A_FMT(OpSize == IR::OpSize::i8Bit || OpSize == IR::OpSize::i16Bit, \"Unsupported {} size: {}\", __func__, OpSize);\n\n if (OpSize == IR::OpSize::i8Bit) {\n setf8(GetReg(Op->Src).W());\n } else {\n setf16(GetReg(Op->Src).W());\n }\n}\n\nDEF_OP(AXFlag) {\n if (CTX->HostFeatures.SupportsFlagM2) {\n axflag();\n } else {\n // AXFLAG is defined in the Arm spec as\n //\n // gt: nzCv -> nzCv\n // lt: Nzcv -> nzcv <==> 1 + 0\n // eq: nZCv -> nZCv <==> 1 + (~0)\n // un: nzCV -> nZcv <==> 0 + 0\n //\n // For the latter 3 cases, we therefore get the right NZCV by adding V_inv\n // to (eq ? ~0 : 0). The remaining case is forced with ccmn.\n auto V_inv = GetReg(IROp->Args[0]);\n csetm(ARMEmitter::Size::i64Bit, TMP1, ARMEmitter::Condition::CC_EQ);\n ccmn(ARMEmitter::Size::i64Bit, V_inv, TMP1, ARMEmitter::StatusFlags {0x2} /* nzCv */, ARMEmitter::Condition::CC_LE);\n }\n}\n\nDEF_OP(Parity) {\n auto Op = IROp->C();\n auto Raw = GetReg(Op->Raw);\n auto Dest = GetReg(Node);\n\n // Cascade to calculate parity of bottom 8-bits to bottom bit.\n eor(ARMEmitter::Size::i32Bit, TMP1, Raw, Raw, ARMEmitter::ShiftType::LSR, 4);\n eor(ARMEmitter::Size::i32Bit, TMP1, TMP1, TMP1, ARMEmitter::ShiftType::LSR, 2);\n\n if (Op->Invert) {\n eon(ARMEmitter::Size::i32Bit, Dest, TMP1, TMP1, ARMEmitter::ShiftType::LSR, 1);\n } else {\n eor(ARMEmitter::Size::i32Bit, Dest, TMP1, TMP1, ARMEmitter::ShiftType::LSR, 1);\n }\n\n // The above sequence leaves garbage in the upper bits.\n if (Op->Mask) {\n and_(ARMEmitter::Size::i32Bit, Dest, Dest, 1);\n }\n}\n\nDEF_OP(CondAddNZCV) {\n auto Op = IROp->C();\n\n ARMEmitter::StatusFlags Flags = (ARMEmitter::StatusFlags)Op->FalseNZCV;\n uint64_t Const = 0;\n auto Src1 = GetZeroableReg(Op->Src1);\n\n if (IsInlineConstant(Op->Src2, &Const)) {\n ccmn(ConvertSize48(IROp), Src1, Const, Flags, MapCC(Op->Cond));\n } else {\n ccmn(ConvertSize48(IROp), Src1, GetReg(Op->Src2), Flags, MapCC(Op->Cond));\n }\n}\n\nDEF_OP(CondSubNZCV) {\n auto Op = IROp->C();\n\n ARMEmitter::StatusFlags Flags = (ARMEmitter::StatusFlags)Op->FalseNZCV;\n uint64_t Const = 0;\n auto Src1 = GetZeroableReg(Op->Src1);\n\n if (IsInlineConstant(Op->Src2, &Const)) {\n ccmp(ConvertSize48(IROp), Src1, Const, Flags, MapCC(Op->Cond));\n } else {\n ccmp(ConvertSize48(IROp), Src1, GetReg(Op->Src2), Flags, MapCC(Op->Cond));\n }\n}\n\nDEF_OP(Neg) {\n auto Op = IROp->C();\n\n if (Op->Cond == IR::CondClass::AL) {\n neg(ConvertSize48(IROp), GetReg(Node), GetReg(Op->Src));\n } else {\n cneg(ConvertSize48(IROp), GetReg(Node), GetReg(Op->Src), MapCC(Op->Cond));\n }\n}\n\nDEF_OP(Mul) {\n auto Op = IROp->C();\n\n mul(ConvertSize48(IROp), GetReg(Node), GetReg(Op->Src1), GetReg(Op->Src2));\n}\n\nDEF_OP(UMul) {\n auto Op = IROp->C();\n\n mul(ConvertSize48(IROp), GetReg(Node), GetReg(Op->Src1), GetReg(Op->Src2));\n}\n\nDEF_OP(UMull) {\n auto Op = IROp->C();\n umull(GetReg(Node).X(), GetReg(Op->Src1).W(), GetReg(Op->Src2).W());\n}\n\nDEF_OP(SMull) {\n auto Op = IROp->C();\n smull(GetReg(Node).X(), GetReg(Op->Src1).W(), GetReg(Op->Src2).W());\n}\n\nDEF_OP(MulH) {\n auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n LOGMAN_THROW_A_FMT(OpSize == IR::OpSize::i32Bit || OpSize == IR::OpSize::i64Bit, \"Unsupported {} size: {}\", __func__, OpSize);\n\n const auto Dst = GetReg(Node);\n const auto Src1 = GetReg(Op->Src1);\n const auto Src2 = GetReg(Op->Src2);\n\n if (OpSize == IR::OpSize::i32Bit) {\n sxtw(TMP1, Src1.W());\n sxtw(TMP2, Src2.W());\n mul(ARMEmitter::Size::i32Bit, Dst, TMP1, TMP2);\n ubfx(ARMEmitter::Size::i32Bit, Dst, Dst, 32, 32);\n } else {\n smulh(Dst.X(), Src1.X(), Src2.X());\n }\n}\n\nDEF_OP(UMulH) {\n auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n LOGMAN_THROW_A_FMT(OpSize == IR::OpSize::i32Bit || OpSize == IR::OpSize::i64Bit, \"Unsupported {} size: {}\", __func__, OpSize);\n\n const auto Dst = GetReg(Node);\n const auto Src1 = GetReg(Op->Src1);\n const auto Src2 = GetReg(Op->Src2);\n\n if (OpSize == IR::OpSize::i32Bit) {\n uxtw(ARMEmitter::Size::i64Bit, TMP1, Src1);\n uxtw(ARMEmitter::Size::i64Bit, TMP2, Src2);\n mul(ARMEmitter::Size::i64Bit, Dst, TMP1, TMP2);\n ubfx(ARMEmitter::Size::i64Bit, Dst, Dst, 32, 32);\n } else {\n umulh(Dst.X(), Src1.X(), Src2.X());\n }\n}\n\nDEF_OP(Orlshl) {\n auto Op = IROp->C();\n const auto Dst = GetReg(Node);\n const auto Src1 = GetReg(Op->Src1);\n\n uint64_t Const;\n if (IsInlineConstant(Op->Src2, &Const)) {\n orr(ConvertSize(IROp), Dst, Src1, Const << Op->BitShift);\n } else {\n const auto Src2 = GetReg(Op->Src2);\n orr(ConvertSize(IROp), Dst, Src1, Src2, ARMEmitter::ShiftType::LSL, Op->BitShift);\n }\n}\n\nDEF_OP(Orlshr) {\n auto Op = IROp->C();\n\n const auto Dst = GetReg(Node);\n const auto Src1 = GetReg(Op->Src1);\n\n uint64_t Const;\n if (IsInlineConstant(Op->Src2, &Const)) {\n orr(ConvertSize(IROp), Dst, Src1, Const >> Op->BitShift);\n } else {\n const auto Src2 = GetReg(Op->Src2);\n orr(ConvertSize(IROp), Dst, Src1, Src2, ARMEmitter::ShiftType::LSR, Op->BitShift);\n }\n}\n\nDEF_OP(Ornror) {\n auto Op = IROp->C();\n\n const auto Dst = GetReg(Node);\n const auto Src1 = GetReg(Op->Src1);\n\n const auto Src2 = GetReg(Op->Src2);\n orn(ConvertSize(IROp), Dst, Src1, Src2, ARMEmitter::ShiftType::ROR, Op->BitShift);\n}\n\nDEF_OP(AndWithFlags) {\n auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n const auto EmitSize = ConvertSize(IROp);\n\n uint64_t Const;\n const auto Dst = GetReg(Node);\n auto Src1 = GetReg(Op->Src1);\n\n // See TestNZ\n if (OpSize < IR::OpSize::i32Bit) {\n if (IsInlineConstant(Op->Src2, &Const)) {\n and_(EmitSize, Dst, Src1, Const);\n } else {\n auto Src2 = GetReg(Op->Src2);\n\n if (Src1 != Src2) {\n and_(EmitSize, Dst, Src1, Src2);\n } else if (Dst != Src1) {\n mov(ARMEmitter::Size::i64Bit, Dst, Src1);\n }\n }\n\n unsigned Shift = 32 - IR::OpSizeAsBits(OpSize);\n cmn(EmitSize, ARMEmitter::Reg::zr, Dst, ARMEmitter::ShiftType::LSL, Shift);\n } else {\n if (IsInlineConstant(Op->Src2, &Const)) {\n ands(EmitSize, Dst, Src1, Const);\n } else {\n const auto Src2 = GetReg(Op->Src2);\n ands(EmitSize, Dst, Src1, Src2);\n }\n }\n}\n\nDEF_OP(AndShift) {\n auto Op = IROp->C();\n\n and_(ConvertSize48(IROp), GetReg(Node), GetReg(Op->Src1), GetReg(Op->Src2), ConvertIRShiftType(Op->Shift), Op->ShiftAmount);\n}\n\nDEF_OP(XorShift) {\n auto Op = IROp->C();\n\n eor(ConvertSize48(IROp), GetReg(Node), GetReg(Op->Src1), GetReg(Op->Src2), ConvertIRShiftType(Op->Shift), Op->ShiftAmount);\n}\n\nDEF_OP(XornShift) {\n auto Op = IROp->C();\n\n eon(ConvertSize48(IROp), GetReg(Node), GetReg(Op->Src1), GetReg(Op->Src2), ConvertIRShiftType(Op->Shift), Op->ShiftAmount);\n}\n\nDEF_OP(Ashr) {\n auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n const auto EmitSize = ConvertSize(IROp);\n\n const auto Dst = GetReg(Node);\n const auto Src1 = GetReg(Op->Src1);\n\n uint64_t Const;\n if (IsInlineConstant(Op->Src2, &Const)) {\n if (OpSize >= IR::OpSize::i32Bit) {\n asr(EmitSize, Dst, Src1, (unsigned int)Const);\n } else {\n sbfx(EmitSize, TMP1, Src1, 0, IR::OpSizeAsBits(OpSize));\n asr(EmitSize, Dst, TMP1, (unsigned int)Const);\n ubfx(EmitSize, Dst, Dst, 0, IR::OpSizeAsBits(OpSize));\n }\n } else {\n const auto Src2 = GetReg(Op->Src2);\n if (OpSize >= IR::OpSize::i32Bit) {\n asrv(EmitSize, Dst, Src1, Src2);\n } else {\n sbfx(EmitSize, TMP1, Src1, 0, IR::OpSizeAsBits(OpSize));\n asrv(EmitSize, Dst, TMP1, Src2);\n ubfx(EmitSize, Dst, Dst, 0, IR::OpSizeAsBits(OpSize));\n }\n }\n}\n\nDEF_OP(ShiftFlags) {\n auto Op = IROp->C();\n const auto OpSize = Op->Size;\n const auto EmitSize = OpSize == IR::OpSize::i64Bit ? ARMEmitter::Size::i64Bit : ARMEmitter::Size::i32Bit;\n\n const auto PFOutput = GetReg(Node);\n const auto PFInput = GetReg(Op->PFInput);\n const auto Dst = GetReg(Op->Result);\n const auto Src1 = GetReg(Op->Src1);\n const auto Src2 = GetReg(Op->Src2);\n\n bool PFBlocked = (PFOutput == Dst) || (PFOutput == Src1) || (PFOutput == Src2);\n const auto PFTemp = PFBlocked ? TMP4 : PFOutput;\n\n // Set the output outside the branch to avoid needing an extra leg of the\n // branch. We specifically do not hardcode the PF register anywhere (relying\n // on a tied SRA register instead) to avoid fighting with RA.\n if (PFTemp != PFInput) {\n mov(ARMEmitter::Size::i64Bit, PFTemp, PFInput);\n }\n\n // We need to mask the source before comparing it. We don't just skip flag\n // updates for Src2=0 but anything that masks to zero.\n and_(ARMEmitter::Size::i32Bit, TMP1, Src2, OpSize == IR::OpSize::i64Bit ? 0x3f : 0x1f);\n\n ARMEmitter::ForwardLabel Done;\n (void)cbz(EmitSize, TMP1, &Done);\n {\n // PF/SF/ZF/OF\n if (OpSize >= IR::OpSize::i32Bit) {\n ands(EmitSize, PFTemp, Dst, Dst);\n } else {\n unsigned Shift = 32 - (IR::OpSizeToSize(OpSize) * 8);\n cmn(EmitSize, ARMEmitter::Reg::zr, Dst, ARMEmitter::ShiftType::LSL, Shift);\n mov(ARMEmitter::Size::i64Bit, PFTemp, Dst);\n }\n\n auto CFWord = TMP1;\n unsigned CFBit = 0;\n\n // Extract the last bit shifted in to CF\n if (Op->Shift == IR::ShiftType::LSL) {\n if (OpSize >= IR::OpSize::i32Bit) {\n neg(EmitSize, CFWord, Src2);\n lsrv(EmitSize, CFWord, Src1, CFWord);\n } else {\n CFWord = Dst.X();\n CFBit = IR::OpSizeToSize(OpSize) * 8;\n }\n } else {\n sub(ARMEmitter::Size::i64Bit, CFWord, Src2, 1);\n lsrv(EmitSize, CFWord, Src1, CFWord);\n }\n\n if (Op->InvertCF) {\n mvn(ARMEmitter::Size::i64Bit, TMP1, CFWord);\n CFWord = TMP1;\n }\n\n bool SetOF = Op->Shift != IR::ShiftType::ASR;\n if (SetOF) {\n // Only defined when Shift is 1 else undefined\n // OF flag is set if a sign change occurred\n eor(EmitSize, TMP3, Src1, Dst);\n }\n\n if (CTX->HostFeatures.SupportsFlagM) {\n rmif(CFWord, (CFBit - 1) % 64, (1 << 1) /* C */);\n\n if (SetOF) {\n rmif(TMP3, IR::OpSizeToSize(OpSize) * 8 - 1, (1 << 0) /* V */);\n }\n } else {\n mrs(TMP2, ARMEmitter::SystemRegister::NZCV);\n\n if (CFBit != 0) {\n lsr(ARMEmitter::Size::i64Bit, TMP1, CFWord, CFBit);\n CFWord = TMP1;\n }\n\n bfi(ARMEmitter::Size::i32Bit, TMP2, CFWord, 29 /* C */, 1);\n\n if (SetOF) {\n lsr(EmitSize, TMP3, TMP3, IR::OpSizeToSize(OpSize) * 8 - 1);\n bfi(ARMEmitter::Size::i32Bit, TMP2, TMP3, 28 /* V */, 1);\n }\n\n msr(ARMEmitter::SystemRegister::NZCV, TMP2);\n }\n }\n (void)Bind(&Done);\n\n // TODO: Make RA less dumb so this can't happen (e.g. with late-kill).\n if (PFOutput != PFTemp) {\n mov(ARMEmitter::Size::i64Bit, PFOutput, PFTemp);\n }\n}\n\nDEF_OP(RotateFlags) {\n auto Op = IROp->C();\n const auto Result = GetReg(Op->Result);\n const auto Shift = GetReg(Op->Shift);\n const bool Left = Op->Left;\n const auto EmitSize = Op->Size == IR::OpSize::i64Bit ? ARMEmitter::Size::i64Bit : ARMEmitter::Size::i32Bit;\n\n // If shift=0, flags are unaffected. Wrap the whole implementation in a cbz.\n ARMEmitter::ForwardLabel Done;\n (void)cbz(EmitSize, Shift, &Done);\n {\n // Extract the last bit shifted in to CF\n const auto BitSize = IR::OpSizeToSize(Op->Size) * 8;\n unsigned CFBit = Left ? 0 : BitSize - 1;\n\n // For ROR, OF is the XOR of the new CF bit and the most significant bit of the result.\n // For ROL, OF is the LSB and MSB XOR'd together.\n // OF is architecturally only defined for 1-bit rotate.\n eor(ARMEmitter::Size::i64Bit, TMP1, Result, Result, ARMEmitter::ShiftType::LSR, Left ? BitSize - 1 : 1);\n unsigned OFBit = Left ? 0 : BitSize - 2;\n\n // Invert result so we get inverted carry.\n mvn(ARMEmitter::Size::i64Bit, TMP2, Result);\n\n if (CTX->HostFeatures.SupportsFlagM) {\n rmif(TMP2, (CFBit - 1) % 64, 1 << 1 /* nzCv */);\n rmif(TMP1, OFBit, 1 << 0 /* nzcV */);\n } else {\n if (OFBit != 0) {\n lsr(EmitSize, TMP1, TMP1, OFBit);\n }\n if (CFBit != 0) {\n lsr(EmitSize, TMP2, TMP2, CFBit);\n }\n\n mrs(TMP3, ARMEmitter::SystemRegister::NZCV);\n bfi(ARMEmitter::Size::i32Bit, TMP3, TMP1, 28 /* V */, 1);\n bfi(ARMEmitter::Size::i32Bit, TMP3, TMP2, 29 /* C */, 1);\n msr(ARMEmitter::SystemRegister::NZCV, TMP3);\n }\n }\n (void)Bind(&Done);\n}\n\nDEF_OP(Extr) {\n auto Op = IROp->C();\n const auto Dst = GetReg(Node);\n const auto Upper = GetReg(Op->Upper);\n const auto Lower = GetReg(Op->Lower);\n\n extr(ConvertSize48(IROp), Dst, Upper, Lower, Op->LSB);\n}\n\nDEF_OP(PDep) {\n auto Op = IROp->C();\n const auto EmitSize = ConvertSize48(IROp);\n\n const auto Dest = GetReg(Node);\n\n // We can't clobber these\n const auto OrigInput = GetReg(Op->Input);\n const auto OrigMask = GetReg(Op->Mask);\n\n if (CTX->HostFeatures.SupportsSVEBitPerm) {\n // SVE added support for PDEP but it needs to be done in a vector register.\n if (EmitSize == ARMEmitter::Size::i32Bit) {\n fmov(ARMEmitter::Size::i32Bit, VTMP1.S(), OrigInput.W());\n fmov(ARMEmitter::Size::i32Bit, VTMP2.S(), OrigMask.W());\n bdep(ARMEmitter::SubRegSize::i32Bit, VTMP1.Z(), VTMP1.Z(), VTMP2.Z());\n umov(Dest, VTMP1, 0);\n } else {\n fmov(ARMEmitter::Size::i64Bit, VTMP1.D(), OrigInput.X());\n fmov(ARMEmitter::Size::i64Bit, VTMP2.D(), OrigMask.X());\n bdep(ARMEmitter::SubRegSize::i64Bit, VTMP1.Z(), VTMP1.Z(), VTMP2.Z());\n umov(Dest, VTMP1, 0);\n }\n } else {\n // PDep implementation follows the ideas from\n // http://0x80.pl/articles/pdep-soft-emu.html ... Basically, iterate the *set*\n // bits only, which will be faster than the naive implementation as long as\n // there are enough holes in the mask.\n //\n // The specific arm64 assembly used is based on the sequence that clang\n // generates for the C code, giving context to the scheduling yielding better\n // ILP than I would do by hand. The registers are allocated by hand however,\n // to fit within the tight constraints we have here withot spilling. Also, we\n // use cbz/cbnz for conditional branching to avoid clobbering NZCV.\n\n // So we have shadow as temporaries\n const auto Input = TMP1.R();\n const auto Mask = TMP2.R();\n\n // these get used variously as scratch\n const auto T0 = TMP3.R();\n const auto T1 = TMP4.R();\n\n ARMEmitter::BackwardLabel NextBit;\n ARMEmitter::ForwardLabel Done;\n\n // First, copy the input/mask, since we'll be clobbering. Copy as 64-bit to\n // make this 0-uop on Firestorm.\n mov(ARMEmitter::Size::i64Bit, Input, OrigInput);\n mov(ARMEmitter::Size::i64Bit, Mask, OrigMask);\n\n // Now, they're copied, so we can start setting Dest (even if it overlaps with\n // one of them). Handle early exit case\n mov(EmitSize, Dest, 0);\n (void)cbz(EmitSize, OrigMask, &Done);\n\n // Setup for first iteration\n neg(EmitSize, T0, Mask);\n and_(EmitSize, T0, T0, Mask);\n\n // Main loop\n (void)Bind(&NextBit);\n sbfx(EmitSize, T1, Input, 0, 1);\n eor(EmitSize, Mask, Mask, T0);\n and_(EmitSize, T0, T1, T0);\n neg(EmitSize, T1, Mask);\n orr(EmitSize, Dest, Dest, T0);\n lsr(EmitSize, Input, Input, 1);\n and_(EmitSize, T0, Mask, T1);\n (void)cbnz(EmitSize, T0, &NextBit);\n\n // All done with nothing to do.\n (void)Bind(&Done);\n }\n}\n\nDEF_OP(PExt) {\n auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n const auto OpSizeBitsM1 = IR::OpSizeAsBits(OpSize) - 1;\n const auto EmitSize = ConvertSize48(IROp);\n\n const auto Input = GetReg(Op->Input);\n const auto Mask = GetReg(Op->Mask);\n const auto Dest = GetReg(Node);\n\n if (CTX->HostFeatures.SupportsSVEBitPerm) {\n // SVE added support for PEXT but it needs to be done in a vector register.\n if (EmitSize == ARMEmitter::Size::i32Bit) {\n fmov(ARMEmitter::Size::i32Bit, VTMP1.S(), Input.W());\n fmov(ARMEmitter::Size::i32Bit, VTMP2.S(), Mask.W());\n bext(ARMEmitter::SubRegSize::i32Bit, VTMP1.Z(), VTMP1.Z(), VTMP2.Z());\n umov(Dest, VTMP1, 0);\n } else {\n fmov(ARMEmitter::Size::i64Bit, VTMP1.D(), Input.X());\n fmov(ARMEmitter::Size::i64Bit, VTMP2.D(), Mask.X());\n bext(ARMEmitter::SubRegSize::i64Bit, VTMP1.Z(), VTMP1.Z(), VTMP2.Z());\n umov(Dest, VTMP1, 0);\n }\n } else {\n const auto MaskReg = TMP1;\n const auto BitReg = TMP2;\n const auto ValueReg = TMP3;\n\n ARMEmitter::ForwardLabel EarlyExit;\n ARMEmitter::BackwardLabel NextBit;\n ARMEmitter::ForwardLabel Done;\n\n (void)cbz(EmitSize, Mask, &EarlyExit);\n mov(EmitSize, MaskReg, Mask);\n mov(EmitSize, ValueReg, Input);\n mov(EmitSize, Dest, ARMEmitter::Reg::zr);\n\n // Main loop\n (void)Bind(&NextBit);\n (void)cbz(EmitSize, MaskReg, &Done);\n clz(EmitSize, BitReg, MaskReg);\n lslv(EmitSize, ValueReg, ValueReg, BitReg);\n lslv(EmitSize, MaskReg, MaskReg, BitReg);\n extr(EmitSize, Dest, Dest, ValueReg, OpSizeBitsM1);\n bfc(EmitSize, MaskReg, OpSizeBitsM1, 1);\n (void)b(&NextBit);\n\n // Early exit\n (void)Bind(&EarlyExit);\n mov(EmitSize, Dest, ARMEmitter::Reg::zr);\n\n // All done with nothing to do.\n (void)Bind(&Done);\n }\n}\n\nDEF_OP(Div) {\n auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto Quotient = GetReg(Op->OutQuotient);\n const auto Remainder = GetReg(Op->OutRemainder);\n auto Lower = GetReg(Op->Lower);\n auto Divisor = GetReg(Op->Divisor);\n\n if (Op->Upper.IsInvalid()) {\n const auto EmitSize = ConvertSize(IROp);\n\n if (OpSize == IR::OpSize::i8Bit) {\n sxtb(EmitSize, TMP1, Lower);\n sxtb(EmitSize, TMP2, Divisor);\n\n Lower = TMP1;\n Divisor = TMP2;\n } else if (OpSize == IR::OpSize::i16Bit) {\n sxth(EmitSize, TMP1, Lower);\n sxth(EmitSize, TMP2, Divisor);\n\n Lower = TMP1;\n Divisor = TMP2;\n }\n\n sdiv(EmitSize, Quotient, Lower, Divisor);\n msub(EmitSize, Remainder, Quotient, Divisor, Lower);\n return;\n }\n\n const auto EmitSize = OpSize >= IR::OpSize::i32Bit ? ARMEmitter::Size::i64Bit : ARMEmitter::Size::i32Bit;\n const auto Upper = GetReg(Op->Upper);\n\n // Each source is OpSize in size\n // So you can have up to a 128bit divide from x86-64\n switch (OpSize) {\n case IR::OpSize::i16Bit: {\n uxth(EmitSize, TMP1, Lower);\n bfi(EmitSize, TMP1, Upper, 16, 16);\n sxth(EmitSize, TMP2, Divisor);\n sdiv(EmitSize, Quotient, TMP1, TMP2);\n msub(EmitSize, Remainder, Quotient, TMP2, TMP1);\n break;\n }\n case IR::OpSize::i32Bit: {\n // TODO: 32-bit operation should be guaranteed not to leave garbage in the upper bits.\n mov(EmitSize, TMP1, Lower);\n bfi(EmitSize, TMP1, Upper, 32, 32);\n sxtw(TMP2, Divisor.W());\n sdiv(EmitSize, Quotient, TMP1, TMP2);\n msub(EmitSize, Remainder, Quotient, TMP2, TMP1);\n break;\n }\n case IR::OpSize::i64Bit: {\n ARMEmitter::ForwardLabel Only64Bit {};\n ARMEmitter::ForwardLabel LongDIVRet {};\n\n // Check if the upper bits match the top bit of the lower 64-bits\n // Sign extend the top bit of lower bits\n sbfx(EmitSize, TMP1, Lower, 63, 1);\n eor(EmitSize, TMP1, TMP1, Upper);\n\n // If the sign bit matches then the result is zero\n (void)cbz(EmitSize, TMP1, &Only64Bit);\n\n // Long divide\n {\n mov(EmitSize, TMP1, Upper);\n mov(EmitSize, TMP2, Lower);\n mov(EmitSize, TMP3, Divisor);\n\n ldr(TMP4, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.LDIVHandler));\n\n str(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, -16);\n blr(TMP4);\n ldr(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, 16);\n\n // Move results to the destination registers\n mov(EmitSize, Quotient, TMP1);\n mov(EmitSize, Remainder, TMP2);\n\n // Skip 64-bit path\n (void)b(&LongDIVRet);\n }\n\n (void)Bind(&Only64Bit);\n // 64-Bit only\n {\n sdiv(EmitSize, Quotient, Lower, Divisor);\n msub(EmitSize, Remainder, Quotient, Divisor, Lower);\n }\n\n (void)Bind(&LongDIVRet);\n break;\n }\n default: LOGMAN_MSG_A_FMT(\"Unknown DIV Size: {}\", OpSize); break;\n }\n}\n\nDEF_OP(UDiv) {\n auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto Quotient = GetReg(Op->OutQuotient);\n const auto Remainder = GetReg(Op->OutRemainder);\n const auto Lower = GetReg(Op->Lower);\n const auto Divisor = GetReg(Op->Divisor);\n\n // Each source is OpSize in size\n // So you can have up to a 128bit divide from x86-64=\n if (Op->Upper.IsInvalid()) {\n const auto EmitSize = ConvertSize(IROp);\n udiv(EmitSize, Quotient, Lower, Divisor);\n msub(EmitSize, Remainder, Quotient, Divisor, Lower);\n return;\n }\n\n const auto EmitSize = OpSize >= IR::OpSize::i32Bit ? ARMEmitter::Size::i64Bit : ARMEmitter::Size::i32Bit;\n const auto Upper = GetReg(Op->Upper);\n\n switch (OpSize) {\n case IR::OpSize::i16Bit: {\n uxth(EmitSize, TMP1, Lower);\n bfi(EmitSize, TMP1, Upper, 16, 16);\n udiv(EmitSize, Quotient, TMP1, Divisor);\n msub(EmitSize, Remainder, Quotient, Divisor, TMP1);\n break;\n }\n case IR::OpSize::i32Bit: {\n // We need to mask divisor if we have Upper bits, since the frontend does\n // not on the hope that we can optimize to use the path above.\n mov(ARMEmitter::Size::i32Bit, TMP2, Divisor);\n\n // TODO: 32-bit operation should be guaranteed not to leave garbage in the upper bits.\n mov(EmitSize, TMP1, Lower);\n bfi(EmitSize, TMP1, Upper, 32, 32);\n udiv(EmitSize, Quotient, TMP1, TMP2);\n msub(EmitSize, Remainder, Quotient, TMP2, TMP1);\n break;\n }\n case IR::OpSize::i64Bit: {\n ARMEmitter::ForwardLabel Only64Bit {};\n ARMEmitter::ForwardLabel LongDIVRet {};\n\n // Check the upper bits for zero\n // If the upper bits are zero then we can do a 64-bit divide\n (void)cbz(EmitSize, Upper, &Only64Bit);\n\n // Long divide\n {\n mov(EmitSize, TMP1, Upper);\n mov(EmitSize, TMP2, Lower);\n mov(EmitSize, TMP3, Divisor);\n\n ldr(TMP4, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.LUDIVHandler));\n\n str(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, -16);\n blr(TMP4);\n ldr(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, 16);\n\n // Move results to the destination registers\n mov(EmitSize, Quotient, TMP1);\n mov(EmitSize, Remainder, TMP2);\n\n // Skip 64-bit path\n (void)b(&LongDIVRet);\n }\n\n (void)Bind(&Only64Bit);\n // 64-Bit only\n {\n udiv(EmitSize, Quotient, Lower, Divisor);\n msub(EmitSize, Remainder, Quotient, Divisor, Lower);\n }\n\n (void)Bind(&LongDIVRet);\n break;\n }\n default: LOGMAN_MSG_A_FMT(\"Unknown LUDIV Size: {}\", OpSize); break;\n }\n}\n\nDEF_OP(Not) {\n auto Op = IROp->C();\n\n const auto Dst = GetReg(Node);\n const auto Src = GetReg(Op->Src);\n\n mvn(ConvertSize48(IROp), Dst, Src);\n}\n\nDEF_OP(Popcount) {\n auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto Dst = GetReg(Node);\n const auto Src = GetReg(Op->Src);\n\n if (CTX->HostFeatures.SupportsCSSC) {\n switch (OpSize) {\n case IR::OpSize::i8Bit:\n uxtb(ARMEmitter::Size::i32Bit, Dst, Src);\n cnt(ARMEmitter::Size::i32Bit, Dst, Dst);\n break;\n case IR::OpSize::i16Bit:\n uxth(ARMEmitter::Size::i32Bit, Dst, Src);\n cnt(ARMEmitter::Size::i32Bit, Dst, Dst);\n break;\n case IR::OpSize::i32Bit: cnt(ARMEmitter::Size::i32Bit, Dst, Src); break;\n case IR::OpSize::i64Bit: cnt(ARMEmitter::Size::i64Bit, Dst, Src); break;\n default: LOGMAN_MSG_A_FMT(\"Unsupported Popcount size: {}\", OpSize);\n }\n } else {\n switch (OpSize) {\n case IR::OpSize::i8Bit:\n fmov(ARMEmitter::Size::i32Bit, VTMP1.S(), Src);\n // only use lowest byte\n cnt(ARMEmitter::SubRegSize::i8Bit, VTMP1.D(), VTMP1.D());\n break;\n case IR::OpSize::i16Bit:\n fmov(ARMEmitter::Size::i32Bit, VTMP1.S(), Src);\n cnt(ARMEmitter::SubRegSize::i8Bit, VTMP1.D(), VTMP1.D());\n // only count two lowest bytes\n addp(ARMEmitter::SubRegSize::i8Bit, VTMP1.D(), VTMP1.D(), VTMP1.D());\n break;\n case IR::OpSize::i32Bit:\n fmov(ARMEmitter::Size::i32Bit, VTMP1.S(), Src);\n cnt(ARMEmitter::SubRegSize::i8Bit, VTMP1.D(), VTMP1.D());\n // fmov has zero extended, unused bytes are zero\n addv(ARMEmitter::SubRegSize::i8Bit, VTMP1.D(), VTMP1.D());\n break;\n case IR::OpSize::i64Bit:\n fmov(ARMEmitter::Size::i64Bit, VTMP1.D(), Src);\n cnt(ARMEmitter::SubRegSize::i8Bit, VTMP1.D(), VTMP1.D());\n // fmov has zero extended, unused bytes are zero\n addv(ARMEmitter::SubRegSize::i8Bit, VTMP1.D(), VTMP1.D());\n break;\n default: LOGMAN_MSG_A_FMT(\"Unsupported Popcount size: {}\", OpSize);\n }\n\n umov(Dst, VTMP1, 0);\n }\n}\n\nDEF_OP(FindLSB) {\n auto Op = IROp->C();\n const auto EmitSize = ConvertSize(IROp);\n\n const auto Dst = GetReg(Node);\n const auto Src = GetReg(Op->Src);\n\n // We assume the source is nonzero, so we can just rbit+clz without worrying\n // about upper garbage for smaller types.\n rbit(EmitSize, TMP1, Src);\n clz(EmitSize, Dst, TMP1);\n}\n\nDEF_OP(FindMSB) {\n auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n LOGMAN_THROW_A_FMT(OpSize == IR::OpSize::i16Bit || OpSize == IR::OpSize::i32Bit || OpSize == IR::OpSize::i64Bit,\n \"Unsupported {} size: {}\", __func__, OpSize);\n const auto EmitSize = ConvertSize(IROp);\n\n const auto Dst = GetReg(Node);\n const auto Src = GetReg(Op->Src);\n\n movz(ARMEmitter::Size::i64Bit, TMP1, IR::OpSizeAsBits(OpSize) - 1);\n\n if (OpSize == IR::OpSize::i16Bit) {\n lsl(EmitSize, Dst, Src, 16);\n clz(EmitSize, Dst, Dst);\n } else {\n clz(EmitSize, Dst, Src);\n }\n\n sub(ARMEmitter::Size::i64Bit, Dst, TMP1, Dst);\n}\n\nDEF_OP(FindTrailingZeroes) {\n auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n LOGMAN_THROW_A_FMT(OpSize == IR::OpSize::i16Bit || OpSize == IR::OpSize::i32Bit || OpSize == IR::OpSize::i64Bit,\n \"Unsupported {} size: {}\", __func__, OpSize);\n const auto EmitSize = ConvertSize(IROp);\n\n const auto Dst = GetReg(Node);\n const auto Src = GetReg(Op->Src);\n\n rbit(EmitSize, Dst, Src);\n\n if (OpSize == IR::OpSize::i16Bit) {\n // This orr does two things. First, if the (masked) source is zero, it\n // reverses to zero in the top so it forces clz to return 16. Second, it\n // ensures garbage in the upper bits of the source don't affect clz, because\n // they'll rbit to garbage in the bottom below the 0x8000 and be ignored by\n // the clz. So we handle Src upper garbage without explicitly masking.\n orr(EmitSize, Dst, Dst, 0x8000);\n }\n\n clz(EmitSize, Dst, Dst);\n}\n\nDEF_OP(CountLeadingZeroes) {\n auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n LOGMAN_THROW_A_FMT(OpSize == IR::OpSize::i16Bit || OpSize == IR::OpSize::i32Bit || OpSize == IR::OpSize::i64Bit,\n \"Unsupported {} size: {}\", __func__, OpSize);\n const auto EmitSize = ConvertSize(IROp);\n\n const auto Dst = GetReg(Node);\n const auto Src = GetReg(Op->Src);\n\n if (OpSize == IR::OpSize::i16Bit) {\n // Expressing as lsl+orr+clz clears away any garbage in the upper bits\n // (alternatively could do uxth+clz+sub.. equal cost in total).\n lsl(EmitSize, Dst, Src, 16);\n orr(EmitSize, Dst, Dst, 0x8000);\n clz(EmitSize, Dst, Dst);\n } else {\n clz(EmitSize, Dst, Src);\n }\n}\n\nDEF_OP(Rev) {\n auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n LOGMAN_THROW_A_FMT(OpSize == IR::OpSize::i16Bit || OpSize == IR::OpSize::i32Bit || OpSize == IR::OpSize::i64Bit,\n \"Unsupported {} size: {}\", __func__, OpSize);\n const auto EmitSize = ConvertSize(IROp);\n\n const auto Dst = GetReg(Node);\n const auto Src = GetReg(Op->Src);\n\n rev(EmitSize, Dst, Src);\n if (OpSize == IR::OpSize::i16Bit) {\n lsr(EmitSize, Dst, Dst, 16);\n }\n}\n\nDEF_OP(Rbit) {\n auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n LOGMAN_THROW_A_FMT(OpSize == IR::OpSize::i32Bit || OpSize == IR::OpSize::i64Bit, \"Unsupported {} size: {}\", __func__, OpSize);\n const auto EmitSize = ConvertSize48(IROp);\n\n const auto Dst = GetReg(Node);\n const auto Src = GetReg(Op->Src);\n\n rbit(EmitSize, Dst, Src);\n}\n\nDEF_OP(Bfi) {\n auto Op = IROp->C();\n const auto EmitSize = ConvertSize(IROp);\n\n const auto Dst = GetReg(Node);\n const auto SrcDst = GetReg(Op->Dest);\n const auto Src = GetReg(Op->Src);\n\n if (Dst == SrcDst) {\n // If Dst and SrcDst match then this turns in to a simple BFI instruction.\n bfi(EmitSize, Dst, Src, Op->lsb, Op->Width);\n } else if (Dst != Src) {\n // If the destination isn't the source then we can move the DstSrc and insert directly.\n //\n // The move is 64-bit to allow register renaming, the upper bits don't\n // matter because of the bfi's EmitSize.\n mov(ARMEmitter::Size::i64Bit, Dst, SrcDst);\n bfi(EmitSize, Dst, Src, Op->lsb, Op->Width);\n } else {\n // Destination didn't match the dst source register.\n // TODO: Inefficient until FEX can have RA constraints here.\n mov(EmitSize, TMP1, SrcDst);\n bfi(EmitSize, TMP1, Src, Op->lsb, Op->Width);\n\n if (IROp->Size >= IR::OpSize::i32Bit) {\n mov(EmitSize, Dst, TMP1.R());\n } else {\n ubfx(EmitSize, Dst, TMP1, 0, IR::OpSizeAsBits(IROp->Size));\n }\n }\n}\n\nDEF_OP(Bfxil) {\n auto Op = IROp->C();\n const auto EmitSize = ConvertSize(IROp);\n\n const auto Dst = GetReg(Node);\n const auto SrcDst = GetReg(Op->Dest);\n const auto Src = GetReg(Op->Src);\n\n if (Dst == SrcDst) {\n // If Dst and SrcDst match then this turns in to a single instruction.\n bfxil(EmitSize, Dst, Src, Op->lsb, Op->Width);\n } else if (Dst != Src) {\n // If the destination isn't the source then we can move the DstSrc and insert directly.\n mov(EmitSize, Dst, SrcDst);\n bfxil(EmitSize, Dst, Src, Op->lsb, Op->Width);\n } else {\n // Destination didn't match the dst source register.\n // TODO: Inefficient until FEX can have RA constraints here.\n mov(EmitSize, TMP1, SrcDst);\n bfxil(EmitSize, TMP1, Src, Op->lsb, Op->Width);\n mov(EmitSize, Dst, TMP1.R());\n }\n}\n\nDEF_OP(Bfe) {\n auto Op = IROp->C();\n LOGMAN_THROW_A_FMT(IROp->Size <= IR::OpSize::i64Bit, \"OpSize is too large for BFE: {}\", IROp->Size);\n LOGMAN_THROW_A_FMT(Op->Width != 0, \"Invalid BFE width of 0\");\n const auto EmitSize = ConvertSize(IROp);\n\n const auto Dst = GetReg(Node);\n const auto Src = GetReg(Op->Src);\n\n if (Op->lsb == 0 && Op->Width == 32) {\n mov(ARMEmitter::Size::i32Bit, Dst, Src);\n } else if (Op->lsb == 0 && Op->Width == 64) {\n LOGMAN_THROW_A_FMT(IROp->Size == IR::OpSize::i64Bit, \"Must be 64-bit wide register\");\n mov(ARMEmitter::Size::i64Bit, Dst, Src);\n } else {\n ubfx(EmitSize, Dst, Src, Op->lsb, Op->Width);\n }\n}\n\nDEF_OP(Sbfe) {\n auto Op = IROp->C();\n const auto Dst = GetReg(Node);\n const auto Src = GetReg(Op->Src);\n\n sbfx(ConvertSize(IROp), Dst, Src, Op->lsb, Op->Width);\n}\n\nDEF_OP(MaskGenerateFromBitWidth) {\n auto Op = IROp->C();\n auto BitWidth = GetReg(Op->BitWidth);\n\n LoadConstant(ARMEmitter::Size::i64Bit, TMP1, -1);\n cmp(ARMEmitter::Size::i64Bit, BitWidth, 0);\n lslv(ARMEmitter::Size::i64Bit, TMP2, TMP1, BitWidth);\n csinv(ARMEmitter::Size::i64Bit, GetReg(Node), TMP1, TMP2, ARMEmitter::Condition::CC_EQ);\n}\n\nDEF_OP(Select) {\n auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n const auto EmitSize = ConvertSize(IROp);\n const auto CompareEmitSize = Op->CompareSize == IR::OpSize::i64Bit ? ARMEmitter::Size::i64Bit : ARMEmitter::Size::i32Bit;\n\n uint64_t Const;\n auto cc = MapCC(Op->Cond);\n\n if (IsGPR(Op->Cmp1)) {\n const auto Src1 = GetReg(Op->Cmp1);\n\n if (IsInlineConstant(Op->Cmp2, &Const)) {\n cmp(CompareEmitSize, Src1, Const);\n } else {\n const auto Src2 = GetReg(Op->Cmp2);\n cmp(CompareEmitSize, Src1, Src2);\n }\n } else if (IsFPR(Op->Cmp1)) {\n const auto Src1 = GetVReg(Op->Cmp1);\n const auto Src2 = GetVReg(Op->Cmp2);\n fcmp(Op->CompareSize == IR::OpSize::i64Bit ? ARMEmitter::ScalarRegSize::i64Bit : ARMEmitter::ScalarRegSize::i32Bit, Src1, Src2);\n } else {\n LOGMAN_MSG_A_FMT(\"Select: Expected GPR or FPR\");\n }\n\n uint64_t const_true, const_false;\n bool is_const_true = IsInlineConstant(Op->TrueVal, &const_true);\n bool is_const_false = IsInlineConstant(Op->FalseVal, &const_false);\n\n uint64_t all_ones = OpSize == IR::OpSize::i64Bit ? 0xffff'ffff'ffff'ffffull : 0xffff'ffffull;\n\n ARMEmitter::Register Dst = GetReg(Node);\n\n if (is_const_true || is_const_false) {\n if (is_const_false != true || is_const_true != true || !(const_true == 1 || const_true == all_ones) || const_false != 0) {\n LOGMAN_MSG_A_FMT(\"Select: Unsupported compare inline parameters\");\n }\n\n if (const_true == all_ones) {\n csetm(EmitSize, Dst, cc);\n } else {\n cset(EmitSize, Dst, cc);\n }\n } else {\n csel(EmitSize, Dst, GetReg(Op->TrueVal), GetReg(Op->FalseVal), cc);\n }\n}\n\nDEF_OP(NZCVSelect) {\n auto Op = IROp->C();\n const auto EmitSize = ConvertSize(IROp);\n\n auto cc = MapCC(Op->Cond);\n\n uint64_t const_true, const_false;\n bool is_const_true = IsInlineConstant(Op->TrueVal, &const_true);\n bool is_const_false = IsInlineConstant(Op->FalseVal, &const_false);\n\n uint64_t all_ones = IROp->Size == IR::OpSize::i64Bit ? 0xffff'ffff'ffff'ffffull : 0xffff'ffffull;\n\n ARMEmitter::Register Dst = GetReg(Node);\n\n if (is_const_true) {\n if (is_const_false != true || !(const_true == 1 || const_true == all_ones) || const_false != 0) {\n LOGMAN_MSG_A_FMT(\"NZCVSelect: Unsupported constant\");\n }\n\n if (const_true == all_ones) {\n csetm(EmitSize, Dst, cc);\n } else {\n cset(EmitSize, Dst, cc);\n }\n } else {\n csel(EmitSize, Dst, GetReg(Op->TrueVal), GetZeroableReg(Op->FalseVal), cc);\n }\n}\n\nDEF_OP(NZCVSelectV) {\n auto Op = IROp->C();\n\n auto cc = MapCC(Op->Cond);\n const auto SubRegSize = ConvertSubRegSizePair248(IROp);\n fcsel(SubRegSize.Scalar, GetVReg(Node), GetVReg(Op->TrueVal), GetVReg(Op->FalseVal), cc);\n}\n\nDEF_OP(NZCVSelectIncrement) {\n auto Op = IROp->C();\n\n csinc(ConvertSize(IROp), GetReg(Node), GetReg(Op->TrueVal), GetZeroableReg(Op->FalseVal), MapCC(Op->Cond));\n}\n\nDEF_OP(VExtractToGPR) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n constexpr auto AVXRegBitSize = Core::CPUState::XMM_AVX_REG_SIZE * 8;\n constexpr auto SSERegBitSize = Core::CPUState::XMM_SSE_REG_SIZE * 8;\n const auto ElementSizeBits = IR::OpSizeAsBits(Op->Header.ElementSize);\n\n const auto Offset = ElementSizeBits * Op->Index;\n const auto Is256Bit = Offset >= SSERegBitSize;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto Dst = GetReg(Node);\n const auto Vector = GetVReg(Op->Vector);\n\n const auto PerformMove = [&](const ARMEmitter::VRegister reg, int index) {\n switch (OpSize) {\n case IR::OpSize::i8Bit: umov(Dst, Vector, index); break;\n case IR::OpSize::i16Bit: umov(Dst, Vector, index); break;\n case IR::OpSize::i32Bit: umov(Dst, Vector, index); break;\n case IR::OpSize::i64Bit: umov(Dst, Vector, index); break;\n default: LOGMAN_MSG_A_FMT(\"Unhandled ExtractElementSize: {}\", OpSize); break;\n }\n };\n\n if (Offset < SSERegBitSize) {\n // Desired data lies within the lower 128-bit lane, so we\n // can treat the operation as a 128-bit operation, even\n // when acting on larger register sizes.\n PerformMove(Vector, Op->Index);\n } else {\n LOGMAN_THROW_A_FMT(Is256Bit, \"Can't perform 256-bit extraction with op side: {}\", OpSize);\n LOGMAN_THROW_A_FMT(Offset < AVXRegBitSize, \"Trying to extract element outside bounds of register. Offset={}, Index={}\", Offset, Op->Index);\n\n // We need to use the upper 128-bit lane, so lets move it down.\n // Inverting our dedicated predicate for 128-bit operations selects\n // all of the top lanes. We can then compact those into a temporary.\n const auto CompactPred = ARMEmitter::PReg::p0;\n not_(CompactPred, PRED_TMP_32B.Zeroing(), PRED_TMP_16B);\n compact(ARMEmitter::SubRegSize::i64Bit, VTMP1.Z(), CompactPred, Vector.Z());\n\n // Sanitize the zero-based index to work on the now-moved\n // upper half of the vector.\n const auto SanitizedIndex = [OpSize, Op] {\n switch (OpSize) {\n case IR::OpSize::i8Bit: return Op->Index - 16;\n case IR::OpSize::i16Bit: return Op->Index - 8;\n case IR::OpSize::i32Bit: return Op->Index - 4;\n case IR::OpSize::i64Bit: return Op->Index - 2;\n default: LOGMAN_MSG_A_FMT(\"Unhandled OpSize: {}\", OpSize); return 0;\n }\n }();\n\n // Move the value from the now-low-lane data.\n PerformMove(VTMP1, SanitizedIndex);\n }\n}\n\nDEF_OP(Float_ToGPR_ZS) {\n auto Op = IROp->C();\n\n ARMEmitter::Register Dst = GetReg(Node);\n ARMEmitter::VRegister Src = GetVReg(Op->Scalar);\n\n if (Op->SrcElementSize == IR::OpSize::i64Bit) {\n fcvtzs(ConvertSize(IROp), Dst, Src.D());\n } else {\n fcvtzs(ConvertSize(IROp), Dst, Src.S());\n }\n}\n\nDEF_OP(Float_ToGPR_S) {\n auto Op = IROp->C();\n\n ARMEmitter::Register Dst = GetReg(Node);\n ARMEmitter::VRegister Src = GetVReg(Op->Scalar);\n\n if (Op->SrcElementSize == IR::OpSize::i64Bit) {\n frinti(VTMP1.D(), Src.D());\n fcvtzs(ConvertSize(IROp), Dst, VTMP1.D());\n } else {\n frinti(VTMP1.S(), Src.S());\n fcvtzs(ConvertSize(IROp), Dst, VTMP1.S());\n }\n}\n\nDEF_OP(FCmp) {\n auto Op = IROp->C();\n const auto EmitSubSize = Op->ElementSize == IR::OpSize::i64Bit ? ARMEmitter::ScalarRegSize::i64Bit : ARMEmitter::ScalarRegSize::i32Bit;\n\n ARMEmitter::VRegister Scalar1 = GetVReg(Op->Scalar1);\n ARMEmitter::VRegister Scalar2 = GetVReg(Op->Scalar2);\n\n fcmp(EmitSubSize, Scalar1, Scalar2);\n}\n\n} // namespace FEXCore::CPU\n" }, { "path": "FEXCore/Source/Interface/Core/JIT/Arm64Relocations.cpp", "content": "// SPDX-License-Identifier: MIT\n/*\n$info$\ntags: backend|arm64\ndesc: relocation logic of the arm64 splatter backend\n$end_info$\n*/\n#include \"Interface/Context/Context.h\"\n#include \"Interface/Core/JIT/JITClass.h\"\n\n#include \n\nnamespace FEXCore::CPU {\nuint64_t GetNamedSymbolLiteral(FEXCore::Context::ContextImpl& CTX, FEXCore::CPU::RelocNamedSymbolLiteral::NamedSymbol Op) {\n switch (Op) {\n case FEXCore::CPU::RelocNamedSymbolLiteral::NamedSymbol::SYMBOL_LITERAL_EXITFUNCTION_LINKER:\n return CTX.Dispatcher->GetExitFunctionLinkerAddress();\n\n default: ERROR_AND_DIE_FMT(\"Unknown named symbol literal: {}\", static_cast(Op));\n }\n}\n\nvoid Arm64JITCore::InsertNamedThunkRelocation(ARMEmitter::Register Reg, const IR::SHA256Sum& Sum) {\n Relocation MoveABI {};\n MoveABI.NamedThunkMove.Header = {.Offset = GetCursorOffset(), .Type = FEXCore::CPU::RelocationTypes::RELOC_NAMED_THUNK_MOVE};\n MoveABI.NamedThunkMove.Symbol = Sum;\n MoveABI.NamedThunkMove.RegisterIndex = Reg.Idx();\n\n uint64_t Pointer = reinterpret_cast(EmitterCTX->ThunkHandler->LookupThunk(Sum));\n\n // Pointers are required to fit within 48-bit VA space.\n // TODO: Force 6-byte `MaxSize`, with zext extension to 64-bit. Current code not smart enough to handle negatives.\n LoadConstant(ARMEmitter::Size::i64Bit, Reg, Pointer, FEXCore::CPU::Arm64Emitter::PadType::AUTOPAD);\n Relocations.emplace_back(MoveABI);\n}\n\nArm64JITCore::NamedSymbolLiteralPair Arm64JITCore::InsertNamedSymbolLiteral(FEXCore::CPU::RelocNamedSymbolLiteral::NamedSymbol Op) {\n uint64_t Pointer = GetNamedSymbolLiteral(*CTX, Op);\n\n NamedSymbolLiteralPair Lit {\n .Lit = Pointer,\n .MoveABI =\n {\n .NamedSymbolLiteral =\n {\n .Header =\n {\n .Offset = 0, // Set by PlaceNamedSymbolLiteral\n .Type = FEXCore::CPU::RelocationTypes::RELOC_NAMED_SYMBOL_LITERAL,\n },\n .Symbol = Op,\n },\n },\n };\n return Lit;\n}\n\nvoid Arm64JITCore::PlaceNamedSymbolLiteral(NamedSymbolLiteralPair Lit) {\n switch (Lit.MoveABI.Header.Type) {\n case RelocationTypes::RELOC_NAMED_SYMBOL_LITERAL:\n case RelocationTypes::RELOC_GUEST_RIP_LITERAL: {\n Lit.MoveABI.Header.Offset = GetCursorOffset();\n break;\n }\n\n default: ERROR_AND_DIE_FMT(\"Unknown relocation type for {}\", __FUNCTION__);\n }\n\n BindOrRestart(&Lit.Loc);\n dc64(Lit.Lit);\n Relocations.emplace_back(Lit.MoveABI);\n}\n\nauto Arm64JITCore::InsertGuestRIPLiteral(uint64_t GuestRIP) -> NamedSymbolLiteralPair {\n return {\n .Lit = GuestRIP,\n .MoveABI =\n {\n .GuestRIP = {.Header =\n {\n .Offset = 0, // Set by PlaceNamedSymbolLiteral\n .Type = FEXCore::CPU::RelocationTypes::RELOC_GUEST_RIP_LITERAL,\n },\n // NOTE: Cache serialization will subtract the guest binary base address later to produce consistency results\n .GuestRIP = GuestRIP},\n },\n };\n}\n\nvoid Arm64JITCore::InsertGuestRIPMove(ARMEmitter::Register Reg, uint64_t Constant) {\n Relocation MoveABI {};\n MoveABI.GuestRIP.Header = {.Offset = GetCursorOffset(), .Type = FEXCore::CPU::RelocationTypes::RELOC_GUEST_RIP_MOVE};\n // NOTE: Cache serialization will subtract the guest binary base address later to produce consistency results\n MoveABI.GuestRIP.GuestRIP = Constant;\n MoveABI.GuestRIP.RegisterIndex = Reg.Idx();\n\n // Pointers are required to fit within 48-bit VA space.\n // TODO: Force 6-byte `MaxSize`, with sign extension to 64-bit. Current code not smart enough to handle negatives.\n // 48-bit sign extension works because x86-64 guests only receive 47-bit VA space, with 48-bit being reserved for kernel.\n // Additional quirk, \"canonical\" 48-bit pointers on x86-64, sign extend the 48-bit as well (Which is why kernel pointers are negative).\n LoadConstant(ARMEmitter::Size::i64Bit, Reg, Constant, FEXCore::CPU::Arm64Emitter::PadType::AUTOPAD);\n Relocations.emplace_back(MoveABI);\n}\n\nfextl::vector Arm64JITCore::TakeRelocations(uint64_t GuestBaseAddress) {\n // Rebase relocations to library base address\n for (auto& Relocation : Relocations) {\n switch (Relocation.Header.Type) {\n case FEXCore::CPU::RelocationTypes::RELOC_GUEST_RIP_MOVE:\n case FEXCore::CPU::RelocationTypes::RELOC_GUEST_RIP_LITERAL: {\n Relocation.GuestRIP.GuestRIP -= GuestBaseAddress;\n break;\n }\n default:;\n }\n }\n\n return std::move(Relocations);\n}\n\n} // namespace FEXCore::CPU\n" }, { "path": "FEXCore/Source/Interface/Core/JIT/AtomicOps.cpp", "content": "// SPDX-License-Identifier: MIT\n/*\n$info$\ntags: backend|arm64\n$end_info$\n*/\n\n#include \"Interface/Context/Context.h\"\n#include \"Interface/Core/Dispatcher/Dispatcher.h\"\n#include \"Interface/Core/JIT/JITClass.h\"\n\nnamespace FEXCore::CPU {\nDEF_OP(CASPair) {\n auto Op = IROp->C();\n LOGMAN_THROW_A_FMT(IROp->ElementSize == IR::OpSize::i32Bit || IROp->ElementSize == IR::OpSize::i64Bit, \"Wrong element size\");\n // Size is the size of each pair element\n auto Dst0 = GetReg(Op->OutLo);\n auto Dst1 = GetReg(Op->OutHi);\n auto Expected0 = GetReg(Op->ExpectedLo);\n auto Expected1 = GetReg(Op->ExpectedHi);\n auto Desired0 = GetReg(Op->DesiredLo);\n auto Desired1 = GetReg(Op->DesiredHi);\n auto MemSrc = GetReg(Op->Addr);\n\n const auto EmitSize = IROp->ElementSize == IR::OpSize::i64Bit ? ARMEmitter::Size::i64Bit : ARMEmitter::Size::i32Bit;\n if (CTX->HostFeatures.SupportsAtomics) {\n // RA has heuristics to try to pair sources, but we need to handle the cases\n // where they fail. We do so by moving to temporaries. Note we use 64-bit\n // moves here even for 32-bit cmpxchg, for the Firestorm register renamer.\n if (Desired1.Idx() != (Desired0.Idx() + 1) || Desired0.Idx() & 1) {\n mov(ARMEmitter::Size::i64Bit, TMP1, Desired0);\n mov(ARMEmitter::Size::i64Bit, TMP2, Desired1);\n Desired0 = TMP1;\n Desired1 = TMP2;\n }\n\n auto CaspalDst0 = Dst0;\n auto CaspalDst1 = Dst1;\n if (CaspalDst1.Idx() != (CaspalDst0.Idx() + 1) || CaspalDst0.Idx() & 1) {\n CaspalDst0 = TMP3;\n CaspalDst1 = TMP4;\n }\n\n // We can't clobber the source, these moves are inherently required due to\n // ISA limitations. But by making them 64-bit, Firestorm can rename.\n mov(ARMEmitter::Size::i64Bit, CaspalDst0, Expected0);\n mov(ARMEmitter::Size::i64Bit, CaspalDst1, Expected1);\n caspal(EmitSize, CaspalDst0, CaspalDst1, Desired0, Desired1, MemSrc);\n\n if (CaspalDst0 != Dst0) {\n mov(ARMEmitter::Size::i64Bit, Dst0, CaspalDst0);\n mov(ARMEmitter::Size::i64Bit, Dst1, CaspalDst1);\n }\n } else {\n // Save NZCV so we don't have to mark this op as clobbering NZCV (the\n // SupportsAtomics does not clobber atomics and this !SupportsAtomics path\n // is so slow it's not worth the complexity of splitting the IR op.). We\n // clobber NZCV inside the hot loop and we can't replace cmp/ccmp/b.ne with\n // something NZCV-preserving without requiring an extra instruction.\n mrs(TMP1, ARMEmitter::SystemRegister::NZCV);\n\n ARMEmitter::BackwardLabel LoopTop;\n ARMEmitter::ForwardLabel LoopNotExpected;\n ARMEmitter::ForwardLabel LoopExpected;\n (void)Bind(&LoopTop);\n\n // This instruction sequence must be synced with HandleCASPAL_Armv8.\n ldaxp(EmitSize, TMP2, TMP3, MemSrc);\n cmp(EmitSize, TMP2, Expected0);\n ccmp(EmitSize, TMP3, Expected1, ARMEmitter::StatusFlags::None, ARMEmitter::Condition::CC_EQ);\n (void)b(ARMEmitter::Condition::CC_NE, &LoopNotExpected);\n stlxp(EmitSize, TMP2, Desired0, Desired1, MemSrc);\n (void)cbnz(EmitSize, TMP2, &LoopTop);\n mov(EmitSize, Dst0, Expected0);\n mov(EmitSize, Dst1, Expected1);\n\n (void)b(&LoopExpected);\n\n (void)Bind(&LoopNotExpected);\n mov(EmitSize, Dst0, TMP2.R());\n mov(EmitSize, Dst1, TMP3.R());\n // exclusive monitor needs to be cleared here\n // Might have hit the case where ldaxr was hit but stlxr wasn't\n clrex();\n (void)Bind(&LoopExpected);\n\n // Restore\n msr(ARMEmitter::SystemRegister::NZCV, TMP1);\n }\n}\n\nDEF_OP(CAS) {\n auto Op = IROp->C();\n const auto EmitSize = ConvertSize(IROp);\n const auto SubEmitSize = ConvertSubRegSize8(IROp->Size);\n // DataSrc = *Src1\n // if (DataSrc == Src3) { *Src1 == Src2; } Src2 = DataSrc\n // This will write to memory! Careful!\n\n auto Expected = GetReg(Op->Expected);\n auto Desired = GetReg(Op->Desired);\n auto MemSrc = GetReg(Op->Addr);\n auto Dst = GetReg(Node);\n\n if (CTX->HostFeatures.SupportsAtomics) {\n if (Expected == Dst && Dst != MemSrc && Dst != Desired) {\n casal(SubEmitSize, Dst, Desired, MemSrc);\n } else {\n mov(EmitSize, TMP2, Expected);\n casal(SubEmitSize, TMP2, Desired, MemSrc);\n mov(EmitSize, Dst, TMP2.R());\n }\n } else {\n ARMEmitter::BackwardLabel LoopTop;\n ARMEmitter::ForwardLabel LoopNotExpected;\n ARMEmitter::ForwardLabel LoopExpected;\n (void)Bind(&LoopTop);\n ldaxr(SubEmitSize, TMP2, MemSrc);\n if (IROp->Size == IR::OpSize::i8Bit) {\n cmp(EmitSize, TMP2, Expected, ARMEmitter::ExtendedType::UXTB, 0);\n } else if (IROp->Size == IR::OpSize::i16Bit) {\n cmp(EmitSize, TMP2, Expected, ARMEmitter::ExtendedType::UXTH, 0);\n } else {\n cmp(EmitSize, TMP2, Expected);\n }\n (void)b(ARMEmitter::Condition::CC_NE, &LoopNotExpected);\n stlxr(SubEmitSize, TMP3, Desired, MemSrc);\n (void)cbnz(EmitSize, TMP3, &LoopTop);\n mov(EmitSize, Dst, Expected);\n (void)b(&LoopExpected);\n\n (void)Bind(&LoopNotExpected);\n mov(EmitSize, Dst, TMP2.R());\n // exclusive monitor needs to be cleared here\n // Might have hit the case where ldaxr was hit but stlxr wasn't\n clrex();\n (void)Bind(&LoopExpected);\n }\n}\n\nDEF_OP(AtomicSwap) {\n auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n LOGMAN_THROW_A_FMT(\n OpSize == IR::OpSize::i64Bit || OpSize == IR::OpSize::i32Bit || OpSize == IR::OpSize::i16Bit || OpSize == IR::OpSize::i8Bit, \"Unexpecte\"\n \"d CAS \"\n \"size\");\n\n auto MemSrc = GetReg(Op->Addr);\n auto Src = GetReg(Op->Value);\n\n const auto EmitSize = ConvertSize(IROp);\n const auto SubEmitSize = OpSize == IR::OpSize::i64Bit ? ARMEmitter::SubRegSize::i64Bit :\n OpSize == IR::OpSize::i32Bit ? ARMEmitter::SubRegSize::i32Bit :\n OpSize == IR::OpSize::i16Bit ? ARMEmitter::SubRegSize::i16Bit :\n ARMEmitter::SubRegSize::i8Bit;\n\n if (CTX->HostFeatures.SupportsAtomics) {\n ldswpal(SubEmitSize, Src, GetReg(Node), MemSrc);\n } else {\n ARMEmitter::BackwardLabel LoopTop;\n (void)Bind(&LoopTop);\n ldaxr(SubEmitSize, TMP2, MemSrc);\n stlxr(SubEmitSize, TMP4, Src, MemSrc);\n (void)cbnz(EmitSize, TMP4, &LoopTop);\n ubfm(EmitSize, GetReg(Node), TMP2, 0, IR::OpSizeAsBits(OpSize) - 1);\n }\n}\n\nDEF_OP(AtomicFetchAdd) {\n auto Op = IROp->C();\n const auto EmitSize = ConvertSize(IROp);\n const auto SubEmitSize = ConvertSubRegSize8(IROp->Size);\n\n auto MemSrc = GetReg(Op->Addr);\n auto Src = GetReg(Op->Value);\n\n if (CTX->HostFeatures.SupportsAtomics) {\n ldaddal(SubEmitSize, Src, GetReg(Node), MemSrc);\n } else {\n ARMEmitter::BackwardLabel LoopTop;\n (void)Bind(&LoopTop);\n ldaxr(SubEmitSize, TMP2, MemSrc);\n add(EmitSize, TMP3, TMP2, Src);\n stlxr(SubEmitSize, TMP4, TMP3, MemSrc);\n (void)cbnz(EmitSize, TMP4, &LoopTop);\n mov(EmitSize, GetReg(Node), TMP2.R());\n }\n}\n\nDEF_OP(AtomicFetchSub) {\n auto Op = IROp->C();\n const auto EmitSize = ConvertSize(IROp);\n const auto SubEmitSize = ConvertSubRegSize8(IROp->Size);\n\n auto MemSrc = GetReg(Op->Addr);\n auto Src = GetReg(Op->Value);\n\n if (CTX->HostFeatures.SupportsAtomics) {\n neg(EmitSize, TMP2, Src);\n ldaddal(SubEmitSize, TMP2, GetReg(Node), MemSrc);\n } else {\n ARMEmitter::BackwardLabel LoopTop;\n (void)Bind(&LoopTop);\n ldaxr(SubEmitSize, TMP2, MemSrc);\n sub(EmitSize, TMP3, TMP2, Src);\n stlxr(SubEmitSize, TMP4, TMP3, MemSrc);\n (void)cbnz(EmitSize, TMP4, &LoopTop);\n mov(EmitSize, GetReg(Node), TMP2.R());\n }\n}\n\nDEF_OP(AtomicFetchAnd) {\n auto Op = IROp->C();\n const auto EmitSize = ConvertSize(IROp);\n const auto SubEmitSize = ConvertSubRegSize8(IROp->Size);\n\n auto MemSrc = GetReg(Op->Addr);\n auto Src = GetReg(Op->Value);\n\n if (CTX->HostFeatures.SupportsAtomics) {\n mvn(EmitSize, TMP2, Src);\n ldclral(SubEmitSize, TMP2, GetReg(Node), MemSrc);\n } else {\n ARMEmitter::BackwardLabel LoopTop;\n (void)Bind(&LoopTop);\n ldaxr(SubEmitSize, TMP2, MemSrc);\n and_(EmitSize, TMP3, TMP2, Src);\n stlxr(SubEmitSize, TMP4, TMP3, MemSrc);\n (void)cbnz(EmitSize, TMP4, &LoopTop);\n mov(EmitSize, GetReg(Node), TMP2.R());\n }\n}\n\nDEF_OP(AtomicFetchCLR) {\n auto Op = IROp->C();\n const auto EmitSize = ConvertSize(IROp);\n const auto SubEmitSize = ConvertSubRegSize8(IROp->Size);\n\n auto MemSrc = GetReg(Op->Addr);\n auto Src = GetReg(Op->Value);\n\n if (CTX->HostFeatures.SupportsAtomics) {\n ldclral(SubEmitSize, Src, GetReg(Node), MemSrc);\n } else {\n ARMEmitter::BackwardLabel LoopTop;\n (void)Bind(&LoopTop);\n ldaxr(SubEmitSize, TMP2, MemSrc);\n bic(EmitSize, TMP3, TMP2, Src);\n stlxr(SubEmitSize, TMP4, TMP3, MemSrc);\n (void)cbnz(EmitSize, TMP4, &LoopTop);\n mov(EmitSize, GetReg(Node), TMP2.R());\n }\n}\n\nDEF_OP(AtomicFetchOr) {\n auto Op = IROp->C();\n const auto EmitSize = ConvertSize(IROp);\n const auto SubEmitSize = ConvertSubRegSize8(IROp->Size);\n\n auto MemSrc = GetReg(Op->Addr);\n auto Src = GetReg(Op->Value);\n\n if (CTX->HostFeatures.SupportsAtomics) {\n ldsetal(SubEmitSize, Src, GetReg(Node), MemSrc);\n } else {\n ARMEmitter::BackwardLabel LoopTop;\n (void)Bind(&LoopTop);\n ldaxr(SubEmitSize, TMP2, MemSrc);\n orr(EmitSize, TMP3, TMP2, Src);\n stlxr(SubEmitSize, TMP4, TMP3, MemSrc);\n (void)cbnz(EmitSize, TMP4, &LoopTop);\n mov(EmitSize, GetReg(Node), TMP2.R());\n }\n}\n\nDEF_OP(AtomicFetchXor) {\n auto Op = IROp->C();\n const auto EmitSize = ConvertSize(IROp);\n const auto SubEmitSize = ConvertSubRegSize8(IROp->Size);\n\n auto MemSrc = GetReg(Op->Addr);\n auto Src = GetReg(Op->Value);\n\n if (CTX->HostFeatures.SupportsAtomics) {\n ldeoral(SubEmitSize, Src, GetReg(Node), MemSrc);\n } else {\n ARMEmitter::BackwardLabel LoopTop;\n (void)Bind(&LoopTop);\n ldaxr(SubEmitSize, TMP2, MemSrc);\n eor(EmitSize, TMP3, TMP2, Src);\n stlxr(SubEmitSize, TMP4, TMP3, MemSrc);\n (void)cbnz(EmitSize, TMP4, &LoopTop);\n mov(EmitSize, GetReg(Node), TMP2.R());\n }\n}\n\nDEF_OP(AtomicFetchNeg) {\n auto Op = IROp->C();\n const auto EmitSize = ConvertSize(IROp);\n const auto SubEmitSize = ConvertSubRegSize8(IROp->Size);\n\n auto MemSrc = GetReg(Op->Addr);\n\n if (CTX->HostFeatures.SupportsAtomics) {\n // Use a CAS loop to avoid needing to emulate unaligned LLSC atomics\n ldr(SubEmitSize, TMP2, MemSrc);\n ARMEmitter::BackwardLabel LoopTop;\n (void)Bind(&LoopTop);\n mov(EmitSize, TMP4, TMP2);\n neg(EmitSize, TMP3, TMP2);\n casal(SubEmitSize, TMP2, TMP3, MemSrc);\n sub(EmitSize, TMP3, TMP2, TMP4);\n (void)cbnz(EmitSize, TMP3, &LoopTop);\n mov(EmitSize, GetReg(Node), TMP2.R());\n } else {\n ARMEmitter::BackwardLabel LoopTop;\n (void)Bind(&LoopTop);\n ldaxr(SubEmitSize, TMP2, MemSrc);\n neg(EmitSize, TMP3, TMP2);\n stlxr(SubEmitSize, TMP4, TMP3, MemSrc);\n (void)cbnz(EmitSize, TMP4, &LoopTop);\n mov(EmitSize, GetReg(Node), TMP2.R());\n }\n}\n\nDEF_OP(TelemetrySetValue) {\n#ifndef FEX_DISABLE_TELEMETRY\n auto Op = IROp->C();\n auto Src = GetReg(Op->Value);\n\n ldr(TMP2, STATE_PTR_IDX(CpuStateFrame, Pointers.TelemetryValueAddresses, Op->TelemetryValueIndex));\n\n // Cortex fuses cmp+cset.\n cmp(ARMEmitter::Size::i32Bit, Src, 0);\n cset(ARMEmitter::Size::i32Bit, TMP1, ARMEmitter::Condition::CC_NE);\n\n if (CTX->HostFeatures.SupportsAtomics) {\n stsetl(ARMEmitter::SubRegSize::i64Bit, TMP1, TMP2);\n } else {\n ARMEmitter::BackwardLabel LoopTop;\n (void)Bind(&LoopTop);\n ldaxr(ARMEmitter::SubRegSize::i64Bit, TMP3, TMP2);\n orr(ARMEmitter::Size::i32Bit, TMP3, TMP3, Src);\n stlxr(ARMEmitter::SubRegSize::i64Bit, TMP3, TMP3, TMP2);\n (void)cbnz(ARMEmitter::Size::i32Bit, TMP3, &LoopTop);\n }\n#endif\n}\n\n} // namespace FEXCore::CPU\n" }, { "path": "FEXCore/Source/Interface/Core/JIT/BranchOps.cpp", "content": "// SPDX-License-Identifier: MIT\n/*\n$info$\ntags: backend|arm64\n$end_info$\n*/\n\n#include \"Interface/Context/Context.h\"\n#include \"FEXCore/IR/IR.h\"\n#include \"Interface/Core/LookupCache.h\"\n\n#include \"Interface/Core/JIT/JITClass.h\"\n\n#include \n#include \n#include \n#include \n#include \n\nnamespace FEXCore::CPU {\n\nDEF_OP(CallbackReturn) {\n // spill back to CTX\n SpillStaticRegs(TMP1);\n\n // First we must reset the stack\n ResetStack();\n\n // We can now lower the ref counter again\n\n ldr(ARMEmitter::WReg::w2, STATE, offsetof(FEXCore::Core::CpuStateFrame, SignalHandlerRefCounter));\n sub(ARMEmitter::Size::i32Bit, ARMEmitter::Reg::r2, ARMEmitter::Reg::r2, 1);\n str(ARMEmitter::WReg::w2, STATE, offsetof(FEXCore::Core::CpuStateFrame, SignalHandlerRefCounter));\n\n // We need to adjust an additional 8 bytes to get back to the original \"misaligned\" RSP state\n ldr(ARMEmitter::XReg::x2, STATE, offsetof(FEXCore::Core::CpuStateFrame, State.gregs[X86State::REG_RSP]));\n add(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r2, ARMEmitter::Reg::r2, 8);\n str(ARMEmitter::XReg::x2, STATE, offsetof(FEXCore::Core::CpuStateFrame, State.gregs[X86State::REG_RSP]));\n\n PopCalleeSavedRegisters();\n\n // Return to the thunk\n ret();\n}\n\nDEF_OP(ExitFunction) {\n auto Op = IROp->C();\n\n ResetStack();\n\n if (CTX->HostFeatures.IsInstCountCI) [[unlikely]] {\n // Emit function end marker\n udf(0x420F);\n }\n\n uint64_t NewRIP;\n\n if (IsInlineConstant(Op->NewRIP, &NewRIP) || IsInlineEntrypointOffset(Op->NewRIP, &NewRIP)) {\n#ifdef ARCHITECTURE_arm64ec\n if (NewRIP < EC_CODE_BITMAP_MAX_ADDRESS && RtlIsEcCode(NewRIP)) {\n str(REG_CALLRET_SP, STATE_PTR(CpuStateFrame, State.callret_sp));\n add(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::rsp, StaticRegisters[X86State::REG_RSP], 0);\n InsertGuestRIPMove(EC_CALL_CHECKER_PC_REG, NewRIP);\n ldr(TMP2, STATE_PTR(CpuStateFrame, Pointers.ExitFunctionEC));\n br(TMP2);\n } else {\n#endif\n // In order to support direct branches without constantly hitting the L1 cache, we emit a call to a block linker,\n // this will compile the branch target block when it is hit and replace the branch to the linker at the callsite\n // with a direct branch to the destination block. Upon invalidation of the target block the backpatch is undone.\n //\n // In addition, to avoid needing to lookup in the cache for returns and any indirect branch prediction penalty,\n // a shadow stack of pairs is maintained, acting as a first level cache for any\n // return operations. As the guest may not balance calls and returns exactly, an exception handler is expected to\n // be installed by the frontend, to reset the shadow stack to the middle of its valid bounds on overflow/underflow.\n // This shadow stack is also cleared on block invalidation operations or codebuffer switches, to ensure all pointed-to\n // host code is always valid.\n\n // This code will be backpatched by Arm64JITCore_ExitFunctionLink, below is an enumeration of all the possible cases.\n // Jump thunks are emitted in JIT.cpp after compilation of the entire multiblock.\n //\n // Call with known return block - unlinked\n // 00: adr TMP1, 0xC\n // 04: stp RetReg, TMP1, [SpReg, -0x10]!\n // 08: bl JmpThunk00\n // JmpThunk00:\n // 00: b 0x8\n // 04: br TMP1\n // 08: ldr TMP1, \n // 0c: blr TMP1\n // 10: HostCode\n // 18: GuestRIP\n // 20: CallerOffset\n //\n // Call with known return block after backpatching - linked in branch immediate range\n // 00: adr TMP1, 0xC\n // 04: stp RetReg, TMP1, [SpReg, -0x10]!\n // 08: bl HostCode - MODIFIED\n //\n // Call with known return block after backpatching - linked out of range\n // 00: adr TMP1, 0xC\n // 04: stp RetReg, TMP1, [SpReg, -0x10]!\n // 08: bl JmpThunk00\n // JmpThunk00:\n // 00: ldr TMP1, 0x10 - MODIFIED 2nd\n // 04: br TMP1\n // 08: ldr TMP1, \n // 0c: blr TMP1\n // 10: HostCode - MODIFIED 1st\n // 18: GuestRIP\n // 20: CallerOffset\n //\n // Jump - unlinked\n // 00: b JmpThunk00\n // JmpThunk00:\n // 00: b 0x8\n // 04: br TMP1\n // 08: ldr TMP1, \n // 0c: blr TMP1\n // 10: HostCode\n // 18: GuestRIP\n // 20: CallerOffset\n //\n // Jump after backpatching - linked in branch immediate range\n // 00: b HostCode - MODIFIED\n //\n // Jump after backpatching - linked out of range\n // 00: b JmpThunk00\n // JmpThunk00:\n // 00: ldr TMP1, 0x10 - MODIFIED 2nd\n // 04: br TMP1\n // 08: ldr TMP1, \n // 0c: blr TMP1\n // 10: HostCode - MODIFIED 1st\n // 18: GuestRIP\n // 20: CallerOffset\n\n ARMEmitter::ForwardLabel l_BranchHost;\n ARMEmitter::ForwardLabel l_CallReturn;\n if (Op->Hint == IR::BranchHint::Call) {\n if (!Op->CallReturnBlock.IsInvalid()) {\n auto CallReturnAddressReg = GetReg(Op->CallReturnAddress).X();\n PendingCallReturnTargetLabel = &CallReturnTargets.try_emplace(Op->CallReturnBlock.ID()).first->second;\n (void)adr(TMP1, &l_CallReturn);\n stp(CallReturnAddressReg, TMP1, REG_CALLRET_SP, -0x10);\n } else {\n stp(ARMEmitter::XReg::zr, ARMEmitter::XReg::zr, REG_CALLRET_SP, -0x10);\n }\n } else if (Op->Hint == IR::BranchHint::CheckTF) {\n ARMEmitter::ForwardLabel TFUnset;\n ldrb(TMP1, STATE_PTR(CpuStateFrame, State.flags[X86State::RFLAG_TF_RAW_LOC]));\n (void)cbz(ARMEmitter::Size::i32Bit, TMP1, &TFUnset);\n InsertGuestRIPMove(TMP1, NewRIP);\n str(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, State.rip));\n ldr(TMP2, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.DispatcherLoopTop));\n blr(TMP2);\n (void)Bind(&TFUnset);\n }\n\n EmitLinkedBranch(NewRIP, Op->Hint == IR::BranchHint::Call);\n (void)Bind(&l_CallReturn);\n#ifdef ARCHITECTURE_arm64ec\n }\n#endif\n } else {\n ARMEmitter::ForwardLabel SkipFullLookup;\n auto RipReg = GetReg(Op->NewRIP);\n\n if (Op->Hint == IR::BranchHint::Return) {\n // First try to pop from the call-ret stack, otherwise follow the normal path (but ending in a ret)\n ldp(TMP1, TMP2, REG_CALLRET_SP, 0x10);\n sub(TMP1, TMP1, RipReg.X());\n (void)cbz(ARMEmitter::Size::i64Bit, TMP1, &SkipFullLookup);\n }\n\n // L1 Cache\n ldp(TMP1, TMP2, STATE, offsetof(FEXCore::Core::CpuStateFrame, State.L1Pointer));\n\n // Calculate (tmp1 + ((ripreg & L1_ENTRIES_MASK) << 4)) for the address\n // L1Mask is pre-shifted.\n and_(ARMEmitter::Size::i64Bit, TMP2, TMP2, RipReg, ARMEmitter::ShiftType::LSL, FEXCore::ilog2(sizeof(LookupCache::LookupCacheEntry)));\n add(TMP1, TMP1, TMP2);\n\n ldp(TMP2, TMP1, TMP1, 0);\n\n // Note: sub+cbnz used over cmp+br to preserve flags.\n sub(TMP1, TMP1, RipReg.X());\n (void)cbz(ARMEmitter::Size::i64Bit, TMP1, &SkipFullLookup);\n ldr(TMP2, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.DispatcherLoopTop));\n str(RipReg.X(), STATE, offsetof(FEXCore::Core::CpuStateFrame, State.rip));\n\n (void)Bind(&SkipFullLookup);\n if (Op->Hint == IR::BranchHint::Call) {\n ARMEmitter::ForwardLabel l_CallReturn;\n if (!Op->CallReturnBlock.IsInvalid()) {\n auto CallReturnAddressReg = GetReg(Op->CallReturnAddress).X();\n PendingCallReturnTargetLabel = &CallReturnTargets.try_emplace(Op->CallReturnBlock.ID()).first->second;\n (void)adr(TMP1, &l_CallReturn);\n stp(CallReturnAddressReg, TMP1, REG_CALLRET_SP, -0x10);\n } else {\n stp(ARMEmitter::XReg::zr, ARMEmitter::XReg::zr, REG_CALLRET_SP, -0x10);\n }\n blr(TMP2);\n (void)Bind(&l_CallReturn);\n } else if (Op->Hint == IR::BranchHint::Return) {\n ret(TMP2);\n } else {\n br(TMP2);\n }\n }\n}\n\nDEF_OP(Jump) {\n const auto Op = IROp->C();\n\n PendingTargetLabel = JumpTarget(Op->TargetBlock);\n}\n\nDEF_OP(CondJump) {\n auto Op = IROp->C();\n\n auto TrueTargetLabel = JumpTarget(Op->TrueBlock);\n\n if (Op->FromNZCV) {\n b_OrRestart(MapCC(Op->Cond), TrueTargetLabel);\n } else {\n uint64_t Const;\n const bool isConst = IsInlineConstant(Op->Cmp2, &Const);\n\n auto Reg = GetReg(Op->Cmp1);\n const auto Size = Op->CompareSize == IR::OpSize::i32Bit ? ARMEmitter::Size::i32Bit : ARMEmitter::Size::i64Bit;\n\n LOGMAN_THROW_A_FMT(IsGPR(Op->Cmp1), \"CondJump: Expected GPR\");\n LOGMAN_THROW_A_FMT(isConst, \"CondJump: Expected constant source\");\n\n if (Op->Cond == IR::CondClass::EQ) {\n LOGMAN_THROW_A_FMT(Const == 0, \"CondJump: Expected 0 source\");\n cbz_OrRestart(Size, Reg, TrueTargetLabel);\n } else if (Op->Cond == IR::CondClass::NEQ) {\n LOGMAN_THROW_A_FMT(Const == 0, \"CondJump: Expected 0 source\");\n cbnz_OrRestart(Size, Reg, TrueTargetLabel);\n } else if (Op->Cond == IR::CondClass::TSTZ) {\n LOGMAN_THROW_A_FMT(Const < 64, \"CondJump: Expected valid bit source\");\n tbz_OrRestart(Reg, Const, TrueTargetLabel);\n } else if (Op->Cond == IR::CondClass::TSTNZ) {\n LOGMAN_THROW_A_FMT(Const < 64, \"CondJump: Expected valid bit source\");\n tbnz_OrRestart(Reg, Const, TrueTargetLabel);\n } else {\n LOGMAN_THROW_A_FMT(false, \"CondJump expected simple condition\");\n }\n }\n\n PendingTargetLabel = JumpTarget(Op->FalseBlock);\n}\n\nDEF_OP(Syscall) {\n auto Op = IROp->C();\n // Arguments are passed as follows:\n // X0: SyscallHandler\n // X1: ThreadState\n // X2: Pointer to SyscallArguments\n\n PushDynamicRegs(TMP1);\n\n uint32_t GPRSpillMask = ~0U;\n uint32_t FPRSpillMask = ~0U;\n\n SpillStaticRegs(TMP1, {\n .GPRSpillMask = GPRSpillMask,\n .FPRSpillMask = FPRSpillMask,\n });\n\n // Now that we are spilled, store in the state that we are in a syscall\n // Still without overwriting registers that matter\n // 16bit LoadConstant to be a single instruction\n // This gives the signal handler a value to check to see if we are in a syscall at all\n LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, GPRSpillMask & 0xFFFF);\n str(ARMEmitter::XReg::x0, STATE, offsetof(FEXCore::Core::CpuStateFrame, InSyscallInfo));\n\n uint64_t SPOffset = AlignUp(FEXCore::HLE::SyscallArguments::MAX_ARGS * 8, 16);\n sub(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::rsp, ARMEmitter::Reg::rsp, SPOffset);\n for (uint32_t i = 0; i < FEXCore::HLE::SyscallArguments::MAX_ARGS; ++i) {\n if (Op->Header.Args[i].IsInvalid()) {\n continue;\n }\n str(GetReg(Op->Header.Args[i]).X(), ARMEmitter::Reg::rsp, i * 8);\n }\n\n ldr(ARMEmitter::XReg::x0, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.SyscallHandlerObj));\n ldr(ARMEmitter::XReg::x3, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.SyscallHandlerFunc));\n mov(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r1, STATE.R());\n\n // SP supporting move\n add(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r2, ARMEmitter::Reg::rsp, 0);\n if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] {\n GenerateIndirectRuntimeCall(ARMEmitter::Reg::r3);\n } else {\n blr(ARMEmitter::Reg::r3);\n }\n\n add(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::rsp, ARMEmitter::Reg::rsp, SPOffset);\n\n // Result is now in x0\n // Fix the stack and any values that were stepped on\n FillStaticRegs({\n .OptionalReg = ARMEmitter::Reg::r1,\n .OptionalReg2 = ARMEmitter::Reg::r2,\n .GPRFillMask = GPRSpillMask,\n .FPRFillMask = FPRSpillMask,\n });\n\n // Now the registers we've spilled are back in their original host registers\n // We can safely claim we are no longer in a syscall\n str(ARMEmitter::XReg::zr, STATE, offsetof(FEXCore::Core::CpuStateFrame, InSyscallInfo));\n\n PopDynamicRegs();\n\n const auto OSABI = CTX->SyscallHandler->GetOSABI();\n\n if (OSABI != FEXCore::HLE::SyscallOSABI::OS_GENERIC) {\n // Move result to its destination register.\n // Only if `NORETURNEDRESULT` wasn't set, otherwise we might overwrite the CPUState refilled with `FillStaticRegs`\n mov(ARMEmitter::Size::i64Bit, GetReg(Node), ARMEmitter::Reg::r0);\n }\n}\n\nDEF_OP(Thunk) {\n auto Op = IROp->C();\n // Arguments are passed as follows:\n // X0: CTX\n // X1: Args (from guest stack)\n\n // spill to ctx before ra64 spill\n SpillStaticRegs(TMP1, {\n .NZCV = false,\n });\n\n PushDynamicRegs(TMP1);\n\n mov(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, GetReg(Op->ArgPtr));\n\n InsertNamedThunkRelocation(ARMEmitter::Reg::r2, Op->ThunkNameHash);\n if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] {\n GenerateIndirectRuntimeCall(ARMEmitter::Reg::r2);\n } else {\n blr(ARMEmitter::Reg::r2);\n }\n\n PopDynamicRegs();\n\n // load from ctx after ra64 refill\n FillStaticRegs({\n .NZCV = false,\n });\n}\n\nDEF_OP(ValidateCode) {\n auto Op = IROp->C();\n auto OldCode = Op->CodeOriginal.data();\n auto Base = GetReg(Op->Header.Args[0]).X();\n int len = Op->CodeLength;\n int Offset = 0;\n ARMEmitter::ForwardLabel Fail;\n\n const auto Dst = GetReg(Node);\n\n auto EmitCheck = [&](size_t Size, auto&& LoadData) {\n while (len >= Size) {\n LoadData();\n sub(ARMEmitter::Size::i64Bit, TMP1, TMP1, TMP2);\n cbnz_OrRestart(ARMEmitter::Size::i64Bit, TMP1, &Fail);\n len -= Size;\n Offset += Size;\n }\n };\n\n EmitCheck(8, [&]() {\n ldr(TMP1, Base, Offset);\n LoadConstant(ARMEmitter::Size::i64Bit, TMP2, *(const uint64_t*)(OldCode + Offset));\n });\n\n EmitCheck(4, [&]() {\n ldr(TMP1.W(), Base, Offset);\n LoadConstant(ARMEmitter::Size::i32Bit, TMP2, *(const uint32_t*)(OldCode + Offset));\n });\n\n EmitCheck(2, [&]() {\n ldrh(TMP1.W(), Base, Offset);\n LoadConstant(ARMEmitter::Size::i32Bit, TMP2, *(const uint16_t*)(OldCode + Offset));\n });\n\n EmitCheck(1, [&]() {\n ldrb(TMP1.W(), Base, Offset);\n LoadConstant(ARMEmitter::Size::i32Bit, TMP2, *(const uint8_t*)(OldCode + Offset));\n });\n\n ARMEmitter::ForwardLabel End;\n LoadConstant(ARMEmitter::Size::i32Bit, Dst, 0);\n b_OrRestart(&End);\n BindOrRestart(&Fail);\n LoadConstant(ARMEmitter::Size::i32Bit, Dst, 1);\n BindOrRestart(&End);\n}\n\nDEF_OP(ThreadRemoveCodeEntry) {\n PushDynamicRegs(TMP4);\n SpillStaticRegs(TMP4);\n\n // Arguments are passed as follows:\n // X0: Thread\n // X1: RIP\n mov(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, STATE.R());\n\n // TODO: Relocations don't seem to be wired up to this...?\n LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r1, Entry, CPU::Arm64Emitter::PadType::AUTOPAD);\n\n ldr(ARMEmitter::XReg::x2, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.ThreadRemoveCodeEntryFromJIT));\n if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] {\n GenerateIndirectRuntimeCall(ARMEmitter::Reg::r2);\n } else {\n blr(ARMEmitter::Reg::r2);\n }\n FillStaticRegs();\n\n // Fix the stack and any values that were stepped on\n PopDynamicRegs();\n}\n\nDEF_OP(CPUID) {\n auto Op = IROp->C();\n\n isb();\n mov(ARMEmitter::Size::i64Bit, TMP2, GetReg(Op->Function));\n mov(ARMEmitter::Size::i64Bit, TMP3, GetReg(Op->Leaf));\n\n PushDynamicRegs(TMP4);\n SpillStaticRegs(TMP4);\n\n // x0 = CPUID Handler\n // x1 = CPUID Function\n // x2 = CPUID Leaf\n ldr(ARMEmitter::XReg::x0, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.CPUIDObj));\n ldr(ARMEmitter::XReg::x3, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.CPUIDFunction));\n\n if (!TMP_ABIARGS) {\n mov(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r1, TMP2);\n mov(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r2, TMP3);\n }\n\n if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] {\n GenerateIndirectRuntimeCall<__uint128_t, void*, uint64_t, uint64_t>(ARMEmitter::Reg::r3);\n } else {\n blr(ARMEmitter::Reg::r3);\n }\n\n if (!TMP_ABIARGS) {\n mov(ARMEmitter::Size::i64Bit, TMP1, ARMEmitter::Reg::r0);\n mov(ARMEmitter::Size::i64Bit, TMP2, ARMEmitter::Reg::r1);\n }\n\n FillStaticRegs();\n\n PopDynamicRegs();\n\n // Results are in x0, x1\n // Results want to be 4xi32 scalars\n mov(ARMEmitter::Size::i32Bit, GetReg(Op->OutEAX), TMP1);\n mov(ARMEmitter::Size::i32Bit, GetReg(Op->OutECX), TMP2);\n ubfx(ARMEmitter::Size::i64Bit, GetReg(Op->OutEBX), TMP1, 32, 32);\n ubfx(ARMEmitter::Size::i64Bit, GetReg(Op->OutEDX), TMP2, 32, 32);\n}\n\nDEF_OP(XGetBV) {\n auto Op = IROp->C();\n\n PushDynamicRegs(TMP4);\n SpillStaticRegs(TMP4);\n\n mov(ARMEmitter::Size::i32Bit, ARMEmitter::Reg::r1, GetReg(Op->Function));\n\n // x0 = CPUID Handler\n // x1 = XCR Function\n ldr(ARMEmitter::XReg::x0, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.CPUIDObj));\n ldr(ARMEmitter::XReg::x2, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.XCRFunction));\n if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] {\n GenerateIndirectRuntimeCall(ARMEmitter::Reg::r2);\n } else {\n blr(ARMEmitter::Reg::r2);\n }\n\n if (!TMP_ABIARGS) {\n mov(ARMEmitter::Size::i64Bit, TMP1, ARMEmitter::Reg::r0);\n }\n\n FillStaticRegs();\n\n PopDynamicRegs();\n\n // Results are in x0, need to split into i32 parts\n mov(ARMEmitter::Size::i32Bit, GetReg(Op->OutEAX), TMP1);\n ubfx(ARMEmitter::Size::i64Bit, GetReg(Op->OutEDX), TMP1, 32, 32);\n}\n\n} // namespace FEXCore::CPU\n" }, { "path": "FEXCore/Source/Interface/Core/JIT/ConversionOps.cpp", "content": "// SPDX-License-Identifier: MIT\n/*\n$info$\ntags: backend|arm64\n$end_info$\n*/\n\n#include \"Interface/Core/JIT/JITClass.h\"\n#include \"Interface/Context/Context.h\"\n\nnamespace FEXCore::CPU {\nDEF_OP(VInsGPR) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto DestIdx = Op->DestIdx;\n const auto ElementSize = Op->Header.ElementSize;\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto SubEmitSize = ConvertSubRegSize8(IROp);\n const auto ElementsPer128Bit = IR::NumElements(IR::OpSize::i128Bit, ElementSize);\n\n const auto Dst = GetVReg(Node);\n const auto DestVector = GetVReg(Op->DestVector);\n const auto Src = GetReg(Op->Src);\n\n if (HostSupportsSVE256 && Is256Bit) {\n const auto ElementSizeBits = IR::OpSizeAsBits(ElementSize);\n const auto Offset = ElementSizeBits * DestIdx;\n\n const auto SSEBitSize = Core::CPUState::XMM_SSE_REG_SIZE * 8;\n const auto InUpperLane = Offset >= SSEBitSize;\n\n // This is going to be a little gross. Pls forgive me.\n // Since SVE has the whole vector length agnostic programming\n // thing going on, we can't exactly freely insert entries into\n // arbitrary locations in the vector.\n //\n // SVE *does* have INSR, however this only shifts the entire\n // vector to the left by an element size and inserts a value\n // at the beginning of the vector. Not *quite* what we need.\n // (though INSR *is* very useful for other things).\n //\n // The idea is (in the case of the upper lane), move the upper\n // lane down, insert into it and recombine with the lower lane.\n //\n // In the case of the lower lane, insert and then recombine with\n // the upper lane.\n\n if (InUpperLane) {\n // Move the upper lane down for the insertion.\n const auto CompactPred = ARMEmitter::PReg::p0;\n not_(CompactPred, PRED_TMP_32B.Zeroing(), PRED_TMP_16B);\n compact(ARMEmitter::SubRegSize::i64Bit, VTMP1.Z(), CompactPred, DestVector.Z());\n }\n\n // Put data in place for destructive SPLICE below.\n mov(Dst.Z(), DestVector.Z());\n\n // Inserts the GPR value into the given V register.\n // Also automatically adjusts the index in the case of using the\n // moved upper lane.\n const auto Insert = [&](const ARMEmitter::VRegister& reg, int index) {\n if (InUpperLane) {\n index -= ElementsPer128Bit;\n }\n ins(SubEmitSize, reg, index, Src);\n };\n\n if (InUpperLane) {\n Insert(VTMP1, DestIdx);\n splice(ARMEmitter::SubRegSize::i64Bit, Dst.Z(), PRED_TMP_16B, Dst.Z(), VTMP1.Z());\n } else {\n Insert(Dst, DestIdx);\n splice(ARMEmitter::SubRegSize::i64Bit, Dst.Z(), PRED_TMP_16B, Dst.Z(), DestVector.Z());\n }\n } else {\n // No need to move if Dst and DestVector alias one another.\n if (Dst != DestVector) {\n mov(Dst.Q(), DestVector.Q());\n }\n ins(SubEmitSize, Dst, DestIdx, Src);\n }\n}\n\nDEF_OP(VCastFromGPR) {\n auto Op = IROp->C();\n auto Dst = GetVReg(Node);\n auto Src = GetReg(Op->Src);\n\n switch (Op->Header.ElementSize) {\n case IR::OpSize::i8Bit:\n uxtb(ARMEmitter::Size::i32Bit, TMP1, Src);\n fmov(ARMEmitter::Size::i32Bit, Dst.S(), TMP1);\n break;\n case IR::OpSize::i16Bit:\n uxth(ARMEmitter::Size::i32Bit, TMP1, Src);\n fmov(ARMEmitter::Size::i32Bit, Dst.S(), TMP1);\n break;\n case IR::OpSize::i32Bit: fmov(ARMEmitter::Size::i32Bit, Dst.S(), Src); break;\n case IR::OpSize::i64Bit: fmov(ARMEmitter::Size::i64Bit, Dst.D(), Src); break;\n default: LOGMAN_MSG_A_FMT(\"Unknown castGPR element size: {}\", Op->Header.ElementSize);\n }\n}\n\nDEF_OP(VLoadTwoGPRs) {\n const auto Op = IROp->C();\n\n const auto Dst = GetVReg(Node);\n const auto SrcLower = GetReg(Op->Lower);\n const auto SrcUpper = GetReg(Op->Upper);\n fmov(ARMEmitter::Size::i64Bit, Dst.D(), SrcLower);\n fmov(ARMEmitter::Size::i64Bit, Dst.D(), SrcUpper, true);\n}\n\nDEF_OP(VDupFromGPR) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto Dst = GetVReg(Node);\n const auto Src = GetReg(Op->Src);\n\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto SubEmitSize = ConvertSubRegSize8(IROp);\n\n if (HostSupportsSVE256 && Is256Bit) {\n dup(SubEmitSize, Dst.Z(), Src);\n } else {\n dup(SubEmitSize, Dst.Q(), Src);\n }\n}\n\nDEF_OP(Float_FromGPR_S) {\n const auto Op = IROp->C();\n\n const uint16_t ElementSize = IR::OpSizeToSize(Op->Header.ElementSize);\n const uint16_t Conv = (ElementSize << 8) | IR::OpSizeToSize(Op->SrcElementSize);\n\n auto Dst = GetVReg(Node);\n auto Src = GetReg(Op->Src);\n\n switch (Conv) {\n case 0x0204: { // Half <- int32_t\n scvtf(ARMEmitter::Size::i32Bit, Dst.H(), Src);\n break;\n }\n case 0x0208: { // Half <- int64_t\n scvtf(ARMEmitter::Size::i64Bit, Dst.H(), Src);\n break;\n }\n case 0x0404: { // Float <- int32_t\n scvtf(ARMEmitter::Size::i32Bit, Dst.S(), Src);\n break;\n }\n case 0x0408: { // Float <- int64_t\n scvtf(ARMEmitter::Size::i64Bit, Dst.S(), Src);\n break;\n }\n case 0x0804: { // Double <- int32_t\n scvtf(ARMEmitter::Size::i32Bit, Dst.D(), Src);\n break;\n }\n case 0x0808: { // Double <- int64_t\n scvtf(ARMEmitter::Size::i64Bit, Dst.D(), Src);\n break;\n }\n default:\n LOGMAN_MSG_A_FMT(\"Unhandled conversion mask: Mask=0x{:04x}, ElementSize={}, SrcElementSize={}\", Conv, ElementSize, Op->SrcElementSize);\n break;\n }\n}\n\nDEF_OP(Float_FToF) {\n auto Op = IROp->C();\n const uint16_t Conv = (IR::OpSizeToSize(Op->Header.ElementSize) << 8) | IR::OpSizeToSize(Op->SrcElementSize);\n\n auto Dst = GetVReg(Node);\n auto Src = GetVReg(Op->Scalar);\n\n switch (Conv) {\n case 0x0204: { // Half <- Float\n fcvt(Dst.H(), Src.S());\n break;\n }\n case 0x0208: { // Half <- Double\n fcvt(Dst.H(), Src.D());\n break;\n }\n case 0x0402: { // Float <- Half\n fcvt(Dst.S(), Src.H());\n break;\n }\n case 0x0802: { // Double <- Half\n fcvt(Dst.D(), Src.H());\n break;\n }\n case 0x0804: { // Double <- Float\n fcvt(Dst.D(), Src.S());\n break;\n }\n case 0x0408: { // Float <- Double\n fcvt(Dst.S(), Src.D());\n break;\n }\n default: LOGMAN_MSG_A_FMT(\"Unknown FCVT sizes: 0x{:x}\", Conv);\n }\n}\n\nDEF_OP(Vector_SToF) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto ElementSize = Op->Header.ElementSize;\n const auto SubEmitSize = ConvertSubRegSize248(IROp);\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto Dst = GetVReg(Node);\n const auto Vector = GetVReg(Op->Vector);\n if (HostSupportsSVE256 && Is256Bit) {\n const auto Mask = PRED_TMP_32B;\n scvtf(Dst.Z(), SubEmitSize, Mask.Merging(), Vector.Z(), SubEmitSize);\n } else {\n if (OpSize == ElementSize) {\n if (ElementSize == IR::OpSize::i64Bit) {\n scvtf(ARMEmitter::ScalarRegSize::i64Bit, Dst.D(), Vector.D());\n } else if (ElementSize == IR::OpSize::i32Bit) {\n scvtf(ARMEmitter::ScalarRegSize::i32Bit, Dst.S(), Vector.S());\n } else {\n scvtf(ARMEmitter::ScalarRegSize::i16Bit, Dst.H(), Vector.H());\n }\n } else {\n if (OpSize == IR::OpSize::i64Bit) {\n scvtf(SubEmitSize, Dst.D(), Vector.D());\n } else {\n scvtf(SubEmitSize, Dst.Q(), Vector.Q());\n }\n }\n }\n}\n\nDEF_OP(Vector_FToZS) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto ElementSize = Op->Header.ElementSize;\n const auto SubEmitSize = ConvertSubRegSize248(IROp);\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto Dst = GetVReg(Node);\n const auto Vector = GetVReg(Op->Vector);\n if (HostSupportsSVE256 && Is256Bit) {\n const auto Mask = PRED_TMP_32B;\n fcvtzs(Dst.Z(), SubEmitSize, Mask.Merging(), Vector.Z(), SubEmitSize);\n } else {\n if (OpSize == ElementSize) {\n if (ElementSize == IR::OpSize::i64Bit) {\n fcvtzs(ARMEmitter::ScalarRegSize::i64Bit, Dst.D(), Vector.D());\n } else if (ElementSize == IR::OpSize::i32Bit) {\n fcvtzs(ARMEmitter::ScalarRegSize::i32Bit, Dst.S(), Vector.S());\n } else {\n fcvtzs(ARMEmitter::ScalarRegSize::i16Bit, Dst.H(), Vector.H());\n }\n } else {\n if (OpSize == IR::OpSize::i64Bit) {\n fcvtzs(SubEmitSize, Dst.D(), Vector.D());\n } else {\n fcvtzs(SubEmitSize, Dst.Q(), Vector.Q());\n }\n }\n }\n}\n\nDEF_OP(Vector_FToS) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto SubEmitSize = ConvertSubRegSize248(IROp);\n\n const auto Dst = GetVReg(Node);\n const auto Vector = GetVReg(Op->Vector);\n\n if (HostSupportsSVE256 && Is256Bit) {\n const auto Mask = PRED_TMP_32B;\n frinti(SubEmitSize, Dst.Z(), Mask.Merging(), Vector.Z());\n fcvtzs(Dst.Z(), SubEmitSize, Mask.Merging(), Dst.Z(), SubEmitSize);\n } else {\n const auto Dst = GetVReg(Node);\n const auto Vector = GetVReg(Op->Vector);\n if (OpSize == IR::OpSize::i64Bit) {\n frinti(SubEmitSize, Dst.D(), Vector.D());\n fcvtzs(SubEmitSize, Dst.D(), Dst.D());\n } else {\n frinti(SubEmitSize, Dst.Q(), Vector.Q());\n fcvtzs(SubEmitSize, Dst.Q(), Dst.Q());\n }\n }\n}\n\nDEF_OP(Vector_FToF) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto ElementSize = Op->Header.ElementSize;\n const auto SubEmitSize = ConvertSubRegSize248(IROp);\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto Conv = (IR::OpSizeToSize(ElementSize) << 8) | IR::OpSizeToSize(Op->SrcElementSize);\n\n const auto Dst = GetVReg(Node);\n const auto Vector = GetVReg(Op->Vector);\n\n if (HostSupportsSVE256 && Is256Bit) {\n // Curiously, FCVTLT and FCVTNT have no bottom variants,\n // and also interesting is that FCVTLT will iterate the\n // source vector by accessing each odd element and storing\n // them consecutively in the destination.\n //\n // FCVTNT is somewhat like the opposite. It will read each\n // consecutive element, but store each result into every odd\n // element in the destination vector.\n //\n // We need to undo the behavior of FCVTNT with UZP2. In the case\n // of FCVTLT, we instead need to set the vector up with ZIP1, so\n // that the elements will be processed correctly.\n\n const auto Mask = PRED_TMP_32B.Merging();\n\n switch (Conv) {\n case 0x0402: { // Float <- Half\n zip1(ARMEmitter::SubRegSize::i16Bit, Dst.Z(), Vector.Z(), Vector.Z());\n fcvtlt(ARMEmitter::SubRegSize::i32Bit, Dst.Z(), Mask, Dst.Z());\n break;\n }\n case 0x0804: { // Double <- Float\n zip1(ARMEmitter::SubRegSize::i32Bit, Dst.Z(), Vector.Z(), Vector.Z());\n fcvtlt(ARMEmitter::SubRegSize::i64Bit, Dst.Z(), Mask, Dst.Z());\n break;\n }\n case 0x0204: { // Half <- Float\n fcvtnt(ARMEmitter::SubRegSize::i16Bit, Dst.Z(), Mask, Vector.Z());\n uzp2(ARMEmitter::SubRegSize::i16Bit, Dst.Z(), Dst.Z(), Dst.Z());\n break;\n }\n case 0x0408: { // Float <- Double\n fcvtnt(ARMEmitter::SubRegSize::i32Bit, Dst.Z(), Mask, Vector.Z());\n uzp2(ARMEmitter::SubRegSize::i32Bit, Dst.Z(), Dst.Z(), Dst.Z());\n break;\n }\n default: LOGMAN_MSG_A_FMT(\"Unknown Vector_FToF Type : 0x{:04x}\", Conv); break;\n }\n } else {\n switch (Conv) {\n case 0x0402: // Float <- Half\n case 0x0804: { // Double <- Float\n fcvtl(SubEmitSize, Dst.D(), Vector.D());\n break;\n }\n case 0x0204: // Half <- Float\n case 0x0408: { // Float <- Double\n fcvtn(SubEmitSize, Dst.D(), Vector.D());\n break;\n }\n default: LOGMAN_MSG_A_FMT(\"Unknown Vector_FToF Type : 0x{:04x}\", Conv); break;\n }\n }\n}\n\nDEF_OP(VFCVTL2) {\n const auto Op = IROp->C();\n\n const auto SubEmitSize = ConvertSubRegSize248(IROp);\n\n const auto Dst = GetVReg(Node);\n const auto Vector = GetVReg(Op->Vector);\n\n fcvtl2(SubEmitSize, Dst.D(), Vector.D());\n}\n\nDEF_OP(VFCVTN2) {\n const auto Op = IROp->C();\n\n const auto SubEmitSize = ConvertSubRegSize248(IROp);\n\n const auto Dst = GetVReg(Node);\n const auto VectorLower = GetVReg(Op->VectorLower);\n const auto VectorUpper = GetVReg(Op->VectorUpper);\n\n auto Lower = VectorLower;\n if (Dst != VectorLower) {\n mov(VTMP1.Q(), VectorLower.Q());\n Lower = VTMP1;\n }\n\n fcvtn2(SubEmitSize, Lower.Q(), VectorUpper.Q());\n\n if (Dst != VectorLower) {\n mov(Dst.Q(), Lower.Q());\n }\n}\n\nDEF_OP(Vector_FToI) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto ElementSize = Op->Header.ElementSize;\n const auto SubEmitSize = ConvertSubRegSize248(IROp);\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto Dst = GetVReg(Node);\n const auto Vector = GetVReg(Op->Vector);\n\n if (HostSupportsSVE256 && Is256Bit) {\n const auto Mask = PRED_TMP_32B.Merging();\n\n switch (Op->Round) {\n case IR::RoundMode::Nearest: frintn(SubEmitSize, Dst.Z(), Mask, Vector.Z()); break;\n case IR::RoundMode::NegInfinity: frintm(SubEmitSize, Dst.Z(), Mask, Vector.Z()); break;\n case IR::RoundMode::PosInfinity: frintp(SubEmitSize, Dst.Z(), Mask, Vector.Z()); break;\n case IR::RoundMode::TowardsZero: frintz(SubEmitSize, Dst.Z(), Mask, Vector.Z()); break;\n case IR::RoundMode::Host: frinti(SubEmitSize, Dst.Z(), Mask, Vector.Z()); break;\n }\n } else {\n const auto IsScalar = ElementSize == OpSize;\n\n if (IsScalar) {\n// Since we have multiple overloads of the same name (e.g.\n// frinti having AdvSIMD, AdvSIMD scalar, and an SVE version),\n// we can't just use a lambda without some seriously ugly casting.\n// This is fairly self-contained otherwise.\n#define ROUNDING_FN(name) \\\n if (ElementSize == IR::OpSize::i16Bit) { \\\n name(Dst.H(), Vector.H()); \\\n } else if (ElementSize == IR::OpSize::i32Bit) { \\\n name(Dst.S(), Vector.S()); \\\n } else if (ElementSize == IR::OpSize::i64Bit) { \\\n name(Dst.D(), Vector.D()); \\\n } else { \\\n FEX_UNREACHABLE; \\\n }\n\n switch (Op->Round) {\n case IR::RoundMode::Nearest: ROUNDING_FN(frintn); break;\n case IR::RoundMode::NegInfinity: ROUNDING_FN(frintm); break;\n case IR::RoundMode::PosInfinity: ROUNDING_FN(frintp); break;\n case IR::RoundMode::TowardsZero: ROUNDING_FN(frintz); break;\n case IR::RoundMode::Host: ROUNDING_FN(frinti); break;\n }\n\n#undef ROUNDING_FN\n } else {\n switch (Op->Round) {\n case IR::RoundMode::Nearest: frintn(SubEmitSize, Dst.Q(), Vector.Q()); break;\n case IR::RoundMode::NegInfinity: frintm(SubEmitSize, Dst.Q(), Vector.Q()); break;\n case IR::RoundMode::PosInfinity: frintp(SubEmitSize, Dst.Q(), Vector.Q()); break;\n case IR::RoundMode::TowardsZero: frintz(SubEmitSize, Dst.Q(), Vector.Q()); break;\n case IR::RoundMode::Host: frinti(SubEmitSize, Dst.Q(), Vector.Q()); break;\n }\n }\n }\n}\n\nDEF_OP(Vector_FToISized) {\n const auto Op = IROp->C();\n\n const auto ElementSize = Op->Header.ElementSize;\n const auto SubEmitSize = ConvertSubRegSize248(IROp);\n LOGMAN_THROW_A_FMT(IROp->Size != IR::OpSize::i256Bit, \"256-bit not wired up, though we could change that\");\n LOGMAN_THROW_A_FMT(CTX->HostFeatures.SupportsFRINTTS, \"Need FRINTTS for Vector_FToISized\");\n\n const auto Dst = GetVReg(Node);\n const auto Vector = GetVReg(Op->Vector);\n\n if (ElementSize == IROp->Size) {\n// See above\n#define ROUNDING_FN(name) \\\n if (ElementSize == IR::OpSize::i32Bit) { \\\n name(Dst.S(), Vector.S()); \\\n } else if (ElementSize == IR::OpSize::i64Bit) { \\\n name(Dst.D(), Vector.D()); \\\n } else { \\\n FEX_UNREACHABLE; \\\n }\n\n if (Op->IntSize == IR::OpSize::i64Bit) {\n if (Op->HostRound) {\n ROUNDING_FN(frint64x);\n } else {\n ROUNDING_FN(frint64z);\n }\n } else {\n if (Op->HostRound) {\n ROUNDING_FN(frint32x);\n } else {\n ROUNDING_FN(frint32z);\n }\n }\n\n#undef ROUNDING_FN\n } else {\n if (Op->IntSize == IR::OpSize::i64Bit) {\n if (Op->HostRound) {\n frint64x(SubEmitSize, Dst.Q(), Vector.Q());\n } else {\n frint64z(SubEmitSize, Dst.Q(), Vector.Q());\n }\n } else {\n if (Op->HostRound) {\n frint32x(SubEmitSize, Dst.Q(), Vector.Q());\n } else {\n frint32z(SubEmitSize, Dst.Q(), Vector.Q());\n }\n }\n }\n}\n\nDEF_OP(Vector_F64ToI32) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n const auto Round = Op->Round;\n\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto Dst = GetVReg(Node);\n const auto Vector = GetVReg(Op->Vector);\n if (HostSupportsSVE128 || HostSupportsSVE256) {\n const auto Mask = Is256Bit ? PRED_TMP_32B.Merging() : PRED_TMP_16B.Merging();\n // First step is to round the f64 values to integrals (frint*)\n // Then convert to integers using fcvtzs.\n auto CVTReg = Dst.Z();\n switch (Round) {\n case IR::RoundMode::Nearest: frintn(ARMEmitter::SubRegSize::i64Bit, Dst.Z(), Mask, Vector.Z()); break;\n case IR::RoundMode::NegInfinity: frintm(ARMEmitter::SubRegSize::i64Bit, Dst.Z(), Mask, Vector.Z()); break;\n case IR::RoundMode::PosInfinity: frintp(ARMEmitter::SubRegSize::i64Bit, Dst.Z(), Mask, Vector.Z()); break;\n case IR::RoundMode::TowardsZero: CVTReg = Vector.Z(); break;\n case IR::RoundMode::Host: frinti(ARMEmitter::SubRegSize::i64Bit, Dst.Z(), Mask, Vector.Z()); break;\n }\n\n fcvtzs(Dst.Z(), ARMEmitter::SubRegSize::i32Bit, Mask, CVTReg, ARMEmitter::SubRegSize::i64Bit);\n\n ///< Fixup format of register that fcvtzs returns.\n uzp1(ARMEmitter::SubRegSize::i32Bit, Dst.Z(), Dst.Z(), Dst.Z());\n if (Op->EnsureZeroUpperHalf) {\n ///< Match CVTPD2DQ/CVTTPD2DQ behaviour if necessary by zeroing the upper bits here.\n if (Is256Bit) {\n mov(Dst.Q(), Dst.Q());\n } else {\n mov(Dst.D(), Dst.D());\n }\n }\n } else {\n // This has a known precision issue that isn't easily resolvable without throwing away performance.\n // Doing the conversion in multi-stage steps has an issue that you can lose precision in the f32->i32 step if your source was f64.\n // To get around this with ASIMD FEX needs to use fcvtzs (Scalar, Integer, to GPR) for each F64 to be directly converted to i32.\n // This is a very costly transform that the SVE path doesn't need to do since it supports f64->i32 directly.\n // If this precision issue is necessary then we can add an option for it in the future.\n\n ///< Round float to integral depending on rounding mode.\n switch (Round) {\n case IR::RoundMode::Nearest: frintn(ARMEmitter::SubRegSize::i64Bit, Dst.Q(), Vector.Q()); break;\n case IR::RoundMode::NegInfinity: frintm(ARMEmitter::SubRegSize::i64Bit, Dst.Q(), Vector.Q()); break;\n case IR::RoundMode::PosInfinity: frintp(ARMEmitter::SubRegSize::i64Bit, Dst.Q(), Vector.Q()); break;\n case IR::RoundMode::TowardsZero: frintz(ARMEmitter::SubRegSize::i64Bit, Dst.Q(), Vector.Q()); break;\n case IR::RoundMode::Host: frinti(ARMEmitter::SubRegSize::i64Bit, Dst.Q(), Vector.Q()); break;\n }\n\n // Now narrow from f64 to f32.\n fcvtn(ARMEmitter::SubRegSize::i32Bit, Dst.Q(), Dst.Q());\n\n ///< Convert the two F32 integrals to real integers.\n fcvtzs(ARMEmitter::SubRegSize::i32Bit, Dst.D(), Dst.D());\n }\n}\n\n} // namespace FEXCore::CPU\n" }, { "path": "FEXCore/Source/Interface/Core/JIT/DebugData.h", "content": "// SPDX-License-Identifier: MIT\n#pragma once\n\n#include \n#include \n\n#include \n\nnamespace FEXCore::CPU {\nunion Relocation;\n} // namespace FEXCore::CPU\n\nnamespace FEXCore::Core {\nstruct DebugDataSubblock {\n uint32_t HostCodeOffset;\n uint32_t HostCodeSize;\n};\n\nstruct DebugDataGuestOpcode {\n uint64_t GuestEntryOffset;\n ptrdiff_t HostEntryOffset;\n};\n\n/**\n * @brief Contains debug data for a block of code for later debugger analysis\n *\n * Needs to remain around for as long as the code could be executed at least\n */\nstruct DebugData : public FEXCore::Allocator::FEXAllocOperators {\n uint64_t HostCodeSize; ///< The size of the code generated in the host JIT\n fextl::vector Subblocks;\n fextl::vector GuestOpcodes;\n fextl::vector* Relocations;\n};\n} // namespace FEXCore::Core\n" }, { "path": "FEXCore/Source/Interface/Core/JIT/EncryptionOps.cpp", "content": "// SPDX-License-Identifier: MIT\n/*\n$info$\ntags: backend|arm64\n$end_info$\n*/\n\n#include \"Interface/Core/JIT/JITClass.h\"\n\nnamespace FEXCore::CPU {\n\nDEF_OP(VAESImc) {\n auto Op = IROp->C();\n aesimc(GetVReg(Node), GetVReg(Op->Vector));\n}\n\nDEF_OP(VAESEnc) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto Dst = GetVReg(Node);\n const auto Key = GetVReg(Op->Key);\n const auto State = GetVReg(Op->State);\n const auto ZeroReg = GetVReg(Op->ZeroReg);\n\n LOGMAN_THROW_A_FMT(OpSize == IR::OpSize::i128Bit, \"Currently only supports 128-bit operations.\");\n\n if (Dst == State && Dst != Key) {\n // Optimal case in which Dst already contains the starting state.\n // This matches the common case of XMM AES.\n aese(Dst.Q(), ZeroReg.Q());\n aesmc(Dst.Q(), Dst.Q());\n eor(Dst.Q(), Dst.Q(), Key.Q());\n } else {\n mov(VTMP1.Q(), State.Q());\n aese(VTMP1, ZeroReg.Q());\n aesmc(VTMP1, VTMP1);\n eor(Dst.Q(), VTMP1.Q(), Key.Q());\n }\n}\n\nDEF_OP(VAESEncLast) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto Dst = GetVReg(Node);\n const auto Key = GetVReg(Op->Key);\n const auto State = GetVReg(Op->State);\n const auto ZeroReg = GetVReg(Op->ZeroReg);\n\n LOGMAN_THROW_A_FMT(OpSize == IR::OpSize::i128Bit, \"Currently only supports 128-bit operations.\");\n\n if (Dst == State && Dst != Key) {\n // Optimal case in which Dst already contains the starting state.\n // This matches the common case of XMM AES.\n aese(Dst.Q(), ZeroReg.Q());\n eor(Dst.Q(), Dst.Q(), Key.Q());\n } else {\n mov(VTMP1.Q(), State.Q());\n aese(VTMP1, ZeroReg.Q());\n eor(Dst.Q(), VTMP1.Q(), Key.Q());\n }\n}\n\nDEF_OP(VAESDec) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto Dst = GetVReg(Node);\n const auto Key = GetVReg(Op->Key);\n const auto State = GetVReg(Op->State);\n const auto ZeroReg = GetVReg(Op->ZeroReg);\n\n LOGMAN_THROW_A_FMT(OpSize == IR::OpSize::i128Bit, \"Currently only supports 128-bit operations.\");\n\n if (Dst == State && Dst != Key) {\n // Optimal case in which Dst already contains the starting state.\n // This matches the common case of XMM AES.\n aesd(Dst.Q(), ZeroReg.Q());\n aesimc(Dst.Q(), Dst.Q());\n eor(Dst.Q(), Dst.Q(), Key.Q());\n } else {\n mov(VTMP1.Q(), State.Q());\n aesd(VTMP1, ZeroReg.Q());\n aesimc(VTMP1, VTMP1);\n eor(Dst.Q(), VTMP1.Q(), Key.Q());\n }\n}\n\nDEF_OP(VAESDecLast) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto Dst = GetVReg(Node);\n const auto Key = GetVReg(Op->Key);\n const auto State = GetVReg(Op->State);\n const auto ZeroReg = GetVReg(Op->ZeroReg);\n\n LOGMAN_THROW_A_FMT(OpSize == IR::OpSize::i128Bit, \"Currently only supports 128-bit operations.\");\n\n if (Dst == State && Dst != Key) {\n // Optimal case in which Dst already contains the starting state.\n // This matches the common case of XMM AES.\n aesd(Dst.Q(), ZeroReg.Q());\n eor(Dst.Q(), Dst.Q(), Key.Q());\n } else {\n mov(VTMP1.Q(), State.Q());\n aesd(VTMP1, ZeroReg.Q());\n eor(Dst.Q(), VTMP1.Q(), Key.Q());\n }\n}\n\nDEF_OP(VAESKeyGenAssist) {\n auto Op = IROp->C();\n const auto Dst = GetVReg(Node);\n const auto Src = GetVReg(Op->Src);\n const auto Swizzle = GetVReg(Op->KeyGenTBLSwizzle);\n auto ZeroReg = GetVReg(Op->ZeroReg);\n\n if (Dst == ZeroReg) {\n // Seriously? ZeroReg ended up being the destination register?\n // Just copy it over in this case...\n mov(VTMP1.Q(), ZeroReg.Q());\n ZeroReg = VTMP1;\n }\n\n if (Dst != Src) {\n mov(Dst.Q(), Src.Q());\n }\n\n // Do a \"regular\" AESE step\n aese(Dst, ZeroReg.Q());\n\n // Now EOR in the RCON\n if (Op->RCON) {\n tbl(Dst.Q(), Dst.Q(), Swizzle.Q());\n\n LoadConstant(ARMEmitter::Size::i64Bit, TMP1, static_cast(Op->RCON) << 32);\n dup(ARMEmitter::SubRegSize::i64Bit, VTMP2.Q(), TMP1);\n eor(Dst.Q(), Dst.Q(), VTMP2.Q());\n } else {\n tbl(Dst.Q(), Dst.Q(), Swizzle.Q());\n }\n}\n\nDEF_OP(CRC32) {\n auto Op = IROp->C();\n\n const auto Dst = GetReg(Node);\n const auto Src1 = GetReg(Op->Src1);\n const auto Src2 = GetReg(Op->Src2);\n\n switch (Op->SrcSize) {\n case IR::OpSize::i8Bit: crc32cb(Dst.W(), Src1.W(), Src2.W()); break;\n case IR::OpSize::i16Bit: crc32ch(Dst.W(), Src1.W(), Src2.W()); break;\n case IR::OpSize::i32Bit: crc32cw(Dst.W(), Src1.W(), Src2.W()); break;\n case IR::OpSize::i64Bit: crc32cx(Dst.X(), Src1.X(), Src2.X()); break;\n default: LOGMAN_MSG_A_FMT(\"Unknown CRC32 size: {}\", Op->SrcSize);\n }\n}\n\nDEF_OP(VSha1H) {\n auto Op = IROp->C();\n\n const auto Dst = GetVReg(Node);\n const auto Src = GetVReg(Op->Src);\n\n sha1h(Dst.S(), Src.S());\n}\n\nDEF_OP(VSha1C) {\n auto Op = IROp->C();\n\n const auto Dst = GetVReg(Node);\n const auto Src1 = GetVReg(Op->Src1);\n const auto Src2 = GetVReg(Op->Src2);\n const auto Src3 = GetVReg(Op->Src3);\n\n if (Dst == Src1) {\n sha1c(Dst, Src2.S(), Src3);\n } else if (Dst != Src2 && Dst != Src3) {\n mov(Dst.Q(), Src1.Q());\n sha1c(Dst, Src2.S(), Src3);\n } else {\n mov(VTMP1.Q(), Src1.Q());\n sha1c(VTMP1, Src2.S(), Src3);\n mov(Dst.Q(), VTMP1.Q());\n }\n}\n\nDEF_OP(VSha1M) {\n auto Op = IROp->C();\n\n const auto Dst = GetVReg(Node);\n const auto Src1 = GetVReg(Op->Src1);\n const auto Src2 = GetVReg(Op->Src2);\n const auto Src3 = GetVReg(Op->Src3);\n\n if (Dst == Src1) {\n sha1m(Dst, Src2.S(), Src3);\n } else if (Dst != Src2 && Dst != Src3) {\n mov(Dst.Q(), Src1.Q());\n sha1m(Dst, Src2.S(), Src3);\n } else {\n mov(VTMP1.Q(), Src1.Q());\n sha1m(VTMP1, Src2.S(), Src3);\n mov(Dst.Q(), VTMP1.Q());\n }\n}\n\nDEF_OP(VSha1P) {\n auto Op = IROp->C();\n\n const auto Dst = GetVReg(Node);\n const auto Src1 = GetVReg(Op->Src1);\n const auto Src2 = GetVReg(Op->Src2);\n const auto Src3 = GetVReg(Op->Src3);\n\n if (Dst == Src1) {\n sha1p(Dst, Src2.S(), Src3);\n } else if (Dst != Src2 && Dst != Src3) {\n mov(Dst.Q(), Src1.Q());\n sha1p(Dst, Src2.S(), Src3);\n } else {\n mov(VTMP1.Q(), Src1.Q());\n sha1p(VTMP1, Src2.S(), Src3);\n mov(Dst.Q(), VTMP1.Q());\n }\n}\n\nDEF_OP(VSha1SU1) {\n auto Op = IROp->C();\n\n const auto Dst = GetVReg(Node);\n const auto Src1 = GetVReg(Op->Src1);\n const auto Src2 = GetVReg(Op->Src2);\n\n if (Dst == Src1) {\n sha1su1(Dst, Src2);\n } else if (Dst != Src2) {\n mov(Dst.Q(), Src1.Q());\n sha1su1(Dst, Src2);\n } else {\n mov(VTMP1.Q(), Src1.Q());\n sha1su1(VTMP1, Src2);\n mov(Dst.Q(), VTMP1.Q());\n }\n}\n\nDEF_OP(VSha256H) {\n auto Op = IROp->C();\n\n const auto Dst = GetVReg(Node);\n const auto Src1 = GetVReg(Op->Src1);\n const auto Src2 = GetVReg(Op->Src2);\n const auto Src3 = GetVReg(Op->Src3);\n\n if (Dst == Src1) {\n sha256h(Dst, Src2, Src3);\n } else if (Dst != Src2 && Dst != Src3) {\n mov(Dst.Q(), Src1.Q());\n sha256h(Dst, Src2, Src3);\n } else {\n mov(VTMP1.Q(), Src1.Q());\n sha256h(VTMP1, Src2, Src3);\n mov(Dst.Q(), VTMP1.Q());\n }\n}\n\nDEF_OP(VSha256H2) {\n auto Op = IROp->C();\n\n const auto Dst = GetVReg(Node);\n const auto Src1 = GetVReg(Op->Src1);\n const auto Src2 = GetVReg(Op->Src2);\n const auto Src3 = GetVReg(Op->Src3);\n\n if (Dst == Src1) {\n sha256h2(Dst, Src2, Src3);\n } else if (Dst != Src2 && Dst != Src3) {\n mov(Dst.Q(), Src1.Q());\n sha256h2(Dst, Src2, Src3);\n } else {\n mov(VTMP1.Q(), Src1.Q());\n sha256h2(VTMP1, Src2, Src3);\n mov(Dst.Q(), VTMP1.Q());\n }\n}\n\nDEF_OP(VSha256U0) {\n auto Op = IROp->C();\n\n const auto Dst = GetVReg(Node);\n const auto Src1 = GetVReg(Op->Src1);\n const auto Src2 = GetVReg(Op->Src2);\n\n if (Dst == Src1) {\n sha256su0(Dst, Src2);\n } else {\n mov(VTMP1.Q(), Src1.Q());\n sha256su0(VTMP1, Src2);\n mov(Dst.Q(), VTMP1.Q());\n }\n}\n\nDEF_OP(VSha256U1) {\n auto Op = IROp->C();\n\n const auto Dst = GetVReg(Node);\n const auto Src1 = GetVReg(Op->Src1);\n const auto Src2 = GetVReg(Op->Src2);\n\n if (Dst != Src1 && Dst != Src2) {\n movi(ARMEmitter::SubRegSize::i64Bit, Dst.Q(), 0);\n sha256su1(Dst, Src1, Src2);\n } else {\n movi(ARMEmitter::SubRegSize::i64Bit, VTMP1.Q(), 0);\n sha256su1(VTMP1, Src1, Src2);\n mov(Dst.Q(), VTMP1.Q());\n }\n}\n\nDEF_OP(PCLMUL) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto Dst = GetVReg(Node);\n const auto Src1 = GetVReg(Op->Src1);\n const auto Src2 = GetVReg(Op->Src2);\n\n LOGMAN_THROW_A_FMT(OpSize == IR::OpSize::i128Bit, \"Currently only supports 128-bit operations.\");\n\n switch (Op->Selector) {\n case 0b00000000: pmull(ARMEmitter::SubRegSize::i128Bit, Dst.D(), Src1.D(), Src2.D()); break;\n case 0b00000001:\n dup(ARMEmitter::SubRegSize::i64Bit, VTMP1.Q(), Src1.Q(), 1);\n pmull(ARMEmitter::SubRegSize::i128Bit, Dst.D(), VTMP1.D(), Src2.D());\n break;\n case 0b00010000:\n dup(ARMEmitter::SubRegSize::i64Bit, VTMP1.Q(), Src2.Q(), 1);\n pmull(ARMEmitter::SubRegSize::i128Bit, Dst.D(), VTMP1.D(), Src1.D());\n break;\n case 0b00010001: pmull2(ARMEmitter::SubRegSize::i128Bit, Dst.Q(), Src1.Q(), Src2.Q()); break;\n default: LOGMAN_MSG_A_FMT(\"Unknown PCLMUL selector: {}\", Op->Selector); break;\n }\n}\n\n} // namespace FEXCore::CPU\n" }, { "path": "FEXCore/Source/Interface/Core/JIT/JIT.cpp", "content": "// SPDX-License-Identifier: MIT\n/*\n$info$\nglossary: Splatter ~ a code generator backend that concatenates configurable macros instead of doing isel\nglossary: IR ~ Intermediate Representation, our high-level opcode representation, loosely modeling arm64\nglossary: SSA ~ Single Static Assignment, a form of representing IR in memory\nglossary: Basic Block ~ A block of instructions with no control flow, terminated by control flow\nglossary: Fragment ~ A Collection of basic blocks, possibly an entire guest function or a subset of it\ntags: backend|arm64\ndesc: Main glue logic of the arm64 splatter backend\n$end_info$\n*/\n\n#include \"Interface/Context/Context.h\"\n#include \"Interface/Core/LookupCache.h\"\n#include \"Interface/Core/Dispatcher/Dispatcher.h\"\n#include \"Interface/Core/Interpreter/InterpreterOps.h\"\n#include \"Interface/Core/JIT/DebugData.h\"\n#include \"Interface/Core/JIT/JITClass.h\"\n#include \"Interface/IR/Passes/RegisterAllocationPass.h\"\n\n#include \"Utils/MemberFunctionToPointer.h\"\n#include \"Utils/variable_length_integer.h\"\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n#include \n#include \n#include \n\nnamespace {\nstruct DivRem {\n uint64_t Quotient;\n uint64_t Remainder;\n};\n\nstatic struct DivRem LUDIV(uint64_t SrcHigh, uint64_t SrcLow, uint64_t Divisor) {\n __uint128_t Source = (static_cast<__uint128_t>(SrcHigh) << 64) | SrcLow;\n\n return {\n .Quotient = (uint64_t)(Source / Divisor),\n .Remainder = (uint64_t)(Source % Divisor),\n };\n}\n\nstatic struct DivRem\nLDIV(uint64_t SrcHigh, uint64_t SrcLow, int64_t Divisor) {\n __int128_t Source = (static_cast<__uint128_t>(SrcHigh) << 64) | SrcLow;\n\n return {\n .Quotient = (uint64_t)(Source / Divisor),\n .Remainder = (uint64_t)(Source % Divisor),\n };\n}\n\nstatic void\nPrintValue(uint64_t Value) {\n LogMan::Msg::DFmt(\"Value: 0x{:x}\", Value);\n}\n\nstatic void PrintVectorValue(uint64_t Value, uint64_t ValueUpper) {\n LogMan::Msg::DFmt(\"Value: 0x{:016x}'{:016x}\", ValueUpper, Value);\n}\n} // namespace\n\nnamespace FEXCore::CPU {\n\nvoid Arm64JITCore::Op_Unhandled(const IR::IROp_Header* IROp, IR::Ref Node) {\n FallbackInfo Info;\n if (!InterpreterOps::GetFallbackHandler(IROp, &Info)) {\n#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED\n LOGMAN_MSG_A_FMT(\"Unhandled IR Op: {}\", FEXCore::IR::GetName(IROp->Op));\n#endif\n } else {\n auto FillF80x2Result = [&](auto DstLo, auto DstHi) {\n mov(DstLo.Q(), VTMP1.Q());\n mov(DstHi.Q(), VTMP2.Q());\n };\n\n auto FillF64x2Result = [&](auto DstLo, auto DstHi) {\n fmov(DstLo.D(), VTMP1.D());\n fmov(DstHi.D(), VTMP2.D());\n };\n\n auto FillF80Result = [&]() {\n const auto Dst = GetVReg(Node);\n mov(Dst.Q(), VTMP1.Q());\n };\n\n auto FillF64Result = [&]() {\n const auto Dst = GetVReg(Node);\n fmov(Dst.D(), VTMP1.D());\n };\n\n auto FillF32Result = [&]() {\n const auto Dst = GetVReg(Node);\n fmov(Dst.S(), VTMP1.S());\n };\n\n auto FillI64Result = [&]() {\n const auto Dst = GetReg(Node);\n mov(Dst.X(), TMP1);\n };\n\n auto FillI32Result = [&]() {\n const auto Dst = GetReg(Node);\n mov(Dst.W(), TMP1.W());\n };\n\n auto FillI16Result = [&]() {\n const auto Dst = GetReg(Node);\n mov(Dst.W(), TMP1.W());\n };\n\n switch (Info.ABI) {\n case FABI_F80_I16_F32_PTR: {\n // Linux Reg/Win32 Reg:\n // tmp4 (x4/x13): FallbackHandler\n // x30: return\n // vtmp1 (v0/v16): source\n str(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, -16);\n\n const auto Src1 = GetVReg(IROp->Args[0]);\n fmov(VTMP1.S(), Src1.S());\n\n ldr(TMP1, FALLBACK_HANDLER_OFFSET(Info.HandlerIndex, ABIHandler));\n ldr(TMP4, FALLBACK_HANDLER_OFFSET(Info.HandlerIndex, Func));\n blr(TMP1);\n\n ldr(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, 16);\n FillF80Result();\n } break;\n\n case FABI_F80_I16_F64_PTR: {\n // Linux Reg/Win32 Reg:\n // tmp4 (x4/x13): FallbackHandler\n // x30: return\n // vtmp1 (v0/v16): source\n str(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, -16);\n\n const auto Src1 = GetVReg(IROp->Args[0]);\n fmov(VTMP1.D(), Src1.D());\n\n ldr(TMP1, FALLBACK_HANDLER_OFFSET(Info.HandlerIndex, ABIHandler));\n ldr(TMP4, FALLBACK_HANDLER_OFFSET(Info.HandlerIndex, Func));\n blr(TMP1);\n\n ldr(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, 16);\n FillF80Result();\n } break;\n\n case FABI_F80_I16_I16_PTR:\n case FABI_F80_I16_I32_PTR: {\n // Linux Reg/Win32 Reg:\n // tmp4 (x4/x13): FallbackHandler\n // x30: return\n // tmp2 (x1/x11): source\n str(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, -16);\n\n const auto Src1 = GetReg(IROp->Args[0]);\n\n // Need to sign or zero extend this for the dispatcher handler.\n if (Info.ABI == FABI_F80_I16_I16_PTR) {\n sxth(ARMEmitter::Size::i32Bit, TMP2, Src1);\n } else {\n mov(ARMEmitter::Size::i32Bit, TMP2, Src1);\n }\n\n ldr(TMP1, FALLBACK_HANDLER_OFFSET(Info.HandlerIndex, ABIHandler));\n ldr(TMP4, FALLBACK_HANDLER_OFFSET(Info.HandlerIndex, Func));\n blr(TMP1);\n\n ldr(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, 16);\n FillF80Result();\n } break;\n\n case FABI_F32_I16_F80_PTR: {\n // Linux Reg/Win32 Reg:\n // tmp4 (x4/x13): FallbackHandler\n // x30: return\n // vtmp1 (v0/v16): source\n str(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, -16);\n\n const auto Src1 = GetVReg(IROp->Args[0]);\n mov(VTMP1.Q(), Src1.Q());\n\n ldr(TMP1, FALLBACK_HANDLER_OFFSET(Info.HandlerIndex, ABIHandler));\n ldr(TMP4, FALLBACK_HANDLER_OFFSET(Info.HandlerIndex, Func));\n blr(TMP1);\n\n ldr(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, 16);\n FillF32Result();\n } break;\n\n case FABI_F64_I16_F80_PTR: {\n // Linux Reg/Win32 Reg:\n // tmp4 (x4/x13): FallbackHandler\n // x30: return\n // vtmp1 (v0/v16): source\n str(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, -16);\n\n const auto Src1 = GetVReg(IROp->Args[0]);\n mov(VTMP1.Q(), Src1.Q());\n\n ldr(TMP1, FALLBACK_HANDLER_OFFSET(Info.HandlerIndex, ABIHandler));\n ldr(TMP4, FALLBACK_HANDLER_OFFSET(Info.HandlerIndex, Func));\n blr(TMP1);\n\n ldr(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, 16);\n FillF64Result();\n } break;\n\n case FABI_F64_F64_PTR: {\n // Linux Reg/Win32 Reg:\n // tmp4 (x4/x13): FallbackHandler\n // x30: return\n // vtmp1 (v0/v16): vector source\n str(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, -16);\n\n const auto Src1 = GetVReg(IROp->Args[0]);\n fmov(VTMP1.D(), Src1.D());\n\n ldr(TMP1, FALLBACK_HANDLER_OFFSET(Info.HandlerIndex, ABIHandler));\n ldr(TMP4, FALLBACK_HANDLER_OFFSET(Info.HandlerIndex, Func));\n blr(TMP1);\n\n ldr(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, 16);\n FillF64Result();\n } break;\n case FABI_F64x2_F64_PTR: {\n // Linux Reg/Win32 Reg:\n // tmp4 (x4/x13): FallbackHandler\n // x30: return\n // vtmp1 (v0/v16): vector source\n // vtmp2 (v1/v16): vector source\n#ifdef VIXL_SIMULATOR\n LOGMAN_THROW_A_FMT(CTX->Config.DisableVixlIndirectCalls, \"Vector register pairs unsupported by simulator currently\");\n#endif\n str(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, -16);\n\n const auto Src1 = GetVReg(IROp->Args[0]);\n const auto DstLo = GetVReg(IROp->Args[1]);\n const auto DstHi = GetVReg(IROp->Args[2]);\n\n fmov(VTMP1.D(), Src1.D());\n\n ldr(TMP1, FALLBACK_HANDLER_OFFSET(Info.HandlerIndex, ABIHandler));\n ldr(TMP4, FALLBACK_HANDLER_OFFSET(Info.HandlerIndex, Func));\n blr(TMP1);\n\n ldr(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, 16);\n FillF64x2Result(DstLo, DstHi);\n } break;\n\n case FABI_F64_F64_F64_PTR: {\n // Linux Reg/Win32 Reg:\n // tmp4 (x4/x13): FallbackHandler\n // x30: return\n // vtmp1 (v0/v16): vector source 1\n // vtmp2 (v1/v17): vector source 2\n str(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, -16);\n\n const auto Src1 = GetVReg(IROp->Args[0]);\n const auto Src2 = GetVReg(IROp->Args[1]);\n\n fmov(VTMP1.D(), Src1.D());\n fmov(VTMP2.D(), Src2.D());\n\n ldr(TMP1, FALLBACK_HANDLER_OFFSET(Info.HandlerIndex, ABIHandler));\n ldr(TMP4, FALLBACK_HANDLER_OFFSET(Info.HandlerIndex, Func));\n blr(TMP1);\n\n ldr(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, 16);\n FillF64Result();\n } break;\n\n case FABI_I16_I16_F80_PTR: {\n // Linux Reg/Win32 Reg:\n // tmp4 (x4/x13): FallbackHandler\n // x30: return\n // vtmp1 (v0/v16): source\n str(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, -16);\n\n const auto Src1 = GetVReg(IROp->Args[0]);\n mov(VTMP1.Q(), Src1.Q());\n\n ldr(TMP1, FALLBACK_HANDLER_OFFSET(Info.HandlerIndex, ABIHandler));\n ldr(TMP4, FALLBACK_HANDLER_OFFSET(Info.HandlerIndex, Func));\n blr(TMP1);\n\n ldr(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, 16);\n FillI16Result();\n } break;\n\n case FABI_I32_I16_F80_PTR: {\n // Linux Reg/Win32 Reg:\n // tmp4 (x4/x13): FallbackHandler\n // x30: return\n // vtmp1 (v0/v16): source\n str(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, -16);\n\n const auto Src1 = GetVReg(IROp->Args[0]);\n mov(VTMP1.Q(), Src1.Q());\n\n ldr(TMP1, FALLBACK_HANDLER_OFFSET(Info.HandlerIndex, ABIHandler));\n ldr(TMP4, FALLBACK_HANDLER_OFFSET(Info.HandlerIndex, Func));\n blr(TMP1);\n\n ldr(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, 16);\n FillI32Result();\n } break;\n\n case FABI_I64_I16_F80_PTR: {\n // Linux Reg/Win32 Reg:\n // tmp4 (x4/x13): FallbackHandler\n // x30: return\n // vtmp1 (v0/v16): source\n str(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, -16);\n\n const auto Src1 = GetVReg(IROp->Args[0]);\n mov(VTMP1.Q(), Src1.Q());\n\n ldr(TMP1, FALLBACK_HANDLER_OFFSET(Info.HandlerIndex, ABIHandler));\n ldr(TMP4, FALLBACK_HANDLER_OFFSET(Info.HandlerIndex, Func));\n blr(TMP1);\n\n ldr(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, 16);\n FillI64Result();\n } break;\n\n case FABI_I64_I16_F80_F80_PTR: {\n // Linux Reg/Win32 Reg:\n // tmp4 (x4/x13): FallbackHandler\n // x30: return\n // vtmp1 (v0/v16): vector source 1\n // vtmp2 (v1/v17): vector source 2\n str(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, -16);\n\n const auto Src1 = GetVReg(IROp->Args[0]);\n const auto Src2 = GetVReg(IROp->Args[1]);\n mov(VTMP1.Q(), Src1.Q());\n mov(VTMP2.Q(), Src2.Q());\n\n ldr(TMP1, FALLBACK_HANDLER_OFFSET(Info.HandlerIndex, ABIHandler));\n ldr(TMP4, FALLBACK_HANDLER_OFFSET(Info.HandlerIndex, Func));\n blr(TMP1);\n\n ldr(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, 16);\n FillI64Result();\n } break;\n\n case FABI_F80_I16_F80_PTR: {\n // Linux Reg/Win32 Reg:\n // tmp4 (x4/x13): FallbackHandler\n // x30: return\n // vtmp1 (v0/v16): vector source 1\n str(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, -16);\n\n const auto Src1 = GetVReg(IROp->Args[0]);\n mov(VTMP1.Q(), Src1.Q());\n\n ldr(TMP1, FALLBACK_HANDLER_OFFSET(Info.HandlerIndex, ABIHandler));\n ldr(TMP4, FALLBACK_HANDLER_OFFSET(Info.HandlerIndex, Func));\n blr(TMP1);\n\n ldr(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, 16);\n FillF80Result();\n } break;\n\n case FABI_F80x2_I16_F80_PTR: {\n // Linux Reg/Win32 Reg:\n // tmp4 (x4/x13): FallbackHandler\n // x30: return\n // vtmp1 (v0/v16): vector source 1\n // vtmp2 (v1/v16): vector source 2\n#ifdef VIXL_SIMULATOR\n LOGMAN_THROW_A_FMT(CTX->Config.DisableVixlIndirectCalls, \"Vector register pairs unsupported by simulator currently\");\n#endif\n str(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, -16);\n\n const auto Src1 = GetVReg(IROp->Args[0]);\n const auto DstLo = GetVReg(IROp->Args[1]);\n const auto DstHi = GetVReg(IROp->Args[2]);\n\n mov(VTMP1.Q(), Src1.Q());\n\n ldr(TMP1, FALLBACK_HANDLER_OFFSET(Info.HandlerIndex, ABIHandler));\n ldr(TMP4, FALLBACK_HANDLER_OFFSET(Info.HandlerIndex, Func));\n blr(TMP1);\n\n ldr(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, 16);\n FillF80x2Result(DstLo, DstHi);\n } break;\n\n case FABI_F80_I16_F80_F80_PTR: {\n // Linux Reg/Win32 Reg:\n // tmp4 (x4/x13): FallbackHandler\n // x30: return\n // vtmp1 (v0/v16): vector source 1\n // vtmp2 (v1/v17): vector source 2\n str(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, -16);\n\n const auto Src1 = GetVReg(IROp->Args[0]);\n const auto Src2 = GetVReg(IROp->Args[1]);\n\n mov(VTMP1.Q(), Src1.Q());\n mov(VTMP2.Q(), Src2.Q());\n\n ldr(TMP1, FALLBACK_HANDLER_OFFSET(Info.HandlerIndex, ABIHandler));\n ldr(TMP4, FALLBACK_HANDLER_OFFSET(Info.HandlerIndex, Func));\n blr(TMP1);\n\n ldr(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, 16);\n FillF80Result();\n } break;\n\n case FABI_I32_I64_I64_V128_V128_I16: {\n // Linux Reg/Win32 Reg:\n // stack: FallbackHandler\n // x30: return\n // vtmp1 (v0/v16): vector source 1\n // vtmp2 (v1/v17): vector source 2\n // tmp1 (x0/x10): source 1\n // tmp2 (x1/x11): source 2\n // tmp3 (x2/x12): source 3\n const auto Op = IROp->C();\n ldr(TMP4, FALLBACK_HANDLER_OFFSET(Info.HandlerIndex, ABIHandler));\n ldr(TMP1, FALLBACK_HANDLER_OFFSET(Info.HandlerIndex, Func));\n\n stp(TMP1, ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, -16);\n\n const auto SrcRAX = GetReg(Op->RAX);\n const auto SrcRDX = GetReg(Op->RDX);\n const auto Control = Op->Control;\n\n mov(TMP1, SrcRAX.X());\n mov(TMP2, SrcRDX.X());\n movz(ARMEmitter::Size::i32Bit, TMP3, Control);\n\n const auto Src1 = GetVReg(Op->LHS);\n const auto Src2 = GetVReg(Op->RHS);\n\n mov(VTMP1.Q(), Src1.Q());\n mov(VTMP2.Q(), Src2.Q());\n\n blr(TMP4);\n\n ldp(ARMEmitter::XReg::zr, ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, 16);\n FillI32Result();\n } break;\n case FABI_I32_V128_V128_I16: {\n // Linux Reg/Win32 Reg:\n // tmp4 (x4/x13): FallbackHandler\n // x30: return\n // vtmp1 (v0/v16): vector source 1\n // vtmp2 (v1/v17): vector source 2\n // tmp1 (x0/x10): source 1\n const auto Op = IROp->C();\n str(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, -16);\n\n const auto Src1 = GetVReg(Op->LHS);\n const auto Src2 = GetVReg(Op->RHS);\n const auto Control = Op->Control;\n\n mov(VTMP1.Q(), Src1.Q());\n mov(VTMP2.Q(), Src2.Q());\n movz(ARMEmitter::Size::i32Bit, TMP1, Control);\n\n ldr(TMP2, FALLBACK_HANDLER_OFFSET(Info.HandlerIndex, ABIHandler));\n ldr(TMP4, FALLBACK_HANDLER_OFFSET(Info.HandlerIndex, Func));\n blr(TMP2);\n\n ldr(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, 16);\n FillI32Result();\n } break;\n case FABI_UNKNOWN:\n default:\n#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED\n LOGMAN_MSG_A_FMT(\"Unhandled IR Fallback ABI: {} {}\", FEXCore::IR::GetName(IROp->Op), ToUnderlying(Info.ABI));\n#endif\n break;\n }\n }\n}\n\nstatic void DirectBlockDelinker(FEXCore::Context::ExitFunctionLinkData* Record, bool Call) {\n uintptr_t JumpThunkStartAddress = reinterpret_cast(Record) - 0x10;\n uintptr_t CallerAddress = JumpThunkStartAddress + Record->CallerOffset;\n auto BranchOffset = JumpThunkStartAddress / 4 - CallerAddress / 4;\n\n // Replace the patched callsite with a branch to the jump thunk.\n uint32_t BranchInst = 0;\n ARMEmitter::Emitter BranchEmit(reinterpret_cast(&BranchInst), 4);\n if (Call) {\n BranchEmit.bl(BranchOffset);\n } else {\n BranchEmit.b(BranchOffset);\n }\n\n std::atomic_ref(*reinterpret_cast(CallerAddress)).store(BranchInst, std::memory_order::relaxed);\n ARMEmitter::Emitter::ClearICache(reinterpret_cast(CallerAddress), 4);\n}\n\nstatic void IndirectBlockDelinker(FEXCore::Context::ExitFunctionLinkData* Record) {\n uintptr_t JumpThunkStartAddress = reinterpret_cast(Record) - 0x10;\n uint32_t BranchInst = 0;\n ARMEmitter::Emitter BranchEmit(reinterpret_cast(&BranchInst), 4);\n // Restore branch +2 instructions to jump to the linker block\n BranchEmit.b(0x2);\n\n std::atomic_ref(*reinterpret_cast(JumpThunkStartAddress)).store(BranchInst, std::memory_order::relaxed);\n ARMEmitter::Emitter::ClearICache(reinterpret_cast(JumpThunkStartAddress), 4);\n\n // No need to reset HostCode here as the exit linker pointer is stored separately, and if the block is relinked it will be updated.\n}\n\nuint64_t Arm64JITCore::ExitFunctionLink(FEXCore::Core::CpuStateFrame* Frame, FEXCore::Context::ExitFunctionLinkData* Record) {\n auto Thread = Frame->Thread;\n bool TFSet = Thread->CurrentFrame->State.flags[X86State::RFLAG_TF_RAW_LOC];\n uintptr_t HostCode {};\n auto GuestRip = Record->GuestRIP;\n\n if (TFSet) {\n // If TF is set, the cache must be skipped as different code needs to be generated.\n Frame->State.rip = GuestRip;\n return Frame->Pointers.DispatcherLoopTop;\n } else {\n {\n // Guard the LookupCache lock with the code invalidation mutex, to avoid issues with forking\n auto lk_inval =\n GuardSignalDeferringSection(static_cast(Thread->CTX)->CodeInvalidationMutex, Thread);\n HostCode = Thread->LookupCache->FindBlock(Thread, GuestRip);\n }\n if (!HostCode) {\n // Hold a reference to the code buffer, to avoid linking unmapped code if compilation triggers a recreation.\n auto CodeBuffer = static_cast(Thread->CPUBackend.get())->CurrentCodeBuffer;\n HostCode = static_cast(Thread->CTX)->CompileBlock(Frame, GuestRip, 0);\n if (Thread->LookupCache->Shared != CodeBuffer->LookupCache.get()) {\n return HostCode;\n }\n }\n }\n\n // See ExitFunction in BranchOps.cpp for an assembly level view of the handled cases.\n uintptr_t JumpThunkStartAddress = reinterpret_cast(Record) - 0x10;\n uintptr_t CallerAddress = JumpThunkStartAddress + Record->CallerOffset;\n auto BranchOffset = HostCode / 4 - CallerAddress / 4;\n\n uint32_t ExpectedKnownCallMarkerInst = 0;\n ARMEmitter::Emitter ExpectedKnownCallMarkerEmit(reinterpret_cast(&ExpectedKnownCallMarkerInst), 4);\n ExpectedKnownCallMarkerEmit.adr(TMP1, 0xC);\n\n // Guard the LookupCache lock with the code invalidation mutex, to avoid issues with forking\n auto lk_inval = GuardSignalDeferringSection(static_cast(Thread->CTX)->CodeInvalidationMutex, Thread);\n\n // Lock here is necessary to prevent simultaneous linking and delinking\n auto lk = Thread->LookupCache->AcquireWriteLock();\n\n // For non-calls, this would extend into the block's code, however that's fine as an out-of-range adr would never\n // be generated avoiding any false positives.\n uintptr_t KnownCallMarkerAddr = CallerAddress - 0x8;\n uint32_t KnownCallMarkerInst = *reinterpret_cast(KnownCallMarkerAddr);\n if (ARMEmitter::Emitter::IsInt26(BranchOffset)) {\n // Directly patch the callsite with the appropriate branch instruction.\n uint32_t BranchInst = 0;\n ARMEmitter::Emitter BranchEmit(reinterpret_cast(&BranchInst), 4);\n\n if (KnownCallMarkerInst == ExpectedKnownCallMarkerInst) {\n BranchEmit.bl(BranchOffset);\n Thread->LookupCache->AddBlockLink(\n GuestRip, Record, [](FEXCore::Context::ExitFunctionLinkData* Record) { DirectBlockDelinker(Record, true); }, lk);\n } else {\n BranchEmit.b(BranchOffset);\n Thread->LookupCache->AddBlockLink(\n GuestRip, Record, [](FEXCore::Context::ExitFunctionLinkData* Record) { DirectBlockDelinker(Record, false); }, lk);\n }\n\n std::atomic_ref(*reinterpret_cast(CallerAddress)).store(BranchInst, std::memory_order::relaxed);\n ARMEmitter::Emitter::ClearICache(reinterpret_cast(CallerAddress), 4);\n } else {\n // This case is common between calls and jumps as the thunk callsite can be left untouched.\n std::atomic_ref(Record->HostCode).store(HostCode, std::memory_order::seq_cst);\n#ifdef ARCHITECTURE_arm64\n // Make memory write visible to other threads reading the same location\n asm volatile(\"dc cvau, %0; dsb ish\" : : \"r\"(Record->HostCode) :);\n#endif\n\n uint32_t LdrInst = 0;\n ARMEmitter::Emitter LdrEmit(reinterpret_cast(&LdrInst), 4);\n LdrEmit.ldr(TMP1, reinterpret_cast(&Record->HostCode) - JumpThunkStartAddress);\n std::atomic_ref(*reinterpret_cast(JumpThunkStartAddress)).store(LdrInst, std::memory_order::relaxed);\n ARMEmitter::Emitter::ClearICache(reinterpret_cast(JumpThunkStartAddress), 4);\n\n Thread->LookupCache->AddBlockLink(GuestRip, Record, IndirectBlockDelinker, lk);\n }\n\n return HostCode;\n}\n\nvoid Arm64JITCore::Op_NoOp(const IR::IROp_Header* IROp, IR::Ref Node) {}\n\nArm64JITCore::Arm64JITCore(FEXCore::Context::ContextImpl* ctx, FEXCore::Core::InternalThreadState* Thread)\n : CPUBackend(*ctx, Thread)\n , Arm64Emitter(ctx)\n , HostSupportsSVE128 {ctx->HostFeatures.SupportsSVE128}\n , HostSupportsSVE256 {ctx->HostFeatures.SupportsSVE256}\n , HostSupportsAVX256 {ctx->HostFeatures.SupportsAVX && ctx->HostFeatures.SupportsSVE256}\n , HostSupportsRPRES {ctx->HostFeatures.SupportsRPRES}\n , HostSupportsAFP {ctx->HostFeatures.SupportsAFP}\n , CTX {ctx}\n , TempAllocator(ctx->CPUBackendAllocator, 0) {\n\n RAPass = Thread->PassManager->GetPass(\"RA\");\n\n RAPass->AddRegisters(IR::RegClass::GPR, GeneralRegisters.size());\n RAPass->AddRegisters(IR::RegClass::GPRFixed, StaticRegisters.size());\n RAPass->AddRegisters(IR::RegClass::FPR, GeneralFPRegisters.size());\n RAPass->AddRegisters(IR::RegClass::FPRFixed, StaticFPRegisters.size());\n RAPass->PairRegs = PairRegisters;\n\n {\n // Set up pointers that the JIT needs to load\n\n // Common\n auto& Ptrs = ThreadState->CurrentFrame->Pointers;\n\n Ptrs.PrintValue = reinterpret_cast(PrintValue);\n Ptrs.PrintVectorValue = reinterpret_cast(PrintVectorValue);\n Ptrs.ThreadRemoveCodeEntryFromJIT = reinterpret_cast(&Context::ContextImpl::ThreadRemoveCodeEntryFromJit);\n Ptrs.MonoBackpatcherWrite = reinterpret_cast(&Context::ContextImpl::MonoBackpatcherWrite);\n Ptrs.CPUIDObj = reinterpret_cast(&CTX->CPUID);\n\n {\n FEXCore::Utils::MemberFunctionToPointerCast PMF(&FEXCore::CPUIDEmu::RunFunction);\n Ptrs.CPUIDFunction = PMF.GetConvertedPointer();\n }\n\n {\n FEXCore::Utils::MemberFunctionToPointerCast PMF(&FEXCore::CPUIDEmu::RunXCRFunction);\n Ptrs.XCRFunction = PMF.GetConvertedPointer();\n }\n\n {\n FEXCore::Utils::MemberFunctionToPointerCast PMF(&FEXCore::HLE::SyscallHandler::HandleSyscall);\n Ptrs.SyscallHandlerObj = reinterpret_cast(CTX->SyscallHandler);\n Ptrs.SyscallHandlerFunc = PMF.GetVTableEntry(CTX->SyscallHandler);\n }\n Ptrs.ExitFunctionLink = reinterpret_cast(&Arm64JITCore::ExitFunctionLink);\n Ptrs.LUDIV = reinterpret_cast(LUDIV);\n Ptrs.LDIV = reinterpret_cast(LDIV);\n }\n\n CurrentCodeBuffer = CodeBuffers.GetLatest();\n ThreadState->LookupCache->Shared = CurrentCodeBuffer->LookupCache.get();\n}\n\nvoid Arm64JITCore::EmitDetectionString() {\n const char JITString[] = \"FEXJIT::Arm64JITCore::\";\n EmitString(JITString);\n Align();\n}\n\nvoid Arm64JITCore::ClearCache() {\n // NOTE: Holding on to the reference here is required to ensure validity of the WriteLock mutex\n auto PrevCodeBuffer = CurrentCodeBuffer;\n auto lk = PrevCodeBuffer->LookupCache->AcquireWriteLock();\n\n auto CodeBuffer = GetEmptyCodeBuffer();\n SetBuffer(CodeBuffer->Ptr, CodeBuffer->AllocatedSize);\n EmitDetectionString();\n\n ThreadState->LookupCache->ChangeGuestToHostMapping(*PrevCodeBuffer, *CurrentCodeBuffer->LookupCache, lk);\n}\n\nArm64JITCore::~Arm64JITCore() {}\n\nbool Arm64JITCore::IsInlineConstant(const IR::OrderedNodeWrapper& WNode, uint64_t* Value) const {\n if (WNode.IsImmediate()) {\n return false;\n }\n\n auto OpHeader = IR->GetOp(WNode);\n\n if (OpHeader->Op == IR::IROps::OP_INLINECONSTANT) {\n auto Op = OpHeader->C();\n if (Value) {\n *Value = Op->Constant;\n }\n return true;\n } else {\n return false;\n }\n}\n\nbool Arm64JITCore::IsInlineEntrypointOffset(const IR::OrderedNodeWrapper& WNode, uint64_t* Value) const {\n if (WNode.IsImmediate()) {\n return false;\n }\n\n auto OpHeader = IR->GetOp(WNode);\n\n if (OpHeader->Op == IR::IROps::OP_INLINEENTRYPOINTOFFSET) {\n auto Op = OpHeader->C();\n if (Value) {\n uint64_t Mask = ~0ULL;\n const auto Size = OpHeader->Size;\n if (Size == IR::OpSize::i32Bit) {\n Mask = 0xFFFF'FFFFULL;\n }\n *Value = (Entry + Op->Offset) & Mask;\n }\n return true;\n } else {\n return false;\n }\n}\n\nvoid Arm64JITCore::EmitTFCheck() {\n ARMEmitter::ForwardLabel l_TFUnset;\n ARMEmitter::ForwardLabel l_TFBlocked;\n\n // Note that this needs to be before the below suspend checks, as X86 checks this flag immediately after executing an instruction.\n ldrb(TMP1, STATE_PTR(CpuStateFrame, State.flags[X86State::RFLAG_TF_RAW_LOC]));\n\n (void)cbz(ARMEmitter::Size::i32Bit, TMP1, &l_TFUnset);\n\n // X86 semantically checks TF after executing each instruction, so e.g. setting a context with TF set will execute a single instruction\n // and then raise an exception. However on the FEX side this is simpler to implement by checking at the start of each instruction, handle this by having bit 1 being unset in the flag state indicate that TF is blocked for a single instruction.\n (void)tbz(TMP1, 1, &l_TFBlocked);\n\n // Block TF for a single instruction when the frontend jumps to a new context by unsetting bit 1.\n ldrb(TMP1, STATE_PTR(CpuStateFrame, State.flags[X86State::RFLAG_TF_RAW_LOC]));\n and_(ARMEmitter::Size::i32Bit, TMP1, TMP1, ~(1 << 1));\n strb(TMP1, STATE_PTR(CpuStateFrame, State.flags[X86State::RFLAG_TF_RAW_LOC]));\n\n Core::CpuStateFrame::SynchronousFaultDataStruct State = {\n .FaultToTopAndGeneratedException = 1,\n .Signal = Core::FAULT_SIGTRAP,\n .TrapNo = X86State::X86_TRAPNO_DB,\n .si_code = 2,\n .err_code = 0,\n };\n\n uint64_t Constant {};\n memcpy(&Constant, &State, sizeof(State));\n\n LoadConstant(ARMEmitter::Size::i64Bit, TMP1, Constant);\n str(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, SynchronousFaultData));\n ldr(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.GuestSignal_SIGTRAP));\n br(TMP1);\n\n (void)Bind(&l_TFBlocked);\n // If TF was blocked for this instruction, unblock it for the next.\n LoadConstant(ARMEmitter::Size::i32Bit, TMP1, 0b11);\n strb(TMP1, STATE_PTR(CpuStateFrame, State.flags[X86State::RFLAG_TF_RAW_LOC]));\n (void)Bind(&l_TFUnset);\n}\n\nvoid Arm64JITCore::EmitSuspendInterruptCheck() {\n if (CTX->Config.NeedsPendingInterruptFaultCheck) {\n // Trigger a fault if there are any pending interrupts\n // Used only for suspend on WIN32 at the moment\n strb(ARMEmitter::XReg::zr, STATE,\n offsetof(FEXCore::Core::InternalThreadState, InterruptFaultPage) - offsetof(FEXCore::Core::InternalThreadState, BaseFrameState));\n }\n\n#ifdef ARCHITECTURE_arm64ec\n static constexpr uint16_t SuspendMagic {0xCAFE};\n\n ldr(TMP2.W(), STATE_PTR(CpuStateFrame, SuspendDoorbell));\n ARMEmitter::ForwardLabel l_NoSuspend;\n (void)cbz(ARMEmitter::Size::i32Bit, TMP2, &l_NoSuspend);\n brk(SuspendMagic);\n (void)Bind(&l_NoSuspend);\n#endif\n}\n\nvoid Arm64JITCore::EmitEntryPoint(ARMEmitter::BackwardLabel& HeaderLabel, bool CheckTF) {\n // Get the address of the JITCodeHeader and store in to the core state.\n // Two instruction cost, each 1 cycle.\n adr_OrRestart(TMP1, &HeaderLabel);\n str(TMP1, STATE, offsetof(FEXCore::Core::CPUState, InlineJITBlockHeader));\n\n if (CheckTF) {\n EmitTFCheck();\n }\n\n if (SpillSlots) {\n const auto TotalSpillSlotsSize = SpillSlots * MaxSpillSlotSize;\n\n if (ARMEmitter::IsImmAddSub(TotalSpillSlotsSize)) {\n sub(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::rsp, ARMEmitter::Reg::rsp, TotalSpillSlotsSize);\n } else {\n LoadConstant(ARMEmitter::Size::i64Bit, TMP1, TotalSpillSlotsSize);\n sub(ARMEmitter::Size::i64Bit, ARMEmitter::XReg::rsp, ARMEmitter::XReg::rsp, TMP1, ARMEmitter::ExtendedType::LSL_64, 0);\n }\n }\n\n EmitSuspendInterruptCheck();\n}\n\nCPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry, uint64_t Size, bool SingleInst, const FEXCore::IR::IRListView* IR,\n FEXCore::Core::DebugData* DebugData, bool CheckTF) {\n FEXCORE_PROFILE_SCOPED(\"Arm64::CompileCode\");\n\n const auto PrevNumAllocations = Relocations.size();\n\n this->Entry = Entry;\n this->DebugData = DebugData;\n this->IR = IR;\n RequiresFarARM64Jumps = false;\n SSANodeMultiplier = 24;\n\n // Prepare restart via long jump in case branch encoding fails.\n // This uses UncheckedLongJump since we don't implement std::longjmp in WoA setups\n switch (static_cast(FEXCore::UncheckedLongJump::SetJump(ThreadState->RestartJump))) {\n case RestartOptions::Control::Incoming:\n // Nothing\n break;\n case RestartOptions::Control::EnableFarARM64Jumps: RequiresFarARM64Jumps = true; break;\n case RestartOptions::Control::NeedsLargerJITSpace:\n // Get rid of the claimed buffer immediately, we can't fit in it at all.\n TempAllocator.UnclaimBuffer();\n SSANodeMultiplier *= 2;\n break;\n default: LOGMAN_MSG_A_FMT(\"Unhandled Arm64 restart condition!\");\n }\n\n uint32_t SSACount = IR->GetSSACount();\n JumpTargets.clear();\n CallReturnTargets.clear();\n PendingJumpThunks.clear();\n JumpTargets.resize(IR->GetHeader()->BlockCount, {});\n Relocations.resize(PrevNumAllocations, FEXCore::CPU::Relocation::Default()); // Discard any relocations generated from a previous attempt\n\n CodeData.EntryPoints.clear();\n\n // Fairly excessive buffer range to make sure we don't overflow\n // One page baseline, plus SSANodeMultipler bytes, plus another page for guard page.\n const uint32_t DesiredBufferRange = AlignUp(FEXCore::Utils::FEX_PAGE_SIZE * 2 + SSACount * SSANodeMultiplier, FEXCore::Utils::FEX_PAGE_SIZE);\n\n // JIT output is first written to a temporary buffer and later relocated to the CodeBuffer.\n // This minimizes lock contention of CodeBufferWriteMutex.\n auto TempCodeBufferInfo = TempAllocator.ReownOrClaimBufferWithSize(DesiredBufferRange);\n auto TempCodeBuffer = TempCodeBufferInfo.Ptr;\n const uint32_t UsableBufferRange = TempCodeBufferInfo.Size - FEXCore::Utils::FEX_PAGE_SIZE;\n\n SetBuffer(TempCodeBuffer, UsableBufferRange);\n\n ThreadState->JITGuardPage = reinterpret_cast(TempCodeBuffer) + UsableBufferRange;\n ThreadState->JITGuardOverflowArgument = FEXCore::ToUnderlying(RestartOptions::Control::NeedsLargerJITSpace);\n\n CodeData.BlockBegin = GetCursorAddress();\n\n // Put the code header at the start of the data block.\n ARMEmitter::BackwardLabel JITCodeHeaderLabel {};\n (void)Bind(&JITCodeHeaderLabel);\n JITCodeHeader* CodeHeader = GetCursorAddress();\n CursorIncrement(sizeof(JITCodeHeader));\n\n auto CodeBegin = GetCursorAddress();\n\n // AAPCS64\n // r30 = LR\n // r29 = FP\n // r19..r28 = Callee saved\n // r18 = Platform Register (Matters if we target Windows or iOS)\n // r16..r17 = Inter-procedure scratch\n // r9..r15 = Temp\n // r8 = Indirect Result\n // r0...r7 = Parameter/Results\n //\n // FPRS:\n // v8..v15 = (lower 64bits) Callee saved\n\n // Our allocation:\n // X0 = ThreadState\n // X1 = MemBase\n //\n // X1-X3 = Temp\n // X4-r18 = RA\n\n SpillSlots = IR->SpillSlots();\n\n PendingTargetLabel = nullptr;\n PendingCallReturnTargetLabel = nullptr;\n\n for (auto [BlockNode, BlockHeader] : IR->GetBlocks()) {\n using namespace FEXCore::IR;\n auto BlockIROp = BlockHeader->CW();\n#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED\n LOGMAN_THROW_A_FMT(BlockIROp->Header.Op == IR::OP_CODEBLOCK, \"IR type failed to be a code block\");\n#endif\n\n auto BlockStartHostCode = GetCursorAddress();\n {\n const auto Node = IR->GetID(BlockNode);\n const auto Target = &JumpTargets[BlockIROp->ID];\n\n // if there's a pending branch, and it is not fall-through\n if (PendingTargetLabel && PendingTargetLabel != Target) {\n if (PendingTargetLabel->Backward.Location) {\n EmitSuspendInterruptCheck();\n }\n b_OrRestart(PendingTargetLabel);\n PendingTargetLabel = nullptr;\n }\n\n if (BlockIROp->EntryPoint) {\n uint64_t BlockStartRIP = Entry + BlockIROp->GuestEntryOffset;\n\n const auto IsReturnTarget = CallReturnTargets.try_emplace(Node).first;\n if (PendingTargetLabel) {\n // If there is a fallthrough branch to this block, skip over the entrypoint code.\n b_OrRestart(Target);\n } else if (PendingCallReturnTargetLabel && PendingCallReturnTargetLabel != &IsReturnTarget->second) {\n // If we just emitted a call, but the block we're now emitting is not the return block so don't fallthrough.\n b_OrRestart(PendingCallReturnTargetLabel);\n }\n PendingCallReturnTargetLabel = nullptr;\n\n BindOrRestart(&IsReturnTarget->second);\n CodeData.EntryPoints.emplace(BlockStartRIP, GetCursorAddress());\n DebugData->GuestOpcodes.push_back({BlockIROp->GuestEntryOffset, GetCursorAddress() - CodeData.BlockBegin});\n\n EmitEntryPoint(JITCodeHeaderLabel, CheckTF);\n }\n\n if (PendingCallReturnTargetLabel) {\n // If there is still a pending call return target, then the block we're emitting is not the return block so don't fallthrough.\n b_OrRestart(PendingCallReturnTargetLabel);\n PendingCallReturnTargetLabel = nullptr;\n }\n PendingTargetLabel = nullptr;\n\n BindOrRestart(Target);\n }\n\n for (auto [CodeNode, IROp] : IR->GetCode(BlockNode)) {\n switch (IROp->Op) {\n#define REGISTER_OP(op, x) \\\n case FEXCore::IR::IROps::OP_##op: Op_##x(IROp, CodeNode); break\n\n#define IROP_DISPATCH_DISPATCH\n#include \n#undef REGISTER_OP\n\n default: Op_Unhandled(IROp, CodeNode); break;\n }\n }\n\n DebugData->Subblocks.push_back({static_cast(BlockStartHostCode - CodeData.BlockBegin),\n static_cast(GetCursorAddress() - BlockStartHostCode)});\n }\n\n // Make sure last branch is generated. It certainly can't be eliminated here.\n if (PendingTargetLabel) {\n if (PendingTargetLabel->Backward.Location) {\n EmitSuspendInterruptCheck();\n }\n b_OrRestart(PendingTargetLabel);\n }\n PendingTargetLabel = nullptr;\n\n ARMEmitter::ForwardLabel l_ExitLink;\n for (auto& PendingJumpThunk : PendingJumpThunks) {\n // Align as 64-bit atomics are used on the HostCode field.\n Align(8);\n\n ARMEmitter::ForwardLabel l_DoLink;\n uint64_t ThunkAddress = GetCursorAddress();\n BindOrRestart(&PendingJumpThunk.Label);\n b_OrRestart(&l_DoLink);\n br(TMP1);\n BindOrRestart(&l_DoLink);\n ldr(TMP1, &l_ExitLink);\n blr(TMP1);\n\n // This is a ExitFunctionLinkData struct\n BindOrRestart(&l_ExitLink);\n dc64(0); // HostCode\n PlaceNamedSymbolLiteral(InsertGuestRIPLiteral(PendingJumpThunk.GuestRIP)); // GuestRIP\n dc64(PendingJumpThunk.CallerAddress - ThunkAddress); // CallerOffset\n }\n\n BindOrRestart(&l_ExitLink);\n PlaceNamedSymbolLiteral(InsertNamedSymbolLiteral(RelocNamedSymbolLiteral::NamedSymbol::SYMBOL_LITERAL_EXITFUNCTION_LINKER));\n\n // CodeSize not including the header or tail data.\n const uint64_t CodeOnlySize = GetCursorAddress() - CodeBegin;\n\n // Add the JitCodeTail (written later)\n Align(alignof(JITCodeTail));\n const auto JITBlockTailLocation = GetCursorAddress();\n CodeHeader->OffsetToBlockTail = JITBlockTailLocation - CodeData.BlockBegin;\n\n JITCodeTail JITBlockTail {\n .RIP = Entry,\n .GuestSize = Size,\n .SpinLockFutex = 0,\n .SingleInst = SingleInst,\n };\n\n // Entries that live after the JITCodeTail.\n // These entries correlate JIT code regions with guest RIP regions.\n // Using these entries FEX is able to reconstruct the guest RIP accurately when an instruction cause a signal fault.\n // Packed using two variable length integer entries to ensure the size isn't too large.\n // These smaller sizes means that each entry is relative to each other instead of absolute offset from the start of the JIT block.\n // When reconstructing the RIP, each entry must be walked linearly and accumulated with the previous entries.\n // This is a trade-off between compression inside the JIT code space and execution time when reconstruction the RIP.\n // RIP reconstruction when faulting is less likely so we are requiring the accumulation.\n //\n // struct {\n // // The Host PC offset from the previous entry.\n // FEXCore::Utils::vl64 HostPCOffset;\n // // How much to offset the RIP from the previous entry.\n // FEXCore::Utils::vl64 GuestRIPOffset;\n // };\n\n const auto JITRIPEntriesBegin = JITBlockTailLocation + sizeof(JITBlockTail);\n auto JITRIPEntriesLocation = JITRIPEntriesBegin;\n\n {\n // Store the RIP entries.\n JITBlockTail.NumberOfRIPEntries = DebugData->GuestOpcodes.size();\n JITBlockTail.OffsetToRIPEntries = JITRIPEntriesBegin - JITBlockTailLocation;\n uintptr_t CurrentRIPOffset = 0;\n uint64_t CurrentPCOffset = 0;\n\n for (size_t i = 0; i < DebugData->GuestOpcodes.size(); i++) {\n const auto& GuestOpcode = DebugData->GuestOpcodes[i];\n int64_t HostPCOffset = GuestOpcode.HostEntryOffset - CurrentPCOffset;\n int64_t GuestRIPOffset = GuestOpcode.GuestEntryOffset - CurrentRIPOffset;\n\n JITRIPEntriesLocation += FEXCore::Utils::vl64pair::Encode(JITRIPEntriesLocation, HostPCOffset, GuestRIPOffset);\n\n CurrentPCOffset = GuestOpcode.HostEntryOffset;\n CurrentRIPOffset = GuestOpcode.GuestEntryOffset;\n }\n }\n\n SetCursorOffset(JITRIPEntriesLocation - CodeData.BlockBegin);\n Align();\n\n CodeData.Size = GetCursorAddress() - CodeData.BlockBegin;\n\n // Finalize and write block tail data\n JITBlockTail.Size = CodeData.Size;\n {\n auto PrevCur = GetCursorOffset();\n memcpy(JITBlockTailLocation, &JITBlockTail, sizeof(JITBlockTail));\n SetCursorOffset(JITBlockTailLocation - CodeData.BlockBegin + offsetof(JITCodeTail, RIP));\n PlaceNamedSymbolLiteral(InsertGuestRIPLiteral(JITBlockTail.RIP));\n SetCursorOffset(PrevCur);\n }\n\n // Migrate the compile output from temporary storage to the actual CodeBuffer.\n // This can block progress in other compiling threads, so the duration of the lock should be as small as possible.\n {\n auto CodeBufferLock = std::unique_lock {CodeBuffers.CodeBufferWriteMutex};\n\n // Query size of generated code\n const auto TempSize = GetCursorOffset();\n\n // Bring CodeBuffer up to date\n {\n LOGMAN_THROW_A_FMT(CurrentCodeBuffer->LookupCache.get() == ThreadState->LookupCache->Shared, \"INVARIANT VIOLATED: SharedLookupCache \"\n \"doesn't match up!\\n\");\n if (auto Prev = CheckCodeBufferUpdate()) {\n Allocator::VirtualDontNeed(ThreadState->CallRetStackBase, FEXCore::Core::InternalThreadState::CALLRET_STACK_SIZE);\n auto lk = ThreadState->LookupCache->AcquireWriteLock();\n ThreadState->LookupCache->ChangeGuestToHostMapping(*Prev, *CurrentCodeBuffer->LookupCache, lk);\n }\n\n // NOTE: 16-byte alignment of the new cursor offset must be preserved for block linking records\n SetBuffer(CurrentCodeBuffer->Ptr, CurrentCodeBuffer->AllocatedSize);\n SetCursorOffset(CodeBuffers.LatestOffset);\n Align16B();\n if ((GetCursorOffset() + TempSize) > CurrentCodeBuffer->UsableSize()) {\n CTX->ClearCodeCache(ThreadState);\n }\n\n CodeBuffers.LatestOffset = GetCursorOffset();\n }\n\n // Adjust host addresses\n const auto Delta = GetCursorAddress() - CodeData.BlockBegin;\n CodeData.BlockBegin += Delta;\n for (auto& EntryPoint : CodeData.EntryPoints) {\n EntryPoint.second += Delta;\n }\n CodeBegin += Delta;\n\n for (std::size_t Idx = PrevNumAllocations; Idx != Relocations.size(); ++Idx) {\n Relocations[Idx].Header.Offset += CodeBuffers.LatestOffset;\n }\n\n // Copy over CodeBuffer contents\n memcpy(GetCursorAddress(), TempCodeBuffer, TempSize);\n SetCursorOffset(CodeBuffers.LatestOffset + TempSize);\n\n CodeBuffers.LatestOffset = GetCursorOffset();\n }\n\n TempAllocator.DelayedDisownBuffer();\n\n ClearICache(CodeBegin, CodeOnlySize);\n\n#ifdef VIXL_DISASSEMBLER\n if (Disassemble() & FEXCore::Config::Disassemble::STATS) {\n auto HeaderOp = IR->GetHeader();\n LOGMAN_THROW_A_FMT(HeaderOp->Header.Op == IR::OP_IRHEADER, \"First op wasn't IRHeader\");\n\n LogMan::Msg::IFmt(\"RIP: 0x{:x}\", Entry);\n LogMan::Msg::IFmt(\"Guest Code instructions: {}\", HeaderOp->NumHostInstructions);\n LogMan::Msg::IFmt(\"Host Code instructions: {}\", CodeOnlySize >> 2);\n LogMan::Msg::IFmt(\"Blow-up Amt: {}x\", double(CodeOnlySize >> 2) / double(HeaderOp->NumHostInstructions));\n }\n\n if (Disassemble() & FEXCore::Config::Disassemble::BLOCKS) {\n const auto DisasmBegin = reinterpret_cast(CodeBegin);\n const auto DisasmEnd = reinterpret_cast(CodeBegin + CodeOnlySize);\n LogMan::Msg::IFmt(\"Disassemble Begin\");\n for (auto PCToDecode = DisasmBegin; PCToDecode < DisasmEnd; PCToDecode += 4) {\n DisasmDecoder->Decode(PCToDecode);\n auto Output = Disasm->GetOutput();\n LogMan::Msg::IFmt(\"{}\", Output);\n }\n LogMan::Msg::IFmt(\"Disassemble End\");\n }\n#endif\n\n DebugData->HostCodeSize = CodeData.Size;\n DebugData->Relocations = &Relocations;\n\n this->IR = nullptr;\n\n return std::move(CodeData);\n}\n\nvoid Arm64JITCore::ResetStack() {\n if (SpillSlots == 0) {\n return;\n }\n\n const auto TotalSpillSlotsSize = SpillSlots * MaxSpillSlotSize;\n\n if (ARMEmitter::IsImmAddSub(TotalSpillSlotsSize)) {\n add(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::rsp, ARMEmitter::Reg::rsp, TotalSpillSlotsSize);\n } else {\n // Too big to fit in a 12bit immediate\n LoadConstant(ARMEmitter::Size::i64Bit, TMP1, TotalSpillSlotsSize);\n add(ARMEmitter::Size::i64Bit, ARMEmitter::XReg::rsp, ARMEmitter::XReg::rsp, TMP1, ARMEmitter::ExtendedType::LSL_64, 0);\n }\n}\n\nfextl::unique_ptr CreateArm64JITCore(FEXCore::Context::ContextImpl* ctx, FEXCore::Core::InternalThreadState* Thread) {\n return fextl::make_unique(ctx, Thread);\n}\n\n} // namespace FEXCore::CPU\n" }, { "path": "FEXCore/Source/Interface/Core/JIT/JITClass.h", "content": "// SPDX-License-Identifier: MIT\n/*\n$info$\ntags: backend|arm64\n$end_info$\n*/\n\n#pragma once\n\n#include \"Interface/Core/ArchHelpers/Arm64Emitter.h\"\n#include \"Interface/Core/CPUBackend.h\"\n#include \"Interface/Core/Dispatcher/Dispatcher.h\"\n#include \"Interface/Core/JIT/Relocations.h\"\n#include \"Interface/IR/IR.h\"\n#include \"Interface/IR/IntrusiveIRList.h\"\n#include \"Interface/IR/RegisterAllocationData.h\"\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n#include \n\n#include \n#include \n#include \n#include \n#include \n#include \n\nnamespace FEXCore::Core {\nstruct InternalThreadState;\n}\nnamespace FEXCore::Context {\nstruct ExitFunctionLinkData;\n}\nnamespace FEXCore::IR {\nclass RegisterAllocationPass;\n}\n\nnamespace FEXCore::CPU {\nclass Arm64JITCore final : public CPUBackend, public Arm64Emitter {\npublic:\n explicit Arm64JITCore(FEXCore::Context::ContextImpl* ctx, FEXCore::Core::InternalThreadState* Thread);\n ~Arm64JITCore() override;\n\n [[nodiscard]]\n CPUBackend::CompiledCode CompileCode(uint64_t Entry, uint64_t Size, bool SingleInst, const FEXCore::IR::IRListView* IR,\n FEXCore::Core::DebugData* DebugData, bool CheckTF) override;\n\n void ClearCache() override;\n\n void ClearRelocations() override {\n Relocations.clear();\n }\n\nprivate:\n const bool HostSupportsSVE128 {};\n const bool HostSupportsSVE256 {};\n const bool HostSupportsAVX256 {};\n const bool HostSupportsRPRES {};\n const bool HostSupportsAFP {};\n\n struct RestartOptions {\n enum class Control : uint64_t {\n Incoming = 0,\n EnableFarARM64Jumps = 1,\n NeedsLargerJITSpace = 2,\n };\n };\n\n // FEXCore makes assumptions in the JIT about certain conditions being true.\n // In the rare case when those assumptions are broken, FEX needs to safely restart the JIT.\n RestartOptions RestartControl {};\n bool RequiresFarARM64Jumps {};\n // Default to 6 instructions per SSA node.\n uint32_t SSANodeMultiplier {24};\n\n ARMEmitter::BiDirectionalLabel* PendingTargetLabel {};\n ARMEmitter::BiDirectionalLabel* PendingCallReturnTargetLabel {};\n FEXCore::Context::ContextImpl* CTX {};\n const FEXCore::IR::IRListView* IR {};\n uint64_t Entry {};\n CPUBackend::CompiledCode CodeData {};\n\n fextl::vector JumpTargets;\n\n ARMEmitter::BiDirectionalLabel* JumpTarget(IR::OrderedNodeWrapper Node) {\n auto Block = IR->GetOp(Node);\n return &JumpTargets[Block->ID];\n }\n\n fextl::map CallReturnTargets;\n\n struct PendingJumpThunk {\n uint64_t CallerAddress;\n uint64_t GuestRIP;\n ARMEmitter::ForwardLabel Label;\n };\n fextl::vector PendingJumpThunks;\n\n Utils::PoolBufferWithTimedRetirement TempAllocator;\n\n static uint64_t ExitFunctionLink(FEXCore::Core::CpuStateFrame* Frame, FEXCore::Context::ExitFunctionLinkData* Record);\n\n [[nodiscard]]\n ARMEmitter::Register GetReg(IR::PhysicalRegister Reg) const {\n const auto RegClass = Reg.AsRegClass();\n\n LOGMAN_THROW_A_FMT(RegClass == IR::RegClass::GPRFixed || RegClass == IR::RegClass::GPR, \"Unexpected Class: {}\", Reg.Class);\n\n if (RegClass == IR::RegClass::GPRFixed) {\n return StaticRegisters[Reg.Reg];\n } else if (RegClass == IR::RegClass::GPR) {\n return GeneralRegisters[Reg.Reg];\n }\n\n FEX_UNREACHABLE;\n }\n\n [[nodiscard]]\n ARMEmitter::Register GetReg(IR::Ref Node) const {\n return GetReg(IR::PhysicalRegister(Node));\n }\n\n [[nodiscard]]\n ARMEmitter::Register GetReg(IR::OrderedNodeWrapper Wrap) const {\n return GetReg(IR::PhysicalRegister(Wrap));\n }\n\n [[nodiscard]]\n ARMEmitter::VRegister GetVReg(IR::PhysicalRegister Reg) const {\n const auto RegClass = Reg.AsRegClass();\n\n LOGMAN_THROW_A_FMT(RegClass == IR::RegClass::FPRFixed || RegClass == IR::RegClass::FPR, \"Unexpected Class: {}\", Reg.Class);\n\n if (RegClass == IR::RegClass::FPRFixed) {\n return StaticFPRegisters[Reg.Reg];\n } else if (RegClass == IR::RegClass::FPR) {\n return GeneralFPRegisters[Reg.Reg];\n }\n\n FEX_UNREACHABLE;\n }\n\n [[nodiscard]]\n ARMEmitter::VRegister GetVReg(IR::Ref Node) const {\n return GetVReg(IR::PhysicalRegister(Node));\n }\n\n [[nodiscard]]\n ARMEmitter::VRegister GetVReg(IR::OrderedNodeWrapper Wrap) const {\n return GetVReg(IR::PhysicalRegister(Wrap));\n }\n\n [[nodiscard]]\n static IR::RegClass GetRegClass(IR::Ref Node) {\n return IR::PhysicalRegister(Node).AsRegClass();\n }\n\n [[nodiscard]]\n ARMEmitter::Register GetZeroableReg(IR::OrderedNodeWrapper Src) const {\n uint64_t Const;\n if (IsInlineConstant(Src, &Const)) {\n LOGMAN_THROW_A_FMT(Const == 0, \"Only valid constant\");\n return ARMEmitter::Reg::zr;\n } else {\n return GetReg(Src);\n }\n }\n\n // Converts IR-base shift type to ARMEmitter shift type.\n // Will be a no-op, only a type conversion since the two definitions match.\n [[nodiscard]]\n static ARMEmitter::ShiftType ConvertIRShiftType(IR::ShiftType Shift) {\n return Shift == IR::ShiftType::LSL ? ARMEmitter::ShiftType::LSL :\n Shift == IR::ShiftType::LSR ? ARMEmitter::ShiftType::LSR :\n Shift == IR::ShiftType::ASR ? ARMEmitter::ShiftType::ASR :\n ARMEmitter::ShiftType::ROR;\n }\n\n [[nodiscard]]\n static ARMEmitter::Size ConvertSize(const IR::IROp_Header* Op) {\n return Op->Size == IR::OpSize::i64Bit ? ARMEmitter::Size::i64Bit : ARMEmitter::Size::i32Bit;\n }\n\n [[nodiscard]]\n static ARMEmitter::Size ConvertSize48(const IR::IROp_Header* Op) {\n LOGMAN_THROW_A_FMT(Op->Size == IR::OpSize::i32Bit || Op->Size == IR::OpSize::i64Bit, \"Invalid size\");\n return ConvertSize(Op);\n }\n\n [[nodiscard]]\n static ARMEmitter::Size ConvertSize(IR::OpSize Size) {\n return Size == IR::OpSize::i64Bit ? ARMEmitter::Size::i64Bit : ARMEmitter::Size::i32Bit;\n }\n\n [[nodiscard]]\n static ARMEmitter::SubRegSize ConvertSubRegSize16(IR::OpSize ElementSize) {\n LOGMAN_THROW_A_FMT(ElementSize == IR::OpSize::i8Bit || ElementSize == IR::OpSize::i16Bit || ElementSize == IR::OpSize::i32Bit ||\n ElementSize == IR::OpSize::i64Bit || ElementSize == IR::OpSize::i128Bit,\n \"Invalid size\");\n return ElementSize == IR::OpSize::i8Bit ? ARMEmitter::SubRegSize::i8Bit :\n ElementSize == IR::OpSize::i16Bit ? ARMEmitter::SubRegSize::i16Bit :\n ElementSize == IR::OpSize::i32Bit ? ARMEmitter::SubRegSize::i32Bit :\n ElementSize == IR::OpSize::i64Bit ? ARMEmitter::SubRegSize::i64Bit :\n ARMEmitter::SubRegSize::i128Bit;\n }\n\n [[nodiscard]]\n static ARMEmitter::SubRegSize ConvertSubRegSize16(const IR::IROp_Header* Op) {\n return ConvertSubRegSize16(Op->ElementSize);\n }\n\n [[nodiscard]]\n static ARMEmitter::SubRegSize ConvertSubRegSize8(IR::OpSize ElementSize) {\n LOGMAN_THROW_A_FMT(ElementSize != IR::OpSize::i128Bit, \"Invalid size\");\n return ConvertSubRegSize16(ElementSize);\n }\n\n [[nodiscard]]\n static ARMEmitter::SubRegSize ConvertSubRegSize8(const IR::IROp_Header* Op) {\n return ConvertSubRegSize8(Op->ElementSize);\n }\n\n [[nodiscard]]\n static ARMEmitter::SubRegSize ConvertSubRegSize4(const IR::IROp_Header* Op) {\n LOGMAN_THROW_A_FMT(Op->ElementSize != IR::OpSize::i64Bit, \"Invalid size\");\n return ConvertSubRegSize8(Op);\n }\n\n [[nodiscard]]\n static ARMEmitter::SubRegSize ConvertSubRegSize248(const IR::IROp_Header* Op) {\n LOGMAN_THROW_A_FMT(Op->ElementSize != IR::OpSize::i8Bit, \"Invalid size\");\n return ConvertSubRegSize8(Op);\n }\n\n [[nodiscard]]\n static ARMEmitter::VectorRegSizePair ConvertSubRegSizePair16(const IR::IROp_Header* Op) {\n return ARMEmitter::ToVectorSizePair(ConvertSubRegSize16(Op));\n }\n\n [[nodiscard]]\n static ARMEmitter::VectorRegSizePair ConvertSubRegSizePair8(const IR::IROp_Header* Op) {\n LOGMAN_THROW_A_FMT(Op->ElementSize != IR::OpSize::i128Bit, \"Invalid size\");\n return ConvertSubRegSizePair16(Op);\n }\n\n [[nodiscard]]\n static ARMEmitter::VectorRegSizePair ConvertSubRegSizePair248(const IR::IROp_Header* Op) {\n LOGMAN_THROW_A_FMT(Op->ElementSize != IR::OpSize::i8Bit, \"Invalid size\");\n return ConvertSubRegSizePair8(Op);\n }\n\n [[nodiscard]]\n static ARMEmitter::Condition MapCC(IR::CondClass Cond) {\n switch (Cond) {\n case IR::CondClass::EQ: return ARMEmitter::Condition::CC_EQ;\n case IR::CondClass::NEQ: return ARMEmitter::Condition::CC_NE;\n case IR::CondClass::SGE: return ARMEmitter::Condition::CC_GE;\n case IR::CondClass::SLT: return ARMEmitter::Condition::CC_LT;\n case IR::CondClass::SGT: return ARMEmitter::Condition::CC_GT;\n case IR::CondClass::SLE: return ARMEmitter::Condition::CC_LE;\n case IR::CondClass::UGE: return ARMEmitter::Condition::CC_CS;\n case IR::CondClass::ULT: return ARMEmitter::Condition::CC_CC;\n case IR::CondClass::UGT: return ARMEmitter::Condition::CC_HI;\n case IR::CondClass::ULE: return ARMEmitter::Condition::CC_LS;\n case IR::CondClass::FLU: return ARMEmitter::Condition::CC_LT;\n case IR::CondClass::FGE: return ARMEmitter::Condition::CC_GE;\n case IR::CondClass::FLEU: return ARMEmitter::Condition::CC_LE;\n case IR::CondClass::FGT: return ARMEmitter::Condition::CC_GT;\n case IR::CondClass::FU:\n case IR::CondClass::VS: return ARMEmitter::Condition::CC_VS;\n case IR::CondClass::FNU:\n case IR::CondClass::VC: return ARMEmitter::Condition::CC_VC;\n case IR::CondClass::MI: return ARMEmitter::Condition::CC_MI;\n case IR::CondClass::PL: return ARMEmitter::Condition::CC_PL;\n default: LOGMAN_MSG_A_FMT(\"Unsupported compare type\"); return ARMEmitter::Condition::CC_NV;\n }\n }\n\n [[nodiscard]]\n static bool IsFPR(IR::RegClass Class) {\n return Class == IR::RegClass::FPR || Class == IR::RegClass::FPRFixed;\n }\n\n [[nodiscard]]\n static bool IsGPR(IR::RegClass Class) {\n return Class == IR::RegClass::GPR || Class == IR::RegClass::GPRFixed;\n }\n\n [[nodiscard]]\n static bool IsGPR(IR::Ref Node) {\n return IsGPR(GetRegClass(Node));\n }\n\n [[nodiscard]]\n static bool IsFPR(IR::Ref Node) {\n return IsFPR(GetRegClass(Node));\n }\n\n [[nodiscard]]\n static bool IsGPR(IR::OrderedNodeWrapper Wrap) {\n return IsGPR(IR::PhysicalRegister(Wrap).AsRegClass());\n }\n\n [[nodiscard]]\n static bool IsFPR(IR::OrderedNodeWrapper Wrap) {\n return IsFPR(IR::PhysicalRegister(Wrap).AsRegClass());\n }\n\n [[nodiscard]]\n ARMEmitter::ExtendedMemOperand GenerateMemOperand(IR::OpSize AccessSize, ARMEmitter::Register Base, IR::OrderedNodeWrapper Offset,\n IR::MemOffsetType OffsetType, uint8_t OffsetScale);\n\n [[nodiscard]]\n ARMEmitter::Register ApplyMemOperand(IR::OpSize AccessSize, ARMEmitter::Register Base, ARMEmitter::Register Tmp,\n IR::OrderedNodeWrapper Offset, IR::MemOffsetType OffsetType, uint8_t OffsetScale);\n\n // NOTE: Will use TMP1 as a way to encode immediates that happen to fall outside\n // the limits of the scalar plus immediate variant of SVE load/stores.\n //\n // TMP1 is safe to use again once this memory operand is used with its\n // equivalent loads or stores that this was called for.\n [[nodiscard]]\n ARMEmitter::SVEMemOperand GenerateSVEMemOperand(IR::OpSize AccessSize, ARMEmitter::Register Base, IR::OrderedNodeWrapper Offset,\n IR::MemOffsetType OffsetType, uint8_t OffsetScale);\n\n [[nodiscard]]\n bool IsInlineConstant(const IR::OrderedNodeWrapper& Node, uint64_t* Value = nullptr) const;\n [[nodiscard]]\n bool IsInlineEntrypointOffset(const IR::OrderedNodeWrapper& WNode, uint64_t* Value) const;\n\n struct LiveRange {\n uint32_t Begin;\n uint32_t End;\n };\n\n void EmitLinkedBranch(uint64_t GuestRIP, bool Call) {\n PendingJumpThunks.push_back({GetCursorAddress(), GuestRIP, {}});\n auto& Thunk = PendingJumpThunks.back();\n BindOrRestart(&Thunk.Label);\n if (Call) {\n bl_OrRestart(&Thunk.Label);\n } else {\n b_OrRestart(&Thunk.Label);\n }\n }\n\n // Restart helpers\n template\n void bl_OrRestart(T* Label) {\n if (bl(Label) == ARMEmitter::BranchEncodeSucceeded::Success) {\n return;\n }\n\n // We can support this but currently unnecessary.\n ERROR_AND_DIE_FMT(\"Tried to branch larger than 128MB away!\");\n FEXCore::UncheckedLongJump::LongJump(ThreadState->RestartJump, FEXCore::ToUnderlying(RestartOptions::Control::EnableFarARM64Jumps));\n }\n\n template\n void b_OrRestart(T* Label) {\n if (b(Label) == ARMEmitter::BranchEncodeSucceeded::Success) {\n return;\n }\n\n // We can support this but currently unnecessary.\n ERROR_AND_DIE_FMT(\"Tried to branch larger than 128MB away!\");\n FEXCore::UncheckedLongJump::LongJump(ThreadState->RestartJump, FEXCore::ToUnderlying(RestartOptions::Control::EnableFarARM64Jumps));\n }\n\n template\n void b_OrRestart(ARMEmitter::Condition Cond, T* Label) {\n if (RequiresFarARM64Jumps) {\n ARMEmitter::ForwardLabel Skip {};\n // Wrap a manual Cond check around an unconditional branch; this can encode larger offsets\n (void)b(InvertCondition(Cond), &Skip);\n if (b(Label) == ARMEmitter::BranchEncodeSucceeded::Failure) {\n ERROR_AND_DIE_FMT(\"Tried to branch larger than 128MB away!\");\n }\n\n (void)Bind(&Skip);\n return;\n }\n\n if (b(Cond, Label) == ARMEmitter::BranchEncodeSucceeded::Success) {\n return;\n }\n\n FEXCore::UncheckedLongJump::LongJump(ThreadState->RestartJump, FEXCore::ToUnderlying(RestartOptions::Control::EnableFarARM64Jumps));\n }\n\n template\n void cbz_OrRestart(ARMEmitter::Size s, ARMEmitter::Register rt, T* Label) {\n if (RequiresFarARM64Jumps) {\n ARMEmitter::ForwardLabel Skip {};\n // Wrap a manual Cond check around an unconditional branch; this can encode larger offsets\n (void)cbnz(s, rt, &Skip);\n if (b(Label) == ARMEmitter::BranchEncodeSucceeded::Failure) {\n ERROR_AND_DIE_FMT(\"Tried to branch larger than 128MB away!\");\n }\n\n (void)Bind(&Skip);\n return;\n }\n\n if (cbz(s, rt, Label) == ARMEmitter::BranchEncodeSucceeded::Success) {\n return;\n }\n\n FEXCore::UncheckedLongJump::LongJump(ThreadState->RestartJump, FEXCore::ToUnderlying(RestartOptions::Control::EnableFarARM64Jumps));\n }\n\n template\n void cbnz_OrRestart(ARMEmitter::Size s, ARMEmitter::Register rt, T* Label) {\n if (RequiresFarARM64Jumps) {\n ARMEmitter::ForwardLabel Skip {};\n // Wrap a manual Cond check around an unconditional branch; this can encode larger offsets\n (void)cbz(s, rt, &Skip);\n if (b(Label) == ARMEmitter::BranchEncodeSucceeded::Failure) {\n ERROR_AND_DIE_FMT(\"Tried to branch larger than 128MB away!\");\n }\n\n (void)Bind(&Skip);\n return;\n }\n\n if (cbnz(s, rt, Label) == ARMEmitter::BranchEncodeSucceeded::Success) {\n return;\n }\n\n FEXCore::UncheckedLongJump::LongJump(ThreadState->RestartJump, FEXCore::ToUnderlying(RestartOptions::Control::EnableFarARM64Jumps));\n }\n\n template\n void tbz_OrRestart(ARMEmitter::Register rt, uint32_t Bit, T* Label) {\n if (RequiresFarARM64Jumps) {\n ARMEmitter::ForwardLabel Skip {};\n // Wrap a manual Cond check around an unconditional branch; this can encode larger offsets\n (void)tbnz(rt, Bit, &Skip);\n if (b(Label) == ARMEmitter::BranchEncodeSucceeded::Failure) {\n ERROR_AND_DIE_FMT(\"Tried to branch larger than 128MB away!\");\n }\n\n (void)Bind(&Skip);\n return;\n }\n\n if (tbz(rt, Bit, Label) == ARMEmitter::BranchEncodeSucceeded::Success) {\n return;\n }\n\n FEXCore::UncheckedLongJump::LongJump(ThreadState->RestartJump, FEXCore::ToUnderlying(RestartOptions::Control::EnableFarARM64Jumps));\n }\n\n template\n void tbnz_OrRestart(ARMEmitter::Register rt, uint32_t Bit, T* Label) {\n if (RequiresFarARM64Jumps) {\n ARMEmitter::ForwardLabel Skip {};\n // Wrap a manual Cond check around an unconditional branch; this can encode larger offsets\n (void)tbz(rt, Bit, &Skip);\n if (b(Label) == ARMEmitter::BranchEncodeSucceeded::Failure) {\n ERROR_AND_DIE_FMT(\"Tried to branch larger than 128MB away!\");\n }\n\n (void)Bind(&Skip);\n return;\n }\n\n if (tbnz(rt, Bit, Label) == ARMEmitter::BranchEncodeSucceeded::Success) {\n return;\n }\n\n FEXCore::UncheckedLongJump::LongJump(ThreadState->RestartJump, FEXCore::ToUnderlying(RestartOptions::Control::EnableFarARM64Jumps));\n }\n\n template\n void adr_OrRestart(ARMEmitter::Register rd, T* Label) {\n if (RequiresFarARM64Jumps) {\n if (LongAddressGen(rd, Label) == ARMEmitter::BranchEncodeSucceeded::Failure) {\n ERROR_AND_DIE_FMT(\"Unable to encode long ADR.\");\n }\n return;\n }\n if (adr(rd, Label) == ARMEmitter::BranchEncodeSucceeded::Success) {\n return;\n }\n\n FEXCore::UncheckedLongJump::LongJump(ThreadState->RestartJump, FEXCore::ToUnderlying(RestartOptions::Control::EnableFarARM64Jumps));\n }\n\n template\n void adrp_OrRestart(ARMEmitter::Register rd, T* Label) {\n if (RequiresFarARM64Jumps) {\n if (LongAddressGen(rd, Label) == ARMEmitter::BranchEncodeSucceeded::Failure) {\n ERROR_AND_DIE_FMT(\"Unable to encode long ADRP.\");\n }\n return;\n }\n if (adrp(rd, Label) == ARMEmitter::BranchEncodeSucceeded::Success) {\n return;\n }\n\n FEXCore::UncheckedLongJump::LongJump(ThreadState->RestartJump, FEXCore::ToUnderlying(RestartOptions::Control::EnableFarARM64Jumps));\n }\n\n template\n void BindOrRestart(T* Label) {\n if (Bind(Label)) {\n return;\n }\n\n if (RequiresFarARM64Jumps) {\n // This should have been caught before this point.\n ERROR_AND_DIE_FMT(\"Unhandled long bind\");\n return;\n }\n\n FEXCore::UncheckedLongJump::LongJump(ThreadState->RestartJump, FEXCore::ToUnderlying(RestartOptions::Control::EnableFarARM64Jumps));\n }\n\n // This is purely a debugging aid for developers to see if they are in JIT code space when inspecting raw memory\n void EmitDetectionString();\n IR::RegisterAllocationPass* RAPass {};\n FEXCore::Core::DebugData* DebugData {};\n\n void ResetStack();\n /**\n * @name Relocations\n * @{ */\n\n /**\n * @brief A literal pair relocation object for named symbol literals\n */\n struct NamedSymbolLiteralPair {\n ARMEmitter::ForwardLabel Loc;\n uint64_t Lit;\n Relocation MoveABI {};\n };\n\n /**\n * @brief Inserts a thunk relocation\n *\n * @param Reg - The GPR to move the thunk handler in to\n * @param Sum - The hash of the thunk\n */\n void InsertNamedThunkRelocation(ARMEmitter::Register Reg, const IR::SHA256Sum& Sum);\n\n /**\n * @brief Inserts a guest GPR move relocation\n *\n * @param Reg - The GPR to move the guest RIP in to\n * @param Constant - The guest RIP that will be relocated\n */\n void InsertGuestRIPMove(ARMEmitter::Register Reg, uint64_t Constant);\n\n /**\n * @brief Inserts a named symbol as a literal in memory\n *\n * Need to use `PlaceNamedSymbolLiteral` with the return value to place the literal in the desired location\n *\n * @param Op The named symbol to place\n *\n * @return A temporary `NamedSymbolLiteralPair`\n */\n NamedSymbolLiteralPair InsertNamedSymbolLiteral(FEXCore::CPU::RelocNamedSymbolLiteral::NamedSymbol Op);\n\n /**\n * @brief Inserts a relocation for a constant value relative to the guest entrypoint\n *\n * @param Reg - The GPR to move the guest RIP in to\n * @param Constant - The guest RIP that will be relocated\n */\n NamedSymbolLiteralPair InsertGuestRIPLiteral(uint64_t GuestRIP);\n\n /**\n * @brief Place the named symbol literal relocation in memory\n *\n * @param Lit - Which literal to place\n */\n void PlaceNamedSymbolLiteral(NamedSymbolLiteralPair Lit);\n\n fextl::vector Relocations;\n\n /**\n * Returns any relocations generated since the last call to TakeRelocations.\n *\n * GuestBaseAddress must match the base virtual address to which the\n * input x86 binary is mapped.\n */\n fextl::vector TakeRelocations(uint64_t GuestBaseAddress) override;\n\n /** @} */\n\n uint32_t SpillSlots {};\n using OpType = void (Arm64JITCore::*)(const IR::IROp_Header* IROp, IR::Ref Node);\n\n using ScalarFMAOpCaller =\n std::function;\n void VFScalarFMAOperation(IR::OpSize OpSize, IR::OpSize ElementSize, ScalarFMAOpCaller ScalarEmit, ARMEmitter::VRegister Dst,\n ARMEmitter::VRegister Upper, ARMEmitter::VRegister Vector1, ARMEmitter::VRegister Vector2,\n ARMEmitter::VRegister Addend);\n using ScalarBinaryOpCaller = std::function;\n void VFScalarOperation(IR::OpSize OpSize, IR::OpSize ElementSize, bool ZeroUpperBits, ScalarBinaryOpCaller ScalarEmit,\n ARMEmitter::VRegister Dst, ARMEmitter::VRegister Vector1, ARMEmitter::VRegister Vector2);\n using ScalarUnaryOpCaller = std::function SrcVar)>;\n void VFScalarUnaryOperation(IR::OpSize OpSize, IR::OpSize ElementSize, bool ZeroUpperBits, ScalarUnaryOpCaller ScalarEmit,\n ARMEmitter::VRegister Dst, ARMEmitter::VRegister Vector1,\n std::variant Vector2);\n\n void Emulate128BitGather(IR::OpSize Size, IR::OpSize ElementSize, ARMEmitter::VRegister Dst, ARMEmitter::VRegister IncomingDst,\n std::optional BaseAddr, ARMEmitter::VRegister VectorIndexLow,\n std::optional VectorIndexHigh, ARMEmitter::VRegister MaskReg, IR::OpSize VectorIndexSize,\n size_t DataElementOffsetStart, size_t IndexElementOffsetStart, uint8_t OffsetScale, IR::OpSize AddrSize);\n\n void EmitTFCheck();\n\n void EmitSuspendInterruptCheck();\n\n void EmitEntryPoint(ARMEmitter::BackwardLabel& HeaderLabel, bool CheckTF);\n\n#define DEF_OP(x) void Op_##x(IR::IROp_Header const* IROp, IR::Ref Node)\n\n ///< Unhandled handler\n DEF_OP(Unhandled);\n\n ///< No-op Handler\n DEF_OP(NoOp);\n\n#define IROP_DISPATCH_DEFS\n#include \n#undef DEF_OP\n};\n\n#define DEF_OP(x) void Arm64JITCore::Op_##x(IR::IROp_Header const* IROp, IR::Ref Node)\n\n[[nodiscard]]\nfextl::unique_ptr CreateArm64JITCore(FEXCore::Context::ContextImpl* ctx, FEXCore::Core::InternalThreadState* Thread);\n\n} // namespace FEXCore::CPU\n" }, { "path": "FEXCore/Source/Interface/Core/JIT/MemoryOps.cpp", "content": "// SPDX-License-Identifier: MIT\n/*\n$info$\ntags: backend|arm64\n$end_info$\n*/\n\n#include \"FEXCore/Core/X86Enums.h\"\n#include \"FEXCore/Utils/LogManager.h\"\n#include \"Interface/Context/Context.h\"\n#include \"Interface/Core/ArchHelpers/Arm64Emitter.h\"\n#include \"Interface/Core/CPUID.h\"\n#include \"Interface/Core/JIT/JITClass.h\"\n#include \"Interface/IR/RegisterAllocationData.h\"\n#include \n#include \n\nnamespace FEXCore::CPU {\n\nDEF_OP(LoadContext) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n if (Op->Class == IR::RegClass::GPR) {\n auto Dst = GetReg(Node);\n\n switch (OpSize) {\n case IR::OpSize::i8Bit: ldrb(Dst, STATE, Op->Offset); break;\n case IR::OpSize::i16Bit: ldrh(Dst, STATE, Op->Offset); break;\n case IR::OpSize::i32Bit: ldr(Dst.W(), STATE, Op->Offset); break;\n case IR::OpSize::i64Bit: ldr(Dst.X(), STATE, Op->Offset); break;\n default: LOGMAN_MSG_A_FMT(\"Unhandled LoadContext size: {}\", OpSize); break;\n }\n } else {\n auto Dst = GetVReg(Node);\n\n switch (OpSize) {\n case IR::OpSize::i8Bit: ldrb(Dst, STATE, Op->Offset); break;\n case IR::OpSize::i16Bit: ldrh(Dst, STATE, Op->Offset); break;\n case IR::OpSize::i32Bit: ldr(Dst.S(), STATE, Op->Offset); break;\n case IR::OpSize::i64Bit: ldr(Dst.D(), STATE, Op->Offset); break;\n case IR::OpSize::i128Bit: ldr(Dst.Q(), STATE, Op->Offset); break;\n case IR::OpSize::i256Bit:\n mov(TMP1, Op->Offset);\n ld1b(Dst.Z(), PRED_TMP_32B.Zeroing(), STATE, TMP1);\n break;\n default: LOGMAN_MSG_A_FMT(\"Unhandled LoadContext size: {}\", OpSize); break;\n }\n }\n}\n\nDEF_OP(LoadContextPair) {\n const auto Op = IROp->C();\n\n if (Op->Class == IR::RegClass::GPR) {\n const auto Dst1 = GetReg(Op->OutValue1);\n const auto Dst2 = GetReg(Op->OutValue2);\n\n switch (IROp->Size) {\n case IR::OpSize::i32Bit: ldp(Dst1.W(), Dst2.W(), STATE, Op->Offset); break;\n case IR::OpSize::i64Bit: ldp(Dst1.X(), Dst2.X(), STATE, Op->Offset); break;\n default: LOGMAN_MSG_A_FMT(\"Unhandled LoadMemPair size: {}\", IROp->Size); break;\n }\n } else {\n const auto Dst1 = GetVReg(Op->OutValue1);\n const auto Dst2 = GetVReg(Op->OutValue2);\n\n switch (IROp->Size) {\n case IR::OpSize::i32Bit: ldp(Dst1.S(), Dst2.S(), STATE, Op->Offset); break;\n case IR::OpSize::i64Bit: ldp(Dst1.D(), Dst2.D(), STATE, Op->Offset); break;\n case IR::OpSize::i128Bit: ldp(Dst1.Q(), Dst2.Q(), STATE, Op->Offset); break;\n default: LOGMAN_MSG_A_FMT(\"Unhandled LoadMemPair size: {}\", IROp->Size); break;\n }\n }\n}\n\nDEF_OP(StoreContext) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n if (Op->Class == IR::RegClass::GPR) {\n auto Src = GetZeroableReg(Op->Value);\n\n switch (OpSize) {\n case IR::OpSize::i8Bit: strb(Src, STATE, Op->Offset); break;\n case IR::OpSize::i16Bit: strh(Src, STATE, Op->Offset); break;\n case IR::OpSize::i32Bit: str(Src.W(), STATE, Op->Offset); break;\n case IR::OpSize::i64Bit: str(Src.X(), STATE, Op->Offset); break;\n default: LOGMAN_MSG_A_FMT(\"Unhandled StoreContext size: {}\", OpSize); break;\n }\n } else {\n const auto Src = GetVReg(Op->Value);\n\n switch (OpSize) {\n case IR::OpSize::i8Bit: strb(Src, STATE, Op->Offset); break;\n case IR::OpSize::i16Bit: strh(Src, STATE, Op->Offset); break;\n case IR::OpSize::i32Bit: str(Src.S(), STATE, Op->Offset); break;\n case IR::OpSize::i64Bit: str(Src.D(), STATE, Op->Offset); break;\n case IR::OpSize::i128Bit: str(Src.Q(), STATE, Op->Offset); break;\n case IR::OpSize::i256Bit:\n mov(TMP1, Op->Offset);\n st1b(Src.Z(), PRED_TMP_32B, STATE, TMP1);\n break;\n default: LOGMAN_MSG_A_FMT(\"Unhandled StoreContext size: {}\", OpSize); break;\n }\n }\n}\n\nDEF_OP(StoreContextPair) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n if (Op->Class == IR::RegClass::GPR) {\n auto Src1 = GetZeroableReg(Op->Value1);\n auto Src2 = GetZeroableReg(Op->Value2);\n\n switch (OpSize) {\n case IR::OpSize::i32Bit: stp(Src1.W(), Src2.W(), STATE, Op->Offset); break;\n case IR::OpSize::i64Bit: stp(Src1.X(), Src2.X(), STATE, Op->Offset); break;\n default: LOGMAN_MSG_A_FMT(\"Unhandled StoreContext size: {}\", OpSize); break;\n }\n } else {\n const auto Src1 = GetVReg(Op->Value1);\n const auto Src2 = GetVReg(Op->Value2);\n\n switch (OpSize) {\n case IR::OpSize::i32Bit: stp(Src1.S(), Src2.S(), STATE, Op->Offset); break;\n case IR::OpSize::i64Bit: stp(Src1.D(), Src2.D(), STATE, Op->Offset); break;\n case IR::OpSize::i128Bit: stp(Src1.Q(), Src2.Q(), STATE, Op->Offset); break;\n default: LOGMAN_MSG_A_FMT(\"Unhandled StoreContextPair size: {}\", OpSize); break;\n }\n }\n}\n\nDEF_OP(LoadRegister) {\n const auto Op = IROp->C();\n\n if (Op->Class == IR::RegClass::GPR) {\n LOGMAN_THROW_A_FMT(Op->Reg < StaticRegisters.size(), \"out of range reg\");\n\n mov(GetReg(Node).X(), StaticRegisters[Op->Reg].X());\n } else if (Op->Class == IR::RegClass::FPR) {\n const auto regSize = HostSupportsAVX256 ? IR::OpSize::i256Bit : IR::OpSize::i128Bit;\n LOGMAN_THROW_A_FMT(Op->Reg < StaticFPRegisters.size(), \"out of range reg\");\n LOGMAN_THROW_A_FMT(IROp->Size == regSize, \"expected sized\");\n\n const auto guest = StaticFPRegisters[Op->Reg];\n const auto host = GetVReg(Node);\n\n if (HostSupportsAVX256) {\n mov(ARMEmitter::SubRegSize::i64Bit, host.Z(), PRED_TMP_32B.Merging(), guest.Z());\n } else {\n mov(host.Q(), guest.Q());\n }\n } else {\n LOGMAN_THROW_A_FMT(false, \"Unhandled Op->Class {}\", Op->Class);\n }\n}\n\nDEF_OP(LoadPF) {\n const auto reg = StaticRegisters[StaticRegisters.size() - 2];\n\n if (GetReg(Node).Idx() != reg.Idx()) {\n mov(GetReg(Node).X(), reg.X());\n }\n}\n\nDEF_OP(LoadAF) {\n const auto reg = StaticRegisters[StaticRegisters.size() - 1];\n\n if (GetReg(Node).Idx() != reg.Idx()) {\n mov(GetReg(Node).X(), reg.X());\n }\n}\n\nDEF_OP(StoreRegister) {\n const auto Op = IROp->C();\n const auto Reg = IR::PhysicalRegister(Node);\n const auto RegClass = Reg.AsRegClass();\n\n if (RegClass == IR::RegClass::GPRFixed) {\n // Always use 64-bit, it's faster. Upper bits ignored for 32-bit mode.\n mov(ARMEmitter::Size::i64Bit, GetReg(Reg), GetReg(Op->Value));\n } else if (RegClass == IR::RegClass::FPRFixed) {\n const auto regSize = HostSupportsAVX256 ? IR::OpSize::i256Bit : IR::OpSize::i128Bit;\n LOGMAN_THROW_A_FMT(IROp->Size == regSize, \"expected sized\");\n\n const auto guest = GetVReg(Reg);\n const auto host = GetVReg(Op->Value);\n\n if (HostSupportsAVX256) {\n mov(ARMEmitter::SubRegSize::i64Bit, guest.Z(), PRED_TMP_32B.Merging(), host.Z());\n } else {\n mov(guest.Q(), host.Q());\n }\n } else {\n LOGMAN_THROW_A_FMT(false, \"Unhandled Op->Class {}\", RegClass);\n }\n}\n\nDEF_OP(StorePF) {\n const auto Op = IROp->C();\n const auto reg = StaticRegisters[StaticRegisters.size() - 2];\n const auto Src = GetReg(Op->Value);\n\n if (Src.Idx() != reg.Idx()) {\n // Always use 64-bit, it's faster. Upper bits ignored for 32-bit mode.\n mov(ARMEmitter::Size::i64Bit, reg, Src);\n }\n}\n\nDEF_OP(StoreAF) {\n const auto Op = IROp->C();\n const auto reg = StaticRegisters[StaticRegisters.size() - 1];\n const auto Src = GetReg(Op->Value);\n\n if (Src.Idx() != reg.Idx()) {\n // Always use 64-bit, it's faster. Upper bits ignored for 32-bit mode.\n mov(ARMEmitter::Size::i64Bit, reg, Src);\n }\n}\n\nDEF_OP(LoadContextIndexed) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto Index = GetReg(Op->Index);\n\n if (Op->Class == IR::RegClass::GPR) {\n switch (Op->Stride) {\n case 1:\n case 2:\n case 4:\n case 8: {\n add(ARMEmitter::Size::i64Bit, TMP1, STATE, Index, ARMEmitter::ShiftType::LSL, FEXCore::ilog2(Op->Stride));\n const auto Dst = GetReg(Node);\n switch (OpSize) {\n case IR::OpSize::i8Bit: ldrb(Dst, TMP1, Op->BaseOffset); break;\n case IR::OpSize::i16Bit: ldrh(Dst, TMP1, Op->BaseOffset); break;\n case IR::OpSize::i32Bit: ldr(Dst.W(), TMP1, Op->BaseOffset); break;\n case IR::OpSize::i64Bit: ldr(Dst.X(), TMP1, Op->BaseOffset); break;\n default: LOGMAN_MSG_A_FMT(\"Unhandled LoadContextIndexed size: {}\", OpSize); break;\n }\n break;\n }\n case 16: LOGMAN_MSG_A_FMT(\"Invalid Class load of size 16\"); break;\n default: LOGMAN_MSG_A_FMT(\"Unhandled LoadContextIndexed stride: {}\", Op->Stride); break;\n }\n } else {\n switch (Op->Stride) {\n case 1:\n case 2:\n case 4:\n case 8:\n case 16:\n case 32: {\n add(ARMEmitter::Size::i64Bit, TMP1, STATE, Index, ARMEmitter::ShiftType::LSL, FEXCore::ilog2(Op->Stride));\n const auto Dst = GetVReg(Node);\n\n switch (OpSize) {\n case IR::OpSize::i8Bit: ldrb(Dst, TMP1, Op->BaseOffset); break;\n case IR::OpSize::i16Bit: ldrh(Dst, TMP1, Op->BaseOffset); break;\n case IR::OpSize::i32Bit: ldr(Dst.S(), TMP1, Op->BaseOffset); break;\n case IR::OpSize::i64Bit: ldr(Dst.D(), TMP1, Op->BaseOffset); break;\n case IR::OpSize::i128Bit:\n if (Op->BaseOffset % 16 == 0) {\n ldr(Dst.Q(), TMP1, Op->BaseOffset);\n } else {\n add(ARMEmitter::Size::i64Bit, TMP1, TMP1, Op->BaseOffset);\n ldur(Dst.Q(), TMP1, Op->BaseOffset);\n }\n break;\n case IR::OpSize::i256Bit:\n mov(TMP2, Op->BaseOffset);\n ld1b(Dst.Z(), PRED_TMP_32B.Zeroing(), TMP1, TMP2);\n break;\n default: LOGMAN_MSG_A_FMT(\"Unhandled LoadContextIndexed size: {}\", OpSize); break;\n }\n break;\n }\n default: LOGMAN_MSG_A_FMT(\"Unhandled LoadContextIndexed stride: {}\", Op->Stride); break;\n }\n }\n}\n\nDEF_OP(StoreContextIndexed) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto Index = GetReg(Op->Index);\n\n if (Op->Class == IR::RegClass::GPR) {\n const auto Value = GetReg(Op->Value);\n\n switch (Op->Stride) {\n case 1:\n case 2:\n case 4:\n case 8: {\n add(ARMEmitter::Size::i64Bit, TMP1, STATE, Index, ARMEmitter::ShiftType::LSL, FEXCore::ilog2(Op->Stride));\n\n switch (OpSize) {\n case IR::OpSize::i8Bit: strb(Value, TMP1, Op->BaseOffset); break;\n case IR::OpSize::i16Bit: strh(Value, TMP1, Op->BaseOffset); break;\n case IR::OpSize::i32Bit: str(Value.W(), TMP1, Op->BaseOffset); break;\n case IR::OpSize::i64Bit: str(Value.X(), TMP1, Op->BaseOffset); break;\n default: LOGMAN_MSG_A_FMT(\"Unhandled StoreContextIndexed size: {}\", OpSize); break;\n }\n break;\n }\n case 16: LOGMAN_MSG_A_FMT(\"Invalid Class store of size 16\"); break;\n default: LOGMAN_MSG_A_FMT(\"Unhandled StoreContextIndexed stride: {}\", Op->Stride); break;\n }\n } else {\n const auto Value = GetVReg(Op->Value);\n\n switch (Op->Stride) {\n case 1:\n case 2:\n case 4:\n case 8:\n case 16:\n case 32: {\n add(ARMEmitter::Size::i64Bit, TMP1, STATE, Index, ARMEmitter::ShiftType::LSL, FEXCore::ilog2(Op->Stride));\n\n switch (OpSize) {\n case IR::OpSize::i8Bit: strb(Value, TMP1, Op->BaseOffset); break;\n case IR::OpSize::i16Bit: strh(Value, TMP1, Op->BaseOffset); break;\n case IR::OpSize::i32Bit: str(Value.S(), TMP1, Op->BaseOffset); break;\n case IR::OpSize::i64Bit: str(Value.D(), TMP1, Op->BaseOffset); break;\n case IR::OpSize::i128Bit:\n if (Op->BaseOffset % 16 == 0) {\n str(Value.Q(), TMP1, Op->BaseOffset);\n } else {\n add(ARMEmitter::Size::i64Bit, TMP1, TMP1, Op->BaseOffset);\n stur(Value.Q(), TMP1, Op->BaseOffset);\n }\n break;\n case IR::OpSize::i256Bit:\n mov(TMP2, Op->BaseOffset);\n st1b(Value.Z(), PRED_TMP_32B, TMP1, TMP2);\n break;\n default: LOGMAN_MSG_A_FMT(\"Unhandled StoreContextIndexed size: {}\", OpSize); break;\n }\n break;\n }\n default: LOGMAN_MSG_A_FMT(\"Unhandled StoreContextIndexed stride: {}\", Op->Stride); break;\n }\n }\n}\n\nDEF_OP(FormContextAddress) {\n const auto Op = IROp->C();\n const auto Index = GetReg(Op->Index);\n const auto Dst = GetReg(Node);\n\n switch (Op->Stride) {\n case 1:\n case 2:\n case 4:\n case 8:\n case 16:\n case 32: {\n add(ARMEmitter::Size::i64Bit, Dst, STATE, Index, ARMEmitter::ShiftType::LSL, FEXCore::ilog2(Op->Stride));\n break;\n }\n default: LOGMAN_MSG_A_FMT(\"Unhandled FormContextAddress stride: {}\", Op->Stride); break;\n }\n}\n\nDEF_OP(SpillRegister) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n const uint32_t SlotOffset = Op->Slot * MaxSpillSlotSize;\n\n if (Op->Class == IR::RegClass::GPR) {\n const auto Src = GetReg(Op->Value);\n switch (OpSize) {\n case IR::OpSize::i8Bit: {\n if (SlotOffset > LSByteMaxUnsignedOffset) {\n LoadConstant(ARMEmitter::Size::i64Bit, TMP1, SlotOffset);\n strb(Src, ARMEmitter::Reg::rsp, TMP1.R(), ARMEmitter::ExtendedType::LSL_64, 0);\n } else {\n strb(Src, ARMEmitter::Reg::rsp, SlotOffset);\n }\n break;\n }\n case IR::OpSize::i16Bit: {\n if (SlotOffset > LSHalfMaxUnsignedOffset) {\n LoadConstant(ARMEmitter::Size::i64Bit, TMP1, SlotOffset);\n strh(Src, ARMEmitter::Reg::rsp, TMP1.R(), ARMEmitter::ExtendedType::LSL_64, 0);\n } else {\n strh(Src, ARMEmitter::Reg::rsp, SlotOffset);\n }\n break;\n }\n case IR::OpSize::i32Bit: {\n if (SlotOffset > LSWordMaxUnsignedOffset) {\n LoadConstant(ARMEmitter::Size::i64Bit, TMP1, SlotOffset);\n str(Src.W(), ARMEmitter::Reg::rsp, TMP1.R(), ARMEmitter::ExtendedType::LSL_64, 0);\n } else {\n str(Src.W(), ARMEmitter::Reg::rsp, SlotOffset);\n }\n break;\n }\n case IR::OpSize::i64Bit: {\n if (SlotOffset > LSDWordMaxUnsignedOffset) {\n LoadConstant(ARMEmitter::Size::i64Bit, TMP1, SlotOffset);\n str(Src.X(), ARMEmitter::Reg::rsp, TMP1.R(), ARMEmitter::ExtendedType::LSL_64, 0);\n } else {\n str(Src.X(), ARMEmitter::Reg::rsp, SlotOffset);\n }\n break;\n }\n default: LOGMAN_MSG_A_FMT(\"Unhandled SpillRegister size: {}\", OpSize); break;\n }\n } else if (Op->Class == FEXCore::IR::RegClass::FPR) {\n const auto Src = GetVReg(Op->Value);\n\n switch (OpSize) {\n case IR::OpSize::i32Bit: {\n if (SlotOffset > LSWordMaxUnsignedOffset) {\n LoadConstant(ARMEmitter::Size::i64Bit, TMP1, SlotOffset);\n str(Src.S(), ARMEmitter::Reg::rsp, TMP1.R(), ARMEmitter::ExtendedType::LSL_64, 0);\n } else {\n str(Src.S(), ARMEmitter::Reg::rsp, SlotOffset);\n }\n break;\n }\n case IR::OpSize::i64Bit: {\n if (SlotOffset > LSDWordMaxUnsignedOffset) {\n LoadConstant(ARMEmitter::Size::i64Bit, TMP1, SlotOffset);\n str(Src.D(), ARMEmitter::Reg::rsp, TMP1.R(), ARMEmitter::ExtendedType::LSL_64, 0);\n } else {\n str(Src.D(), ARMEmitter::Reg::rsp, SlotOffset);\n }\n break;\n }\n case IR::OpSize::i128Bit: {\n if (SlotOffset > LSQWordMaxUnsignedOffset) {\n LoadConstant(ARMEmitter::Size::i64Bit, TMP1, SlotOffset);\n str(Src.Q(), ARMEmitter::Reg::rsp, TMP1.R(), ARMEmitter::ExtendedType::LSL_64, 0);\n } else {\n str(Src.Q(), ARMEmitter::Reg::rsp, SlotOffset);\n }\n break;\n }\n case IR::OpSize::i256Bit: {\n mov(TMP3, SlotOffset);\n st1b(Src.Z(), PRED_TMP_32B, ARMEmitter::Reg::rsp, TMP3);\n break;\n }\n default: LOGMAN_MSG_A_FMT(\"Unhandled SpillRegister size: {}\", OpSize); break;\n }\n } else {\n LOGMAN_MSG_A_FMT(\"Unhandled SpillRegister class: {}\", Op->Class);\n }\n}\n\nDEF_OP(FillRegister) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n const uint32_t SlotOffset = Op->Slot * MaxSpillSlotSize;\n\n if (Op->Class == IR::RegClass::GPR) {\n const auto Dst = GetReg(Node);\n switch (OpSize) {\n case IR::OpSize::i8Bit: {\n if (SlotOffset > LSByteMaxUnsignedOffset) {\n LoadConstant(ARMEmitter::Size::i64Bit, TMP1, SlotOffset);\n ldrb(Dst, ARMEmitter::Reg::rsp, TMP1.R(), ARMEmitter::ExtendedType::LSL_64, 0);\n } else {\n ldrb(Dst, ARMEmitter::Reg::rsp, SlotOffset);\n }\n break;\n }\n case IR::OpSize::i16Bit: {\n if (SlotOffset > LSHalfMaxUnsignedOffset) {\n LoadConstant(ARMEmitter::Size::i64Bit, TMP1, SlotOffset);\n ldrh(Dst, ARMEmitter::Reg::rsp, TMP1.R(), ARMEmitter::ExtendedType::LSL_64, 0);\n } else {\n ldrh(Dst, ARMEmitter::Reg::rsp, SlotOffset);\n }\n break;\n }\n case IR::OpSize::i32Bit: {\n if (SlotOffset > LSWordMaxUnsignedOffset) {\n LoadConstant(ARMEmitter::Size::i64Bit, TMP1, SlotOffset);\n ldr(Dst.W(), ARMEmitter::Reg::rsp, TMP1.R(), ARMEmitter::ExtendedType::LSL_64, 0);\n } else {\n ldr(Dst.W(), ARMEmitter::Reg::rsp, SlotOffset);\n }\n break;\n }\n case IR::OpSize::i64Bit: {\n if (SlotOffset > LSDWordMaxUnsignedOffset) {\n LoadConstant(ARMEmitter::Size::i64Bit, TMP1, SlotOffset);\n ldr(Dst.X(), ARMEmitter::Reg::rsp, TMP1.R(), ARMEmitter::ExtendedType::LSL_64, 0);\n } else {\n ldr(Dst.X(), ARMEmitter::Reg::rsp, SlotOffset);\n }\n break;\n }\n default: LOGMAN_MSG_A_FMT(\"Unhandled FillRegister size: {}\", OpSize); break;\n }\n } else if (Op->Class == FEXCore::IR::RegClass::FPR) {\n const auto Dst = GetVReg(Node);\n\n switch (OpSize) {\n case IR::OpSize::i32Bit: {\n if (SlotOffset > LSWordMaxUnsignedOffset) {\n LoadConstant(ARMEmitter::Size::i64Bit, TMP1, SlotOffset);\n ldr(Dst.S(), ARMEmitter::Reg::rsp, TMP1.R(), ARMEmitter::ExtendedType::LSL_64, 0);\n } else {\n ldr(Dst.S(), ARMEmitter::Reg::rsp, SlotOffset);\n }\n break;\n }\n case IR::OpSize::i64Bit: {\n if (SlotOffset > LSDWordMaxUnsignedOffset) {\n LoadConstant(ARMEmitter::Size::i64Bit, TMP1, SlotOffset);\n ldr(Dst.D(), ARMEmitter::Reg::rsp, TMP1.R(), ARMEmitter::ExtendedType::LSL_64, 0);\n } else {\n ldr(Dst.D(), ARMEmitter::Reg::rsp, SlotOffset);\n }\n break;\n }\n case IR::OpSize::i128Bit: {\n if (SlotOffset > LSQWordMaxUnsignedOffset) {\n LoadConstant(ARMEmitter::Size::i64Bit, TMP1, SlotOffset);\n ldr(Dst.Q(), ARMEmitter::Reg::rsp, TMP1.R(), ARMEmitter::ExtendedType::LSL_64, 0);\n } else {\n ldr(Dst.Q(), ARMEmitter::Reg::rsp, SlotOffset);\n }\n break;\n }\n case IR::OpSize::i256Bit: {\n mov(TMP3, SlotOffset);\n ld1b(Dst.Z(), PRED_TMP_32B.Zeroing(), ARMEmitter::Reg::rsp, TMP3);\n break;\n }\n default: LOGMAN_MSG_A_FMT(\"Unhandled FillRegister size: {}\", OpSize); break;\n }\n } else {\n LOGMAN_MSG_A_FMT(\"Unhandled FillRegister class: {}\", Op->Class);\n }\n}\n\nDEF_OP(LoadNZCV) {\n auto Dst = GetReg(Node);\n\n mrs(Dst, ARMEmitter::SystemRegister::NZCV);\n}\n\nDEF_OP(StoreNZCV) {\n auto Op = IROp->C();\n\n msr(ARMEmitter::SystemRegister::NZCV, GetReg(Op->Value));\n}\n\nDEF_OP(LoadDF) {\n auto Dst = GetReg(Node);\n auto Flag = X86State::RFLAG_DF_RAW_LOC;\n\n // DF needs sign extension to turn 0x1/0xFF into 1/-1\n ldrsb(Dst.X(), STATE, ARRAY_OFFSETOF(FEXCore::Core::CPUState, flags, Flag));\n}\n\nDEF_OP(ContextClear) {\n auto Op = IROp->C();\n if (CTX->HostFeatures.SupportsCLZERO) {\n // We can use CLZero directly when hardware supports it.\n // Provides a fairly generous speed-up on Ampere1A hardware.\n // TODO: When FEAT_MOPS hardware ships, test memset using MOPS.\n for (size_t i = 0; i < Op->Size; i += 64) {\n add(ARMEmitter::Size::i64Bit, TMP1, STATE.R(), Op->Offset + i);\n dc(ARMEmitter::DataCacheOperation::ZVA, TMP1);\n }\n } else {\n movi(ARMEmitter::SubRegSize::i64Bit, VTMP1.Q(), 0);\n for (size_t i = 0; i < Op->Size; i += 32) {\n stp(VTMP1.Q(), VTMP1.Q(), STATE.R(), Op->Offset + i);\n }\n }\n}\n\nARMEmitter::ExtendedMemOperand Arm64JITCore::GenerateMemOperand(\n IR::OpSize AccessSize, ARMEmitter::Register Base, IR::OrderedNodeWrapper Offset, IR::MemOffsetType OffsetType, uint8_t OffsetScale) {\n if (Offset.IsInvalid()) {\n return ARMEmitter::ExtendedMemOperand(Base.X(), ARMEmitter::IndexType::OFFSET, 0);\n } else {\n if (OffsetScale != 1 && OffsetScale != IR::OpSizeToSize(AccessSize)) {\n LOGMAN_MSG_A_FMT(\"Unhandled GenerateMemOperand OffsetScale: {}\", OffsetScale);\n }\n uint64_t Const;\n if (IsInlineConstant(Offset, &Const)) {\n return ARMEmitter::ExtendedMemOperand(Base.X(), ARMEmitter::IndexType::OFFSET, Const);\n } else {\n auto RegOffset = GetReg(Offset);\n switch (OffsetType) {\n case IR::MemOffsetType::SXTX:\n return ARMEmitter::ExtendedMemOperand(Base.X(), RegOffset.X(), ARMEmitter::ExtendedType::SXTX, FEXCore::ilog2(OffsetScale));\n case IR::MemOffsetType::UXTW:\n return ARMEmitter::ExtendedMemOperand(Base.X(), RegOffset.X(), ARMEmitter::ExtendedType::UXTW, FEXCore::ilog2(OffsetScale));\n case IR::MemOffsetType::SXTW:\n return ARMEmitter::ExtendedMemOperand(Base.X(), RegOffset.X(), ARMEmitter::ExtendedType::SXTW, FEXCore::ilog2(OffsetScale));\n default: LOGMAN_MSG_A_FMT(\"Unhandled GenerateMemOperand OffsetType: {}\", OffsetType); break;\n }\n }\n }\n\n FEX_UNREACHABLE;\n}\n\nARMEmitter::Register Arm64JITCore::ApplyMemOperand(IR::OpSize AccessSize, ARMEmitter::Register Base, ARMEmitter::Register Tmp,\n IR::OrderedNodeWrapper Offset, IR::MemOffsetType OffsetType, uint8_t OffsetScale) {\n if (Offset.IsInvalid()) {\n return Base;\n }\n\n if (OffsetScale != 1 && OffsetScale != IR::OpSizeToSize(AccessSize)) {\n LOGMAN_MSG_A_FMT(\"Unhandled OffsetScale: {}\", OffsetScale);\n }\n\n uint64_t Const;\n if (IsInlineConstant(Offset, &Const)) {\n if (Const == 0) {\n return Base;\n }\n LoadConstant(ARMEmitter::Size::i64Bit, Tmp, Const);\n add(ARMEmitter::Size::i64Bit, Tmp, Base, Tmp, ARMEmitter::ShiftType::LSL, FEXCore::ilog2(OffsetScale));\n } else {\n auto RegOffset = GetReg(Offset);\n switch (OffsetType) {\n case IR::MemOffsetType::SXTX:\n add(ARMEmitter::Size::i64Bit, Tmp, Base, RegOffset, ARMEmitter::ExtendedType::SXTX, FEXCore::ilog2(OffsetScale));\n break;\n\n case IR::MemOffsetType::UXTW:\n add(ARMEmitter::Size::i64Bit, Tmp, Base, RegOffset, ARMEmitter::ExtendedType::UXTW, FEXCore::ilog2(OffsetScale));\n break;\n\n case IR::MemOffsetType::SXTW:\n add(ARMEmitter::Size::i64Bit, Tmp, Base, RegOffset, ARMEmitter::ExtendedType::SXTW, FEXCore::ilog2(OffsetScale));\n break;\n\n default: LOGMAN_MSG_A_FMT(\"Unhandled OffsetType: {}\", OffsetType); break;\n }\n }\n return Tmp;\n}\n\nARMEmitter::SVEMemOperand Arm64JITCore::GenerateSVEMemOperand(IR::OpSize AccessSize, ARMEmitter::Register Base, IR::OrderedNodeWrapper Offset,\n IR::MemOffsetType OffsetType, [[maybe_unused]] uint8_t OffsetScale) {\n if (Offset.IsInvalid()) {\n return ARMEmitter::SVEMemOperand(Base.X(), 0);\n }\n\n uint64_t Const {};\n if (IsInlineConstant(Offset, &Const)) {\n if (Const == 0) {\n return ARMEmitter::SVEMemOperand(Base.X(), 0);\n }\n\n const auto SignedConst = static_cast(Const);\n const auto SignedSVESize = static_cast(HostSupportsSVE256 ? Core::CPUState::XMM_AVX_REG_SIZE : Core::CPUState::XMM_SSE_REG_SIZE);\n\n const auto IsCleanlyDivisible = (SignedConst % SignedSVESize) == 0;\n const auto Index = SignedConst / SignedSVESize;\n\n // SVE's immediate variants of load stores are quite limited in terms\n // of immediate range. They also operate on a by-vector-length basis.\n //\n // e.g. On a 256-bit SVE capable system:\n //\n // LD1B Dst.B, Predicate/Z, [Reg, #1, MUL VL]\n //\n // Will add 32 to the base register as the offset\n //\n // So if we have a constant that cleanly lies along a 256-bit offset\n // and is also within the limitations of the immediate of -8 to 7\n // then we can encode it as an immediate offset.\n //\n if (IsCleanlyDivisible && Index >= -8 && Index <= 7) {\n return ARMEmitter::SVEMemOperand(Base.X(), static_cast(Index));\n }\n\n // If we can't do that for whatever reason, then unfortunately, we need\n // to move it over to a temporary to use as an offset.\n mov(TMP1, Const);\n return ARMEmitter::SVEMemOperand(Base.X(), TMP1);\n }\n\n // Otherwise handle it like normal.\n // Note that we do nothing with the offset type and offset scale,\n // since SVE loads and stores don't have the ability to perform an\n // optional extension or shift as part of their behavior.\n LOGMAN_THROW_A_FMT(OffsetType == IR::MemOffsetType::SXTX, \"Currently only the default offset type (SXTX) is supported.\");\n\n const auto RegOffset = GetReg(Offset);\n return ARMEmitter::SVEMemOperand(Base.X(), RegOffset.X());\n}\n\nDEF_OP(LoadMem) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto MemReg = GetReg(Op->Addr);\n const auto MemSrc = GenerateMemOperand(OpSize, MemReg, Op->Offset, Op->OffsetType, Op->OffsetScale);\n\n if (Op->Class == IR::RegClass::GPR) {\n const auto Dst = GetReg(Node);\n\n switch (OpSize) {\n case IR::OpSize::i8Bit: ldrb(Dst, MemSrc); break;\n case IR::OpSize::i16Bit: ldrh(Dst, MemSrc); break;\n case IR::OpSize::i32Bit: ldr(Dst.W(), MemSrc); break;\n case IR::OpSize::i64Bit: ldr(Dst.X(), MemSrc); break;\n default: LOGMAN_MSG_A_FMT(\"Unhandled LoadMem size: {}\", OpSize); break;\n }\n } else {\n const auto Dst = GetVReg(Node);\n\n switch (OpSize) {\n case IR::OpSize::i8Bit: ldrb(Dst, MemSrc); break;\n case IR::OpSize::i16Bit: ldrh(Dst, MemSrc); break;\n case IR::OpSize::i32Bit: ldr(Dst.S(), MemSrc); break;\n case IR::OpSize::i64Bit: ldr(Dst.D(), MemSrc); break;\n case IR::OpSize::i128Bit: ldr(Dst.Q(), MemSrc); break;\n case IR::OpSize::i256Bit: {\n LOGMAN_THROW_A_FMT(HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n const auto Operand = GenerateSVEMemOperand(OpSize, MemReg, Op->Offset, Op->OffsetType, Op->OffsetScale);\n ld1b(Dst.Z(), PRED_TMP_32B.Zeroing(), Operand);\n break;\n }\n default: LOGMAN_MSG_A_FMT(\"Unhandled LoadMem size: {}\", OpSize); break;\n }\n }\n}\n\nDEF_OP(LoadMemPair) {\n const auto Op = IROp->C();\n const auto Addr = GetReg(Op->Addr);\n\n if (Op->Class == IR::RegClass::GPR) {\n const auto Dst1 = GetReg(Op->OutValue1);\n const auto Dst2 = GetReg(Op->OutValue2);\n\n switch (IROp->Size) {\n case IR::OpSize::i32Bit: ldp(Dst1.W(), Dst2.W(), Addr, Op->Offset); break;\n case IR::OpSize::i64Bit: ldp(Dst1.X(), Dst2.X(), Addr, Op->Offset); break;\n default: LOGMAN_MSG_A_FMT(\"Unhandled LoadMemPair size: {}\", IROp->Size); break;\n }\n } else {\n const auto Dst1 = GetVReg(Op->OutValue1);\n const auto Dst2 = GetVReg(Op->OutValue2);\n\n switch (IROp->Size) {\n case IR::OpSize::i32Bit: ldp(Dst1.S(), Dst2.S(), Addr, Op->Offset); break;\n case IR::OpSize::i64Bit: ldp(Dst1.D(), Dst2.D(), Addr, Op->Offset); break;\n case IR::OpSize::i128Bit: ldp(Dst1.Q(), Dst2.Q(), Addr, Op->Offset); break;\n default: LOGMAN_MSG_A_FMT(\"Unhandled LoadMemPair size: {}\", IROp->Size); break;\n }\n }\n}\n\nDEF_OP(LoadMemTSO) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto MemReg = GetReg(Op->Addr);\n\n if (Op->Class == IR::RegClass::GPR) {\n LOGMAN_THROW_A_FMT(Op->Offset.IsInvalid() || CTX->HostFeatures.SupportsTSOImm9, \"unexpected offset\");\n LOGMAN_THROW_A_FMT(Op->OffsetScale == 1, \"unexpected offset scale\");\n LOGMAN_THROW_A_FMT(Op->OffsetType == IR::MemOffsetType::SXTX, \"unexpected offset type\");\n }\n\n if (CTX->HostFeatures.SupportsTSOImm9 && Op->Class == IR::RegClass::GPR) {\n const auto Dst = GetReg(Node);\n uint64_t Offset = 0;\n if (!Op->Offset.IsInvalid()) {\n bool IsInline = IsInlineConstant(Op->Offset, &Offset);\n LOGMAN_THROW_A_FMT(IsInline, \"expected immediate\");\n }\n\n if (OpSize == IR::OpSize::i8Bit) {\n // 8bit load is always aligned to natural alignment\n const auto Dst = GetReg(Node);\n ldapurb(Dst, MemReg, Offset);\n } else {\n switch (OpSize) {\n case IR::OpSize::i16Bit: ldapurh(Dst, MemReg, Offset); break;\n case IR::OpSize::i32Bit: ldapur(Dst.W(), MemReg, Offset); break;\n case IR::OpSize::i64Bit: ldapur(Dst.X(), MemReg, Offset); break;\n default: LOGMAN_MSG_A_FMT(\"Unhandled LoadMemTSO size: {}\", OpSize); break;\n }\n // Half-barrier once back-patched.\n nop();\n }\n } else if (CTX->HostFeatures.SupportsRCPC && Op->Class == IR::RegClass::GPR) {\n const auto Dst = GetReg(Node);\n if (OpSize == IR::OpSize::i8Bit) {\n // 8bit load is always aligned to natural alignment\n ldaprb(Dst.W(), MemReg);\n } else {\n switch (OpSize) {\n case IR::OpSize::i16Bit: ldaprh(Dst.W(), MemReg); break;\n case IR::OpSize::i32Bit: ldapr(Dst.W(), MemReg); break;\n case IR::OpSize::i64Bit: ldapr(Dst.X(), MemReg); break;\n default: LOGMAN_MSG_A_FMT(\"Unhandled LoadMemTSO size: {}\", OpSize); break;\n }\n // Half-barrier once back-patched.\n nop();\n }\n } else if (Op->Class == IR::RegClass::GPR) {\n const auto Dst = GetReg(Node);\n if (OpSize == IR::OpSize::i8Bit) {\n // 8bit load is always aligned to natural alignment\n ldarb(Dst, MemReg);\n } else {\n switch (OpSize) {\n case IR::OpSize::i16Bit: ldarh(Dst, MemReg); break;\n case IR::OpSize::i32Bit: ldar(Dst.W(), MemReg); break;\n case IR::OpSize::i64Bit: ldar(Dst.X(), MemReg); break;\n default: LOGMAN_MSG_A_FMT(\"Unhandled LoadMemTSO size: {}\", OpSize); break;\n }\n // Half-barrier once back-patched.\n nop();\n }\n } else {\n const auto Dst = GetVReg(Node);\n const auto MemSrc = GenerateMemOperand(OpSize, MemReg, Op->Offset, Op->OffsetType, Op->OffsetScale);\n switch (OpSize) {\n case IR::OpSize::i8Bit: ldrb(Dst, MemSrc); break;\n case IR::OpSize::i16Bit: ldrh(Dst, MemSrc); break;\n case IR::OpSize::i32Bit: ldr(Dst.S(), MemSrc); break;\n case IR::OpSize::i64Bit: ldr(Dst.D(), MemSrc); break;\n case IR::OpSize::i128Bit: ldr(Dst.Q(), MemSrc); break;\n case IR::OpSize::i256Bit: {\n LOGMAN_THROW_A_FMT(HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n const auto MemSrc = GenerateSVEMemOperand(OpSize, MemReg, Op->Offset, Op->OffsetType, Op->OffsetScale);\n ld1b(Dst.Z(), PRED_TMP_32B.Zeroing(), MemSrc);\n break;\n }\n default: LOGMAN_MSG_A_FMT(\"Unhandled LoadMemTSO size: {}\", OpSize); break;\n }\n if (CTX->IsVectorAtomicTSOEnabled()) {\n // Half-barrier.\n dmb(ARMEmitter::BarrierScope::ISHLD);\n }\n }\n}\n\nDEF_OP(VLoadVectorMasked) {\n\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n const auto SubRegSize = ConvertSubRegSize8(IROp);\n\n const auto CMPPredicate = ARMEmitter::PReg::p0;\n const auto GoverningPredicate = Is256Bit ? PRED_TMP_32B : PRED_TMP_16B;\n\n const auto Dst = GetVReg(Node);\n const auto MaskReg = GetVReg(Op->Mask);\n const auto MemReg = GetReg(Op->Addr);\n\n if (HostSupportsSVE128 || HostSupportsSVE256) {\n const auto MemSrc = GenerateSVEMemOperand(OpSize, MemReg, Op->Offset, Op->OffsetType, Op->OffsetScale);\n\n // Check if the sign bit is set for the given element size.\n cmplt(SubRegSize, CMPPredicate, GoverningPredicate.Zeroing(), MaskReg.Z(), 0);\n\n switch (IROp->ElementSize) {\n case IR::OpSize::i8Bit: {\n ld1b(Dst.Z(), CMPPredicate.Zeroing(), MemSrc);\n break;\n }\n case IR::OpSize::i16Bit: {\n ld1h(Dst.Z(), CMPPredicate.Zeroing(), MemSrc);\n break;\n }\n case IR::OpSize::i32Bit: {\n ld1w(Dst.Z(), CMPPredicate.Zeroing(), MemSrc);\n break;\n }\n case IR::OpSize::i64Bit: {\n ld1d(Dst.Z(), CMPPredicate.Zeroing(), MemSrc);\n break;\n }\n default: break;\n }\n } else {\n const auto PerformMove = [this](IR::OpSize ElementSize, const ARMEmitter::Register Dst, const ARMEmitter::VRegister Vector, int index) {\n switch (ElementSize) {\n case IR::OpSize::i8Bit: umov(Dst, Vector, index); break;\n case IR::OpSize::i16Bit: umov(Dst, Vector, index); break;\n case IR::OpSize::i32Bit: umov(Dst, Vector, index); break;\n case IR::OpSize::i64Bit: umov(Dst, Vector, index); break;\n default: LOGMAN_MSG_A_FMT(\"Unhandled ExtractElementSize: {}\", ElementSize); break;\n }\n };\n\n // Prepare yourself adventurer. For a masked load without instructions that implement it.\n LOGMAN_THROW_A_FMT(OpSize == IR::OpSize::i128Bit, \"Only supports 128-bit without SVE256\");\n size_t NumElements = IR::NumElements(IROp->Size, IROp->ElementSize);\n\n // Use VTMP1 as the temporary destination\n auto TempDst = VTMP1;\n auto WorkingReg = TMP1;\n auto TempMemReg = MemReg;\n movi(ARMEmitter::SubRegSize::i64Bit, TempDst.Q(), 0);\n uint64_t Const {};\n if (Op->Offset.IsInvalid()) {\n // Intentional no-op.\n } else if (IsInlineConstant(Op->Offset, &Const)) {\n TempMemReg = TMP2;\n add(ARMEmitter::Size::i64Bit, TMP2, MemReg, Const);\n } else {\n LOGMAN_MSG_A_FMT(\"Complex addressing requested and not supported!\");\n }\n\n const uint64_t ElementSizeInBits = IR::OpSizeAsBits(IROp->ElementSize);\n for (size_t i = 0; i < NumElements; ++i) {\n // Extract the mask element.\n PerformMove(IROp->ElementSize, WorkingReg, MaskReg, i);\n\n // If the sign bit is zero then skip the load\n ARMEmitter::ForwardLabel Skip {};\n (void)tbz(WorkingReg, ElementSizeInBits - 1, &Skip);\n // Do the gather load for this element into the destination\n switch (IROp->ElementSize) {\n case IR::OpSize::i8Bit: ld1(TempDst.Q(), i, TempMemReg); break;\n case IR::OpSize::i16Bit: ld1(TempDst.Q(), i, TempMemReg); break;\n case IR::OpSize::i32Bit: ld1(TempDst.Q(), i, TempMemReg); break;\n case IR::OpSize::i64Bit: ld1(TempDst.Q(), i, TempMemReg); break;\n case IR::OpSize::i128Bit: ldr(TempDst.Q(), TempMemReg, 0); break;\n default: LOGMAN_MSG_A_FMT(\"Unhandled {} size: {}\", __func__, IROp->ElementSize); return;\n }\n\n (void)Bind(&Skip);\n\n if ((i + 1) != NumElements) {\n // Handle register rename to save a move.\n auto WorkingReg = TempMemReg;\n TempMemReg = TMP2;\n add(ARMEmitter::Size::i64Bit, TempMemReg, WorkingReg, IR::OpSizeToSize(IROp->ElementSize));\n }\n }\n\n // Move result.\n mov(Dst.Q(), TempDst.Q());\n }\n}\n\nDEF_OP(VStoreVectorMasked) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n const auto SubRegSize = ConvertSubRegSize8(IROp);\n\n const auto CMPPredicate = ARMEmitter::PReg::p0;\n const auto GoverningPredicate = Is256Bit ? PRED_TMP_32B : PRED_TMP_16B;\n\n const auto RegData = GetVReg(Op->Data);\n const auto MaskReg = GetVReg(Op->Mask);\n const auto MemReg = GetReg(Op->Addr);\n if (HostSupportsSVE128 || HostSupportsSVE256) {\n const auto MemDst = GenerateSVEMemOperand(OpSize, MemReg, Op->Offset, Op->OffsetType, Op->OffsetScale);\n\n // Check if the sign bit is set for the given element size.\n cmplt(SubRegSize, CMPPredicate, GoverningPredicate.Zeroing(), MaskReg.Z(), 0);\n\n switch (IROp->ElementSize) {\n case IR::OpSize::i8Bit: {\n st1b(RegData.Z(), CMPPredicate.Zeroing(), MemDst);\n break;\n }\n case IR::OpSize::i16Bit: {\n st1h(RegData.Z(), CMPPredicate.Zeroing(), MemDst);\n break;\n }\n case IR::OpSize::i32Bit: {\n st1w(RegData.Z(), CMPPredicate.Zeroing(), MemDst);\n break;\n }\n case IR::OpSize::i64Bit: {\n st1d(RegData.Z(), CMPPredicate.Zeroing(), MemDst);\n break;\n }\n default: break;\n }\n } else {\n const auto PerformMove = [this](IR::OpSize ElementSize, const ARMEmitter::Register Dst, const ARMEmitter::VRegister Vector, int index) {\n switch (ElementSize) {\n case IR::OpSize::i8Bit: umov(Dst, Vector, index); break;\n case IR::OpSize::i16Bit: umov(Dst, Vector, index); break;\n case IR::OpSize::i32Bit: umov(Dst, Vector, index); break;\n case IR::OpSize::i64Bit: umov(Dst, Vector, index); break;\n default: LOGMAN_MSG_A_FMT(\"Unhandled ExtractElementSize: {}\", ElementSize); break;\n }\n };\n\n // Prepare yourself adventurer. For a masked store without instructions that implement it.\n LOGMAN_THROW_A_FMT(OpSize == IR::OpSize::i128Bit, \"Only supports 128-bit without SVE256\");\n size_t NumElements = IR::NumElements(IROp->Size, IROp->ElementSize);\n\n // Use VTMP1 as the temporary destination\n auto WorkingReg = TMP1;\n auto TempMemReg = MemReg;\n\n uint64_t Const {};\n if (Op->Offset.IsInvalid()) {\n // Intentional no-op.\n } else if (IsInlineConstant(Op->Offset, &Const)) {\n TempMemReg = TMP2;\n add(ARMEmitter::Size::i64Bit, TMP2, MemReg, Const);\n } else {\n LOGMAN_MSG_A_FMT(\"Complex addressing requested and not supported!\");\n }\n\n const uint64_t ElementSizeInBits = IR::OpSizeAsBits(IROp->ElementSize);\n for (size_t i = 0; i < NumElements; ++i) {\n // Extract the mask element.\n PerformMove(IROp->ElementSize, WorkingReg, MaskReg, i);\n\n // If the sign bit is zero then skip the load\n ARMEmitter::ForwardLabel Skip {};\n (void)tbz(WorkingReg, ElementSizeInBits - 1, &Skip);\n // Do the gather load for this element into the destination\n switch (IROp->ElementSize) {\n case IR::OpSize::i8Bit: st1(RegData.Q(), i, TempMemReg); break;\n case IR::OpSize::i16Bit: st1(RegData.Q(), i, TempMemReg); break;\n case IR::OpSize::i32Bit: st1(RegData.Q(), i, TempMemReg); break;\n case IR::OpSize::i64Bit: st1(RegData.Q(), i, TempMemReg); break;\n case IR::OpSize::i128Bit: str(RegData.Q(), TempMemReg, 0); break;\n default: LOGMAN_MSG_A_FMT(\"Unhandled {} size: {}\", __func__, IROp->ElementSize); return;\n }\n\n (void)Bind(&Skip);\n\n if ((i + 1) != NumElements) {\n // Handle register rename to save a move.\n auto WorkingReg = TempMemReg;\n TempMemReg = TMP2;\n add(ARMEmitter::Size::i64Bit, TempMemReg, WorkingReg, IR::OpSizeToSize(IROp->ElementSize));\n }\n }\n }\n}\n\nvoid Arm64JITCore::Emulate128BitGather(IR::OpSize Size, IR::OpSize ElementSize, ARMEmitter::VRegister Dst,\n ARMEmitter::VRegister IncomingDst, std::optional BaseAddr,\n ARMEmitter::VRegister VectorIndexLow, std::optional VectorIndexHigh,\n ARMEmitter::VRegister MaskReg, IR::OpSize VectorIndexSize, size_t DataElementOffsetStart,\n size_t IndexElementOffsetStart, uint8_t OffsetScale, IR::OpSize AddrSize) {\n LOGMAN_THROW_A_FMT(ElementSize >= IR::OpSize::i8Bit && ElementSize <= IR::OpSize::i64Bit, \"Invalid element size\");\n\n const auto PerformSMove = [this](IR::OpSize ElementSize, const ARMEmitter::Register Dst, const ARMEmitter::VRegister Vector, int index) {\n switch (ElementSize) {\n case IR::OpSize::i8Bit: smov(Dst.X(), Vector, index); break;\n case IR::OpSize::i16Bit: smov(Dst.X(), Vector, index); break;\n case IR::OpSize::i32Bit: smov(Dst.X(), Vector, index); break;\n case IR::OpSize::i64Bit: umov(Dst.X(), Vector, index); break;\n default: LOGMAN_MSG_A_FMT(\"Unhandled ExtractElementSize: {}\", ElementSize); break;\n }\n };\n\n const auto PerformMove = [this](IR::OpSize ElementSize, const ARMEmitter::Register Dst, const ARMEmitter::VRegister Vector, int index) {\n switch (ElementSize) {\n case IR::OpSize::i8Bit: umov(Dst, Vector, index); break;\n case IR::OpSize::i16Bit: umov(Dst, Vector, index); break;\n case IR::OpSize::i32Bit: umov(Dst, Vector, index); break;\n case IR::OpSize::i64Bit: umov(Dst, Vector, index); break;\n default: LOGMAN_MSG_A_FMT(\"Unhandled ExtractElementSize: {}\", ElementSize); break;\n }\n };\n\n // FEX needs to use a temporary destination vector register in a couple of instances.\n // When Dst overlaps MaskReg, VectorIndexLow, or VectorIndexHigh\n // Due to x86 gather instruction limitations, it is highly likely that a destination temporary isn't required.\n const bool NeedsDestTmp = Dst == MaskReg || Dst == VectorIndexLow || (VectorIndexHigh.has_value() && Dst == *VectorIndexHigh);\n\n // If the incoming destination isn't the destination then we need to move.\n const bool NeedsIncomingDestMove = Dst != IncomingDst || NeedsDestTmp;\n\n ///< Adventurers beware, emulated ASIMD style gather masked load operation.\n // Number of elements to load is calculated by the number of index elements available.\n size_t NumAddrElements = (VectorIndexHigh.has_value() ? 32 : 16) / IR::OpSizeToSize(VectorIndexSize);\n // The number of elements is clamped by the resulting register size.\n size_t NumDataElements = std::min(IR::OpSizeToSize(Size) / IR::OpSizeToSize(ElementSize), NumAddrElements);\n\n size_t IndexElementsSizeBytes = NumAddrElements * IR::OpSizeToSize(VectorIndexSize);\n if (IndexElementsSizeBytes > 16) {\n // We must have a high register in this case.\n LOGMAN_THROW_A_FMT(VectorIndexHigh.has_value(), \"Need High vector index register!\");\n }\n\n auto ResultReg = Dst;\n if (NeedsDestTmp) {\n // Use VTMP1 as the temporary destination\n ResultReg = VTMP1;\n }\n auto WorkingReg = TMP1;\n auto TempMemReg = TMP2;\n const uint64_t ElementSizeInBits = IR::OpSizeToSize(ElementSize) * 8;\n\n if (NeedsIncomingDestMove) {\n mov(ResultReg.Q(), IncomingDst.Q());\n }\n\n for (size_t i = DataElementOffsetStart, IndexElement = IndexElementOffsetStart; i < NumDataElements; ++i, ++IndexElement) {\n ARMEmitter::ForwardLabel Skip {};\n // Extract mask element\n PerformMove(ElementSize, WorkingReg, MaskReg, i);\n\n // Skip if the mask's sign bit isn't set\n (void)tbz(WorkingReg, ElementSizeInBits - 1, &Skip);\n\n // Extract Index Element\n if ((IndexElement * IR::OpSizeToSize(VectorIndexSize)) >= 16) {\n // Fetch from the high index register.\n PerformSMove(VectorIndexSize, WorkingReg, *VectorIndexHigh, IndexElement - (16 / IR::OpSizeToSize(VectorIndexSize)));\n } else {\n // Fetch from the low index register.\n PerformSMove(VectorIndexSize, WorkingReg, VectorIndexLow, IndexElement);\n }\n\n // Calculate memory position for this gather load\n if (BaseAddr.has_value()) {\n if (VectorIndexSize == IR::OpSize::i32Bit) {\n add(ConvertSize(AddrSize), TempMemReg, *BaseAddr, WorkingReg, ARMEmitter::ExtendedType::SXTW, FEXCore::ilog2(OffsetScale));\n } else {\n add(ConvertSize(AddrSize), TempMemReg, *BaseAddr, WorkingReg, ARMEmitter::ShiftType::LSL, FEXCore::ilog2(OffsetScale));\n }\n } else {\n ///< In this case we have no base address, All addresses come from the vector register itself\n if (VectorIndexSize == IR::OpSize::i32Bit) {\n // Sign extend and shift in to the 64-bit register\n sbfiz(ConvertSize(AddrSize), TempMemReg, WorkingReg, FEXCore::ilog2(OffsetScale), 32);\n } else {\n lsl(ConvertSize(AddrSize), TempMemReg, WorkingReg, FEXCore::ilog2(OffsetScale));\n }\n }\n\n // Now that the address is calculated. Do the load.\n switch (ElementSize) {\n case IR::OpSize::i8Bit: ld1(ResultReg.Q(), i, TempMemReg); break;\n case IR::OpSize::i16Bit: ld1(ResultReg.Q(), i, TempMemReg); break;\n case IR::OpSize::i32Bit: ld1(ResultReg.Q(), i, TempMemReg); break;\n case IR::OpSize::i64Bit: ld1(ResultReg.Q(), i, TempMemReg); break;\n case IR::OpSize::i128Bit: ldr(ResultReg.Q(), TempMemReg, 0); break;\n default: LOGMAN_MSG_A_FMT(\"Unhandled {} size: {}\", __func__, ElementSize); FEX_UNREACHABLE;\n }\n\n (void)Bind(&Skip);\n }\n\n if (NeedsDestTmp) {\n // Move result.\n mov(Dst.Q(), ResultReg.Q());\n }\n}\n\nDEF_OP(VLoadVectorGatherMasked) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto VectorIndexSize = Op->VectorIndexElementSize;\n const auto OffsetScale = Op->OffsetScale;\n const auto DataElementOffsetStart = Op->DataElementOffsetStart;\n const auto IndexElementOffsetStart = Op->IndexElementOffsetStart;\n\n ///< This IR operation handles discontiguous masked gather loadstore instructions. Some things to note about its behaviour.\n /// - VSIB behaviour is mostly entirely exposed in the IR operation directly.\n /// - Displacement is the only value missing as that can be added directly to AddrBase.\n /// - VectorIndex{Low,High} contains the index offsets for each element getting loaded.\n /// - These element sizes are decoupled from the resulting element size. These can be 32-bit or 64-bit.\n /// - When the element size is 32-bit then the value is zero-extended to the full 64-bit address calculation\n /// - When loading a 128-bit result with 64-bit VectorIndex Elements, this requires the use of both VectorIndexLow and VectorIndexHigh\n /// to get enough pointers.\n /// - When VectorIndexElementSize and OffsetScale matches Arm64 SVE behaviour then the operation becomes more optimal\n /// - When the behaviour doesn't match then it gets decomposed to ASIMD style masked load.\n /// - AddrBase also doesn't need to exist\n /// - If the instruction is using 64-bit vector indexing or 32-bit addresses where the top-bit isn't set then this is valid!\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto Dst = GetVReg(Node);\n const auto IncomingDst = GetVReg(Op->Incoming);\n\n const auto MaskReg = GetVReg(Op->Mask);\n std::optional BaseAddr = !Op->AddrBase.IsInvalid() ? std::make_optional(GetReg(Op->AddrBase)) : std::nullopt;\n const auto VectorIndexLow = GetVReg(Op->VectorIndexLow);\n std::optional VectorIndexHigh =\n !Op->VectorIndexHigh.IsInvalid() ? std::make_optional(GetVReg(Op->VectorIndexHigh)) : std::nullopt;\n\n ///< If the host supports SVE and the offset scale matches SVE limitations then it can do an SVE style load.\n const bool SupportsSVELoad = (HostSupportsSVE128 || HostSupportsSVE256) &&\n (OffsetScale == 1 || OffsetScale == IR::OpSizeToSize(VectorIndexSize)) &&\n VectorIndexSize == IROp->ElementSize && Op->AddrSize == IR::OpSize::i64Bit;\n\n if (SupportsSVELoad) {\n uint8_t SVEScale = FEXCore::ilog2(OffsetScale);\n ARMEmitter::SVEModType ModType = ARMEmitter::SVEModType::MOD_NONE;\n if (VectorIndexSize == IR::OpSize::i32Bit) {\n ModType = ARMEmitter::SVEModType::MOD_SXTW;\n } else if (VectorIndexSize == IR::OpSize::i64Bit && OffsetScale != 1) {\n ModType = ARMEmitter::SVEModType::MOD_LSL;\n }\n\n const auto SubRegSize = ConvertSubRegSize8(IROp);\n\n const auto CMPPredicate = ARMEmitter::PReg::p0;\n const auto GoverningPredicate = Is256Bit ? PRED_TMP_32B : PRED_TMP_16B;\n\n // Check if the sign bit is set for the given element size.\n cmplt(SubRegSize, CMPPredicate, GoverningPredicate.Zeroing(), MaskReg.Z(), 0);\n auto TempDst = VTMP1;\n\n // No need to load a temporary register in the case that we weren't provided a base address and there is no scaling.\n ARMEmitter::SVEMemOperand MemDst {ARMEmitter::SVEMemOperand(VectorIndexLow.Z(), 0)};\n if (BaseAddr.has_value() || OffsetScale != 1) {\n ARMEmitter::Register AddrReg = TMP1;\n if (BaseAddr.has_value()) {\n AddrReg = GetReg(Op->AddrBase);\n } else {\n ///< OpcodeDispatcher didn't provide a Base address while SVE requires one.\n LoadConstant(ARMEmitter::Size::i64Bit, AddrReg, 0);\n }\n MemDst = ARMEmitter::SVEMemOperand(AddrReg.X(), VectorIndexLow.Z(), ModType, SVEScale);\n }\n\n switch (IROp->ElementSize) {\n case IR::OpSize::i8Bit: {\n ld1b(TempDst.Z(), CMPPredicate.Zeroing(), MemDst);\n break;\n }\n case IR::OpSize::i16Bit: {\n ld1h(TempDst.Z(), CMPPredicate.Zeroing(), MemDst);\n break;\n }\n case IR::OpSize::i32Bit: {\n ld1w(TempDst.Z(), CMPPredicate.Zeroing(), MemDst);\n break;\n }\n case IR::OpSize::i64Bit: {\n ld1d(TempDst.Z(), CMPPredicate.Zeroing(), MemDst);\n break;\n }\n default: break;\n }\n\n ///< Merge elements based on predicate.\n sel(SubRegSize, Dst.Z(), CMPPredicate, TempDst.Z(), IncomingDst.Z());\n } else {\n LOGMAN_THROW_A_FMT(!Is256Bit, \"Can't emulate this gather load in the backend! Programming error!\");\n Emulate128BitGather(IROp->Size, IROp->ElementSize, Dst, IncomingDst, BaseAddr, VectorIndexLow, VectorIndexHigh, MaskReg,\n VectorIndexSize, DataElementOffsetStart, IndexElementOffsetStart, OffsetScale, Op->AddrSize);\n }\n}\n\nDEF_OP(VLoadVectorGatherMaskedQPS) {\n const auto Op = IROp->C();\n\n /// This instruction behaves similarly to the non-QPS version except for some STRICT limitations\n /// - Only supports 32-bit element data size!\n /// - Only supports 64-bit element address size!\n /// - Only masks elements based on 32-bit element data size! (NOT ADDR SIZE!)\n /// - Optimally uses SVE's `ld1w {zt.D}` variant instruction!\n /// - Only outputs a single 128-bit result, while consuming 128-bit or 256-bit of address indexes!\n /// - Matches VGATHERQPS/VPGATHERQD behaviour!\n const auto OffsetScale = Op->OffsetScale;\n const auto Dst = GetVReg(Node);\n const auto IncomingDst = GetVReg(Op->Incoming);\n\n const auto MaskReg = GetVReg(Op->MaskReg);\n std::optional BaseAddr = !Op->AddrBase.IsInvalid() ? std::make_optional(GetReg(Op->AddrBase)) : std::nullopt;\n const auto VectorIndexLow = GetVReg(Op->VectorIndexLow);\n std::optional VectorIndexHigh =\n !Op->VectorIndexHigh.IsInvalid() ? std::make_optional(GetVReg(Op->VectorIndexHigh)) : std::nullopt;\n\n ///< If the host supports SVE and the offset scale matches SVE limitations then it can do an SVE style load.\n const bool SupportsSVELoad = HostSupportsSVE128 && (OffsetScale == 1 || OffsetScale == 4) && Op->AddrSize == IR::OpSize::i64Bit;\n\n if (SupportsSVELoad) {\n ARMEmitter::SVEModType ModType = ARMEmitter::SVEModType::MOD_NONE;\n if (OffsetScale != 1) {\n ModType = ARMEmitter::SVEModType::MOD_LSL;\n }\n\n const auto CMPPredicate = ARMEmitter::PReg::p0;\n const auto CMPPredicate2 = ARMEmitter::PReg::p1;\n\n const auto GoverningPredicate = PRED_TMP_16B;\n\n // Check if the sign bit is set for the given element size.\n // This will set the predicate bits for elements [0, 1, 2, 3]\n // We then use punpklo to extend the low results to be for 64-bit elements.\n cmplt(ARMEmitter::SubRegSize::i32Bit, CMPPredicate, GoverningPredicate.Zeroing(), MaskReg.Z(), 0);\n punpklo(CMPPredicate2, CMPPredicate);\n auto TempDst = VTMP1;\n\n auto GatherExtend = [this](ARMEmitter::VRegister Dst, std::optional BaseAddr, ARMEmitter::VRegister VectorIndex,\n ARMEmitter::PRegister CMPPredicate, ARMEmitter::SVEModType ModType, uint8_t OffsetScale) {\n // No need to load a temporary register in the case that we weren't provided a base address and there is no scaling.\n uint8_t SVEScale = FEXCore::ilog2(OffsetScale);\n ARMEmitter::SVEMemOperand MemDst {ARMEmitter::SVEMemOperand(VectorIndex.Z(), 0)};\n if (BaseAddr.has_value() || OffsetScale != 1) {\n ARMEmitter::Register AddrReg = TMP1;\n if (BaseAddr.has_value()) {\n AddrReg = *BaseAddr;\n } else {\n ///< OpcodeDispatcher didn't provide a Base address while SVE requires one.\n LoadConstant(ARMEmitter::Size::i64Bit, AddrReg, 0);\n }\n MemDst = ARMEmitter::SVEMemOperand(AddrReg.X(), VectorIndex.Z(), ModType, SVEScale);\n }\n\n ld1w(Dst.Z(), CMPPredicate.Zeroing(), MemDst);\n };\n\n GatherExtend(TempDst, BaseAddr, VectorIndexLow, CMPPredicate2, ModType, OffsetScale);\n\n if (VectorIndexHigh.has_value()) {\n punpkhi(CMPPredicate2, CMPPredicate);\n GatherExtend(VTMP2, BaseAddr, *VectorIndexHigh, CMPPredicate2, ModType, OffsetScale);\n // Move elements to the lower half.\n uzp1(ARMEmitter::SubRegSize::i32Bit, TempDst.Q(), TempDst.Q(), VTMP2.Q());\n ///< Merge elements based on predicate.\n sel(ARMEmitter::SubRegSize::i32Bit, Dst.Z(), CMPPredicate, TempDst.Z(), IncomingDst.Z());\n } else {\n // Move elements to the lower half.\n xtn(ARMEmitter::SubRegSize::i32Bit, TempDst.Q(), TempDst.Q());\n ///< Merge elements based on predicate.\n sel(ARMEmitter::SubRegSize::i32Bit, Dst.Z(), CMPPredicate, TempDst.Z(), IncomingDst.Z());\n }\n } else {\n Emulate128BitGather(IR::OpSize::i128Bit, IR::OpSize::i32Bit, Dst, IncomingDst, BaseAddr, VectorIndexLow, VectorIndexHigh, MaskReg,\n IR::OpSize::i64Bit, 0, 0, OffsetScale, Op->AddrSize);\n }\n}\n\nDEF_OP(VLoadVectorElement) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n const auto ElementSize = IROp->ElementSize;\n\n const auto Dst = GetVReg(Node);\n const auto DstSrc = GetVReg(Op->DstSrc);\n const auto MemReg = GetReg(Op->Addr);\n\n LOGMAN_THROW_A_FMT(ElementSize == IR::OpSize::i8Bit || ElementSize == IR::OpSize::i16Bit || ElementSize == IR::OpSize::i32Bit ||\n ElementSize == IR::OpSize::i64Bit || ElementSize == IR::OpSize::i128Bit,\n \"Invalid element \"\n \"size\");\n\n if (Is256Bit) {\n LOGMAN_MSG_A_FMT(\"Unsupported 256-bit VLoadVectorElement\");\n } else {\n if (Dst != DstSrc && ElementSize != IR::OpSize::i128Bit) {\n mov(Dst.Q(), DstSrc.Q());\n }\n switch (ElementSize) {\n case IR::OpSize::i8Bit: ld1(Dst.Q(), Op->Index, MemReg); break;\n case IR::OpSize::i16Bit: ld1(Dst.Q(), Op->Index, MemReg); break;\n case IR::OpSize::i32Bit: ld1(Dst.Q(), Op->Index, MemReg); break;\n case IR::OpSize::i64Bit: ld1(Dst.Q(), Op->Index, MemReg); break;\n case IR::OpSize::i128Bit: ldr(Dst.Q(), MemReg); break;\n default: LOGMAN_MSG_A_FMT(\"Unhandled {} size: {}\", __func__, ElementSize); return;\n }\n }\n\n // Emit a half-barrier if TSO is enabled.\n if (CTX->IsVectorAtomicTSOEnabled()) {\n dmb(ARMEmitter::BarrierScope::ISHLD);\n }\n}\n\nDEF_OP(VStoreVectorElement) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n const auto ElementSize = IROp->ElementSize;\n\n const auto Value = GetVReg(Op->Value);\n const auto MemReg = GetReg(Op->Addr);\n\n LOGMAN_THROW_A_FMT(ElementSize == IR::OpSize::i8Bit || ElementSize == IR::OpSize::i16Bit || ElementSize == IR::OpSize::i32Bit ||\n ElementSize == IR::OpSize::i64Bit || ElementSize == IR::OpSize::i128Bit,\n \"Invalid element \"\n \"size\");\n\n // Emit a half-barrier if TSO is enabled.\n if (CTX->IsVectorAtomicTSOEnabled()) {\n dmb(ARMEmitter::BarrierScope::ISH);\n }\n\n if (Is256Bit) {\n LOGMAN_MSG_A_FMT(\"Unsupported 256-bit {}\", __func__);\n } else {\n switch (ElementSize) {\n case IR::OpSize::i8Bit: st1(Value.Q(), Op->Index, MemReg); break;\n case IR::OpSize::i16Bit: st1(Value.Q(), Op->Index, MemReg); break;\n case IR::OpSize::i32Bit: st1(Value.Q(), Op->Index, MemReg); break;\n case IR::OpSize::i64Bit: st1(Value.Q(), Op->Index, MemReg); break;\n case IR::OpSize::i128Bit: str(Value.Q(), MemReg); break;\n default: LOGMAN_MSG_A_FMT(\"Unhandled {} size: {}\", __func__, ElementSize); return;\n }\n }\n}\n\nDEF_OP(VBroadcastFromMem) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n const auto ElementSize = IROp->ElementSize;\n\n const auto Dst = GetVReg(Node);\n const auto MemReg = GetReg(Op->Address);\n\n LOGMAN_THROW_A_FMT(ElementSize == IR::OpSize::i8Bit || ElementSize == IR::OpSize::i16Bit || ElementSize == IR::OpSize::i32Bit ||\n ElementSize == IR::OpSize::i64Bit || ElementSize == IR::OpSize::i128Bit,\n \"Invalid element \"\n \"size\");\n\n if (Is256Bit && HostSupportsSVE256) {\n const auto GoverningPredicate = PRED_TMP_32B.Zeroing();\n\n switch (ElementSize) {\n case IR::OpSize::i8Bit: ld1rb(ARMEmitter::SubRegSize::i8Bit, Dst.Z(), GoverningPredicate, MemReg); break;\n case IR::OpSize::i16Bit: ld1rh(ARMEmitter::SubRegSize::i16Bit, Dst.Z(), GoverningPredicate, MemReg); break;\n case IR::OpSize::i32Bit: ld1rw(ARMEmitter::SubRegSize::i32Bit, Dst.Z(), GoverningPredicate, MemReg); break;\n case IR::OpSize::i64Bit: ld1rd(Dst.Z(), GoverningPredicate, MemReg); break;\n case IR::OpSize::i128Bit: ld1rqb(Dst.Z(), GoverningPredicate, MemReg); break;\n default: LOGMAN_MSG_A_FMT(\"Unhandled VBroadcastFromMem size: {}\", ElementSize); return;\n }\n } else {\n switch (ElementSize) {\n case IR::OpSize::i8Bit: ld1r(Dst.Q(), MemReg); break;\n case IR::OpSize::i16Bit: ld1r(Dst.Q(), MemReg); break;\n case IR::OpSize::i32Bit: ld1r(Dst.Q(), MemReg); break;\n case IR::OpSize::i64Bit: ld1r(Dst.Q(), MemReg); break;\n case IR::OpSize::i128Bit:\n // Normal load, like ld1rqb with 128-bit regs.\n ldr(Dst.Q(), MemReg);\n break;\n default: LOGMAN_MSG_A_FMT(\"Unhandled VBroadcastFromMem size: {}\", ElementSize); return;\n }\n }\n\n // Emit a half-barrier if TSO is enabled.\n if (CTX->IsVectorAtomicTSOEnabled()) {\n dmb(ARMEmitter::BarrierScope::ISHLD);\n }\n}\n\nDEF_OP(Push) {\n const auto Op = IROp->C();\n const auto ValueSize = IR::OpSizeToSize(Op->ValueSize);\n auto Src = GetReg(Op->Value);\n const auto AddrSrc = GetReg(Op->Addr);\n const auto Dst = GetReg(Node);\n\n bool NeedsMoveAfterwards = false;\n if (Dst != AddrSrc) {\n if (Dst == Src) {\n NeedsMoveAfterwards = true;\n // Need to be careful here, incoming source might be reused afterwards.\n } else {\n // RA constraints would let this always be true.\n mov(IROp->Size == IR::OpSize::i64Bit ? ARMEmitter::Size::i64Bit : ARMEmitter::Size::i32Bit, Dst, AddrSrc);\n }\n }\n\n if (Src == AddrSrc) {\n // If the data source is the address source then we need to do some additional work.\n // This is because it is undefined behaviour to do a writeback on store operation where dest == src.\n // In the case of writeback where the source is the address there are multiple behaviours.\n // - SIGILL - Apple Silicon Behaviour\n // - Stores original value - Cortex behaviour\n // - Stores value after pre-index adjust adjust - Vixl simulator behaviour.\n // - Undefined value stored\n // - Undefined behaviour(!)\n\n // In this path Src can end up overlapping both AddrSrc and Dst.\n // Move the data to a temporary and store from there instead.\n mov(TMP1, Src.X());\n Src = TMP1;\n }\n\n if (NeedsMoveAfterwards) {\n switch (ValueSize) {\n case 1: {\n sturb(Src.W(), AddrSrc, -ValueSize);\n break;\n }\n case 2: {\n sturh(Src.W(), AddrSrc, -ValueSize);\n break;\n }\n case 4: {\n stur(Src.W(), AddrSrc, -ValueSize);\n break;\n }\n case 8: {\n stur(Src.X(), AddrSrc, -ValueSize);\n break;\n }\n default: {\n LOGMAN_MSG_A_FMT(\"Unhandled {} size: {}\", __func__, ValueSize);\n break;\n }\n }\n\n sub(IROp->Size == IR::OpSize::i64Bit ? ARMEmitter::Size::i64Bit : ARMEmitter::Size::i32Bit, Dst, AddrSrc, ValueSize);\n } else {\n switch (ValueSize) {\n case 1: {\n strb(Src.W(), Dst, -ValueSize);\n break;\n }\n case 2: {\n strh(Src.W(), Dst, -ValueSize);\n break;\n }\n case 4: {\n str(Src.W(), Dst, -ValueSize);\n break;\n }\n case 8: {\n str(Src.X(), Dst, -ValueSize);\n break;\n }\n default: {\n LOGMAN_MSG_A_FMT(\"Unhandled {} size: {}\", __func__, ValueSize);\n break;\n }\n }\n }\n}\n\nDEF_OP(PushTwo) {\n const auto Op = IROp->C();\n const auto ValueSize = IR::OpSizeToSize(Op->ValueSize);\n auto Src1 = GetReg(Op->Value1);\n auto Src2 = GetReg(Op->Value2);\n const auto Dst = GetReg(Op->Addr);\n\n switch (ValueSize) {\n case 4: {\n stp(Src1.W(), Src2.W(), Dst, -2 * ValueSize);\n break;\n }\n case 8: {\n stp(Src1.X(), Src2.X(), Dst, -2 * ValueSize);\n break;\n }\n default: {\n LOGMAN_MSG_A_FMT(\"Unhandled {} size: {}\", __func__, ValueSize);\n break;\n }\n }\n}\n\nDEF_OP(Pop) {\n const auto Op = IROp->C();\n const auto Size = IR::OpSizeToSize(Op->Size);\n const auto Addr = GetReg(Op->InoutAddr);\n const auto Dst = GetReg(Op->OutValue);\n\n LOGMAN_THROW_A_FMT(Dst != Addr, \"Invalid\");\n\n switch (Size) {\n case 1: {\n ldrb(Dst.W(), Addr, Size);\n break;\n }\n case 2: {\n ldrh(Dst.W(), Addr, Size);\n break;\n }\n case 4: {\n ldr(Dst.W(), Addr, Size);\n break;\n }\n case 8: {\n ldr(Dst.X(), Addr, Size);\n break;\n }\n default: {\n LOGMAN_MSG_A_FMT(\"Unhandled {} size: {}\", __func__, Op->Size);\n break;\n }\n }\n}\n\nDEF_OP(PopTwo) {\n const auto Op = IROp->C();\n const auto Size = IR::OpSizeToSize(Op->Size);\n const auto Addr = GetReg(Op->InoutAddr);\n auto Dst1 = GetReg(Op->OutValue1);\n const auto Dst2 = GetReg(Op->OutValue2);\n\n // ldp x, x is invalid. Explicitly discard the first destination to encode.\n if (Dst1 == Dst2) {\n Dst1 = ARMEmitter::Reg::zr;\n }\n\n LOGMAN_THROW_A_FMT(Dst1 != Addr && Dst2 != Addr, \"Invalid\");\n LOGMAN_THROW_A_FMT(Dst1 != Dst2, \"Invalid\");\n\n switch (Size) {\n case 4: {\n ldp(Dst1.W(), Dst2.W(), Addr, 2 * Size);\n break;\n }\n case 8: {\n ldp(Dst1.X(), Dst2.X(), Addr, 2 * Size);\n break;\n }\n default: {\n LOGMAN_MSG_A_FMT(\"Unhandled {} size: {}\", __func__, Op->Size);\n break;\n }\n }\n}\n\nDEF_OP(StoreMem) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto MemReg = GetReg(Op->Addr);\n const auto MemSrc = GenerateMemOperand(OpSize, MemReg, Op->Offset, Op->OffsetType, Op->OffsetScale);\n\n if (Op->Class == IR::RegClass::GPR) {\n const auto Src = GetZeroableReg(Op->Value);\n switch (OpSize) {\n case IR::OpSize::i8Bit: strb(Src, MemSrc); break;\n case IR::OpSize::i16Bit: strh(Src, MemSrc); break;\n case IR::OpSize::i32Bit: str(Src.W(), MemSrc); break;\n case IR::OpSize::i64Bit: str(Src.X(), MemSrc); break;\n default: LOGMAN_MSG_A_FMT(\"Unhandled StoreMem size: {}\", OpSize); break;\n }\n } else {\n const auto Src = GetVReg(Op->Value);\n\n switch (OpSize) {\n case IR::OpSize::i8Bit: {\n strb(Src, MemSrc);\n break;\n }\n case IR::OpSize::i16Bit: {\n strh(Src, MemSrc);\n break;\n }\n case IR::OpSize::i32Bit: {\n str(Src.S(), MemSrc);\n break;\n }\n case IR::OpSize::i64Bit: {\n str(Src.D(), MemSrc);\n break;\n }\n case IR::OpSize::i128Bit: {\n str(Src.Q(), MemSrc);\n break;\n }\n case IR::OpSize::i256Bit: {\n LOGMAN_THROW_A_FMT(HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n const auto MemSrc = GenerateSVEMemOperand(OpSize, MemReg, Op->Offset, Op->OffsetType, Op->OffsetScale);\n st1b(Src.Z(), PRED_TMP_32B, MemSrc);\n break;\n }\n default: LOGMAN_MSG_A_FMT(\"Unhandled StoreMem size: {}\", OpSize); break;\n }\n }\n}\n\nDEF_OP(StoreMemX87SVEOptPredicate) {\n const auto Op = IROp->C();\n const auto Predicate = PRED_X87_SVEOPT;\n\n LOGMAN_THROW_A_FMT(HostSupportsSVE128 || HostSupportsSVE256, \"StoreMemX87SVEOptPredicate needs SVE support\");\n\n const auto RegData = GetVReg(Op->Value);\n const auto MemReg = GetReg(Op->Addr);\n const auto MemDst = ARMEmitter::SVEMemOperand(MemReg.X(), 0);\n\n switch (IROp->ElementSize) {\n case IR::OpSize::i8Bit: {\n st1b(RegData.Z(), Predicate, MemDst);\n break;\n }\n case IR::OpSize::i16Bit: {\n st1h(RegData.Z(), Predicate, MemDst);\n break;\n }\n case IR::OpSize::i32Bit: {\n st1w(RegData.Z(), Predicate, MemDst);\n break;\n }\n case IR::OpSize::i64Bit: {\n st1d(RegData.Z(), Predicate, MemDst);\n break;\n }\n default: LOGMAN_MSG_A_FMT(\"Unhandled {} element size: {}\", __func__, IROp->ElementSize); break;\n }\n}\n\nDEF_OP(LoadMemX87SVEOptPredicate) {\n const auto Op = IROp->C();\n const auto Dst = GetVReg(Node);\n const auto Predicate = PRED_X87_SVEOPT;\n const auto MemReg = GetReg(Op->Addr);\n\n LOGMAN_THROW_A_FMT(HostSupportsSVE128 || HostSupportsSVE256, \"LoadMemX87SVEOptPredicate needs SVE support\");\n\n const auto MemDst = ARMEmitter::SVEMemOperand(MemReg.X(), 0);\n\n switch (IROp->ElementSize) {\n case IR::OpSize::i8Bit: {\n ld1b(Dst.Z(), Predicate.Zeroing(), MemDst);\n break;\n }\n case IR::OpSize::i16Bit: {\n ld1h(Dst.Z(), Predicate.Zeroing(), MemDst);\n break;\n }\n case IR::OpSize::i32Bit: {\n ld1w(Dst.Z(), Predicate.Zeroing(), MemDst);\n break;\n }\n case IR::OpSize::i64Bit: {\n ld1d(Dst.Z(), Predicate.Zeroing(), MemDst);\n break;\n }\n default: LOGMAN_MSG_A_FMT(\"Unhandled {} element size: {}\", __func__, IROp->ElementSize); break;\n }\n}\n\nDEF_OP(StoreMemPair) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n const auto Addr = GetReg(Op->Addr);\n\n if (Op->Class == IR::RegClass::GPR) {\n const auto Src1 = GetZeroableReg(Op->Value1);\n const auto Src2 = GetZeroableReg(Op->Value2);\n switch (OpSize) {\n case IR::OpSize::i32Bit: stp(Src1.W(), Src2.W(), Addr, Op->Offset); break;\n case IR::OpSize::i64Bit: stp(Src1.X(), Src2.X(), Addr, Op->Offset); break;\n default: LOGMAN_MSG_A_FMT(\"Unhandled StoreMem size: {}\", OpSize); break;\n }\n } else {\n const auto Src1 = GetVReg(Op->Value1);\n const auto Src2 = GetVReg(Op->Value2);\n\n switch (OpSize) {\n case IR::OpSize::i32Bit: stp(Src1.S(), Src2.S(), Addr, Op->Offset); break;\n case IR::OpSize::i64Bit: stp(Src1.D(), Src2.D(), Addr, Op->Offset); break;\n case IR::OpSize::i128Bit: stp(Src1.Q(), Src2.Q(), Addr, Op->Offset); break;\n default: LOGMAN_MSG_A_FMT(\"Unhandled StoreMemPair size: {}\", OpSize); break;\n }\n }\n}\n\nDEF_OP(StoreMemTSO) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto MemReg = GetReg(Op->Addr);\n\n if (Op->Class == IR::RegClass::GPR) {\n LOGMAN_THROW_A_FMT(Op->Offset.IsInvalid() || CTX->HostFeatures.SupportsTSOImm9, \"unexpected offset\");\n LOGMAN_THROW_A_FMT(Op->OffsetScale == 1, \"unexpected offset scale\");\n LOGMAN_THROW_A_FMT(Op->OffsetType == IR::MemOffsetType::SXTX, \"unexpected offset type\");\n }\n\n if (CTX->HostFeatures.SupportsTSOImm9 && Op->Class == IR::RegClass::GPR) {\n const auto Src = GetZeroableReg(Op->Value);\n uint64_t Offset = 0;\n if (!Op->Offset.IsInvalid()) {\n bool IsInline = IsInlineConstant(Op->Offset, &Offset);\n LOGMAN_THROW_A_FMT(IsInline, \"expected immediate\");\n }\n\n if (OpSize == IR::OpSize::i8Bit) {\n // 8bit load is always aligned to natural alignment\n stlurb(Src, MemReg, Offset);\n } else {\n // Half-barrier once back-patched.\n nop();\n switch (OpSize) {\n case IR::OpSize::i16Bit: stlurh(Src, MemReg, Offset); break;\n case IR::OpSize::i32Bit: stlur(Src.W(), MemReg, Offset); break;\n case IR::OpSize::i64Bit: stlur(Src.X(), MemReg, Offset); break;\n default: LOGMAN_MSG_A_FMT(\"Unhandled StoreMemTSO size: {}\", OpSize); break;\n }\n }\n } else if (Op->Class == IR::RegClass::GPR) {\n const auto Src = GetZeroableReg(Op->Value);\n\n if (OpSize == IR::OpSize::i8Bit) {\n // 8bit load is always aligned to natural alignment\n stlrb(Src, MemReg);\n } else {\n // Half-barrier once back-patched.\n nop();\n switch (OpSize) {\n case IR::OpSize::i16Bit: stlrh(Src, MemReg); break;\n case IR::OpSize::i32Bit: stlr(Src.W(), MemReg); break;\n case IR::OpSize::i64Bit: stlr(Src.X(), MemReg); break;\n default: LOGMAN_MSG_A_FMT(\"Unhandled StoreMemTSO size: {}\", OpSize); break;\n }\n }\n } else {\n if (CTX->IsVectorAtomicTSOEnabled()) {\n // Half-Barrier.\n dmb(ARMEmitter::BarrierScope::ISH);\n }\n const auto Src = GetVReg(Op->Value);\n const auto MemSrc = GenerateMemOperand(OpSize, MemReg, Op->Offset, Op->OffsetType, Op->OffsetScale);\n switch (OpSize) {\n case IR::OpSize::i8Bit: strb(Src, MemSrc); break;\n case IR::OpSize::i16Bit: strh(Src, MemSrc); break;\n case IR::OpSize::i32Bit: str(Src.S(), MemSrc); break;\n case IR::OpSize::i64Bit: str(Src.D(), MemSrc); break;\n case IR::OpSize::i128Bit: str(Src.Q(), MemSrc); break;\n case IR::OpSize::i256Bit: {\n LOGMAN_THROW_A_FMT(HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n const auto Operand = GenerateSVEMemOperand(OpSize, MemReg, Op->Offset, Op->OffsetType, Op->OffsetScale);\n st1b(Src.Z(), PRED_TMP_32B, Operand);\n break;\n }\n default: LOGMAN_MSG_A_FMT(\"Unhandled StoreMemTSO size: {}\", OpSize); break;\n }\n }\n}\n\nDEF_OP(MemSet) {\n const auto Op = IROp->C();\n\n const bool IsAtomic = CTX->IsMemcpyAtomicTSOEnabled();\n const auto Size = IR::OpSizeToSize(Op->Size);\n const auto MemReg = GetReg(Op->Addr);\n const auto Value = GetZeroableReg(Op->Value);\n const auto Length = GetReg(Op->Length);\n const auto Dst = GetReg(Node);\n\n uint64_t DirectionConstant;\n bool DirectionIsInline = IsInlineConstant(Op->Direction, &DirectionConstant);\n ARMEmitter::Register DirectionReg = ARMEmitter::Reg::r0;\n if (!DirectionIsInline) {\n DirectionReg = GetReg(Op->Direction);\n }\n\n // If Direction > 0 then:\n // MemReg is incremented (by size)\n // else:\n // MemReg is decremented (by size)\n //\n // Counter is decremented regardless.\n\n ARMEmitter::ForwardLabel BackwardImpl {};\n ARMEmitter::ForwardLabel Done {};\n\n mov(TMP1, Length.X());\n if (Op->Prefix.IsInvalid()) {\n mov(TMP2, MemReg.X());\n } else {\n const auto Prefix = GetReg(Op->Prefix);\n add(TMP2, Prefix.X(), MemReg.X());\n }\n\n if (!DirectionIsInline) {\n // Backward or forwards implementation depends on flag\n (void)tbnz(DirectionReg, 1, &BackwardImpl);\n }\n\n auto MemStore = [this](auto Value, uint32_t OpSize, int32_t Size) {\n switch (OpSize) {\n case 1: strb(Value.W(), TMP2, Size); break;\n case 2: strh(Value.W(), TMP2, Size); break;\n case 4: str(Value.W(), TMP2, Size); break;\n case 8: str(Value.X(), TMP2, Size); break;\n default: LOGMAN_MSG_A_FMT(\"Unhandled {} size: {}\", __func__, Size); break;\n }\n };\n\n auto MemStoreTSO = [this](auto Value, uint32_t OpSize, int32_t Size) {\n if (OpSize == 1) {\n // 8bit load is always aligned to natural alignment\n stlrb(Value.W(), TMP2);\n } else {\n nop();\n switch (OpSize) {\n case 2: stlrh(Value.W(), TMP2); break;\n case 4: stlr(Value.W(), TMP2); break;\n case 8: stlr(Value.X(), TMP2); break;\n default: LOGMAN_MSG_A_FMT(\"Unhandled {} size: {}\", __func__, Size); break;\n }\n }\n\n if (Size >= 0) {\n add(ARMEmitter::Size::i64Bit, TMP2, TMP2, OpSize);\n } else {\n sub(ARMEmitter::Size::i64Bit, TMP2, TMP2, OpSize);\n }\n };\n\n const auto SubRegSize = Size == 1 ? ARMEmitter::SubRegSize::i8Bit :\n Size == 2 ? ARMEmitter::SubRegSize::i16Bit :\n Size == 4 ? ARMEmitter::SubRegSize::i32Bit :\n Size == 8 ? ARMEmitter::SubRegSize::i64Bit :\n ARMEmitter::SubRegSize::i8Bit;\n\n auto EmitMemset = [&](int32_t Direction) {\n const int32_t SizeDirection = Size * Direction;\n const bool IsBackwards = Direction == -1;\n\n // Sets the result to the final address written depending on\n // whether or not the memset is forwards or backwards.\n const auto MakeFinalAddress = [&] {\n if (IsBackwards) {\n switch (Size) {\n case 1: sub(Dst.X(), MemReg.X(), Length.X()); break;\n case 2:\n case 4:\n case 8: sub(Dst.X(), MemReg.X(), Length.X(), ARMEmitter::ShiftType::LSL, FEXCore::ilog2(Size)); break;\n default: LOGMAN_MSG_A_FMT(\"Unhandled MemSet size: {}\", Size); break;\n }\n } else {\n switch (Size) {\n case 1: add(Dst.X(), MemReg.X(), Length.X()); break;\n case 2:\n case 4:\n case 8: add(Dst.X(), MemReg.X(), Length.X(), ARMEmitter::ShiftType::LSL, FEXCore::ilog2(Size)); break;\n default: LOGMAN_MSG_A_FMT(\"Unhandled MemSet size: {}\", Size); break;\n }\n }\n };\n\n ARMEmitter::BiDirectionalLabel AgainInternal {};\n ARMEmitter::ForwardLabel DoneInternal {};\n\n // Early exit if zero count.\n (void)cbz(ARMEmitter::Size::i64Bit, TMP1, &DoneInternal);\n\n if (!IsAtomic) {\n if (CTX->HostFeatures.SupportsMOPS) {\n const bool Is8Bit = SubRegSize == ARMEmitter::SubRegSize::i8Bit;\n\n // We can handle 8-bit memsets and any other size that happens\n // to be using an inlined zero value (resulting in the use of ZR).\n //\n // NOTE:\n // Strictly speaking, this can also be trivially expanded to handle other sizes\n // that happen to use any value that could fit inside a byte if the need\n // arises. This does increase branching and code generation, however, since\n // we'd still need to emit the fallback in the event a value for a larger size\n // falls outside the range of a byte instead of only generating the MOPS code.\n if (Is8Bit || Value == ARMEmitter::Reg::zr) {\n // If we're performing a non-byte-sized zeroing operation then we need to\n // scale the counter accordingly. (e.g. a 64-bit memset of size 2 needs to\n // be turned into an 8-bit memset of size 16)\n if (!Is8Bit) {\n lsl(ARMEmitter::Size::i64Bit, TMP1, TMP1, FEXCore::ToUnderlying(SubRegSize));\n }\n\n // If backwards, then we need to adjust the starting address because\n // set{p, m, e} memset forwards, so we need to slide this bad boy\n // back like: (address - count) + 1.\n //\n // This lets us offset the address such that we can treat a backwards\n // memset as if it were a forwards one.\n if (IsBackwards) {\n sub(TMP2, TMP2, TMP1);\n add(ARMEmitter::Size::i64Bit, TMP2, TMP2, 1);\n }\n\n // Unfortunately set operations fiddle with NZCV, so we need to preserve it.\n mrs(TMP3, ARMEmitter::SystemRegister::NZCV);\n setp(TMP2, TMP1, Value.X());\n setm(TMP2, TMP1, Value.X());\n sete(TMP2, TMP1, Value.X());\n msr(ARMEmitter::SystemRegister::NZCV, TMP3);\n\n MakeFinalAddress();\n (void)Bind(&DoneInternal);\n return;\n }\n }\n\n ARMEmitter::ForwardLabel AgainInternal256Exit {};\n ARMEmitter::BackwardLabel AgainInternal256 {};\n ARMEmitter::ForwardLabel AgainInternal128Exit {};\n ARMEmitter::BackwardLabel AgainInternal128 {};\n\n if (IsBackwards) {\n sub(ARMEmitter::Size::i64Bit, TMP2, TMP2, 32 - Size);\n }\n\n // Keep the counter one copy ahead, so that underflow can be used to detect when to fallback\n // to the copy unit size copy loop for the last chunk.\n // Do this in two parts, to fallback to the byte by byte loop if size < 32, and to the\n // single copy loop if size < 64.\n sub(ARMEmitter::Size::i64Bit, TMP1, TMP1, 32 / Size);\n (void)tbnz(TMP1, 63, &AgainInternal128Exit);\n\n // Fill VTMP2 with the set pattern\n dup(SubRegSize, VTMP2.Q(), Value);\n\n sub(ARMEmitter::Size::i64Bit, TMP1, TMP1, 32 / Size);\n (void)tbnz(TMP1, 63, &AgainInternal256Exit);\n\n (void)Bind(&AgainInternal256);\n stp(VTMP2.Q(), VTMP2.Q(), TMP2, 32 * Direction);\n stp(VTMP2.Q(), VTMP2.Q(), TMP2, 32 * Direction);\n sub(ARMEmitter::Size::i64Bit, TMP1, TMP1, 64 / Size);\n (void)tbz(TMP1, 63, &AgainInternal256);\n\n (void)Bind(&AgainInternal256Exit);\n add(ARMEmitter::Size::i64Bit, TMP1, TMP1, 64 / Size);\n (void)cbz(ARMEmitter::Size::i64Bit, TMP1, &DoneInternal);\n\n sub(ARMEmitter::Size::i64Bit, TMP1, TMP1, 32 / Size);\n (void)tbnz(TMP1, 63, &AgainInternal128Exit);\n (void)Bind(&AgainInternal128);\n stp(VTMP2.Q(), VTMP2.Q(), TMP2, 32 * Direction);\n sub(ARMEmitter::Size::i64Bit, TMP1, TMP1, 32 / Size);\n (void)tbz(TMP1, 63, &AgainInternal128);\n\n (void)Bind(&AgainInternal128Exit);\n add(ARMEmitter::Size::i64Bit, TMP1, TMP1, 32 / Size);\n (void)cbz(ARMEmitter::Size::i64Bit, TMP1, &DoneInternal);\n\n if (IsBackwards) {\n add(ARMEmitter::Size::i64Bit, TMP2, TMP2, 32 - Size);\n }\n }\n\n (void)Bind(&AgainInternal);\n if (IsAtomic) {\n MemStoreTSO(Value, Size, SizeDirection);\n } else {\n MemStore(Value, Size, SizeDirection);\n }\n sub(ARMEmitter::Size::i64Bit, TMP1, TMP1, 1);\n (void)cbnz(ARMEmitter::Size::i64Bit, TMP1, &AgainInternal);\n\n (void)Bind(&DoneInternal);\n\n MakeFinalAddress();\n };\n\n if (DirectionIsInline) {\n LOGMAN_THROW_A_FMT(DirectionConstant == 1 || DirectionConstant == -1, \"unexpected direction\");\n EmitMemset(DirectionConstant);\n } else {\n // Emit forward direction memset then backward direction memset.\n for (int32_t Direction : {1, -1}) {\n EmitMemset(Direction);\n\n if (Direction == 1) {\n (void)b(&Done);\n (void)Bind(&BackwardImpl);\n }\n }\n\n (void)Bind(&Done);\n // Destination already set to the final pointer.\n }\n}\n\nDEF_OP(MemCpy) {\n const auto Op = IROp->C();\n\n const bool IsAtomic = CTX->IsMemcpyAtomicTSOEnabled();\n const auto Size = IR::OpSizeToSize(Op->Size);\n const auto MemRegDest = GetReg(Op->Dest);\n const auto MemRegSrc = GetReg(Op->Src);\n\n const auto Length = GetReg(Op->Length);\n uint64_t DirectionConstant;\n bool DirectionIsInline = IsInlineConstant(Op->Direction, &DirectionConstant);\n ARMEmitter::Register DirectionReg = ARMEmitter::Reg::r0;\n if (!DirectionIsInline) {\n DirectionReg = GetReg(Op->Direction);\n }\n\n auto Dst0 = GetReg(Op->OutDstAddress);\n auto Dst1 = GetReg(Op->OutSrcAddress);\n // If Direction > 0 then:\n // MemRegDest is incremented (by size)\n // MemRegSrc is incremented (by size)\n // else:\n // MemRegDest is decremented (by size)\n // MemRegSrc is decremented (by size)\n //\n // Counter is decremented regardless.\n\n ARMEmitter::ForwardLabel BackwardImpl {};\n ARMEmitter::ForwardLabel Done {};\n\n mov(TMP1, Length.X());\n mov(TMP2, MemRegDest.X());\n mov(TMP3, MemRegSrc.X());\n\n // TMP1 = Length\n // TMP2 = Dest\n // TMP3 = Src\n // TMP4 = load+store temp value\n\n if (!DirectionIsInline) {\n // Backward or forwards implementation depends on flag\n (void)tbnz(DirectionReg, 1, &BackwardImpl);\n }\n\n auto MemCpy = [this](uint32_t OpSize, int32_t Size) {\n switch (OpSize) {\n case 1:\n ldrb(TMP4.W(), TMP3, Size);\n strb(TMP4.W(), TMP2, Size);\n break;\n case 2:\n ldrh(TMP4.W(), TMP3, Size);\n strh(TMP4.W(), TMP2, Size);\n break;\n case 4:\n ldr(TMP4.W(), TMP3, Size);\n str(TMP4.W(), TMP2, Size);\n break;\n case 8:\n ldr(TMP4, TMP3, Size);\n str(TMP4, TMP2, Size);\n break;\n case 32:\n ldp(VTMP1.Q(), VTMP2.Q(), TMP3, Size);\n stp(VTMP1.Q(), VTMP2.Q(), TMP2, Size);\n break;\n default: LOGMAN_MSG_A_FMT(\"Unhandled {} size: {}\", __func__, Size); break;\n }\n };\n\n auto MemCpyTSO = [this](uint32_t OpSize, int32_t Size) {\n if (CTX->HostFeatures.SupportsRCPC) {\n if (OpSize == 1) {\n // 8bit load is always aligned to natural alignment\n ldaprb(TMP4.W(), TMP3);\n stlrb(TMP4.W(), TMP2);\n } else {\n switch (OpSize) {\n case 2: ldaprh(TMP4.W(), TMP3); break;\n case 4: ldapr(TMP4.W(), TMP3); break;\n case 8: ldapr(TMP4, TMP3); break;\n default: LOGMAN_MSG_A_FMT(\"Unhandled {} size: {}\", __func__, Size); break;\n }\n\n // Placeholders for backpatching barriers (one per load/store)\n nop();\n nop();\n\n switch (OpSize) {\n case 2: stlrh(TMP4.W(), TMP2); break;\n case 4: stlr(TMP4.W(), TMP2); break;\n case 8: stlr(TMP4, TMP2); break;\n default: LOGMAN_MSG_A_FMT(\"Unhandled {} size: {}\", __func__, Size); break;\n }\n }\n } else {\n if (OpSize == 1) {\n // 8bit load is always aligned to natural alignment\n ldarb(TMP4.W(), TMP3);\n stlrb(TMP4.W(), TMP2);\n } else {\n switch (OpSize) {\n case 2: ldarh(TMP4.W(), TMP3); break;\n case 4: ldar(TMP4.W(), TMP3); break;\n case 8: ldar(TMP4, TMP3); break;\n default: LOGMAN_MSG_A_FMT(\"Unhandled {} size: {}\", __func__, Size); break;\n }\n\n // Placeholders for backpatching barriers (one per load/store)\n nop();\n nop();\n\n switch (OpSize) {\n case 2: stlrh(TMP4.W(), TMP2); break;\n case 4: stlr(TMP4.W(), TMP2); break;\n case 8: stlr(TMP4, TMP2); break;\n default: LOGMAN_MSG_A_FMT(\"Unhandled {} size: {}\", __func__, Size); break;\n }\n }\n }\n\n if (Size >= 0) {\n add(ARMEmitter::Size::i64Bit, TMP2, TMP2, OpSize);\n add(ARMEmitter::Size::i64Bit, TMP3, TMP3, OpSize);\n } else {\n sub(ARMEmitter::Size::i64Bit, TMP2, TMP2, OpSize);\n sub(ARMEmitter::Size::i64Bit, TMP3, TMP3, OpSize);\n }\n };\n\n auto EmitMemcpy = [&](int32_t Direction) {\n const int32_t SizeDirection = Size * Direction;\n const bool IsBackwards = Direction == -1;\n\n const auto FinalizeAddresses = [&] {\n if (IsBackwards) {\n switch (Size) {\n case 1:\n sub(Dst0.X(), TMP1, TMP3);\n sub(Dst1.X(), TMP2, TMP3);\n break;\n case 2:\n case 4:\n case 8:\n sub(Dst0.X(), TMP1, TMP3, ARMEmitter::ShiftType::LSL, FEXCore::ilog2(Size));\n sub(Dst1.X(), TMP2, TMP3, ARMEmitter::ShiftType::LSL, FEXCore::ilog2(Size));\n break;\n default: LOGMAN_MSG_A_FMT(\"Unhandled MemCpy size: {}\", Size); break;\n }\n } else {\n switch (Size) {\n case 1:\n add(Dst0.X(), TMP1, TMP3);\n add(Dst1.X(), TMP2, TMP3);\n break;\n case 2:\n case 4:\n case 8:\n add(Dst0.X(), TMP1, TMP3, ARMEmitter::ShiftType::LSL, FEXCore::ilog2(Size));\n add(Dst1.X(), TMP2, TMP3, ARMEmitter::ShiftType::LSL, FEXCore::ilog2(Size));\n break;\n default: LOGMAN_MSG_A_FMT(\"Unhandled MemCpy size: {}\", Size); break;\n }\n }\n };\n\n ARMEmitter::BiDirectionalLabel AgainInternal {};\n ARMEmitter::ForwardLabel DoneInternal {};\n\n // Early exit if zero count.\n (void)cbz(ARMEmitter::Size::i64Bit, TMP1, &DoneInternal);\n\n if (!IsAtomic) {\n if (CTX->HostFeatures.SupportsMOPS) {\n // In the event we have an overlap (gross), we need to fall back\n // to the non-mops copy handler. Since the overlap check needs to\n // make use of NZCV, we need to save it. This can be avoided with\n // ARMv9.6+'s FEAT_CMPBR, but alas, we don't have access to that right now.\n //\n // NOTE: That we need to temporarily trash TMP1 and restore it after the\n // comparison.\n ARMEmitter::ForwardLabel OverlapCase;\n mrs(TMP4, ARMEmitter::SystemRegister::NZCV);\n sub(ARMEmitter::Size::i64Bit, TMP1, TMP2, TMP3);\n cmp(ARMEmitter::Size::i64Bit, TMP1, Length.X());\n mov(TMP1, Length.X());\n (void)bc(ARMEmitter::Condition::CC_LT, &OverlapCase);\n\n // If doing something larger than a byte copy, then we need to scale\n // the counter value accordingly to convert it to bytes.\n if (Size > 1) {\n lsl(ARMEmitter::Size::i64Bit, TMP1, TMP1, FEXCore::ilog2(Size));\n }\n\n // Adjust addresses so that we treat the backward copy as a forward copy\n if (IsBackwards) {\n sub(ARMEmitter::Size::i64Bit, TMP2, TMP2, TMP1);\n sub(ARMEmitter::Size::i64Bit, TMP3, TMP3, TMP1);\n add(ARMEmitter::Size::i64Bit, TMP2, TMP2, Size);\n add(ARMEmitter::Size::i64Bit, TMP3, TMP3, Size);\n }\n\n // Unfortunately copy operations fiddle with NZCV, so we need to preserve it.\n cpyfp(TMP2, TMP3, TMP1);\n cpyfm(TMP2, TMP3, TMP1);\n cpyfe(TMP2, TMP3, TMP1);\n msr(ARMEmitter::SystemRegister::NZCV, TMP4);\n\n (void)b(&DoneInternal);\n\n // Turns out we overlap and need to fall back. Make sure to restore NZCV.\n (void)Bind(&OverlapCase);\n msr(ARMEmitter::SystemRegister::NZCV, TMP4);\n }\n\n ARMEmitter::ForwardLabel AbsPos {};\n ARMEmitter::ForwardLabel AgainInternal256Exit {};\n ARMEmitter::ForwardLabel AgainInternal128Exit {};\n ARMEmitter::BackwardLabel AgainInternal128 {};\n ARMEmitter::BackwardLabel AgainInternal256 {};\n\n sub(ARMEmitter::Size::i64Bit, TMP4, TMP2, TMP3);\n (void)tbz(TMP4, 63, &AbsPos);\n neg(ARMEmitter::Size::i64Bit, TMP4, TMP4);\n (void)Bind(&AbsPos);\n sub(ARMEmitter::Size::i64Bit, TMP4, TMP4, 32);\n (void)tbnz(TMP4, 63, &AgainInternal);\n\n if (IsBackwards) {\n sub(ARMEmitter::Size::i64Bit, TMP2, TMP2, 32 - Size);\n sub(ARMEmitter::Size::i64Bit, TMP3, TMP3, 32 - Size);\n }\n\n // Keep the counter one copy ahead, so that underflow can be used to detect when to fallback\n // to the copy unit size copy loop for the last chunk.\n // Do this in two parts, to fallback to the byte by byte loop if size < 32, and to the\n // single copy loop if size < 64.\n sub(ARMEmitter::Size::i64Bit, TMP1, TMP1, 32 / Size);\n (void)tbnz(TMP1, 63, &AgainInternal128Exit);\n sub(ARMEmitter::Size::i64Bit, TMP1, TMP1, 32 / Size);\n (void)tbnz(TMP1, 63, &AgainInternal256Exit);\n\n (void)Bind(&AgainInternal256);\n MemCpy(32, 32 * Direction);\n MemCpy(32, 32 * Direction);\n sub(ARMEmitter::Size::i64Bit, TMP1, TMP1, 64 / Size);\n (void)tbz(TMP1, 63, &AgainInternal256);\n\n (void)Bind(&AgainInternal256Exit);\n add(ARMEmitter::Size::i64Bit, TMP1, TMP1, 64 / Size);\n (void)cbz(ARMEmitter::Size::i64Bit, TMP1, &DoneInternal);\n\n sub(ARMEmitter::Size::i64Bit, TMP1, TMP1, 32 / Size);\n (void)tbnz(TMP1, 63, &AgainInternal128Exit);\n (void)Bind(&AgainInternal128);\n MemCpy(32, 32 * Direction);\n sub(ARMEmitter::Size::i64Bit, TMP1, TMP1, 32 / Size);\n (void)tbz(TMP1, 63, &AgainInternal128);\n\n (void)Bind(&AgainInternal128Exit);\n add(ARMEmitter::Size::i64Bit, TMP1, TMP1, 32 / Size);\n (void)cbz(ARMEmitter::Size::i64Bit, TMP1, &DoneInternal);\n\n if (IsBackwards) {\n add(ARMEmitter::Size::i64Bit, TMP2, TMP2, 32 - Size);\n add(ARMEmitter::Size::i64Bit, TMP3, TMP3, 32 - Size);\n }\n }\n\n (void)Bind(&AgainInternal);\n if (IsAtomic) {\n MemCpyTSO(Size, SizeDirection);\n } else {\n MemCpy(Size, SizeDirection);\n }\n sub(ARMEmitter::Size::i64Bit, TMP1, TMP1, 1);\n (void)cbnz(ARMEmitter::Size::i64Bit, TMP1, &AgainInternal);\n\n (void)Bind(&DoneInternal);\n\n // Needs to use temporaries just in case of overwrite\n mov(TMP1, MemRegDest.X());\n mov(TMP2, MemRegSrc.X());\n mov(TMP3, Length.X());\n\n FinalizeAddresses();\n };\n\n if (DirectionIsInline) {\n LOGMAN_THROW_A_FMT(DirectionConstant == 1 || DirectionConstant == -1, \"unexpected direction\");\n EmitMemcpy(DirectionConstant);\n } else {\n // Emit forward direction memcpy then backward direction memcpy.\n for (int32_t Direction : {1, -1}) {\n EmitMemcpy(Direction);\n if (Direction == 1) {\n (void)b(&Done);\n (void)Bind(&BackwardImpl);\n }\n }\n (void)Bind(&Done);\n // Destination already set to the final pointer.\n }\n}\n\nDEF_OP(CacheLineClear) {\n if (!CTX->HostFeatures.SupportsCacheMaintenanceOps) {\n dmb(ARMEmitter::BarrierScope::SY);\n return;\n }\n\n auto Op = IROp->C();\n\n auto MemReg = GetReg(Op->Addr);\n\n // Clear dcache only\n // icache doesn't matter here since the guest application shouldn't be calling clflush on JIT code.\n if (CTX->HostFeatures.DCacheLineSize >= 64U) {\n dc(ARMEmitter::DataCacheOperation::CIVAC, MemReg);\n } else {\n auto CurrentWorkingReg = MemReg.X();\n for (size_t i = 0; i < std::max(1U, CTX->HostFeatures.DCacheLineSize / 64U); ++i) {\n dc(ARMEmitter::DataCacheOperation::CIVAC, TMP1);\n add(ARMEmitter::Size::i64Bit, TMP1, CurrentWorkingReg, CTX->HostFeatures.DCacheLineSize);\n CurrentWorkingReg = TMP1;\n }\n }\n\n if (Op->Serialize) {\n // If requested, serialized all of the data cache operations.\n dsb(ARMEmitter::BarrierScope::ISH);\n }\n}\n\nDEF_OP(CacheLineClean) {\n if (!CTX->HostFeatures.SupportsCacheMaintenanceOps) {\n dmb(ARMEmitter::BarrierScope::ST);\n return;\n }\n\n auto Op = IROp->C();\n\n auto MemReg = GetReg(Op->Addr);\n\n // Clean dcache only\n if (CTX->HostFeatures.DCacheLineSize >= 64U) {\n dc(ARMEmitter::DataCacheOperation::CVAC, MemReg);\n } else {\n auto CurrentWorkingReg = MemReg.X();\n for (size_t i = 0; i < std::max(1U, CTX->HostFeatures.DCacheLineSize / 64U); ++i) {\n dc(ARMEmitter::DataCacheOperation::CVAC, TMP1);\n add(ARMEmitter::Size::i64Bit, TMP1, CurrentWorkingReg, CTX->HostFeatures.DCacheLineSize);\n CurrentWorkingReg = TMP1;\n }\n }\n}\n\nDEF_OP(CacheLineZero) {\n auto Op = IROp->C();\n\n auto MemReg = GetReg(Op->Addr);\n\n if (CTX->HostFeatures.SupportsCLZERO) {\n // We can use this instruction directly\n dc(ARMEmitter::DataCacheOperation::ZVA, MemReg);\n } else {\n // We must walk the cacheline ourselves\n // Force cacheline alignment\n and_(ARMEmitter::Size::i64Bit, TMP1, MemReg, ~(CPUIDEmu::CACHELINE_SIZE - 1));\n // This will end up being four STPs\n // Depending on uarch it could be slightly more efficient in instructions emitted\n // and uops to use vector pair STP, but we want the non-temporal bit specifically here\n for (size_t i = 0; i < CPUIDEmu::CACHELINE_SIZE; i += 16) {\n stnp(ARMEmitter::XReg::zr, ARMEmitter::XReg::zr, TMP1, i);\n }\n }\n}\n\nDEF_OP(Prefetch) {\n auto Op = IROp->C();\n const auto MemReg = GetReg(Op->Addr);\n\n // Access size is only ever handled as 8-byte. Even though it is accesssed as a cacheline.\n const auto MemSrc = GenerateMemOperand(IR::OpSize::i64Bit, MemReg, Op->Offset, Op->OffsetType, Op->OffsetScale);\n\n size_t LUT = (Op->Stream ? 1 : 0) | ((Op->CacheLevel - 1) << 1) | (Op->ForStore ? 1U << 3 : 0);\n\n constexpr static std::array PrefetchType = {\n ARMEmitter::Prefetch::PLDL1KEEP,\n ARMEmitter::Prefetch::PLDL1STRM,\n\n ARMEmitter::Prefetch::PLDL2KEEP,\n ARMEmitter::Prefetch::PLDL2STRM,\n\n ARMEmitter::Prefetch::PLDL3KEEP,\n ARMEmitter::Prefetch::PLDL3STRM,\n\n // Gap of two.\n // 0b0'11'0\n ARMEmitter::Prefetch::PLDL1STRM,\n // 0b0'11'1\n ARMEmitter::Prefetch::PLDL1STRM,\n\n ARMEmitter::Prefetch::PSTL1KEEP,\n ARMEmitter::Prefetch::PSTL1STRM,\n\n ARMEmitter::Prefetch::PSTL2KEEP,\n ARMEmitter::Prefetch::PSTL2STRM,\n\n ARMEmitter::Prefetch::PSTL3KEEP,\n ARMEmitter::Prefetch::PSTL3STRM,\n };\n\n prfm(PrefetchType[LUT], MemSrc);\n}\n\nDEF_OP(VStoreNonTemporal) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n const auto Is128Bit = OpSize == IR::OpSize::i128Bit;\n\n const auto Value = GetVReg(Op->Value);\n const auto MemReg = GetReg(Op->Addr);\n const auto Offset = Op->Offset;\n\n if (Is256Bit) {\n const auto GoverningPredicate = PRED_TMP_32B.Zeroing();\n const auto OffsetScaled = Offset / 32;\n stnt1b(Value.Z(), GoverningPredicate, MemReg, OffsetScaled);\n } else if (Is128Bit && HostSupportsSVE128) {\n const auto GoverningPredicate = PRED_TMP_16B.Zeroing();\n const auto OffsetScaled = Offset / 16;\n stnt1b(Value.Z(), GoverningPredicate, MemReg, OffsetScaled);\n } else {\n // Treat the non-temporal store as a regular vector store in this case for compatibility\n str(Value.Q(), MemReg, Offset);\n }\n}\n\nDEF_OP(VStoreNonTemporalPair) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto Is128Bit = OpSize == IR::OpSize::i128Bit;\n LOGMAN_THROW_A_FMT(Is128Bit, \"This IR operation only operates at 128-bit wide\");\n\n const auto ValueLow = GetVReg(Op->ValueLow);\n const auto ValueHigh = GetVReg(Op->ValueHigh);\n\n const auto MemReg = GetReg(Op->Addr);\n const auto Offset = Op->Offset;\n\n stnp(ValueLow.Q(), ValueHigh.Q(), MemReg, Offset);\n}\n\nDEF_OP(VLoadNonTemporal) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n const auto Is128Bit = OpSize == IR::OpSize::i128Bit;\n\n const auto Dst = GetVReg(Node);\n const auto MemReg = GetReg(Op->Addr);\n const auto Offset = Op->Offset;\n\n if (Is256Bit) {\n const auto GoverningPredicate = PRED_TMP_32B.Zeroing();\n const auto OffsetScaled = Offset / 32;\n ldnt1b(Dst.Z(), GoverningPredicate, MemReg, OffsetScaled);\n } else if (Is128Bit && HostSupportsSVE128) {\n const auto GoverningPredicate = PRED_TMP_16B.Zeroing();\n const auto OffsetScaled = Offset / 16;\n ldnt1b(Dst.Z(), GoverningPredicate, MemReg, OffsetScaled);\n } else {\n // Treat the non-temporal store as a regular vector store in this case for compatibility\n ldr(Dst.Q(), MemReg, Offset);\n }\n}\n\n} // namespace FEXCore::CPU\n" }, { "path": "FEXCore/Source/Interface/Core/JIT/MiscOps.cpp", "content": "// SPDX-License-Identifier: MIT\n/*\n$info$\ntags: backend|arm64\n$end_info$\n*/\n\n#ifndef _WIN32\n#include \n#endif\n\n#include \"Interface/Context/Context.h\"\n#include \"Interface/Core/JIT/DebugData.h\"\n#include \"Interface/Core/JIT/JITClass.h\"\n\n#include \n#include \n#include \n\nnamespace FEXCore::CPU {\n\nDEF_OP(WFET) {\n auto Op = IROp->C();\n const auto Lower = GetReg(Op->Lower);\n const auto Upper = GetReg(Op->Upper);\n\n // Combine registers.\n mov(ARMEmitter::Size::i64Bit, TMP1, Lower);\n bfi(ARMEmitter::Size::i64Bit, TMP1, Upper, 32, 32);\n if (CTX->Config.TSCScale) {\n // Scale back to ARM64 TSC scale if necessary\n lsr(ARMEmitter::Size::i64Bit, TMP1, TMP1, CTX->Config.TSCScale);\n }\n\n // Clear the exclusive monitor so it can't spuriously wake up with that event.\n clrex();\n\n // Execute wfet to wait until the TSC.\n wfet(TMP1);\n}\n\nDEF_OP(GuestOpcode) {\n auto Op = IROp->C();\n // metadata\n DebugData->GuestOpcodes.push_back({Op->GuestEntryOffset, GetCursorAddress() - CodeData.BlockBegin});\n}\n\nDEF_OP(Fence) {\n auto Op = IROp->C();\n switch (Op->Fence) {\n case IR::FenceType::Load: dmb(ARMEmitter::BarrierScope::LD); break;\n case IR::FenceType::LoadStore: dmb(ARMEmitter::BarrierScope::SY); break;\n case IR::FenceType::Store: dmb(ARMEmitter::BarrierScope::ST); break;\n case IR::FenceType::Inst: isb(); break;\n default: LOGMAN_MSG_A_FMT(\"Unknown Fence: {}\", Op->Fence); break;\n }\n}\n\nDEF_OP(Break) {\n auto Op = IROp->C();\n\n // First we must reset the stack\n ResetStack();\n\n Core::CpuStateFrame::SynchronousFaultDataStruct State = {\n .FaultToTopAndGeneratedException = 1,\n .Signal = Op->Reason.Signal,\n .TrapNo = Op->Reason.TrapNumber,\n .si_code = Op->Reason.si_code,\n .err_code = Op->Reason.ErrorRegister,\n };\n\n uint64_t Constant {};\n memcpy(&Constant, &State, sizeof(State));\n\n LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r1, Constant);\n str(ARMEmitter::XReg::x1, STATE, offsetof(FEXCore::Core::CpuStateFrame, SynchronousFaultData));\n\n switch (Op->Reason.Signal) {\n case Core::FAULT_SIGILL:\n ldr(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.GuestSignal_SIGILL));\n br(TMP1);\n break;\n case Core::FAULT_SIGTRAP:\n ldr(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.GuestSignal_SIGTRAP));\n br(TMP1);\n break;\n case Core::FAULT_SIGSEGV:\n ldr(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.GuestSignal_SIGSEGV));\n br(TMP1);\n break;\n default:\n ldr(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.GuestSignal_SIGTRAP));\n br(TMP1);\n break;\n }\n}\n\nDEF_OP(GetRoundingMode) {\n auto Dst = GetReg(Node);\n mrs(Dst, ARMEmitter::SystemRegister::FPCR);\n ubfx(ARMEmitter::Size::i64Bit, Dst, Dst, 22, 3);\n\n // FTZ is already in the correct location\n // Rounding mode is different\n //\n // Need to remap rounding mode from order nearest, pos inf, neg inf, toward\n // zero. Just swapping 01 and 10. That's a bitfield reverse. Round mode is in\n // bottom two bits. After reversing as a 32-bit operation, it'll be in [31:30]\n // and ripe for reinsertion back at 0.\n static_assert(FEXCore::ToUnderlying(IR::RoundMode::Nearest) == 0);\n static_assert(FEXCore::ToUnderlying(IR::RoundMode::NegInfinity) == 1);\n static_assert(FEXCore::ToUnderlying(IR::RoundMode::PosInfinity) == 2);\n static_assert(FEXCore::ToUnderlying(IR::RoundMode::TowardsZero) == 3);\n\n rbit(ARMEmitter::Size::i32Bit, TMP1, Dst);\n bfi(ARMEmitter::Size::i64Bit, Dst, TMP1, 30, 2);\n}\n\nDEF_OP(SetRoundingMode) {\n auto Op = IROp->C();\n auto Src = GetReg(Op->RoundMode);\n auto MXCSR = GetReg(Op->MXCSR);\n\n // As above, setup the rounding flags in [31:30]\n rbit(ARMEmitter::Size::i32Bit, TMP2, Src);\n // and extract\n lsr(ARMEmitter::Size::i32Bit, TMP2, TMP2, 30);\n\n mrs(TMP1, ARMEmitter::SystemRegister::FPCR);\n\n // vixl simulator doesn't support anything beyond ties-to-even rounding\n if (CTX->Config.DisableVixlIndirectCalls) [[likely]] {\n // Insert the rounding flags\n bfi(ARMEmitter::Size::i64Bit, TMP1, TMP2, 22, 2);\n }\n\n // Insert the FTZ flag\n lsr(ARMEmitter::Size::i64Bit, TMP2, Src, 2);\n bfi(ARMEmitter::Size::i64Bit, TMP1, TMP2, 24, 1);\n\n if (Op->SetDAZ && HostSupportsAFP) {\n // Extract DAZ from MXCSR and insert to in FPCR.FIZ\n bfxil(ARMEmitter::Size::i64Bit, TMP1, MXCSR, 6, 1);\n }\n\n // Now save the new FPCR\n msr(ARMEmitter::SystemRegister::FPCR, TMP1);\n}\n\nDEF_OP(PushRoundingMode) {\n auto Op = IROp->C();\n auto Dest = GetReg(Node);\n\n // Save the old rounding mode\n mrs(Dest, ARMEmitter::SystemRegister::FPCR);\n\n // vixl simulator doesn't support anything beyond ties-to-even rounding\n if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] {\n return;\n }\n\n // Insert the rounding flags, reversing the mode bits as above\n if (Op->RoundMode == 3) {\n orr(ARMEmitter::Size::i64Bit, TMP1, Dest, 3 << 22);\n } else if (Op->RoundMode == 0) {\n and_(ARMEmitter::Size::i64Bit, TMP1, Dest, ~(3 << 22));\n } else {\n LOGMAN_THROW_A_FMT(Op->RoundMode == 1 || Op->RoundMode == 2, \"expect a valid round mode\");\n\n and_(ARMEmitter::Size::i64Bit, TMP1, Dest, ~(Op->RoundMode << 22));\n orr(ARMEmitter::Size::i64Bit, TMP1, TMP1, (Op->RoundMode == 2 ? 1 : 2) << 22);\n }\n\n // Now save the new FPCR\n msr(ARMEmitter::SystemRegister::FPCR, TMP1);\n}\n\nDEF_OP(PopRoundingMode) {\n auto Op = IROp->C();\n msr(ARMEmitter::SystemRegister::FPCR, GetReg(Op->FPCR));\n}\n\nDEF_OP(Print) {\n auto Op = IROp->C();\n\n PushDynamicRegs(TMP1);\n SpillStaticRegs(TMP1);\n\n if (IsGPR(Op->Value)) {\n mov(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, GetReg(Op->Value));\n ldr(ARMEmitter::XReg::x3, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.PrintValue));\n } else {\n fmov(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, GetVReg(Op->Value), false);\n fmov(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r1, GetVReg(Op->Value), true);\n ldr(ARMEmitter::XReg::x3, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.PrintVectorValue));\n }\n\n if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] {\n if (IsGPR(Op->Value)) {\n GenerateIndirectRuntimeCall(ARMEmitter::Reg::r3);\n } else {\n GenerateIndirectRuntimeCall(ARMEmitter::Reg::r3);\n }\n } else {\n blr(ARMEmitter::Reg::r3);\n }\n\n FillStaticRegs();\n PopDynamicRegs();\n}\n\nDEF_OP(ProcessorID) {\n if (CTX->HostFeatures.SupportsCPUIndexInTPIDRRO) {\n mrs(GetReg(Node), ARMEmitter::SystemRegister::TPIDRRO_EL0);\n return;\n }\n#ifdef _WIN32\n else {\n // If on Windows and TPIDRRO isn't supported (like in wine), then this is a programming error.\n ERROR_AND_DIE_FMT(\"Unsupported\");\n }\n#else\n // We always need to spill x8 since we can't know if it is live at this SSA location\n uint32_t SpillMask = 1U << 8;\n\n // Ordering is incredibly important here\n // We must spill any overlapping registers first THEN claim we are in a syscall without invalidating state at all\n // Only spill the registers that intersect with our usage\n SpillStaticRegs(TMP1, {\n .GPRSpillMask = SpillMask,\n .FPRs = false,\n });\n\n // Now that we are spilled, store in the state that we are in a syscall\n // Still without overwriting registers that matter\n // 16bit LoadConstant to be a single instruction\n // We must always spill at least one register (x8) so this value always has a bit set\n // This gives the signal handler a value to check to see if we are in a syscall at all\n LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, SpillMask & 0xFFFF);\n str(ARMEmitter::XReg::x0, STATE, offsetof(FEXCore::Core::CpuStateFrame, InSyscallInfo));\n\n // Allocate some temporary space for storing the uint32_t CPU and Node IDs\n sub(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::rsp, ARMEmitter::Reg::rsp, 16);\n\n // Load the getcpu syscall number\n#if defined(ARCHITECTURE_x86_64)\n // Just to ensure the syscall number doesn't change if compiled for an x86_64 host.\n constexpr auto GetCPUSyscallNum = 0xa8;\n#else\n constexpr auto GetCPUSyscallNum = SYS_getcpu;\n#endif\n LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r8, GetCPUSyscallNum);\n\n // CPU pointer in x0\n add(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, ARMEmitter::Reg::rsp, 0);\n // Node in x1\n add(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r1, ARMEmitter::Reg::rsp, 4);\n\n svc(0);\n // On updated signal mask we can receive a signal RIGHT HERE\n\n // Load the values returned by the kernel\n ldp(ARMEmitter::WReg::w0, ARMEmitter::WReg::w1, ARMEmitter::Reg::rsp);\n // Deallocate stack space\n sub(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::rsp, ARMEmitter::Reg::rsp, 16);\n\n // Now that we are done in the syscall we need to carefully peel back the state\n // First unspill the registers from before\n\n FillStaticRegs({\n .OptionalReg = ARMEmitter::Reg::r8,\n .OptionalReg2 = ARMEmitter::Reg::r2,\n .GPRFillMask = SpillMask,\n .FPRs = false,\n });\n\n // Now the registers we've spilled are back in their original host registers\n // We can safely claim we are no longer in a syscall\n str(ARMEmitter::XReg::zr, STATE, offsetof(FEXCore::Core::CpuStateFrame, InSyscallInfo));\n\n // Now store the result in the destination in the expected format\n // uint32_t Res = (node << 12) | cpu;\n // CPU is in w0\n // Node is in w1\n orr(ARMEmitter::Size::i64Bit, GetReg(Node), ARMEmitter::Reg::r0, ARMEmitter::Reg::r1, ARMEmitter::ShiftType::LSL, 12);\n#endif\n}\n\nDEF_OP(RDRAND) {\n auto Op = IROp->C();\n\n mrs(GetReg(Node), Op->GetReseeded ? ARMEmitter::SystemRegister::RNDRRS : ARMEmitter::SystemRegister::RNDR);\n}\n\nDEF_OP(Yield) {\n yield();\n}\n\nDEF_OP(MonoBackpatcherWrite) {\n auto Op = IROp->C();\n\n mov(ARMEmitter::Size::i64Bit, TMP3, GetReg(Op->Addr));\n mov(ARMEmitter::Size::i64Bit, TMP4, GetReg(Op->Value));\n\n PushDynamicRegs(TMP1);\n SpillStaticRegs(TMP1);\n\n mov(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, STATE.R());\n mov(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r1, IR::OpSizeToSize(Op->Size));\n\n if (!TMP_ABIARGS) {\n mov(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r2, TMP3);\n mov(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r3, TMP4);\n }\n\n#ifdef ARCHITECTURE_arm64ec\n ldr(TMP2, ARMEmitter::XReg::x18, TEB_CPU_AREA_OFFSET);\n LoadConstant(ARMEmitter::Size::i32Bit, TMP1, 1);\n strb(TMP1.W(), TMP2, CPU_AREA_IN_SYSCALL_CALLBACK_OFFSET);\n#endif\n\n ldr(ARMEmitter::XReg::x4, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.MonoBackpatcherWrite));\n if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] {\n GenerateIndirectRuntimeCall(ARMEmitter::Reg::r4);\n } else {\n blr(ARMEmitter::Reg::r4);\n }\n\n#ifdef ARCHITECTURE_arm64ec\n ldr(TMP2, ARMEmitter::XReg::x18, TEB_CPU_AREA_OFFSET);\n strb(ARMEmitter::WReg::zr, TMP2, CPU_AREA_IN_SYSCALL_CALLBACK_OFFSET);\n#endif\n\n FillStaticRegs();\n PopDynamicRegs();\n}\n\n} // namespace FEXCore::CPU\n" }, { "path": "FEXCore/Source/Interface/Core/JIT/MoveOps.cpp", "content": "// SPDX-License-Identifier: MIT\n/*\n$info$\ntags: backend|arm64\n$end_info$\n*/\n\n#include \"Interface/Core/JIT/JITClass.h\"\n\nnamespace FEXCore::CPU {\nDEF_OP(Copy) {\n auto Op = IROp->C();\n\n mov(ARMEmitter::Size::i64Bit, GetReg(Node), GetReg(Op->Source));\n}\n\nDEF_OP(RMWHandle) {\n mov(ARMEmitter::Size::i64Bit, GetReg(Node), GetReg(IROp->Args[0]));\n}\n\n} // namespace FEXCore::CPU\n" }, { "path": "FEXCore/Source/Interface/Core/JIT/Relocations.h", "content": "// SPDX-License-Identifier: MIT\n#pragma once\n#include \n#include \n\nnamespace FEXCore::Context {\nclass ContextImpl;\n}\n\nnamespace FEXCore::CPU {\nenum class RelocationTypes : uint32_t {\n // 8 byte literal in memory for symbol\n // Aligned to struct RelocNamedSymbolLiteral\n RELOC_NAMED_SYMBOL_LITERAL,\n\n // Fixed size named thunk move\n // 4 instruction constant generation\n // Aligned to struct RelocNamedThunkMove\n RELOC_NAMED_THUNK_MOVE,\n\n // 8 byte literal (relative to binary base address)\n RELOC_GUEST_RIP_LITERAL,\n\n // Fixed size guest RIP move\n // 4 instruction constant generation\n // Aligned to struct RelocGuestRIP\n RELOC_GUEST_RIP_MOVE,\n};\n\nstruct FEX_PACKED RelocationHeader final {\n // Offset to the relocated host code data\n uint64_t Offset {};\n\n RelocationTypes Type;\n};\n\nstruct RelocNamedSymbolLiteral final {\n enum class NamedSymbol : uint32_t {\n ///< Thread specific relocations\n // JIT Literal pointers\n SYMBOL_LITERAL_EXITFUNCTION_LINKER,\n };\n\n RelocationHeader Header {};\n\n NamedSymbol Symbol;\n\n uint32_t Pad[8];\n};\n\nstruct RelocNamedThunkMove final {\n RelocationHeader Header {};\n\n // GPR index the constant is being moved to\n uint32_t RegisterIndex;\n\n // The thunk SHA256 hash\n IR::SHA256Sum Symbol;\n};\n\nstruct RelocGuestRIP final {\n RelocationHeader Header {};\n\n // GPR index the constant is being moved to (for non-literal relocations)\n uint8_t RegisterIndex;\n\n char Pad[3];\n\n // The base RIP (to be moved by the register for non-literal relocations).\n // In a serialized code cache, this is relative to the binary base address.\n uint64_t GuestRIP;\n\n uint32_t pad2[6] {};\n};\n\nunion Relocation {\n // Clang 16 Can't default-initialize this union\n static Relocation Default() {\n#if __clang_major__ < 17\n Relocation Ret {.Header {}};\n memset(&Ret, 0, sizeof(Ret));\n return Ret;\n#else\n return {};\n#endif\n }\n\n RelocationHeader Header {};\n\n RelocNamedSymbolLiteral NamedSymbolLiteral;\n // This makes our union of relocations at least 48 bytes\n // It might be more efficient to not use a union\n RelocNamedThunkMove NamedThunkMove;\n\n RelocGuestRIP GuestRIP;\n};\n\nuint64_t GetNamedSymbolLiteral(FEXCore::Context::ContextImpl&, RelocNamedSymbolLiteral::NamedSymbol);\n\n} // namespace FEXCore::CPU\n" }, { "path": "FEXCore/Source/Interface/Core/JIT/VectorOps.cpp", "content": "// SPDX-License-Identifier: MIT\n/*\n$info$\ntags: backend|arm64\n$end_info$\n*/\n\n#include \"Interface/Core/Dispatcher/Dispatcher.h\"\n#include \"Interface/Core/JIT/JITClass.h\"\n\n#include \n\nnamespace FEXCore::CPU {\n\n#define DEF_UNOP(FEXOp, ARMOp, ScalarCase) \\\n DEF_OP(FEXOp) { \\\n const auto Op = IROp->C(); \\\n const auto OpSize = IROp->Size; \\\n \\\n const auto ElementSize = Op->Header.ElementSize; \\\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit; \\\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__); \\\n const auto SubRegSize = ConvertSubRegSize8(IROp); \\\n \\\n const auto Dst = GetVReg(Node); \\\n const auto Src = GetVReg(Op->Vector); \\\n \\\n if (HostSupportsSVE256 && Is256Bit) { \\\n ARMOp(SubRegSize, Dst.Z(), PRED_TMP_32B.Merging(), Src.Z()); \\\n } else { \\\n if (ElementSize == OpSize && ScalarCase) { \\\n ARMOp(SubRegSize, Dst.D(), Src.D()); \\\n } else { \\\n ARMOp(SubRegSize, Dst.Q(), Src.Q()); \\\n } \\\n } \\\n }\n\n#define DEF_BITOP(FEXOp, ARMOp) \\\n DEF_OP(FEXOp) { \\\n const auto Op = IROp->C(); \\\n const auto OpSize = IROp->Size; \\\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit; \\\n const auto Is128Bit = OpSize == IR::OpSize::i128Bit; \\\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__); \\\n \\\n const auto Dst = GetVReg(Node); \\\n const auto Vector1 = GetVReg(Op->Vector1); \\\n const auto Vector2 = GetVReg(Op->Vector2); \\\n \\\n if (HostSupportsSVE256 && Is256Bit) { \\\n ARMOp(Dst.Z(), Vector1.Z(), Vector2.Z()); \\\n } else if (Is128Bit) { \\\n ARMOp(Dst.Q(), Vector1.Q(), Vector2.Q()); \\\n } else { \\\n ARMOp(Dst.D(), Vector1.D(), Vector2.D()); \\\n } \\\n }\n\n#define DEF_BINOP(FEXOp, ARMOp) \\\n DEF_OP(FEXOp) { \\\n const auto Op = IROp->C(); \\\n const auto OpSize = IROp->Size; \\\n \\\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit; \\\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__); \\\n const auto SubRegSize = ConvertSubRegSize8(IROp); \\\n \\\n const auto Dst = GetVReg(Node); \\\n const auto Vector1 = GetVReg(Op->Vector1); \\\n const auto Vector2 = GetVReg(Op->Vector2); \\\n \\\n if (HostSupportsSVE256 && Is256Bit) { \\\n ARMOp(SubRegSize, Dst.Z(), Vector1.Z(), Vector2.Z()); \\\n } else { \\\n ARMOp(SubRegSize, Dst.Q(), Vector1.Q(), Vector2.Q()); \\\n } \\\n }\n\n#define DEF_ZIPOP(FEXOp, ARMOp) \\\n DEF_OP(FEXOp) { \\\n const auto Op = IROp->C(); \\\n const auto OpSize = IROp->Size; \\\n \\\n const auto SubRegSize = ConvertSubRegSize8(IROp); \\\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit; \\\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__); \\\n \\\n const auto Dst = GetVReg(Node); \\\n const auto VectorLower = GetVReg(Op->VectorLower); \\\n const auto VectorUpper = GetVReg(Op->VectorUpper); \\\n \\\n if (HostSupportsSVE256 && Is256Bit) { \\\n ARMOp(SubRegSize, Dst.Z(), VectorLower.Z(), VectorUpper.Z()); \\\n } else { \\\n if (OpSize == IR::OpSize::i64Bit) { \\\n ARMOp(SubRegSize, Dst.D(), VectorLower.D(), VectorUpper.D()); \\\n } else { \\\n ARMOp(SubRegSize, Dst.Q(), VectorLower.Q(), VectorUpper.Q()); \\\n } \\\n } \\\n }\n\n#define DEF_FUNOP(FEXOp, ARMOp) \\\n DEF_OP(FEXOp) { \\\n const auto Op = IROp->C(); \\\n const auto OpSize = IROp->Size; \\\n \\\n const auto ElementSize = Op->Header.ElementSize; \\\n const auto SubRegSize = ConvertSubRegSize248(IROp); \\\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit; \\\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__); \\\n \\\n const auto Dst = GetVReg(Node); \\\n const auto Src = GetVReg(Op->Vector); \\\n \\\n if (HostSupportsSVE256 && Is256Bit) { \\\n ARMOp(SubRegSize, Dst.Z(), PRED_TMP_32B.Merging(), Src.Z()); \\\n } else { \\\n if (ElementSize == OpSize) { \\\n switch (ElementSize) { \\\n case IR::OpSize::i16Bit: { \\\n ARMOp(Dst.H(), Src.H()); \\\n break; \\\n } \\\n case IR::OpSize::i32Bit: { \\\n ARMOp(Dst.S(), Src.S()); \\\n break; \\\n } \\\n case IR::OpSize::i64Bit: { \\\n ARMOp(Dst.D(), Src.D()); \\\n break; \\\n } \\\n default: break; \\\n } \\\n } else { \\\n ARMOp(SubRegSize, Dst.Q(), Src.Q()); \\\n } \\\n } \\\n }\n\n#define DEF_FBINOP(FEXOp, ARMOp) \\\n DEF_OP(FEXOp) { \\\n const auto Op = IROp->C(); \\\n const auto OpSize = IROp->Size; \\\n \\\n const auto ElementSize = Op->Header.ElementSize; \\\n const auto SubRegSize = ConvertSubRegSize248(IROp); \\\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit; \\\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__); \\\n const auto IsScalar = ElementSize == OpSize; \\\n \\\n const auto Dst = GetVReg(Node); \\\n const auto Vector1 = GetVReg(Op->Vector1); \\\n const auto Vector2 = GetVReg(Op->Vector2); \\\n \\\n if (HostSupportsSVE256 && Is256Bit) { \\\n ARMOp(SubRegSize, Dst.Z(), Vector1.Z(), Vector2.Z()); \\\n } else { \\\n if (IsScalar) { \\\n switch (ElementSize) { \\\n case IR::OpSize::i16Bit: { \\\n ARMOp(Dst.H(), Vector1.H(), Vector2.H()); \\\n break; \\\n } \\\n case IR::OpSize::i32Bit: { \\\n ARMOp(Dst.S(), Vector1.S(), Vector2.S()); \\\n break; \\\n } \\\n case IR::OpSize::i64Bit: { \\\n ARMOp(Dst.D(), Vector1.D(), Vector2.D()); \\\n break; \\\n } \\\n default: break; \\\n } \\\n } else { \\\n ARMOp(SubRegSize, Dst.Q(), Vector1.Q(), Vector2.Q()); \\\n } \\\n } \\\n }\n\n#define DEF_FBINOP_SCALAR_INSERT(FEXOp, ARMOp) \\\n DEF_OP(FEXOp) { \\\n const auto Op = IROp->C(); \\\n const auto ElementSize = Op->Header.ElementSize; \\\n const auto SubRegSize = ConvertSubRegSizePair248(IROp); \\\n \\\n auto ScalarEmit = [this, SubRegSize](ARMEmitter::VRegister Dst, ARMEmitter::VRegister Src1, ARMEmitter::VRegister Src2) { \\\n ARMOp(SubRegSize.Scalar, Dst, Src1, Src2); \\\n }; \\\n \\\n const auto Dst = GetVReg(Node); \\\n const auto Vector1 = GetVReg(Op->Vector1); \\\n const auto Vector2 = GetVReg(Op->Vector2); \\\n \\\n VFScalarOperation(IROp->Size, ElementSize, Op->ZeroUpperBits, ScalarEmit, Dst, Vector1, Vector2); \\\n }\n\n#define DEF_FMAOP_SCALAR_INSERT(FEXOp, ARMOp) \\\n DEF_OP(FEXOp) { \\\n const auto Op = IROp->C(); \\\n const auto ElementSize = Op->Header.ElementSize; \\\n \\\n auto ScalarEmit = [this, ElementSize](ARMEmitter::VRegister Dst, ARMEmitter::VRegister Src1, ARMEmitter::VRegister Src2, \\\n ARMEmitter::VRegister Src3) { \\\n if (ElementSize == IR::OpSize::i16Bit) { \\\n ARMOp(Dst.H(), Src1.H(), Src2.H(), Src3.H()); \\\n } else if (ElementSize == IR::OpSize::i32Bit) { \\\n ARMOp(Dst.S(), Src1.S(), Src2.S(), Src3.S()); \\\n } else if (ElementSize == IR::OpSize::i64Bit) { \\\n ARMOp(Dst.D(), Src1.D(), Src2.D(), Src3.D()); \\\n } \\\n }; \\\n \\\n const auto Dst = GetVReg(Node); \\\n const auto Upper = GetVReg(Op->Upper); \\\n const auto Vector1 = GetVReg(Op->Vector1); \\\n const auto Vector2 = GetVReg(Op->Vector2); \\\n const auto Addend = GetVReg(Op->Addend); \\\n \\\n VFScalarFMAOperation(IROp->Size, ElementSize, ScalarEmit, Dst, Upper, Vector1, Vector2, Addend); \\\n }\n\nDEF_UNOP(VAbs, abs, true)\nDEF_UNOP(VPopcount, cnt, true)\nDEF_UNOP(VNeg, neg, false)\nDEF_UNOP(VFNeg, fneg, false)\n\nDEF_BITOP(VAnd, and_)\nDEF_BITOP(VAndn, bic)\nDEF_BITOP(VOr, orr)\nDEF_BITOP(VXor, eor)\n\nDEF_BINOP(VAdd, add)\nDEF_BINOP(VSub, sub)\nDEF_BINOP(VUQAdd, uqadd)\nDEF_BINOP(VUQSub, uqsub)\nDEF_BINOP(VSQAdd, sqadd)\nDEF_BINOP(VSQSub, sqsub)\n\nDEF_ZIPOP(VZip, zip1)\nDEF_ZIPOP(VZip2, zip2)\nDEF_ZIPOP(VUnZip, uzp1)\nDEF_ZIPOP(VUnZip2, uzp2)\nDEF_ZIPOP(VTrn, trn1)\nDEF_ZIPOP(VTrn2, trn2)\n\nDEF_FUNOP(VFSqrt, fsqrt)\nDEF_FUNOP(VFAbs, fabs)\n\nDEF_FBINOP(VFAdd, fadd)\nDEF_FBINOP(VFSub, fsub)\nDEF_FBINOP(VFMul, fmul)\n\nDEF_FBINOP_SCALAR_INSERT(VFAddScalarInsert, fadd)\nDEF_FBINOP_SCALAR_INSERT(VFSubScalarInsert, fsub)\nDEF_FBINOP_SCALAR_INSERT(VFMulScalarInsert, fmul)\nDEF_FBINOP_SCALAR_INSERT(VFDivScalarInsert, fdiv)\n\nDEF_FMAOP_SCALAR_INSERT(VFMLAScalarInsert, fmadd)\nDEF_FMAOP_SCALAR_INSERT(VFMLSScalarInsert, fnmsub)\nDEF_FMAOP_SCALAR_INSERT(VFNMLAScalarInsert, fmsub)\nDEF_FMAOP_SCALAR_INSERT(VFNMLSScalarInsert, fnmadd)\n\nvoid Arm64JITCore::VFScalarFMAOperation(IR::OpSize OpSize, IR::OpSize ElementSize, ScalarFMAOpCaller ScalarEmit, ARMEmitter::VRegister Dst,\n ARMEmitter::VRegister Upper, ARMEmitter::VRegister Vector1, ARMEmitter::VRegister Vector2,\n ARMEmitter::VRegister Addend) {\n LOGMAN_THROW_A_FMT(OpSize == IR::OpSize::i128Bit, \"256-bit unsupported\", __func__);\n\n LOGMAN_THROW_A_FMT(ElementSize == IR::OpSize::i16Bit || ElementSize == IR::OpSize::i32Bit || ElementSize == IR::OpSize::i64Bit, \"Invalid \"\n \"size\");\n const auto SubRegSize = ARMEmitter::ToVectorSizePair(ElementSize == IR::OpSize::i16Bit ? ARMEmitter::SubRegSize::i16Bit :\n ElementSize == IR::OpSize::i32Bit ? ARMEmitter::SubRegSize::i32Bit :\n ARMEmitter::SubRegSize::i64Bit);\n\n if (Dst != Upper) {\n // If destination is not tied, move the upper bits to the destination first.\n mov(Dst.Q(), Upper.Q());\n }\n\n if (HostSupportsAFP && Dst == Addend) {\n ///< Exactly matches ARM scalar FMA semantics\n // If the host CPU supports AFP then scalar does an insert without modifying upper bits.\n ScalarEmit(Dst, Vector1, Vector2, Addend);\n } else {\n // Host doesn't support AFP, need to emit in to a temporary then insert.\n ScalarEmit(VTMP1, Vector1, Vector2, Addend);\n ins(SubRegSize.Vector, Dst.Q(), 0, VTMP1.Q(), 0);\n }\n}\n\n// VFScalarOperation performs the operation described through ScalarEmit between Vector1 and Vector2,\n// storing it into Dst. This is a scalar operation, so the only lowest element of each vector is used for the operation.\n// The result is stored into the destination. The untouched bits of the destination come from Vector1, unless it's a 256 vector\n// and ZeroUpperBits is true, in which case the upper bits are zero.\nvoid Arm64JITCore::VFScalarOperation(IR::OpSize OpSize, IR::OpSize ElementSize, bool ZeroUpperBits, ScalarBinaryOpCaller ScalarEmit,\n ARMEmitter::VRegister Dst, ARMEmitter::VRegister Vector1, ARMEmitter::VRegister Vector2) {\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n LOGMAN_THROW_A_FMT(Is256Bit || !ZeroUpperBits, \"128-bit operation doesn't support ZeroUpperBits in {}\", __func__);\n\n // Bit of a tricky detail.\n // The upper bits of the destination comes from Vector1.\n LOGMAN_THROW_A_FMT(ElementSize == IR::OpSize::i16Bit || ElementSize == IR::OpSize::i32Bit || ElementSize == IR::OpSize::i64Bit, \"Invalid \"\n \"size\");\n const auto SubRegSize = ARMEmitter::ToVectorSizePair(ElementSize == IR::OpSize::i16Bit ? ARMEmitter::SubRegSize::i16Bit :\n ElementSize == IR::OpSize::i32Bit ? ARMEmitter::SubRegSize::i32Bit :\n ARMEmitter::SubRegSize::i64Bit);\n\n constexpr auto Predicate = ARMEmitter::PReg::p0;\n\n if (Dst == Vector1) {\n if (ZeroUpperBits) {\n // When zeroing the upper 128-bits we just use an ASIMD move.\n mov(Dst.Q(), Vector1.Q());\n }\n\n if (HostSupportsAFP) {\n // If the host CPU supports AFP then scalar does an insert without modifying upper bits.\n ScalarEmit(Dst, Vector1, Vector2);\n } else {\n // If AFP is unsupported then the operation result goes in to a temporary.\n // and then it gets inserted.\n ScalarEmit(VTMP1, Vector1, Vector2);\n if (!ZeroUpperBits && Is256Bit) {\n ptrue(SubRegSize.Vector, Predicate, ARMEmitter::PredicatePattern::SVE_VL1);\n mov(SubRegSize.Vector, Dst.Z(), Predicate.Merging(), VTMP1.Z());\n } else {\n ins(SubRegSize.Vector, Dst.Q(), 0, VTMP1.Q(), 0);\n }\n }\n } else if (Dst != Vector2) { // Dst different from both Vector1 and Vector2\n if (Is256Bit && !ZeroUpperBits) {\n mov(Dst.Z(), Vector1.Z());\n } else {\n mov(Dst.Q(), Vector1.Q());\n }\n\n if (HostSupportsAFP) {\n ScalarEmit(Dst, Vector1, Vector2);\n } else {\n ScalarEmit(VTMP1, Vector1, Vector2);\n if (!ZeroUpperBits && Is256Bit) {\n ptrue(SubRegSize.Vector, Predicate, ARMEmitter::PredicatePattern::SVE_VL1);\n mov(SubRegSize.Vector, Dst.Z(), Predicate.Merging(), VTMP1.Z());\n } else {\n ins(SubRegSize.Vector, Dst.Q(), 0, VTMP1.Q(), 0);\n }\n }\n } else { // Dst same as Vector2\n\n ScalarEmit(VTMP1, Vector1, Vector2);\n\n if (!ZeroUpperBits && Is256Bit) {\n mov(Dst.Z(), Vector1.Z());\n ptrue(SubRegSize.Vector, Predicate, ARMEmitter::PredicatePattern::SVE_VL1);\n mov(SubRegSize.Vector, Dst.Z(), Predicate.Merging(), VTMP1.Z());\n } else {\n mov(Dst.Q(), Vector1.Q());\n ins(SubRegSize.Vector, Dst.Q(), 0, VTMP1.Q(), 0);\n }\n }\n}\n\n// Similarly to VFScalarOperation it performs the operation described through ScalarEmit operating on Vector2.\n// However the result of the scalar operation is inserted into Vector1 and moved to Destination.\n// The untouched bits of the destination come from Vector1, unless it's a 256 vector\n// and ZeroUpperBits is true, in which case the upper bits are zero.\nvoid Arm64JITCore::VFScalarUnaryOperation(IR::OpSize OpSize, IR::OpSize ElementSize, bool ZeroUpperBits, ScalarUnaryOpCaller ScalarEmit,\n ARMEmitter::VRegister Dst, ARMEmitter::VRegister Vector1,\n std::variant Vector2) {\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n LOGMAN_THROW_A_FMT(Is256Bit || !ZeroUpperBits, \"128-bit operation doesn't support ZeroUpperBits in {}\", __func__);\n\n LOGMAN_THROW_A_FMT(ElementSize == IR::OpSize::i16Bit || ElementSize == IR::OpSize::i32Bit || ElementSize == IR::OpSize::i64Bit, \"Invalid \"\n \"size\");\n const auto SubRegSize = ARMEmitter::ToVectorSizePair(ElementSize == IR::OpSize::i16Bit ? ARMEmitter::SubRegSize::i16Bit :\n ElementSize == IR::OpSize::i32Bit ? ARMEmitter::SubRegSize::i32Bit :\n ARMEmitter::SubRegSize::i64Bit);\n\n constexpr auto Predicate = ARMEmitter::PReg::p0;\n bool DstOverlapsVector2 = false;\n if (const auto* Vector2Reg = std::get_if(&Vector2)) {\n DstOverlapsVector2 = Dst == *Vector2Reg;\n }\n\n if (Dst == Vector1) {\n if (ZeroUpperBits) {\n // When zeroing the upper 128-bits we just use an ASIMD move.\n mov(Dst.Q(), Vector1.Q());\n }\n\n if (HostSupportsAFP) { // or Dst (here Dst == Vector1)\n // If the host CPU supports AFP then scalar does an insert without modifying upper bits.\n ScalarEmit(Dst, Vector2);\n } else {\n // If AFP is unsupported then the operation result goes in to a temporary.\n // and then it gets inserted.\n ScalarEmit(VTMP1, Vector2);\n if (!ZeroUpperBits && Is256Bit) {\n ptrue(SubRegSize.Vector, Predicate, ARMEmitter::PredicatePattern::SVE_VL1);\n mov(SubRegSize.Vector, Dst.Z(), Predicate.Merging(), VTMP1.Z());\n } else {\n ins(SubRegSize.Vector, Dst.Q(), 0, VTMP1.Q(), 0);\n }\n }\n } else if (!DstOverlapsVector2) {\n if (!ZeroUpperBits && Is256Bit) {\n mov(Dst.Z(), Vector1.Z());\n } else {\n mov(Dst.Q(), Vector1.Q());\n }\n\n if (HostSupportsAFP) {\n ScalarEmit(Dst, Vector2);\n } else {\n ScalarEmit(VTMP1, Vector2);\n if (!ZeroUpperBits && Is256Bit) {\n ptrue(SubRegSize.Vector, Predicate, ARMEmitter::PredicatePattern::SVE_VL1);\n mov(SubRegSize.Vector, Dst.Z(), Predicate.Merging(), VTMP1.Z());\n } else {\n ins(SubRegSize.Vector, Dst.Q(), 0, VTMP1.Q(), 0);\n }\n }\n } else {\n // Destination intersects Vector2, can't do anything optimal in this case.\n // Do the scalar operation first and then move and insert.\n ScalarEmit(VTMP1, Vector2);\n\n if (!ZeroUpperBits && Is256Bit) {\n mov(Dst.Z(), Vector1.Z());\n ptrue(SubRegSize.Vector, Predicate, ARMEmitter::PredicatePattern::SVE_VL1);\n mov(SubRegSize.Vector, Dst.Z(), Predicate.Merging(), VTMP1.Z());\n } else {\n mov(Dst.Q(), Vector1.Q());\n ins(SubRegSize.Vector, Dst.Q(), 0, VTMP1.Q(), 0);\n }\n }\n}\n\nDEF_OP(VFMinScalarInsert) {\n const auto Op = IROp->C();\n const auto ElementSize = Op->Header.ElementSize;\n const auto SubRegSize = ConvertSubRegSizePair248(IROp);\n\n auto ScalarEmit = [this, SubRegSize](ARMEmitter::VRegister Dst, ARMEmitter::VRegister Src1, ARMEmitter::VRegister Src2) {\n if (HostSupportsAFP) {\n // AFP.AH lets fmin behave like x86 min\n fmin(SubRegSize.Scalar, Dst, Src1, Src2);\n } else {\n // Only take the first operand if it is strictly less. Otherwise take\n // the second. This emulates all the weird x86 rules for signed zero and\n // NaNs. No, they're not IEEE-754 semantics.\n fcmp(SubRegSize.Scalar, Src1, Src2);\n fcsel(SubRegSize.Scalar, Dst, Src1, Src2, ARMEmitter::Condition::CC_MI);\n }\n };\n\n // Bit of a tricky detail.\n // The upper bits of the destination comes from the first source.\n const auto Dst = GetVReg(Node);\n const auto Vector1 = GetVReg(Op->Vector1);\n const auto Vector2 = GetVReg(Op->Vector2);\n\n VFScalarOperation(IROp->Size, ElementSize, Op->ZeroUpperBits, ScalarEmit, Dst, Vector1, Vector2);\n}\n\nDEF_OP(VFMaxScalarInsert) {\n const auto Op = IROp->C();\n const auto ElementSize = Op->Header.ElementSize;\n const auto SubRegSize = ConvertSubRegSizePair248(IROp);\n\n // AFP can make this more optimal.\n auto ScalarEmit = [this, SubRegSize](ARMEmitter::VRegister Dst, ARMEmitter::VRegister Src1, ARMEmitter::VRegister Src2) {\n if (HostSupportsAFP) {\n // AFP.AH lets fmax behave like x86 max\n fmax(SubRegSize.Scalar, Dst, Src1, Src2);\n } else {\n // Only take the first operand if it is strictly greater. See fmin.\n fcmp(SubRegSize.Scalar, Src1, Src2);\n fcsel(SubRegSize.Scalar, Dst, Src1, Src2, ARMEmitter::Condition::CC_GT);\n }\n };\n\n // Bit of a tricky detail.\n // The upper bits of the destination comes from the first source.\n const auto Dst = GetVReg(Node);\n const auto Vector1 = GetVReg(Op->Vector1);\n const auto Vector2 = GetVReg(Op->Vector2);\n\n VFScalarOperation(IROp->Size, ElementSize, Op->ZeroUpperBits, ScalarEmit, Dst, Vector1, Vector2);\n}\n\nDEF_OP(VFSqrtScalarInsert) {\n const auto Op = IROp->C();\n const auto ElementSize = Op->Header.ElementSize;\n const auto SubRegSize = ConvertSubRegSizePair248(IROp);\n\n auto ScalarEmit = [this, SubRegSize](ARMEmitter::VRegister Dst, std::variant SrcVar) {\n auto Src = *std::get_if(&SrcVar);\n fsqrt(SubRegSize.Scalar, Dst, Src);\n };\n\n // Bit of a tricky detail.\n // The upper bits of the destination comes from the first source.\n const auto Dst = GetVReg(Node);\n const auto Vector1 = GetVReg(Op->Vector1);\n const auto Vector2 = GetVReg(Op->Vector2);\n\n VFScalarUnaryOperation(IROp->Size, ElementSize, Op->ZeroUpperBits, ScalarEmit, Dst, Vector1, Vector2);\n}\n\nDEF_OP(VFRSqrtScalarInsert) {\n const auto Op = IROp->C();\n const auto ElementSize = Op->Header.ElementSize;\n const auto SubRegSize = ConvertSubRegSizePair248(IROp);\n\n auto ScalarEmit = [this, SubRegSize](ARMEmitter::VRegister Dst, std::variant SrcVar) {\n auto Src = *std::get_if(&SrcVar);\n\n fmov(SubRegSize.Scalar, VTMP1.Q(), 1.0f);\n fsqrt(SubRegSize.Scalar, VTMP2, Src);\n if (HostSupportsAFP) {\n fdiv(SubRegSize.Scalar, VTMP1, VTMP1, VTMP2);\n ins(SubRegSize.Vector, Dst, 0, VTMP1, 0);\n } else {\n fdiv(SubRegSize.Scalar, Dst, VTMP1, VTMP2);\n }\n };\n\n auto ScalarEmitRPRES = [this, SubRegSize](ARMEmitter::VRegister Dst, std::variant SrcVar) {\n auto Src = *std::get_if(&SrcVar);\n frsqrte(SubRegSize.Scalar, Dst.D(), Src.D());\n };\n\n std::array Handlers = {\n ScalarEmit,\n ScalarEmitRPRES,\n };\n const auto HandlerIndex = ElementSize == IR::OpSize::i32Bit && HostSupportsRPRES ? 1 : 0;\n\n // Bit of a tricky detail.\n // The upper bits of the destination comes from the first source.\n const auto Dst = GetVReg(Node);\n const auto Vector1 = GetVReg(Op->Vector1);\n const auto Vector2 = GetVReg(Op->Vector2);\n\n VFScalarUnaryOperation(IROp->Size, ElementSize, Op->ZeroUpperBits, Handlers[HandlerIndex], Dst, Vector1, Vector2);\n}\n\nDEF_OP(VFRecpScalarInsert) {\n const auto Op = IROp->C();\n const auto ElementSize = Op->Header.ElementSize;\n const auto SubRegSize = ConvertSubRegSizePair248(IROp);\n\n auto ScalarEmit = [this, SubRegSize](ARMEmitter::VRegister Dst, std::variant SrcVar) {\n auto Src = *std::get_if(&SrcVar);\n\n fmov(SubRegSize.Scalar, VTMP1.Q(), 1.0f);\n if (HostSupportsAFP) {\n fdiv(SubRegSize.Scalar, VTMP1, VTMP1, Src);\n ins(SubRegSize.Vector, Dst, 0, VTMP1, 0);\n } else {\n fdiv(SubRegSize.Scalar, Dst, VTMP1, Src);\n }\n };\n\n auto ScalarEmitRPRES = [this, SubRegSize](ARMEmitter::VRegister Dst, std::variant SrcVar) {\n auto Src = *std::get_if(&SrcVar);\n frecpe(SubRegSize.Scalar, Dst, Src);\n };\n\n std::array Handlers = {\n ScalarEmit,\n ScalarEmitRPRES,\n };\n const auto HandlerIndex = ElementSize == IR::OpSize::i32Bit && HostSupportsRPRES ? 1 : 0;\n\n // Bit of a tricky detail.\n // The upper bits of the destination comes from the first source.\n const auto Dst = GetVReg(Node);\n const auto Vector1 = GetVReg(Op->Vector1);\n const auto Vector2 = GetVReg(Op->Vector2);\n\n VFScalarUnaryOperation(IROp->Size, ElementSize, Op->ZeroUpperBits, Handlers[HandlerIndex], Dst, Vector1, Vector2);\n}\n\nDEF_OP(VFToFScalarInsert) {\n const auto Op = IROp->C();\n const auto ElementSize = Op->Header.ElementSize;\n const uint16_t Conv = (IR::OpSizeToSize(Op->Header.ElementSize) << 8) | IR::OpSizeToSize(Op->SrcElementSize);\n\n auto ScalarEmit = [this, Conv](ARMEmitter::VRegister Dst, std::variant SrcVar) {\n auto Src = *std::get_if(&SrcVar);\n\n switch (Conv) {\n case 0x0204: { // Half <- Float\n fcvt(Dst.H(), Src.S());\n break;\n }\n case 0x0208: { // Half <- Double\n fcvt(Dst.H(), Src.D());\n break;\n }\n case 0x0402: { // Float <- Half\n fcvt(Dst.S(), Src.H());\n break;\n }\n case 0x0802: { // Double <- Half\n fcvt(Dst.D(), Src.H());\n break;\n }\n case 0x0804: { // Double <- Float\n fcvt(Dst.D(), Src.S());\n break;\n }\n case 0x0408: { // Float <- Double\n fcvt(Dst.S(), Src.D());\n break;\n }\n default: LOGMAN_MSG_A_FMT(\"Unknown FCVT sizes: 0x{:x}\", Conv);\n }\n };\n\n // Bit of a tricky detail.\n // The upper bits of the destination comes from the first source.\n const auto Dst = GetVReg(Node);\n const auto Vector1 = GetVReg(Op->Vector1);\n const auto Vector2 = GetVReg(Op->Vector2);\n\n VFScalarUnaryOperation(IROp->Size, ElementSize, Op->ZeroUpperBits, ScalarEmit, Dst, Vector1, Vector2);\n}\n\nDEF_OP(VSToFVectorInsert) {\n const auto Op = IROp->C();\n const auto ElementSize = Op->Header.ElementSize;\n const auto HasTwoElements = Op->HasTwoElements;\n\n LOGMAN_THROW_A_FMT(ElementSize == IR::OpSize::i32Bit || ElementSize == IR::OpSize::i64Bit, \"Invalid size\");\n if (HasTwoElements) {\n LOGMAN_THROW_A_FMT(ElementSize == IR::OpSize::i32Bit, \"Can't have two elements for 8-byte size\");\n }\n\n auto ScalarEmit = [this, ElementSize, HasTwoElements](ARMEmitter::VRegister Dst, std::variant SrcVar) {\n auto Src = *std::get_if(&SrcVar);\n if (ElementSize == IR::OpSize::i32Bit) {\n if (HasTwoElements) {\n scvtf(ARMEmitter::SubRegSize::i32Bit, Dst.D(), Src.D());\n } else {\n scvtf(ARMEmitter::ScalarRegSize::i32Bit, Dst.S(), Src.S());\n }\n } else {\n scvtf(ARMEmitter::ScalarRegSize::i64Bit, Dst.D(), Src.D());\n }\n };\n\n // Bit of a tricky detail.\n // The upper bits of the destination comes from the first source.\n const auto Dst = GetVReg(Node);\n const auto Vector1 = GetVReg(Op->Vector1);\n const auto Vector2 = GetVReg(Op->Vector2);\n\n // Claim the element size is 8-bytes.\n // Might be scalar 8-byte (cvtsi2ss xmm0, rax)\n // Might be vector i32v2 (cvtpi2ps xmm0, mm0)\n if (!HasTwoElements) {\n VFScalarUnaryOperation(IROp->Size, ElementSize, Op->ZeroUpperBits, ScalarEmit, Dst, Vector1, Vector2);\n return;\n }\n\n // Dealing with the odd case of this being actually a vector operation rather than scalar.\n const auto Is256Bit = IROp->Size == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n constexpr auto Predicate = ARMEmitter::PReg::p0;\n\n ScalarEmit(VTMP1, Vector2);\n if (!Op->ZeroUpperBits && Is256Bit) {\n if (Dst != Vector1) {\n mov(Dst.Z(), Vector1.Z());\n }\n ptrue(ARMEmitter::SubRegSize::i64Bit, Predicate, ARMEmitter::PredicatePattern::SVE_VL1);\n mov(ARMEmitter::SubRegSize::i64Bit, Dst.Z(), Predicate.Merging(), VTMP1.Z());\n } else {\n if (Dst != Vector1) {\n mov(Dst.Q(), Vector1.Q());\n }\n ins(ARMEmitter::SubRegSize::i64Bit, Dst.Q(), 0, VTMP1.Q(), 0);\n }\n}\n\nDEF_OP(VSToFGPRInsert) {\n const auto Op = IROp->C();\n\n const auto ElementSize = Op->Header.ElementSize;\n const uint16_t Conv = (IR::OpSizeToSize(ElementSize) << 8) | IR::OpSizeToSize(Op->SrcElementSize);\n\n auto ScalarEmit = [this, Conv](ARMEmitter::VRegister Dst, std::variant SrcVar) {\n auto Src = *std::get_if(&SrcVar);\n\n switch (Conv) {\n case 0x0204: { // Half <- int32_t\n scvtf(ARMEmitter::Size::i32Bit, Dst.H(), Src);\n break;\n }\n case 0x0208: { // Half <- int64_t\n scvtf(ARMEmitter::Size::i64Bit, Dst.H(), Src);\n break;\n }\n case 0x0404: { // Float <- int32_t\n scvtf(ARMEmitter::Size::i32Bit, Dst.S(), Src);\n break;\n }\n case 0x0408: { // Float <- int64_t\n scvtf(ARMEmitter::Size::i64Bit, Dst.S(), Src);\n break;\n }\n case 0x0804: { // Double <- int32_t\n scvtf(ARMEmitter::Size::i32Bit, Dst.D(), Src);\n break;\n }\n case 0x0808: { // Double <- int64_t\n scvtf(ARMEmitter::Size::i64Bit, Dst.D(), Src);\n break;\n }\n default: LOGMAN_MSG_A_FMT(\"Unhandled conversion mask: Mask=0x{:04x}\", Conv); break;\n }\n };\n\n\n // Bit of a tricky detail.\n // The upper bits of the destination comes from the first source.\n const auto Dst = GetVReg(Node);\n const auto Vector = GetVReg(Op->Vector);\n const auto GPR = GetReg(Op->Src);\n\n VFScalarUnaryOperation(IROp->Size, ElementSize, Op->ZeroUpperBits, ScalarEmit, Dst, Vector, GPR);\n}\n\nDEF_OP(VFToIScalarInsert) {\n const auto Op = IROp->C();\n const auto ElementSize = Op->Header.ElementSize;\n const auto SubRegSize = ConvertSubRegSizePair248(IROp);\n const auto RoundMode = Op->Round;\n\n auto ScalarEmit = [this, SubRegSize, RoundMode](ARMEmitter::VRegister Dst, std::variant SrcVar) {\n auto Src = *std::get_if(&SrcVar);\n\n switch (RoundMode) {\n case IR::RoundMode::Nearest: frintn(SubRegSize.Scalar, Dst, Src); break;\n case IR::RoundMode::NegInfinity: frintm(SubRegSize.Scalar, Dst, Src); break;\n case IR::RoundMode::PosInfinity: frintp(SubRegSize.Scalar, Dst, Src); break;\n case IR::RoundMode::TowardsZero: frintz(SubRegSize.Scalar, Dst, Src); break;\n case IR::RoundMode::Host: frinti(SubRegSize.Scalar, Dst, Src); break;\n }\n };\n\n // Bit of a tricky detail.\n // The upper bits of the destination comes from the first source.\n const auto Dst = GetVReg(Node);\n const auto Vector1 = GetVReg(Op->Vector1);\n const auto Vector2 = GetVReg(Op->Vector2);\n\n VFScalarUnaryOperation(IROp->Size, ElementSize, Op->ZeroUpperBits, ScalarEmit, Dst, Vector1, Vector2);\n}\n\nDEF_OP(VFCMPScalarInsert) {\n const auto Op = IROp->C();\n const auto ElementSize = Op->Header.ElementSize;\n const auto SubRegSize = ConvertSubRegSizePair248(IROp);\n\n const auto ZeroUpperBits = Op->ZeroUpperBits;\n const auto Is256Bit = IROp->Size == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n auto ScalarEmitEQ = [this, SubRegSize](ARMEmitter::VRegister Dst, ARMEmitter::VRegister Src1, ARMEmitter::VRegister Src2) {\n switch (SubRegSize.Scalar) {\n case ARMEmitter::ScalarRegSize::i16Bit: {\n fcmeq(Dst.H(), Src2.H(), Src1.H());\n break;\n }\n case ARMEmitter::ScalarRegSize::i32Bit:\n case ARMEmitter::ScalarRegSize::i64Bit: fcmeq(SubRegSize.Scalar, Dst, Src2, Src1); break;\n default: break;\n }\n };\n auto ScalarEmitLT = [this, SubRegSize](ARMEmitter::VRegister Dst, ARMEmitter::VRegister Src1, ARMEmitter::VRegister Src2) {\n switch (SubRegSize.Scalar) {\n case ARMEmitter::ScalarRegSize::i16Bit: {\n fcmgt(Dst.H(), Src2.H(), Src1.H());\n break;\n }\n case ARMEmitter::ScalarRegSize::i32Bit:\n case ARMEmitter::ScalarRegSize::i64Bit: fcmgt(SubRegSize.Scalar, Dst, Src2, Src1); break;\n default: break;\n }\n };\n auto ScalarEmitLE = [this, SubRegSize](ARMEmitter::VRegister Dst, ARMEmitter::VRegister Src1, ARMEmitter::VRegister Src2) {\n switch (SubRegSize.Scalar) {\n case ARMEmitter::ScalarRegSize::i16Bit: {\n fcmge(Dst.H(), Src2.H(), Src1.H());\n break;\n }\n case ARMEmitter::ScalarRegSize::i32Bit:\n case ARMEmitter::ScalarRegSize::i64Bit: fcmge(SubRegSize.Scalar, Dst, Src2, Src1); break;\n default: break;\n }\n };\n auto ScalarEmitUNO = [this, SubRegSize, ZeroUpperBits, Is256Bit](ARMEmitter::VRegister Dst, ARMEmitter::VRegister Src1,\n ARMEmitter::VRegister Src2) {\n switch (SubRegSize.Scalar) {\n case ARMEmitter::ScalarRegSize::i16Bit: {\n fcmge(VTMP1.H(), Src1.H(), Src2.H());\n fcmgt(VTMP2.H(), Src2.H(), Src1.H());\n break;\n }\n case ARMEmitter::ScalarRegSize::i32Bit:\n case ARMEmitter::ScalarRegSize::i64Bit:\n fcmge(SubRegSize.Scalar, VTMP1, Src1, Src2);\n fcmgt(SubRegSize.Scalar, VTMP2, Src2, Src1);\n break;\n default: break;\n }\n // If the destination is a temporary then it is going to do an insert after the operation.\n // This means this operation can avoid a redundant insert in this case.\n const bool DstIsTemp = Dst == VTMP1;\n\n // Combine results and invert directly in VTMP1.\n orr(VTMP1.D(), VTMP1.D(), VTMP2.D());\n mvn(ARMEmitter::SubRegSize::i8Bit, VTMP1.D(), VTMP1.D());\n\n if (!DstIsTemp) {\n // If the destination doesn't overlap VTMP1, then we need to insert the final result.\n // This only happens in the case that the host supports AFP.\n if (!ZeroUpperBits && Is256Bit) {\n constexpr auto Predicate = ARMEmitter::PReg::p0;\n ptrue(SubRegSize.Vector, Predicate, ARMEmitter::PredicatePattern::SVE_VL1);\n mov(SubRegSize.Vector, Dst.Z(), Predicate.Merging(), VTMP1.Z());\n } else {\n ins(SubRegSize.Vector, Dst.Q(), 0, VTMP1.Q(), 0);\n }\n }\n };\n auto ScalarEmitNEQ = [this, SubRegSize, ZeroUpperBits, Is256Bit](ARMEmitter::VRegister Dst, ARMEmitter::VRegister Src1,\n ARMEmitter::VRegister Src2) {\n switch (SubRegSize.Scalar) {\n case ARMEmitter::ScalarRegSize::i16Bit: {\n fcmeq(VTMP1.H(), Src2.H(), Src1.H());\n break;\n }\n case ARMEmitter::ScalarRegSize::i32Bit:\n case ARMEmitter::ScalarRegSize::i64Bit: fcmeq(SubRegSize.Scalar, VTMP1, Src2, Src1); break;\n default: break;\n }\n // If the destination is a temporary then it is going to do an insert after the operation.\n // This means this operation can avoid a redundant insert in this case.\n const bool DstIsTemp = Dst == VTMP1;\n\n // Invert directly in VTMP1.\n mvn(ARMEmitter::SubRegSize::i8Bit, VTMP1.D(), VTMP1.D());\n\n if (!DstIsTemp) {\n // If the destination doesn't overlap VTMP1, then we need to insert the final result.\n // This only happens in the case that the host supports AFP.\n if (!ZeroUpperBits && Is256Bit) {\n constexpr auto Predicate = ARMEmitter::PReg::p0;\n ptrue(SubRegSize.Vector, Predicate, ARMEmitter::PredicatePattern::SVE_VL1);\n mov(SubRegSize.Vector, Dst.Z(), Predicate.Merging(), VTMP1.Z());\n } else {\n ins(SubRegSize.Vector, Dst.Q(), 0, VTMP1.Q(), 0);\n }\n }\n };\n auto ScalarEmitORD = [this, SubRegSize, ZeroUpperBits, Is256Bit](ARMEmitter::VRegister Dst, ARMEmitter::VRegister Src1,\n ARMEmitter::VRegister Src2) {\n switch (SubRegSize.Scalar) {\n case ARMEmitter::ScalarRegSize::i16Bit: {\n fcmge(VTMP1.H(), Src1.H(), Src2.H());\n fcmgt(VTMP2.H(), Src2.H(), Src1.H());\n break;\n }\n case ARMEmitter::ScalarRegSize::i32Bit:\n case ARMEmitter::ScalarRegSize::i64Bit:\n fcmge(SubRegSize.Scalar, VTMP1, Src1, Src2);\n fcmgt(SubRegSize.Scalar, VTMP2, Src2, Src1);\n break;\n default: break;\n }\n // If the destination is a temporary then it is going to do an insert after the operation.\n // This means this operation can avoid a redundant insert in this case.\n const bool DstIsTemp = Dst == VTMP1;\n\n // Combine results directly in VTMP1.\n orr(VTMP1.D(), VTMP1.D(), VTMP2.D());\n\n if (!DstIsTemp) {\n // If the destination doesn't overlap VTMP1, then we need to insert the final result.\n // This only happens in the case that the host supports AFP.\n if (!ZeroUpperBits && Is256Bit) {\n constexpr auto Predicate = ARMEmitter::PReg::p0;\n ptrue(SubRegSize.Vector, Predicate, ARMEmitter::PredicatePattern::SVE_VL1);\n mov(SubRegSize.Vector, Dst.Z(), Predicate.Merging(), VTMP1.Z());\n } else {\n ins(SubRegSize.Vector, Dst.Q(), 0, VTMP1.Q(), 0);\n }\n }\n };\n\n std::array Funcs = {{\n ScalarEmitEQ,\n ScalarEmitLT,\n ScalarEmitLE,\n ScalarEmitUNO,\n ScalarEmitNEQ,\n ScalarEmitORD,\n }};\n\n // Bit of a tricky detail.\n // The upper bits of the destination comes from the first source.\n const auto Dst = GetVReg(Node);\n const auto Vector1 = GetVReg(Op->Vector1);\n const auto Vector2 = GetVReg(Op->Vector2);\n\n VFScalarOperation(IROp->Size, ElementSize, Op->ZeroUpperBits, Funcs[FEXCore::ToUnderlying(Op->Op)], Dst, Vector1, Vector2);\n}\n\nDEF_OP(VectorImm) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n const auto ElementSize = Op->Header.ElementSize;\n const auto SubRegSize = ConvertSubRegSize8(IROp);\n\n const auto Dst = GetVReg(Node);\n\n if (HostSupportsSVE256 && Is256Bit) {\n LOGMAN_THROW_A_FMT(Op->ShiftAmount == 0, \"SVE VectorImm doesn't support a shift\");\n if (ElementSize > IR::OpSize::i8Bit && (Op->Immediate & 0x80)) {\n // SVE dup uses sign extension where VectorImm wants zext\n LoadConstant(ARMEmitter::Size::i64Bit, TMP1, Op->Immediate);\n dup(SubRegSize, Dst.Z(), TMP1);\n } else {\n dup_imm(SubRegSize, Dst.Z(), static_cast(Op->Immediate));\n }\n } else {\n if (ElementSize == IR::OpSize::i64Bit) {\n // movi with 64bit element size doesn't do what we want here\n LoadConstant(ARMEmitter::Size::i64Bit, TMP1, static_cast(Op->Immediate) << Op->ShiftAmount);\n dup(SubRegSize, Dst.Q(), TMP1.R());\n } else {\n movi(SubRegSize, Dst.Q(), Op->Immediate, Op->ShiftAmount);\n }\n }\n}\n\nDEF_OP(LoadNamedVectorConstant) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto Dst = GetVReg(Node);\n switch (Op->Constant) {\n case FEXCore::IR::NamedVectorConstant::NAMED_VECTOR_ZERO: movi(ARMEmitter::SubRegSize::i64Bit, Dst.Q(), 0); return;\n default:\n // Intentionally doing nothing.\n break;\n }\n\n if (HostSupportsSVE128) {\n switch (Op->Constant) {\n case FEXCore::IR::NamedVectorConstant::NAMED_VECTOR_MOVMSKPS_SHIFT: index(ARMEmitter::SubRegSize::i32Bit, Dst.Z(), 0, 1); return;\n default:\n // Intentionally doing nothing.\n break;\n }\n }\n // Load the pointer.\n auto GenerateMemOperand = [this](IR::OpSize OpSize, uint32_t NamedConstant, ARMEmitter::Register Base) {\n const auto ConstantOffset = ARRAY_OFFSETOF(FEXCore::Core::CpuStateFrame, Pointers.NamedVectorConstants, NamedConstant);\n\n if (ConstantOffset <= 255 || // Unscaled 9-bit signed\n ((ConstantOffset & (IR::OpSizeToSize(OpSize) - 1)) == 0 &&\n FEXCore::DividePow2(ConstantOffset, IR::OpSizeToSize(OpSize)) <= 4095)) /* 12-bit unsigned scaled */ {\n return ARMEmitter::ExtendedMemOperand(Base.X(), ARMEmitter::IndexType::OFFSET, ConstantOffset);\n }\n\n ldr(TMP1, STATE_PTR_IDX(CpuStateFrame, Pointers.NamedVectorConstantPointers, NamedConstant));\n return ARMEmitter::ExtendedMemOperand(TMP1, ARMEmitter::IndexType::OFFSET, 0);\n };\n\n if (OpSize == IR::OpSize::i256Bit) {\n // Handle SVE 32-byte variant upfront.\n ldr(TMP1, STATE_PTR_IDX(CpuStateFrame, Pointers.NamedVectorConstantPointers, Op->Constant));\n ld1b(Dst.Z(), PRED_TMP_32B.Zeroing(), TMP1, 0);\n return;\n }\n\n auto MemOperand = GenerateMemOperand(OpSize, Op->Constant, STATE);\n switch (OpSize) {\n case IR::OpSize::i8Bit: ldrb(Dst, MemOperand); break;\n case IR::OpSize::i16Bit: ldrh(Dst, MemOperand); break;\n case IR::OpSize::i32Bit: ldr(Dst.S(), MemOperand); break;\n case IR::OpSize::i64Bit: ldr(Dst.D(), MemOperand); break;\n case IR::OpSize::i128Bit: ldr(Dst.Q(), MemOperand); break;\n default: LOGMAN_MSG_A_FMT(\"Unhandled {} size: {}\", __func__, OpSize); break;\n }\n}\nDEF_OP(LoadNamedVectorIndexedConstant) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto Dst = GetVReg(Node);\n\n // Load the pointer.\n ldr(TMP1, STATE_PTR_IDX(CpuStateFrame, Pointers.IndexedNamedVectorConstantPointers, Op->Constant));\n\n switch (OpSize) {\n case IR::OpSize::i8Bit: ldrb(Dst, TMP1, Op->Index); break;\n case IR::OpSize::i16Bit: ldrh(Dst, TMP1, Op->Index); break;\n case IR::OpSize::i32Bit: ldr(Dst.S(), TMP1, Op->Index); break;\n case IR::OpSize::i64Bit: ldr(Dst.D(), TMP1, Op->Index); break;\n case IR::OpSize::i128Bit: ldr(Dst.Q(), TMP1, Op->Index); break;\n case IR::OpSize::i256Bit: {\n add(ARMEmitter::Size::i64Bit, TMP1, TMP1, Op->Index);\n ld1b(Dst.Z(), PRED_TMP_32B.Zeroing(), TMP1, 0);\n break;\n }\n default: LOGMAN_MSG_A_FMT(\"Unhandled {} size: {}\", __func__, OpSize); break;\n }\n}\n\nDEF_OP(VMov) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto Dst = GetVReg(Node);\n const auto Source = GetVReg(Op->Source);\n\n switch (OpSize) {\n case IR::OpSize::i8Bit: {\n movi(ARMEmitter::SubRegSize::i64Bit, VTMP1.Q(), 0);\n ins(ARMEmitter::SubRegSize::i8Bit, VTMP1, 0, Source, 0);\n mov(Dst.Q(), VTMP1.Q());\n break;\n }\n case IR::OpSize::i16Bit: {\n movi(ARMEmitter::SubRegSize::i64Bit, VTMP1.Q(), 0);\n ins(ARMEmitter::SubRegSize::i16Bit, VTMP1, 0, Source, 0);\n mov(Dst.Q(), VTMP1.Q());\n break;\n }\n case IR::OpSize::i32Bit: {\n movi(ARMEmitter::SubRegSize::i64Bit, VTMP1.Q(), 0);\n ins(ARMEmitter::SubRegSize::i32Bit, VTMP1, 0, Source, 0);\n mov(Dst.Q(), VTMP1.Q());\n break;\n }\n case IR::OpSize::i64Bit: {\n mov(Dst.D(), Source.D());\n break;\n }\n case IR::OpSize::i128Bit: {\n if (HostSupportsSVE256 || Dst.Idx() != Source.Idx()) {\n mov(Dst.Q(), Source.Q());\n }\n break;\n }\n case IR::OpSize::i256Bit: {\n // NOTE: If, in the distant future we support larger moves, or registers\n // (*cough* AVX-512 *cough*) make sure to change this to treat\n // 256-bit moves with zero extending behavior instead of doing only\n // a regular SVE move into a 512-bit register.\n if (Dst.Idx() != Source.Idx()) {\n mov(Dst.Z(), Source.Z());\n }\n break;\n }\n default: LOGMAN_MSG_A_FMT(\"Unknown Op Size: {}\", OpSize); break;\n }\n}\n\nDEF_OP(VAddP) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n const auto IsScalar = OpSize == IR::OpSize::i64Bit;\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n const auto SubRegSize = ConvertSubRegSize8(IROp);\n\n const auto Dst = GetVReg(Node);\n const auto VectorLower = GetVReg(Op->VectorLower);\n const auto VectorUpper = GetVReg(Op->VectorUpper);\n\n if (HostSupportsSVE256 && Is256Bit) {\n const auto Pred = PRED_TMP_32B.Merging();\n\n // SVE ADDP is a destructive operation, so we need a temporary\n movprfx(VTMP1.Z(), VectorLower.Z());\n\n // Unlike Adv. SIMD's version of ADDP, which acts like it concats the\n // upper vector onto the end of the lower vector and then performs\n // pairwise addition, the SVE version actually interleaves the\n // results of the pairwise addition (gross!), so we need to undo that.\n addp(SubRegSize, VTMP1.Z(), Pred, VTMP1.Z(), VectorUpper.Z());\n uzp1(SubRegSize, Dst.Z(), VTMP1.Z(), VTMP1.Z());\n uzp2(SubRegSize, VTMP2.Z(), VTMP1.Z(), VTMP1.Z());\n\n // Merge upper half with lower half.\n splice(ARMEmitter::SubRegSize::i64Bit, Dst.Z(), PRED_TMP_16B, Dst.Z(), VTMP2.Z());\n } else {\n if (IsScalar) {\n addp(SubRegSize, Dst.D(), VectorLower.D(), VectorUpper.D());\n } else {\n addp(SubRegSize, Dst.Q(), VectorLower.Q(), VectorUpper.Q());\n }\n }\n}\n\nDEF_OP(VOrn) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n const auto Is128Bit = OpSize == IR::OpSize::i128Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto Dst = GetVReg(Node);\n const auto Vector1 = GetVReg(Op->Vector1);\n const auto Vector2 = GetVReg(Op->Vector2);\n\n if (HostSupportsSVE256 && Is256Bit) {\n const auto Pred = PRED_TMP_32B.Merging();\n not_(ARMEmitter::SubRegSize::i8Bit, VTMP1.Z(), Pred, Vector2.Z());\n orr(Dst.Z(), Vector1.Z(), VTMP1.Z());\n } else if (Is128Bit) {\n orn(Dst.Q(), Vector1.Q(), Vector2.Q());\n } else {\n orn(Dst.D(), Vector1.D(), Vector2.D());\n }\n}\n\nDEF_OP(VFAddV) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto ElementSize = Op->Header.ElementSize;\n const auto SubRegSize = ConvertSubRegSizePair248(IROp);\n\n const auto Dst = GetVReg(Node);\n const auto Vector = GetVReg(Op->Vector);\n\n LOGMAN_THROW_A_FMT(OpSize == IR::OpSize::i128Bit || OpSize == IR::OpSize::i256Bit, \"Only AVX and SSE size supported\");\n if (HostSupportsSVE256 && Is256Bit) {\n const auto Pred = PRED_TMP_32B.Merging();\n faddv(SubRegSize.Vector, Dst, Pred, Vector.Z());\n }\n if (HostSupportsSVE128) {\n const auto Pred = PRED_TMP_16B.Merging();\n faddv(SubRegSize.Vector, Dst, Pred, Vector.Z());\n } else {\n // ASIMD doesn't support faddv, need to use multiple faddp to match behaviour.\n if (ElementSize == IR::OpSize::i32Bit) {\n faddp(SubRegSize.Vector, Dst.Q(), Vector.Q(), Vector.Q());\n faddp(SubRegSize.Scalar, Dst, Dst);\n } else {\n faddp(SubRegSize.Scalar, Dst, Vector);\n }\n }\n}\n\nDEF_OP(VAddV) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto ElementSize = Op->Header.ElementSize;\n const auto SubRegSize = ConvertSubRegSizePair8(IROp);\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto Dst = GetVReg(Node);\n const auto Vector = GetVReg(Op->Vector);\n\n if (HostSupportsSVE256 && Is256Bit) {\n // SVE doesn't have an equivalent ADDV instruction, so we make do\n // by performing two Adv. SIMD ADDV operations on the high and low\n // 128-bit lanes and then sum them up.\n\n const auto Mask = PRED_TMP_32B.Zeroing();\n const auto CompactPred = ARMEmitter::PReg::p0;\n\n // Select all our upper elements to run ADDV over them.\n not_(CompactPred, Mask, PRED_TMP_16B);\n compact(ARMEmitter::SubRegSize::i64Bit, VTMP1.Z(), CompactPred, Vector.Z());\n\n addv(SubRegSize.Vector, VTMP2.Q(), Vector.Q());\n addv(SubRegSize.Vector, VTMP1.Q(), VTMP1.Q());\n add(SubRegSize.Vector, Dst.Q(), VTMP1.Q(), VTMP2.Q());\n } else {\n if (ElementSize == IR::OpSize::i64Bit) {\n addp(SubRegSize.Scalar, Dst, Vector);\n } else {\n addv(SubRegSize.Vector, Dst.Q(), Vector.Q());\n }\n }\n}\n\nDEF_OP(VUMinV) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto SubRegSize = ConvertSubRegSize8(IROp);\n\n const auto Dst = GetVReg(Node);\n const auto Vector = GetVReg(Op->Vector);\n\n if (HostSupportsSVE256 && Is256Bit) {\n const auto Pred = PRED_TMP_32B;\n uminv(SubRegSize, Dst, Pred, Vector.Z());\n } else {\n // Vector\n uminv(SubRegSize, Dst.Q(), Vector.Q());\n }\n}\n\nDEF_OP(VUMaxV) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto SubRegSize = ConvertSubRegSize8(IROp);\n\n const auto Dst = GetVReg(Node);\n const auto Vector = GetVReg(Op->Vector);\n\n if (HostSupportsSVE256 && Is256Bit) {\n const auto Pred = PRED_TMP_32B;\n umaxv(SubRegSize, Dst, Pred, Vector.Z());\n } else {\n // Vector\n umaxv(SubRegSize, Dst.Q(), Vector.Q());\n }\n}\n\nDEF_OP(VURAvg) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto SubRegSize = ConvertSubRegSize8(IROp);\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto Dst = GetVReg(Node);\n const auto Vector1 = GetVReg(Op->Vector1);\n const auto Vector2 = GetVReg(Op->Vector2);\n\n if (HostSupportsSVE256 && Is256Bit) {\n const auto Mask = PRED_TMP_32B.Merging();\n\n // Trivial cases where we already have source data to be averaged in\n // the destination register. We can just do the operation in place.\n if (Dst == Vector1) {\n urhadd(SubRegSize, Dst.Z(), Mask, Dst.Z(), Vector2.Z());\n } else if (Dst == Vector2) {\n urhadd(SubRegSize, Dst.Z(), Mask, Dst.Z(), Vector1.Z());\n } else {\n // SVE URHADD is a destructive operation, but we know that\n // we don't have any source/destination aliasing happening here\n // so we can safely move one of the source operands into the destination.\n movprfx(Dst.Z(), Vector1.Z());\n urhadd(SubRegSize, Dst.Z(), Mask, Dst.Z(), Vector2.Z());\n }\n } else {\n urhadd(SubRegSize, Dst.Q(), Vector1.Q(), Vector2.Q());\n }\n}\n\nDEF_OP(VFAddP) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto SubRegSize = ConvertSubRegSize248(IROp);\n\n const auto Dst = GetVReg(Node);\n const auto VectorLower = GetVReg(Op->VectorLower);\n const auto VectorUpper = GetVReg(Op->VectorUpper);\n\n if (HostSupportsSVE256 && Is256Bit) {\n const auto Pred = PRED_TMP_32B.Merging();\n\n // SVE FADDP is a destructive operation, so we need a temporary\n movprfx(VTMP1.Z(), VectorLower.Z());\n\n // Unlike Adv. SIMD's version of FADDP, which acts like it concats the\n // upper vector onto the end of the lower vector and then performs\n // pairwise addition, the SVE version actually interleaves the\n // results of the pairwise addition (gross!), so we need to undo that.\n faddp(SubRegSize, VTMP1.Z(), Pred, VTMP1.Z(), VectorUpper.Z());\n uzp1(SubRegSize, Dst.Z(), VTMP1.Z(), VTMP1.Z());\n uzp2(SubRegSize, VTMP2.Z(), VTMP1.Z(), VTMP1.Z());\n\n // Merge upper half with lower half.\n splice(ARMEmitter::SubRegSize::i64Bit, Dst.Z(), PRED_TMP_16B, Dst.Z(), VTMP2.Z());\n } else {\n faddp(SubRegSize, Dst.Q(), VectorLower.Q(), VectorUpper.Q());\n }\n}\n\nDEF_OP(VFDiv) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto ElementSize = Op->Header.ElementSize;\n const auto SubRegSize = ConvertSubRegSize248(IROp);\n const auto IsScalar = ElementSize == OpSize;\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto Dst = GetVReg(Node);\n const auto Vector1 = GetVReg(Op->Vector1);\n const auto Vector2 = GetVReg(Op->Vector2);\n\n if (HostSupportsSVE256 && Is256Bit) {\n const auto Mask = PRED_TMP_32B.Merging();\n\n if (Dst == Vector1) {\n // Trivial case where we already have source data to be divided in the\n // destination register. We can just divide by Vector2 and be done with it.\n fdiv(SubRegSize, Dst.Z(), Mask, Dst.Z(), Vector2.Z());\n } else if (Dst == Vector2) {\n // If the destination aliases the second vector, then we need\n // to use a temp.\n movprfx(VTMP1.Z(), Vector1.Z());\n fdiv(SubRegSize, VTMP1.Z(), Mask, VTMP1.Z(), Vector2.Z());\n mov(Dst.Z(), VTMP1.Z());\n } else {\n // If no registers alias the destination, then we can move directly\n // into the destination and then divide.\n movprfx(Dst.Z(), Vector1.Z());\n fdiv(SubRegSize, Dst.Z(), Mask, Dst.Z(), Vector2.Z());\n }\n } else {\n if (IsScalar) {\n switch (ElementSize) {\n case IR::OpSize::i16Bit: {\n fdiv(Dst.H(), Vector1.H(), Vector2.H());\n break;\n }\n case IR::OpSize::i32Bit: {\n fdiv(Dst.S(), Vector1.S(), Vector2.S());\n break;\n }\n case IR::OpSize::i64Bit: {\n fdiv(Dst.D(), Vector1.D(), Vector2.D());\n break;\n }\n default: break;\n }\n } else {\n fdiv(SubRegSize, Dst.Q(), Vector1.Q(), Vector2.Q());\n }\n }\n}\n\nDEF_OP(VFMin) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto ElementSize = Op->Header.ElementSize;\n const auto SubRegSize = ConvertSubRegSize248(IROp);\n const auto IsScalar = ElementSize == OpSize;\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto Dst = GetVReg(Node);\n const auto Vector1 = GetVReg(Op->Vector1);\n const auto Vector2 = GetVReg(Op->Vector2);\n\n // NOTE: We don't directly use FMIN** here for any of the implementations,\n // because it has undesirable NaN handling behavior (it sets\n // entries either to the incoming NaN value*, or the default NaN\n // depending on FPCR flags set). We want behavior that sets NaN\n // entries to zero for the comparison result.\n //\n // * - Not exactly (differs slightly with SNaNs), but close enough for the explanation\n // ** - Unless the host supports AFP.AH, which allows FMIN/FMAX to select the second source element as expected of x86.\n\n if (HostSupportsSVE256 && Is256Bit) {\n const auto Mask = PRED_TMP_32B;\n const auto ComparePred = ARMEmitter::PReg::p0;\n\n // General idea:\n // 1. Compare greater than against the two vectors\n // 2. Invert the resulting values in the predicate register.\n // 3. Move the first vector into a temporary\n // 4. Merge all the elements that correspond to the inverted\n // predicate bits from the second vector into the\n // same temporary.\n // 5. Move temporary into the destination register and we're done.\n fcmgt(SubRegSize, ComparePred, Mask.Zeroing(), Vector2.Z(), Vector1.Z());\n not_(ComparePred, Mask.Zeroing(), ComparePred);\n\n if (Dst == Vector1) {\n // Trivial case where Vector1 is also the destination.\n // We don't need to move any data around in this case (aside from the merge).\n mov(SubRegSize, Dst.Z(), ComparePred.Merging(), Vector2.Z());\n } else {\n mov(VTMP1.Z(), Vector1.Z());\n mov(SubRegSize, VTMP1.Z(), ComparePred.Merging(), Vector2.Z());\n mov(Dst.Z(), VTMP1.Z());\n }\n } else {\n LOGMAN_THROW_A_FMT(!IsScalar, \"should use VFMinScalarInsert instead\");\n\n if (HostSupportsAFP) {\n // AFP.AH lets fmin behave like x86 min\n fmin(SubRegSize, Dst.Q(), Vector1.Q(), Vector2.Q());\n return;\n }\n\n if (Dst == Vector1) {\n // Destination is already Vector1, need to insert Vector2 on false.\n fcmgt(SubRegSize, VTMP1.Q(), Vector2.Q(), Vector1.Q());\n bif(Dst.Q(), Vector2.Q(), VTMP1.Q());\n } else if (Dst == Vector2) {\n // Destination is already Vector2, Invert arguments and insert Vector1 on false.\n fcmgt(SubRegSize, VTMP1.Q(), Vector1.Q(), Vector2.Q());\n bif(Dst.Q(), Vector1.Q(), VTMP1.Q());\n } else {\n // Dst is not either source, need a move.\n fcmgt(SubRegSize, VTMP1.Q(), Vector2.Q(), Vector1.Q());\n mov(Dst.Q(), Vector1.Q());\n bif(Dst.Q(), Vector2.Q(), VTMP1.Q());\n }\n }\n}\n\nDEF_OP(VFMax) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto ElementSize = Op->Header.ElementSize;\n const auto SubRegSize = ConvertSubRegSize248(IROp);\n const auto IsScalar = ElementSize == OpSize;\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto Dst = GetVReg(Node);\n const auto Vector1 = GetVReg(Op->Vector1);\n const auto Vector2 = GetVReg(Op->Vector2);\n\n // NOTE: See VFMin implementation for reasons why we\n // don't just use FMAX/FMIN for these implementations.\n\n if (HostSupportsSVE256 && Is256Bit) {\n const auto Mask = PRED_TMP_32B;\n const auto ComparePred = ARMEmitter::PReg::p0;\n\n fcmgt(SubRegSize, ComparePred, Mask.Zeroing(), Vector2.Z(), Vector1.Z());\n\n if (Dst == Vector1) {\n // Trivial case where Vector1 is also the destination.\n // We don't need to move any data around in this case (aside from the merge).\n mov(SubRegSize, Dst.Z(), ComparePred.Merging(), Vector2.Z());\n } else {\n mov(VTMP1.Z(), Vector1.Z());\n mov(SubRegSize, VTMP1.Z(), ComparePred.Merging(), Vector2.Z());\n mov(Dst.Z(), VTMP1.Z());\n }\n } else {\n LOGMAN_THROW_A_FMT(!IsScalar, \"should use VFMaxScalarInsert instead\");\n\n if (HostSupportsAFP) {\n // AFP.AH lets fmax behave like x86 max\n fmax(SubRegSize, Dst.Q(), Vector1.Q(), Vector2.Q());\n return;\n }\n\n if (Dst == Vector1) {\n // Destination is already Vector1, need to insert Vector2 on true.\n fcmgt(SubRegSize, VTMP1.Q(), Vector2.Q(), Vector1.Q());\n bit(Dst.Q(), Vector2.Q(), VTMP1.Q());\n } else if (Dst == Vector2) {\n // Destination is already Vector2, Invert arguments and insert Vector1 on true.\n fcmgt(SubRegSize, VTMP1.Q(), Vector1.Q(), Vector2.Q());\n bit(Dst.Q(), Vector1.Q(), VTMP1.Q());\n } else {\n // Dst is not either source, need a move.\n fcmgt(SubRegSize, VTMP1.Q(), Vector2.Q(), Vector1.Q());\n mov(Dst.Q(), Vector1.Q());\n bit(Dst.Q(), Vector2.Q(), VTMP1.Q());\n }\n }\n}\n\nDEF_OP(VFRecp) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto ElementSize = Op->Header.ElementSize;\n const auto SubRegSize = ConvertSubRegSizePair16(IROp);\n const auto IsScalar = Op->Header.ElementSize == OpSize;\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto Dst = GetVReg(Node);\n const auto Vector = GetVReg(Op->Vector);\n\n if (HostSupportsSVE256 && Is256Bit) {\n const auto Pred = PRED_TMP_32B.Merging();\n\n if (ElementSize == IR::OpSize::i32Bit && HostSupportsRPRES) {\n // RPRES gives enough precision for this.\n frecpe(SubRegSize.Vector, Dst.Z(), Vector.Z());\n return;\n }\n\n fmov(SubRegSize.Vector, VTMP1.Z(), 1.0);\n fdiv(SubRegSize.Vector, VTMP1.Z(), Pred, VTMP1.Z(), Vector.Z());\n mov(Dst.Z(), VTMP1.Z());\n } else {\n if (IsScalar) {\n if (ElementSize == IR::OpSize::i32Bit && HostSupportsRPRES) {\n // RPRES gives enough precision for this.\n frecpe(SubRegSize.Scalar, Dst.S(), Vector.S());\n return;\n }\n\n fmov(SubRegSize.Scalar, VTMP1.Q(), 1.0f);\n switch (ElementSize) {\n case IR::OpSize::i16Bit: {\n fdiv(Dst.H(), VTMP1.H(), Vector.H());\n break;\n }\n case IR::OpSize::i32Bit: {\n fdiv(Dst.S(), VTMP1.S(), Vector.S());\n break;\n }\n case IR::OpSize::i64Bit: {\n fdiv(Dst.D(), VTMP1.D(), Vector.D());\n break;\n }\n default: {\n LOGMAN_MSG_A_FMT(\"Unexpected ElementSize for {}\", __func__);\n FEX_UNREACHABLE;\n }\n }\n } else {\n if (ElementSize == IR::OpSize::i32Bit && HostSupportsRPRES) {\n // RPRES gives enough precision for this.\n if (OpSize == IR::OpSize::i64Bit) {\n frecpe(SubRegSize.Vector, Dst.D(), Vector.D());\n } else {\n frecpe(SubRegSize.Vector, Dst.Q(), Vector.Q());\n }\n return;\n }\n\n fmov(SubRegSize.Vector, VTMP1.Q(), 1.0f);\n fdiv(SubRegSize.Vector, Dst.Q(), VTMP1.Q(), Vector.Q());\n }\n }\n}\n\nDEF_OP(VFRecpPrecision) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n const auto ElementSize = Op->Header.ElementSize;\n\n LOGMAN_THROW_A_FMT((OpSize == IR::OpSize::i64Bit || OpSize == IR::OpSize::i32Bit) && ElementSize == IR::OpSize::i32Bit,\n \"Unexpected sizes for operation.\", __func__);\n\n const auto SubRegSize = ConvertSubRegSizePair16(IROp);\n const auto IsScalar = OpSize == ElementSize;\n\n const auto Dst = GetVReg(Node);\n const auto Vector = GetVReg(Op->Vector);\n\n if (IsScalar) {\n if (ElementSize == IR::OpSize::i32Bit && HostSupportsRPRES) {\n // Not enough precision so we need to improve it with frecps\n frecpe(SubRegSize.Scalar, VTMP1.S(), Vector.S());\n frecps(SubRegSize.Scalar, VTMP2.S(), VTMP1.S(), Vector.S());\n fmul(SubRegSize.Scalar, Dst.S(), VTMP1.S(), VTMP2.S());\n return;\n }\n\n fmov(SubRegSize.Scalar, VTMP1.Q(), 1.0f);\n // Element size is known to be 32bits\n fdiv(Dst.S(), VTMP1.S(), Vector.S());\n } else { // Vector operation - Opsize 64bits, elementsize 32bits\n if (HostSupportsRPRES) {\n frecpe(SubRegSize.Vector, VTMP1.D(), Vector.D());\n frecps(SubRegSize.Vector, VTMP2.D(), VTMP1.D(), Vector.D());\n fmul(SubRegSize.Vector, Dst.D(), VTMP1.D(), VTMP2.D());\n return;\n }\n\n // No RPRES, so normal division\n fmov(SubRegSize.Vector, VTMP1.Q(), 1.0f);\n fdiv(SubRegSize.Vector, Dst.Q(), VTMP1.Q(), Vector.Q());\n }\n}\n\nDEF_OP(VFRSqrt) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto ElementSize = Op->Header.ElementSize;\n const auto SubRegSize = ConvertSubRegSizePair16(IROp);\n const auto IsScalar = ElementSize == OpSize;\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto Dst = GetVReg(Node);\n const auto Vector = GetVReg(Op->Vector);\n\n if (HostSupportsSVE256 && Is256Bit) {\n const auto Pred = PRED_TMP_32B.Merging();\n if (ElementSize == IR::OpSize::i32Bit && HostSupportsRPRES) {\n // RPRES gives enough precision for this.\n frsqrte(SubRegSize.Vector, Dst.Z(), Vector.Z());\n return;\n }\n\n fsqrt(SubRegSize.Vector, VTMP1.Z(), Pred, Vector.Z());\n fmov(SubRegSize.Vector, Dst.Z(), 1.0);\n fdiv(SubRegSize.Vector, Dst.Z(), Pred, Dst.Z(), VTMP1.Z());\n } else {\n if (IsScalar) {\n if (ElementSize == IR::OpSize::i32Bit && HostSupportsRPRES) {\n // RPRES gives enough precision for this.\n frsqrte(SubRegSize.Scalar, Dst.S(), Vector.S());\n return;\n }\n\n fmov(SubRegSize.Scalar, VTMP1.Q(), 1.0);\n switch (ElementSize) {\n case IR::OpSize::i16Bit: {\n fsqrt(VTMP2.H(), Vector.H());\n fdiv(Dst.H(), VTMP1.H(), VTMP2.H());\n break;\n }\n case IR::OpSize::i32Bit: {\n fsqrt(VTMP2.S(), Vector.S());\n fdiv(Dst.S(), VTMP1.S(), VTMP2.S());\n break;\n }\n case IR::OpSize::i64Bit: {\n fsqrt(VTMP2.D(), Vector.D());\n fdiv(Dst.D(), VTMP1.D(), VTMP2.D());\n break;\n }\n default: break;\n }\n } else {\n if (ElementSize == IR::OpSize::i32Bit && HostSupportsRPRES) {\n // RPRES gives enough precision for this.\n if (OpSize == IR::OpSize::i64Bit) {\n frsqrte(SubRegSize.Vector, Dst.D(), Vector.D());\n } else {\n frsqrte(SubRegSize.Vector, Dst.Q(), Vector.Q());\n }\n return;\n }\n\n fmov(SubRegSize.Vector, VTMP1.Q(), 1.0);\n fsqrt(SubRegSize.Vector, VTMP2.Q(), Vector.Q());\n fdiv(SubRegSize.Vector, Dst.Q(), VTMP1.Q(), VTMP2.Q());\n }\n }\n}\n\nDEF_OP(VFRSqrtPrecision) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n const auto ElementSize = Op->Header.ElementSize;\n\n\n LOGMAN_THROW_A_FMT((OpSize == IR::OpSize::i64Bit || OpSize == IR::OpSize::i32Bit) && ElementSize == IR::OpSize::i32Bit,\n \"Unexpected sizes for operation.\", __func__);\n\n const auto SubRegSize = ConvertSubRegSizePair16(IROp);\n const auto IsScalar = ElementSize == OpSize;\n\n const auto Dst = GetVReg(Node);\n const auto Vector = GetVReg(Op->Vector);\n\n if (IsScalar) {\n if (HostSupportsRPRES) {\n frsqrte(SubRegSize.Scalar, VTMP1.S(), Vector.S());\n // Improve initial estimate which is not good enough.\n fmul(SubRegSize.Scalar, VTMP2.S(), VTMP1.S(), VTMP1.S());\n frsqrts(SubRegSize.Scalar, VTMP2.S(), VTMP2.S(), Vector.S());\n fmul(SubRegSize.Scalar, Dst.S(), VTMP1.S(), VTMP2.S());\n return;\n }\n\n fmov(SubRegSize.Scalar, VTMP1.Q(), 1.0);\n // element size is known to be 32bits\n fsqrt(VTMP2.S(), Vector.S());\n fdiv(Dst.S(), VTMP1.S(), VTMP2.S());\n } else {\n if (HostSupportsRPRES) {\n frsqrte(SubRegSize.Vector, VTMP1.D(), Vector.D());\n // Improve initial estimate which is not good enough.\n fmul(SubRegSize.Vector, VTMP2.D(), VTMP1.D(), VTMP1.D());\n frsqrts(SubRegSize.Vector, VTMP2.D(), VTMP2.D(), Vector.D());\n fmul(SubRegSize.Vector, Dst.D(), VTMP1.D(), VTMP2.D());\n return;\n }\n fmov(SubRegSize.Vector, VTMP1.Q(), 1.0);\n fsqrt(SubRegSize.Vector, VTMP2.Q(), Vector.Q());\n fdiv(SubRegSize.Vector, Dst.Q(), VTMP1.Q(), VTMP2.Q());\n }\n}\n\nDEF_OP(VNot) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto Dst = GetVReg(Node);\n const auto Vector = GetVReg(Op->Vector);\n\n if (HostSupportsSVE256 && Is256Bit) {\n not_(ARMEmitter::SubRegSize::i8Bit, Dst.Z(), PRED_TMP_32B.Merging(), Vector.Z());\n } else {\n mvn(ARMEmitter::SubRegSize::i8Bit, Dst.Q(), Vector.Q());\n }\n}\n\nDEF_OP(VUMin) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto ElementSize = Op->Header.ElementSize;\n const auto SubRegSize = ConvertSubRegSize16(IROp);\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto Dst = GetVReg(Node);\n const auto Vector1 = GetVReg(Op->Vector1);\n const auto Vector2 = GetVReg(Op->Vector2);\n\n if (HostSupportsSVE256 && Is256Bit) {\n const auto Pred = PRED_TMP_32B.Merging();\n\n // In any case where the destination aliases one of the source vectors\n // then we can just perform the UMIN in place.\n if (Dst == Vector1) {\n umin(SubRegSize, Dst.Z(), Pred, Dst.Z(), Vector2.Z());\n } else if (Dst == Vector2) {\n umin(SubRegSize, Dst.Z(), Pred, Dst.Z(), Vector1.Z());\n } else {\n // SVE UMIN is a destructive operation, but we know nothing is\n // aliasing the destination by this point, so we can move into\n // the destination without needing a temporary.\n movprfx(Dst.Z(), Vector1.Z());\n umin(SubRegSize, Dst.Z(), Pred, Dst.Z(), Vector2.Z());\n }\n } else {\n switch (ElementSize) {\n case IR::OpSize::i8Bit:\n case IR::OpSize::i16Bit:\n case IR::OpSize::i32Bit: {\n umin(SubRegSize, Dst.Q(), Vector1.Q(), Vector2.Q());\n break;\n }\n case IR::OpSize::i64Bit: {\n cmhi(SubRegSize, VTMP1.Q(), Vector2.Q(), Vector1.Q());\n mov(VTMP2.Q(), Vector1.Q());\n bif(VTMP2.Q(), Vector2.Q(), VTMP1.Q());\n mov(Dst.Q(), VTMP2.Q());\n break;\n }\n default: break;\n }\n }\n}\n\nDEF_OP(VSMin) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto ElementSize = Op->Header.ElementSize;\n const auto SubRegSize = ConvertSubRegSize16(IROp);\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto Dst = GetVReg(Node);\n const auto Vector1 = GetVReg(Op->Vector1);\n const auto Vector2 = GetVReg(Op->Vector2);\n\n if (HostSupportsSVE256 && Is256Bit) {\n const auto Pred = PRED_TMP_32B.Merging();\n\n // In any case where the destination aliases one of the source vectors\n // then we can just perform the SMIN in place.\n if (Dst == Vector1) {\n smin(SubRegSize, Dst.Z(), Pred, Dst.Z(), Vector2.Z());\n } else if (Dst == Vector2) {\n smin(SubRegSize, Dst.Z(), Pred, Dst.Z(), Vector1.Z());\n } else {\n // SVE SMIN is a destructive operation, but we know nothing is\n // aliasing the destination by this point, so we can move into\n // the destination without needing a temporary.\n movprfx(Dst.Z(), Vector1.Z());\n smin(SubRegSize, Dst.Z(), Pred, Dst.Z(), Vector2.Z());\n }\n } else {\n switch (ElementSize) {\n case IR::OpSize::i8Bit:\n case IR::OpSize::i16Bit:\n case IR::OpSize::i32Bit: {\n smin(SubRegSize, Dst.Q(), Vector1.Q(), Vector2.Q());\n break;\n }\n case IR::OpSize::i64Bit: {\n cmgt(SubRegSize, VTMP1.Q(), Vector1.Q(), Vector2.Q());\n mov(VTMP2.Q(), Vector1.Q());\n bif(VTMP2.Q(), Vector2.Q(), VTMP1.Q());\n mov(Dst.Q(), VTMP2.Q());\n break;\n }\n default: break;\n }\n }\n}\n\nDEF_OP(VUMax) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto ElementSize = Op->Header.ElementSize;\n const auto SubRegSize = ConvertSubRegSize16(IROp);\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto Dst = GetVReg(Node);\n const auto Vector1 = GetVReg(Op->Vector1);\n const auto Vector2 = GetVReg(Op->Vector2);\n\n if (HostSupportsSVE256 && Is256Bit) {\n const auto Pred = PRED_TMP_32B.Merging();\n\n // In any case where the destination aliases one of the source vectors\n // then we can just perform the UMAX in place.\n if (Dst == Vector1) {\n umax(SubRegSize, Dst.Z(), Pred, Dst.Z(), Vector2.Z());\n } else if (Dst == Vector2) {\n umax(SubRegSize, Dst.Z(), Pred, Dst.Z(), Vector1.Z());\n } else {\n // SVE UMAX is a destructive operation, but we know nothing is\n // aliasing the destination by this point, so we can move into\n // the destination without needing a temporary.\n movprfx(Dst.Z(), Vector1.Z());\n umax(SubRegSize, Dst.Z(), Pred, Dst.Z(), Vector2.Z());\n }\n } else {\n switch (ElementSize) {\n case IR::OpSize::i8Bit:\n case IR::OpSize::i16Bit:\n case IR::OpSize::i32Bit: {\n umax(SubRegSize, Dst.Q(), Vector1.Q(), Vector2.Q());\n break;\n }\n case IR::OpSize::i64Bit: {\n cmhi(SubRegSize, VTMP1.Q(), Vector2.Q(), Vector1.Q());\n mov(VTMP2.Q(), Vector1.Q());\n bif(VTMP2.Q(), Vector2.Q(), VTMP1.Q());\n mov(Dst.Q(), VTMP2.Q());\n break;\n }\n default: break;\n }\n }\n}\n\nDEF_OP(VSMax) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto ElementSize = Op->Header.ElementSize;\n const auto SubRegSize = ConvertSubRegSize16(IROp);\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto Dst = GetVReg(Node);\n const auto Vector1 = GetVReg(Op->Vector1);\n const auto Vector2 = GetVReg(Op->Vector2);\n\n if (HostSupportsSVE256 && Is256Bit) {\n const auto Pred = PRED_TMP_32B.Merging();\n\n // In any case where the destination aliases one of the source vectors\n // then we can just perform the SMAX in place.\n if (Dst == Vector1) {\n smax(SubRegSize, Dst.Z(), Pred, Dst.Z(), Vector2.Z());\n } else if (Dst == Vector2) {\n smax(SubRegSize, Dst.Z(), Pred, Dst.Z(), Vector1.Z());\n } else {\n // SVE SMAX is a destructive operation, but we know nothing is\n // aliasing the destination by this point, so we can move into\n // the destination without needing a temporary.\n movprfx(Dst.Z(), Vector1.Z());\n smax(SubRegSize, Dst.Z(), Pred, Dst.Z(), Vector2.Z());\n }\n } else {\n switch (ElementSize) {\n case IR::OpSize::i8Bit:\n case IR::OpSize::i16Bit:\n case IR::OpSize::i32Bit: {\n smax(SubRegSize, Dst.Q(), Vector1.Q(), Vector2.Q());\n break;\n }\n case IR::OpSize::i64Bit: {\n cmgt(SubRegSize, VTMP1.Q(), Vector2.Q(), Vector1.Q());\n mov(VTMP2.Q(), Vector1.Q());\n bif(VTMP2.Q(), Vector2.Q(), VTMP1.Q());\n mov(Dst.Q(), VTMP2.Q());\n break;\n }\n default: break;\n }\n }\n}\n\nDEF_OP(VBSL) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto Is128Bit = OpSize == IR::OpSize::i128Bit;\n\n const auto Dst = GetVReg(Node);\n const auto VectorFalse = GetVReg(Op->VectorFalse);\n const auto VectorTrue = GetVReg(Op->VectorTrue);\n const auto VectorMask = GetVReg(Op->VectorMask);\n\n if (HostSupportsSVE256 && Is256Bit) {\n // NOTE: Slight parameter difference from ASIMD\n // ASIMD -> BSL Mask, True, False\n // SVE -> BSL True, True, False, Mask\n // ASIMD -> BIT True, False, Mask\n // ASIMD -> BIF False, True, Mask\n if (Dst == VectorTrue) {\n // Trivial case where we can perform the operation in place.\n bsl(Dst.Z(), Dst.Z(), VectorFalse.Z(), VectorMask.Z());\n } else {\n movprfx(VTMP1.Z(), VectorTrue.Z());\n bsl(VTMP1.Z(), VTMP1.Z(), VectorFalse.Z(), VectorMask.Z());\n mov(Dst.Z(), VTMP1.Z());\n }\n } else if (!HostSupportsSVE256 && HostSupportsSVE128 && Is128Bit && Dst != VectorFalse && Dst != VectorTrue && Dst != VectorMask) {\n // Needs to move but SVE movprfx+bsl is slightly more efficient than ASIMD mov+bsl on CPUs that support\n // movprfx fusion and NOT zero-cycle vector register moves.\n movprfx(Dst.Z(), VectorTrue.Z());\n bsl(Dst.Z(), Dst.Z(), VectorFalse.Z(), VectorMask.Z());\n } else {\n if (VectorMask == Dst) {\n // Can use BSL without any moves.\n if (OpSize == IR::OpSize::i64Bit) {\n bsl(Dst.D(), VectorTrue.D(), VectorFalse.D());\n } else {\n bsl(Dst.Q(), VectorTrue.Q(), VectorFalse.Q());\n }\n } else if (VectorTrue == Dst) {\n // Can use BIF without any moves.\n if (OpSize == IR::OpSize::i64Bit) {\n bif(Dst.D(), VectorFalse.D(), VectorMask.D());\n } else {\n bif(Dst.Q(), VectorFalse.Q(), VectorMask.Q());\n }\n } else if (VectorFalse == Dst) {\n // Can use BIT without any moves.\n if (OpSize == IR::OpSize::i64Bit) {\n bit(Dst.D(), VectorTrue.D(), VectorMask.D());\n } else {\n bit(Dst.Q(), VectorTrue.Q(), VectorMask.Q());\n }\n } else {\n // Needs moves.\n if (OpSize == IR::OpSize::i64Bit) {\n mov(Dst.D(), VectorMask.D());\n bsl(Dst.D(), VectorTrue.D(), VectorFalse.D());\n } else {\n mov(Dst.Q(), VectorMask.Q());\n bsl(Dst.Q(), VectorTrue.Q(), VectorFalse.Q());\n }\n }\n }\n}\n\nDEF_OP(VCMPEQ) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto ElementSize = Op->Header.ElementSize;\n const auto SubRegSize = ConvertSubRegSizePair16(IROp);\n const auto IsScalar = ElementSize == OpSize;\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto Dst = GetVReg(Node);\n const auto Vector1 = GetVReg(Op->Vector1);\n const auto Vector2 = GetVReg(Op->Vector2);\n\n if (HostSupportsSVE256 && Is256Bit) {\n // FIXME: We should rework this op to avoid the NZCV spill/fill dance.\n mrs(TMP1, ARMEmitter::SystemRegister::NZCV);\n\n const auto Mask = PRED_TMP_32B.Zeroing();\n const auto ComparePred = ARMEmitter::PReg::p0;\n\n // General idea is to compare for equality, not the equal vals\n // from one of the registers, then or both together to make the\n // relevant equal entries all 1s.\n cmpeq(SubRegSize.Vector, ComparePred, Mask, Vector1.Z(), Vector2.Z());\n not_(SubRegSize.Vector, VTMP1.Z(), ComparePred.Merging(), Vector1.Z());\n movprfx(SubRegSize.Vector, Dst.Z(), ComparePred.Zeroing(), Vector1.Z());\n orr(SubRegSize.Vector, Dst.Z(), ComparePred.Merging(), Dst.Z(), VTMP1.Z());\n\n // Restore NZCV\n msr(ARMEmitter::SystemRegister::NZCV, TMP1);\n } else {\n if (IsScalar) {\n cmeq(SubRegSize.Scalar, Dst, Vector1, Vector2);\n } else {\n cmeq(SubRegSize.Vector, Dst.Q(), Vector1.Q(), Vector2.Q());\n }\n }\n}\n\nDEF_OP(VCMPEQZ) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto ElementSize = Op->Header.ElementSize;\n const auto SubRegSize = ConvertSubRegSizePair16(IROp);\n const auto IsScalar = ElementSize == OpSize;\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto Dst = GetVReg(Node);\n const auto Vector = GetVReg(Op->Vector);\n\n if (HostSupportsSVE256 && Is256Bit) {\n const auto Mask = PRED_TMP_32B.Zeroing();\n const auto ComparePred = ARMEmitter::PReg::p0;\n\n // FIXME: We should rework this op to avoid the NZCV spill/fill dance.\n mrs(TMP1, ARMEmitter::SystemRegister::NZCV);\n\n // Ensure no junk is in the temp (important for ensuring\n // non-equal entries remain as zero).\n mov_imm(ARMEmitter::SubRegSize::i64Bit, VTMP1.Z(), 0);\n // Unlike with VCMPEQ, we can skip needing to bitwise OR the\n // final results, since if our elements are equal to zero,\n // we just need to bitwise NOT them and they're already set\n // to all 1s.\n cmpeq(SubRegSize.Vector, ComparePred, Mask, Vector.Z(), 0);\n not_(SubRegSize.Vector, VTMP1.Z(), ComparePred.Merging(), Vector.Z());\n mov(Dst.Z(), VTMP1.Z());\n\n // Restore NZCV\n msr(ARMEmitter::SystemRegister::NZCV, TMP1);\n } else {\n if (IsScalar) {\n cmeq(SubRegSize.Scalar, Dst, Vector);\n } else {\n cmeq(SubRegSize.Vector, Dst.Q(), Vector.Q());\n }\n }\n}\n\nDEF_OP(VCMPGT) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto ElementSize = Op->Header.ElementSize;\n const auto SubRegSize = ConvertSubRegSizePair16(IROp);\n const auto IsScalar = ElementSize == OpSize;\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto Dst = GetVReg(Node);\n const auto Vector1 = GetVReg(Op->Vector1);\n const auto Vector2 = GetVReg(Op->Vector2);\n\n if (HostSupportsSVE256 && Is256Bit) {\n const auto Mask = PRED_TMP_32B.Zeroing();\n const auto ComparePred = ARMEmitter::PReg::p0;\n\n // FIXME: We should rework this op to avoid the NZCV spill/fill dance.\n mrs(TMP1, ARMEmitter::SystemRegister::NZCV);\n\n // General idea is to compare for greater-than, bitwise NOT\n // the valid values, then ORR the NOTed values with the original\n // values to form entries that are all 1s.\n cmpgt(SubRegSize.Vector, ComparePred, Mask, Vector1.Z(), Vector2.Z());\n not_(SubRegSize.Vector, VTMP1.Z(), ComparePred.Merging(), Vector1.Z());\n movprfx(SubRegSize.Vector, Dst.Z(), ComparePred.Zeroing(), Vector1.Z());\n orr(SubRegSize.Vector, Dst.Z(), ComparePred.Merging(), Dst.Z(), VTMP1.Z());\n\n // Restore NZCV\n msr(ARMEmitter::SystemRegister::NZCV, TMP1);\n } else {\n if (IsScalar) {\n cmgt(SubRegSize.Scalar, Dst, Vector1, Vector2);\n } else {\n cmgt(SubRegSize.Vector, Dst.Q(), Vector1.Q(), Vector2.Q());\n }\n }\n}\n\nDEF_OP(VCMPGTZ) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto ElementSize = Op->Header.ElementSize;\n const auto SubRegSize = ConvertSubRegSizePair16(IROp);\n const auto IsScalar = ElementSize == OpSize;\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto Dst = GetVReg(Node);\n const auto Vector = GetVReg(Op->Vector);\n\n if (HostSupportsSVE256 && Is256Bit) {\n const auto Mask = PRED_TMP_32B.Zeroing();\n const auto ComparePred = ARMEmitter::PReg::p0;\n\n // FIXME: We should rework this op to avoid the NZCV spill/fill dance.\n mrs(TMP1, ARMEmitter::SystemRegister::NZCV);\n\n // Ensure no junk is in the temp (important for ensuring\n // non greater-than values remain as zero).\n mov_imm(ARMEmitter::SubRegSize::i64Bit, VTMP1.Z(), 0);\n cmpgt(SubRegSize.Vector, ComparePred, Mask, Vector.Z(), 0);\n not_(SubRegSize.Vector, VTMP1.Z(), ComparePred.Merging(), Vector.Z());\n orr(SubRegSize.Vector, VTMP1.Z(), ComparePred.Merging(), VTMP1.Z(), Vector.Z());\n mov(Dst.Z(), VTMP1.Z());\n\n // Restore NZCV\n msr(ARMEmitter::SystemRegister::NZCV, TMP1);\n } else {\n if (IsScalar) {\n cmgt(SubRegSize.Scalar, Dst, Vector);\n } else {\n cmgt(SubRegSize.Vector, Dst.Q(), Vector.Q());\n }\n }\n}\n\nDEF_OP(VCMPLTZ) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto ElementSize = Op->Header.ElementSize;\n const auto SubRegSize = ConvertSubRegSizePair16(IROp);\n const auto IsScalar = ElementSize == OpSize;\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto Dst = GetVReg(Node);\n const auto Vector = GetVReg(Op->Vector);\n\n if (HostSupportsSVE256 && Is256Bit) {\n const auto Mask = PRED_TMP_32B.Zeroing();\n const auto ComparePred = ARMEmitter::PReg::p0;\n\n // FIXME: We should rework this op to avoid the NZCV spill/fill dance.\n mrs(TMP1, ARMEmitter::SystemRegister::NZCV);\n\n // Ensure no junk is in the temp (important for ensuring\n // non less-than values remain as zero).\n mov_imm(ARMEmitter::SubRegSize::i64Bit, VTMP1.Z(), 0);\n cmplt(SubRegSize.Vector, ComparePred, Mask, Vector.Z(), 0);\n not_(SubRegSize.Vector, VTMP1.Z(), ComparePred.Merging(), Vector.Z());\n orr(SubRegSize.Vector, VTMP1.Z(), ComparePred.Merging(), VTMP1.Z(), Vector.Z());\n mov(Dst.Z(), VTMP1.Z());\n\n // Restore NZCV\n msr(ARMEmitter::SystemRegister::NZCV, TMP1);\n } else {\n if (IsScalar) {\n cmlt(SubRegSize.Scalar, Dst, Vector);\n } else {\n cmlt(SubRegSize.Vector, Dst.Q(), Vector.Q());\n }\n }\n}\n\nDEF_OP(VFCMPEQ) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto ElementSize = Op->Header.ElementSize;\n const auto SubRegSize = ConvertSubRegSizePair248(IROp);\n const auto IsScalar = ElementSize == OpSize;\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto Dst = GetVReg(Node);\n const auto Vector1 = GetVReg(Op->Vector1);\n const auto Vector2 = GetVReg(Op->Vector2);\n\n if (HostSupportsSVE256 && Is256Bit) {\n const auto Mask = PRED_TMP_32B.Zeroing();\n const auto ComparePred = ARMEmitter::PReg::p0;\n\n fcmeq(SubRegSize.Vector, ComparePred, Mask, Vector1.Z(), Vector2.Z());\n not_(SubRegSize.Vector, VTMP1.Z(), ComparePred.Merging(), Vector1.Z());\n movprfx(SubRegSize.Vector, Dst.Z(), ComparePred.Zeroing(), Vector1.Z());\n orr(SubRegSize.Vector, Dst.Z(), ComparePred.Merging(), Dst.Z(), VTMP1.Z());\n } else {\n if (IsScalar) {\n switch (ElementSize) {\n case IR::OpSize::i16Bit: {\n fcmeq(Dst.H(), Vector1.H(), Vector2.H());\n break;\n }\n case IR::OpSize::i32Bit:\n case IR::OpSize::i64Bit: fcmeq(SubRegSize.Scalar, Dst, Vector1, Vector2); break;\n default: break;\n }\n } else {\n fcmeq(SubRegSize.Vector, Dst.Q(), Vector1.Q(), Vector2.Q());\n }\n }\n}\n\nDEF_OP(VFCMPNEQ) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto ElementSize = Op->Header.ElementSize;\n const auto SubRegSize = ConvertSubRegSizePair248(IROp);\n const auto IsScalar = ElementSize == OpSize;\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto Dst = GetVReg(Node);\n const auto Vector1 = GetVReg(Op->Vector1);\n const auto Vector2 = GetVReg(Op->Vector2);\n\n if (HostSupportsSVE256 && Is256Bit) {\n const auto Mask = PRED_TMP_32B.Zeroing();\n const auto ComparePred = ARMEmitter::PReg::p0;\n\n fcmne(SubRegSize.Vector, ComparePred, Mask, Vector1.Z(), Vector2.Z());\n not_(SubRegSize.Vector, VTMP1.Z(), ComparePred.Merging(), Vector1.Z());\n movprfx(SubRegSize.Vector, Dst.Z(), ComparePred.Zeroing(), Vector1.Z());\n orr(SubRegSize.Vector, Dst.Z(), ComparePred.Merging(), Dst.Z(), VTMP1.Z());\n } else {\n if (IsScalar) {\n switch (ElementSize) {\n case IR::OpSize::i16Bit: {\n fcmeq(Dst.H(), Vector1.H(), Vector2.H());\n break;\n }\n case IR::OpSize::i32Bit:\n case IR::OpSize::i64Bit: fcmeq(SubRegSize.Scalar, Dst, Vector1, Vector2); break;\n default: break;\n }\n mvn(ARMEmitter::SubRegSize::i8Bit, Dst.D(), Dst.D());\n } else {\n fcmeq(SubRegSize.Vector, Dst.Q(), Vector1.Q(), Vector2.Q());\n mvn(ARMEmitter::SubRegSize::i8Bit, Dst.Q(), Dst.Q());\n }\n }\n}\n\nDEF_OP(VFCMPLT) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto ElementSize = Op->Header.ElementSize;\n const auto SubRegSize = ConvertSubRegSizePair248(IROp);\n const auto IsScalar = ElementSize == OpSize;\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto Dst = GetVReg(Node);\n const auto Vector1 = GetVReg(Op->Vector1);\n const auto Vector2 = GetVReg(Op->Vector2);\n\n if (HostSupportsSVE256 && Is256Bit) {\n const auto Mask = PRED_TMP_32B.Zeroing();\n const auto ComparePred = ARMEmitter::PReg::p0;\n\n fcmgt(SubRegSize.Vector, ComparePred, Mask, Vector2.Z(), Vector1.Z());\n not_(SubRegSize.Vector, VTMP1.Z(), ComparePred.Merging(), Vector2.Z());\n movprfx(SubRegSize.Vector, Dst.Z(), ComparePred.Zeroing(), Vector2.Z());\n orr(SubRegSize.Vector, Dst.Z(), ComparePred.Merging(), Dst.Z(), VTMP1.Z());\n } else {\n if (IsScalar) {\n switch (ElementSize) {\n case IR::OpSize::i16Bit: {\n fcmgt(Dst.H(), Vector2.H(), Vector1.H());\n break;\n }\n case IR::OpSize::i32Bit:\n case IR::OpSize::i64Bit: fcmgt(SubRegSize.Scalar, Dst, Vector2, Vector1); break;\n default: break;\n }\n } else {\n fcmgt(SubRegSize.Vector, Dst.Q(), Vector2.Q(), Vector1.Q());\n }\n }\n}\n\nDEF_OP(VFCMPGT) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto ElementSize = Op->Header.ElementSize;\n const auto SubRegSize = ConvertSubRegSizePair248(IROp);\n const auto IsScalar = ElementSize == OpSize;\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto Dst = GetVReg(Node);\n const auto Vector1 = GetVReg(Op->Vector1);\n const auto Vector2 = GetVReg(Op->Vector2);\n\n if (HostSupportsSVE256 && Is256Bit) {\n const auto Mask = PRED_TMP_32B.Zeroing();\n const auto ComparePred = ARMEmitter::PReg::p0;\n\n fcmgt(SubRegSize.Vector, ComparePred, Mask, Vector1.Z(), Vector2.Z());\n not_(SubRegSize.Vector, VTMP1.Z(), ComparePred.Merging(), Vector1.Z());\n movprfx(SubRegSize.Vector, Dst.Z(), ComparePred.Zeroing(), Vector1.Z());\n orr(SubRegSize.Vector, Dst.Z(), ComparePred.Merging(), Dst.Z(), VTMP1.Z());\n } else {\n if (IsScalar) {\n switch (ElementSize) {\n case IR::OpSize::i16Bit: {\n fcmgt(Dst.H(), Vector1.H(), Vector2.H());\n break;\n }\n case IR::OpSize::i32Bit:\n case IR::OpSize::i64Bit: fcmgt(SubRegSize.Scalar, Dst, Vector1, Vector2); break;\n default: break;\n }\n } else {\n fcmgt(SubRegSize.Vector, Dst.Q(), Vector1.Q(), Vector2.Q());\n }\n }\n}\n\nDEF_OP(VFCMPLE) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto ElementSize = Op->Header.ElementSize;\n const auto SubRegSize = ConvertSubRegSizePair248(IROp);\n const auto IsScalar = ElementSize == OpSize;\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto Dst = GetVReg(Node);\n const auto Vector1 = GetVReg(Op->Vector1);\n const auto Vector2 = GetVReg(Op->Vector2);\n\n if (HostSupportsSVE256 && Is256Bit) {\n const auto Mask = PRED_TMP_32B.Zeroing();\n const auto ComparePred = ARMEmitter::PReg::p0;\n\n fcmge(SubRegSize.Vector, ComparePred, Mask, Vector2.Z(), Vector1.Z());\n not_(SubRegSize.Vector, VTMP1.Z(), ComparePred.Merging(), Vector2.Z());\n movprfx(SubRegSize.Vector, Dst.Z(), ComparePred.Zeroing(), Vector2.Z());\n orr(SubRegSize.Vector, Dst.Z(), ComparePred.Merging(), Dst.Z(), VTMP1.Z());\n } else {\n if (IsScalar) {\n switch (ElementSize) {\n case IR::OpSize::i16Bit: {\n fcmge(Dst.H(), Vector2.H(), Vector1.H());\n break;\n }\n case IR::OpSize::i32Bit:\n case IR::OpSize::i64Bit: fcmge(SubRegSize.Scalar, Dst, Vector2, Vector1); break;\n default: break;\n }\n } else {\n fcmge(SubRegSize.Vector, Dst.Q(), Vector2.Q(), Vector1.Q());\n }\n }\n}\n\nDEF_OP(VFCMPORD) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto ElementSize = Op->Header.ElementSize;\n const auto SubRegSize = ConvertSubRegSizePair248(IROp);\n const auto IsScalar = ElementSize == OpSize;\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto Dst = GetVReg(Node);\n const auto Vector1 = GetVReg(Op->Vector1);\n const auto Vector2 = GetVReg(Op->Vector2);\n\n if (HostSupportsSVE256 && Is256Bit) {\n const auto Mask = PRED_TMP_32B.Zeroing();\n const auto ComparePred = ARMEmitter::PReg::p0;\n\n // The idea is like comparing for unordered, but we just\n // invert the predicate from the comparison to instead\n // select all ordered elements in the vector.\n fcmuo(SubRegSize.Vector, ComparePred, Mask, Vector1.Z(), Vector2.Z());\n not_(ComparePred, Mask, ComparePred);\n not_(SubRegSize.Vector, VTMP1.Z(), ComparePred.Merging(), Vector1.Z());\n movprfx(SubRegSize.Vector, Dst.Z(), ComparePred.Zeroing(), Vector1.Z());\n orr(SubRegSize.Vector, Dst.Z(), ComparePred.Merging(), Dst.Z(), VTMP1.Z());\n } else {\n if (IsScalar) {\n switch (ElementSize) {\n case IR::OpSize::i16Bit: {\n fcmge(VTMP1.H(), Vector1.H(), Vector2.H());\n fcmgt(VTMP2.H(), Vector2.H(), Vector1.H());\n orr(Dst.D(), VTMP1.D(), VTMP2.D());\n break;\n }\n case IR::OpSize::i32Bit:\n case IR::OpSize::i64Bit:\n fcmge(SubRegSize.Scalar, VTMP1, Vector1, Vector2);\n fcmgt(SubRegSize.Scalar, VTMP2, Vector2, Vector1);\n orr(Dst.D(), VTMP1.D(), VTMP2.D());\n break;\n default: break;\n }\n } else {\n fcmge(SubRegSize.Vector, VTMP1.Q(), Vector1.Q(), Vector2.Q());\n fcmgt(SubRegSize.Vector, VTMP2.Q(), Vector2.Q(), Vector1.Q());\n orr(Dst.Q(), VTMP1.Q(), VTMP2.Q());\n }\n }\n}\n\nDEF_OP(VFCMPUNO) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto ElementSize = Op->Header.ElementSize;\n const auto SubRegSize = ConvertSubRegSizePair248(IROp);\n const auto IsScalar = ElementSize == OpSize;\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto Dst = GetVReg(Node);\n const auto Vector1 = GetVReg(Op->Vector1);\n const auto Vector2 = GetVReg(Op->Vector2);\n\n if (HostSupportsSVE256 && Is256Bit) {\n const auto Mask = PRED_TMP_32B.Zeroing();\n const auto ComparePred = ARMEmitter::PReg::p0;\n\n fcmuo(SubRegSize.Vector, ComparePred, Mask, Vector1.Z(), Vector2.Z());\n not_(SubRegSize.Vector, VTMP1.Z(), ComparePred.Merging(), Vector1.Z());\n movprfx(SubRegSize.Vector, Dst.Z(), ComparePred.Zeroing(), Vector1.Z());\n orr(SubRegSize.Vector, Dst.Z(), ComparePred.Merging(), Dst.Z(), VTMP1.Z());\n } else {\n if (IsScalar) {\n switch (ElementSize) {\n case IR::OpSize::i16Bit: {\n fcmge(VTMP1.H(), Vector1.H(), Vector2.H());\n fcmgt(VTMP2.H(), Vector2.H(), Vector1.H());\n orr(Dst.D(), VTMP1.D(), VTMP2.D());\n mvn(ARMEmitter::SubRegSize::i8Bit, Dst.D(), Dst.D());\n break;\n }\n case IR::OpSize::i32Bit:\n case IR::OpSize::i64Bit:\n fcmge(SubRegSize.Scalar, VTMP1, Vector1, Vector2);\n fcmgt(SubRegSize.Scalar, VTMP2, Vector2, Vector1);\n orr(Dst.D(), VTMP1.D(), VTMP2.D());\n mvn(ARMEmitter::SubRegSize::i8Bit, Dst.D(), Dst.D());\n break;\n default: break;\n }\n } else {\n fcmge(SubRegSize.Vector, VTMP1.Q(), Vector1.Q(), Vector2.Q());\n fcmgt(SubRegSize.Vector, VTMP2.Q(), Vector2.Q(), Vector1.Q());\n orr(Dst.Q(), VTMP1.Q(), VTMP2.Q());\n mvn(ARMEmitter::SubRegSize::i8Bit, Dst.Q(), Dst.Q());\n }\n }\n}\n\nDEF_OP(VUShl) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto ElementSize = IROp->ElementSize;\n const auto SubRegSize = ConvertSubRegSize8(IROp);\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto MaxShift = IR::OpSizeAsBits(ElementSize);\n\n const auto Dst = GetVReg(Node);\n auto ShiftVector = GetVReg(Op->ShiftVector);\n const auto Vector = GetVReg(Op->Vector);\n const auto RangeCheck = Op->RangeCheck;\n\n if (HostSupportsSVE256 && Is256Bit) {\n const auto Mask = PRED_TMP_32B.Merging();\n\n if (RangeCheck) {\n dup_imm(SubRegSize, VTMP2.Z(), MaxShift);\n umin(SubRegSize, VTMP2.Z(), Mask, VTMP2.Z(), ShiftVector.Z());\n ShiftVector = VTMP2;\n }\n\n if (Dst == ShiftVector) {\n // If destination aliases the shift vector then we need to move it temporarily.\n mov(VTMP2.Z(), ShiftVector.Z());\n ShiftVector = VTMP2;\n }\n\n // If Dst aliases Vector, then we can skip the move.\n if (Dst != Vector) {\n movprfx(Dst.Z(), Vector.Z());\n }\n lsl(SubRegSize, Dst.Z(), Mask, Dst.Z(), ShiftVector.Z());\n } else {\n if (RangeCheck) {\n if (ElementSize < IR::OpSize::i64Bit) {\n movi(SubRegSize, VTMP1.Q(), MaxShift);\n umin(SubRegSize, VTMP1.Q(), VTMP1.Q(), ShiftVector.Q());\n } else {\n LoadConstant(ARMEmitter::Size::i64Bit, TMP1, MaxShift);\n dup(SubRegSize, VTMP1.Q(), TMP1.R());\n\n // UMIN is silly on Adv.SIMD and doesn't have a variant that handles 64-bit elements\n cmhi(SubRegSize, VTMP2.Q(), ShiftVector.Q(), VTMP1.Q());\n bif(VTMP1.Q(), ShiftVector.Q(), VTMP2.Q());\n }\n ShiftVector = VTMP1;\n }\n\n ushl(SubRegSize, Dst.Q(), Vector.Q(), ShiftVector.Q());\n }\n}\n\nDEF_OP(VUShr) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto ElementSize = IROp->ElementSize;\n const auto SubRegSize = ConvertSubRegSize8(IROp);\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto MaxShift = IR::OpSizeAsBits(ElementSize);\n\n const auto Dst = GetVReg(Node);\n auto ShiftVector = GetVReg(Op->ShiftVector);\n const auto Vector = GetVReg(Op->Vector);\n const auto RangeCheck = Op->RangeCheck;\n\n if (HostSupportsSVE256 && Is256Bit) {\n const auto Mask = PRED_TMP_32B.Merging();\n\n if (RangeCheck) {\n dup_imm(SubRegSize, VTMP2.Z(), MaxShift);\n umin(SubRegSize, VTMP2.Z(), Mask, VTMP2.Z(), ShiftVector.Z());\n ShiftVector = VTMP2;\n }\n\n if (Dst == ShiftVector) {\n // If destination aliases the shift vector then we need to move it temporarily.\n mov(VTMP2.Z(), ShiftVector.Z());\n ShiftVector = VTMP2;\n }\n\n // If Dst aliases Vector, then we can skip the move.\n if (Dst != Vector) {\n movprfx(Dst.Z(), Vector.Z());\n }\n lsr(SubRegSize, Dst.Z(), Mask, Dst.Z(), ShiftVector.Z());\n } else {\n if (RangeCheck) {\n if (ElementSize < IR::OpSize::i64Bit) {\n movi(SubRegSize, VTMP1.Q(), MaxShift);\n umin(SubRegSize, VTMP1.Q(), VTMP1.Q(), ShiftVector.Q());\n } else {\n LoadConstant(ARMEmitter::Size::i64Bit, TMP1, MaxShift);\n dup(SubRegSize, VTMP1.Q(), TMP1.R());\n\n // UMIN is silly on Adv.SIMD and doesn't have a variant that handles 64-bit elements\n cmhi(SubRegSize, VTMP2.Q(), ShiftVector.Q(), VTMP1.Q());\n bif(VTMP1.Q(), ShiftVector.Q(), VTMP2.Q());\n }\n ShiftVector = VTMP1;\n }\n\n // Need to invert shift values to perform a right shift with USHL\n // (USHR only has an immediate variant).\n neg(SubRegSize, VTMP1.Q(), ShiftVector.Q());\n ushl(SubRegSize, Dst.Q(), Vector.Q(), VTMP1.Q());\n }\n}\n\nDEF_OP(VSShr) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto ElementSize = IROp->ElementSize;\n const auto SubRegSize = ConvertSubRegSize8(IROp);\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto MaxShift = IR::OpSizeAsBits(ElementSize) - 1;\n const auto RangeCheck = Op->RangeCheck;\n\n const auto Dst = GetVReg(Node);\n auto ShiftVector = GetVReg(Op->ShiftVector);\n const auto Vector = GetVReg(Op->Vector);\n\n if (HostSupportsSVE256 && Is256Bit) {\n const auto Mask = PRED_TMP_32B.Merging();\n\n if (RangeCheck) {\n dup_imm(SubRegSize, VTMP1.Z(), MaxShift);\n umin(SubRegSize, VTMP1.Z(), Mask, VTMP1.Z(), ShiftVector.Z());\n ShiftVector = VTMP1;\n }\n\n if (Dst == ShiftVector) {\n // If destination aliases the shift vector then we need to move it temporarily.\n mov(VTMP1.Z(), ShiftVector.Z());\n ShiftVector = VTMP1;\n }\n\n // If Dst aliases Vector, then we can skip the move.\n if (Dst != Vector) {\n movprfx(Dst.Z(), Vector.Z());\n }\n asr(SubRegSize, Dst.Z(), Mask, Dst.Z(), ShiftVector.Z());\n } else {\n if (RangeCheck) {\n if (ElementSize < IR::OpSize::i64Bit) {\n movi(SubRegSize, VTMP1.Q(), MaxShift);\n umin(SubRegSize, VTMP1.Q(), VTMP1.Q(), ShiftVector.Q());\n } else {\n LoadConstant(ARMEmitter::Size::i64Bit, TMP1, MaxShift);\n dup(SubRegSize, VTMP1.Q(), TMP1.R());\n\n // UMIN is silly on Adv.SIMD and doesn't have a variant that handles 64-bit elements\n cmhi(SubRegSize, VTMP2.Q(), ShiftVector.Q(), VTMP1.Q());\n bif(VTMP1.Q(), ShiftVector.Q(), VTMP2.Q());\n }\n ShiftVector = VTMP1;\n }\n\n // Need to invert shift values to perform a right shift with SSHL\n // (SSHR only has an immediate variant).\n neg(SubRegSize, VTMP1.Q(), ShiftVector.Q());\n sshl(SubRegSize, Dst.Q(), Vector.Q(), VTMP1.Q());\n }\n}\n\nDEF_OP(VUShlS) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto SubRegSize = ConvertSubRegSize16(IROp);\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto Dst = GetVReg(Node);\n const auto ShiftScalar = GetVReg(Op->ShiftScalar);\n const auto Vector = GetVReg(Op->Vector);\n\n if (HostSupportsSVE256 && Is256Bit) {\n const auto Mask = PRED_TMP_32B.Merging();\n\n // NOTE: SVE LSL is a destructive operation, so we need to\n // move the vector into the destination if they don't\n // already alias.\n dup(SubRegSize, VTMP1.Z(), ShiftScalar.Z(), 0);\n if (Dst != Vector) {\n movprfx(Dst.Z(), Vector.Z());\n }\n lsl(SubRegSize, Dst.Z(), Mask, Dst.Z(), VTMP1.Z());\n } else {\n dup(SubRegSize, VTMP1.Q(), ShiftScalar.Q(), 0);\n ushl(SubRegSize, Dst.Q(), Vector.Q(), VTMP1.Q());\n }\n}\n\nDEF_OP(VUShrS) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto SubRegSize = ConvertSubRegSize16(IROp);\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto Dst = GetVReg(Node);\n const auto ShiftScalar = GetVReg(Op->ShiftScalar);\n const auto Vector = GetVReg(Op->Vector);\n\n if (HostSupportsSVE256 && Is256Bit) {\n const auto Mask = PRED_TMP_32B.Merging();\n\n // NOTE: SVE LSR is a destructive operation, so we need to\n // move the vector into the destination if they don't\n // already alias.\n dup(SubRegSize, VTMP1.Z(), ShiftScalar.Z(), 0);\n if (Dst != Vector) {\n movprfx(Dst.Z(), Vector.Z());\n }\n lsr(SubRegSize, Dst.Z(), Mask, Dst.Z(), VTMP1.Z());\n } else {\n dup(SubRegSize, VTMP1.Q(), ShiftScalar.Q(), 0);\n neg(SubRegSize, VTMP1.Q(), VTMP1.Q());\n ushl(SubRegSize, Dst.Q(), Vector.Q(), VTMP1.Q());\n }\n}\n\nDEF_OP(VUShrSWide) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto ElementSize = Op->Header.ElementSize;\n const auto SubRegSize = ConvertSubRegSize8(IROp);\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto Dst = GetVReg(Node);\n const auto ShiftScalar = GetVReg(Op->ShiftScalar);\n const auto Vector = GetVReg(Op->Vector);\n\n if (HostSupportsSVE256 && Is256Bit) {\n const auto Mask = PRED_TMP_32B.Merging();\n\n dup(ARMEmitter::SubRegSize::i64Bit, VTMP1.Z(), ShiftScalar.Z(), 0);\n if (Dst != Vector) {\n // NOTE: SVE LSR is a destructive operation.\n movprfx(Dst.Z(), Vector.Z());\n }\n if (ElementSize == IR::OpSize::i64Bit) {\n lsr(SubRegSize, Dst.Z(), Mask, Dst.Z(), VTMP1.Z());\n } else {\n lsr_wide(SubRegSize, Dst.Z(), Mask, Dst.Z(), VTMP1.Z());\n }\n } else if (HostSupportsSVE128) {\n const auto Mask = PRED_TMP_16B.Merging();\n\n auto ShiftRegister = ShiftScalar;\n if (OpSize > IR::OpSize::i64Bit) {\n // SVE wide shifts don't need to duplicate the low bits unless the OpSize is 16-bytes\n // Slightly more optimal for 8-byte opsize.\n dup(ARMEmitter::SubRegSize::i64Bit, VTMP1.Z(), ShiftScalar.Z(), 0);\n ShiftRegister = VTMP1;\n }\n\n if (Dst == ShiftRegister) {\n // If destination aliases the shift vector then we need to move it temporarily.\n mov(VTMP1.Z(), ShiftRegister.Z());\n ShiftRegister = VTMP1;\n }\n\n if (Dst != Vector) {\n // NOTE: SVE LSR is a destructive operation.\n movprfx(Dst.Z(), Vector.Z());\n }\n if (ElementSize == IR::OpSize::i64Bit) {\n lsr(SubRegSize, Dst.Z(), Mask, Dst.Z(), ShiftRegister.Z());\n } else {\n lsr_wide(SubRegSize, Dst.Z(), Mask, Dst.Z(), ShiftRegister.Z());\n }\n } else {\n // uqshl + ushr of 57-bits leaves 7-bits remaining.\n // This saturates the 64-bit shift value from an arbitrary 64-bit length\n // variable to maximum of 0x7F.\n // This allows the shift to fit within the width of the signed 8-bits\n // that ASIMD's vector shift requires.\n uqshl(ARMEmitter::ScalarRegSize::i64Bit, VTMP1, ShiftScalar, 57);\n ushr(ARMEmitter::ScalarRegSize::i64Bit, VTMP1, VTMP1, 57);\n dup(SubRegSize, VTMP1.Q(), VTMP1.Q(), 0);\n neg(SubRegSize, VTMP1.Q(), VTMP1.Q());\n ushl(SubRegSize, Dst.Q(), Vector.Q(), VTMP1.Q());\n }\n}\n\nDEF_OP(VSShrSWide) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto ElementSize = Op->Header.ElementSize;\n const auto SubRegSize = ConvertSubRegSize8(IROp);\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto Dst = GetVReg(Node);\n const auto ShiftScalar = GetVReg(Op->ShiftScalar);\n const auto Vector = GetVReg(Op->Vector);\n\n if (HostSupportsSVE256 && Is256Bit) {\n const auto Mask = PRED_TMP_32B.Merging();\n\n dup(ARMEmitter::SubRegSize::i64Bit, VTMP1.Z(), ShiftScalar.Z(), 0);\n if (Dst != Vector) {\n // NOTE: SVE LSR is a destructive operation.\n movprfx(Dst.Z(), Vector.Z());\n }\n if (ElementSize == IR::OpSize::i64Bit) {\n asr(SubRegSize, Dst.Z(), Mask, Dst.Z(), VTMP1.Z());\n } else {\n asr_wide(SubRegSize, Dst.Z(), Mask, Dst.Z(), VTMP1.Z());\n }\n } else if (HostSupportsSVE128) {\n const auto Mask = PRED_TMP_16B.Merging();\n\n auto ShiftRegister = ShiftScalar;\n if (OpSize > IR::OpSize::i64Bit) {\n // SVE wide shifts don't need to duplicate the low bits unless the OpSize is 16-bytes\n // Slightly more optimal for 8-byte opsize.\n dup(ARMEmitter::SubRegSize::i64Bit, VTMP1.Z(), ShiftScalar.Z(), 0);\n ShiftRegister = VTMP1;\n }\n\n if (Dst == ShiftRegister) {\n // If destination aliases the shift vector then we need to move it temporarily.\n mov(VTMP1.Z(), ShiftRegister.Z());\n ShiftRegister = VTMP1;\n }\n\n if (Dst != Vector) {\n // NOTE: SVE LSR is a destructive operation.\n movprfx(Dst.Z(), Vector.Z());\n }\n if (ElementSize == IR::OpSize::i64Bit) {\n asr(SubRegSize, Dst.Z(), Mask, Dst.Z(), ShiftRegister.Z());\n } else {\n asr_wide(SubRegSize, Dst.Z(), Mask, Dst.Z(), ShiftRegister.Z());\n }\n } else {\n // uqshl + ushr of 57-bits leaves 7-bits remaining.\n // This saturates the 64-bit shift value from an arbitrary 64-bit length\n // variable to maximum of 0x7F.\n // This allows the shift to fit within the width of the signed 8-bits\n // that ASIMD's vector shift requires.\n uqshl(ARMEmitter::ScalarRegSize::i64Bit, VTMP1, ShiftScalar, 57);\n ushr(ARMEmitter::ScalarRegSize::i64Bit, VTMP1, VTMP1, 57);\n dup(SubRegSize, VTMP1.Q(), VTMP1.Q(), 0);\n neg(SubRegSize, VTMP1.Q(), VTMP1.Q());\n sshl(SubRegSize, Dst.Q(), Vector.Q(), VTMP1.Q());\n }\n}\n\nDEF_OP(VUShlSWide) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto ElementSize = Op->Header.ElementSize;\n const auto SubRegSize = ConvertSubRegSize8(IROp);\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto Dst = GetVReg(Node);\n const auto ShiftScalar = GetVReg(Op->ShiftScalar);\n const auto Vector = GetVReg(Op->Vector);\n\n if (HostSupportsSVE256 && Is256Bit) {\n const auto Mask = PRED_TMP_32B.Merging();\n\n dup(ARMEmitter::SubRegSize::i64Bit, VTMP1.Z(), ShiftScalar.Z(), 0);\n if (Dst != Vector) {\n // NOTE: SVE LSR is a destructive operation.\n movprfx(Dst.Z(), Vector.Z());\n }\n if (ElementSize == IR::OpSize::i64Bit) {\n lsl(SubRegSize, Dst.Z(), Mask, Dst.Z(), VTMP1.Z());\n } else {\n lsl_wide(SubRegSize, Dst.Z(), Mask, Dst.Z(), VTMP1.Z());\n }\n } else if (HostSupportsSVE128) {\n const auto Mask = PRED_TMP_16B.Merging();\n\n auto ShiftRegister = ShiftScalar;\n if (OpSize > IR::OpSize::i64Bit) {\n // SVE wide shifts don't need to duplicate the low bits unless the OpSize is 16-bytes\n // Slightly more optimal for 8-byte opsize.\n dup(ARMEmitter::SubRegSize::i64Bit, VTMP1.Z(), ShiftScalar.Z(), 0);\n ShiftRegister = VTMP1;\n }\n\n if (Dst == ShiftRegister) {\n // If destination aliases the shift vector then we need to move it temporarily.\n mov(VTMP1.Z(), ShiftRegister.Z());\n ShiftRegister = VTMP1;\n }\n\n if (Dst != Vector) {\n // NOTE: SVE LSR is a destructive operation.\n movprfx(Dst.Z(), Vector.Z());\n }\n if (ElementSize == IR::OpSize::i64Bit) {\n lsl(SubRegSize, Dst.Z(), Mask, Dst.Z(), ShiftRegister.Z());\n } else {\n lsl_wide(SubRegSize, Dst.Z(), Mask, Dst.Z(), ShiftRegister.Z());\n }\n } else {\n // uqshl + ushr of 57-bits leaves 7-bits remaining.\n // This saturates the 64-bit shift value from an arbitrary 64-bit length\n // variable to maximum of 0x7F.\n // This allows the shift to fit within the width of the signed 8-bits\n // that ASIMD's vector shift requires.\n uqshl(ARMEmitter::ScalarRegSize::i64Bit, VTMP1, ShiftScalar, 57);\n ushr(ARMEmitter::ScalarRegSize::i64Bit, VTMP1, VTMP1, 57);\n dup(SubRegSize, VTMP1.Q(), VTMP1.Q(), 0);\n ushl(SubRegSize, Dst.Q(), Vector.Q(), VTMP1.Q());\n }\n}\n\nDEF_OP(VSShrS) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto SubRegSize = ConvertSubRegSize16(IROp);\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto Dst = GetVReg(Node);\n const auto ShiftScalar = GetVReg(Op->ShiftScalar);\n const auto Vector = GetVReg(Op->Vector);\n\n if (HostSupportsSVE256 && Is256Bit) {\n const auto Mask = PRED_TMP_32B.Merging();\n\n // NOTE: SVE ASR is a destructive operation, so we need to\n // move the vector into the destination if they don't\n // already alias.\n dup(SubRegSize, VTMP1.Z(), ShiftScalar.Z(), 0);\n if (Dst != Vector) {\n movprfx(Dst.Z(), Vector.Z());\n }\n asr(SubRegSize, Dst.Z(), Mask, Dst.Z(), VTMP1.Z());\n } else {\n dup(SubRegSize, VTMP1.Q(), ShiftScalar.Q(), 0);\n neg(SubRegSize, VTMP1.Q(), VTMP1.Q());\n sshl(SubRegSize, Dst.Q(), Vector.Q(), VTMP1.Q());\n }\n}\n\nDEF_OP(VInsElement) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto ElementSize = Op->Header.ElementSize;\n const auto SubRegSize = ConvertSubRegSize16(IROp);\n\n const uint32_t DestIdx = Op->DestIdx;\n const uint32_t SrcIdx = Op->SrcIdx;\n\n const auto Dst = GetVReg(Node);\n const auto SrcVector = GetVReg(Op->SrcVector);\n auto Reg = GetVReg(Op->DestVector);\n\n if (HostSupportsSVE256 && Is256Bit) {\n // Broadcast our source value across a temporary,\n // then combine with the destination.\n dup(SubRegSize, VTMP2.Z(), SrcVector.Z(), SrcIdx);\n\n // We don't need to move the data unnecessarily if\n // DestVector just so happens to also be the IR op\n // destination.\n if (Dst != Reg) {\n mov(Dst.Z(), Reg.Z());\n }\n\n constexpr auto Predicate = ARMEmitter::PReg::p0;\n\n if (ElementSize == IR::OpSize::i128Bit) {\n if (DestIdx == 0) {\n mov(ARMEmitter::SubRegSize::i8Bit, Dst.Z(), PRED_TMP_16B.Merging(), VTMP2.Z());\n } else {\n not_(Predicate, PRED_TMP_32B.Zeroing(), PRED_TMP_16B);\n mov(ARMEmitter::SubRegSize::i8Bit, Dst.Z(), Predicate.Merging(), VTMP2.Z());\n }\n } else {\n const auto UpperBound = 16 >> FEXCore::ilog2(IR::OpSizeToSize(ElementSize));\n const auto TargetElement = static_cast(DestIdx) - UpperBound;\n\n // FIXME: We should rework this op to avoid the NZCV spill/fill dance.\n mrs(TMP1, ARMEmitter::SystemRegister::NZCV);\n\n index(SubRegSize, VTMP1.Z(), -UpperBound, 1);\n cmpeq(SubRegSize, Predicate, PRED_TMP_32B.Zeroing(), VTMP1.Z(), TargetElement);\n mov(SubRegSize, Dst.Z(), Predicate.Merging(), VTMP2.Z());\n\n // Restore NZCV\n msr(ARMEmitter::SystemRegister::NZCV, TMP1);\n }\n } else {\n // If nothing aliases the destination, then we can just\n // move the DestVector over and directly insert.\n if (Dst != Reg && Dst != SrcVector) {\n mov(Dst.Q(), Reg.Q());\n ins(SubRegSize, Dst.Q(), DestIdx, SrcVector.Q(), SrcIdx);\n return;\n }\n\n // If our vector data to insert into is within a register\n // that aliases the destination, then we can avoid using a\n // temporary and just perform the insert.\n //\n // Otherwise, if the source vector to select from aliases\n // the destination, then we hit the worst case where we\n // need to use a temporary to avoid clobbering data.\n if (Dst != Reg) {\n mov(VTMP1.Q(), Reg.Q());\n Reg = VTMP1;\n }\n\n ins(SubRegSize, Reg.Q(), DestIdx, SrcVector.Q(), SrcIdx);\n\n if (Dst != Reg) {\n mov(Dst.Q(), Reg.Q());\n }\n }\n}\n\nDEF_OP(VDupElement) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto Index = Op->Index;\n const auto SubRegSize = ConvertSubRegSize16(IROp);\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto Is128Bit = OpSize == IR::OpSize::i128Bit;\n\n const auto Dst = GetVReg(Node);\n const auto Vector = GetVReg(Op->Vector);\n\n if (HostSupportsSVE256 && Is256Bit) {\n dup(SubRegSize, Dst.Z(), Vector.Z(), Index);\n } else {\n if (Is128Bit) {\n dup(SubRegSize, Dst.Q(), Vector.Q(), Index);\n } else {\n dup(SubRegSize, Dst.D(), Vector.D(), Index);\n }\n }\n}\n\nDEF_OP(VExtr) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n // AArch64 ext op has bit arrangement as [Vm:Vn] so arguments need to be swapped\n const auto Dst = GetVReg(Node);\n auto UpperBits = GetVReg(Op->VectorLower);\n auto LowerBits = GetVReg(Op->VectorUpper);\n\n const auto ElementSize = Op->Header.ElementSize;\n auto Index = Op->Index;\n\n if (Index >= IR::OpSizeToSize(OpSize)) {\n // Upper bits have moved in to the lower bits\n LowerBits = UpperBits;\n\n // Upper bits are all now zero\n UpperBits = VTMP1;\n movi(ARMEmitter::SubRegSize::i64Bit, VTMP1.Q(), 0);\n Index -= IR::OpSizeToSize(OpSize);\n }\n\n const auto CopyFromByte = Index * IR::OpSizeToSize(ElementSize);\n\n if (HostSupportsSVE256 && Is256Bit) {\n if (Dst == LowerBits) {\n // Trivial case where we don't need to do any moves\n ext(Dst.Z(), Dst.Z(), UpperBits.Z(), CopyFromByte);\n } else if (Dst == UpperBits) {\n movprfx(VTMP2.Z(), LowerBits.Z());\n ext(VTMP2.Z(), VTMP2.Z(), UpperBits.Z(), CopyFromByte);\n mov(Dst.Z(), VTMP2.Z());\n } else {\n // No registers alias the destination, so we can safely move into it.\n movprfx(Dst.Z(), LowerBits.Z());\n ext(Dst.Z(), Dst.Z(), UpperBits.Z(), CopyFromByte);\n }\n } else {\n if (OpSize == IR::OpSize::i64Bit) {\n ext(Dst.D(), LowerBits.D(), UpperBits.D(), CopyFromByte);\n } else {\n ext(Dst.Q(), LowerBits.Q(), UpperBits.Q(), CopyFromByte);\n }\n }\n}\n\nDEF_OP(VUShrI) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto BitShift = Op->BitShift;\n const auto ElementSize = Op->Header.ElementSize;\n const auto SubRegSize = ConvertSubRegSize8(IROp);\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto Dst = GetVReg(Node);\n const auto Vector = GetVReg(Op->Vector);\n\n if (BitShift >= IR::OpSizeAsBits(ElementSize)) {\n movi(ARMEmitter::SubRegSize::i64Bit, Dst.Q(), 0);\n } else {\n if (HostSupportsSVE256 && Is256Bit) {\n const auto Mask = PRED_TMP_32B.Merging();\n\n if (BitShift == 0) {\n if (Dst != Vector) {\n mov(Dst.Z(), Vector.Z());\n }\n } else {\n // SVE LSR is destructive, so lets set up the destination if\n // Vector doesn't already alias it.\n if (Dst != Vector) {\n movprfx(Dst.Z(), Vector.Z());\n }\n lsr(SubRegSize, Dst.Z(), Mask, Dst.Z(), BitShift);\n }\n } else {\n if (BitShift == 0) {\n if (Dst != Vector) {\n mov(Dst.Q(), Vector.Q());\n }\n } else {\n ushr(SubRegSize, Dst.Q(), Vector.Q(), BitShift);\n }\n }\n }\n}\n\nDEF_OP(VUShraI) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto BitShift = Op->BitShift;\n const auto SubRegSize = ConvertSubRegSize8(IROp);\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto Dst = GetVReg(Node);\n const auto DestVector = GetVReg(Op->DestVector);\n const auto Vector = GetVReg(Op->Vector);\n\n if (HostSupportsSVE256 && Is256Bit) {\n if (Dst == DestVector) {\n usra(SubRegSize, Dst.Z(), Vector.Z(), BitShift);\n } else {\n if (Dst != Vector) {\n mov(Dst.Z(), DestVector.Z());\n usra(SubRegSize, Dst.Z(), Vector.Z(), BitShift);\n } else {\n mov(VTMP1.Z(), DestVector.Z());\n usra(SubRegSize, Dst.Z(), Vector.Z(), BitShift);\n mov(Dst.Z(), VTMP1.Z());\n }\n }\n } else {\n if (Dst == DestVector) {\n usra(SubRegSize, Dst.Q(), Vector.Q(), BitShift);\n } else {\n if (Dst != Vector) {\n mov(Dst.Q(), DestVector.Q());\n usra(SubRegSize, Dst.Q(), Vector.Q(), BitShift);\n } else {\n mov(VTMP1.Q(), DestVector.Q());\n usra(SubRegSize, VTMP1.Q(), Vector.Q(), BitShift);\n mov(Dst.Q(), VTMP1.Q());\n }\n }\n }\n}\n\nDEF_OP(VSShrI) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto ElementSize = Op->Header.ElementSize;\n LOGMAN_THROW_A_FMT(ElementSize >= IR::OpSize::i8Bit && ElementSize <= IR::OpSize::i64Bit, \"Invalid element size\");\n const auto SubRegSize = ConvertSubRegSize8(IROp);\n const auto Shift = std::min(IR::OpSizeAsBits(ElementSize) - 1, Op->BitShift);\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto Dst = GetVReg(Node);\n const auto Vector = GetVReg(Op->Vector);\n\n if (HostSupportsSVE256 && Is256Bit) {\n const auto Mask = PRED_TMP_32B.Merging();\n\n if (Shift == 0) {\n if (Dst != Vector) {\n mov(Dst.Z(), Vector.Z());\n }\n } else {\n // SVE ASR is destructive, so lets set up the destination if\n // Vector doesn't already alias it.\n if (Dst != Vector) {\n movprfx(Dst.Z(), Vector.Z());\n }\n asr(SubRegSize, Dst.Z(), Mask, Dst.Z(), Shift);\n }\n } else {\n if (Shift == 0) {\n if (Dst != Vector) {\n mov(Dst.Q(), Vector.Q());\n }\n } else {\n sshr(SubRegSize, Dst.Q(), Vector.Q(), Shift);\n }\n }\n}\n\nDEF_OP(VShlI) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto BitShift = Op->BitShift;\n const auto ElementSize = Op->Header.ElementSize;\n const auto SubRegSize = ConvertSubRegSize8(IROp);\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto Dst = GetVReg(Node);\n const auto Vector = GetVReg(Op->Vector);\n\n if (BitShift >= IR::OpSizeAsBits(ElementSize)) {\n movi(ARMEmitter::SubRegSize::i64Bit, Dst.Q(), 0);\n } else {\n if (HostSupportsSVE256 && Is256Bit) {\n const auto Mask = PRED_TMP_32B.Merging();\n\n if (BitShift == 0) {\n if (Dst != Vector) {\n mov(Dst.Z(), Vector.Z());\n }\n } else {\n // SVE LSL is destructive, so lets set up the destination if\n // Vector doesn't already alias it.\n if (Dst != Vector) {\n movprfx(Dst.Z(), Vector.Z());\n }\n lsl(SubRegSize, Dst.Z(), Mask, Dst.Z(), BitShift);\n }\n } else {\n if (BitShift == 0) {\n if (Dst != Vector) {\n mov(Dst.Q(), Vector.Q());\n }\n } else {\n shl(SubRegSize, Dst.Q(), Vector.Q(), BitShift);\n }\n }\n }\n}\n\nDEF_OP(VUShrNI) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto BitShift = Op->BitShift;\n const auto SubRegSize = ConvertSubRegSize4(IROp);\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto Dst = GetVReg(Node);\n const auto Vector = GetVReg(Op->Vector);\n\n if (HostSupportsSVE256 && Is256Bit) {\n shrnb(SubRegSize, Dst.Z(), Vector.Z(), BitShift);\n uzp1(SubRegSize, Dst.Z(), Dst.Z(), Dst.Z());\n } else {\n if (BitShift == 0) {\n xtn(SubRegSize, Dst.D(), Vector.D());\n } else {\n shrn(SubRegSize, Dst.D(), Vector.D(), BitShift);\n }\n }\n}\n\nDEF_OP(VUShrNI2) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto BitShift = Op->BitShift;\n const auto SubRegSize = ConvertSubRegSize8(IROp);\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto Dst = GetVReg(Node);\n const auto VectorLower = GetVReg(Op->VectorLower);\n const auto VectorUpper = GetVReg(Op->VectorUpper);\n\n if (HostSupportsSVE256 && Is256Bit) {\n const auto Mask = PRED_TMP_16B;\n\n shrnb(SubRegSize, VTMP2.Z(), VectorUpper.Z(), BitShift);\n uzp1(SubRegSize, VTMP2.Z(), VTMP2.Z(), VTMP2.Z());\n\n if (Dst != VectorLower) {\n movprfx(Dst.Z(), VectorLower.Z());\n }\n splice(SubRegSize, Dst.Z(), Mask, Dst.Z(), VTMP2.Z());\n } else {\n auto Lower = VectorLower;\n if (Dst != VectorLower) {\n mov(VTMP1.Q(), VectorLower.Q());\n Lower = VTMP1;\n }\n\n shrn2(SubRegSize, Lower.Q(), VectorUpper.Q(), BitShift);\n\n if (Dst != VectorLower) {\n mov(Dst.Q(), Lower.Q());\n }\n }\n}\n\nDEF_OP(VSXTL) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto SubRegSize = ConvertSubRegSize248(IROp);\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto Dst = GetVReg(Node);\n const auto Vector = GetVReg(Op->Vector);\n\n if ((HostSupportsSVE128 && !Is256Bit && !HostSupportsSVE256) || (HostSupportsSVE256 && Is256Bit)) {\n sunpklo(SubRegSize, Dst.Z(), Vector.Z());\n } else {\n sxtl(SubRegSize, Dst.D(), Vector.D());\n }\n}\n\nDEF_OP(VSXTL2) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto SubRegSize = ConvertSubRegSize248(IROp);\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto Dst = GetVReg(Node);\n const auto Vector = GetVReg(Op->Vector);\n\n if ((HostSupportsSVE128 && !Is256Bit && !HostSupportsSVE256) || (HostSupportsSVE256 && Is256Bit)) {\n sunpkhi(SubRegSize, Dst.Z(), Vector.Z());\n } else {\n sxtl2(SubRegSize, Dst.Q(), Vector.Q());\n }\n}\n\nDEF_OP(VSSHLL) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto SubRegSize = ConvertSubRegSize248(IROp);\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto Dst = GetVReg(Node);\n const auto Vector = GetVReg(Op->Vector);\n const auto BitShift = Op->BitShift;\n LOGMAN_THROW_A_FMT(BitShift < IR::OpSizeAsBits(IROp->ElementSize / 2), \"Bitshift size too large for source element size: {} < {}\",\n BitShift, IR::OpSizeAsBits(IROp->ElementSize / 2));\n\n if (Is256Bit) {\n sunpklo(SubRegSize, Dst.Z(), Vector.Z());\n lsl(SubRegSize, Dst.Z(), Dst.Z(), BitShift);\n } else {\n sshll(SubRegSize, Dst.D(), Vector.D(), BitShift);\n }\n}\n\nDEF_OP(VSSHLL2) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto SubRegSize = ConvertSubRegSize248(IROp);\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto Dst = GetVReg(Node);\n const auto Vector = GetVReg(Op->Vector);\n const auto BitShift = Op->BitShift;\n LOGMAN_THROW_A_FMT(BitShift < IR::OpSizeAsBits(IROp->ElementSize / 2), \"Bitshift size too large for source element size: {} < {}\",\n BitShift, IR::OpSizeAsBits(IROp->ElementSize / 2));\n\n if (Is256Bit) {\n sunpkhi(SubRegSize, Dst.Z(), Vector.Z());\n lsl(SubRegSize, Dst.Z(), Dst.Z(), BitShift);\n } else {\n sshll2(SubRegSize, Dst.Q(), Vector.Q(), BitShift);\n }\n}\n\nDEF_OP(VUXTL) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto SubRegSize = ConvertSubRegSize248(IROp);\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto Dst = GetVReg(Node);\n const auto Vector = GetVReg(Op->Vector);\n\n if ((HostSupportsSVE128 && !Is256Bit && !HostSupportsSVE256) || (HostSupportsSVE256 && Is256Bit)) {\n uunpklo(SubRegSize, Dst.Z(), Vector.Z());\n } else {\n uxtl(SubRegSize, Dst.D(), Vector.D());\n }\n}\n\nDEF_OP(VUXTL2) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto SubRegSize = ConvertSubRegSize248(IROp);\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto Dst = GetVReg(Node);\n const auto Vector = GetVReg(Op->Vector);\n\n if ((HostSupportsSVE128 && !Is256Bit && !HostSupportsSVE256) || (HostSupportsSVE256 && Is256Bit)) {\n uunpkhi(SubRegSize, Dst.Z(), Vector.Z());\n } else {\n uxtl2(SubRegSize, Dst.Q(), Vector.Q());\n }\n}\n\nDEF_OP(VSQXTN) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto SubRegSize = ConvertSubRegSize4(IROp);\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto Dst = GetVReg(Node);\n const auto Vector = GetVReg(Op->Vector);\n\n if (HostSupportsSVE256 && Is256Bit) {\n // Note that SVE SQXTNB and SQXTNT are a tad different\n // in behavior compared to most other [name]B and [name]T\n // instructions.\n //\n // Most other bottom and top instructions operate\n // on even (bottom) or odd (top) elements and store each\n // result into the next subsequent element in the destination\n // vector\n //\n // SQXTNB and SQXTNT will operate on the same elements regardless\n // of which one is chosen, but will instead place results from\n // the operation into either each subsequent even (bottom) element\n // or odd (top) element. However the bottom instruction will zero the\n // odd elements out in the destination vector, while the top instruction\n // will leave the even elements alone (in a behavior similar to Adv.SIMD's\n // SQXTN/SQXTN2 instructions).\n //\n // e.g. consider this 64-bit (for brevity) vector with four 16-bit elements:\n //\n // ╔═══════════╗╔═══════════╗╔═══════════╗╔═══════════╗\n // ║ Value 3 ║║ Value 2 ║║ Value 1 ║║ Value 0 ║\n // ╚═══════════╝╚═══════════╝╚═══════════╝╚═══════════╝\n //\n // SQXTNB Dst.VnB, Src.VnH will result in:\n //\n // ╔═════╗╔═════╗╔═════╗╔═════╗╔═════╗╔═════╗╔═════╗╔═════╗\n // ║ 0 ║║ V3 ║║ 0 ║║ V2 ║║ 0 ║║ V1 ║║ 0 ║║ V0 ║\n // ╚═════╝╚═════╝╚═════╝╚═════╝╚═════╝╚═════╝╚═════╝╚═════╝\n //\n // This is kind of convenient, considering we only need\n // to use the bottom variant and then concatenate all the\n // even elements with SVE UZP1.\n\n sqxtnb(SubRegSize, Dst.Z(), Vector.Z());\n uzp1(SubRegSize, Dst.Z(), Dst.Z(), Dst.Z());\n } else {\n sqxtn(SubRegSize, Dst, Vector);\n }\n}\n\nDEF_OP(VSQXTN2) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto SubRegSize = ConvertSubRegSize4(IROp);\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto Dst = GetVReg(Node);\n const auto VectorLower = GetVReg(Op->VectorLower);\n const auto VectorUpper = GetVReg(Op->VectorUpper);\n\n if (HostSupportsSVE256 && Is256Bit) {\n // We use the 16 byte mask due to how SPLICE works. We only\n // want to get at the first 16 bytes in the lower vector, so\n // that SPLICE will then begin copying the first 16 bytes\n // from the upper vector and begin placing them after the\n // previously copied lower 16 bytes.\n const auto Mask = PRED_TMP_16B;\n\n sqxtnb(SubRegSize, VTMP2.Z(), VectorUpper.Z());\n uzp1(SubRegSize, VTMP2.Z(), VTMP2.Z(), VTMP2.Z());\n\n // Need to use the destructive variant of SPLICE, since\n // the constructive variant requires a register list, and\n // we can't guarantee VectorLower and VectorUpper will always\n // have consecutive indexes with one another.\n if (Dst != VectorLower) {\n movprfx(Dst.Z(), VectorLower.Z());\n }\n splice(SubRegSize, Dst.Z(), Mask, Dst.Z(), VTMP2.Z());\n } else {\n if (OpSize == IR::OpSize::i64Bit) {\n sqxtn(SubRegSize, VTMP2, VectorUpper);\n mov(Dst.Q(), VectorLower.Q());\n ins(ARMEmitter::SubRegSize::i32Bit, Dst, 1, VTMP2, 0);\n } else {\n mov(VTMP1.Q(), VectorLower.Q());\n sqxtn2(SubRegSize, VTMP1, VectorUpper);\n mov(Dst.Q(), VTMP1.Q());\n }\n }\n}\n\nDEF_OP(VSQXTNPair) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto SubRegSize = ConvertSubRegSize4(IROp);\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto Dst = GetVReg(Node);\n const auto VectorLower = GetVReg(Op->VectorLower);\n auto VectorUpper = GetVReg(Op->VectorUpper);\n\n if (HostSupportsSVE256 && Is256Bit) {\n // This combines the SVE versions of VSQXTN/VSQXTN2.\n // Upper VSQXTN2 handling.\n // Doing upper first to ensure it doesn't get overwritten by lower calculation.\n const auto Mask = PRED_TMP_16B;\n\n sqxtnb(SubRegSize, VTMP2.Z(), VectorUpper.Z());\n uzp1(SubRegSize, VTMP2.Z(), VTMP2.Z(), VTMP2.Z());\n\n // Look at those implementations for details about this.\n // Lower VSQXTN handling.\n sqxtnb(SubRegSize, Dst.Z(), VectorLower.Z());\n uzp1(SubRegSize, Dst.Z(), Dst.Z(), Dst.Z());\n\n // Merge.\n splice(SubRegSize, Dst.Z(), Mask, Dst.Z(), VTMP2.Z());\n } else {\n if (OpSize == IR::OpSize::i64Bit) {\n zip1(ARMEmitter::SubRegSize::i64Bit, Dst.Q(), VectorLower.Q(), VectorUpper.Q());\n sqxtn(SubRegSize, Dst, Dst);\n } else {\n if (Dst == VectorUpper) {\n // If the destination overlaps the upper then we need to move it temporarily.\n mov(VTMP1.Q(), VectorUpper.Q());\n VectorUpper = VTMP1;\n }\n sqxtn(SubRegSize, Dst, VectorLower);\n sqxtn2(SubRegSize, Dst, VectorUpper);\n }\n }\n}\n\nDEF_OP(VSQXTUN) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto SubRegSize = ConvertSubRegSize8(IROp);\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto Dst = GetVReg(Node);\n const auto Vector = GetVReg(Op->Vector);\n\n if (HostSupportsSVE256 && Is256Bit) {\n sqxtunb(SubRegSize, Dst.Z(), Vector.Z());\n uzp1(SubRegSize, Dst.Z(), Dst.Z(), Dst.Z());\n } else {\n sqxtun(SubRegSize, Dst, Vector);\n }\n}\n\nDEF_OP(VSQXTUN2) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto SubRegSize = ConvertSubRegSize8(IROp);\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto Dst = GetVReg(Node);\n const auto VectorLower = GetVReg(Op->VectorLower);\n const auto VectorUpper = GetVReg(Op->VectorUpper);\n\n if (HostSupportsSVE256 && Is256Bit) {\n // NOTE: See VSQXTN2 implementation for an in-depth explanation\n // of everything going on here.\n\n const auto Mask = PRED_TMP_16B;\n\n sqxtunb(SubRegSize, VTMP2.Z(), VectorUpper.Z());\n uzp1(SubRegSize, VTMP2.Z(), VTMP2.Z(), VTMP2.Z());\n\n if (Dst != VectorLower) {\n movprfx(Dst.Z(), VectorLower.Z());\n }\n splice(SubRegSize, Dst.Z(), Mask, Dst.Z(), VTMP2.Z());\n } else {\n if (OpSize == IR::OpSize::i64Bit) {\n sqxtun(SubRegSize, VTMP2, VectorUpper);\n mov(Dst.Q(), VectorLower.Q());\n ins(ARMEmitter::SubRegSize::i32Bit, Dst, 1, VTMP2, 0);\n } else {\n auto Lower = VectorLower;\n if (Dst != VectorLower) {\n mov(VTMP1.Q(), VectorLower.Q());\n Lower = VTMP1;\n }\n\n sqxtun2(SubRegSize, Lower, VectorUpper);\n\n if (Dst != VectorLower) {\n mov(Dst.Q(), Lower.Q());\n }\n }\n }\n}\n\nDEF_OP(VSQXTUNPair) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto SubRegSize = ConvertSubRegSize4(IROp);\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto Dst = GetVReg(Node);\n const auto VectorLower = GetVReg(Op->VectorLower);\n auto VectorUpper = GetVReg(Op->VectorUpper);\n\n if (HostSupportsSVE256 && Is256Bit) {\n // This combines the SVE versions of VSQXTUN/VSQXTUN2.\n // Upper VSQXTUN2 handling.\n // Doing upper first to ensure it doesn't get overwritten by lower calculation.\n const auto Mask = PRED_TMP_16B;\n\n sqxtunb(SubRegSize, VTMP2.Z(), VectorUpper.Z());\n uzp1(SubRegSize, VTMP2.Z(), VTMP2.Z(), VTMP2.Z());\n\n // Look at those implementations for details about this.\n // Lower VSQXTUN handling.\n sqxtunb(SubRegSize, Dst.Z(), VectorLower.Z());\n uzp1(SubRegSize, Dst.Z(), Dst.Z(), Dst.Z());\n\n // Merge.\n splice(SubRegSize, Dst.Z(), Mask, Dst.Z(), VTMP2.Z());\n } else {\n if (OpSize == IR::OpSize::i64Bit) {\n zip1(ARMEmitter::SubRegSize::i64Bit, Dst.Q(), VectorLower.Q(), VectorUpper.Q());\n sqxtun(SubRegSize, Dst, Dst);\n } else {\n if (Dst == VectorUpper) {\n // If the destination overlaps the upper then we need to move it temporarily.\n mov(VTMP1.Q(), VectorUpper.Q());\n VectorUpper = VTMP1;\n }\n sqxtun(SubRegSize, Dst, VectorLower);\n sqxtun2(SubRegSize, Dst, VectorUpper);\n }\n }\n}\n\nDEF_OP(VSRSHR) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto SubRegSize = ConvertSubRegSize16(IROp);\n\n const auto Dst = GetVReg(Node);\n const auto Vector = GetVReg(Op->Vector);\n const auto BitShift = Op->BitShift;\n\n if (HostSupportsSVE256 && Is256Bit) {\n const auto Mask = PRED_TMP_32B.Merging();\n // SVE SRSHR is destructive, so lets set up the destination\n // in the event we Dst and Vector don't alias.\n if (Dst != Vector) {\n movprfx(Dst.Z(), Vector.Z());\n }\n srshr(SubRegSize, Dst.Z(), Mask, Dst.Z(), BitShift);\n } else {\n if (OpSize == IR::OpSize::i64Bit) {\n srshr(SubRegSize, Dst.D(), Vector.D(), BitShift);\n } else {\n srshr(SubRegSize, Dst.Q(), Vector.Q(), BitShift);\n }\n }\n}\n\nDEF_OP(VSQSHL) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto SubRegSize = ConvertSubRegSize8(IROp);\n\n const auto Dst = GetVReg(Node);\n const auto Vector = GetVReg(Op->Vector);\n const auto BitShift = Op->BitShift;\n\n if (HostSupportsSVE256 && Is256Bit) {\n const auto Mask = PRED_TMP_32B.Merging();\n // SVE SQSHL is destructive, so lets set up the destination\n // in the event Dst and Vector don't alias\n if (Dst != Vector) {\n movprfx(Dst.Z(), Vector.Z());\n }\n sqshl(SubRegSize, Dst.Z(), Mask, Dst.Z(), BitShift);\n } else {\n if (OpSize == IR::OpSize::i64Bit) {\n sqshl(SubRegSize, Dst.D(), Vector.D(), BitShift);\n } else {\n sqshl(SubRegSize, Dst.Q(), Vector.Q(), BitShift);\n }\n }\n}\n\nDEF_OP(VMul) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto SubRegSize = ConvertSubRegSize16(IROp);\n\n const auto Dst = GetVReg(Node);\n const auto Vector1 = GetVReg(Op->Vector1);\n const auto Vector2 = GetVReg(Op->Vector2);\n\n if (HostSupportsSVE256 && Is256Bit) {\n mul(SubRegSize, Dst.Z(), Vector1.Z(), Vector2.Z());\n } else {\n mul(SubRegSize, Dst.Q(), Vector1.Q(), Vector2.Q());\n }\n}\n\nDEF_OP(VUMull) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto SubRegSize = ConvertSubRegSize248(IROp);\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto Dst = GetVReg(Node);\n const auto Vector1 = GetVReg(Op->Vector1);\n const auto Vector2 = GetVReg(Op->Vector2);\n\n if (HostSupportsSVE256 && Is256Bit) {\n umullb(SubRegSize, VTMP1.Z(), Vector1.Z(), Vector2.Z());\n umullt(SubRegSize, VTMP2.Z(), Vector1.Z(), Vector2.Z());\n zip1(SubRegSize, Dst.Z(), VTMP1.Z(), VTMP2.Z());\n } else {\n umull(SubRegSize, Dst.D(), Vector1.D(), Vector2.D());\n }\n}\n\nDEF_OP(VSMull) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto SubRegSize = ConvertSubRegSize248(IROp);\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto Dst = GetVReg(Node);\n const auto Vector1 = GetVReg(Op->Vector1);\n const auto Vector2 = GetVReg(Op->Vector2);\n\n if (HostSupportsSVE256 && Is256Bit) {\n smullb(SubRegSize, VTMP1.Z(), Vector1.Z(), Vector2.Z());\n smullt(SubRegSize, VTMP2.Z(), Vector1.Z(), Vector2.Z());\n zip1(SubRegSize, Dst.Z(), VTMP1.Z(), VTMP2.Z());\n } else {\n smull(SubRegSize, Dst.D(), Vector1.D(), Vector2.D());\n }\n}\n\nDEF_OP(VUMull2) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto SubRegSize = ConvertSubRegSize248(IROp);\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto Dst = GetVReg(Node);\n const auto Vector1 = GetVReg(Op->Vector1);\n const auto Vector2 = GetVReg(Op->Vector2);\n\n if (HostSupportsSVE256 && Is256Bit) {\n umullb(SubRegSize, VTMP1.Z(), Vector1.Z(), Vector2.Z());\n umullt(SubRegSize, VTMP2.Z(), Vector1.Z(), Vector2.Z());\n zip2(SubRegSize, Dst.Z(), VTMP1.Z(), VTMP2.Z());\n } else {\n umull2(SubRegSize, Dst.Q(), Vector1.Q(), Vector2.Q());\n }\n}\n\nDEF_OP(VSMull2) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto SubRegSize = ConvertSubRegSize248(IROp);\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto Dst = GetVReg(Node);\n const auto Vector1 = GetVReg(Op->Vector1);\n const auto Vector2 = GetVReg(Op->Vector2);\n\n if (HostSupportsSVE256 && Is256Bit) {\n smullb(SubRegSize, VTMP1.Z(), Vector1.Z(), Vector2.Z());\n smullt(SubRegSize, VTMP2.Z(), Vector1.Z(), Vector2.Z());\n zip2(SubRegSize, Dst.Z(), VTMP1.Z(), VTMP2.Z());\n } else {\n smull2(SubRegSize, Dst.Q(), Vector1.Q(), Vector2.Q());\n }\n}\n\nDEF_OP(VUMulH) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto ElementSize = Op->Header.ElementSize;\n const auto SubRegSize = ConvertSubRegSize8(IROp);\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto Is128Bit = OpSize == IR::OpSize::i128Bit;\n\n const auto Dst = GetVReg(Node);\n const auto Vector1 = GetVReg(Op->Vector1);\n const auto Vector2 = GetVReg(Op->Vector2);\n\n const auto SubRegSizeLarger = ElementSize == IR::OpSize::i8Bit ? ARMEmitter::SubRegSize::i16Bit :\n ElementSize == IR::OpSize::i16Bit ? ARMEmitter::SubRegSize::i32Bit :\n ElementSize == IR::OpSize::i32Bit ? ARMEmitter::SubRegSize::i64Bit :\n ARMEmitter::SubRegSize::i8Bit;\n\n if (HostSupportsSVE256 && Is256Bit) {\n umulh(SubRegSize, Dst.Z(), Vector1.Z(), Vector2.Z());\n } else if (HostSupportsSVE128 && Is128Bit) {\n if (HostSupportsSVE256) {\n // Do predicated to ensure upper-bits get zero as expected\n const auto Mask = PRED_TMP_16B.Merging();\n\n if (Dst == Vector1) {\n umulh(SubRegSize, Dst.Z(), Mask, Dst.Z(), Vector2.Z());\n } else if (Dst == Vector2) {\n umulh(SubRegSize, Dst.Z(), Mask, Dst.Z(), Vector1.Z());\n } else {\n // Destination register doesn't overlap either source.\n // NOTE: SVE umulh (predicated) is a destructive operation.\n movprfx(Dst.Z(), Vector1.Z());\n umulh(SubRegSize, Dst.Z(), Mask, Dst.Z(), Vector2.Z());\n }\n } else {\n umulh(SubRegSize, Dst.Z(), Vector1.Z(), Vector2.Z());\n }\n } else if (OpSize == IR::OpSize::i64Bit) {\n umull(SubRegSizeLarger, Dst.D(), Vector1.D(), Vector2.D());\n shrn(SubRegSize, Dst.D(), Dst.D(), IR::OpSizeAsBits(ElementSize));\n } else {\n // ASIMD doesn't have a umulh. Need to emulate.\n umull2(SubRegSizeLarger, VTMP1.Q(), Vector1.Q(), Vector2.Q());\n umull(SubRegSizeLarger, Dst.D(), Vector1.D(), Vector2.D());\n uzp2(SubRegSize, Dst.Q(), Dst.Q(), VTMP1.Q());\n }\n}\n\nDEF_OP(VSMulH) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto ElementSize = Op->Header.ElementSize;\n const auto SubRegSize = ConvertSubRegSize8(IROp);\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto Is128Bit = OpSize == IR::OpSize::i128Bit;\n\n const auto Dst = GetVReg(Node);\n const auto Vector1 = GetVReg(Op->Vector1);\n const auto Vector2 = GetVReg(Op->Vector2);\n\n const auto SubRegSizeLarger = ElementSize == IR::OpSize::i8Bit ? ARMEmitter::SubRegSize::i16Bit :\n ElementSize == IR::OpSize::i16Bit ? ARMEmitter::SubRegSize::i32Bit :\n ElementSize == IR::OpSize::i32Bit ? ARMEmitter::SubRegSize::i64Bit :\n ARMEmitter::SubRegSize::i8Bit;\n\n if (HostSupportsSVE256 && Is256Bit) {\n smulh(SubRegSize, Dst.Z(), Vector1.Z(), Vector2.Z());\n } else if (HostSupportsSVE128 && Is128Bit) {\n if (HostSupportsSVE256) {\n // Do predicated to ensure upper-bits get zero as expected\n const auto Mask = PRED_TMP_16B.Merging();\n\n if (Dst == Vector1) {\n smulh(SubRegSize, Dst.Z(), Mask, Dst.Z(), Vector2.Z());\n } else if (Dst == Vector2) {\n smulh(SubRegSize, Dst.Z(), Mask, Dst.Z(), Vector1.Z());\n } else {\n // Destination register doesn't overlap either source.\n // NOTE: SVE umulh (predicated) is a destructive operation.\n movprfx(Dst.Z(), Vector1.Z());\n smulh(SubRegSize, Dst.Z(), Mask, Dst.Z(), Vector2.Z());\n }\n } else {\n smulh(SubRegSize, Dst.Z(), Vector1.Z(), Vector2.Z());\n }\n } else if (OpSize == IR::OpSize::i64Bit) {\n smull(SubRegSizeLarger, Dst.D(), Vector1.D(), Vector2.D());\n shrn(SubRegSize, Dst.D(), Dst.D(), IR::OpSizeAsBits(ElementSize));\n } else {\n // ASIMD doesn't have a umulh. Need to emulate.\n smull2(SubRegSizeLarger, VTMP1.Q(), Vector1.Q(), Vector2.Q());\n smull(SubRegSizeLarger, Dst.D(), Vector1.D(), Vector2.D());\n uzp2(SubRegSize, Dst.Q(), Dst.Q(), VTMP1.Q());\n }\n}\n\nDEF_OP(VUABDL) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto SubRegSize = ConvertSubRegSize248(IROp);\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto Dst = GetVReg(Node);\n const auto Vector1 = GetVReg(Op->Vector1);\n const auto Vector2 = GetVReg(Op->Vector2);\n\n if (HostSupportsSVE256 && Is256Bit) {\n // To mimic the behavior of AdvSIMD UABDL, we need to get the\n // absolute difference of the even elements (UADBLB), get the\n // absolute difference of the odd elemenets (UABDLT), then\n // interleave the results in both vectors together.\n\n uabdlb(SubRegSize, VTMP1.Z(), Vector1.Z(), Vector2.Z());\n uabdlt(SubRegSize, VTMP2.Z(), Vector1.Z(), Vector2.Z());\n zip1(SubRegSize, Dst.Z(), VTMP1.Z(), VTMP2.Z());\n } else {\n uabdl(SubRegSize, Dst.D(), Vector1.D(), Vector2.D());\n }\n}\n\nDEF_OP(VUABDL2) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto SubRegSize = ConvertSubRegSize248(IROp);\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto Dst = GetVReg(Node);\n const auto Vector1 = GetVReg(Op->Vector1);\n const auto Vector2 = GetVReg(Op->Vector2);\n\n if (HostSupportsSVE256 && Is256Bit) {\n // To mimic the behavior of AdvSIMD UABDL, we need to get the\n // absolute difference of the even elements (UADBLB), get the\n // absolute difference of the odd elemenets (UABDLT), then\n // interleave the results in both vectors together.\n\n uabdlb(SubRegSize, VTMP1.Z(), Vector1.Z(), Vector2.Z());\n uabdlt(SubRegSize, VTMP2.Z(), Vector1.Z(), Vector2.Z());\n zip2(SubRegSize, Dst.Z(), VTMP1.Z(), VTMP2.Z());\n } else {\n uabdl2(SubRegSize, Dst.Q(), Vector1.Q(), Vector2.Q());\n }\n}\n\nDEF_OP(VTBL1) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto Dst = GetVReg(Node);\n const auto VectorIndices = GetVReg(Op->VectorIndices);\n const auto VectorTable = GetVReg(Op->VectorTable);\n\n switch (OpSize) {\n case IR::OpSize::i64Bit: {\n tbl(Dst.D(), VectorTable.Q(), VectorIndices.D());\n break;\n }\n case IR::OpSize::i128Bit: {\n tbl(Dst.Q(), VectorTable.Q(), VectorIndices.Q());\n break;\n }\n case IR::OpSize::i256Bit: {\n LOGMAN_THROW_A_FMT(HostSupportsSVE256, \"Host does not support SVE. Cannot perform 256-bit table lookup\");\n\n tbl(ARMEmitter::SubRegSize::i8Bit, Dst.Z(), VectorTable.Z(), VectorIndices.Z());\n break;\n }\n default: LOGMAN_MSG_A_FMT(\"Unknown OpSize: {}\", OpSize); break;\n }\n}\n\nDEF_OP(VTBL2) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto Dst = GetVReg(Node);\n const auto VectorIndices = GetVReg(Op->VectorIndices);\n auto VectorTable1 = GetVReg(Op->VectorTable1);\n auto VectorTable2 = GetVReg(Op->VectorTable2);\n\n if (!ARMEmitter::AreVectorsSequential(VectorTable1, VectorTable2)) {\n // Vector registers aren't sequential, need to move to temporaries.\n if (OpSize == IR::OpSize::i256Bit) {\n mov(VTMP1.Z(), VectorTable1.Z());\n mov(VTMP2.Z(), VectorTable2.Z());\n } else {\n mov(VTMP1.Q(), VectorTable1.Q());\n mov(VTMP2.Q(), VectorTable2.Q());\n }\n\n static_assert(ARMEmitter::AreVectorsSequential(VTMP1, VTMP2), \"VTMP1 and VTMP2 must be sequential in order to use double-table \"\n \"TBL\");\n VectorTable1 = VTMP1;\n VectorTable2 = VTMP2;\n }\n\n switch (OpSize) {\n case IR::OpSize::i64Bit: {\n tbl(Dst.D(), VectorTable1.Q(), VectorTable2.Q(), VectorIndices.D());\n break;\n }\n case IR::OpSize::i128Bit: {\n tbl(Dst.Q(), VectorTable1.Q(), VectorTable2.Q(), VectorIndices.Q());\n break;\n }\n case IR::OpSize::i256Bit: {\n LOGMAN_THROW_A_FMT(HostSupportsSVE256, \"Host does not support SVE. Cannot perform 256-bit table lookup\");\n\n tbl(ARMEmitter::SubRegSize::i8Bit, Dst.Z(), VectorTable1.Z(), VectorTable2.Z(), VectorIndices.Z());\n break;\n }\n default: LOGMAN_MSG_A_FMT(\"Unknown OpSize: {}\", OpSize); break;\n }\n}\n\nDEF_OP(VTBX1) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto Dst = GetVReg(Node);\n const auto VectorSrcDst = GetVReg(Op->VectorSrcDst);\n const auto VectorIndices = GetVReg(Op->VectorIndices);\n const auto VectorTable = GetVReg(Op->VectorTable);\n\n if (Dst != VectorSrcDst) {\n switch (OpSize) {\n case IR::OpSize::i64Bit: {\n mov(VTMP1.D(), VectorSrcDst.D());\n tbx(VTMP1.D(), VectorTable.Q(), VectorIndices.D());\n mov(Dst.D(), VTMP1.D());\n break;\n }\n case IR::OpSize::i128Bit: {\n mov(VTMP1.Q(), VectorSrcDst.Q());\n tbx(VTMP1.Q(), VectorTable.Q(), VectorIndices.Q());\n mov(Dst.Q(), VTMP1.Q());\n break;\n }\n case IR::OpSize::i256Bit: {\n LOGMAN_THROW_A_FMT(HostSupportsSVE256, \"Host does not support SVE. Cannot perform 256-bit table lookup\");\n mov(VTMP1.Z(), VectorSrcDst.Z());\n tbx(ARMEmitter::SubRegSize::i8Bit, VTMP1.Z(), VectorTable.Z(), VectorIndices.Z());\n mov(Dst.Z(), VTMP1.Z());\n break;\n }\n default: LOGMAN_MSG_A_FMT(\"Unknown OpSize: {}\", OpSize); break;\n }\n } else {\n switch (OpSize) {\n case IR::OpSize::i64Bit: {\n tbx(VectorSrcDst.D(), VectorTable.Q(), VectorIndices.D());\n break;\n }\n case IR::OpSize::i128Bit: {\n tbx(VectorSrcDst.Q(), VectorTable.Q(), VectorIndices.Q());\n break;\n }\n case IR::OpSize::i256Bit: {\n LOGMAN_THROW_A_FMT(HostSupportsSVE256, \"Host does not support SVE. Cannot perform 256-bit table lookup\");\n\n tbx(ARMEmitter::SubRegSize::i8Bit, VectorSrcDst.Z(), VectorTable.Z(), VectorIndices.Z());\n break;\n }\n default: LOGMAN_MSG_A_FMT(\"Unknown OpSize: {}\", OpSize); break;\n }\n }\n}\n\nDEF_OP(VRev32) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto ElementSize = Op->Header.ElementSize;\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto Dst = GetVReg(Node);\n const auto Vector = GetVReg(Op->Vector);\n\n LOGMAN_THROW_A_FMT(ElementSize == IR::OpSize::i8Bit || ElementSize == IR::OpSize::i16Bit, \"Invalid size\");\n const auto SubRegSize = ElementSize == IR::OpSize::i8Bit ? ARMEmitter::SubRegSize::i8Bit : ARMEmitter::SubRegSize::i16Bit;\n\n if (HostSupportsSVE256 && Is256Bit) {\n const auto Mask = PRED_TMP_32B.Merging();\n\n switch (ElementSize) {\n case IR::OpSize::i8Bit: {\n revb(ARMEmitter::SubRegSize::i32Bit, Dst.Z(), Mask, Vector.Z());\n break;\n }\n case IR::OpSize::i16Bit: {\n revh(ARMEmitter::SubRegSize::i32Bit, Dst.Z(), Mask, Vector.Z());\n break;\n }\n default: LOGMAN_MSG_A_FMT(\"Invalid Element Size: {}\", ElementSize); break;\n }\n } else {\n if (OpSize == IR::OpSize::i64Bit) {\n rev32(SubRegSize, Dst.D(), Vector.D());\n } else {\n rev32(SubRegSize, Dst.Q(), Vector.Q());\n }\n }\n}\n\n\nDEF_OP(VRev64) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto ElementSize = Op->Header.ElementSize;\n const auto SubRegSize = ConvertSubRegSize4(IROp);\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto Dst = GetVReg(Node);\n const auto Vector = GetVReg(Op->Vector);\n\n if (HostSupportsSVE256 && Is256Bit) {\n const auto Mask = PRED_TMP_32B.Merging();\n\n switch (ElementSize) {\n case IR::OpSize::i8Bit: {\n revb(ARMEmitter::SubRegSize::i64Bit, Dst.Z(), Mask, Vector.Z());\n break;\n }\n case IR::OpSize::i16Bit: {\n revh(ARMEmitter::SubRegSize::i64Bit, Dst.Z(), Mask, Vector.Z());\n break;\n }\n case IR::OpSize::i32Bit: {\n revw(ARMEmitter::SubRegSize::i64Bit, Dst.Z(), Mask, Vector.Z());\n break;\n }\n default: LOGMAN_MSG_A_FMT(\"Invalid Element Size: {}\", ElementSize); break;\n }\n } else {\n if (OpSize == IR::OpSize::i64Bit) {\n rev64(SubRegSize, Dst.D(), Vector.D());\n } else {\n rev64(SubRegSize, Dst.Q(), Vector.Q());\n }\n }\n}\n\nDEF_OP(VFCADD) {\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto SubRegSize = ConvertSubRegSize248(IROp);\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto Dst = GetVReg(Node);\n const auto Vector1 = GetVReg(Op->Vector1);\n const auto Vector2 = GetVReg(Op->Vector2);\n\n LOGMAN_THROW_A_FMT(Op->Rotate == 90 || Op->Rotate == 270, \"Invalidate Rotate\");\n const auto Rotate = Op->Rotate == 90 ? ARMEmitter::Rotation::ROTATE_90 : ARMEmitter::Rotation::ROTATE_270;\n\n if (HostSupportsSVE256 && Is256Bit) {\n const auto Mask = PRED_TMP_32B.Merging();\n\n if (Dst == Vector1) {\n // Trivial case where we already have first vector in the destination\n // register. We can just do the operation in place.\n fcadd(SubRegSize, Dst.Z(), Mask, Vector1.Z(), Vector2.Z(), Rotate);\n } else if (Dst == Vector2) {\n // SVE FCADD is a destructive operation, so we need\n // a temporary for performing operations.\n movprfx(VTMP1.Z(), Vector1.Z());\n fcadd(SubRegSize, VTMP1.Z(), Mask, VTMP1.Z(), Vector2.Z(), Rotate);\n mov(Dst.Z(), VTMP1.Z());\n } else {\n // We have no source/dest aliasing, so we can move into the destination.\n movprfx(Dst.Z(), Vector1.Z());\n fcadd(SubRegSize, Dst.Z(), Mask, Dst.Z(), Vector2.Z(), Rotate);\n }\n } else {\n if (OpSize == IR::OpSize::i64Bit) {\n fcadd(SubRegSize, Dst.D(), Vector1.D(), Vector2.D(), Rotate);\n } else {\n fcadd(SubRegSize, Dst.Q(), Vector1.Q(), Vector2.Q(), Rotate);\n }\n }\n}\n\nDEF_OP(VFMLA) {\n ///< Dest = (Vector1 * Vector2) + Addend\n // Matches:\n // - SVE - FMLA\n // - ASIMD - FMLA\n // - Scalar - FMADD\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto SubRegSize = ConvertSubRegSize248(IROp);\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto Dst = GetVReg(Node);\n const auto Vector1 = GetVReg(Op->Vector1);\n const auto Vector2 = GetVReg(Op->Vector2);\n const auto VectorAddend = GetVReg(Op->Addend);\n\n if (HostSupportsSVE256 && Is256Bit) {\n const auto Mask = PRED_TMP_32B.Merging();\n ARMEmitter::VRegister DestTmp = Dst;\n if (Dst != VectorAddend) {\n if (Dst != Vector1 && Dst != Vector2) {\n DestTmp = Dst;\n } else {\n DestTmp = VTMP1;\n }\n mov(DestTmp.Z(), VectorAddend.Z());\n }\n\n fmla(SubRegSize, DestTmp.Z(), Mask, Vector1.Z(), Vector2.Z());\n if (Dst != DestTmp) {\n mov(Dst.Z(), DestTmp.Z());\n }\n } else {\n if (IROp->ElementSize == OpSize) {\n if (IROp->ElementSize == IR::OpSize::i16Bit) {\n fmadd(Dst.H(), Vector1.H(), Vector2.H(), VectorAddend.H());\n } else if (IROp->ElementSize == IR::OpSize::i32Bit) {\n fmadd(Dst.S(), Vector1.S(), Vector2.S(), VectorAddend.S());\n } else if (IROp->ElementSize == IR::OpSize::i64Bit) {\n fmadd(Dst.D(), Vector1.D(), Vector2.D(), VectorAddend.D());\n }\n return;\n }\n ARMEmitter::VRegister DestTmp = Dst;\n if (Dst != VectorAddend) {\n if (Dst != Vector1 && Dst != Vector2) {\n DestTmp = Dst;\n } else {\n DestTmp = VTMP1;\n }\n mov(DestTmp.Q(), VectorAddend.Q());\n }\n if (OpSize == IR::OpSize::i128Bit) {\n fmla(SubRegSize, DestTmp.Q(), Vector1.Q(), Vector2.Q());\n } else {\n fmla(SubRegSize, DestTmp.D(), Vector1.D(), Vector2.D());\n }\n\n if (Dst != DestTmp) {\n mov(Dst.Q(), DestTmp.Q());\n }\n }\n}\n\nDEF_OP(VFMLS) {\n ///< Dest = (Vector1 * Vector2) - Addend\n // Matches:\n // - SVE - FNMLS\n // - ASIMD - FMLA (With negated addend)\n // - Scalar - FNMSUB\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto SubRegSize = ConvertSubRegSize248(IROp);\n const auto Is128Bit = OpSize == IR::OpSize::i128Bit;\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto Dst = GetVReg(Node);\n const auto Vector1 = GetVReg(Op->Vector1);\n const auto Vector2 = GetVReg(Op->Vector2);\n const auto VectorAddend = GetVReg(Op->Addend);\n\n if (HostSupportsSVE256 && Is256Bit) {\n const auto Mask = PRED_TMP_32B.Merging();\n ARMEmitter::VRegister DestTmp = Dst;\n if (Dst != VectorAddend) {\n if (Dst != Vector1 && Dst != Vector2) {\n DestTmp = Dst;\n } else {\n DestTmp = VTMP1;\n }\n mov(DestTmp.Z(), VectorAddend.Z());\n }\n\n fnmls(SubRegSize, DestTmp.Z(), Mask, Vector1.Z(), Vector2.Z());\n if (Dst != DestTmp) {\n mov(Dst.Z(), DestTmp.Z());\n }\n } else if (HostSupportsSVE128 && Is128Bit) {\n const auto Mask = PRED_TMP_16B.Merging();\n ARMEmitter::VRegister DestTmp = Dst;\n if (Dst != VectorAddend) {\n if (Dst != Vector1 && Dst != Vector2) {\n DestTmp = Dst;\n } else {\n DestTmp = VTMP1;\n }\n mov(DestTmp.Z(), VectorAddend.Z());\n }\n\n fnmls(SubRegSize, DestTmp.Z(), Mask, Vector1.Z(), Vector2.Z());\n if (Dst != DestTmp) {\n mov(Dst.Z(), DestTmp.Z());\n }\n } else {\n if (IROp->ElementSize == OpSize) {\n if (IROp->ElementSize == IR::OpSize::i16Bit) {\n fnmsub(Dst.H(), Vector1.H(), Vector2.H(), VectorAddend.H());\n } else if (IROp->ElementSize == IR::OpSize::i32Bit) {\n fnmsub(Dst.S(), Vector1.S(), Vector2.S(), VectorAddend.S());\n } else if (IROp->ElementSize == IR::OpSize::i64Bit) {\n fnmsub(Dst.D(), Vector1.D(), Vector2.D(), VectorAddend.D());\n }\n return;\n }\n\n // Addend needs to get negated to match correct behaviour here.\n ARMEmitter::VRegister DestTmp = Dst;\n if (Dst == Vector1 || Dst == Vector2) {\n DestTmp = VTMP1;\n }\n\n if (Is128Bit) {\n fneg(SubRegSize, DestTmp.Q(), VectorAddend.Q());\n } else {\n fneg(SubRegSize, DestTmp.D(), VectorAddend.D());\n }\n\n if (Is128Bit) {\n fmla(SubRegSize, DestTmp.Q(), Vector1.Q(), Vector2.Q());\n } else {\n fmla(SubRegSize, DestTmp.D(), Vector1.D(), Vector2.D());\n }\n\n if (DestTmp != Dst) {\n if (Is128Bit) {\n mov(Dst.Q(), DestTmp.Q());\n } else {\n mov(Dst.D(), DestTmp.D());\n }\n }\n }\n}\n\nDEF_OP(VFNMLA) {\n ///< Dest = (-Vector1 * Vector2) + Addend\n // Matches:\n // - SVE - FMLS\n // - ASIMD - FMLS\n // - Scalar - FMSUB\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto SubRegSize = ConvertSubRegSize248(IROp);\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto Dst = GetVReg(Node);\n const auto Vector1 = GetVReg(Op->Vector1);\n const auto Vector2 = GetVReg(Op->Vector2);\n const auto VectorAddend = GetVReg(Op->Addend);\n\n if (HostSupportsSVE256 && Is256Bit) {\n const auto Mask = PRED_TMP_32B.Merging();\n ARMEmitter::VRegister DestTmp = Dst;\n if (Dst != VectorAddend) {\n if (Dst != Vector1 && Dst != Vector2) {\n DestTmp = Dst;\n } else {\n DestTmp = VTMP1;\n }\n mov(DestTmp.Z(), VectorAddend.Z());\n }\n\n fmls(SubRegSize, DestTmp.Z(), Mask, Vector1.Z(), Vector2.Z());\n if (Dst != DestTmp) {\n mov(Dst.Z(), DestTmp.Z());\n }\n } else {\n if (IROp->ElementSize == OpSize) {\n if (IROp->ElementSize == IR::OpSize::i16Bit) {\n fmsub(Dst.H(), Vector1.H(), Vector2.H(), VectorAddend.H());\n } else if (IROp->ElementSize == IR::OpSize::i32Bit) {\n fmsub(Dst.S(), Vector1.S(), Vector2.S(), VectorAddend.S());\n } else if (IROp->ElementSize == IR::OpSize::i64Bit) {\n fmsub(Dst.D(), Vector1.D(), Vector2.D(), VectorAddend.D());\n }\n return;\n }\n\n ARMEmitter::VRegister DestTmp = Dst;\n if (Dst != VectorAddend) {\n if (Dst != Vector1 && Dst != Vector2) {\n DestTmp = Dst;\n } else {\n DestTmp = VTMP1;\n }\n mov(DestTmp.Q(), VectorAddend.Q());\n }\n if (OpSize == IR::OpSize::i128Bit) {\n fmls(SubRegSize, DestTmp.Q(), Vector1.Q(), Vector2.Q());\n } else {\n fmls(SubRegSize, DestTmp.D(), Vector1.D(), Vector2.D());\n }\n\n if (Dst != DestTmp) {\n mov(Dst.Q(), DestTmp.Q());\n }\n }\n}\n\nDEF_OP(VFNMLS) {\n ///< Dest = (-Vector1 * Vector2) - Addend\n // Matches:\n // - SVE - FNMLA\n // - ASIMD - FMLS (With Negated addend)\n // - Scalar - FNMADD\n\n const auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n\n const auto SubRegSize = ConvertSubRegSize248(IROp);\n const auto Is256Bit = OpSize == IR::OpSize::i256Bit;\n LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, \"Need SVE256 support in order to use {} with 256-bit operation\", __func__);\n\n const auto Is128Bit = OpSize == IR::OpSize::i128Bit;\n\n const auto Dst = GetVReg(Node);\n const auto Vector1 = GetVReg(Op->Vector1);\n const auto Vector2 = GetVReg(Op->Vector2);\n const auto VectorAddend = GetVReg(Op->Addend);\n\n if (HostSupportsSVE256 && Is256Bit) {\n const auto Mask = PRED_TMP_32B.Merging();\n ARMEmitter::VRegister DestTmp = Dst;\n if (Dst != VectorAddend) {\n if (Dst != Vector1 && Dst != Vector2) {\n DestTmp = Dst;\n } else {\n DestTmp = VTMP1;\n }\n mov(DestTmp.Z(), VectorAddend.Z());\n }\n\n fnmla(SubRegSize, DestTmp.Z(), Mask, Vector1.Z(), Vector2.Z());\n if (Dst != DestTmp) {\n mov(Dst.Z(), DestTmp.Z());\n }\n } else if (HostSupportsSVE128 && Is128Bit) {\n const auto Mask = PRED_TMP_16B.Merging();\n ARMEmitter::VRegister DestTmp = Dst;\n if (Dst != VectorAddend) {\n if (Dst != Vector1 && Dst != Vector2) {\n DestTmp = Dst;\n } else {\n DestTmp = VTMP1;\n }\n mov(DestTmp.Z(), VectorAddend.Z());\n }\n\n fnmla(SubRegSize, DestTmp.Z(), Mask, Vector1.Z(), Vector2.Z());\n if (Dst != DestTmp) {\n mov(Dst.Z(), DestTmp.Z());\n }\n } else {\n if (IROp->ElementSize == OpSize) {\n if (IROp->ElementSize == IR::OpSize::i16Bit) {\n fnmadd(Dst.H(), Vector1.H(), Vector2.H(), VectorAddend.H());\n } else if (IROp->ElementSize == IR::OpSize::i32Bit) {\n fnmadd(Dst.S(), Vector1.S(), Vector2.S(), VectorAddend.S());\n } else if (IROp->ElementSize == IR::OpSize::i64Bit) {\n fnmadd(Dst.D(), Vector1.D(), Vector2.D(), VectorAddend.D());\n }\n return;\n }\n\n // Addend needs to get negated to match correct behaviour here.\n ARMEmitter::VRegister DestTmp = Dst;\n if (Dst == Vector1 || Dst == Vector2) {\n DestTmp = VTMP1;\n }\n\n if (Is128Bit) {\n fneg(SubRegSize, DestTmp.Q(), VectorAddend.Q());\n } else {\n fneg(SubRegSize, DestTmp.D(), VectorAddend.D());\n }\n\n if (Is128Bit) {\n fmls(SubRegSize, DestTmp.Q(), Vector1.Q(), Vector2.Q());\n } else {\n fmls(SubRegSize, DestTmp.D(), Vector1.D(), Vector2.D());\n }\n\n if (DestTmp != Dst) {\n if (Is128Bit) {\n mov(Dst.Q(), DestTmp.Q());\n } else {\n mov(Dst.D(), DestTmp.D());\n }\n }\n }\n}\n\nDEF_OP(VFCopySign) {\n auto Op = IROp->C();\n const auto OpSize = IROp->Size;\n const auto SubRegSize = ConvertSubRegSize248(IROp);\n\n ARMEmitter::VRegister Magnitude = GetVReg(Op->Vector1);\n ARMEmitter::VRegister Sign = GetVReg(Op->Vector2);\n\n // We don't assign explicity to Dst but Dst and Magniture are tied to the same register.\n // Similar in semantics to C's copysignf.\n switch (OpSize) {\n case IR::OpSize::i64Bit:\n movi(SubRegSize, VTMP1.D(), 0x80, 24);\n bit(Magnitude.D(), Sign.D(), VTMP1.D());\n break;\n case IR::OpSize::i128Bit:\n movi(SubRegSize, VTMP1.Q(), 0x80, 24);\n bit(Magnitude.Q(), Sign.Q(), VTMP1.Q());\n break;\n default: LOGMAN_MSG_A_FMT(\"Unsupported element size for operation {}\", __func__); FEX_UNREACHABLE;\n }\n}\n\nDEF_OP(F64SIN) {\n const auto Op = IROp->C();\n const auto Src = GetVReg(Op->Src);\n const auto Dst = GetVReg(Node);\n\n fmov(VTMP1.D(), Src.D());\n ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.F64SinHandler));\n str(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, -16);\n blr(TMP1);\n ldr(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, 16);\n fmov(Dst.D(), VTMP1.D());\n}\n\nDEF_OP(F64COS) {\n const auto Op = IROp->C();\n const auto Src = GetVReg(Op->Src);\n const auto Dst = GetVReg(Node);\n\n fmov(VTMP1.D(), Src.D());\n ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.F64CosHandler));\n str(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, -16);\n blr(TMP1);\n ldr(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, 16);\n fmov(Dst.D(), VTMP1.D());\n}\n\nDEF_OP(F64TAN) {\n const auto Op = IROp->C();\n const auto Src = GetVReg(Op->Src);\n const auto Dst = GetVReg(Node);\n\n fmov(VTMP1.D(), Src.D());\n ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.F64TanHandler));\n str(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, -16);\n blr(TMP1);\n ldr(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, 16);\n fmov(Dst.D(), VTMP1.D());\n}\n\n\n} // namespace FEXCore::CPU\n" }, { "path": "FEXCore/Source/Interface/Core/LookupCache.cpp", "content": "// SPDX-License-Identifier: MIT\n/*\n$info$\ntags: glue|block-database\ndesc: Stores information about blocks, and provides C++ implementations to lookup the blocks\n$end_info$\n*/\n\n#include \n#include \n#include \n\n#include \"Interface/Context/Context.h\"\n#include \"Interface/Core/LookupCache.h\"\n\nnamespace FEXCore {\nGuestToHostMap::GuestToHostMap()\n : BlockLinks_mbr {\"FEXMem_BlockLinks\"} {\n BlockLinks_pma = fextl::make_unique>(&BlockLinks_mbr);\n // Setup our PMR map.\n BlockLinks = BlockLinks_pma->new_object();\n}\n\nLookupCache::LookupCache(FEXCore::Context::ContextImpl* CTX)\n : ctx {CTX} {\n\n TotalCacheSize = ctx->Config.VirtualMemSize / FEXCore::Utils::FEX_PAGE_SIZE * 8 + CODE_SIZE + MAX_L1_SIZE;\n\n // Block cache ends up looking like this\n // PageMemoryMap[VirtualMemoryRegion >> 12]\n // |\n // v\n // PageMemory[Memory & (VIRTUAL_PAGE_SIZE - 1)]\n // |\n // v\n // Pointer to Code\n //\n // Allocate a region of memory that we can use to back our block pointers\n // We need one pointer per page of virtual memory\n // At 64GB of virtual memory this will allocate 128MB of virtual memory space\n PagePointer = reinterpret_cast(FEXCore::Allocator::VirtualAlloc(TotalCacheSize, false, false));\n LOGMAN_THROW_A_FMT(PagePointer != -1ULL, \"Failed to allocate PagePointer\");\n\n // Disable THP on the Lookup cache.\n FEXCore::Allocator::VirtualTHPControl(reinterpret_cast(PagePointer), TotalCacheSize, FEXCore::Allocator::THPControl::Disable);\n\n FEXCore::Allocator::VirtualName(\"FEXMem_Lookup\", reinterpret_cast(PagePointer),\n ctx->Config.VirtualMemSize / FEXCore::Utils::FEX_PAGE_SIZE * 8 + CODE_SIZE);\n CTX->SyscallHandler->MarkOvercommitRange(PagePointer, TotalCacheSize);\n\n // Allocate our memory backing our pages\n // We need 32KB per guest page (One pointer per byte)\n // XXX: We can drop down to 16KB if we store 4byte offsets from the code base\n // We currently limit to 128MB of real memory for caching for the total cache size.\n // Can end up being inefficient if we compile a small number of blocks per page\n PageMemory = PagePointer + ctx->Config.VirtualMemSize / FEXCore::Utils::FEX_PAGE_SIZE * 8;\n\n // L1 Cache\n L1Pointer = PageMemory + CODE_SIZE;\n FEXCore::Allocator::VirtualName(\"FEXMem_Lookup_L1\", reinterpret_cast(L1Pointer), MAX_L1_SIZE);\n\n VirtualMemSize = ctx->Config.VirtualMemSize;\n\n if (DynamicL1Cache()) {\n // Start at minimum size when dynamic.\n L1PointerMask = MIN_L1_ENTRIES - 1;\n } else {\n // Start at maximum instead.\n L1PointerMask = MAX_L1_ENTRIES - 1;\n }\n}\n\nLookupCache::~LookupCache() {\n FEXCore::Allocator::VirtualFree(reinterpret_cast(PagePointer), TotalCacheSize);\n ctx->SyscallHandler->UnmarkOvercommitRange(PagePointer, TotalCacheSize);\n\n // No need to free BlockLinks map.\n // These will get freed when their memory allocators are deallocated.\n}\n\nvoid LookupCache::ClearL2Cache(const FEXCore::LookupCacheBaseLockToken& lk) {\n // Clear out the page memory\n // PagePointer and PageMemory are sequential with each other. Clear both at once.\n FEXCore::Allocator::VirtualDontNeed(reinterpret_cast(PagePointer),\n ctx->Config.VirtualMemSize / FEXCore::Utils::FEX_PAGE_SIZE * 8 + CODE_SIZE, false);\n AllocateOffset = 0;\n}\n\nvoid LookupCache::ClearThreadLocalCaches(const LookupCacheWriteLockToken&) {\n // Clear L1 and L2 by clearing the full cache.\n FEXCore::Allocator::VirtualDontNeed(reinterpret_cast(PagePointer), TotalCacheSize, false);\n CachedCodePages.clear();\n}\n\nvoid LookupCache::ClearCache(const LookupCacheWriteLockToken& lk) {\n // Clear L1 and L2 by clearing the full cache.\n ClearThreadLocalCaches(lk);\n Shared->ClearCache(lk);\n}\n\nvoid GuestToHostMap::ClearCache(const LookupCacheWriteLockToken&) {\n // Allocate a new pointer from the BlockLinks pma again.\n BlockLinks = BlockLinks_pma->new_object();\n // All code is gone, clear the block list\n BlockList.clear();\n}\n\n} // namespace FEXCore\n" }, { "path": "FEXCore/Source/Interface/Core/LookupCache.h", "content": "// SPDX-License-Identifier: MIT\n#pragma once\n#include \"Interface/Context/Context.h\"\n#include \n#include \n#include \"Utils/WritePriorityMutex.h\"\n\n#include \n#include \n#include \n#include \n#include \n#include \n\n#include \n#include \n#include \n#include \n\nnamespace FEXCore {\nstruct LookupCacheBaseLockToken {\nprotected:\n // Protected constructor - only derived classes can construct\n LookupCacheBaseLockToken() = default;\n};\n\nstruct LookupCacheWriteLockToken : public LookupCacheBaseLockToken {\nprivate:\n // Only constructible by GuestToHostMap\n friend struct GuestToHostMap;\n LookupCacheWriteLockToken(FEXCore::Utils::WritePriorityMutex::Mutex& Mutex)\n : Lock {Mutex} {}\n std::lock_guard Lock;\n};\n\nstruct LookupCacheReadLockToken : public LookupCacheBaseLockToken {\nprivate:\n // Only constructible by GuestToHostMap\n friend struct GuestToHostMap;\n LookupCacheReadLockToken(FEXCore::Utils::WritePriorityMutex::Mutex& Mutex)\n : Lock {Mutex} {}\n std::shared_lock Lock;\n};\n\nstruct GuestToHostMap {\n FEXCore::Utils::WritePriorityMutex::Mutex Lock {};\n\n [[nodiscard]]\n LookupCacheWriteLockToken AcquireWriteLock() {\n return LookupCacheWriteLockToken {Lock};\n }\n\n [[nodiscard]]\n LookupCacheReadLockToken AcquireReadLock() {\n return LookupCacheReadLockToken {Lock};\n }\n\n struct BlockLinkTag {\n uint64_t GuestDestination;\n FEXCore::Context::ExitFunctionLinkData* HostLink;\n\n bool operator<(const BlockLinkTag& other) const {\n if (GuestDestination < other.GuestDestination) {\n return true;\n } else if (GuestDestination == other.GuestDestination) {\n return HostLink < other.HostLink;\n } else {\n return false;\n }\n }\n };\n\n // Use a monotonic buffer resource to allocate both the std::pmr::map and its members.\n // This allows us to quickly clear the block link map by clearing the monotonic allocator.\n // If we had allocated the block link map without the MBR, then clearing the map would require slowly\n // walking each block member and destructing objects.\n //\n // This makes `BlockLinks` look like a raw pointer that could memory leak, but since it is backed by the MBR, it won't.\n fextl::pmr::named_monotonic_page_buffer_resource BlockLinks_mbr;\n using BlockLinksMapType = std::pmr::map;\n fextl::unique_ptr> BlockLinks_pma;\n BlockLinksMapType* BlockLinks;\n\n struct BlockEntry {\n uint64_t HostCode;\n fextl::vector CodePages;\n };\n\n fextl::robin_map BlockList;\n\n fextl::map> CodePages;\n\n GuestToHostMap();\n\n // Adds to Guest -> Host code mapping\n const BlockEntry& AddBlockMapping(uint64_t Address, const fextl::vector& CodePages, void* HostCode, const LookupCacheWriteLockToken&) {\n // This may replace an existing mapping\n // NOTE: Generally no previous entry should exist, however there is one exception:\n // If the backend updates the active thread's CodeBuffer, the new associated LookupCache\n // may already contain the block address. Since is comparatively rare, we'll just leak\n // one of the two blocks in this case.\n return BlockList.insert_or_assign(Address, BlockEntry {(uintptr_t)HostCode, CodePages}).first->second;\n }\n\n const BlockEntry* FindBlock(uint64_t Address, const LookupCacheReadLockToken&) {\n auto HostCode = BlockList.find(Address);\n if (HostCode == BlockList.end()) {\n return nullptr;\n }\n return &HostCode->second;\n }\n\n bool Erase(uint64_t Address, const LookupCacheWriteLockToken&) {\n // Sever any links to this block\n auto lower = BlockLinks->lower_bound({Address, nullptr});\n auto upper = BlockLinks->upper_bound({Address, reinterpret_cast(UINTPTR_MAX)});\n for (auto it = lower; it != upper; it = BlockLinks->erase(it)) {\n it->second(it->first.HostLink);\n }\n\n // Remove from BlockList\n return BlockList.erase(Address) != 0;\n }\n\n void InvalidateRange(uint64_t Start, uint64_t Length) {\n auto lk = AcquireWriteLock();\n\n auto lower = CodePages.lower_bound(Start >> 12);\n auto upper = CodePages.upper_bound((Start + Length - 1) >> 12);\n\n for (auto it = lower; it != upper; it++) {\n for (const auto& Entry : it->second) {\n Erase(Entry, lk);\n }\n }\n CodePages.erase(lower, upper);\n }\n\n void AddBlockLink(uint64_t GuestDestination, FEXCore::Context::ExitFunctionLinkData* HostLink,\n const FEXCore::Context::BlockDelinkerFunc& delinker, const LookupCacheWriteLockToken&) {\n BlockLinks->insert({{GuestDestination, HostLink}, delinker});\n }\n\n bool AddBlockExecutableRange(const std::ranges::input_range auto& Addresses, uint64_t Start, uint64_t Length, const LookupCacheWriteLockToken&) {\n bool rv = false;\n\n for (auto CurrentPage = Start >> 12, EndPage = (Start + Length - 1) >> 12; CurrentPage <= EndPage; CurrentPage++) {\n auto& CodePage = CodePages[CurrentPage];\n rv |= CodePage.empty();\n CodePage.insert(CodePage.end(), Addresses.begin(), Addresses.end());\n }\n\n return rv;\n }\n\n void ClearCache(const LookupCacheWriteLockToken&);\n};\n\nclass LookupCache {\npublic:\n struct LookupCacheEntry {\n uintptr_t HostCode;\n uintptr_t GuestCode;\n };\n\n LookupCache(FEXCore::Context::ContextImpl* CTX);\n ~LookupCache();\n\n // Swaps out the underlying GuestToHostMap and clears all associated caches.\n // This interface requires the previous CodeBuffer to be provided despite not using it. This ensures the shared write lock is still valid.\n void ChangeGuestToHostMapping([[maybe_unused]] CPU::CodeBuffer& Prev, GuestToHostMap& NewMap, const LookupCacheWriteLockToken& lk) {\n ClearThreadLocalCaches(lk);\n Shared = &NewMap;\n }\n\n uintptr_t FindBlock(FEXCore::Core::InternalThreadState* Thread, uint64_t Address) {\n // Try L1, no lock needed\n auto& L1Entry = reinterpret_cast(L1Pointer)[Address & L1PointerMask];\n if (L1Entry.GuestCode == Address) {\n return L1Entry.HostCode;\n }\n\n // L2 and L3 need to be locked\n uintptr_t HostPtr {};\n {\n std::optional> LockTime(\n Thread->ThreadStats ? &Thread->ThreadStats->AccumulatedCacheReadLockTime : nullptr);\n auto lk = Shared->AcquireReadLock();\n LockTime.reset();\n\n if (!DisableL2Cache()) {\n // Try L2\n const auto PageIndex = (Address & (VirtualMemSize - 1)) >> 12;\n const auto PageOffset = Address & (0x0FFF);\n\n const auto Pointers = reinterpret_cast(PagePointer);\n auto LocalPagePointer = Pointers[PageIndex];\n\n // Do we a page pointer for this address?\n if (LocalPagePointer) {\n // Find there pointer for the address in the blocks\n auto BlockPointers = reinterpret_cast(LocalPagePointer);\n\n if (BlockPointers[PageOffset].GuestCode == Address) {\n L1Entry.GuestCode = Address;\n L1Entry.HostCode = BlockPointers[PageOffset].HostCode;\n HostPtr = L1Entry.HostCode;\n }\n }\n }\n\n if (!HostPtr) {\n // Try L3\n auto Entry = Shared->FindBlock(Address, lk);\n if (Entry) {\n CacheBlockMapping(Address, *Entry, false, lk);\n HostPtr = Entry->HostCode;\n }\n }\n }\n\n if (HostPtr && DynamicL1Cache()) {\n UpdateDynamicL1Stats(Thread);\n }\n\n FEXCORE_PROFILE_INSTANT_INCREMENT(Thread, AccumulatedCacheMissCount, 1);\n\n return HostPtr;\n }\n\n void UpdateDynamicL1Stats(FEXCore::Core::InternalThreadState* Thread) {\n // If host pointer was found in L2 or L3, then add it to the counter.\n // Keeping track not L1 misses, but specifically L2/L3 hits.\n ++L2L3CacheHits;\n\n const auto CurrentTime = std::chrono::system_clock::now();\n const auto Period = CurrentTime - LastPeriod;\n if (Period >= SamplePeriod) {\n // If larger than the sample period then check if we need to increase L1 cache size.\n const double AveragePerSecond = static_cast(L2L3CacheHits) /\n static_cast(std::chrono::duration_cast(Period).count()) * 1000.0;\n\n if (AveragePerSecond >= DynamicL1CacheIncreaseCountHeuristic()) {\n if (CurrentL1Entries < MAX_L1_ENTRIES) {\n CurrentL1Entries <<= 1;\n L1PointerMask = CurrentL1Entries - 1;\n\n // Update the thread's L1 pointer mask to increase how much cache it uses.\n // Since we're in C-code, this is safe to update here.\n Thread->CurrentFrame->State.L1Mask = GetScaledL1PointerMask();\n }\n } else if (AveragePerSecond < DynamicL1CacheDecreaseCountHeuristic()) {\n if (CurrentL1Entries > MIN_L1_ENTRIES) {\n CurrentL1Entries >>= 1;\n L1PointerMask = CurrentL1Entries - 1;\n\n // Madvise the entries that we are dropping. Gives the memory back to the OS.\n LookupCacheEntry* FirstZeroL1Entry = &reinterpret_cast(L1Pointer)[CurrentL1Entries];\n size_t ZeroMemorySize = (MAX_L1_ENTRIES - CurrentL1Entries) * sizeof(LookupCacheEntry);\n FEXCore::Allocator::VirtualDontNeed(FirstZeroL1Entry, ZeroMemorySize, false);\n\n // Update the thread's L1 pointer mask to increase how much cache it uses.\n // Since we're in C-code, this is safe to update here.\n Thread->CurrentFrame->State.L1Mask = GetScaledL1PointerMask();\n }\n }\n\n // Update Last period to start again.\n LastPeriod = CurrentTime;\n L2L3CacheHits = 0;\n }\n }\n\n GuestToHostMap* Shared = nullptr;\n\n // Appends a list of Block {Address} to CodePages [Start, Start + Length)\n // Returns true if new pages are marked as containing code\n bool AddBlockExecutableRange(FEXCore::Core::InternalThreadState* Thread, const fextl::set& Addresses, uint64_t Start, uint64_t Length) {\n std::optional> LockTime(\n Thread->ThreadStats ? &Thread->ThreadStats->AccumulatedCacheWriteLockTime : nullptr);\n auto lk = Shared->AcquireWriteLock();\n LockTime.reset();\n\n return Shared->AddBlockExecutableRange(Addresses, Start, Length, lk);\n }\n\n // Adds to Guest -> Host code mapping\n void AddBlockMapping(FEXCore::Core::InternalThreadState* Thread, uint64_t Address, const fextl::vector& CodePages, void* HostCode) {\n std::optional> LockTime(\n Thread->ThreadStats ? &Thread->ThreadStats->AccumulatedCacheWriteLockTime : nullptr);\n auto lk = Shared->AcquireWriteLock();\n LockTime.reset();\n\n const auto& Entry = Shared->AddBlockMapping(Address, CodePages, HostCode, lk);\n\n // There is no need to update L1 or L2, they will get updated on first lookup\n // However, adding to L1 here increases performance\n CacheBlockMapping(Address, Entry, true, lk);\n }\n\n // Invalidates L1/L2 for a given guest block\n void InvalidateCache(uint64_t Address, const LookupCacheWriteLockToken& lk) {\n // Do L1\n auto& L1Entry = reinterpret_cast(L1Pointer)[Address & L1PointerMask];\n if (L1Entry.GuestCode == Address) {\n L1Entry.GuestCode = 0;\n // Leave L1Entry.HostCode as is, so that concurrent lookups won't read a null pointer\n // This is a soft guarantee for cross thread invalidation, as atomics are not used\n // and it hasn't been thoroughly tested\n }\n\n if (!DisableL2Cache()) {\n // Do full map\n Address = Address & (VirtualMemSize - 1);\n uint64_t PageOffset = Address & (0x0FFF);\n Address >>= 12;\n\n uintptr_t* Pointers = reinterpret_cast(PagePointer);\n uint64_t LocalPagePointer = Pointers[Address];\n if (!LocalPagePointer) {\n // Page for this code didn't even exist, nothing to do\n return;\n }\n\n // Page exists, just set the offset to zero\n auto BlockPointers = reinterpret_cast(LocalPagePointer);\n BlockPointers[PageOffset].GuestCode = 0;\n BlockPointers[PageOffset].HostCode = 0;\n }\n }\n\n // Invalidates all L1/L2 entries for all guest block that intersect the given range\n bool InvalidateCacheRange(uint64_t Start, uint64_t Length) {\n auto lk = Shared->AcquireWriteLock();\n\n auto lower = CachedCodePages.lower_bound(Start >> 12);\n auto upper = CachedCodePages.upper_bound((Start + Length - 1) >> 12);\n\n for (auto it = lower; it != upper; it++) {\n for (const auto& Entry : it->second) {\n InvalidateCache(Entry, lk);\n }\n }\n bool ret = upper != lower;\n CachedCodePages.erase(lower, upper);\n return ret;\n }\n\n void AddBlockLink(uint64_t GuestDestination, FEXCore::Context::ExitFunctionLinkData* HostLink,\n const FEXCore::Context::BlockDelinkerFunc& delinker, const LookupCacheWriteLockToken& lk) {\n Shared->AddBlockLink(GuestDestination, HostLink, delinker, lk);\n }\n\n void ClearCache(const LookupCacheWriteLockToken&);\n void ClearL2Cache(const LookupCacheBaseLockToken&);\n void ClearThreadLocalCaches(const LookupCacheWriteLockToken&);\n\n uintptr_t GetL1Pointer() const {\n return L1Pointer;\n }\n uintptr_t GetScaledL1PointerMask() const {\n return L1PointerMask << FEXCore::ilog2(sizeof(LookupCache::LookupCacheEntry));\n }\n uintptr_t GetPagePointer() const {\n return PagePointer;\n }\n uintptr_t GetVirtualMemorySize() const {\n return VirtualMemSize;\n }\n\n // This needs to be taken before reads or writes to L2, L3, CodePages,\n // and before writes to L1. Concurrent access from a thread that this LookupCache doesn't belong to\n // may only happen during cross thread invalidation (::Erase).\n // All other operations must be done from the owning thread.\n // Some care is taken so that L1 lookups can be done without locks, and even tearing is unlikely to lead to a crash.\n // This approach has not been fully vetted yet.\n // Also note that L1 lookups might be inlined in the JIT Dispatcher and/or block ends.\n auto AcquireWriteLock() {\n return Shared->AcquireWriteLock();\n }\n\nprivate:\n void CacheBlockMapping(uint64_t Address, const GuestToHostMap::BlockEntry& Entry, bool L1Only, const LookupCacheBaseLockToken& lk) {\n for (const auto& CodePage : Entry.CodePages) {\n CachedCodePages[CodePage >> 12].insert(Address);\n }\n\n // Do L1\n auto& L1Entry = reinterpret_cast(L1Pointer)[Address & L1PointerMask];\n L1Entry.GuestCode = Address;\n L1Entry.HostCode = Entry.HostCode;\n\n if (!DisableL2Cache() && !L1Only) {\n // Do ful map\n auto FullAddress = Address;\n Address = Address & (VirtualMemSize - 1);\n\n uint64_t PageOffset = Address & (0x0FFF);\n Address >>= 12;\n\n uintptr_t* Pointers = reinterpret_cast(PagePointer);\n uint64_t LocalPagePointer = Pointers[Address];\n if (!LocalPagePointer) {\n // We don't have a page pointer for this address\n // Allocate one now if we can\n uintptr_t NewPageBacking = AllocateBackingForPage();\n if (!NewPageBacking) {\n // Couldn't allocate, clear L2 and retry\n ClearL2Cache(lk);\n CacheBlockMapping(FullAddress, Entry, false, lk);\n return;\n }\n Pointers[Address] = NewPageBacking;\n LocalPagePointer = NewPageBacking;\n }\n\n // Add the new pointer to the page block\n auto BlockPointers = reinterpret_cast(LocalPagePointer);\n\n // This silently replaces existing mappings\n BlockPointers[PageOffset].GuestCode = FullAddress;\n BlockPointers[PageOffset].HostCode = Entry.HostCode;\n }\n }\n\n uintptr_t AllocateBackingForPage() {\n uintptr_t NewBase = AllocateOffset;\n uintptr_t NewEnd = AllocateOffset + SIZE_PER_PAGE;\n\n if (NewEnd >= CODE_SIZE) {\n // We ran out of block backing space. Need to clear the block cache and tell the JIT cores to clear their caches as well\n // Tell whatever is calling this that it needs to do it.\n return 0;\n }\n\n AllocateOffset = NewEnd;\n return PageMemory + NewBase;\n }\n\n // Maps from a page index to all blocks in the page that have at some point been fetched into L1/L2\n fextl::map> CachedCodePages;\n\n uintptr_t PagePointer;\n uintptr_t PageMemory;\n uintptr_t L1Pointer;\n uintptr_t L1PointerMask;\n\n size_t TotalCacheSize;\n\n // Start with 8k entries in L1 to give 128KB of L1 cache to each thread.\n // Max out at 1 million entries to give each thread 16MB of L1 cache maximum.\n constexpr static size_t MIN_L1_ENTRIES = 8 * 1024; // Must be a power of 2\n constexpr static size_t MAX_L1_ENTRIES = 1 * 1024 * 1024; // Must be a power of 2\n\n constexpr static size_t CODE_SIZE = 128 * 1024 * 1024;\n constexpr static size_t SIZE_PER_PAGE = FEXCore::Utils::FEX_PAGE_SIZE * sizeof(LookupCacheEntry);\n constexpr static size_t MAX_L1_SIZE = MAX_L1_ENTRIES * sizeof(LookupCacheEntry);\n\n size_t AllocateOffset {};\n\n FEXCore::Context::ContextImpl* ctx;\n uint64_t VirtualMemSize {};\n\n size_t CurrentL1Entries = MIN_L1_ENTRIES;\n uint64_t L2L3CacheHits {};\n std::chrono::time_point LastPeriod {};\n constexpr static std::chrono::seconds SamplePeriod {1};\n FEX_CONFIG_OPT(DynamicL1CacheIncreaseCountHeuristic, DYNAMICL1CACHEINCREASECOUNTHEURISTIC);\n FEX_CONFIG_OPT(DynamicL1CacheDecreaseCountHeuristic, DYNAMICL1CACHEDECREASECOUNTHEURISTIC);\n\n FEX_CONFIG_OPT(DynamicL1Cache, DYNAMICL1CACHE);\n FEX_CONFIG_OPT(DisableL2Cache, DISABLEL2CACHE);\n};\n} // namespace FEXCore\n" }, { "path": "FEXCore/Source/Interface/Core/OpcodeDispatcher/AVX_128.cpp", "content": "// SPDX-License-Identifier: MIT\n/*\n$info$\ntags: frontend|x86-to-ir, opcodes|dispatcher-implementations\ndesc: Handles x86/64 AVX instructions to 128-bit IR\n$end_info$\n*/\n\n#include \"Interface/Core/X86Tables/X86Tables.h\"\n\n#include \n#include \"Interface/Core/OpcodeDispatcher.h\"\n\n#include \n#include \n#include \n#include \n\nnamespace FEXCore::IR {\nclass OrderedNode;\n\n#define OpcodeArgs [[maybe_unused]] FEXCore::X86Tables::DecodedOp Op\n\nOpDispatchBuilder::RefPair OpDispatchBuilder::AVX128_LoadSource_WithOpSize(\n const X86Tables::DecodedOp& Op, const X86Tables::DecodedOperand& Operand, uint32_t Flags, bool NeedsHigh, MemoryAccessType AccessType) {\n\n if (Operand.IsGPR()) {\n const auto gpr = Operand.Data.GPR.GPR;\n LOGMAN_THROW_A_FMT(gpr >= FEXCore::X86State::REG_XMM_0 && gpr <= FEXCore::X86State::REG_XMM_15, \"must be AVX reg\");\n const auto gprIndex = gpr - X86State::REG_XMM_0;\n return {\n .Low = AVX128_LoadXMMRegister(gprIndex, false),\n .High = NeedsHigh ? AVX128_LoadXMMRegister(gprIndex, true) : nullptr,\n };\n } else {\n LOGMAN_THROW_A_FMT(IsOperandMem(Operand, true), \"only memory sources\");\n\n if (Operand.IsSIB()) {\n const bool IsVSIB = (Op->Flags & X86Tables::DecodeFlags::FLAG_VSIB_BYTE) != 0;\n LOGMAN_THROW_A_FMT(!IsVSIB, \"VSIB uses LoadVSIB instead\");\n }\n\n const AddressMode A = DecodeAddress(Op, Operand, AccessType, true /* IsLoad */);\n if (NeedsHigh) {\n return _LoadMemPairFPRAutoTSO(OpSize::i128Bit, A, OpSize::i8Bit);\n } else {\n return {.Low = _LoadMemFPRAutoTSO(OpSize::i128Bit, A, OpSize::i8Bit)};\n }\n }\n}\n\nOpDispatchBuilder::RefVSIB\nOpDispatchBuilder::AVX128_LoadVSIB(const X86Tables::DecodedOp& Op, const X86Tables::DecodedOperand& Operand, uint32_t Flags, bool NeedsHigh) {\n const bool IsVSIB = (Op->Flags & X86Tables::DecodeFlags::FLAG_VSIB_BYTE) != 0;\n LOGMAN_THROW_A_FMT((Operand.IsSIB() || Operand.IsSIBRelocation()) && IsVSIB, \"Trying to load VSIB for something that isn't the correct \"\n \"type!\");\n\n // VSIB is a very special case which has a ton of encoded data.\n // Get it in a format we can reason about.\n\n const auto Index_gpr = Operand.Data.SIB.Index;\n const auto Base_gpr = Operand.Data.SIB.Base;\n LOGMAN_THROW_A_FMT(Index_gpr >= FEXCore::X86State::REG_XMM_0 && Index_gpr <= FEXCore::X86State::REG_XMM_15, \"must be AVX reg\");\n LOGMAN_THROW_A_FMT(Base_gpr == FEXCore::X86State::REG_INVALID || (Base_gpr >= FEXCore::X86State::REG_RAX && Base_gpr <= FEXCore::X86State::REG_R15),\n \"Base must be a GPR.\");\n const auto Index_XMM_gpr = Index_gpr - X86State::REG_XMM_0;\n\n OpDispatchBuilder::RefVSIB A {\n .Low = AVX128_LoadXMMRegister(Index_XMM_gpr, false),\n .High = NeedsHigh ? AVX128_LoadXMMRegister(Index_XMM_gpr, true) : Invalid(),\n .BaseAddr = Base_gpr != FEXCore::X86State::REG_INVALID ? LoadGPRRegister(Base_gpr, OpSize::i64Bit, 0, false) : nullptr,\n .Scale = Operand.Data.SIB.Scale,\n };\n\n if (Operand.IsSIBRelocation()) {\n auto EPOffset = _EntrypointOffset(OpSize::i64Bit, Operand.Data.SIB.Offset);\n if (A.BaseAddr) {\n A.BaseAddr = Add(OpSize::i64Bit, EPOffset, A.BaseAddr);\n } else {\n A.BaseAddr = EPOffset;\n }\n } else {\n A.Displacement = static_cast(Operand.Data.SIB.Offset);\n }\n\n return A;\n}\n\nvoid OpDispatchBuilder::AVX128_StoreResult_WithOpSize(FEXCore::X86Tables::DecodedOp Op, const FEXCore::X86Tables::DecodedOperand& Operand,\n const RefPair Src, MemoryAccessType AccessType) {\n if (Operand.IsGPR()) {\n const auto gpr = Operand.Data.GPR.GPR;\n LOGMAN_THROW_A_FMT(gpr >= FEXCore::X86State::REG_XMM_0 && gpr <= FEXCore::X86State::REG_XMM_15, \"expected AVX register\");\n const auto gprIndex = gpr - X86State::REG_XMM_0;\n\n if (Src.Low) {\n AVX128_StoreXMMRegister(gprIndex, Src.Low, false);\n }\n\n if (Src.High) {\n AVX128_StoreXMMRegister(gprIndex, Src.High, true);\n }\n } else {\n AddressMode A = DecodeAddress(Op, Operand, AccessType, false /* IsLoad */);\n\n if (Src.High) {\n _StoreMemPairFPRAutoTSO(OpSize::i128Bit, A, Src.Low, Src.High, OpSize::i8Bit);\n } else {\n _StoreMemFPRAutoTSO(OpSize::i128Bit, A, Src.Low, OpSize::i8Bit);\n }\n }\n}\n\nRef OpDispatchBuilder::AVX128_LoadXMMRegister(uint32_t XMM, bool High) {\n if (High) {\n return LoadContext(AVXHigh0Index + XMM);\n } else {\n return LoadXMMRegister(XMM);\n }\n}\n\nvoid OpDispatchBuilder::AVX128_StoreXMMRegister(uint32_t XMM, const Ref Src, bool High) {\n if (High) {\n StoreContext(AVXHigh0Index + XMM, Src);\n } else {\n StoreXMMRegister(XMM, Src);\n }\n}\n\nvoid OpDispatchBuilder::AVX128_VMOVAPS(OpcodeArgs) {\n // Reg <- Mem or Reg <- Reg\n const auto SrcSize = GetSrcSize(Op);\n const auto Is128Bit = SrcSize == Core::CPUState::XMM_SSE_REG_SIZE;\n\n if (Is128Bit) {\n // Zero upper 128-bits\n auto Src = AVX128_LoadSource_WithOpSize(Op, Op->Src[0], Op->Flags, false);\n\n ///< Zero upper bits when destination is GPR.\n if (Op->Dest.IsGPR()) {\n Src.High = LoadZeroVector(OpSize::i128Bit);\n }\n AVX128_StoreResult_WithOpSize(Op, Op->Dest, Src);\n } else {\n // Copy or memory load\n auto Src = AVX128_LoadSource_WithOpSize(Op, Op->Src[0], Op->Flags, true);\n AVX128_StoreResult_WithOpSize(Op, Op->Dest, Src);\n }\n}\n\nvoid OpDispatchBuilder::AVX128_VMOVScalarImpl(OpcodeArgs, IR::OpSize ElementSize) {\n if (Op->Dest.IsGPR() && Op->Src[0].IsGPR() && Op->Src[1].IsGPR()) {\n // VMOVSS/SD xmm1, xmm2, xmm3\n // Lower 128-bits are merged\n // Upper 128-bits are zero'd\n auto Src1 = AVX128_LoadSource_WithOpSize(Op, Op->Src[0], Op->Flags, false);\n auto Src2 = AVX128_LoadSource_WithOpSize(Op, Op->Src[1], Op->Flags, false);\n Ref Result = _VInsElement(OpSize::i128Bit, ElementSize, 0, 0, Src1.Low, Src2.Low);\n auto High = LoadZeroVector(OpSize::i128Bit);\n AVX128_StoreResult_WithOpSize(Op, Op->Dest, RefPair {.Low = Result, .High = High});\n } else if (Op->Dest.IsGPR()) {\n // VMOVSS/SD xmm1, mem32/mem64\n Ref Src = LoadSourceFPR_WithOpSize(Op, Op->Src[1], ElementSize, Op->Flags);\n auto High = LoadZeroVector(OpSize::i128Bit);\n AVX128_StoreResult_WithOpSize(Op, Op->Dest, RefPair {.Low = Src, .High = High});\n } else {\n // VMOVSS/SD mem32/mem64, xmm1\n auto Src = AVX128_LoadSource_WithOpSize(Op, Op->Src[1], Op->Flags, false);\n StoreResultFPR_WithOpSize(Op, Op->Dest, Src.Low, ElementSize);\n }\n}\n\nvoid OpDispatchBuilder::AVX128_VMOVSD(OpcodeArgs) {\n AVX128_VMOVScalarImpl(Op, OpSize::i64Bit);\n}\n\nvoid OpDispatchBuilder::AVX128_VMOVSS(OpcodeArgs) {\n AVX128_VMOVScalarImpl(Op, OpSize::i32Bit);\n}\n\nvoid OpDispatchBuilder::AVX128_VectorALU(OpcodeArgs, IROps IROp, IR::OpSize ElementSize) {\n const auto SrcSize = GetSrcSize(Op);\n const auto Is128Bit = SrcSize == Core::CPUState::XMM_SSE_REG_SIZE;\n\n auto Src1 = AVX128_LoadSource_WithOpSize(Op, Op->Src[0], Op->Flags, !Is128Bit);\n auto Src2 = AVX128_LoadSource_WithOpSize(Op, Op->Src[1], Op->Flags, !Is128Bit);\n DeriveOp(Result_Low, IROp, _VAdd(OpSize::i128Bit, ElementSize, Src1.Low, Src2.Low));\n\n if (Is128Bit) {\n auto High = LoadZeroVector(OpSize::i128Bit);\n AVX128_StoreResult_WithOpSize(Op, Op->Dest, RefPair {.Low = Result_Low, .High = High});\n } else {\n DeriveOp(Result_High, IROp, _VAdd(OpSize::i128Bit, ElementSize, Src1.High, Src2.High));\n AVX128_StoreResult_WithOpSize(Op, Op->Dest, RefPair {.Low = Result_Low, .High = Result_High});\n }\n}\n\nvoid OpDispatchBuilder::AVX128_VectorUnary(OpcodeArgs, IROps IROp, IR::OpSize ElementSize) {\n const auto SrcSize = GetSrcSize(Op);\n const auto Is128Bit = SrcSize == Core::CPUState::XMM_SSE_REG_SIZE;\n\n auto Src = AVX128_LoadSource_WithOpSize(Op, Op->Src[0], Op->Flags, !Is128Bit);\n DeriveOp(Result_Low, IROp, _VFSqrt(OpSize::i128Bit, ElementSize, Src.Low));\n\n if (Is128Bit) {\n auto High = LoadZeroVector(OpSize::i128Bit);\n AVX128_StoreResult_WithOpSize(Op, Op->Dest, RefPair {.Low = Result_Low, .High = High});\n } else {\n DeriveOp(Result_High, IROp, _VFSqrt(OpSize::i128Bit, ElementSize, Src.High));\n AVX128_StoreResult_WithOpSize(Op, Op->Dest, RefPair {.Low = Result_Low, .High = Result_High});\n }\n}\n\nvoid OpDispatchBuilder::AVX128_VectorUnaryImpl(OpcodeArgs, IR::OpSize SrcSize, IR::OpSize ElementSize,\n std::function Helper) {\n const auto Is128Bit = SrcSize == OpSize::i128Bit;\n\n auto Src = AVX128_LoadSource_WithOpSize(Op, Op->Src[0], Op->Flags, !Is128Bit);\n RefPair Result {};\n Result.Low = Helper(ElementSize, Src.Low);\n\n if (Is128Bit) {\n Result.High = LoadZeroVector(OpSize::i128Bit);\n } else {\n Result.High = Helper(ElementSize, Src.High);\n }\n\n AVX128_StoreResult_WithOpSize(Op, Op->Dest, Result);\n}\n\nvoid OpDispatchBuilder::AVX128_VectorBinaryImpl(OpcodeArgs, IR::OpSize SrcSize, IR::OpSize ElementSize,\n std::function Helper) {\n const auto Is128Bit = SrcSize == OpSize::i128Bit;\n\n auto Src1 = AVX128_LoadSource_WithOpSize(Op, Op->Src[0], Op->Flags, !Is128Bit);\n auto Src2 = AVX128_LoadSource_WithOpSize(Op, Op->Src[1], Op->Flags, !Is128Bit);\n RefPair Result {};\n Result.Low = Helper(ElementSize, Src1.Low, Src2.Low);\n\n if (Is128Bit) {\n Result.High = LoadZeroVector(OpSize::i128Bit);\n } else {\n Result.High = Helper(ElementSize, Src1.High, Src2.High);\n }\n\n AVX128_StoreResult_WithOpSize(Op, Op->Dest, Result);\n}\n\nvoid OpDispatchBuilder::AVX128_VectorTrinaryImpl(OpcodeArgs, IR::OpSize SrcSize, IR::OpSize ElementSize, Ref Src3,\n std::function Helper) {\n const auto Is128Bit = SrcSize == OpSize::i128Bit;\n\n auto Src1 = AVX128_LoadSource_WithOpSize(Op, Op->Src[0], Op->Flags, !Is128Bit);\n auto Src2 = AVX128_LoadSource_WithOpSize(Op, Op->Src[1], Op->Flags, !Is128Bit);\n RefPair Result {};\n Result.Low = Helper(ElementSize, Src1.Low, Src2.Low, Src3);\n\n if (Is128Bit) {\n Result.High = LoadZeroVector(OpSize::i128Bit);\n } else {\n Result.High = Helper(ElementSize, Src1.High, Src2.High, Src3);\n }\n\n AVX128_StoreResult_WithOpSize(Op, Op->Dest, Result);\n}\n\nvoid OpDispatchBuilder::AVX128_VectorShiftWideImpl(OpcodeArgs, IR::OpSize ElementSize, IROps IROp) {\n const auto Is128Bit = GetSrcSize(Op) == Core::CPUState::XMM_SSE_REG_SIZE;\n\n auto Src1 = AVX128_LoadSource_WithOpSize(Op, Op->Src[0], Op->Flags, !Is128Bit);\n auto Src2 = AVX128_LoadSource_WithOpSize(Op, Op->Src[1], Op->Flags, false);\n\n // Incoming element size for the shift source is always 8-bytes in the lower register.\n DeriveOp(Low, IROp, _VUShrSWide(OpSize::i128Bit, ElementSize, Src1.Low, Src2.Low));\n\n RefPair Result {};\n Result.Low = Low;\n\n if (Is128Bit) {\n Result.High = LoadZeroVector(OpSize::i128Bit);\n } else {\n DeriveOp(High, IROp, _VUShrSWide(OpSize::i128Bit, ElementSize, Src1.High, Src2.Low));\n Result.High = High;\n }\n\n AVX128_StoreResult_WithOpSize(Op, Op->Dest, Result);\n}\n\nvoid OpDispatchBuilder::AVX128_VectorShiftImmImpl(OpcodeArgs, IR::OpSize ElementSize, IROps IROp) {\n const auto DstSize = GetDstSize(Op);\n const auto Is128Bit = DstSize == Core::CPUState::XMM_SSE_REG_SIZE;\n const uint64_t ShiftConstant = Op->Src[1].Literal();\n\n auto Src = AVX128_LoadSource_WithOpSize(Op, Op->Src[0], Op->Flags, !Is128Bit);\n RefPair Result {};\n\n if (ShiftConstant == 0) [[unlikely]] {\n Result = Src;\n } else {\n DeriveOp(Low, IROp, _VUShrI(OpSize::i128Bit, ElementSize, Src.Low, ShiftConstant));\n Result.Low = Low;\n\n if (!Is128Bit) {\n DeriveOp(High, IROp, _VUShrI(OpSize::i128Bit, ElementSize, Src.High, ShiftConstant));\n Result.High = High;\n }\n }\n\n if (Is128Bit) {\n Result.High = LoadZeroVector(OpSize::i128Bit);\n }\n\n AVX128_StoreResult_WithOpSize(Op, Op->Dest, Result);\n}\n\nvoid OpDispatchBuilder::AVX128_VectorXOR(OpcodeArgs) {\n // Special case for vector xor with itself being the optimal way for x86 to zero vector registers.\n if (Op->Src[0].IsGPR() && Op->Src[1].IsGPR() && Op->Src[0].Data.GPR.GPR == Op->Src[1].Data.GPR.GPR) {\n AVX128_StoreResult_WithOpSize(Op, Op->Dest, AVX128_Zext(LoadZeroVector(OpSize::i128Bit)));\n return;\n }\n\n ///< Regular code path\n AVX128_VectorALU(Op, OP_VXOR, OpSize::i128Bit);\n}\n\nvoid OpDispatchBuilder::AVX128_VZERO(OpcodeArgs) {\n const auto DstSize = GetDstSize(Op);\n const auto IsVZEROALL = DstSize == Core::CPUState::XMM_AVX_REG_SIZE;\n const auto NumRegs = Is64BitMode ? 16U : 8U;\n\n if (IsVZEROALL) {\n // NOTE: Despite the name being VZEROALL, this will still only ever\n // zero out up to the first 16 registers (even on AVX-512, where we have 32 registers)\n Ref ZeroVector {};\n\n for (uint32_t i = 0; i < NumRegs; i++) {\n // Explicitly not caching named vector zero. This ensures that every register gets movi #0.0 directly.\n ZeroVector = LoadUncachedZeroVector(OpSize::i128Bit);\n AVX128_StoreXMMRegister(i, ZeroVector, false);\n }\n\n InvalidateHighAVXRegisters();\n _ContextClear(offsetof(FEXCore::Core::CPUState, avx_high), sizeof(FEXCore::Core::CPUState::avx_high[0]) * NumRegs);\n } else {\n // Likewise, VZEROUPPER will only ever zero only up to the first 16 registers\n InvalidateHighAVXRegisters();\n _ContextClear(offsetof(FEXCore::Core::CPUState, avx_high), sizeof(FEXCore::Core::CPUState::avx_high[0]) * NumRegs);\n }\n}\n\nvoid OpDispatchBuilder::AVX128_MOVVectorNT(OpcodeArgs) {\n const auto SrcSize = GetSrcSize(Op);\n const auto Is128Bit = SrcSize == Core::CPUState::XMM_SSE_REG_SIZE;\n\n if (Op->Dest.IsGPR()) {\n ///< MOVNTDQA load non-temporal comes from SSE4.1 and is extended by AVX/AVX2.\n RefPair Src {};\n Ref SrcAddr = LoadSourceGPR(Op, Op->Src[0], Op->Flags, {.LoadData = false});\n Src.Low = _VLoadNonTemporal(OpSize::i128Bit, SrcAddr, 0);\n\n if (Is128Bit) {\n Src.High = LoadZeroVector(OpSize::i128Bit);\n } else {\n Src.High = _VLoadNonTemporal(OpSize::i128Bit, SrcAddr, 16);\n }\n AVX128_StoreResult_WithOpSize(Op, Op->Dest, Src);\n } else {\n auto Src = AVX128_LoadSource_WithOpSize(Op, Op->Src[0], Op->Flags, !Is128Bit, MemoryAccessType::STREAM);\n Ref Dest = LoadSourceGPR(Op, Op->Dest, Op->Flags, {.LoadData = false});\n\n if (Is128Bit) {\n // Single store non-temporal for 128-bit operations.\n _VStoreNonTemporal(OpSize::i128Bit, Src.Low, Dest, 0);\n } else {\n // For a 256-bit store, use a non-temporal store pair\n _VStoreNonTemporalPair(OpSize::i128Bit, Src.Low, Src.High, Dest, 0);\n }\n }\n}\n\nvoid OpDispatchBuilder::AVX128_MOVQ(OpcodeArgs) {\n RefPair Src {};\n if (Op->Src[0].IsGPR()) {\n Src = AVX128_LoadSource_WithOpSize(Op, Op->Src[0], Op->Flags, false);\n } else {\n Src.Low = LoadSourceFPR_WithOpSize(Op, Op->Src[0], OpSize::i64Bit, Op->Flags);\n }\n\n // This instruction is a bit special that if the destination is a register then it'll ZEXT the 64bit source to 256bit\n if (Op->Dest.IsGPR()) {\n // Zero bits [127:64] as well.\n Src.Low = VZeroExtendOperand(OpSize::i64Bit, Op->Src[0], Src.Low);\n Ref ZeroVector = LoadZeroVector(OpSize::i128Bit);\n Src.High = ZeroVector;\n AVX128_StoreResult_WithOpSize(Op, Op->Dest, Src);\n } else {\n StoreResultFPR_WithOpSize(Op, Op->Dest, Src.Low, OpSize::i64Bit, OpSize::i64Bit);\n }\n}\n\nvoid OpDispatchBuilder::AVX128_VMOVLP(OpcodeArgs) {\n auto Src1 = AVX128_LoadSource_WithOpSize(Op, Op->Src[0], Op->Flags, false);\n\n if (!Op->Dest.IsGPR()) {\n ///< VMOVLPS/PD mem64, xmm1\n StoreResultFPR_WithOpSize(Op, Op->Dest, Src1.Low, OpSize::i64Bit, OpSize::i64Bit);\n } else if (!Op->Src[1].IsGPR()) {\n ///< VMOVLPS/PD xmm1, xmm2, mem64\n // Bits[63:0] come from Src2[63:0]\n // Bits[127:64] come from Src1[127:64]\n auto Src2 = MakeSegmentAddress(Op, Op->Src[1]);\n Ref Result_Low = _VLoadVectorElement(OpSize::i128Bit, OpSize::i64Bit, Src1.Low, 0, Src2);\n Ref ZeroVector = LoadZeroVector(OpSize::i128Bit);\n\n AVX128_StoreResult_WithOpSize(Op, Op->Dest, RefPair {.Low = Result_Low, .High = ZeroVector});\n } else {\n ///< VMOVHLPS/PD xmm1, xmm2, xmm3\n auto Src2 = AVX128_LoadSource_WithOpSize(Op, Op->Src[1], Op->Flags, false);\n\n Ref Result_Low = _VInsElement(OpSize::i128Bit, OpSize::i64Bit, 0, 1, Src1.Low, Src2.Low);\n Ref ZeroVector = LoadZeroVector(OpSize::i128Bit);\n\n AVX128_StoreResult_WithOpSize(Op, Op->Dest, RefPair {.Low = Result_Low, .High = ZeroVector});\n }\n}\n\nvoid OpDispatchBuilder::AVX128_VMOVHP(OpcodeArgs) {\n auto Src1 = AVX128_LoadSource_WithOpSize(Op, Op->Src[0], Op->Flags, false);\n\n if (!Op->Dest.IsGPR()) {\n ///< VMOVHPS/PD mem64, xmm1\n // Need to store Bits[127:64]. Use a vector element store.\n auto Dest = MakeSegmentAddress(Op, Op->Dest);\n _VStoreVectorElement(OpSize::i128Bit, OpSize::i64Bit, Src1.Low, 1, Dest);\n } else if (!Op->Src[1].IsGPR()) {\n ///< VMOVHPS/PD xmm2, xmm1, mem64\n auto Src2 = MakeSegmentAddress(Op, Op->Src[1]);\n\n // Bits[63:0] come from Src1[63:0]\n // Bits[127:64] come from Src2[63:0]\n Ref Result_Low = _VLoadVectorElement(OpSize::i128Bit, OpSize::i64Bit, Src1.Low, 1, Src2);\n Ref ZeroVector = LoadZeroVector(OpSize::i128Bit);\n\n AVX128_StoreResult_WithOpSize(Op, Op->Dest, RefPair {.Low = Result_Low, .High = ZeroVector});\n } else {\n // VMOVLHPS xmm1, xmm2, xmm3\n auto Src2 = AVX128_LoadSource_WithOpSize(Op, Op->Src[1], Op->Flags, false);\n\n Ref Result_Low = _VZip(OpSize::i128Bit, OpSize::i64Bit, Src1.Low, Src2.Low);\n Ref ZeroVector = LoadZeroVector(OpSize::i128Bit);\n\n AVX128_StoreResult_WithOpSize(Op, Op->Dest, RefPair {.Low = Result_Low, .High = ZeroVector});\n }\n}\n\nvoid OpDispatchBuilder::AVX128_VMOVDDUP(OpcodeArgs) {\n const auto SrcSize = GetSrcSize(Op);\n const auto Is128Bit = SrcSize == Core::CPUState::XMM_SSE_REG_SIZE;\n\n const auto IsSrcGPR = Op->Src[0].IsGPR();\n\n RefPair Src {};\n if (IsSrcGPR) {\n Src = AVX128_LoadSource_WithOpSize(Op, Op->Src[0], Op->Flags, !Is128Bit);\n } else {\n // Accesses from memory are a little weird.\n // 128-bit operation only loads 8-bytes.\n // 256-bit operation loads a full 32-bytes.\n if (Is128Bit) {\n Src.Low = LoadSourceFPR_WithOpSize(Op, Op->Src[0], OpSize::i64Bit, Op->Flags);\n } else {\n Src = AVX128_LoadSource_WithOpSize(Op, Op->Src[0], Op->Flags, true);\n }\n }\n\n if (Is128Bit) {\n // Duplicate Src[63:0] in to low 128-bits\n auto Result_Low = _VDupElement(OpSize::i128Bit, OpSize::i64Bit, Src.Low, 0);\n Ref ZeroVector = LoadZeroVector(OpSize::i128Bit);\n\n AVX128_StoreResult_WithOpSize(Op, Op->Dest, RefPair {.Low = Result_Low, .High = ZeroVector});\n } else {\n // Duplicate Src.Low[63:0] in to low 128-bits\n auto Result_Low = _VDupElement(OpSize::i128Bit, OpSize::i64Bit, Src.Low, 0);\n // Duplicate Src.High[63:0] in to high 128-bits\n auto Result_High = _VDupElement(OpSize::i128Bit, OpSize::i64Bit, Src.High, 0);\n AVX128_StoreResult_WithOpSize(Op, Op->Dest, RefPair {.Low = Result_Low, .High = Result_High});\n }\n}\n\nvoid OpDispatchBuilder::AVX128_VMOVSLDUP(OpcodeArgs) {\n AVX128_VectorUnaryImpl(Op, OpSizeFromSrc(Op), OpSize::i32Bit,\n [this](IR::OpSize ElementSize, Ref Src) { return _VTrn(OpSize::i128Bit, ElementSize, Src, Src); });\n}\n\nvoid OpDispatchBuilder::AVX128_VMOVSHDUP(OpcodeArgs) {\n AVX128_VectorUnaryImpl(Op, OpSizeFromSrc(Op), OpSize::i32Bit,\n [this](IR::OpSize ElementSize, Ref Src) { return _VTrn2(OpSize::i128Bit, ElementSize, Src, Src); });\n}\n\nvoid OpDispatchBuilder::AVX128_VBROADCAST(OpcodeArgs, IR::OpSize ElementSize) {\n const auto DstSize = GetDstSize(Op);\n const auto Is128Bit = DstSize == Core::CPUState::XMM_SSE_REG_SIZE;\n RefPair Src {};\n\n if (Op->Src[0].IsGPR()) {\n Src = AVX128_LoadSource_WithOpSize(Op, Op->Src[0], Op->Flags, false);\n if (ElementSize != OpSize::i128Bit) {\n // Only duplicate if not VBROADCASTF128.\n Src.Low = _VDupElement(OpSize::i128Bit, ElementSize, Src.Low, 0);\n }\n } else {\n // Get the address to broadcast from into a GPR.\n Ref Address = MakeSegmentAddress(Op, Op->Src[0], GetGPROpSize());\n Src.Low = _VBroadcastFromMem(OpSize::i128Bit, ElementSize, Address);\n }\n\n if (Is128Bit) {\n Src.High = LoadZeroVector(OpSize::i128Bit);\n } else {\n Src.High = Src.Low;\n }\n\n AVX128_StoreResult_WithOpSize(Op, Op->Dest, Src);\n}\n\nvoid OpDispatchBuilder::AVX128_VPUNPCKL(OpcodeArgs, IR::OpSize ElementSize) {\n AVX128_VectorBinaryImpl(Op, OpSizeFromSrc(Op), ElementSize,\n [this](IR::OpSize _ElementSize, Ref Src1, Ref Src2) { return _VZip(OpSize::i128Bit, _ElementSize, Src1, Src2); });\n}\n\nvoid OpDispatchBuilder::AVX128_VPUNPCKH(OpcodeArgs, IR::OpSize ElementSize) {\n AVX128_VectorBinaryImpl(Op, OpSizeFromSrc(Op), ElementSize,\n [this](IR::OpSize _ElementSize, Ref Src1, Ref Src2) { return _VZip2(OpSize::i128Bit, _ElementSize, Src1, Src2); });\n}\n\nvoid OpDispatchBuilder::AVX128_MOVVectorUnaligned(OpcodeArgs) {\n const auto SrcSize = GetSrcSize(Op);\n const auto Is128Bit = SrcSize == Core::CPUState::XMM_SSE_REG_SIZE;\n\n if (!Is128Bit && Op->Dest.IsGPR() && Op->Src[0].IsGPR() && Op->Dest.Data.GPR.GPR == Op->Src[0].Data.GPR.GPR) {\n // Nop\n return;\n }\n\n auto Src = AVX128_LoadSource_WithOpSize(Op, Op->Src[0], Op->Flags, !Is128Bit);\n\n if (Is128Bit) {\n Src.High = LoadZeroVector(OpSize::i128Bit);\n }\n\n AVX128_StoreResult_WithOpSize(Op, Op->Dest, Src);\n}\n\nvoid OpDispatchBuilder::AVX128_InsertCVTGPR_To_FPR(OpcodeArgs, IR::OpSize DstElementSize) {\n const auto SrcSize = OpSizeFromSrc(Op);\n const auto DstSize = OpSizeFromDst(Op);\n\n auto Src1 = AVX128_LoadSource_WithOpSize(Op, Op->Src[0], Op->Flags, false);\n\n RefPair Result {};\n\n if (Op->Src[1].IsGPR()) {\n // If the source is a GPR then convert directly from the GPR.\n auto Src2 = LoadSourceGPR_WithOpSize(Op, Op->Src[1], GetGPROpSize(), Op->Flags);\n Result.Low = _VSToFGPRInsert(OpSize::i128Bit, DstElementSize, SrcSize, Src1.Low, Src2, false);\n } else if (SrcSize != DstElementSize) {\n // If the source is from memory but the Source size and destination size aren't the same,\n // then it is more optimal to load in to a GPR and convert between GPR->FPR.\n // ARM GPR->FPR conversion supports different size source and destinations while FPR->FPR doesn't.\n auto Src2 = LoadSourceGPR(Op, Op->Src[1], Op->Flags);\n Result.Low = _VSToFGPRInsert(DstSize, DstElementSize, SrcSize, Src1.Low, Src2, false);\n } else {\n // In the case of cvtsi2s{s,d} where the source and destination are the same size,\n // then it is more optimal to load in to the FPR register directly and convert there.\n auto Src2 = LoadSourceFPR(Op, Op->Src[1], Op->Flags);\n // Always signed\n Result.Low = _VSToFVectorInsert(DstSize, DstElementSize, DstElementSize, Src1.Low, Src2, false, false);\n }\n\n const auto Is128Bit = DstSize == OpSize::i128Bit;\n LOGMAN_THROW_A_FMT(Is128Bit, \"Programming Error: This should never occur!\");\n Result.High = LoadZeroVector(OpSize::i128Bit);\n\n AVX128_StoreResult_WithOpSize(Op, Op->Dest, Result);\n}\n\nvoid OpDispatchBuilder::AVX128_CVTFPR_To_GPR(OpcodeArgs, IR::OpSize SrcElementSize, bool HostRoundingMode) {\n // If loading a vector, use the full size, so we don't\n // unnecessarily zero extend the vector. Otherwise, if\n // memory, then we want to load the element size exactly.\n RefPair Src {};\n if (Op->Src[0].IsGPR()) {\n Src = AVX128_LoadSource_WithOpSize(Op, Op->Src[0], Op->Flags, false);\n } else {\n Src.Low = LoadSourceFPR_WithOpSize(Op, Op->Src[0], SrcElementSize, Op->Flags);\n }\n\n Ref Result = CVTFPR_To_GPRImpl(Op, Src.Low, SrcElementSize, HostRoundingMode);\n StoreResultGPR(Op, Result);\n}\n\nvoid OpDispatchBuilder::AVX128_VANDN(OpcodeArgs) {\n AVX128_VectorBinaryImpl(Op, OpSizeFromSrc(Op), OpSize::i128Bit,\n [this](IR::OpSize _ElementSize, Ref Src1, Ref Src2) { return _VAndn(OpSize::i128Bit, _ElementSize, Src2, Src1); });\n}\n\nvoid OpDispatchBuilder::AVX128_VPACKSS(OpcodeArgs, IR::OpSize ElementSize) {\n AVX128_VectorBinaryImpl(Op, OpSizeFromSrc(Op), ElementSize, [this](IR::OpSize _ElementSize, Ref Src1, Ref Src2) {\n return _VSQXTNPair(OpSize::i128Bit, _ElementSize, Src1, Src2);\n });\n}\n\nvoid OpDispatchBuilder::AVX128_VPACKUS(OpcodeArgs, IR::OpSize ElementSize) {\n AVX128_VectorBinaryImpl(Op, OpSizeFromSrc(Op), ElementSize, [this](IR::OpSize _ElementSize, Ref Src1, Ref Src2) {\n return _VSQXTUNPair(OpSize::i128Bit, _ElementSize, Src1, Src2);\n });\n}\n\nRef OpDispatchBuilder::AVX128_PSIGNImpl(IR::OpSize ElementSize, Ref Src1, Ref Src2) {\n Ref Control = _VSQSHL(OpSize::i128Bit, ElementSize, Src2, IR::OpSizeAsBits(ElementSize) - 1);\n Control = _VSRSHR(OpSize::i128Bit, ElementSize, Control, IR::OpSizeAsBits(ElementSize) - 1);\n return _VMul(OpSize::i128Bit, ElementSize, Src1, Control);\n}\n\nvoid OpDispatchBuilder::AVX128_VPSIGN(OpcodeArgs, IR::OpSize ElementSize) {\n AVX128_VectorBinaryImpl(Op, OpSizeFromSrc(Op), ElementSize,\n [this](IR::OpSize _ElementSize, Ref Src1, Ref Src2) { return AVX128_PSIGNImpl(_ElementSize, Src1, Src2); });\n}\n\nvoid OpDispatchBuilder::AVX128_UCOMISx(OpcodeArgs, IR::OpSize ElementSize) {\n const auto SrcSize = Op->Src[0].IsGPR() ? GetGuestVectorLength() : ElementSize;\n\n auto Src1 = AVX128_LoadSource_WithOpSize(Op, Op->Dest, Op->Flags, false);\n\n RefPair Src2 {};\n\n // Careful here, if the source is from a GPR then we want to load the full 128-bit lower half.\n // If it is memory then we only want to load the element size.\n if (Op->Src[0].IsGPR()) {\n Src2 = AVX128_LoadSource_WithOpSize(Op, Op->Src[0], Op->Flags, false);\n } else {\n Src2.Low = LoadSourceFPR_WithOpSize(Op, Op->Src[0], SrcSize, Op->Flags);\n }\n\n Comiss(ElementSize, Src1.Low, Src2.Low);\n}\n\nvoid OpDispatchBuilder::AVX128_VectorScalarInsertALU(OpcodeArgs, FEXCore::IR::IROps IROp, IR::OpSize ElementSize) {\n // We load the full vector width when dealing with a source vector,\n // so that we don't do any unnecessary zero extension to the scalar\n // element that we're going to operate on.\n const auto SrcSize = OpSizeFromSrc(Op);\n\n auto Src1 = AVX128_LoadSource_WithOpSize(Op, Op->Src[0], Op->Flags, false);\n RefPair Src2 {};\n if (Op->Src[1].IsGPR()) {\n Src2 = AVX128_LoadSource_WithOpSize(Op, Op->Src[1], Op->Flags, false);\n } else {\n Src2.Low = LoadSourceFPR_WithOpSize(Op, Op->Src[1], SrcSize, Op->Flags);\n }\n\n // If OpSize == ElementSize then it only does the lower scalar op\n DeriveOp(Result_Low, IROp, _VFAddScalarInsert(OpSize::i128Bit, ElementSize, Src1.Low, Src2.Low, false));\n auto High = LoadZeroVector(OpSize::i128Bit);\n AVX128_StoreResult_WithOpSize(Op, Op->Dest, RefPair {.Low = Result_Low, .High = High});\n}\n\nvoid OpDispatchBuilder::AVX128_VFCMP(OpcodeArgs, IR::OpSize ElementSize) {\n const uint8_t CompType = Op->Src[2].Literal();\n\n struct {\n FEXCore::X86Tables::DecodedOp Op;\n uint32_t CompType {};\n } Capture {\n .Op = Op,\n .CompType = CompType & 0b11111u,\n };\n\n AVX128_VectorBinaryImpl(Op, OpSizeFromSrc(Op), ElementSize, [this, &Capture](IR::OpSize _ElementSize, Ref Src1, Ref Src2) {\n return VFCMPOpImpl(OpSize::i128Bit, _ElementSize, Src1, Src2, Capture.CompType);\n });\n}\n\nvoid OpDispatchBuilder::AVX128_InsertScalarFCMP(OpcodeArgs, IR::OpSize ElementSize) {\n // We load the full vector width when dealing with a source vector,\n // so that we don't do any unnecessary zero extension to the scalar\n // element that we're going to operate on.\n const auto SrcSize = OpSizeFromSrc(Op);\n\n auto Src1 = AVX128_LoadSource_WithOpSize(Op, Op->Src[0], Op->Flags, false);\n RefPair Src2 {};\n\n if (Op->Src[1].IsGPR()) {\n Src2 = AVX128_LoadSource_WithOpSize(Op, Op->Src[1], Op->Flags, false);\n } else {\n Src2.Low = LoadSourceFPR_WithOpSize(Op, Op->Src[1], SrcSize, Op->Flags);\n }\n\n const uint8_t CompType = Op->Src[2].Literal();\n\n RefPair Result {};\n Result.Low = InsertScalarFCMPOpImpl(OpSize::i128Bit, OpSize::i128Bit, ElementSize, Src1.Low, Src2.Low, CompType & 0b11111, false);\n Result.High = LoadZeroVector(OpSize::i128Bit);\n AVX128_StoreResult_WithOpSize(Op, Op->Dest, Result);\n}\n\nvoid OpDispatchBuilder::AVX128_MOVBetweenGPR_FPR(OpcodeArgs) {\n if (Op->Dest.IsGPR() && Op->Dest.Data.GPR.GPR >= FEXCore::X86State::REG_XMM_0) {\n ///< XMM <- Reg/Mem\n\n RefPair Result {};\n if (Op->Src[0].IsGPR()) {\n // Loading from GPR and moving to Vector.\n Ref Src = LoadSourceFPR_WithOpSize(Op, Op->Src[0], GetGPROpSize(), Op->Flags);\n // zext to 128bit\n Result.Low = _VCastFromGPR(OpSize::i128Bit, OpSizeFromSrc(Op), Src);\n } else {\n // Loading from Memory as a scalar. Zero extend\n Result.Low = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n }\n\n Result.High = LoadZeroVector(OpSize::i128Bit);\n AVX128_StoreResult_WithOpSize(Op, Op->Dest, Result);\n } else {\n ///< Reg/Mem <- XMM\n auto Src = AVX128_LoadSource_WithOpSize(Op, Op->Src[0], Op->Flags, false);\n\n if (Op->Dest.IsGPR()) {\n auto ElementSize = OpSizeFromDst(Op);\n // Extract element from GPR. Zero extending in the process.\n Src.Low = _VExtractToGPR(OpSizeFromSrc(Op), ElementSize, Src.Low, 0);\n StoreResultGPR(Op, Op->Dest, Src.Low);\n } else {\n // Storing first element to memory.\n Ref Dest = LoadSourceGPR(Op, Op->Dest, Op->Flags, {.LoadData = false});\n _StoreMemFPR(OpSizeFromDst(Op), Dest, Src.Low, OpSize::i8Bit);\n }\n }\n}\n\nvoid OpDispatchBuilder::AVX128_PExtr(OpcodeArgs, IR::OpSize ElementSize) {\n const auto DstSize = OpSizeFromDst(Op);\n\n auto Src = AVX128_LoadSource_WithOpSize(Op, Op->Src[0], Op->Flags, false);\n uint64_t Index = Op->Src[1].Literal();\n\n // Fixup of 32-bit element size.\n // When the element size is 32-bit then it can be overriden as 64-bit because the encoding of PEXTRD/PEXTRQ\n // is the same except that REX.W or VEX.W is set to 1. Incredibly frustrating.\n // Use the destination size as the element size in this case.\n auto OverridenElementSize = ElementSize;\n if (ElementSize == OpSize::i32Bit) {\n OverridenElementSize = DstSize;\n }\n\n // AVX version only operates on 128-bit.\n const uint8_t NumElements = IR::NumElements(std::min(OpSizeFromSrc(Op), OpSize::i128Bit), OverridenElementSize);\n Index &= NumElements - 1;\n\n if (Op->Dest.IsGPR()) {\n const auto GPRSize = GetGPROpSize();\n // Extract already zero extends the result.\n Ref Result = _VExtractToGPR(OpSize::i128Bit, OverridenElementSize, Src.Low, Index);\n StoreResultGPR_WithOpSize(Op, Op->Dest, Result, GPRSize);\n return;\n }\n\n // If we are storing to memory then we store the size of the element extracted\n Ref Dest = MakeSegmentAddress(Op, Op->Dest);\n _VStoreVectorElement(OpSize::i128Bit, OverridenElementSize, Src.Low, Index, Dest);\n}\n\nvoid OpDispatchBuilder::AVX128_ExtendVectorElements(OpcodeArgs, IR::OpSize ElementSize, IR::OpSize DstElementSize, bool Signed) {\n const auto DstSize = OpSizeFromDst(Op);\n\n const auto GetSrc = [&] {\n if (Op->Src[0].IsGPR()) {\n return AVX128_LoadSource_WithOpSize(Op, Op->Src[0], Op->Flags, false).Low;\n } else {\n // For memory operands the 256-bit variant loads twice the size specified in the table.\n const auto Is256Bit = DstSize == OpSize::i256Bit;\n const auto SrcSize = OpSizeFromSrc(Op);\n const auto LoadSize = Is256Bit ? IR::SizeToOpSize(IR::OpSizeToSize(SrcSize) * 2) : SrcSize;\n\n return LoadSourceFPR_WithOpSize(Op, Op->Src[0], LoadSize, Op->Flags);\n }\n };\n\n auto Transform = [=, this](Ref Src) {\n for (auto CurrentElementSize = ElementSize; CurrentElementSize != DstElementSize; CurrentElementSize = CurrentElementSize << 1) {\n if (Signed) {\n Src = _VSXTL(OpSize::i128Bit, CurrentElementSize, Src);\n } else {\n Src = _VUXTL(OpSize::i128Bit, CurrentElementSize, Src);\n }\n }\n return Src;\n };\n\n Ref Src = GetSrc();\n RefPair Result {};\n\n if (DstSize == OpSize::i128Bit) {\n // 128-bit operation is easy, it stays within the single register.\n Result.Low = Transform(Src);\n } else {\n // 256-bit operation is a bit special. It splits the incoming source between lower and upper registers.\n size_t TotalElementCount = IR::NumElements(OpSize::i256Bit, DstElementSize);\n size_t TotalElementsToSplitSize = (TotalElementCount / 2) * IR::OpSizeToSize(ElementSize);\n\n // Split the number of elements in half between lower and upper.\n Ref SrcHigh = _VDupElement(OpSize::i128Bit, IR::SizeToOpSize(TotalElementsToSplitSize), Src, 1);\n Result.Low = Transform(Src);\n Result.High = Transform(SrcHigh);\n }\n\n if (DstSize == OpSize::i128Bit) {\n // Regular zero-extending semantics.\n Result.High = LoadZeroVector(OpSize::i128Bit);\n }\n\n AVX128_StoreResult_WithOpSize(Op, Op->Dest, Result);\n}\n\nvoid OpDispatchBuilder::AVX128_MOVMSK(OpcodeArgs, IR::OpSize ElementSize) {\n const auto SrcSize = OpSizeFromSrc(Op);\n const auto Is128Bit = SrcSize == OpSize::i128Bit;\n\n auto Src = AVX128_LoadSource_WithOpSize(Op, Op->Src[0], Op->Flags, !Is128Bit);\n\n auto Mask8Byte = [this](Ref Src) {\n // UnZip2 the 64-bit elements as 32-bit to get the sign bits closer.\n // Sign bits are now in bit positions 31 and 63 after this.\n Src = _VUnZip2(OpSize::i128Bit, OpSize::i32Bit, Src, Src);\n\n // Extract the low 64-bits to GPR in one move.\n Ref GPR = _VExtractToGPR(OpSize::i128Bit, OpSize::i64Bit, Src, 0);\n // BFI the sign bit in 31 in to 62.\n // Inserting the full lower 32-bits offset 31 so the sign bit ends up at offset 63.\n GPR = _Bfi(OpSize::i64Bit, 32, 31, GPR, GPR);\n // Shift right to only get the two sign bits we care about.\n return _Lshr(OpSize::i64Bit, GPR, Constant(62));\n };\n\n auto Mask4Byte = [this](Ref Src) {\n // Shift all the sign bits to the bottom of their respective elements.\n Src = _VUShrI(OpSize::i128Bit, OpSize::i32Bit, Src, 31);\n // Load the specific 128-bit movmskps shift elements operator.\n auto ConstantUSHL = LoadAndCacheNamedVectorConstant(OpSize::i128Bit, NAMED_VECTOR_MOVMSKPS_SHIFT);\n // Shift the sign bits in to specific locations.\n Src = _VUShl(OpSize::i128Bit, OpSize::i32Bit, Src, ConstantUSHL, false);\n // Add across the vector so the sign bits will end up in bits [3:0]\n Src = _VAddV(OpSize::i128Bit, OpSize::i32Bit, Src);\n // Extract to a GPR.\n return _VExtractToGPR(OpSize::i128Bit, OpSize::i32Bit, Src, 0);\n };\n\n Ref GPR {};\n if (Is128Bit) {\n if (ElementSize == OpSize::i64Bit) {\n GPR = Mask8Byte(Src.Low);\n } else {\n GPR = Mask4Byte(Src.Low);\n }\n } else if (ElementSize == OpSize::i32Bit) {\n auto GPRLow = Mask4Byte(Src.Low);\n auto GPRHigh = Mask4Byte(Src.High);\n GPR = _Orlshl(OpSize::i64Bit, GPRLow, GPRHigh, 4);\n } else {\n auto GPRLow = Mask8Byte(Src.Low);\n auto GPRHigh = Mask8Byte(Src.High);\n GPR = _Orlshl(OpSize::i64Bit, GPRLow, GPRHigh, 2);\n }\n StoreResultGPR_WithOpSize(Op, Op->Dest, GPR, GetGPROpSize());\n}\n\nvoid OpDispatchBuilder::AVX128_MOVMSKB(OpcodeArgs) {\n const auto SrcSize = GetSrcSize(Op);\n const auto Is128Bit = SrcSize == Core::CPUState::XMM_SSE_REG_SIZE;\n\n auto Src = AVX128_LoadSource_WithOpSize(Op, Op->Src[0], Op->Flags, !Is128Bit);\n Ref VMask = LoadAndCacheNamedVectorConstant(OpSize::i128Bit, NAMED_VECTOR_MOVMASKB);\n\n auto Mask1Byte = [this](Ref Src, Ref VMask) {\n auto VCMP = _VCMPLTZ(OpSize::i128Bit, OpSize::i8Bit, Src);\n auto VAnd = _VAnd(OpSize::i128Bit, OpSize::i8Bit, VCMP, VMask);\n\n auto VAdd1 = _VAddP(OpSize::i128Bit, OpSize::i8Bit, VAnd, VAnd);\n auto VAdd2 = _VAddP(OpSize::i128Bit, OpSize::i8Bit, VAdd1, VAdd1);\n auto VAdd3 = _VAddP(OpSize::i64Bit, OpSize::i8Bit, VAdd2, VAdd2);\n\n ///< 16-bits of data per 128-bit\n return _VExtractToGPR(OpSize::i128Bit, OpSize::i16Bit, VAdd3, 0);\n };\n\n Ref Result = Mask1Byte(Src.Low, VMask);\n\n if (!Is128Bit) {\n auto ResultHigh = Mask1Byte(Src.High, VMask);\n Result = _Orlshl(OpSize::i64Bit, Result, ResultHigh, 16);\n }\n\n StoreResultGPR(Op, Result);\n}\n\nvoid OpDispatchBuilder::AVX128_PINSRImpl(OpcodeArgs, IR::OpSize ElementSize, const X86Tables::DecodedOperand& Src1Op,\n const X86Tables::DecodedOperand& Src2Op, const X86Tables::DecodedOperand& Imm) {\n const auto NumElements = IR::NumElements(OpSize::i128Bit, ElementSize);\n const uint64_t Index = Imm.Literal() & (NumElements - 1);\n auto Src1 = AVX128_LoadSource_WithOpSize(Op, Src1Op, Op->Flags, false);\n\n RefPair Result {};\n\n if (Src2Op.IsGPR()) {\n // If the source is a GPR then convert directly from the GPR.\n auto Src2 = LoadSourceGPR_WithOpSize(Op, Src2Op, GetGPROpSize(), Op->Flags);\n Result.Low = _VInsGPR(OpSize::i128Bit, ElementSize, Index, Src1.Low, Src2);\n } else {\n // If loading from memory then we only load the element size\n auto Src2 = MakeSegmentAddress(Op, Src2Op);\n Result.Low = _VLoadVectorElement(OpSize::i128Bit, ElementSize, Src1.Low, Index, Src2);\n }\n\n Result.High = LoadZeroVector(OpSize::i128Bit);\n AVX128_StoreResult_WithOpSize(Op, Op->Dest, Result);\n}\n\nvoid OpDispatchBuilder::AVX128_VPINSRB(OpcodeArgs) {\n AVX128_PINSRImpl(Op, OpSize::i8Bit, Op->Src[0], Op->Src[1], Op->Src[2]);\n}\n\nvoid OpDispatchBuilder::AVX128_VPINSRW(OpcodeArgs) {\n AVX128_PINSRImpl(Op, OpSize::i16Bit, Op->Src[0], Op->Src[1], Op->Src[2]);\n}\n\nvoid OpDispatchBuilder::AVX128_VPINSRDQ(OpcodeArgs) {\n const auto SrcSize = OpSizeFromSrc(Op);\n AVX128_PINSRImpl(Op, SrcSize, Op->Src[0], Op->Src[1], Op->Src[2]);\n}\n\nvoid OpDispatchBuilder::AVX128_VariableShiftImpl(OpcodeArgs, IROps IROp) {\n AVX128_VectorBinaryImpl(Op, OpSizeFromDst(Op), OpSizeFromSrc(Op), [this, IROp](IR::OpSize ElementSize, Ref Src1, Ref Src2) {\n DeriveOp(Shift, IROp, _VUShr(OpSize::i128Bit, ElementSize, Src1, Src2, true));\n return Shift;\n });\n}\n\nvoid OpDispatchBuilder::AVX128_ShiftDoubleImm(OpcodeArgs, ShiftDirection Dir) {\n const auto DstSize = GetDstSize(Op);\n const auto Is128Bit = DstSize == Core::CPUState::XMM_SSE_REG_SIZE;\n const bool Right = Dir == ShiftDirection::RIGHT;\n\n const uint64_t Shift = Op->Src[1].Literal();\n const uint64_t ExtrShift = Right ? Shift : IR::OpSizeToSize(OpSize::i128Bit) - Shift;\n\n auto Src = AVX128_LoadSource_WithOpSize(Op, Op->Src[0], Op->Flags, !Is128Bit);\n\n RefPair Result {};\n if (Shift == 0) [[unlikely]] {\n Result = Src;\n } else if (Shift >= Core::CPUState::XMM_SSE_REG_SIZE) {\n Result.Low = LoadZeroVector(OpSize::i128Bit);\n Result.High = Result.Low;\n } else {\n Ref ZeroVector = LoadZeroVector(OpSize::i128Bit);\n RefPair Zero {ZeroVector, ZeroVector};\n RefPair Src1 = Right ? Zero : Src;\n RefPair Src2 = Right ? Src : Zero;\n\n Result.Low = _VExtr(OpSize::i128Bit, OpSize::i8Bit, Src1.Low, Src2.Low, ExtrShift);\n if (!Is128Bit) {\n Result.High = _VExtr(OpSize::i128Bit, OpSize::i8Bit, Src1.High, Src2.High, ExtrShift);\n }\n }\n\n if (Is128Bit) {\n Result.High = LoadZeroVector(OpSize::i128Bit);\n }\n\n AVX128_StoreResult_WithOpSize(Op, Op->Dest, Result);\n}\n\nvoid OpDispatchBuilder::AVX128_VINSERT(OpcodeArgs) {\n const auto DstSize = GetDstSize(Op);\n const auto Is128Bit = DstSize == Core::CPUState::XMM_SSE_REG_SIZE;\n const auto Selector = Op->Src[2].Literal() & 1;\n\n auto Result = AVX128_LoadSource_WithOpSize(Op, Op->Src[0], Op->Flags, !Is128Bit);\n auto Src2 = AVX128_LoadSource_WithOpSize(Op, Op->Src[1], Op->Flags, false);\n\n if (Selector == 0) {\n // Insert in to Low bits\n Result.Low = Src2.Low;\n } else {\n // Insert in to the High bits\n Result.High = Src2.Low;\n }\n\n AVX128_StoreResult_WithOpSize(Op, Op->Dest, Result);\n}\n\nvoid OpDispatchBuilder::AVX128_VINSERTPS(OpcodeArgs) {\n Ref Result = InsertPSOpImpl(Op, Op->Src[0], Op->Src[1], Op->Src[2]);\n AVX128_StoreResult_WithOpSize(Op, Op->Dest, AVX128_Zext(Result));\n}\n\nvoid OpDispatchBuilder::AVX128_VPHSUB(OpcodeArgs, IR::OpSize ElementSize) {\n AVX128_VectorBinaryImpl(Op, OpSizeFromDst(Op), ElementSize, [this](IR::OpSize _ElementSize, Ref Src1, Ref Src2) {\n return PHSUBOpImpl(OpSize::i128Bit, Src1, Src2, _ElementSize);\n });\n}\n\nvoid OpDispatchBuilder::AVX128_VPHSUBSW(OpcodeArgs) {\n AVX128_VectorBinaryImpl(Op, OpSizeFromDst(Op), OpSize::i16Bit,\n [this](IR::OpSize _ElementSize, Ref Src1, Ref Src2) { return PHSUBSOpImpl(OpSize::i128Bit, Src1, Src2); });\n}\n\nvoid OpDispatchBuilder::AVX128_VADDSUBP(OpcodeArgs, IR::OpSize ElementSize) {\n AVX128_VectorBinaryImpl(Op, OpSizeFromDst(Op), ElementSize, [this](IR::OpSize _ElementSize, Ref Src1, Ref Src2) {\n return ADDSUBPOpImpl(OpSize::i128Bit, _ElementSize, Src1, Src2);\n });\n}\n\nvoid OpDispatchBuilder::AVX128_VPMULL(OpcodeArgs, IR::OpSize ElementSize, bool Signed) {\n LOGMAN_THROW_A_FMT(ElementSize == OpSize::i32Bit, \"Currently only handles 32-bit -> 64-bit\");\n\n AVX128_VectorBinaryImpl(Op, OpSizeFromDst(Op), ElementSize, [&](IR::OpSize _ElementSize, Ref Src1, Ref Src2) -> Ref {\n return PMULLOpImpl(OpSize::i128Bit, _ElementSize, Signed, Src1, Src2);\n });\n}\n\nvoid OpDispatchBuilder::AVX128_VPMULHRSW(OpcodeArgs) {\n AVX128_VectorBinaryImpl(Op, OpSizeFromDst(Op), OpSize::i16Bit,\n [this](IR::OpSize _ElementSize, Ref Src1, Ref Src2) -> Ref { return PMULHRSWOpImpl(OpSize::i128Bit, Src1, Src2); });\n}\n\nvoid OpDispatchBuilder::AVX128_VPMULHW(OpcodeArgs, bool Signed) {\n AVX128_VectorBinaryImpl(Op, OpSizeFromDst(Op), OpSize::i16Bit, [&](IR::OpSize _ElementSize, Ref Src1, Ref Src2) -> Ref {\n if (Signed) {\n return _VSMulH(OpSize::i128Bit, _ElementSize, Src1, Src2);\n } else {\n return _VUMulH(OpSize::i128Bit, _ElementSize, Src1, Src2);\n }\n });\n}\n\nvoid OpDispatchBuilder::AVX128_InsertScalar_CVT_Float_To_Float(OpcodeArgs, IR::OpSize DstElementSize, IR::OpSize SrcElementSize) {\n // Gotta be careful with this operation.\n // It inserts in to the lowest element, retaining the remainder of the lower 128-bits.\n // Then zero extends the top 128-bit.\n const auto SrcSize = Op->Src[1].IsGPR() ? OpSize::i128Bit : SrcElementSize;\n auto Src1 = AVX128_LoadSource_WithOpSize(Op, Op->Src[0], Op->Flags, false);\n Ref Src2 = LoadSourceFPR_WithOpSize(Op, Op->Src[1], SrcSize, Op->Flags, {.AllowUpperGarbage = true});\n\n Ref Result = _VFToFScalarInsert(OpSize::i128Bit, DstElementSize, SrcElementSize, Src1.Low, Src2, false);\n AVX128_StoreResult_WithOpSize(Op, Op->Dest, AVX128_Zext(Result));\n}\n\nvoid OpDispatchBuilder::AVX128_Vector_CVT_Float_To_Float(OpcodeArgs, IR::OpSize DstElementSize, IR::OpSize SrcElementSize) {\n const auto SrcSize = OpSizeFromSrc(Op);\n const auto DstSize = OpSizeFromDst(Op);\n\n const auto IsFloatSrc = SrcElementSize == OpSize::i32Bit;\n auto Is128BitSrc = SrcSize == OpSize::i128Bit;\n auto Is128BitDst = DstSize == OpSize::i128Bit;\n\n ///< Decompose correctly.\n if (DstElementSize > SrcElementSize && !Is128BitDst) {\n Is128BitSrc = true;\n } else if (SrcElementSize > DstElementSize && !Is128BitSrc) {\n Is128BitDst = true;\n }\n\n const auto LoadSize = IsFloatSrc && !Op->Src[0].IsGPR() ? IR::SizeToOpSize(IR::OpSizeToSize(SrcSize) / 2) : SrcSize;\n\n RefPair Src {};\n if (Op->Src[0].IsGPR() || LoadSize >= OpSize::i128Bit) {\n Src = AVX128_LoadSource_WithOpSize(Op, Op->Src[0], Op->Flags, !Is128BitSrc);\n } else {\n // Handle 64-bit memory source.\n // In the case of cvtps2pd xmm, m64.\n Src.Low = LoadSourceFPR_WithOpSize(Op, Op->Src[0], LoadSize, Op->Flags);\n }\n\n RefPair Result {};\n\n auto TransformLow = [&](Ref Src) -> Ref {\n return _Vector_FToF(OpSize::i128Bit, DstElementSize, Src, SrcElementSize);\n };\n\n auto TransformHigh = [&](Ref Src) -> Ref {\n return _VFCVTL2(OpSize::i128Bit, SrcElementSize, Src);\n };\n\n Result.Low = TransformLow(Src.Low);\n if (Is128BitSrc) {\n if (Is128BitDst) {\n // cvtps2pd xmm, xmm or cvtpd2ps xmm, xmm\n // Done here\n } else {\n LOGMAN_THROW_A_FMT(DstElementSize > SrcElementSize, \"cvtpd2ps ymm, xmm doesn't exist\");\n\n // cvtps2pd ymm, xmm\n Result.High = TransformHigh(Src.Low);\n }\n } else {\n // 256-bit src\n LOGMAN_THROW_A_FMT(Is128BitDst, \"Not real: cvt{ps2pd,pd2ps} ymm, ymm\");\n LOGMAN_THROW_A_FMT(DstElementSize < SrcElementSize, \"cvtps2pd xmm, ymm doesn't exist\");\n\n // cvtpd2ps xmm, ymm\n Result.Low = _VInsElement(OpSize::i128Bit, OpSize::i64Bit, 1, 0, Result.Low, TransformLow(Src.High));\n }\n\n if (Is128BitDst) {\n Result = AVX128_Zext(Result.Low);\n }\n\n AVX128_StoreResult_WithOpSize(Op, Op->Dest, Result);\n}\n\nvoid OpDispatchBuilder::AVX128_Vector_CVT_Float_To_Int(OpcodeArgs, IR::OpSize SrcElementSize, bool HostRoundingMode) {\n const auto SrcSize = GetSrcSize(Op);\n\n const auto Is128BitSrc = SrcSize == Core::CPUState::XMM_SSE_REG_SIZE;\n\n // VCVTPD2DQ/VCVTTPD2DQ only use the bottom lane, even for the 256-bit version.\n auto Src = AVX128_LoadSource_WithOpSize(Op, Op->Src[0], Op->Flags, !Is128BitSrc);\n RefPair Result {};\n\n Result.Low = Vector_CVT_Float_To_Int32Impl(Op, OpSize::i128Bit, Src.Low, OpSize::i128Bit, SrcElementSize, HostRoundingMode, Is128BitSrc);\n if (Is128BitSrc) {\n // Zero the upper 128-bit lane of the result.\n Result = AVX128_Zext(Result.Low);\n } else {\n Result.High = Vector_CVT_Float_To_Int32Impl(Op, OpSize::i128Bit, Src.High, OpSize::i128Bit, SrcElementSize, HostRoundingMode, false);\n // Also convert the upper 128-bit lane\n if (SrcElementSize == OpSize::i64Bit) {\n // Zip the two halves together in to the lower 128-bits\n Result.Low = _VZip(OpSize::i128Bit, OpSize::i64Bit, Result.Low, Result.High);\n\n // Zero the upper 128-bit lane of the result.\n Result = AVX128_Zext(Result.Low);\n }\n }\n\n AVX128_StoreResult_WithOpSize(Op, Op->Dest, Result);\n}\n\nvoid OpDispatchBuilder::AVX128_Vector_CVT_Int_To_Float(OpcodeArgs, IR::OpSize SrcElementSize, bool Widen) {\n const auto Size = OpSizeFromDst(Op);\n const auto Is128Bit = Size == OpSize::i128Bit;\n\n RefPair Src = [&] {\n if (Widen && !Op->Src[0].IsGPR()) {\n // If loading a vector, use the full size, so we don't\n // unnecessarily zero extend the vector. Otherwise, if\n // memory, then we want to load the element size exactly.\n const auto LoadSize = IR::SizeToOpSize(8 * (IR::OpSizeToSize(Size) / 16));\n return RefPair {.Low = LoadSourceFPR_WithOpSize(Op, Op->Src[0], LoadSize, Op->Flags)};\n } else {\n return AVX128_LoadSource_WithOpSize(Op, Op->Src[0], Op->Flags, !Is128Bit);\n }\n }();\n\n auto Convert = [&](Ref Src, IROps Op) -> Ref {\n auto ElementSize = SrcElementSize;\n if (Widen) {\n DeriveOp(Extended, Op, _VSXTL(OpSize::i128Bit, ElementSize, Src));\n Src = Extended;\n ElementSize = ElementSize << 1;\n }\n\n return _Vector_SToF(OpSize::i128Bit, ElementSize, Src);\n };\n\n RefPair Result {};\n Result.Low = Convert(Src.Low, IROps::OP_VSXTL);\n\n if (Is128Bit) {\n Result = AVX128_Zext(Result.Low);\n } else {\n if (Widen) {\n Result.High = Convert(Src.Low, IROps::OP_VSXTL2);\n } else {\n Result.High = Convert(Src.High, IROps::OP_VSXTL);\n }\n }\n\n AVX128_StoreResult_WithOpSize(Op, Op->Dest, Result);\n}\n\nvoid OpDispatchBuilder::AVX128_VEXTRACT128(OpcodeArgs) {\n const auto DstIsXMM = Op->Dest.IsGPR();\n const auto Selector = Op->Src[1].Literal() & 0b1;\n\n auto Src = AVX128_LoadSource_WithOpSize(Op, Op->Src[0], Op->Flags, true);\n\n RefPair Result {};\n if (Selector == 0) {\n Result.Low = Src.Low;\n } else {\n Result.Low = Src.High;\n }\n\n if (DstIsXMM) {\n // Only zero the upper-half when destination is XMM, otherwise this is a memory store.\n Result = AVX128_Zext(Result.Low);\n }\n\n AVX128_StoreResult_WithOpSize(Op, Op->Dest, Result);\n}\n\nvoid OpDispatchBuilder::AVX128_VAESImc(OpcodeArgs) {\n ///< 128-bit only.\n AVX128_VectorUnaryImpl(Op, OpSize::i128Bit, OpSize::i128Bit, [this](IR::OpSize, Ref Src) { return _VAESImc(Src); });\n}\n\nvoid OpDispatchBuilder::AVX128_VAESEnc(OpcodeArgs) {\n AVX128_VectorTrinaryImpl(Op, OpSizeFromDst(Op), OpSize::i128Bit, LoadZeroVector(OpSize::i128Bit),\n [this](IR::OpSize, Ref Src1, Ref Src2, Ref Src3) { return _VAESEnc(OpSize::i128Bit, Src1, Src2, Src3); });\n}\n\nvoid OpDispatchBuilder::AVX128_VAESEncLast(OpcodeArgs) {\n AVX128_VectorTrinaryImpl(Op, OpSizeFromDst(Op), OpSize::i128Bit, LoadZeroVector(OpSize::i128Bit),\n [this](IR::OpSize, Ref Src1, Ref Src2, Ref Src3) { return _VAESEncLast(OpSize::i128Bit, Src1, Src2, Src3); });\n}\n\nvoid OpDispatchBuilder::AVX128_VAESDec(OpcodeArgs) {\n AVX128_VectorTrinaryImpl(Op, OpSizeFromDst(Op), OpSize::i128Bit, LoadZeroVector(OpSize::i128Bit),\n [this](IR::OpSize, Ref Src1, Ref Src2, Ref Src3) { return _VAESDec(OpSize::i128Bit, Src1, Src2, Src3); });\n}\n\nvoid OpDispatchBuilder::AVX128_VAESDecLast(OpcodeArgs) {\n AVX128_VectorTrinaryImpl(Op, OpSizeFromDst(Op), OpSize::i128Bit, LoadZeroVector(OpSize::i128Bit),\n [this](IR::OpSize, Ref Src1, Ref Src2, Ref Src3) { return _VAESDecLast(OpSize::i128Bit, Src1, Src2, Src3); });\n}\n\nvoid OpDispatchBuilder::AVX128_VAESKeyGenAssist(OpcodeArgs) {\n ///< 128-bit only.\n const uint64_t RCON = Op->Src[1].Literal();\n auto ZeroRegister = LoadZeroVector(OpSize::i128Bit);\n auto KeyGenSwizzle = LoadAndCacheNamedVectorConstant(OpSize::i128Bit, NAMED_VECTOR_AESKEYGENASSIST_SWIZZLE);\n\n struct {\n Ref ZeroRegister;\n Ref KeyGenSwizzle;\n uint64_t RCON;\n } Capture {\n .ZeroRegister = ZeroRegister,\n .KeyGenSwizzle = KeyGenSwizzle,\n .RCON = RCON,\n };\n\n AVX128_VectorUnaryImpl(Op, OpSize::i128Bit, OpSize::i128Bit, [this, &Capture](IR::OpSize, Ref Src) {\n return _VAESKeyGenAssist(Src, Capture.KeyGenSwizzle, Capture.ZeroRegister, Capture.RCON);\n });\n}\n\nvoid OpDispatchBuilder::AVX128_VPCMPESTRI(OpcodeArgs) {\n PCMPXSTRXOpImpl(Op, true, false);\n\n ///< Does not zero anything.\n}\n\nvoid OpDispatchBuilder::AVX128_VPCMPESTRM(OpcodeArgs) {\n PCMPXSTRXOpImpl(Op, true, true);\n\n ///< Zero the upper 128-bits of hardcoded YMM0\n AVX128_StoreXMMRegister(0, LoadZeroVector(OpSize::i128Bit), true);\n}\n\nvoid OpDispatchBuilder::AVX128_VPCMPISTRI(OpcodeArgs) {\n PCMPXSTRXOpImpl(Op, false, false);\n\n ///< Does not zero anything.\n}\n\nvoid OpDispatchBuilder::AVX128_VPCMPISTRM(OpcodeArgs) {\n PCMPXSTRXOpImpl(Op, false, true);\n\n ///< Zero the upper 128-bits of hardcoded YMM0\n AVX128_StoreXMMRegister(0, LoadZeroVector(OpSize::i128Bit), true);\n}\n\nvoid OpDispatchBuilder::AVX128_PHMINPOSUW(OpcodeArgs) {\n Ref Result = PHMINPOSUWOpImpl(Op);\n AVX128_StoreResult_WithOpSize(Op, Op->Dest, AVX128_Zext(Result));\n}\n\nvoid OpDispatchBuilder::AVX128_VectorRound(OpcodeArgs, IR::OpSize ElementSize) {\n const auto Size = OpSizeFromSrc(Op);\n const auto Mode = Op->Src[1].Literal();\n\n AVX128_VectorUnaryImpl(Op, Size, ElementSize,\n [this, Mode](IR::OpSize ElementSize, Ref Src) { return VectorRoundImpl(OpSize::i128Bit, ElementSize, Src, Mode); });\n}\n\nvoid OpDispatchBuilder::AVX128_InsertScalarRound(OpcodeArgs, IR::OpSize ElementSize) {\n // We load the full vector width when dealing with a source vector,\n // so that we don't do any unnecessary zero extension to the scalar\n // element that we're going to operate on.\n const auto SrcSize = OpSizeFromSrc(Op);\n\n auto Src1 = AVX128_LoadSource_WithOpSize(Op, Op->Src[0], Op->Flags, false);\n RefPair Src2 {};\n if (Op->Src[1].IsGPR()) {\n Src2 = AVX128_LoadSource_WithOpSize(Op, Op->Src[1], Op->Flags, false);\n } else {\n Src2.Low = LoadSourceFPR_WithOpSize(Op, Op->Src[1], SrcSize, Op->Flags);\n }\n\n // If OpSize == ElementSize then it only does the lower scalar op\n const auto SourceMode = TranslateRoundType(Op->Src[2].Literal());\n\n Ref Result = _VFToIScalarInsert(OpSize::i128Bit, ElementSize, Src1.Low, Src2.Low, SourceMode, false);\n AVX128_StoreResult_WithOpSize(Op, Op->Dest, AVX128_Zext(Result));\n}\n\nvoid OpDispatchBuilder::AVX128_VDPP(OpcodeArgs, IR::OpSize ElementSize) {\n const uint64_t Literal = Op->Src[2].Literal();\n\n AVX128_VectorBinaryImpl(Op, OpSizeFromSrc(Op), ElementSize, [this, Literal](IR::OpSize ElementSize, Ref Src1, Ref Src2) {\n return DPPOpImpl(OpSize::i128Bit, Src1, Src2, Literal, ElementSize);\n });\n}\n\nvoid OpDispatchBuilder::AVX128_VPERMQ(OpcodeArgs) {\n ///< Only ever 256-bit.\n auto Src = AVX128_LoadSource_WithOpSize(Op, Op->Src[0], Op->Flags, true);\n const auto Selector = Op->Src[1].Literal();\n\n RefPair Result {};\n\n // Crack the operation in to two halves and implement per half\n uint8_t SelectorLow = Selector & 0b1111;\n uint8_t SelectorHigh = (Selector >> 4) & 0b1111;\n auto SelectLane = [this](uint8_t Selector, RefPair Src) -> Ref {\n LOGMAN_THROW_A_FMT(Selector < 16, \"Selector too large!\");\n\n switch (Selector) {\n case 0b00'00: return _VDupElement(OpSize::i128Bit, OpSize::i64Bit, Src.Low, 0);\n case 0b00'01: return _VExtr(OpSize::i128Bit, OpSize::i8Bit, Src.Low, Src.Low, 8);\n case 0b00'10: return _VZip(OpSize::i128Bit, OpSize::i64Bit, Src.High, Src.Low);\n case 0b00'11: return _VExtr(OpSize::i128Bit, OpSize::i8Bit, Src.Low, Src.High, 8);\n case 0b01'00: return Src.Low;\n case 0b01'01: return _VDupElement(OpSize::i128Bit, OpSize::i64Bit, Src.Low, 1);\n case 0b01'10: return _VInsElement(OpSize::i128Bit, OpSize::i64Bit, 1, 1, Src.High, Src.Low);\n case 0b01'11: return _VTrn2(OpSize::i128Bit, OpSize::i64Bit, Src.High, Src.Low);\n case 0b10'00: return _VZip(OpSize::i128Bit, OpSize::i64Bit, Src.Low, Src.High);\n case 0b10'01: return _VExtr(OpSize::i128Bit, OpSize::i8Bit, Src.High, Src.Low, 8);\n case 0b10'10: return _VDupElement(OpSize::i128Bit, OpSize::i64Bit, Src.High, 0);\n case 0b10'11: return _VExtr(OpSize::i128Bit, OpSize::i8Bit, Src.High, Src.High, 8);\n case 0b11'00: return _VInsElement(OpSize::i128Bit, OpSize::i64Bit, 1, 1, Src.Low, Src.High);\n case 0b11'01: return _VTrn2(OpSize::i128Bit, OpSize::i64Bit, Src.Low, Src.High);\n case 0b11'10: return Src.High;\n case 0b11'11: return _VDupElement(OpSize::i128Bit, OpSize::i64Bit, Src.High, 1);\n default: FEX_UNREACHABLE;\n }\n };\n\n Result.Low = SelectLane(SelectorLow, Src);\n Result.High = SelectorLow == SelectorHigh ? Result.Low : SelectLane(SelectorHigh, Src);\n\n AVX128_StoreResult_WithOpSize(Op, Op->Dest, Result);\n}\n\nvoid OpDispatchBuilder::AVX128_VPSHUFW(OpcodeArgs, bool Low) {\n auto Shuffle = Op->Src[1].Literal();\n\n struct DataPacking {\n OpDispatchBuilder* This;\n uint8_t Shuffle;\n bool Low;\n };\n\n DataPacking Pack {\n .This = this,\n .Shuffle = static_cast(Shuffle),\n .Low = Low,\n };\n\n AVX128_VectorUnaryImpl(Op, OpSizeFromSrc(Op), OpSize::i16Bit, [Pack](IR::OpSize, Ref Src) {\n const auto IndexedVectorConstant = Pack.Low ? FEXCore::IR::IndexNamedVectorConstant::INDEXED_NAMED_VECTOR_PSHUFLW :\n FEXCore::IR::IndexNamedVectorConstant::INDEXED_NAMED_VECTOR_PSHUFHW;\n\n return Pack.This->PShufWLane(OpSize::i128Bit, IndexedVectorConstant, Pack.Low, Src, Pack.Shuffle);\n });\n}\n\nvoid OpDispatchBuilder::AVX128_VSHUF(OpcodeArgs, IR::OpSize ElementSize) {\n const auto SrcSize = GetSrcSize(Op);\n const auto Is128Bit = SrcSize == Core::CPUState::XMM_SSE_REG_SIZE;\n auto Shuffle = Op->Src[2].Literal();\n\n auto Src1 = AVX128_LoadSource_WithOpSize(Op, Op->Src[0], Op->Flags, !Is128Bit);\n auto Src2 = AVX128_LoadSource_WithOpSize(Op, Op->Src[1], Op->Flags, !Is128Bit);\n\n RefPair Result {};\n Result.Low = SHUFOpImpl(Op, OpSize::i128Bit, ElementSize, Src1.Low, Src2.Low, Shuffle);\n\n if (Is128Bit) {\n Result.High = LoadZeroVector(OpSize::i128Bit);\n } else {\n const uint8_t ShiftAmount = ElementSize == OpSize::i32Bit ? 0 : 2;\n Result.High = SHUFOpImpl(Op, OpSize::i128Bit, ElementSize, Src1.High, Src2.High, Shuffle >> ShiftAmount);\n }\n AVX128_StoreResult_WithOpSize(Op, Op->Dest, Result);\n}\n\nvoid OpDispatchBuilder::AVX128_VPERMILImm(OpcodeArgs, IR::OpSize ElementSize) {\n const auto SrcSize = GetSrcSize(Op);\n const auto Is128Bit = SrcSize == Core::CPUState::XMM_SSE_REG_SIZE;\n\n const auto Selector = Op->Src[1].Literal() & 0xFF;\n auto Src = AVX128_LoadSource_WithOpSize(Op, Op->Src[0], Op->Flags, !Is128Bit);\n\n RefPair Result = AVX128_Zext(LoadZeroVector(OpSize::i128Bit));\n\n if (ElementSize == OpSize::i64Bit) {\n auto DoSwizzle64 = [this](Ref Src, uint8_t Selector) -> Ref {\n switch (Selector) {\n case 0b00:\n case 0b11: return _VDupElement(OpSize::i128Bit, OpSize::i64Bit, Src, Selector & 1);\n case 0b01: return _VExtr(OpSize::i128Bit, OpSize::i8Bit, Src, Src, 8);\n case 0b10:\n // No swizzle\n return Src;\n default: FEX_UNREACHABLE;\n }\n };\n Result.Low = DoSwizzle64(Src.Low, Selector & 0b11);\n\n if (!Is128Bit) {\n Result.High = DoSwizzle64(Src.High, (Selector >> 2) & 0b11);\n }\n } else {\n Result.Low = Single128Bit4ByteVectorShuffle(Src.Low, Selector);\n\n if (!Is128Bit) {\n Result.High = Single128Bit4ByteVectorShuffle(Src.High, Selector);\n }\n }\n\n AVX128_StoreResult_WithOpSize(Op, Op->Dest, Result);\n}\n\nvoid OpDispatchBuilder::AVX128_VHADDP(OpcodeArgs, IROps IROp, IR::OpSize ElementSize) {\n AVX128_VectorBinaryImpl(Op, OpSizeFromSrc(Op), ElementSize, [&](IR::OpSize ElementSize, Ref Src1, Ref Src2) {\n DeriveOp(Res, IROp, _VFAddP(OpSize::i128Bit, ElementSize, Src1, Src2));\n return Res;\n });\n}\n\nvoid OpDispatchBuilder::AVX128_VPHADDSW(OpcodeArgs) {\n AVX128_VectorBinaryImpl(Op, OpSizeFromDst(Op), OpSize::i16Bit,\n [this](IR::OpSize _ElementSize, Ref Src1, Ref Src2) { return PHADDSOpImpl(OpSize::i128Bit, Src1, Src2); });\n}\n\nvoid OpDispatchBuilder::AVX128_VPMADDUBSW(OpcodeArgs) {\n AVX128_VectorBinaryImpl(Op, OpSizeFromDst(Op), OpSize::i128Bit,\n [this](IR::OpSize _ElementSize, Ref Src1, Ref Src2) { return PMADDUBSWOpImpl(OpSize::i128Bit, Src1, Src2); });\n}\n\nvoid OpDispatchBuilder::AVX128_VPMADDWD(OpcodeArgs) {\n AVX128_VectorBinaryImpl(Op, OpSizeFromDst(Op), OpSize::i128Bit,\n [this](IR::OpSize _ElementSize, Ref Src1, Ref Src2) { return PMADDWDOpImpl(OpSize::i128Bit, Src1, Src2); });\n}\n\nvoid OpDispatchBuilder::AVX128_VBLEND(OpcodeArgs, IR::OpSize ElementSize) {\n const auto SrcSize = OpSizeFromSrc(Op);\n const auto Is128Bit = SrcSize == OpSize::i128Bit;\n const uint64_t Selector = Op->Src[2].Literal();\n\n ///< High Selector shift depends on element size:\n /// i16Bit: Reuses same bits, no shift\n /// i32Bit: Shift by 4\n /// i64Bit: Shift by 2\n const uint64_t SelectorShift = ElementSize == OpSize::i64Bit ? 2 : ElementSize == OpSize::i32Bit ? 4 : 0;\n\n auto Src1 = AVX128_LoadSource_WithOpSize(Op, Op->Src[0], Op->Flags, !Is128Bit);\n auto Src2 = AVX128_LoadSource_WithOpSize(Op, Op->Src[1], Op->Flags, !Is128Bit);\n\n RefPair Result {};\n Result.Low = VectorBlend(OpSize::i128Bit, ElementSize, Src1.Low, Src2.Low, Selector);\n\n if (Is128Bit) {\n Result = AVX128_Zext(Result.Low);\n } else {\n Result.High = VectorBlend(OpSize::i128Bit, ElementSize, Src1.High, Src2.High, (Selector >> SelectorShift));\n }\n\n AVX128_StoreResult_WithOpSize(Op, Op->Dest, Result);\n}\n\nvoid OpDispatchBuilder::AVX128_VHSUBP(OpcodeArgs, IR::OpSize ElementSize) {\n AVX128_VectorBinaryImpl(Op, OpSizeFromDst(Op), ElementSize,\n [&](IR::OpSize, Ref Src1, Ref Src2) { return HSUBPOpImpl(OpSize::i128Bit, ElementSize, Src1, Src2); });\n}\n\nvoid OpDispatchBuilder::AVX128_VPSHUFB(OpcodeArgs) {\n auto MaskVector = GeneratePSHUFBMask(OpSize::i128Bit);\n AVX128_VectorBinaryImpl(Op, OpSizeFromDst(Op), OpSize::i8Bit, [this, MaskVector](IR::OpSize, Ref Src1, Ref Src2) {\n return PSHUFBOpImpl(OpSize::i128Bit, Src1, Src2, MaskVector);\n });\n}\n\nvoid OpDispatchBuilder::AVX128_VPSADBW(OpcodeArgs) {\n AVX128_VectorBinaryImpl(Op, OpSizeFromDst(Op), OpSize::i8Bit,\n [this](IR::OpSize, Ref Src1, Ref Src2) { return PSADBWOpImpl(OpSize::i128Bit, Src1, Src2); });\n}\n\nvoid OpDispatchBuilder::AVX128_VMPSADBW(OpcodeArgs) {\n const auto SrcSize = GetSrcSize(Op);\n const auto Is128Bit = SrcSize == Core::CPUState::XMM_SSE_REG_SIZE;\n const uint64_t Selector = Op->Src[2].Literal();\n\n auto Src1 = AVX128_LoadSource_WithOpSize(Op, Op->Src[0], Op->Flags, !Is128Bit);\n auto Src2 = AVX128_LoadSource_WithOpSize(Op, Op->Src[1], Op->Flags, !Is128Bit);\n\n RefPair Result {};\n auto ZeroRegister = LoadZeroVector(OpSize::i128Bit);\n\n Result.Low = MPSADBWOpImpl(OpSize::i128Bit, Src1.Low, Src2.Low, Selector);\n\n if (Is128Bit) {\n Result.High = ZeroRegister;\n } else {\n Result.High = MPSADBWOpImpl(OpSize::i128Bit, Src1.High, Src2.High, Selector >> 3);\n }\n\n AVX128_StoreResult_WithOpSize(Op, Op->Dest, Result);\n}\n\nvoid OpDispatchBuilder::AVX128_VPALIGNR(OpcodeArgs) {\n const auto Index = Op->Src[2].Literal();\n const auto Size = OpSizeFromDst(Op);\n const auto SanitizedDstSize = std::min(Size, OpSize::i128Bit);\n\n AVX128_VectorBinaryImpl(Op, Size, SanitizedDstSize, [this, Index](IR::OpSize SanitizedDstSize, Ref Src1, Ref Src2) -> Ref {\n if (Index >= (IR::OpSizeToSize(SanitizedDstSize) * 2)) {\n // If the immediate is greater than both vectors combined then it zeroes the vector\n return LoadZeroVector(OpSize::i128Bit);\n }\n\n if (Index == 0) {\n return Src2;\n }\n\n if (Index == 16) {\n return Src1;\n }\n\n auto SanitizedIndex = Index;\n if (Index > 16) {\n Src2 = Src1;\n Src1 = LoadZeroVector(OpSize::i128Bit);\n SanitizedIndex -= 16;\n }\n\n return _VExtr(OpSize::i128Bit, OpSize::i8Bit, Src1, Src2, SanitizedIndex);\n });\n}\n\nvoid OpDispatchBuilder::AVX128_VMASKMOVImpl(OpcodeArgs, IR::OpSize ElementSize, IR::OpSize DstSize, bool IsStore,\n const X86Tables::DecodedOperand& MaskOp, const X86Tables::DecodedOperand& DataOp) {\n const auto Is128Bit = DstSize == OpSize::i128Bit;\n\n auto Mask = AVX128_LoadSource_WithOpSize(Op, MaskOp, Op->Flags, !Is128Bit);\n\n const auto MakeAddress = [this, Op](const X86Tables::DecodedOperand& Data) {\n return MakeSegmentAddress(Op, Data, GetGPROpSize());\n };\n\n if (IsStore) {\n auto Address = MakeAddress(Op->Dest);\n\n auto Data = AVX128_LoadSource_WithOpSize(Op, DataOp, Op->Flags, !Is128Bit);\n _VStoreVectorMasked(OpSize::i128Bit, ElementSize, Mask.Low, Data.Low, Address, Invalid(), MemOffsetType::SXTX, 1);\n if (!Is128Bit) {\n _VStoreVectorMasked(OpSize::i128Bit, ElementSize, Mask.High, Data.High, Address, _InlineConstant(16), MemOffsetType::SXTX, 1);\n }\n } else {\n auto Address = MakeAddress(DataOp);\n\n RefPair Result {};\n Result.Low = _VLoadVectorMasked(OpSize::i128Bit, ElementSize, Mask.Low, Address, Invalid(), MemOffsetType::SXTX, 1);\n\n if (Is128Bit) {\n Result.High = LoadZeroVector(OpSize::i128Bit);\n } else {\n Result.High = _VLoadVectorMasked(OpSize::i128Bit, ElementSize, Mask.High, Address, _InlineConstant(16), MemOffsetType::SXTX, 1);\n }\n AVX128_StoreResult_WithOpSize(Op, Op->Dest, Result);\n }\n}\n\nvoid OpDispatchBuilder::AVX128_VPMASKMOV(OpcodeArgs, bool IsStore) {\n AVX128_VMASKMOVImpl(Op, OpSizeFromSrc(Op), OpSizeFromDst(Op), IsStore, Op->Src[0], Op->Src[1]);\n}\n\nvoid OpDispatchBuilder::AVX128_VMASKMOV(OpcodeArgs, IR::OpSize ElementSize, bool IsStore) {\n AVX128_VMASKMOVImpl(Op, ElementSize, OpSizeFromDst(Op), IsStore, Op->Src[0], Op->Src[1]);\n}\n\nvoid OpDispatchBuilder::AVX128_MASKMOV(OpcodeArgs) {\n ///< This instruction only supports 128-bit.\n const auto Size = OpSizeFromSrc(Op);\n const auto Is128Bit = Size == OpSize::i128Bit;\n\n auto MaskSrc = AVX128_LoadSource_WithOpSize(Op, Op->Src[0], Op->Flags, !Is128Bit);\n\n // Mask only cares about the top bit of each byte\n MaskSrc.Low = _VCMPLTZ(Size, OpSize::i8Bit, MaskSrc.Low);\n\n // Vector that will overwrite byte elements.\n auto VectorSrc = AVX128_LoadSource_WithOpSize(Op, Op->Dest, Op->Flags, !Is128Bit);\n\n // RDI source (DS prefix by default)\n auto MemDest = MakeSegmentAddress(X86State::REG_RDI, Op->Flags, X86Tables::DecodeFlags::FLAG_DS_PREFIX);\n\n Ref XMMReg = _LoadMemFPR(Size, MemDest, OpSize::i8Bit);\n\n // If the Mask element high bit is set then overwrite the element with the source, else keep the memory variant\n XMMReg = _VBSL(Size, MaskSrc.Low, VectorSrc.Low, XMMReg);\n _StoreMemFPR(Size, MemDest, XMMReg, OpSize::i8Bit);\n}\n\nvoid OpDispatchBuilder::AVX128_VectorVariableBlend(OpcodeArgs, IR::OpSize ElementSize) {\n const auto Size = OpSizeFromSrc(Op);\n const auto Is128Bit = Size == OpSize::i128Bit;\n const auto Src3Selector = Op->Src[2].Literal();\n\n auto Src1 = AVX128_LoadSource_WithOpSize(Op, Op->Src[0], Op->Flags, !Is128Bit);\n auto Src2 = AVX128_LoadSource_WithOpSize(Op, Op->Src[1], Op->Flags, !Is128Bit);\n\n uint8_t MaskRegister = (Src3Selector >> 4) & 0b1111;\n RefPair Mask {.Low = AVX128_LoadXMMRegister(MaskRegister, false)};\n\n if (!Is128Bit) {\n Mask.High = AVX128_LoadXMMRegister(MaskRegister, true);\n }\n\n auto Convert = [&](Ref Src1, Ref Src2, Ref Mask) {\n const auto ElementSizeBits = IR::OpSizeAsBits(ElementSize);\n Ref Shifted = _VSShrI(OpSize::i128Bit, ElementSize, Mask, ElementSizeBits - 1);\n return _VBSL(OpSize::i128Bit, Shifted, Src2, Src1);\n };\n\n RefPair Result {};\n Result.Low = Convert(Src1.Low, Src2.Low, Mask.Low);\n if (!Is128Bit) {\n Result.High = Convert(Src1.High, Src2.High, Mask.High);\n } else {\n Result = AVX128_Zext(Result.Low);\n }\n\n AVX128_StoreResult_WithOpSize(Op, Op->Dest, Result);\n}\n\nvoid OpDispatchBuilder::AVX128_SaveAVXState(Ref MemBase) {\n const auto NumRegs = Is64BitMode ? 16U : 8U;\n\n for (uint32_t i = 0; i < NumRegs; i += 2) {\n RefPair Pair = LoadContextPair(OpSize::i128Bit, AVXHigh0Index + i);\n _StoreMemPairFPR(OpSize::i128Bit, Pair.Low, Pair.High, MemBase, i * 16 + 576);\n }\n}\n\nvoid OpDispatchBuilder::AVX128_RestoreAVXState(Ref MemBase) {\n const auto NumRegs = Is64BitMode ? 16U : 8U;\n\n for (uint32_t i = 0; i < NumRegs; i += 2) {\n auto YMMHRegs = LoadMemPairFPR(OpSize::i128Bit, MemBase, i * 16 + 576);\n\n AVX128_StoreXMMRegister(i, YMMHRegs.Low, true);\n AVX128_StoreXMMRegister(i + 1, YMMHRegs.High, true);\n }\n}\n\nvoid OpDispatchBuilder::AVX128_DefaultAVXState() {\n const auto NumRegs = Is64BitMode ? 16U : 8U;\n\n auto ZeroRegister = LoadZeroVector(OpSize::i128Bit);\n for (uint32_t i = 0; i < NumRegs; i++) {\n AVX128_StoreXMMRegister(i, ZeroRegister, true);\n }\n}\n\nvoid OpDispatchBuilder::AVX128_VPERM2(OpcodeArgs) {\n auto Src1 = AVX128_LoadSource_WithOpSize(Op, Op->Src[0], Op->Flags, true);\n auto Src2 = AVX128_LoadSource_WithOpSize(Op, Op->Src[1], Op->Flags, true);\n const auto Selector = Op->Src[2].Literal();\n\n RefPair Result = AVX128_Zext(LoadZeroVector(OpSize::i128Bit));\n Ref Elements[4] = {Src1.Low, Src1.High, Src2.Low, Src2.High};\n\n if ((Selector & 0b00001000) == 0) {\n Result.Low = Elements[Selector & 0b11];\n }\n\n if ((Selector & 0b10000000) == 0) {\n Result.High = Elements[(Selector >> 4) & 0b11];\n }\n\n AVX128_StoreResult_WithOpSize(Op, Op->Dest, Result);\n}\n\nvoid OpDispatchBuilder::AVX128_VTESTP(OpcodeArgs, IR::OpSize ElementSize) {\n const auto Size = GetSrcSize(Op);\n const auto Is128Bit = Size == Core::CPUState::XMM_SSE_REG_SIZE;\n\n auto Src1 = AVX128_LoadSource_WithOpSize(Op, Op->Dest, Op->Flags, !Is128Bit);\n auto Src2 = AVX128_LoadSource_WithOpSize(Op, Op->Src[0], Op->Flags, !Is128Bit);\n\n // For 128-bit, we use the common path.\n if (Is128Bit) {\n VTESTOpImpl(OpSize::i128Bit, ElementSize, Src1.Low, Src2.Low);\n return;\n }\n\n // For 256-bit, we need to split up the operation. This is nontrivial.\n // Let's go the simple route here.\n Ref ZF, CFInv;\n\n const auto ElementSizeInBits = IR::OpSizeAsBits(ElementSize);\n\n {\n // Calculate ZF first.\n auto AndLow = _VAnd(OpSize::i128Bit, OpSize::i8Bit, Src2.Low, Src1.Low);\n auto AndHigh = _VAnd(OpSize::i128Bit, OpSize::i8Bit, Src2.High, Src1.High);\n\n auto ShiftLow = _VUShrI(OpSize::i128Bit, ElementSize, AndLow, ElementSizeInBits - 1);\n auto ShiftHigh = _VUShrI(OpSize::i128Bit, ElementSize, AndHigh, ElementSizeInBits - 1);\n // Only have the signs now, add it all\n auto AddResult = _VAdd(OpSize::i128Bit, ElementSize, ShiftHigh, ShiftLow);\n Ref AddWide {};\n if (ElementSize == OpSize::i32Bit) {\n AddWide = _VAddV(OpSize::i128Bit, ElementSize, AddResult);\n } else {\n AddWide = _VAddP(OpSize::i128Bit, ElementSize, AddResult, AddResult);\n }\n\n // ExtGPR will either be [0, 8] or [0, 16] If 0 then set Flag.\n ZF = _VExtractToGPR(OpSize::i128Bit, ElementSize, AddWide, 0);\n }\n\n {\n // Calculate CF Second\n auto AndLow = _VAndn(OpSize::i128Bit, OpSize::i8Bit, Src2.Low, Src1.Low);\n auto AndHigh = _VAndn(OpSize::i128Bit, OpSize::i8Bit, Src2.High, Src1.High);\n\n auto ShiftLow = _VUShrI(OpSize::i128Bit, ElementSize, AndLow, ElementSizeInBits - 1);\n auto ShiftHigh = _VUShrI(OpSize::i128Bit, ElementSize, AndHigh, ElementSizeInBits - 1);\n // Only have the signs now, add it all\n auto AddResult = _VAdd(OpSize::i128Bit, ElementSize, ShiftHigh, ShiftLow);\n Ref AddWide {};\n if (ElementSize == OpSize::i32Bit) {\n AddWide = _VAddV(OpSize::i128Bit, ElementSize, AddResult);\n } else {\n AddWide = _VAddP(OpSize::i128Bit, ElementSize, AddResult, AddResult);\n }\n\n // ExtGPR will either be [0, 8] or [0, 16] If 0 then set Flag.\n auto ExtGPR = _VExtractToGPR(OpSize::i128Bit, ElementSize, AddWide, 0);\n CFInv = To01(OpSize::i64Bit, ExtGPR);\n }\n\n // As in PTest, this sets Z appropriately while zeroing the rest of NZCV.\n SetNZ_ZeroCV(OpSize::i32Bit, ZF);\n SetCFInverted(CFInv);\n ZeroPF_AF();\n}\n\nvoid OpDispatchBuilder::AVX128_PTest(OpcodeArgs) {\n const auto Size = GetSrcSize(Op);\n const auto Is128Bit = Size == Core::CPUState::XMM_SSE_REG_SIZE;\n\n auto Src1 = AVX128_LoadSource_WithOpSize(Op, Op->Dest, Op->Flags, !Is128Bit);\n auto Src2 = AVX128_LoadSource_WithOpSize(Op, Op->Src[0], Op->Flags, !Is128Bit);\n\n // For 128-bit, use the common path.\n if (Is128Bit) {\n PTestOpImpl(OpSize::i128Bit, Src1.Low, Src2.Low);\n return;\n }\n\n // For 256-bit, we need to unroll. This is nontrivial.\n Ref Test1Low = _VAnd(OpSize::i128Bit, OpSize::i8Bit, Src1.Low, Src2.Low);\n Ref Test2Low = _VAndn(OpSize::i128Bit, OpSize::i8Bit, Src2.Low, Src1.Low);\n\n Ref Test1High = _VAnd(OpSize::i128Bit, OpSize::i8Bit, Src1.High, Src2.High);\n Ref Test2High = _VAndn(OpSize::i128Bit, OpSize::i8Bit, Src2.High, Src1.High);\n\n // Element size must be less than 32-bit for the sign bit tricks.\n Ref Test1Max = _VUMax(OpSize::i128Bit, OpSize::i16Bit, Test1Low, Test1High);\n Ref Test2Max = _VUMax(OpSize::i128Bit, OpSize::i16Bit, Test2Low, Test2High);\n\n Ref Test1 = _VUMaxV(OpSize::i128Bit, OpSize::i16Bit, Test1Max);\n Ref Test2 = _VUMaxV(OpSize::i128Bit, OpSize::i16Bit, Test2Max);\n\n Test1 = _VExtractToGPR(OpSize::i128Bit, OpSize::i16Bit, Test1, 0);\n Test2 = _VExtractToGPR(OpSize::i128Bit, OpSize::i16Bit, Test2, 0);\n\n Test2 = To01(OpSize::i64Bit, Test2);\n\n // Careful, these flags are different between {V,}PTEST and VTESTP{S,D}\n // Set ZF according to Test1. SF will be zeroed since we do a 32-bit test on\n // the results of a 16-bit value from the UMaxV, so the 32-bit sign bit is\n // cleared even if the 16-bit scalars were negative.\n SetNZ_ZeroCV(OpSize::i32Bit, Test1);\n SetCFInverted(Test2);\n ZeroPF_AF();\n}\n\nvoid OpDispatchBuilder::AVX128_VPERMILReg(OpcodeArgs, IR::OpSize ElementSize) {\n AVX128_VectorBinaryImpl(Op, OpSizeFromSrc(Op), ElementSize, [this](IR::OpSize _ElementSize, Ref Src, Ref Indices) {\n return VPERMILRegOpImpl(OpSize::i128Bit, _ElementSize, Src, Indices);\n });\n}\n\nvoid OpDispatchBuilder::AVX128_VPERMD(OpcodeArgs) {\n // Only 256-bit\n auto Indices = AVX128_LoadSource_WithOpSize(Op, Op->Src[0], Op->Flags, true);\n auto Src = AVX128_LoadSource_WithOpSize(Op, Op->Src[1], Op->Flags, true);\n\n auto DoPerm = [this](RefPair Src, Ref Indices, Ref IndexMask, Ref AddVector) {\n Ref FinalIndices = VPERMDIndices(OpSize::i128Bit, Indices, IndexMask, AddVector);\n return _VTBL2(OpSize::i128Bit, Src.Low, Src.High, FinalIndices);\n };\n\n RefPair Result {};\n\n Ref IndexMask = _VectorImm(OpSize::i128Bit, OpSize::i32Bit, 0b111);\n Ref AddConst = Constant(0x03020100);\n Ref Repeating3210 = _VDupFromGPR(OpSize::i128Bit, OpSize::i32Bit, AddConst);\n\n Result.Low = DoPerm(Src, Indices.Low, IndexMask, Repeating3210);\n Result.High = DoPerm(Src, Indices.High, IndexMask, Repeating3210);\n\n AVX128_StoreResult_WithOpSize(Op, Op->Dest, Result);\n}\n\nvoid OpDispatchBuilder::AVX128_VPCLMULQDQ(OpcodeArgs) {\n if (!CTX->HostFeatures.SupportsPMULL_128Bit) {\n UnimplementedOp(Op);\n return;\n }\n\n const auto Selector = static_cast(Op->Src[2].Literal());\n\n AVX128_VectorBinaryImpl(Op, OpSizeFromSrc(Op), OpSize::iInvalid, [this, Selector](IR::OpSize, Ref Src1, Ref Src2) {\n return _PCLMUL(OpSize::i128Bit, Src1, Src2, Selector & 0b1'0001);\n });\n}\n\n// FMA differences between AArch64 and x86 make this really confusing to remember how things match.\n// Here's a little guide for remembering how these instructions related across the architectures.\n//\n///< AArch64 Vector FMA behaviour\n// FMLA vd, vn, vm\n// - vd = (vn * vm) + vd\n// FMLS vd, vn, vm\n// - vd = (-vn * vm) + vd\n//\n// SVE ONLY! No FNMLA or FNMLS variants until SVE!\n// FMLA zda, pg/m, zn, zm - Ignore predicate here\n// - zda = (zn * zm) + zda\n// FMLS zda, pg/m, zn, zm - Ignore predicate here\n// - zda = (-zn * zm) + zda\n// FNMLA zda, pg/m, zn, zm - Ignore predicate here\n// - zda = (-zn * zm) - zda\n// FNMLS zda, pg/m, zn, zm - Ignore predicate here\n// - zda = (zn * zm) - zda\n//\n///< AArch64 Scalar FMA behaviour (FMA4 versions!)\n// All variants support 16-bit, 32-bit, and 64-bit.\n// FMADD d, n, m, a\n// - d = (n * m) + a\n// FMSUB d, n, m, a\n// - d = (-n * m) + a\n// FNMADD d, n, m, a\n// - d = (-n * m) - a\n// FNMSUB d, n, m, a\n// - d = (n * m) - a\n//\n///< x86 FMA behaviour\n// ## Packed variants\n// - VFMADD{PD,PS}suffix src1, src2, src3/mem\n// - 132 - src1 = (src1 * src3) + src2\n// - 213 - src1 = (src2 * src1) + src3\n// - 231 - src1 = (src2 * src3) + src1\n// ^ Matches ARM FMLA\n//\n// - VFMSUB{PD,PS}suffix src1, src2, src3/mem\n// - 132 - src1 = (src1 * src3) - src2\n// - 213 - src1 = (src2 * src1) - src3\n// - 231 - src1 = (src2 * src3) - src1\n// ^ Matches ARM FMLA with addend negated first\n// ^ Or just SVE FNMLS\n// ^ or scalar FNMSUB\n//\n// - VFNMADD{PD,PS}suffix src1, src2, src3/mem\n// - 132 - src1 = (-src1 * src3) + src2\n// - 213 - src1 = (-src2 * src1) + src3\n// - 231 - src1 = (-src2 * src3) + src1\n// ^ Matches ARM FMLS behaviour! (REALLY CONFUSINGLY NAMED!)\n// ^ Or Scalar FMSUB\n//\n// - VFNMSUB{PD,PS}suffix src1, src2, src3/mem\n// - 132 - src1 = (-src1 * src3) - src2\n// - 213 - src1 = (-src2 * src1) - src3\n// - 231 - src1 = (-src2 * src3) - src1\n// ^ Matches ARM FMLS behaviour with addend negated first! (REALLY CONFUSINGLY NAMED!)\n// ^ Or just SVE FNMLA\n// ^ Or scalar FNMADD\n//\n// - VFNMADDSUB{PD,PS}suffix src1, src2, src3/mem\n// - 132 - src1.odd = (src1.odd * src3.odd) + src2.odd\n// - src1.even = (src1.even * src3.even) - src2.even\n// - 213 - src1.odd = (src2.odd * src1.odd) + src3.odd\n// - src1.even = (src2.even * src1.even) - src3.even\n// - 231 - src1.odd = (src2.odd * src3.odd) + src1.odd\n// - src1.even = (src2.even * src3.even) - src1.even\n// ^ Matches ARM FMLA behaviour with addend.even negated first!\n//\n// - VFNMSUBADD{PD,PS}suffix src1, src2, src3/mem\n// - 132 - src1.odd = (src1.odd * src3.odd) - src2.odd\n// - src1.even = (src1.even * src3.even) + src2.even\n// - 213 - src1.odd = (src2.odd * src1.odd) - src3.odd\n// - src1.even = (src2.even * src1.even) + src3.even\n// - 231 - src1.odd = (src2.odd * src3.odd) - src1.odd\n// - src1.even = (src2.even * src3.even) + src1.even\n// ^ Matches ARM FMLA behaviour with addend.odd negated first!\n//\n// As shown only the 231 suffixed instructions matches AArch64 behaviour.\n// FEX will insert moves to transpose the vectors to match AArch64 behaviour for 132 and 213 variants.\n\nvoid OpDispatchBuilder::AVX128_VFMAImpl(OpcodeArgs, IROps IROp, uint8_t Src1Idx, uint8_t Src2Idx, uint8_t AddendIdx) {\n const auto Size = GetDstSize(Op);\n const auto Is128Bit = Size == Core::CPUState::XMM_SSE_REG_SIZE;\n\n const OpSize ElementSize = Op->Flags & X86Tables::DecodeFlags::FLAG_OPTION_AVX_W ? OpSize::i64Bit : OpSize::i32Bit;\n\n\n auto Dest = AVX128_LoadSource_WithOpSize(Op, Op->Dest, Op->Flags, !Is128Bit);\n auto Src1 = AVX128_LoadSource_WithOpSize(Op, Op->Src[0], Op->Flags, !Is128Bit);\n auto Src2 = AVX128_LoadSource_WithOpSize(Op, Op->Src[1], Op->Flags, !Is128Bit);\n\n RefPair Sources[3] = {Dest, Src1, Src2};\n\n RefPair Result {};\n DeriveOp(Result_Low, IROp, _VFMLA(OpSize::i128Bit, ElementSize, Sources[Src1Idx - 1].Low, Sources[Src2Idx - 1].Low, Sources[AddendIdx - 1].Low));\n Result.Low = Result_Low;\n if (Is128Bit) {\n Result.High = LoadZeroVector(OpSize::i128Bit);\n } else {\n DeriveOp(Result_High, IROp,\n _VFMLA(OpSize::i128Bit, ElementSize, Sources[Src1Idx - 1].High, Sources[Src2Idx - 1].High, Sources[AddendIdx - 1].High));\n Result.High = Result_High;\n }\n AVX128_StoreResult_WithOpSize(Op, Op->Dest, Result);\n}\n\nvoid OpDispatchBuilder::AVX128_VFMAScalarImpl(OpcodeArgs, IROps IROp, uint8_t Src1Idx, uint8_t Src2Idx, uint8_t AddendIdx) {\n const OpSize ElementSize = Op->Flags & X86Tables::DecodeFlags::FLAG_OPTION_AVX_W ? OpSize::i64Bit : OpSize::i32Bit;\n\n auto Dest = AVX128_LoadSource_WithOpSize(Op, Op->Dest, Op->Flags, false).Low;\n auto Src1 = AVX128_LoadSource_WithOpSize(Op, Op->Src[0], Op->Flags, false).Low;\n Ref Src2 {};\n if (Op->Src[1].IsGPR()) {\n Src2 = AVX128_LoadSource_WithOpSize(Op, Op->Src[1], Op->Flags, false).Low;\n } else {\n Src2 = LoadSourceFPR_WithOpSize(Op, Op->Src[1], ElementSize, Op->Flags);\n }\n\n Ref Sources[3] = {Dest, Src1, Src2};\n\n DeriveOp(Result_Low, IROp,\n _VFMLAScalarInsert(OpSize::i128Bit, ElementSize, Dest, Sources[Src1Idx - 1], Sources[Src2Idx - 1], Sources[AddendIdx - 1]));\n AVX128_StoreResult_WithOpSize(Op, Op->Dest, AVX128_Zext(Result_Low));\n}\n\nvoid OpDispatchBuilder::AVX128_VFMAddSubImpl(OpcodeArgs, bool AddSub, uint8_t Src1Idx, uint8_t Src2Idx, uint8_t AddendIdx) {\n const auto Size = GetDstSize(Op);\n const auto Is128Bit = Size == Core::CPUState::XMM_SSE_REG_SIZE;\n\n const OpSize ElementSize = Op->Flags & X86Tables::DecodeFlags::FLAG_OPTION_AVX_W ? OpSize::i64Bit : OpSize::i32Bit;\n\n auto Dest = AVX128_LoadSource_WithOpSize(Op, Op->Dest, Op->Flags, !Is128Bit);\n auto Src1 = AVX128_LoadSource_WithOpSize(Op, Op->Src[0], Op->Flags, !Is128Bit);\n auto Src2 = AVX128_LoadSource_WithOpSize(Op, Op->Src[1], Op->Flags, !Is128Bit);\n\n RefPair Sources[3] = {\n Dest,\n Src1,\n Src2,\n };\n\n RefPair Result {};\n\n Ref ConstantEOR {};\n if (AddSub) {\n ConstantEOR = LoadAndCacheNamedVectorConstant(\n OpSize::i128Bit, ElementSize == OpSize::i32Bit ? NAMED_VECTOR_PADDSUBPS_INVERT : NAMED_VECTOR_PADDSUBPD_INVERT);\n } else {\n ConstantEOR = LoadAndCacheNamedVectorConstant(\n OpSize::i128Bit, ElementSize == OpSize::i32Bit ? NAMED_VECTOR_PSUBADDPS_INVERT : NAMED_VECTOR_PSUBADDPD_INVERT);\n }\n auto InvertedSourceLow = _VXor(OpSize::i128Bit, ElementSize, Sources[AddendIdx - 1].Low, ConstantEOR);\n\n Result.Low = _VFMLA(OpSize::i128Bit, ElementSize, Sources[Src1Idx - 1].Low, Sources[Src2Idx - 1].Low, InvertedSourceLow);\n if (Is128Bit) {\n Result.High = LoadZeroVector(OpSize::i128Bit);\n } else {\n auto InvertedSourceHigh = _VXor(OpSize::i128Bit, ElementSize, Sources[AddendIdx - 1].High, ConstantEOR);\n Result.High = _VFMLA(OpSize::i128Bit, ElementSize, Sources[Src1Idx - 1].High, Sources[Src2Idx - 1].High, InvertedSourceHigh);\n }\n AVX128_StoreResult_WithOpSize(Op, Op->Dest, Result);\n}\n\nOpDispatchBuilder::RefPair OpDispatchBuilder::AVX128_VPGatherImpl(OpcodeArgs, OpSize Size, OpSize ElementLoadSize, OpSize AddrElementSize,\n RefPair Dest, RefPair Mask, RefVSIB VSIB) {\n LOGMAN_THROW_A_FMT(AddrElementSize == OpSize::i32Bit || AddrElementSize == OpSize::i64Bit, \"Unknown address element size\");\n const auto Is128Bit = Size == OpSize::i128Bit;\n\n ///< BaseAddr doesn't need to exist, calculate that here.\n Ref BaseAddr = VSIB.BaseAddr;\n if (BaseAddr && VSIB.Displacement) {\n BaseAddr = Add(OpSize::i64Bit, BaseAddr, VSIB.Displacement);\n } else if (VSIB.Displacement) {\n BaseAddr = Constant(VSIB.Displacement);\n } else if (!BaseAddr) {\n BaseAddr = Invalid();\n }\n\n if (CTX->HostFeatures.SupportsSVE128) {\n if (ElementLoadSize == OpSize::i64Bit && AddrElementSize == OpSize::i32Bit) {\n // In the case that FEX is loading double the amount of data than the number of address bits then we can optimize this case.\n // For 256-bits of data we need to sign extend all four 32-bit address elements to be 64-bit.\n // For 128-bits of data we only need to sign extend the lower two 32-bit address elements.\n LOGMAN_THROW_A_FMT(VSIB.High == Invalid(), \"Need to not have a high VSIB source\");\n\n if (!Is128Bit) {\n VSIB.High = _VSSHLL2(OpSize::i128Bit, OpSize::i32Bit, VSIB.Low, FEXCore::ilog2(VSIB.Scale));\n }\n VSIB.Low = _VSSHLL(OpSize::i128Bit, OpSize::i32Bit, VSIB.Low, FEXCore::ilog2(VSIB.Scale));\n\n ///< Set the scale to one now that it has been prescaled as well.\n VSIB.Scale = 1;\n\n // Set the address element size to 64-bit now that the elements are extended.\n AddrElementSize = OpSize::i64Bit;\n } else if (ElementLoadSize == OpSize::i64Bit && AddrElementSize == OpSize::i64Bit && (VSIB.Scale == 2 || VSIB.Scale == 4)) {\n // SVE gather instructions don't support scaling their vector elements by anything other than 1 or the address element size.\n // Pre-scale 64-bit addresses in the case that scale doesn't match in-order to hit SVE code paths more frequently.\n // Only hit this path if the host supports SVE. Otherwise it's a degradation for the ASIMD codepath.\n VSIB.Low = _VShlI(OpSize::i128Bit, OpSize::i64Bit, VSIB.Low, FEXCore::ilog2(VSIB.Scale));\n if (!Is128Bit) {\n VSIB.High = _VShlI(OpSize::i128Bit, OpSize::i64Bit, VSIB.High, FEXCore::ilog2(VSIB.Scale));\n }\n ///< Set the scale to one now that it has been prescaled.\n VSIB.Scale = 1;\n }\n }\n\n const auto GPRSize = GetGPROpSize();\n auto AddrSize = (Op->Flags & X86Tables::DecodeFlags::FLAG_ADDRESS_SIZE) != 0 ? (GPRSize >> 1) : GPRSize;\n\n RefPair Result {};\n ///< Calculate the low-half.\n Result.Low = _VLoadVectorGatherMasked(OpSize::i128Bit, ElementLoadSize, Dest.Low, Mask.Low, BaseAddr, VSIB.Low, VSIB.High,\n AddrElementSize, VSIB.Scale, 0, 0, AddrSize);\n\n if (Is128Bit) {\n Result.High = LoadZeroVector(OpSize::i128Bit);\n if (AddrElementSize == OpSize::i64Bit && ElementLoadSize == OpSize::i32Bit) {\n // Special case for the 128-bit gather load using 64-bit address indexes with 32-bit results.\n // Only loads two 32-bit elements in to the lower 64-bits of the first destination.\n // Bits [255:65] all become zero.\n Result.Low = _VZip(OpSize::i128Bit, OpSize::i64Bit, Result.Low, Result.High);\n }\n } else {\n RefPair AddrAddressing {};\n\n Ref DestReg = Dest.High;\n Ref MaskReg = Mask.High;\n uint8_t IndexElementOffset {};\n uint8_t DataElementOffset {};\n if (AddrElementSize == ElementLoadSize) {\n // If the address size matches the loading element size then it will be fetching at the same rate between low and high\n AddrAddressing.Low = VSIB.High;\n AddrAddressing.High = Invalid();\n } else if (AddrElementSize == OpSize::i32Bit && ElementLoadSize == OpSize::i64Bit) {\n // If the address element size if half the size of the Element load size then we need to start fetching half-way through the low register.\n AddrAddressing.Low = VSIB.Low;\n AddrAddressing.High = VSIB.High;\n IndexElementOffset = IR::NumElements(OpSize::i128Bit, AddrElementSize) / 2;\n } else if (AddrElementSize == OpSize::i64Bit && ElementLoadSize == OpSize::i32Bit) {\n AddrAddressing.Low = VSIB.High;\n AddrAddressing.High = Invalid();\n DestReg = Result.Low; ///< Start mixing with the low register.\n MaskReg = Mask.Low; ///< Mask starts with the low mask here.\n IndexElementOffset = 0;\n DataElementOffset = IR::NumElements(OpSize::i128Bit, ElementLoadSize) / 2;\n }\n\n ///< Calculate the high-half.\n auto ResultHigh = _VLoadVectorGatherMasked(OpSize::i128Bit, ElementLoadSize, DestReg, MaskReg, BaseAddr, AddrAddressing.Low,\n AddrAddressing.High, AddrElementSize, VSIB.Scale, DataElementOffset, IndexElementOffset, AddrSize);\n\n if (AddrElementSize == OpSize::i64Bit && ElementLoadSize == OpSize::i32Bit) {\n // If we only fetched 128-bits worth of data then the upper-result is all zero.\n Result = AVX128_Zext(ResultHigh);\n } else {\n Result.High = ResultHigh;\n }\n }\n\n return Result;\n}\n\nOpDispatchBuilder::RefPair OpDispatchBuilder::AVX128_VPGatherQPSImpl(OpcodeArgs, Ref Dest, Ref Mask, RefVSIB VSIB) {\n\n ///< BaseAddr doesn't need to exist, calculate that here.\n Ref BaseAddr = VSIB.BaseAddr;\n if (BaseAddr && VSIB.Displacement) {\n BaseAddr = Add(OpSize::i64Bit, BaseAddr, VSIB.Displacement);\n } else if (VSIB.Displacement) {\n BaseAddr = Constant(VSIB.Displacement);\n } else if (!BaseAddr) {\n BaseAddr = Invalid();\n }\n\n bool NeedsSVEScale = (VSIB.Scale == 2 || VSIB.Scale == 8) || (BaseAddr == Invalid() && VSIB.Scale != 1);\n\n if (CTX->HostFeatures.SupportsSVE128 && NeedsSVEScale) {\n // SVE gather instructions don't support scaling their vector elements by anything other than 1 or the address element size.\n // Pre-scale 64-bit addresses in the case that scale doesn't match in-order to hit SVE code paths more frequently.\n // Only hit this path if the host supports SVE. Otherwise it's a degradation for the ASIMD codepath.\n VSIB.Low = _VShlI(OpSize::i128Bit, OpSize::i64Bit, VSIB.Low, FEXCore::ilog2(VSIB.Scale));\n if (VSIB.High != Invalid()) {\n VSIB.High = _VShlI(OpSize::i128Bit, OpSize::i64Bit, VSIB.High, FEXCore::ilog2(VSIB.Scale));\n }\n ///< Set the scale to one now that it has been prescaled.\n VSIB.Scale = 1;\n }\n\n RefPair Result {};\n\n const auto GPRSize = GetGPROpSize();\n auto AddrSize = (Op->Flags & X86Tables::DecodeFlags::FLAG_ADDRESS_SIZE) != 0 ? (GPRSize >> 1) : GPRSize;\n\n ///< Calculate the low-half.\n Result.Low = _VLoadVectorGatherMaskedQPS(OpSize::i128Bit, OpSize::i32Bit, Dest, Mask, BaseAddr, VSIB.Low, VSIB.High, VSIB.Scale, AddrSize);\n Result.High = LoadZeroVector(OpSize::i128Bit);\n if (VSIB.High == Invalid()) {\n // Special case for only loading two floats.\n // The upper 64-bits of the lower lane also gets zero.\n Result.Low = _VZip(OpSize::i128Bit, OpSize::i64Bit, Result.Low, Result.High);\n }\n\n return Result;\n}\n\nvoid OpDispatchBuilder::AVX128_VPGATHER(OpcodeArgs, OpSize AddrElementSize) {\n\n const auto Size = OpSizeFromDst(Op);\n const auto Is128Bit = Size == OpSize::i128Bit;\n\n ///< Element size is determined by W flag.\n const OpSize ElementLoadSize = Op->Flags & X86Tables::DecodeFlags::FLAG_OPTION_AVX_W ? OpSize::i64Bit : OpSize::i32Bit;\n\n // We only need the high address register if the number of data elements is more than what the low half can consume.\n // But also the number of address elements is clamped by the destination size as well.\n const size_t NumDataElements = IR::NumElements(Size, ElementLoadSize);\n const size_t NumAddrElementBytes = std::min(IR::OpSizeToSize(Size), (NumDataElements * IR::OpSizeToSize(AddrElementSize)));\n const bool NeedsHighAddrBytes = NumAddrElementBytes > IR::OpSizeToSize(OpSize::i128Bit);\n\n auto Dest = AVX128_LoadSource_WithOpSize(Op, Op->Dest, Op->Flags, !Is128Bit);\n auto VSIB = AVX128_LoadVSIB(Op, Op->Src[0], Op->Flags, NeedsHighAddrBytes);\n auto Mask = AVX128_LoadSource_WithOpSize(Op, Op->Src[1], Op->Flags, !Is128Bit);\n\n bool NeedsSVEScale = (VSIB.Scale == 2 || VSIB.Scale == 8) || (VSIB.BaseAddr == Invalid() && VSIB.Scale != 1);\n\n const bool NeedsExplicitSVEPath =\n CTX->HostFeatures.SupportsSVE128 && AddrElementSize == OpSize::i32Bit && ElementLoadSize == OpSize::i32Bit && NeedsSVEScale;\n\n RefPair Result {};\n if (NeedsExplicitSVEPath) {\n // Special case for VGATHERDPS/VPGATHERDD (32-bit addresses loading 32-bit elements) that can't use the SVE codepath.\n // The problem is due to the scale not matching SVE limitations, we need to prescale the addresses to be 64-bit.\n auto ScaleVSIBHalf = [this](Ref VSIB, Ref BaseAddr, int32_t Displacement, uint8_t Scale) -> RefVSIB {\n RefVSIB Result {};\n Result.High = _VSSHLL2(OpSize::i128Bit, OpSize::i32Bit, VSIB, FEXCore::ilog2(Scale));\n Result.Low = _VSSHLL(OpSize::i128Bit, OpSize::i32Bit, VSIB, FEXCore::ilog2(Scale));\n\n Result.Displacement = Displacement;\n Result.BaseAddr = BaseAddr;\n\n ///< Set the scale to one now that it has been prescaled as well.\n Result.Scale = 1;\n return Result;\n };\n\n RefVSIB VSIBLow = ScaleVSIBHalf(VSIB.Low, VSIB.BaseAddr, VSIB.Displacement, VSIB.Scale);\n RefVSIB VSIBHigh {};\n\n if (NeedsHighAddrBytes) {\n VSIBHigh = ScaleVSIBHalf(VSIB.High, VSIB.BaseAddr, VSIB.Displacement, VSIB.Scale);\n }\n\n ///< AddressElementSize is now OpSize::i64Bit\n Result = AVX128_VPGatherQPSImpl(Op, Dest.Low, Mask.Low, VSIBLow);\n if (NeedsHighAddrBytes) {\n auto Res = AVX128_VPGatherQPSImpl(Op, Dest.High, Mask.High, VSIBHigh);\n Result.High = Res.Low;\n }\n } else if (AddrElementSize == OpSize::i64Bit && ElementLoadSize == OpSize::i32Bit) {\n Result = AVX128_VPGatherQPSImpl(Op, Dest.Low, Mask.Low, VSIB);\n } else {\n Result = AVX128_VPGatherImpl(Op, Size, ElementLoadSize, AddrElementSize, Dest, Mask, VSIB);\n }\n AVX128_StoreResult_WithOpSize(Op, Op->Dest, Result);\n\n ///< Assume non-faulting behaviour and clear the mask register.\n RefPair ZeroPair {};\n ZeroPair.Low = LoadZeroVector(OpSize::i128Bit);\n ZeroPair.High = ZeroPair.Low;\n AVX128_StoreResult_WithOpSize(Op, Op->Src[1], ZeroPair);\n}\n\nvoid OpDispatchBuilder::AVX128_VCVTPH2PS(OpcodeArgs) {\n const auto DstSize = OpSizeFromDst(Op);\n const auto SrcSize = IR::SizeToOpSize(IR::OpSizeToSize(DstSize) / 2);\n const auto Is128BitSrc = SrcSize == OpSize::i128Bit;\n const auto Is128BitDst = DstSize == OpSize::i128Bit;\n\n RefPair Src {};\n if (Op->Src[0].IsGPR()) {\n Src = AVX128_LoadSource_WithOpSize(Op, Op->Src[0], Op->Flags, !Is128BitSrc);\n } else {\n // In the event that a memory operand is used as the source operand,\n // the access width will always be half the size of the destination vector width\n // (i.e. 128-bit vector -> 64-bit mem, 256-bit vector -> 128-bit mem)\n Src.Low = LoadSourceFPR_WithOpSize(Op, Op->Src[0], SrcSize, Op->Flags);\n }\n\n RefPair Result {};\n Result.Low = _Vector_FToF(OpSize::i128Bit, OpSize::i32Bit, Src.Low, OpSize::i16Bit);\n\n if (Is128BitSrc) {\n Result.High = _VFCVTL2(OpSize::i128Bit, OpSize::i16Bit, Src.Low);\n }\n\n if (Is128BitDst) {\n Result = AVX128_Zext(Result.Low);\n }\n\n AVX128_StoreResult_WithOpSize(Op, Op->Dest, Result);\n}\n\nvoid OpDispatchBuilder::AVX128_VCVTPS2PH(OpcodeArgs) {\n const auto SrcSize = OpSizeFromSrc(Op);\n const auto Is128BitSrc = SrcSize == OpSize::i128Bit;\n const auto StoreSize = Op->Dest.IsGPR() ? OpSize::i128Bit : IR::SizeToOpSize(IR::OpSizeToSize(SrcSize) / 2);\n\n const auto Imm8 = Op->Src[1].Literal();\n const auto UseMXCSR = (Imm8 & 0b100) != 0;\n\n auto Src = AVX128_LoadSource_WithOpSize(Op, Op->Src[0], Op->Flags, !Is128BitSrc);\n\n RefPair Result {};\n\n Ref OldFPCR {};\n if (!UseMXCSR) {\n // No ARM float conversion instructions allow passing in\n // a rounding mode as an immediate. All of them depend on\n // the RM field in the FPCR. And so! We have to do some ugly\n // rounding mode shuffling.\n const auto NewRMode = Imm8 & 0b11;\n OldFPCR = _PushRoundingMode(NewRMode);\n }\n\n Result.Low = _Vector_FToF(OpSize::i128Bit, OpSize::i16Bit, Src.Low, OpSize::i32Bit);\n if (!Is128BitSrc) {\n Result.Low = _VFCVTN2(OpSize::i128Bit, OpSize::i32Bit, Result.Low, Src.High);\n }\n\n if (!UseMXCSR) {\n _PopRoundingMode(OldFPCR);\n }\n\n // We need to eliminate upper junk if we're storing into a register with\n // a 256-bit source (VCVTPS2PH's destination for registers is an XMM).\n if (Op->Src[0].IsGPR() && SrcSize == OpSize::i256Bit) {\n Result = AVX128_Zext(Result.Low);\n }\n\n if (!Op->Dest.IsGPR()) {\n StoreResultFPR_WithOpSize(Op, Op->Dest, Result.Low, StoreSize);\n } else {\n AVX128_StoreResult_WithOpSize(Op, Op->Dest, Result);\n }\n}\n\n} // namespace FEXCore::IR\n" }, { "path": "FEXCore/Source/Interface/Core/OpcodeDispatcher/BaseTables.h", "content": "// SPDX-License-Identifier: MIT\n#pragma once\n#include \"Interface/Core/OpcodeDispatcher.h\"\n\nnamespace FEXCore::IR {\nconstexpr inline DispatchTableEntry OpDispatch_BaseOpTable[] = {\n // Instructions\n {0x00, 6, &OpDispatchBuilder::Bind<&OpDispatchBuilder::ALUOp, FEXCore::IR::IROps::OP_ADD, FEXCore::IR::IROps::OP_ATOMICFETCHADD, 0>},\n\n {0x08, 6, &OpDispatchBuilder::Bind<&OpDispatchBuilder::ALUOp, FEXCore::IR::IROps::OP_OR, FEXCore::IR::IROps::OP_ATOMICFETCHOR, 0>},\n\n {0x10, 6, &OpDispatchBuilder::Bind<&OpDispatchBuilder::ADCOp, 0>},\n\n {0x18, 6, &OpDispatchBuilder::Bind<&OpDispatchBuilder::SBBOp, 0>},\n\n {0x20, 6, &OpDispatchBuilder::Bind<&OpDispatchBuilder::ALUOp, FEXCore::IR::IROps::OP_ANDWITHFLAGS, FEXCore::IR::IROps::OP_ATOMICFETCHAND, 0>},\n\n {0x28, 6, &OpDispatchBuilder::Bind<&OpDispatchBuilder::ALUOp, FEXCore::IR::IROps::OP_SUB, FEXCore::IR::IROps::OP_ATOMICFETCHSUB, 0>},\n\n {0x30, 6, &OpDispatchBuilder::Bind<&OpDispatchBuilder::ALUOp, FEXCore::IR::IROps::OP_XOR, FEXCore::IR::IROps::OP_ATOMICFETCHXOR, 0>},\n\n {0x38, 6, &OpDispatchBuilder::Bind<&OpDispatchBuilder::CMPOp, 0>},\n {0x50, 8, &OpDispatchBuilder::PUSHREGOp},\n {0x58, 8, &OpDispatchBuilder::POPOp},\n {0x68, 1, &OpDispatchBuilder::PUSHOp},\n {0x69, 1, &OpDispatchBuilder::IMUL2SrcOp},\n {0x6A, 1, &OpDispatchBuilder::PUSHOp},\n {0x6B, 1, &OpDispatchBuilder::IMUL2SrcOp},\n {0x6C, 4, &OpDispatchBuilder::PermissionRestrictedOp},\n\n {0x70, 16, &OpDispatchBuilder::CondJUMPOp},\n {0x84, 2, &OpDispatchBuilder::Bind<&OpDispatchBuilder::TESTOp, 0>},\n {0x86, 2, &OpDispatchBuilder::XCHGOp},\n {0x88, 4, &OpDispatchBuilder::Bind<&OpDispatchBuilder::MOVGPROp, 0>},\n\n {0x8C, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::MOVSegOp, false>},\n {0x8D, 1, &OpDispatchBuilder::LEAOp},\n {0x8E, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::MOVSegOp, true>},\n {0x8F, 1, &OpDispatchBuilder::POPOp},\n {0x90, 8, &OpDispatchBuilder::XCHGOp},\n\n {0x98, 1, &OpDispatchBuilder::CDQOp},\n {0x99, 1, &OpDispatchBuilder::CQOOp},\n {0x9B, 1, &OpDispatchBuilder::NOPOp},\n {0x9C, 1, &OpDispatchBuilder::PUSHFOp},\n {0x9D, 1, &OpDispatchBuilder::POPFOp},\n {0x9E, 1, &OpDispatchBuilder::SAHFOp},\n {0x9F, 1, &OpDispatchBuilder::LAHFOp},\n {0xA4, 2, &OpDispatchBuilder::MOVSOp},\n\n {0xA6, 2, &OpDispatchBuilder::CMPSOp},\n {0xA8, 2, &OpDispatchBuilder::Bind<&OpDispatchBuilder::TESTOp, 0>},\n {0xAA, 2, &OpDispatchBuilder::STOSOp},\n {0xAC, 2, &OpDispatchBuilder::LODSOp},\n {0xAE, 2, &OpDispatchBuilder::SCASOp},\n {0xB0, 16, &OpDispatchBuilder::Bind<&OpDispatchBuilder::MOVGPROp, 0>},\n {0xC2, 2, &OpDispatchBuilder::RETOp},\n {0xC8, 1, &OpDispatchBuilder::EnterOp},\n {0xC9, 1, &OpDispatchBuilder::LEAVEOp},\n {0xCA, 2, &OpDispatchBuilder::RETFARIndirectOp},\n {0xCC, 2, &OpDispatchBuilder::INTOp},\n {0xCF, 1, &OpDispatchBuilder::IRETOp},\n {0xD7, 2, &OpDispatchBuilder::XLATOp},\n {0xE0, 3, &OpDispatchBuilder::LoopOp},\n {0xE3, 1, &OpDispatchBuilder::CondJUMPRCXOp},\n {0xE4, 4, &OpDispatchBuilder::PermissionRestrictedOp},\n {0xE8, 1, &OpDispatchBuilder::CALLOp},\n {0xE9, 1, &OpDispatchBuilder::JUMPOp},\n {0xEB, 1, &OpDispatchBuilder::JUMPOp},\n {0xEC, 4, &OpDispatchBuilder::PermissionRestrictedOp},\n {0xF1, 1, &OpDispatchBuilder::INTOp},\n {0xF4, 1, &OpDispatchBuilder::INTOp},\n\n {0xF5, 1, &OpDispatchBuilder::FLAGControlOp},\n {0xF8, 2, &OpDispatchBuilder::FLAGControlOp},\n {0xFA, 2, &OpDispatchBuilder::PermissionRestrictedOp},\n {0xFC, 2, &OpDispatchBuilder::FLAGControlOp},\n};\n} // namespace FEXCore::IR\n" }, { "path": "FEXCore/Source/Interface/Core/OpcodeDispatcher/Crypto.cpp", "content": "// SPDX-License-Identifier: MIT\n/*\n$info$\ntags: frontend|x86-to-ir, opcodes|dispatcher-implementations\ndesc: Handles x86/64 Crypto instructions to IR\n$end_info$\n*/\n\n#include \"Interface/Core/X86Tables/X86Tables.h\"\n\n#include \n#include \"Interface/Core/OpcodeDispatcher.h\"\n\n#include \n\nnamespace FEXCore::IR {\nclass OrderedNode;\n\n#define OpcodeArgs [[maybe_unused]] FEXCore::X86Tables::DecodedOp Op\n\nvoid OpDispatchBuilder::SHA1NEXTEOp(OpcodeArgs) {\n if (!CTX->HostFeatures.SupportsSHA) {\n UnimplementedOp(Op);\n return;\n }\n Ref Dest = LoadSourceFPR(Op, Op->Dest, Op->Flags);\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n\n // ARMv8 SHA1 extension provides a `SHA1H` instruction which does a fixed rotate by 30.\n // This only operates on element 0 rather than element 3. We don't have the luxury of rewriting the x86 SHA algorithm to take advantage of this.\n // Move the element to zero, rotate, and then move back (Using duplicates).\n // Saves one instruction versus that path that doesn't support SHA extension.\n auto Duplicated = _VDupElement(OpSize::i128Bit, OpSize::i32Bit, Dest, 3);\n auto Sha1HRotated = _VSha1H(Duplicated);\n auto RotatedNode = _VDupElement(OpSize::i128Bit, OpSize::i32Bit, Sha1HRotated, 0);\n auto Tmp = _VAdd(OpSize::i128Bit, OpSize::i32Bit, Src, RotatedNode);\n auto Result = _VInsElement(OpSize::i128Bit, OpSize::i32Bit, 3, 3, Src, Tmp);\n\n StoreResultFPR(Op, Result);\n}\n\nvoid OpDispatchBuilder::SHA1MSG1Op(OpcodeArgs) {\n if (!CTX->HostFeatures.SupportsSHA) {\n UnimplementedOp(Op);\n return;\n }\n Ref Dest = LoadSourceFPR(Op, Op->Dest, Op->Flags);\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n\n Ref NewVec = _VExtr(OpSize::i128Bit, OpSize::i64Bit, Dest, Src, 1);\n\n // [W0, W1, W2, W3] ^ [W2, W3, W4, W5]\n Ref Result = _VXor(OpSize::i128Bit, OpSize::i8Bit, Dest, NewVec);\n\n StoreResultFPR(Op, Result);\n}\n\nvoid OpDispatchBuilder::SHA1MSG2Op(OpcodeArgs) {\n if (!CTX->HostFeatures.SupportsSHA) {\n UnimplementedOp(Op);\n return;\n }\n Ref Dest = LoadSourceFPR(Op, Op->Dest, Op->Flags);\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n\n // ARM SHA1 mostly matches x86 semantics, except the input and outputs are both flipped from elements 0,1,2,3 to 3,2,1,0.\n auto Src1 = SHADataShuffle(Dest);\n auto Src2 = SHADataShuffle(Src);\n\n // The result is swizzled differently than expected\n auto Result = SHADataShuffle(_VSha1SU1(Src1, Src2));\n\n StoreResultFPR(Op, Result);\n}\n\nvoid OpDispatchBuilder::SHA1RNDS4Op(OpcodeArgs) {\n if (!CTX->HostFeatures.SupportsSHA) {\n UnimplementedOp(Op);\n return;\n }\n const uint64_t Imm8 = Op->Src[1].Literal() & 0b11;\n Ref Dest = LoadSourceFPR(Op, Op->Dest, Op->Flags);\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n\n Ref Result {};\n Ref ConstantVector {};\n switch (Imm8) {\n case 0:\n ConstantVector = LoadAndCacheNamedVectorConstant(OpSize::i128Bit, FEXCore::IR::NamedVectorConstant::NAMED_VECTOR_SHA1RNDS_K0);\n break;\n case 1:\n ConstantVector = LoadAndCacheNamedVectorConstant(OpSize::i128Bit, FEXCore::IR::NamedVectorConstant::NAMED_VECTOR_SHA1RNDS_K1);\n break;\n case 2:\n ConstantVector = LoadAndCacheNamedVectorConstant(OpSize::i128Bit, FEXCore::IR::NamedVectorConstant::NAMED_VECTOR_SHA1RNDS_K2);\n break;\n case 3:\n ConstantVector = LoadAndCacheNamedVectorConstant(OpSize::i128Bit, FEXCore::IR::NamedVectorConstant::NAMED_VECTOR_SHA1RNDS_K3);\n break;\n }\n\n const auto ZeroRegister = LoadZeroVector(OpSize::i32Bit);\n\n Ref Src1 = SHADataShuffle(Dest);\n Ref Src2 = SHADataShuffle(Src);\n Src2 = _VAdd(OpSize::i128Bit, OpSize::i32Bit, Src2, ConstantVector);\n\n switch (Imm8) {\n case 0: Result = SHADataShuffle(_VSha1C(Src1, ZeroRegister, Src2)); break;\n case 2: Result = SHADataShuffle(_VSha1M(Src1, ZeroRegister, Src2)); break;\n case 1:\n case 3: Result = SHADataShuffle(_VSha1P(Src1, ZeroRegister, Src2)); break;\n }\n\n StoreResultFPR(Op, Result);\n}\n\nvoid OpDispatchBuilder::SHA256MSG1Op(OpcodeArgs) {\n if (!CTX->HostFeatures.SupportsSHA) {\n UnimplementedOp(Op);\n return;\n }\n Ref Dest = LoadSourceFPR(Op, Op->Dest, Op->Flags);\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n\n auto Result = _VSha256U0(Dest, Src);\n\n StoreResultFPR(Op, Result);\n}\n\nvoid OpDispatchBuilder::SHA256MSG2Op(OpcodeArgs) {\n if (!CTX->HostFeatures.SupportsSHA) {\n UnimplementedOp(Op);\n return;\n }\n Ref Dest = LoadSourceFPR(Op, Op->Dest, Op->Flags);\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n\n auto Src1 = _VExtr(OpSize::i128Bit, OpSize::i32Bit, Dest, Dest, 3);\n auto DupDst = _VDupElement(OpSize::i128Bit, OpSize::i32Bit, Dest, 3);\n auto Src2 = _VZip2(OpSize::i128Bit, OpSize::i64Bit, DupDst, Src);\n\n auto Result = _VSha256U1(Src1, Src2);\n\n StoreResultFPR(Op, Result);\n}\n\nvoid OpDispatchBuilder::SHA256RNDS2Op(OpcodeArgs) {\n if (!CTX->HostFeatures.SupportsSHA) {\n UnimplementedOp(Op);\n return;\n }\n Ref Dest = LoadSourceFPR(Op, Op->Dest, Op->Flags);\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n // Hardcoded to XMM0\n auto XMM0 = LoadXMMRegister(0);\n\n auto shuffle_abcd = [this](Ref Src1, Ref Src2) -> Ref {\n // Generates a suitable SHA256 `abcd` configuration from x86 format.\n auto Tmp = _VZip2(OpSize::i128Bit, OpSize::i64Bit, Src2, Src1);\n return _VRev64(OpSize::i128Bit, OpSize::i32Bit, Tmp);\n };\n\n auto shuffle_efgh = [this](Ref Src1, Ref Src2) -> Ref {\n // Generates a suitable SHA256 `efgh` configuration from x86 format.\n auto Tmp = _VZip(OpSize::i128Bit, OpSize::i64Bit, Src2, Src1);\n return _VRev64(OpSize::i128Bit, OpSize::i32Bit, Tmp);\n };\n\n auto ABCD = shuffle_abcd(Dest, Src);\n auto EFGH = shuffle_efgh(Dest, Src);\n\n // x86 uses only the bottom 64-bits of the key, so duplicate to match ARM64 semantics.\n auto Key = _VDupElement(OpSize::i128Bit, OpSize::i64Bit, XMM0, 0);\n\n auto A = _VSha256H(ABCD, EFGH, Key);\n auto B = _VSha256H2(EFGH, ABCD, Key);\n auto Result = shuffle_abcd(A, B);\n\n StoreResultFPR(Op, Result);\n}\n\nvoid OpDispatchBuilder::AESImcOp(OpcodeArgs) {\n if (!CTX->HostFeatures.SupportsAES) {\n UnimplementedOp(Op);\n return;\n }\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n Ref Result = _VAESImc(Src);\n StoreResultFPR(Op, Result);\n}\n\nvoid OpDispatchBuilder::AESEncOp(OpcodeArgs) {\n if (!CTX->HostFeatures.SupportsAES) {\n UnimplementedOp(Op);\n return;\n }\n Ref Dest = LoadSourceFPR(Op, Op->Dest, Op->Flags);\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n Ref Result = _VAESEnc(OpSize::i128Bit, Dest, Src, LoadZeroVector(OpSize::i128Bit));\n StoreResultFPR(Op, Result);\n}\n\nvoid OpDispatchBuilder::VAESEncOp(OpcodeArgs) {\n const auto DstSize = OpSizeFromDst(Op);\n const auto Is128Bit = DstSize == OpSize::i128Bit;\n\n // TODO: Handle 256-bit VAESENC.\n LOGMAN_THROW_A_FMT(Is128Bit, \"256-bit VAESENC unimplemented\");\n\n Ref State = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n Ref Key = LoadSourceFPR(Op, Op->Src[1], Op->Flags);\n Ref Result = _VAESEnc(DstSize, State, Key, LoadZeroVector(DstSize));\n\n StoreResultFPR(Op, Result);\n}\n\nvoid OpDispatchBuilder::AESEncLastOp(OpcodeArgs) {\n if (!CTX->HostFeatures.SupportsAES) {\n UnimplementedOp(Op);\n return;\n }\n Ref Dest = LoadSourceFPR(Op, Op->Dest, Op->Flags);\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n Ref Result = _VAESEncLast(OpSize::i128Bit, Dest, Src, LoadZeroVector(OpSize::i128Bit));\n StoreResultFPR(Op, Result);\n}\n\nvoid OpDispatchBuilder::VAESEncLastOp(OpcodeArgs) {\n const auto DstSize = OpSizeFromDst(Op);\n const auto Is128Bit = DstSize == OpSize::i128Bit;\n\n // TODO: Handle 256-bit VAESENCLAST.\n LOGMAN_THROW_A_FMT(Is128Bit, \"256-bit VAESENCLAST unimplemented\");\n\n Ref State = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n Ref Key = LoadSourceFPR(Op, Op->Src[1], Op->Flags);\n Ref Result = _VAESEncLast(DstSize, State, Key, LoadZeroVector(DstSize));\n\n StoreResultFPR(Op, Result);\n}\n\nvoid OpDispatchBuilder::AESDecOp(OpcodeArgs) {\n if (!CTX->HostFeatures.SupportsAES) {\n UnimplementedOp(Op);\n return;\n }\n Ref Dest = LoadSourceFPR(Op, Op->Dest, Op->Flags);\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n Ref Result = _VAESDec(OpSize::i128Bit, Dest, Src, LoadZeroVector(OpSize::i128Bit));\n StoreResultFPR(Op, Result);\n}\n\nvoid OpDispatchBuilder::VAESDecOp(OpcodeArgs) {\n const auto DstSize = OpSizeFromDst(Op);\n const auto Is128Bit = DstSize == OpSize::i128Bit;\n\n // TODO: Handle 256-bit VAESDEC.\n LOGMAN_THROW_A_FMT(Is128Bit, \"256-bit VAESDEC unimplemented\");\n\n Ref State = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n Ref Key = LoadSourceFPR(Op, Op->Src[1], Op->Flags);\n Ref Result = _VAESDec(DstSize, State, Key, LoadZeroVector(DstSize));\n\n StoreResultFPR(Op, Result);\n}\n\nvoid OpDispatchBuilder::AESDecLastOp(OpcodeArgs) {\n if (!CTX->HostFeatures.SupportsAES) {\n UnimplementedOp(Op);\n return;\n }\n Ref Dest = LoadSourceFPR(Op, Op->Dest, Op->Flags);\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n Ref Result = _VAESDecLast(OpSize::i128Bit, Dest, Src, LoadZeroVector(OpSize::i128Bit));\n StoreResultFPR(Op, Result);\n}\n\nvoid OpDispatchBuilder::VAESDecLastOp(OpcodeArgs) {\n const auto DstSize = OpSizeFromDst(Op);\n const auto Is128Bit = DstSize == OpSize::i128Bit;\n\n // TODO: Handle 256-bit VAESDECLAST.\n LOGMAN_THROW_A_FMT(Is128Bit, \"256-bit VAESDECLAST unimplemented\");\n\n Ref State = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n Ref Key = LoadSourceFPR(Op, Op->Src[1], Op->Flags);\n Ref Result = _VAESDecLast(DstSize, State, Key, LoadZeroVector(DstSize));\n\n StoreResultFPR(Op, Result);\n}\n\nRef OpDispatchBuilder::AESKeyGenAssistImpl(OpcodeArgs) {\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n const uint64_t RCON = Op->Src[1].Literal();\n\n auto KeyGenSwizzle = LoadAndCacheNamedVectorConstant(OpSize::i128Bit, NAMED_VECTOR_AESKEYGENASSIST_SWIZZLE);\n return _VAESKeyGenAssist(Src, KeyGenSwizzle, LoadZeroVector(OpSize::i128Bit), RCON);\n}\n\nvoid OpDispatchBuilder::AESKeyGenAssist(OpcodeArgs) {\n if (!CTX->HostFeatures.SupportsAES) {\n UnimplementedOp(Op);\n return;\n }\n\n Ref Result = AESKeyGenAssistImpl(Op);\n StoreResultFPR(Op, Result);\n}\n\nvoid OpDispatchBuilder::PCLMULQDQOp(OpcodeArgs) {\n if (!CTX->HostFeatures.SupportsPMULL_128Bit) {\n UnimplementedOp(Op);\n return;\n }\n Ref Dest = LoadSourceFPR(Op, Op->Dest, Op->Flags);\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n const auto Selector = static_cast(Op->Src[1].Literal());\n\n auto Res = _PCLMUL(OpSize::i128Bit, Dest, Src, Selector & 0b1'0001);\n StoreResultFPR(Op, Res);\n}\n\nvoid OpDispatchBuilder::VPCLMULQDQOp(OpcodeArgs) {\n if (!CTX->HostFeatures.SupportsPMULL_128Bit) {\n UnimplementedOp(Op);\n return;\n }\n const auto DstSize = OpSizeFromDst(Op);\n\n Ref Src1 = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n Ref Src2 = LoadSourceFPR(Op, Op->Src[1], Op->Flags);\n const auto Selector = static_cast(Op->Src[2].Literal());\n\n Ref Res = _PCLMUL(DstSize, Src1, Src2, Selector & 0b1'0001);\n StoreResultFPR(Op, Res);\n}\n\n} // namespace FEXCore::IR\n" }, { "path": "FEXCore/Source/Interface/Core/OpcodeDispatcher/DDDTables.h", "content": "// SPDX-License-Identifier: MIT\n#pragma once\n#include \"Interface/Core/OpcodeDispatcher.h\"\n\nnamespace FEXCore::IR {\nconstexpr DispatchTableEntry OpDispatch_DDDTable[] = {\n {0x0C, 1, &OpDispatchBuilder::PI2FWOp},\n {0x0D, 1, &OpDispatchBuilder::Vector_CVT_Int_To_Float},\n {0x1C, 1, &OpDispatchBuilder::PF2IWOp},\n {0x1D, 1, &OpDispatchBuilder::Vector_CVT_Float_To_Int},\n\n {0x86, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorUnaryOp, IR::OP_VFRECPPRECISION, OpSize::i32Bit>},\n {0x87, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::RSqrt3DNowOp, false>},\n\n {0x8A, 1, &OpDispatchBuilder::PFNACCOp},\n {0x8E, 1, &OpDispatchBuilder::PFPNACCOp},\n\n {0x90, 1, &OpDispatchBuilder::VPFCMPOp<1>},\n {0x94, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VFMIN, OpSize::i32Bit>},\n {0x96, 1, &OpDispatchBuilder::VectorUnaryDuplicateOp},\n {0x97, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::RSqrt3DNowOp, true>},\n\n {0x9A, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VFSUB, OpSize::i32Bit>},\n {0x9E, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VFADD, OpSize::i32Bit>},\n\n {0xA0, 1, &OpDispatchBuilder::VPFCMPOp<2>},\n {0xA4, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VFMAX, OpSize::i32Bit>},\n // Can be treated as a move\n {0xA6, 1, &OpDispatchBuilder::MOVVectorUnalignedOp},\n {0xA7, 1, &OpDispatchBuilder::MOVVectorUnalignedOp},\n\n {0xAA, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUROp, IR::OP_VFSUB, OpSize::i32Bit>},\n {0xAE, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VFADDP, OpSize::i32Bit>},\n\n {0xB0, 1, &OpDispatchBuilder::VPFCMPOp<0>},\n {0xB4, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VFMUL, OpSize::i32Bit>},\n // Can be treated as a move\n {0xB6, 1, &OpDispatchBuilder::MOVVectorUnalignedOp},\n {0xB7, 1, &OpDispatchBuilder::PMULHRWOp},\n\n {0xBB, 1, &OpDispatchBuilder::PSWAPDOp},\n {0xBF, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VURAVG, OpSize::i8Bit>},\n};\n\n} // namespace FEXCore::IR\n" }, { "path": "FEXCore/Source/Interface/Core/OpcodeDispatcher/Flags.cpp", "content": "// SPDX-License-Identifier: MIT\n/*\n$info$\ntags: frontend|x86-to-ir, opcodes|dispatcher-implementations\ndesc: Handles x86/64 flag generation\n$end_info$\n*/\n\n#include \"Interface/Core/OpcodeDispatcher.h\"\n#include \"Interface/Core/X86Tables/X86Tables.h\"\n\n#include \n#include \n#include \n\n#include \n#include \n\nnamespace FEXCore::IR {\nconstexpr std::array FlagOffsets = {\n FEXCore::X86State::RFLAG_CF_RAW_LOC, FEXCore::X86State::RFLAG_PF_RAW_LOC, FEXCore::X86State::RFLAG_AF_RAW_LOC,\n FEXCore::X86State::RFLAG_ZF_RAW_LOC, FEXCore::X86State::RFLAG_SF_RAW_LOC, FEXCore::X86State::RFLAG_TF_RAW_LOC,\n FEXCore::X86State::RFLAG_IF_LOC, FEXCore::X86State::RFLAG_DF_RAW_LOC, FEXCore::X86State::RFLAG_OF_RAW_LOC,\n FEXCore::X86State::RFLAG_IOPL_LOC, FEXCore::X86State::RFLAG_NT_LOC, FEXCore::X86State::RFLAG_RF_LOC,\n FEXCore::X86State::RFLAG_VM_LOC, FEXCore::X86State::RFLAG_AC_LOC, FEXCore::X86State::RFLAG_VIF_LOC,\n FEXCore::X86State::RFLAG_VIP_LOC, FEXCore::X86State::RFLAG_ID_LOC,\n};\n\nvoid OpDispatchBuilder::ZeroPF_AF() {\n // PF is stored inverted, so invert it when we zero.\n SetRFLAG(Constant(1));\n SetAF(0);\n}\n\nvoid OpDispatchBuilder::SetPackedRFLAG(bool Lower8, Ref Src) {\n size_t NumFlags = FlagOffsets.size();\n if (Lower8) {\n // Calculate flags early.\n // This is only a partial overwrite of flags since OF isn't stored here.\n CalculateDeferredFlags();\n NumFlags = 5;\n }\n\n // PF and CF are both stored inverted, so hoist the invert.\n auto SrcInverted = _Not(OpSize::i32Bit, Src);\n\n for (size_t i = 0; i < NumFlags; ++i) {\n const auto FlagOffset = FlagOffsets[i];\n\n if (FlagOffset == FEXCore::X86State::RFLAG_AF_RAW_LOC) {\n // AF is in bit 4 architecturally, and we need to store it to bit 4 of our\n // AF register, with garbage in the other bits. The extract is deferred.\n // We also defer a XOR with the result bit, which is implemented as XOR\n // with PF[4]. But the _Bfe below reliably zeros bit 4 of the PF byte, so\n // that will be a no-op and we get the right result.\n //\n // So we write out the whole flags byte to AF without an extract.\n static_assert(FEXCore::X86State::RFLAG_AF_RAW_LOC == 4);\n SetRFLAG(Src, FEXCore::X86State::RFLAG_AF_RAW_LOC);\n } else if (FlagOffset == FEXCore::X86State::RFLAG_PF_RAW_LOC || FlagOffset == FEXCore::X86State::RFLAG_CF_RAW_LOC) {\n // PF and CF are both stored parity flipped.\n SetRFLAG(SrcInverted, FlagOffset, FlagOffset, true);\n } else {\n SetRFLAG(Src, FlagOffset, FlagOffset, true);\n }\n }\n\n CFInverted = true;\n}\n\nRef OpDispatchBuilder::GetPackedRFLAG(uint32_t FlagsMask) {\n // Calculate flags early.\n CalculateDeferredFlags();\n\n // SF/ZF and N/Z are together on both arm64 and x86_64, so we special case that.\n bool GetNZ = (FlagsMask & (1 << FEXCore::X86State::RFLAG_SF_RAW_LOC)) && (FlagsMask & (1 << FEXCore::X86State::RFLAG_ZF_RAW_LOC));\n\n // Handle CF first, since it's at bit 0 and hence doesn't need shift or OR.\n LOGMAN_THROW_A_FMT(FlagsMask & (1 << FEXCore::X86State::RFLAG_CF_RAW_LOC), \"CF always handled\");\n static_assert(FEXCore::X86State::RFLAG_CF_RAW_LOC == 0);\n Ref Original = GetRFLAG(FEXCore::X86State::RFLAG_CF_RAW_LOC);\n\n for (size_t i = 0; i < FlagOffsets.size(); ++i) {\n const auto FlagOffset = FlagOffsets[i];\n if (!((1U << FlagOffset) & FlagsMask)) {\n continue;\n }\n\n if ((GetNZ && (FlagOffset == FEXCore::X86State::RFLAG_SF_RAW_LOC || FlagOffset == FEXCore::X86State::RFLAG_ZF_RAW_LOC)) ||\n FlagOffset == FEXCore::X86State::RFLAG_CF_RAW_LOC || FlagOffset == FEXCore::X86State::RFLAG_PF_RAW_LOC) {\n // Already handled\n continue;\n }\n\n // Note that the Bfi only considers the bottom bit of the flag, the rest of\n // the byte is allowed to be garbage.\n Ref Flag;\n if (FlagOffset == FEXCore::X86State::RFLAG_AF_RAW_LOC) {\n Flag = LoadAF();\n } else {\n Flag = GetRFLAG(FlagOffset);\n }\n\n Original = _Orlshl(OpSize::i64Bit, Original, Flag, FlagOffset);\n }\n\n // Raw PF value needs to have its bottom bit masked out and inverted. The\n // naive sequence is and/eor/orlshl. But we can do the inversion implicitly\n // instead.\n if (FlagsMask & (1 << FEXCore::X86State::RFLAG_PF_RAW_LOC)) {\n // Set every bit except the bottommost.\n auto OnesInvPF = _Or(OpSize::i64Bit, LoadPFRaw(false, false), _InlineConstant(~1ull));\n\n // Rotate the bottom bit to the appropriate location for PF, so we get\n // something like 111P1111. Then invert that to get 000p0000. Then OR that\n // into the flags. This is 1 A64 instruction :-)\n auto RightRotation = 64 - FEXCore::X86State::RFLAG_PF_RAW_LOC;\n Original = _Ornror(OpSize::i64Bit, Original, OnesInvPF, RightRotation);\n }\n\n // OR in the SF/ZF flags at the end, allowing the lshr to fold with the OR\n if (GetNZ) {\n static_assert(FEXCore::X86State::RFLAG_SF_RAW_LOC == (FEXCore::X86State::RFLAG_ZF_RAW_LOC + 1));\n auto NZCV = GetNZCV();\n auto NZ = _And(OpSize::i64Bit, NZCV, _InlineConstant(0b11u << 30));\n Original = _Orlshr(OpSize::i64Bit, Original, NZ, 31 - FEXCore::X86State::RFLAG_SF_RAW_LOC);\n }\n\n // The constant is OR'ed in at the end, to avoid a pointless or xzr, #2.\n if ((1U << X86State::RFLAG_RESERVED_LOC) & FlagsMask) {\n Original = _Or(OpSize::i64Bit, Original, _InlineConstant(2));\n }\n\n return Original;\n}\n\nvoid OpDispatchBuilder::CalculateOF(IR::OpSize SrcSize, Ref Res, Ref Src1, Ref Src2, bool Sub) {\n LOGMAN_THROW_A_FMT(SrcSize >= IR::OpSize::i8Bit && SrcSize <= IR::OpSize::i64Bit, \"Invalid size\");\n const auto OpSize = SrcSize == OpSize::i64Bit ? OpSize::i64Bit : OpSize::i32Bit;\n const uint64_t SignBit = IR::OpSizeAsBits(SrcSize) - 1;\n Ref Anded = nullptr;\n\n // For add, OF is set iff the sources have the same sign but the destination\n // sign differs. If we know a source sign, we can simplify the expression: if\n // source 2 is known to be positive, we set OF if source 1 is positive and\n // source 2 is negative. Similarly if source 2 is known negative.\n //\n // For sub, OF is set iff the sources have differing signs and the destination\n // sign matches the second source. If source 2 is known positive, set iff\n // source 1 negative and source 2 positive.\n uint64_t Const;\n if (IsValueConstant(WrapNode(Src2), &Const)) {\n bool Negative = (Const & (1ull << SignBit)) != 0;\n\n if (Negative ^ Sub) {\n Anded = _Andn(OpSize, Src1, Res);\n } else {\n Anded = _Andn(OpSize, Res, Src1);\n }\n } else {\n auto XorOp1 = _Xor(OpSize, Src1, Src2);\n auto XorOp2 = _Xor(OpSize, Res, Src1);\n\n if (Sub) {\n Anded = _And(OpSize, XorOp2, XorOp1);\n } else {\n Anded = _Andn(OpSize, XorOp2, XorOp1);\n }\n }\n\n SetRFLAG(Anded, SignBit, true);\n}\n\nRef OpDispatchBuilder::LoadPFRaw(bool Mask, bool Invert) {\n // Most blocks do not read parity, so PF optimization is gated on this flag.\n CurrentHeader->ReadsParity = true;\n\n // Evaluate parity on the deferred raw value.\n return _Parity(GetRFLAG(FEXCore::X86State::RFLAG_PF_RAW_LOC), Mask, Invert);\n}\n\nRef OpDispatchBuilder::LoadAF() {\n // Read the stored value. This is the XOR of the arguments.\n auto AFWord = GetRFLAG(FEXCore::X86State::RFLAG_AF_RAW_LOC);\n\n // Read the result, stored for PF.\n auto Result = GetRFLAG(FEXCore::X86State::RFLAG_PF_RAW_LOC);\n\n // What's left is to XOR and extract. This is the deferred part. We\n // specifically use a 64-bit Xor here as we don't need masking.\n return _Bfe(OpSize::i32Bit, 1, 4, _Xor(OpSize::i64Bit, AFWord, Result));\n}\n\nvoid OpDispatchBuilder::FixupAF() {\n // The caller has set a desired value of AF in AF[4], regardless of the value\n // of PF. We need to fixup AF[4] so that we get the right value when we XOR in\n // PF[4] later. The easiest solution is to XOR by PF[4], since:\n //\n // (AF[4] ^ PF[4]) ^ PF[4] = AF[4]\n\n auto PFRaw = GetRFLAG(FEXCore::X86State::RFLAG_PF_RAW_LOC);\n auto AFRaw = GetRFLAG(FEXCore::X86State::RFLAG_AF_RAW_LOC);\n\n // Again 64-bit as masking is more expensive.\n Ref XorRes = _Xor(OpSize::i64Bit, AFRaw, PFRaw);\n SetRFLAG(XorRes);\n}\n\nvoid OpDispatchBuilder::SetAFAndFixup(Ref AF) {\n // We have a value of AF, we shift into AF[4]. We need to fixup AF[4] so that\n // we get the right value when we XOR in PF[4] later. The easiest solution is\n // to XOR by PF[4], since:\n //\n // (AF[4] ^ PF[4]) ^ PF[4] = AF[4]\n\n auto PFRaw = GetRFLAG(FEXCore::X86State::RFLAG_PF_RAW_LOC);\n\n Ref XorRes = _XorShift(OpSize::i32Bit, PFRaw, AF, ShiftType::LSL, 4);\n SetRFLAG(XorRes);\n}\n\nvoid OpDispatchBuilder::CalculatePF(Ref Res) {\n // Calculation is entirely deferred until load, just store the 8-bit result.\n SetRFLAG(Res);\n}\n\nvoid OpDispatchBuilder::CalculateAF(Ref Src1, Ref Src2) {\n // We only care about bit 4 in the subsequent XOR. If we'll XOR with 0,\n // there's no sense XOR'ing at all. If we'll XOR with 1, that's just\n // inverting.\n for (unsigned i = 0; i < 2; ++i) {\n Ref SrcA = i ? Src1 : Src2;\n Ref SrcB = i ? Src2 : Src1;\n\n uint64_t Const;\n if (IsValueConstant(WrapNode(SrcA), &Const)) {\n if (Const & (1u << 4)) {\n SetRFLAG(_Not(OpSize::i32Bit, SrcB));\n } else {\n SetRFLAG(SrcB);\n }\n\n return;\n }\n }\n\n // We store the XOR of the arguments. At read time, we XOR with the\n // appropriate bit of the result (available as the PF flag) and extract the\n // appropriate bit. Again 64-bit to avoid masking.\n Ref XorRes = Src1 == Src2 ? Constant(0) : _Xor(OpSize::i64Bit, Src1, Src2);\n SetRFLAG(XorRes);\n}\n\nvoid OpDispatchBuilder::CalculateDeferredFlags() {\n if (NZCVDirty && CachedNZCV) {\n _StoreNZCV(CachedNZCV);\n }\n\n CachedNZCV = nullptr;\n NZCVDirty = false;\n}\n\nRef OpDispatchBuilder::IncrementByCarry(OpSize OpSize, Ref Src) {\n // If CF not inverted, we use .cc since the increment happens when the\n // condition is false. If CF inverted, invert to use .cs. A bit mindbendy.\n return _NZCVSelectIncrement(OpSize, CFInverted ? CondClass::UGE : CondClass::ULT, Src, Src);\n}\n\nRef OpDispatchBuilder::CalculateFlags_ADC(IR::OpSize SrcSize, Ref Src1, Ref Src2) {\n auto OpSize = SrcSize == OpSize::i64Bit ? OpSize::i64Bit : OpSize::i32Bit;\n Ref Res;\n\n CalculateAF(Src1, Src2);\n\n if (SrcSize >= OpSize::i32Bit) {\n RectifyCarryInvert(false);\n HandleNZCV_RMW();\n Res = _AdcWithFlags(OpSize, Src1, Src2);\n CFInverted = false;\n } else {\n // Need to zero-extend for correct comparisons below\n Src2 = ARef(Src2).Bfe(0, IR::OpSizeAsBits(SrcSize)).Ref();\n\n // Note that we do not extend Src2PlusCF, since we depend on proper\n // 32-bit arithmetic to correctly handle the Src2 = 0xffff case.\n Ref Src2PlusCF = IncrementByCarry(OpSize, Src2);\n\n // Need to zero-extend for the comparison.\n Res = Add(OpSize, Src1, Src2PlusCF);\n Res = _Bfe(OpSize, IR::OpSizeAsBits(SrcSize), 0, Res);\n\n // TODO: We can fold that second Bfe in (cmp uxth).\n auto SelectCFInv = Select01(OpSize, CondClass::UGE, Res, Src2PlusCF);\n\n SetNZ_ZeroCV(SrcSize, Res);\n SetCFInverted(SelectCFInv);\n CalculateOF(SrcSize, Res, Src1, Src2, false);\n }\n\n CalculatePF(Res);\n return Res;\n}\n\nRef OpDispatchBuilder::CalculateFlags_SBB(IR::OpSize SrcSize, Ref Src1, Ref Src2) {\n auto OpSize = SrcSize == OpSize::i64Bit ? OpSize::i64Bit : OpSize::i32Bit;\n\n CalculateAF(Src1, Src2);\n\n Ref Res;\n if (SrcSize >= OpSize::i32Bit) {\n // Arm's subtraction has inverted CF from x86, so rectify the input and\n // invert the output.\n RectifyCarryInvert(true);\n HandleNZCV_RMW();\n Res = _SbbWithFlags(OpSize, Src1, Src2);\n CFInverted = true;\n } else {\n // Zero extend for correct comparison behaviour with Src1 = 0xffff.\n Src1 = _Bfe(OpSize, IR::OpSizeAsBits(SrcSize), 0, Src1);\n Src2 = ARef(Src2).Bfe(0, IR::OpSizeAsBits(SrcSize)).Ref();\n\n auto Src2PlusCF = IncrementByCarry(OpSize, Src2);\n\n Res = Sub(OpSize, Src1, Src2PlusCF);\n Res = _Bfe(OpSize, IR::OpSizeAsBits(SrcSize), 0, Res);\n\n auto SelectCFInv = Select01(OpSize, CondClass::UGE, Src1, Src2PlusCF);\n\n SetNZ_ZeroCV(SrcSize, Res);\n SetCFInverted(SelectCFInv);\n CalculateOF(SrcSize, Res, Src1, Src2, true);\n }\n\n CalculatePF(Res);\n return Res;\n}\n\nRef OpDispatchBuilder::CalculateFlags_SUB(IR::OpSize SrcSize, Ref Src1, Ref Src2, bool UpdateCF) {\n // Stash CF before stomping over it\n auto OldCFInv = UpdateCF ? nullptr : GetRFLAG(FEXCore::X86State::RFLAG_CF_RAW_LOC, true);\n\n HandleNZCVWrite();\n\n CalculateAF(Src1, Src2);\n\n Ref Res;\n if (SrcSize >= OpSize::i32Bit) {\n Res = SubWithFlags(SrcSize, Src1, Src2);\n } else {\n _SubNZCV(SrcSize, Src1, Src2);\n Res = Sub(OpSize::i32Bit, Src1, Src2);\n }\n\n CalculatePF(Res);\n\n // If we're updating CF, we need it to be inverted because SubNZCV is inverted\n // from x86. If we're not updating CF, we need to restore the CF since we\n // stomped over it.\n if (UpdateCF) {\n CFInverted = true;\n } else {\n SetCFInverted(OldCFInv);\n }\n\n return Res;\n}\n\nRef OpDispatchBuilder::CalculateFlags_ADD(IR::OpSize SrcSize, Ref Src1, Ref Src2, bool UpdateCF) {\n // Stash CF before stomping over it\n auto OldCFInv = UpdateCF ? nullptr : GetRFLAG(FEXCore::X86State::RFLAG_CF_RAW_LOC, true);\n\n HandleNZCVWrite();\n\n CalculateAF(Src1, Src2);\n\n Ref Res;\n if (SrcSize >= OpSize::i32Bit) {\n Res = AddWithFlags(SrcSize, Src1, Src2);\n } else {\n _AddNZCV(SrcSize, Src1, Src2);\n Res = Add(OpSize::i32Bit, Src1, Src2);\n }\n\n CalculatePF(Res);\n\n // We stomped over CF while calculation flags, restore it.\n if (UpdateCF) {\n // Adds match between x86 and arm64.\n CFInverted = false;\n } else {\n SetCFInverted(OldCFInv);\n }\n\n return Res;\n}\n\nvoid OpDispatchBuilder::CalculateFlags_MUL(IR::OpSize SrcSize, Ref Res, Ref High) {\n HandleNZCVWrite();\n InvalidatePF_AF();\n\n // CF and OF are set if the result of the operation can't be fit in to the destination register\n // If the value can fit then the top bits will be zero\n auto SignBit = _Sbfe(OpSize::i64Bit, 1, IR::OpSizeAsBits(SrcSize) - 1, Res);\n _SubNZCV(OpSize::i64Bit, High, SignBit);\n\n // If High = SignBit, then sets to nZCv. Else sets to nzcV. Since SF/ZF\n // undefined, this does what we need after inverting carry.\n auto Zero = _InlineConstant(0);\n _CondSubNZCV(OpSize::i64Bit, Zero, Zero, CondClass::EQ, 0x1 /* nzcV */);\n CFInverted = true;\n}\n\nvoid OpDispatchBuilder::CalculateFlags_UMUL(Ref High) {\n HandleNZCVWrite();\n InvalidatePF_AF();\n\n auto Zero = _InlineConstant(0);\n const auto Size = GetOpSize(High);\n\n // CF and OF are set if the result of the operation can't be fit in to the destination register\n // The result register will be all zero if it can't fit due to how multiplication behaves\n _SubNZCV(Size, High, Zero);\n\n // If High = 0, then sets to nZCv. Else sets to nzcV. Since SF/ZF undefined,\n // this does what we need.\n _CondSubNZCV(Size, Zero, Zero, CondClass::EQ, 0x1 /* nzcV */);\n CFInverted = true;\n}\n\nvoid OpDispatchBuilder::CalculateFlags_Logical(IR::OpSize SrcSize, Ref Res) {\n InvalidateAF();\n SetNZP_ZeroCV(SrcSize, Res);\n}\n\nvoid OpDispatchBuilder::CalculateFlags_ShiftLeftImmediate(IR::OpSize SrcSize, Ref UnmaskedRes, Ref Src1, uint64_t Shift) {\n // No flags changed if shift is zero\n if (Shift == 0) {\n return;\n }\n\n auto OpSize = SrcSize == OpSize::i64Bit ? OpSize::i64Bit : OpSize::i32Bit;\n\n SetNZ_ZeroCV(SrcSize, UnmaskedRes);\n\n // CF\n {\n // Extract the last bit shifted in to CF. Shift is already masked, but for\n // 8/16-bit it might be >= SrcSizeBits, in which case CF is cleared. There's\n // nothing to do in that case since we already cleared CF above.\n const auto SrcSizeBits = IR::OpSizeAsBits(SrcSize);\n if (Shift < SrcSizeBits) {\n SetCFDirect(Src1, SrcSizeBits - Shift, true);\n }\n }\n\n CalculatePF(UnmaskedRes);\n InvalidateAF();\n\n // OF\n // In the case of left shift. OF is only set from the result of XOR \n if (Shift == 1) {\n auto Xor = _Xor(OpSize, UnmaskedRes, Src1);\n SetRFLAG(Xor, IR::OpSizeAsBits(SrcSize) - 1, true);\n } else {\n // Undefined, we choose to zero as part of SetNZ_ZeroCV\n }\n}\n\nvoid OpDispatchBuilder::CalculateFlags_SignShiftRightImmediate(IR::OpSize SrcSize, Ref Res, Ref Src1, uint64_t Shift) {\n // No flags changed if shift is zero\n if (Shift == 0) {\n return;\n }\n\n SetNZ_ZeroCV(SrcSize, Res);\n\n // Extract the last bit shifted in to CF\n SetCFDirect(Src1, Shift - 1, true);\n\n CalculatePF(Res);\n InvalidateAF();\n\n // OF\n // Only defined when Shift is 1 else undefined. Only is set if the top bit was set to 1 when\n // shifted So it is set to zero. In the undefined case we choose to zero as well. Since it was\n // already zeroed there's nothing to do here.\n}\n\nvoid OpDispatchBuilder::CalculateFlags_ShiftRightImmediateCommon(IR::OpSize SrcSize, Ref Res, Ref Src1, uint64_t Shift) {\n // Set SF and PF. Clobbers OF, but OF only defined for Shift = 1 where it is\n // set below.\n SetNZ_ZeroCV(SrcSize, Res);\n\n // Extract the last bit shifted in to CF\n SetCFDirect(Src1, Shift - 1, true);\n\n CalculatePF(Res);\n InvalidateAF();\n}\n\nvoid OpDispatchBuilder::CalculateFlags_ShiftRightImmediate(IR::OpSize SrcSize, Ref Res, Ref Src1, uint64_t Shift) {\n // No flags changed if shift is zero\n if (Shift == 0) {\n return;\n }\n\n CalculateFlags_ShiftRightImmediateCommon(SrcSize, Res, Src1, Shift);\n\n // OF\n {\n // Only defined when Shift is 1 else undefined\n // Is set to the MSB of the original value\n if (Shift == 1) {\n SetRFLAG(Src1, IR::OpSizeAsBits(SrcSize) - 1, true);\n }\n }\n}\n\nvoid OpDispatchBuilder::CalculateFlags_ShiftRightDoubleImmediate(IR::OpSize SrcSize, Ref Res, Ref Src1, uint64_t Shift) {\n // No flags changed if shift is zero\n if (Shift == 0) {\n return;\n }\n\n const auto OpSize = SrcSize == OpSize::i64Bit ? OpSize::i64Bit : OpSize::i32Bit;\n CalculateFlags_ShiftRightImmediateCommon(SrcSize, Res, Src1, Shift);\n\n // OF\n {\n // Only defined when Shift is 1 else undefined\n // Is set if the MSB bit changes.\n // XOR of Result and Src1\n if (Shift == 1) {\n auto val = _Xor(OpSize, Src1, Res);\n SetRFLAG(val, IR::OpSizeAsBits(SrcSize) - 1, true);\n }\n }\n}\n\nvoid OpDispatchBuilder::CalculateFlags_ZCNT(IR::OpSize SrcSize, Ref Result) {\n // OF, SF, AF, PF all undefined\n // Test ZF of result, SF is undefined so this is ok.\n SetNZ_ZeroCV(SrcSize, Result);\n\n // Now set CF if the Result = SrcSize * 8. Since SrcSize is a power-of-two and\n // Result is <= SrcSize * 8, we equivalently check if the log2(SrcSize * 8)\n // bit is set. No masking is needed because no higher bits could be set.\n unsigned CarryBit = FEXCore::ilog2(IR::OpSizeAsBits(SrcSize));\n SetCFDirect(Result, CarryBit);\n}\n\n} // namespace FEXCore::IR\n" }, { "path": "FEXCore/Source/Interface/Core/OpcodeDispatcher/H0F38Tables.h", "content": "// SPDX-License-Identifier: MIT\n#pragma once\n#include \"Interface/Core/OpcodeDispatcher.h\"\n\nnamespace FEXCore::IR {\n#define OPD(prefix, opcode) (((prefix) << 8) | opcode)\nconstexpr uint16_t PF_38_NONE = 0;\nconstexpr uint16_t PF_38_66 = (1U << 0);\nconstexpr uint16_t PF_38_F2 = (1U << 1);\nconstexpr uint16_t PF_38_F3 = (1U << 2);\n\nconstexpr DispatchTableEntry OpDispatch_H0F38Table[] = {\n {OPD(PF_38_NONE, 0x00), 1, &OpDispatchBuilder::PSHUFBOp},\n {OPD(PF_38_66, 0x00), 1, &OpDispatchBuilder::PSHUFBOp},\n {OPD(PF_38_NONE, 0x01), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VADDP, OpSize::i16Bit>},\n {OPD(PF_38_66, 0x01), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VADDP, OpSize::i16Bit>},\n {OPD(PF_38_NONE, 0x02), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VADDP, OpSize::i32Bit>},\n {OPD(PF_38_66, 0x02), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VADDP, OpSize::i32Bit>},\n {OPD(PF_38_NONE, 0x03), 1, &OpDispatchBuilder::PHADDS},\n {OPD(PF_38_66, 0x03), 1, &OpDispatchBuilder::PHADDS},\n {OPD(PF_38_NONE, 0x04), 1, &OpDispatchBuilder::PMADDUBSW},\n {OPD(PF_38_66, 0x04), 1, &OpDispatchBuilder::PMADDUBSW},\n {OPD(PF_38_NONE, 0x05), 1, &OpDispatchBuilder::PHSUB},\n {OPD(PF_38_66, 0x05), 1, &OpDispatchBuilder::PHSUB},\n {OPD(PF_38_NONE, 0x06), 1, &OpDispatchBuilder::PHSUB},\n {OPD(PF_38_66, 0x06), 1, &OpDispatchBuilder::PHSUB},\n {OPD(PF_38_NONE, 0x07), 1, &OpDispatchBuilder::PHSUBS},\n {OPD(PF_38_66, 0x07), 1, &OpDispatchBuilder::PHSUBS},\n {OPD(PF_38_NONE, 0x08), 1, &OpDispatchBuilder::PSIGN},\n {OPD(PF_38_66, 0x08), 1, &OpDispatchBuilder::PSIGN},\n {OPD(PF_38_NONE, 0x09), 1, &OpDispatchBuilder::PSIGN},\n {OPD(PF_38_66, 0x09), 1, &OpDispatchBuilder::PSIGN},\n {OPD(PF_38_NONE, 0x0A), 1, &OpDispatchBuilder::PSIGN},\n {OPD(PF_38_66, 0x0A), 1, &OpDispatchBuilder::PSIGN},\n {OPD(PF_38_NONE, 0x0B), 1, &OpDispatchBuilder::PMULHRSW},\n {OPD(PF_38_66, 0x0B), 1, &OpDispatchBuilder::PMULHRSW},\n {OPD(PF_38_66, 0x10), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorVariableBlend, OpSize::i8Bit>},\n {OPD(PF_38_66, 0x14), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorVariableBlend, OpSize::i32Bit>},\n {OPD(PF_38_66, 0x15), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorVariableBlend, OpSize::i64Bit>},\n {OPD(PF_38_66, 0x17), 1, &OpDispatchBuilder::PTestOp},\n {OPD(PF_38_NONE, 0x1C), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorUnaryOp, IR::OP_VABS, OpSize::i8Bit>},\n {OPD(PF_38_66, 0x1C), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorUnaryOp, IR::OP_VABS, OpSize::i8Bit>},\n {OPD(PF_38_NONE, 0x1D), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorUnaryOp, IR::OP_VABS, OpSize::i16Bit>},\n {OPD(PF_38_66, 0x1D), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorUnaryOp, IR::OP_VABS, OpSize::i16Bit>},\n {OPD(PF_38_NONE, 0x1E), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorUnaryOp, IR::OP_VABS, OpSize::i32Bit>},\n {OPD(PF_38_66, 0x1E), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorUnaryOp, IR::OP_VABS, OpSize::i32Bit>},\n {OPD(PF_38_66, 0x20), 1, &OpDispatchBuilder::ExtendVectorElements},\n {OPD(PF_38_66, 0x21), 1, &OpDispatchBuilder::ExtendVectorElements},\n {OPD(PF_38_66, 0x22), 1, &OpDispatchBuilder::ExtendVectorElements},\n {OPD(PF_38_66, 0x23), 1, &OpDispatchBuilder::ExtendVectorElements},\n {OPD(PF_38_66, 0x24), 1, &OpDispatchBuilder::ExtendVectorElements},\n {OPD(PF_38_66, 0x25), 1, &OpDispatchBuilder::ExtendVectorElements},\n {OPD(PF_38_66, 0x28), 1, &OpDispatchBuilder::PMULLOp},\n {OPD(PF_38_66, 0x29), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VCMPEQ, OpSize::i64Bit>},\n {OPD(PF_38_66, 0x2A), 1, &OpDispatchBuilder::MOVVectorNTOp},\n {OPD(PF_38_66, 0x2B), 1, &OpDispatchBuilder::PACKUSOp},\n {OPD(PF_38_66, 0x30), 1, &OpDispatchBuilder::ExtendVectorElements},\n {OPD(PF_38_66, 0x31), 1, &OpDispatchBuilder::ExtendVectorElements},\n {OPD(PF_38_66, 0x32), 1, &OpDispatchBuilder::ExtendVectorElements},\n {OPD(PF_38_66, 0x33), 1, &OpDispatchBuilder::ExtendVectorElements},\n {OPD(PF_38_66, 0x34), 1, &OpDispatchBuilder::ExtendVectorElements},\n {OPD(PF_38_66, 0x35), 1, &OpDispatchBuilder::ExtendVectorElements},\n {OPD(PF_38_66, 0x37), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VCMPGT, OpSize::i64Bit>},\n {OPD(PF_38_66, 0x38), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VSMIN, OpSize::i8Bit>},\n {OPD(PF_38_66, 0x39), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VSMIN, OpSize::i32Bit>},\n {OPD(PF_38_66, 0x3A), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VUMIN, OpSize::i16Bit>},\n {OPD(PF_38_66, 0x3B), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VUMIN, OpSize::i32Bit>},\n {OPD(PF_38_66, 0x3C), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VSMAX, OpSize::i8Bit>},\n {OPD(PF_38_66, 0x3D), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VSMAX, OpSize::i32Bit>},\n {OPD(PF_38_66, 0x3E), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VUMAX, OpSize::i16Bit>},\n {OPD(PF_38_66, 0x3F), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VUMAX, OpSize::i32Bit>},\n {OPD(PF_38_66, 0x40), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VMUL, OpSize::i32Bit>},\n {OPD(PF_38_66, 0x41), 1, &OpDispatchBuilder::PHMINPOSUWOp},\n\n {OPD(PF_38_NONE, 0xC8), 1, &OpDispatchBuilder::SHA1NEXTEOp},\n {OPD(PF_38_NONE, 0xC9), 1, &OpDispatchBuilder::SHA1MSG1Op},\n {OPD(PF_38_NONE, 0xCA), 1, &OpDispatchBuilder::SHA1MSG2Op},\n {OPD(PF_38_NONE, 0xCB), 1, &OpDispatchBuilder::SHA256RNDS2Op},\n {OPD(PF_38_NONE, 0xCC), 1, &OpDispatchBuilder::SHA256MSG1Op},\n {OPD(PF_38_NONE, 0xCD), 1, &OpDispatchBuilder::SHA256MSG2Op},\n\n {OPD(PF_38_66, 0xDB), 1, &OpDispatchBuilder::AESImcOp},\n {OPD(PF_38_66, 0xDC), 1, &OpDispatchBuilder::AESEncOp},\n {OPD(PF_38_66, 0xDD), 1, &OpDispatchBuilder::AESEncLastOp},\n {OPD(PF_38_66, 0xDE), 1, &OpDispatchBuilder::AESDecOp},\n {OPD(PF_38_66, 0xDF), 1, &OpDispatchBuilder::AESDecLastOp},\n\n {OPD(PF_38_NONE, 0xF0), 2, &OpDispatchBuilder::MOVBEOp},\n {OPD(PF_38_66, 0xF0), 2, &OpDispatchBuilder::MOVBEOp},\n\n {OPD(PF_38_F2, 0xF0), 1, &OpDispatchBuilder::CRC32},\n {OPD(PF_38_F2, 0xF1), 1, &OpDispatchBuilder::CRC32},\n\n {OPD(PF_38_66 | PF_38_F2, 0xF0), 1, &OpDispatchBuilder::CRC32},\n {OPD(PF_38_66 | PF_38_F2, 0xF1), 1, &OpDispatchBuilder::CRC32},\n\n {OPD(PF_38_66, 0xF6), 1, &OpDispatchBuilder::ADXOp},\n {OPD(PF_38_F3, 0xF6), 1, &OpDispatchBuilder::ADXOp},\n};\n#undef OPD\n\n} // namespace FEXCore::IR\n" }, { "path": "FEXCore/Source/Interface/Core/OpcodeDispatcher/H0F3ATables.h", "content": "// SPDX-License-Identifier: MIT\n#pragma once\n#include \"Interface/Core/OpcodeDispatcher.h\"\n\nnamespace FEXCore::IR {\n#define OPD(REX, prefix, opcode) ((REX << 9) | (prefix << 8) | opcode)\n#define PF_3A_NONE 0\n#define PF_3A_66 1\nconstexpr auto OpDispatchTableGenH0F3A = []() consteval {\n constexpr auto OpDispatchTableGenH0F3AREX = []() consteval {\n constexpr DispatchTableEntry Table[] = {\n {OPD(REX, PF_3A_66, 0x08), 1, &OpDispatchBuilder::VectorRound},\n {OPD(REX, PF_3A_66, 0x09), 1, &OpDispatchBuilder::VectorRound},\n {OPD(REX, PF_3A_66, 0x0A), 1, &OpDispatchBuilder::InsertScalarRound},\n {OPD(REX, PF_3A_66, 0x0B), 1, &OpDispatchBuilder::InsertScalarRound},\n {OPD(REX, PF_3A_66, 0x0C), 1, &OpDispatchBuilder::VectorBlend},\n {OPD(REX, PF_3A_66, 0x0D), 1, &OpDispatchBuilder::VectorBlend},\n {OPD(REX, PF_3A_66, 0x0E), 1, &OpDispatchBuilder::VectorBlend},\n\n {OPD(REX, PF_3A_NONE, 0x0F), 1, &OpDispatchBuilder::PAlignrOp},\n {OPD(REX, PF_3A_66, 0x0F), 1, &OpDispatchBuilder::PAlignrOp},\n\n {OPD(REX, PF_3A_66, 0x14), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PExtrOp, OpSize::i8Bit>},\n {OPD(REX, PF_3A_66, 0x15), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PExtrOp, OpSize::i16Bit>},\n {OPD(REX, PF_3A_66, 0x17), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PExtrOp, OpSize::i32Bit>},\n\n {OPD(REX, PF_3A_66, 0x20), 1, &OpDispatchBuilder::PINSROp},\n {OPD(REX, PF_3A_66, 0x21), 1, &OpDispatchBuilder::InsertPSOp},\n {OPD(REX, PF_3A_66, 0x40), 1, &OpDispatchBuilder::DPPOp},\n {OPD(REX, PF_3A_66, 0x41), 1, &OpDispatchBuilder::DPPOp},\n {OPD(REX, PF_3A_66, 0x42), 1, &OpDispatchBuilder::MPSADBWOp},\n {OPD(REX, PF_3A_66, 0x44), 1, &OpDispatchBuilder::PCLMULQDQOp},\n\n {OPD(REX, PF_3A_66, 0x60), 1, &OpDispatchBuilder::VPCMPESTRMOp},\n {OPD(REX, PF_3A_66, 0x61), 1, &OpDispatchBuilder::VPCMPESTRIOp},\n {OPD(REX, PF_3A_66, 0x62), 1, &OpDispatchBuilder::VPCMPISTRMOp},\n {OPD(REX, PF_3A_66, 0x63), 1, &OpDispatchBuilder::VPCMPISTRIOp},\n\n {OPD(REX, PF_3A_NONE, 0xCC), 1, &OpDispatchBuilder::SHA1RNDS4Op},\n {OPD(REX, PF_3A_66, 0xDF), 1, &OpDispatchBuilder::AESKeyGenAssist},\n\n };\n return std::to_array(Table);\n };\n\n auto REX0 = OpDispatchTableGenH0F3AREX.template operator()<0>();\n auto REX1 = OpDispatchTableGenH0F3AREX.template operator()<1>();\n auto concat = [](const std::array& lhs,\n const std::array& rhs) consteval -> std::array {\n std::array Table {};\n for (size_t i = 0; i < N1; ++i) {\n Table[i] = lhs[i];\n }\n\n for (size_t i = 0; i < N2; ++i) {\n Table[N1 + i] = rhs[i];\n }\n\n return Table;\n };\n return concat(REX0, REX1);\n};\n\nconstexpr auto OpDispatch_H0F3ATableIgnoreREX = OpDispatchTableGenH0F3A();\n\nconstexpr DispatchTableEntry OpDispatch_H0F3ATableNeedsREX0[] = {\n {OPD(0, PF_3A_66, 0x16), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PExtrOp, OpSize::i32Bit>},\n {OPD(0, PF_3A_66, 0x22), 1, &OpDispatchBuilder::PINSROp},\n};\n\n#undef PF_3A_NONE\n#undef PF_3A_66\n\n#undef OPD\n} // namespace FEXCore::IR\n" }, { "path": "FEXCore/Source/Interface/Core/OpcodeDispatcher/PrimaryGroupTables.h", "content": "// SPDX-License-Identifier: MIT\n#pragma once\n#include \"Interface/Core/OpcodeDispatcher.h\"\n\nnamespace FEXCore::IR {\nusing X86Tables::OpToIndex;\n#define OPD(group, prefix, Reg) (((group - FEXCore::X86Tables::TYPE_GROUP_1) << 6) | (prefix) << 3 | (Reg))\nconstexpr DispatchTableEntry OpDispatch_PrimaryGroupTables[] = {\n // GROUP 1\n {OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x80), 0), 1, &OpDispatchBuilder::SecondaryALUOp},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x80), 1), 1, &OpDispatchBuilder::SecondaryALUOp},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x80), 2), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::ADCOp, 1>},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x80), 3), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::SBBOp, 1>},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x80), 4), 1, &OpDispatchBuilder::SecondaryALUOp},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x80), 5), 1, &OpDispatchBuilder::SecondaryALUOp},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x80), 6), 1, &OpDispatchBuilder::SecondaryALUOp},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x80), 7), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::CMPOp, 1>}, // CMP\n\n {OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x81), 0), 1, &OpDispatchBuilder::SecondaryALUOp},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x81), 1), 1, &OpDispatchBuilder::SecondaryALUOp},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x81), 2), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::ADCOp, 1>},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x81), 3), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::SBBOp, 1>},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x81), 4), 1, &OpDispatchBuilder::SecondaryALUOp},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x81), 5), 1, &OpDispatchBuilder::SecondaryALUOp},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x81), 6), 1, &OpDispatchBuilder::SecondaryALUOp},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x81), 7), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::CMPOp, 1>},\n\n {OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x83), 0), 1, &OpDispatchBuilder::SecondaryALUOp},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x83), 1), 1, &OpDispatchBuilder::SecondaryALUOp},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x83), 2), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::ADCOp, 1>},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x83), 3), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::SBBOp, 1>},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x83), 4), 1, &OpDispatchBuilder::SecondaryALUOp},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x83), 5), 1, &OpDispatchBuilder::SecondaryALUOp},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x83), 6), 1, &OpDispatchBuilder::SecondaryALUOp},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x83), 7), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::CMPOp, 1>},\n\n // GROUP 2\n {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xC0), 0), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::RotateOp, true, true, false>},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xC0), 1), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::RotateOp, false, true, false>},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xC0), 2), 1, &OpDispatchBuilder::RCLOp},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xC0), 3), 1, &OpDispatchBuilder::RCROp},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xC0), 4), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::SHLImmediateOp, false>},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xC0), 5), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::SHRImmediateOp, false>},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xC0), 6), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::SHLImmediateOp, false>}, // SAL\n {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xC0), 7), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::ASHROp, true, false>}, // SAR\n\n {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xC1), 0), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::RotateOp, true, true, false>},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xC1), 1), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::RotateOp, false, true, false>},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xC1), 2), 1, &OpDispatchBuilder::RCLOp},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xC1), 3), 1, &OpDispatchBuilder::RCROp},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xC1), 4), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::SHLImmediateOp, false>},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xC1), 5), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::SHRImmediateOp, false>},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xC1), 6), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::SHLImmediateOp, false>}, // SAL\n {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xC1), 7), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::ASHROp, true, false>}, // SAR\n\n {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD0), 0), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::RotateOp, true, true, true>},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD0), 1), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::RotateOp, false, true, true>},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD0), 2), 1, &OpDispatchBuilder::RCLOp1Bit},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD0), 3), 1, &OpDispatchBuilder::RCROp8x1Bit},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD0), 4), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::SHLImmediateOp, true>},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD0), 5), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::SHRImmediateOp, true>}, // 1Bit SHR\n {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD0), 6), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::SHLImmediateOp, true>}, // SAL\n {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD0), 7), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::ASHROp, true, true>}, // SAR\n\n {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD1), 0), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::RotateOp, true, true, true>},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD1), 1), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::RotateOp, false, true, true>},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD1), 2), 1, &OpDispatchBuilder::RCLOp1Bit},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD1), 3), 1, &OpDispatchBuilder::RCROp1Bit},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD1), 4), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::SHLImmediateOp, true>},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD1), 5), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::SHRImmediateOp, true>}, // 1Bit SHR\n {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD1), 6), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::SHLImmediateOp, true>}, // SAL\n {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD1), 7), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::ASHROp, true, true>}, // SAR\n\n {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD2), 0), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::RotateOp, true, false, false>},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD2), 1), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::RotateOp, false, false, false>},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD2), 2), 1, &OpDispatchBuilder::RCLSmallerOp},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD2), 3), 1, &OpDispatchBuilder::RCRSmallerOp},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD2), 4), 1, &OpDispatchBuilder::SHLOp},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD2), 5), 1, &OpDispatchBuilder::SHROp}, // SHR by CL\n {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD2), 6), 1, &OpDispatchBuilder::SHLOp}, // SAL\n {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD2), 7), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::ASHROp, false, false>}, // SAR\n\n {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD3), 0), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::RotateOp, true, false, false>},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD3), 1), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::RotateOp, false, false, false>},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD3), 2), 1, &OpDispatchBuilder::RCLOp},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD3), 3), 1, &OpDispatchBuilder::RCROp},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD3), 4), 1, &OpDispatchBuilder::SHLOp},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD3), 5), 1, &OpDispatchBuilder::SHROp}, // SHR by CL\n {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD3), 6), 1, &OpDispatchBuilder::SHLOp}, // SAL\n {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD3), 7), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::ASHROp, false, false>}, // SAR\n\n // GROUP 3\n {OPD(FEXCore::X86Tables::TYPE_GROUP_3, OpToIndex(0xF6), 0), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::TESTOp, 1>},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_3, OpToIndex(0xF6), 1), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::TESTOp, 1>},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_3, OpToIndex(0xF6), 2), 1, &OpDispatchBuilder::NOTOp},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_3, OpToIndex(0xF6), 3), 1, &OpDispatchBuilder::NEGOp}, // NEG\n {OPD(FEXCore::X86Tables::TYPE_GROUP_3, OpToIndex(0xF6), 4), 1, &OpDispatchBuilder::MULOp},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_3, OpToIndex(0xF6), 5), 1, &OpDispatchBuilder::IMULOp},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_3, OpToIndex(0xF6), 6), 1, &OpDispatchBuilder::DIVOp}, // DIV\n {OPD(FEXCore::X86Tables::TYPE_GROUP_3, OpToIndex(0xF6), 7), 1, &OpDispatchBuilder::IDIVOp}, // IDIV\n\n {OPD(FEXCore::X86Tables::TYPE_GROUP_3, OpToIndex(0xF7), 0), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::TESTOp, 1>},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_3, OpToIndex(0xF7), 1), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::TESTOp, 1>},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_3, OpToIndex(0xF7), 2), 1, &OpDispatchBuilder::NOTOp},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_3, OpToIndex(0xF7), 3), 1, &OpDispatchBuilder::NEGOp}, // NEG\n\n {OPD(FEXCore::X86Tables::TYPE_GROUP_3, OpToIndex(0xF7), 4), 1, &OpDispatchBuilder::MULOp}, // MUL\n {OPD(FEXCore::X86Tables::TYPE_GROUP_3, OpToIndex(0xF7), 5), 1, &OpDispatchBuilder::IMULOp},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_3, OpToIndex(0xF7), 6), 1, &OpDispatchBuilder::DIVOp}, // DIV\n {OPD(FEXCore::X86Tables::TYPE_GROUP_3, OpToIndex(0xF7), 7), 1, &OpDispatchBuilder::IDIVOp}, // IDIV\n\n // GROUP 4\n {OPD(FEXCore::X86Tables::TYPE_GROUP_4, OpToIndex(0xFE), 0), 1, &OpDispatchBuilder::INCOp}, // INC\n {OPD(FEXCore::X86Tables::TYPE_GROUP_4, OpToIndex(0xFE), 1), 1, &OpDispatchBuilder::DECOp}, // DEC\n\n // GROUP 5\n {OPD(FEXCore::X86Tables::TYPE_GROUP_5, OpToIndex(0xFF), 0), 1, &OpDispatchBuilder::INCOp}, // INC\n {OPD(FEXCore::X86Tables::TYPE_GROUP_5, OpToIndex(0xFF), 1), 1, &OpDispatchBuilder::DECOp}, // DEC\n {OPD(FEXCore::X86Tables::TYPE_GROUP_5, OpToIndex(0xFF), 2), 1, &OpDispatchBuilder::CALLAbsoluteOp},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_5, OpToIndex(0xFF), 3), 1, &OpDispatchBuilder::CALLFARIndirectOp},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_5, OpToIndex(0xFF), 4), 1, &OpDispatchBuilder::JUMPAbsoluteOp},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_5, OpToIndex(0xFF), 5), 1, &OpDispatchBuilder::JUMPFARIndirectOp},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_5, OpToIndex(0xFF), 6), 1, &OpDispatchBuilder::PUSHOp},\n\n // GROUP 11\n {OPD(FEXCore::X86Tables::TYPE_GROUP_11, OpToIndex(0xC6), 0), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::MOVGPROp, 1>},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_11, OpToIndex(0xC7), 0), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::MOVGPROp, 1>},\n};\n#undef OPD\n\n} // namespace FEXCore::IR\n" }, { "path": "FEXCore/Source/Interface/Core/OpcodeDispatcher/SecondaryGroupTables.h", "content": "// SPDX-License-Identifier: MIT\n#pragma once\n#include \"Interface/Core/OpcodeDispatcher.h\"\n\nnamespace FEXCore::IR {\n#define OPD(group, prefix, Reg) (((group - FEXCore::X86Tables::TYPE_GROUP_6) << 5) | (prefix) << 3 | (Reg))\nconstexpr uint16_t PF_NONE = 0;\nconstexpr uint16_t PF_F3 = 1;\nconstexpr uint16_t PF_66 = 2;\nconstexpr uint16_t PF_F2 = 3;\nconstexpr DispatchTableEntry OpDispatch_SecondaryGroupTables[] = {\n // GROUP 6\n {OPD(FEXCore::X86Tables::TYPE_GROUP_6, PF_NONE, 3), 1, &OpDispatchBuilder::PermissionRestrictedOp},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_6, PF_F3, 3), 1, &OpDispatchBuilder::PermissionRestrictedOp},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_6, PF_66, 3), 1, &OpDispatchBuilder::PermissionRestrictedOp},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_6, PF_F2, 3), 1, &OpDispatchBuilder::PermissionRestrictedOp},\n\n // GROUP 7\n {OPD(FEXCore::X86Tables::TYPE_GROUP_7, PF_NONE, 0), 1, &OpDispatchBuilder::SGDTOp},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_7, PF_F3, 0), 1, &OpDispatchBuilder::SGDTOp},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_7, PF_66, 0), 1, &OpDispatchBuilder::SGDTOp},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_7, PF_F2, 0), 1, &OpDispatchBuilder::SGDTOp},\n\n {OPD(FEXCore::X86Tables::TYPE_GROUP_7, PF_NONE, 1), 1, &OpDispatchBuilder::SIDTOp},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_7, PF_F3, 1), 1, &OpDispatchBuilder::SIDTOp},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_7, PF_66, 1), 1, &OpDispatchBuilder::SIDTOp},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_7, PF_F2, 1), 1, &OpDispatchBuilder::SIDTOp},\n\n {OPD(FEXCore::X86Tables::TYPE_GROUP_7, PF_NONE, 3), 1, &OpDispatchBuilder::PermissionRestrictedOp},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_7, PF_F3, 3), 1, &OpDispatchBuilder::PermissionRestrictedOp},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_7, PF_66, 3), 1, &OpDispatchBuilder::PermissionRestrictedOp},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_7, PF_F2, 3), 1, &OpDispatchBuilder::PermissionRestrictedOp},\n\n {OPD(FEXCore::X86Tables::TYPE_GROUP_7, PF_NONE, 4), 1, &OpDispatchBuilder::SMSWOp},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_7, PF_F3, 4), 1, &OpDispatchBuilder::SMSWOp},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_7, PF_66, 4), 1, &OpDispatchBuilder::SMSWOp},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_7, PF_F2, 4), 1, &OpDispatchBuilder::SMSWOp},\n\n {OPD(FEXCore::X86Tables::TYPE_GROUP_7, PF_NONE, 6), 1, &OpDispatchBuilder::PermissionRestrictedOp},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_7, PF_F3, 6), 1, &OpDispatchBuilder::PermissionRestrictedOp},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_7, PF_66, 6), 1, &OpDispatchBuilder::PermissionRestrictedOp},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_7, PF_F2, 6), 1, &OpDispatchBuilder::PermissionRestrictedOp},\n\n {OPD(FEXCore::X86Tables::TYPE_GROUP_7, PF_NONE, 7), 1, &OpDispatchBuilder::PermissionRestrictedOp},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_7, PF_F3, 7), 1, &OpDispatchBuilder::PermissionRestrictedOp},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_7, PF_66, 7), 1, &OpDispatchBuilder::PermissionRestrictedOp},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_7, PF_F2, 7), 1, &OpDispatchBuilder::PermissionRestrictedOp},\n\n // GROUP 8\n {OPD(FEXCore::X86Tables::TYPE_GROUP_8, PF_NONE, 4), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::BTOp, 1, BTAction::BTNone>},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_8, PF_F3, 4), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::BTOp, 1, BTAction::BTNone>},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_8, PF_66, 4), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::BTOp, 1, BTAction::BTNone>},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_8, PF_F2, 4), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::BTOp, 1, BTAction::BTNone>},\n\n {OPD(FEXCore::X86Tables::TYPE_GROUP_8, PF_NONE, 5), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::BTOp, 1, BTAction::BTSet>},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_8, PF_F3, 5), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::BTOp, 1, BTAction::BTSet>},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_8, PF_66, 5), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::BTOp, 1, BTAction::BTSet>},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_8, PF_F2, 5), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::BTOp, 1, BTAction::BTSet>},\n\n {OPD(FEXCore::X86Tables::TYPE_GROUP_8, PF_NONE, 6), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::BTOp, 1, BTAction::BTClear>},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_8, PF_F3, 6), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::BTOp, 1, BTAction::BTClear>},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_8, PF_66, 6), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::BTOp, 1, BTAction::BTClear>},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_8, PF_F2, 6), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::BTOp, 1, BTAction::BTClear>},\n\n {OPD(FEXCore::X86Tables::TYPE_GROUP_8, PF_NONE, 7), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::BTOp, 1, BTAction::BTComplement>},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_8, PF_F3, 7), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::BTOp, 1, BTAction::BTComplement>},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_8, PF_66, 7), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::BTOp, 1, BTAction::BTComplement>},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_8, PF_F2, 7), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::BTOp, 1, BTAction::BTComplement>},\n\n // GROUP 9\n {OPD(FEXCore::X86Tables::TYPE_GROUP_9, PF_NONE, 1), 1, &OpDispatchBuilder::CMPXCHGPairOp},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_9, PF_NONE, 6), 1, &OpDispatchBuilder::RDRANDOp},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_9, PF_NONE, 7), 1, &OpDispatchBuilder::RDRANDOp},\n\n {OPD(FEXCore::X86Tables::TYPE_GROUP_9, PF_66, 1), 1, &OpDispatchBuilder::CMPXCHGPairOp},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_9, PF_66, 6), 1, &OpDispatchBuilder::RDRANDOp},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_9, PF_66, 7), 1, &OpDispatchBuilder::RDRANDOp},\n\n {OPD(FEXCore::X86Tables::TYPE_GROUP_9, PF_F2, 1), 1, &OpDispatchBuilder::CMPXCHGPairOp},\n\n {OPD(FEXCore::X86Tables::TYPE_GROUP_9, PF_F3, 1), 1, &OpDispatchBuilder::CMPXCHGPairOp},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_9, PF_F3, 7), 1, &OpDispatchBuilder::RDPIDOp},\n\n // GROUP 12\n {OPD(FEXCore::X86Tables::TYPE_GROUP_12, PF_NONE, 2), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PSRLI, OpSize::i16Bit>},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_12, PF_NONE, 4), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PSRAIOp, OpSize::i16Bit>},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_12, PF_NONE, 6), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PSLLI, OpSize::i16Bit>},\n\n {OPD(FEXCore::X86Tables::TYPE_GROUP_12, PF_66, 2), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PSRLI, OpSize::i16Bit>},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_12, PF_66, 4), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PSRAIOp, OpSize::i16Bit>},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_12, PF_66, 6), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PSLLI, OpSize::i16Bit>},\n\n // GROUP 13\n {OPD(FEXCore::X86Tables::TYPE_GROUP_13, PF_NONE, 2), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PSRLI, OpSize::i32Bit>},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_13, PF_NONE, 4), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PSRAIOp, OpSize::i32Bit>},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_13, PF_NONE, 6), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PSLLI, OpSize::i32Bit>},\n\n {OPD(FEXCore::X86Tables::TYPE_GROUP_13, PF_66, 2), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PSRLI, OpSize::i32Bit>},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_13, PF_66, 4), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PSRAIOp, OpSize::i32Bit>},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_13, PF_66, 6), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PSLLI, OpSize::i32Bit>},\n\n // GROUP 14\n {OPD(FEXCore::X86Tables::TYPE_GROUP_14, PF_NONE, 2), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PSRLI, OpSize::i64Bit>},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_14, PF_NONE, 6), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PSLLI, OpSize::i64Bit>},\n\n {OPD(FEXCore::X86Tables::TYPE_GROUP_14, PF_66, 2), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PSRLI, OpSize::i64Bit>},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_14, PF_66, 3), 1, &OpDispatchBuilder::PSRLDQ},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_14, PF_66, 6), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PSLLI, OpSize::i64Bit>},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_14, PF_66, 7), 1, &OpDispatchBuilder::PSLLDQ},\n\n // GROUP 15\n {OPD(FEXCore::X86Tables::TYPE_GROUP_15, PF_NONE, 0), 1, &OpDispatchBuilder::FXSaveOp},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_15, PF_NONE, 1), 1, &OpDispatchBuilder::FXRStoreOp},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_15, PF_NONE, 2), 1, &OpDispatchBuilder::LDMXCSR},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_15, PF_NONE, 3), 1, &OpDispatchBuilder::STMXCSR},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_15, PF_NONE, 4), 1, &OpDispatchBuilder::XSaveOp},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_15, PF_NONE, 5), 1, &OpDispatchBuilder::LoadFenceOrXRSTOR}, // LFENCE (or XRSTOR)\n {OPD(FEXCore::X86Tables::TYPE_GROUP_15, PF_NONE, 6), 1, &OpDispatchBuilder::MemFenceOrXSAVEOPT}, // MFENCE (or XSAVEOPT)\n {OPD(FEXCore::X86Tables::TYPE_GROUP_15, PF_NONE, 7), 1, &OpDispatchBuilder::StoreFenceOrCLFlush}, // SFENCE (or CLFLUSH)\n\n {OPD(FEXCore::X86Tables::TYPE_GROUP_15, PF_F3, 5), 1, &OpDispatchBuilder::UnimplementedOp},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_15, PF_F3, 6), 1, &OpDispatchBuilder::UMonitorOrCLRSSBSY},\n\n {OPD(FEXCore::X86Tables::TYPE_GROUP_15, PF_66, 6), 1, &OpDispatchBuilder::CLWBOrTPause},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_15, PF_66, 7), 1, &OpDispatchBuilder::CLFLUSHOPT},\n\n {OPD(FEXCore::X86Tables::TYPE_GROUP_15, PF_F2, 6), 1, &OpDispatchBuilder::UMWaitOp},\n\n // GROUP 16\n {OPD(FEXCore::X86Tables::TYPE_GROUP_16, PF_NONE, 0), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::Prefetch, false, true, 1>},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_16, PF_NONE, 1), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::Prefetch, false, false, 1>},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_16, PF_NONE, 2), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::Prefetch, false, false, 2>},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_16, PF_NONE, 3), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::Prefetch, false, false, 3>},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_16, PF_NONE, 4), 4, &OpDispatchBuilder::NOPOp},\n\n {OPD(FEXCore::X86Tables::TYPE_GROUP_16, PF_F3, 0), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::Prefetch, false, true, 1>},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_16, PF_F3, 1), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::Prefetch, false, false, 1>},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_16, PF_F3, 2), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::Prefetch, false, false, 2>},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_16, PF_F3, 3), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::Prefetch, false, false, 3>},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_16, PF_F3, 4), 4, &OpDispatchBuilder::NOPOp},\n\n {OPD(FEXCore::X86Tables::TYPE_GROUP_16, PF_66, 0), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::Prefetch, false, true, 1>},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_16, PF_66, 1), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::Prefetch, false, false, 1>},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_16, PF_66, 2), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::Prefetch, false, false, 2>},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_16, PF_66, 3), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::Prefetch, false, false, 3>},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_16, PF_66, 4), 4, &OpDispatchBuilder::NOPOp},\n\n {OPD(FEXCore::X86Tables::TYPE_GROUP_16, PF_F2, 0), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::Prefetch, false, true, 1>},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_16, PF_F2, 1), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::Prefetch, false, false, 1>},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_16, PF_F2, 2), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::Prefetch, false, false, 2>},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_16, PF_F2, 3), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::Prefetch, false, false, 3>},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_16, PF_F2, 4), 4, &OpDispatchBuilder::NOPOp},\n\n // GROUP 17\n {OPD(FEXCore::X86Tables::TYPE_GROUP_17, PF_66, 0), 1, &OpDispatchBuilder::Extrq_imm},\n\n // GROUP P\n {OPD(FEXCore::X86Tables::TYPE_GROUP_P, PF_NONE, 0), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::Prefetch, false, false, 1>},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_P, PF_NONE, 1), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::Prefetch, true, false, 1>},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_P, PF_NONE, 2), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::Prefetch, true, false, 1>},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_P, PF_NONE, 3), 5, &OpDispatchBuilder::NOPOp},\n\n {OPD(FEXCore::X86Tables::TYPE_GROUP_P, PF_F3, 0), 8, &OpDispatchBuilder::NOPOp},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_P, PF_66, 0), 8, &OpDispatchBuilder::NOPOp},\n {OPD(FEXCore::X86Tables::TYPE_GROUP_P, PF_F2, 0), 8, &OpDispatchBuilder::NOPOp},\n};\n\n#undef OPD\n\n} // namespace FEXCore::IR\n" }, { "path": "FEXCore/Source/Interface/Core/OpcodeDispatcher/SecondaryModRMTables.h", "content": "// SPDX-License-Identifier: MIT\n#pragma once\n#include \"Interface/Core/OpcodeDispatcher.h\"\n\nnamespace FEXCore::IR {\nconstexpr DispatchTableEntry OpDispatch_SecondaryModRMTables[] = {\n // REG /1\n {((0 << 3) | 0), 1, &OpDispatchBuilder::UnimplementedOp},\n {((0 << 3) | 1), 1, &OpDispatchBuilder::UnimplementedOp},\n\n // REG /2\n {((1 << 3) | 0), 1, &OpDispatchBuilder::XGetBVOp},\n\n // REG /3\n {((2 << 3) | 7), 1, &OpDispatchBuilder::PermissionRestrictedOp},\n\n // REG /7\n {((3 << 3) | 0), 1, &OpDispatchBuilder::PermissionRestrictedOp},\n {((3 << 3) | 1), 1, &OpDispatchBuilder::RDTSCPOp},\n {((3 << 3) | 4), 1, &OpDispatchBuilder::CLZeroOp},\n};\n\n} // namespace FEXCore::IR\n" }, { "path": "FEXCore/Source/Interface/Core/OpcodeDispatcher/SecondaryTables.h", "content": "// SPDX-License-Identifier: MIT\n#pragma once\n#include \"Interface/Core/OpcodeDispatcher.h\"\n\nnamespace FEXCore::IR {\nconstexpr DispatchTableEntry OpDispatch_TwoByteOpTable[] = {\n // Instructions\n {0x03, 1, &OpDispatchBuilder::LSLOp},\n {0x06, 1, &OpDispatchBuilder::PermissionRestrictedOp},\n {0x07, 1, &OpDispatchBuilder::PermissionRestrictedOp},\n {0x0B, 1, &OpDispatchBuilder::INTOp},\n {0x0E, 1, &OpDispatchBuilder::X87EMMS},\n\n {0x19, 7, &OpDispatchBuilder::NOPOp}, // NOP with ModRM\n\n {0x20, 4, &OpDispatchBuilder::PermissionRestrictedOp},\n\n {0x30, 1, &OpDispatchBuilder::PermissionRestrictedOp},\n {0x31, 1, &OpDispatchBuilder::RDTSCOp},\n {0x32, 2, &OpDispatchBuilder::PermissionRestrictedOp},\n {0x34, 3, &OpDispatchBuilder::UnimplementedOp},\n\n {0x40, 16, &OpDispatchBuilder::CMOVOp},\n {0x6E, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::MOVBetweenGPR_FPR, OpDispatchBuilder::VectorOpType::MMX>},\n {0x6F, 1, &OpDispatchBuilder::MOVQMMXOp},\n {0x7E, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::MOVBetweenGPR_FPR, OpDispatchBuilder::VectorOpType::MMX>},\n {0x7F, 1, &OpDispatchBuilder::MOVQMMXOp},\n {0x80, 16, &OpDispatchBuilder::CondJUMPOp},\n {0x90, 16, &OpDispatchBuilder::SETccOp},\n {0xA2, 1, &OpDispatchBuilder::CPUIDOp},\n {0xA3, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::BTOp, 0, BTAction::BTNone>}, // BT\n {0xA4, 1, &OpDispatchBuilder::SHLDImmediateOp},\n {0xA5, 1, &OpDispatchBuilder::SHLDOp},\n {0xAB, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::BTOp, 0, BTAction::BTSet>}, // BTS\n {0xAC, 1, &OpDispatchBuilder::SHRDImmediateOp},\n {0xAD, 1, &OpDispatchBuilder::SHRDOp},\n {0xAF, 1, &OpDispatchBuilder::IMUL1SrcOp},\n {0xB0, 2, &OpDispatchBuilder::CMPXCHGOp}, // CMPXCHG\n {0xB3, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::BTOp, 0, BTAction::BTClear>}, // BTR\n {0xB6, 2, &OpDispatchBuilder::MOVZXOp},\n {0xBB, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::BTOp, 0, BTAction::BTComplement>}, // BTC\n {0xBC, 1, &OpDispatchBuilder::BSFOp}, // BSF\n {0xBD, 1, &OpDispatchBuilder::BSROp}, // BSF\n {0xBE, 2, &OpDispatchBuilder::MOVSXOp},\n {0xC0, 2, &OpDispatchBuilder::XADDOp},\n {0xC3, 1, &OpDispatchBuilder::MOVGPRNTOp},\n {0xC4, 1, &OpDispatchBuilder::PINSROp},\n {0xC5, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PExtrOp, OpSize::i16Bit>},\n {0xC8, 8, &OpDispatchBuilder::BSWAPOp},\n\n // SSE\n {0x10, 2, &OpDispatchBuilder::MOVVectorUnalignedOp},\n {0x12, 2, &OpDispatchBuilder::MOVLPOp},\n {0x14, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PUNPCKLOp, OpSize::i32Bit>},\n {0x15, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PUNPCKHOp, OpSize::i32Bit>},\n {0x16, 2, &OpDispatchBuilder::MOVHPDOp},\n {0x28, 2, &OpDispatchBuilder::MOVVectorAlignedOp},\n {0x2A, 1, &OpDispatchBuilder::InsertMMX_To_XMM_Vector_CVT_Int_To_Float},\n {0x2B, 1, &OpDispatchBuilder::MOVVectorNTOp},\n {0x2C, 1, &OpDispatchBuilder::XMM_To_MMX_Vector_CVT_Float_To_Int},\n {0x2D, 1, &OpDispatchBuilder::XMM_To_MMX_Vector_CVT_Float_To_Int},\n {0x2E, 2, &OpDispatchBuilder::UCOMISxOp},\n {0x50, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::MOVMSKOp, OpSize::i32Bit>},\n {0x51, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorUnaryOp, IR::OP_VFSQRT, OpSize::i32Bit>},\n {0x52, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorUnaryOp, IR::OP_VFRSQRT, OpSize::i32Bit>},\n {0x53, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorUnaryOp, IR::OP_VFRECP, OpSize::i32Bit>},\n {0x54, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VAND, OpSize::i128Bit>},\n {0x55, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUROp, IR::OP_VANDN, OpSize::i64Bit>},\n {0x56, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VOR, OpSize::i128Bit>},\n {0x57, 1, &OpDispatchBuilder::VectorXOROp},\n {0x58, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VFADD, OpSize::i32Bit>},\n {0x59, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VFMUL, OpSize::i32Bit>},\n {0x5A, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::Vector_CVT_Float_To_Float, OpSize::i64Bit, OpSize::i32Bit, false>},\n {0x5B, 1, &OpDispatchBuilder::Vector_CVT_Int_To_Float},\n {0x5C, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VFSUB, OpSize::i32Bit>},\n {0x5D, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VFMIN, OpSize::i32Bit>},\n {0x5E, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VFDIV, OpSize::i32Bit>},\n {0x5F, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VFMAX, OpSize::i32Bit>},\n {0x60, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PUNPCKLOp, OpSize::i8Bit>},\n {0x61, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PUNPCKLOp, OpSize::i16Bit>},\n {0x62, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PUNPCKLOp, OpSize::i32Bit>},\n {0x63, 1, &OpDispatchBuilder::PACKSSOp},\n {0x64, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VCMPGT, OpSize::i8Bit>},\n {0x65, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VCMPGT, OpSize::i16Bit>},\n {0x66, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VCMPGT, OpSize::i32Bit>},\n {0x67, 1, &OpDispatchBuilder::PACKUSOp},\n {0x68, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PUNPCKHOp, OpSize::i8Bit>},\n {0x69, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PUNPCKHOp, OpSize::i16Bit>},\n {0x6A, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PUNPCKHOp, OpSize::i32Bit>},\n {0x6B, 1, &OpDispatchBuilder::PACKSSOp},\n {0x70, 1, &OpDispatchBuilder::PSHUFW8ByteOp},\n\n {0x74, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VCMPEQ, OpSize::i8Bit>},\n {0x75, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VCMPEQ, OpSize::i16Bit>},\n {0x76, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VCMPEQ, OpSize::i32Bit>},\n {0x77, 1, &OpDispatchBuilder::X87EMMS},\n\n {0xC2, 1, &OpDispatchBuilder::VFCMPOp},\n {0xC6, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::SHUFOp, OpSize::i32Bit>},\n\n {0xD1, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PSRLDOp, OpSize::i16Bit>},\n {0xD2, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PSRLDOp, OpSize::i32Bit>},\n {0xD3, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PSRLDOp, OpSize::i64Bit>},\n {0xD4, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VADD, OpSize::i64Bit>},\n {0xD5, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VMUL, OpSize::i16Bit>},\n {0xD7, 1, &OpDispatchBuilder::MOVMSKOpOne}, // PMOVMSKB\n {0xD8, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VUQSUB, OpSize::i8Bit>},\n {0xD9, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VUQSUB, OpSize::i16Bit>},\n {0xDA, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VUMIN, OpSize::i8Bit>},\n {0xDB, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VAND, OpSize::i64Bit>},\n {0xDC, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VUQADD, OpSize::i8Bit>},\n {0xDD, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VUQADD, OpSize::i16Bit>},\n {0xDE, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VUMAX, OpSize::i8Bit>},\n {0xDF, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUROp, IR::OP_VANDN, OpSize::i64Bit>},\n {0xE0, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VURAVG, OpSize::i8Bit>},\n {0xE1, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PSRAOp, OpSize::i16Bit>},\n {0xE2, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PSRAOp, OpSize::i32Bit>},\n {0xE3, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VURAVG, OpSize::i16Bit>},\n {0xE4, 1, &OpDispatchBuilder::PMULHW},\n {0xE5, 1, &OpDispatchBuilder::PMULHW},\n {0xE7, 1, &OpDispatchBuilder::MOVVectorNTOp},\n {0xE8, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VSQSUB, OpSize::i8Bit>},\n {0xE9, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VSQSUB, OpSize::i16Bit>},\n {0xEA, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VSMIN, OpSize::i16Bit>},\n {0xEB, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VOR, OpSize::i64Bit>},\n {0xEC, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VSQADD, OpSize::i8Bit>},\n {0xED, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VSQADD, OpSize::i16Bit>},\n {0xEE, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VSMAX, OpSize::i16Bit>},\n {0xEF, 1, &OpDispatchBuilder::VectorXOROp},\n\n {0xF1, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PSLL, OpSize::i16Bit>},\n {0xF2, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PSLL, OpSize::i32Bit>},\n {0xF3, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PSLL, OpSize::i64Bit>},\n {0xF4, 1, &OpDispatchBuilder::PMULLOp},\n {0xF5, 1, &OpDispatchBuilder::PMADDWD},\n {0xF6, 1, &OpDispatchBuilder::PSADBW},\n {0xF7, 1, &OpDispatchBuilder::MASKMOVOp},\n {0xF8, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VSUB, OpSize::i8Bit>},\n {0xF9, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VSUB, OpSize::i16Bit>},\n {0xFA, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VSUB, OpSize::i32Bit>},\n {0xFB, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VSUB, OpSize::i64Bit>},\n {0xFC, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VADD, OpSize::i8Bit>},\n {0xFD, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VADD, OpSize::i16Bit>},\n {0xFE, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VADD, OpSize::i32Bit>},\n\n#ifndef _WIN32\n // FEX reserved instructions\n {0x3E, 1, &OpDispatchBuilder::CallbackReturnOp},\n {0x3F, 1, &OpDispatchBuilder::ThunkOp},\n#endif\n};\n\nconstexpr DispatchTableEntry OpDispatch_SecondaryRepModTables[] = {\n {0x10, 2, &OpDispatchBuilder::MOVSSOp},\n {0x12, 1, &OpDispatchBuilder::VMOVSLDUPOp},\n {0x16, 1, &OpDispatchBuilder::VMOVSHDUPOp},\n {0x2A, 1, &OpDispatchBuilder::InsertCVTGPR_To_FPR},\n {0x2B, 1, &OpDispatchBuilder::MOVVectorNTOp},\n {0x2C, 1, &OpDispatchBuilder::CVTFPR_To_GPR},\n {0x2D, 1, &OpDispatchBuilder::CVTFPR_To_GPR},\n {0x51, 1, &OpDispatchBuilder::VectorScalarUnaryInsertALUOp},\n {0x52, 1, &OpDispatchBuilder::VectorScalarUnaryInsertALUOp},\n {0x53, 1, &OpDispatchBuilder::VectorScalarUnaryInsertALUOp},\n {0x58, 1, &OpDispatchBuilder::VectorScalarInsertALUOp},\n {0x59, 1, &OpDispatchBuilder::VectorScalarInsertALUOp},\n {0x5A, 1, &OpDispatchBuilder::InsertScalar_CVT_Float_To_Float},\n {0x5B, 1, &OpDispatchBuilder::Vector_CVT_Float_To_Int},\n {0x5C, 1, &OpDispatchBuilder::VectorScalarInsertALUOp},\n {0x5D, 1, &OpDispatchBuilder::VectorScalarInsertALUOp},\n {0x5E, 1, &OpDispatchBuilder::VectorScalarInsertALUOp},\n {0x5F, 1, &OpDispatchBuilder::VectorScalarInsertALUOp},\n {0x6F, 1, &OpDispatchBuilder::MOVVectorUnalignedOp},\n {0x70, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PSHUFWOp, false>},\n {0x7E, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::MOVQOp, OpDispatchBuilder::VectorOpType::SSE>},\n {0x7F, 1, &OpDispatchBuilder::MOVVectorUnalignedOp},\n {0xB8, 1, &OpDispatchBuilder::PopcountOp},\n {0xBC, 1, &OpDispatchBuilder::TZCNT},\n {0xBD, 1, &OpDispatchBuilder::LZCNT},\n {0xC2, 1, &OpDispatchBuilder::InsertScalarFCMPOp},\n {0xD6, 1, &OpDispatchBuilder::MOVQ2DQ},\n {0xE6, 1, &OpDispatchBuilder::Vector_CVT_Int_To_Float},\n};\n\nconstexpr DispatchTableEntry OpDispatch_SecondaryRepNEModTables[] = {\n {0x10, 2, &OpDispatchBuilder::MOVSDOp},\n {0x12, 1, &OpDispatchBuilder::MOVDDUPOp},\n {0x2A, 1, &OpDispatchBuilder::InsertCVTGPR_To_FPR},\n {0x2B, 1, &OpDispatchBuilder::MOVVectorNTOp},\n {0x2C, 1, &OpDispatchBuilder::CVTFPR_To_GPR},\n {0x2D, 1, &OpDispatchBuilder::CVTFPR_To_GPR},\n {0x51, 1, &OpDispatchBuilder::VectorScalarUnaryInsertALUOp},\n // x52 = Invalid\n {0x58, 1, &OpDispatchBuilder::VectorScalarInsertALUOp},\n {0x59, 1, &OpDispatchBuilder::VectorScalarInsertALUOp},\n {0x5A, 1, &OpDispatchBuilder::InsertScalar_CVT_Float_To_Float},\n {0x5C, 1, &OpDispatchBuilder::VectorScalarInsertALUOp},\n {0x5D, 1, &OpDispatchBuilder::VectorScalarInsertALUOp},\n {0x5E, 1, &OpDispatchBuilder::VectorScalarInsertALUOp},\n {0x5F, 1, &OpDispatchBuilder::VectorScalarInsertALUOp},\n {0x70, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PSHUFWOp, true>},\n {0x78, 1, &OpDispatchBuilder::Insertq_imm},\n {0x79, 1, &OpDispatchBuilder::Insertq},\n {0x7C, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VFADDP, OpSize::i32Bit>},\n {0x7D, 1, &OpDispatchBuilder::HSUBP},\n {0xD0, 1, &OpDispatchBuilder::ADDSUBPOp},\n {0xD6, 1, &OpDispatchBuilder::MOVQ2DQ},\n {0xC2, 1, &OpDispatchBuilder::InsertScalarFCMPOp},\n {0xE6, 1, &OpDispatchBuilder::Vector_CVT_Float_To_Int},\n {0xF0, 1, &OpDispatchBuilder::MOVVectorUnalignedOp},\n};\n\nconstexpr DispatchTableEntry OpDispatch_SecondaryOpSizeModTables[] = {\n {0x10, 2, &OpDispatchBuilder::MOVVectorUnalignedOp},\n {0x12, 2, &OpDispatchBuilder::MOVLPOp},\n {0x14, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PUNPCKLOp, OpSize::i64Bit>},\n {0x15, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PUNPCKHOp, OpSize::i64Bit>},\n {0x16, 2, &OpDispatchBuilder::MOVHPDOp},\n {0x28, 2, &OpDispatchBuilder::MOVVectorAlignedOp},\n {0x2A, 1, &OpDispatchBuilder::MMX_To_XMM_Vector_CVT_Int_To_Float},\n {0x2B, 1, &OpDispatchBuilder::MOVVectorNTOp},\n {0x2C, 1, &OpDispatchBuilder::XMM_To_MMX_Vector_CVT_Float_To_Int},\n {0x2D, 1, &OpDispatchBuilder::XMM_To_MMX_Vector_CVT_Float_To_Int},\n {0x2E, 2, &OpDispatchBuilder::UCOMISxOp},\n\n {0x50, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::MOVMSKOp, OpSize::i64Bit>},\n {0x51, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorUnaryOp, IR::OP_VFSQRT, OpSize::i64Bit>},\n {0x54, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VAND, OpSize::i128Bit>},\n {0x55, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUROp, IR::OP_VANDN, OpSize::i64Bit>},\n {0x56, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VOR, OpSize::i128Bit>},\n {0x57, 1, &OpDispatchBuilder::VectorXOROp},\n {0x58, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VFADD, OpSize::i64Bit>},\n {0x59, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VFMUL, OpSize::i64Bit>},\n {0x5A, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::Vector_CVT_Float_To_Float, OpSize::i32Bit, OpSize::i64Bit, false>},\n {0x5B, 1, &OpDispatchBuilder::Vector_CVT_Float_To_Int},\n {0x5C, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VFSUB, OpSize::i64Bit>},\n {0x5D, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VFMIN, OpSize::i64Bit>},\n {0x5E, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VFDIV, OpSize::i64Bit>},\n {0x5F, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VFMAX, OpSize::i64Bit>},\n {0x60, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PUNPCKLOp, OpSize::i8Bit>},\n {0x61, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PUNPCKLOp, OpSize::i16Bit>},\n {0x62, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PUNPCKLOp, OpSize::i32Bit>},\n {0x63, 1, &OpDispatchBuilder::PACKSSOp},\n {0x64, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VCMPGT, OpSize::i8Bit>},\n {0x65, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VCMPGT, OpSize::i16Bit>},\n {0x66, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VCMPGT, OpSize::i32Bit>},\n {0x67, 1, &OpDispatchBuilder::PACKUSOp},\n {0x68, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PUNPCKHOp, OpSize::i8Bit>},\n {0x69, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PUNPCKHOp, OpSize::i16Bit>},\n {0x6A, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PUNPCKHOp, OpSize::i32Bit>},\n {0x6B, 1, &OpDispatchBuilder::PACKSSOp},\n {0x6C, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PUNPCKLOp, OpSize::i64Bit>},\n {0x6D, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PUNPCKHOp, OpSize::i64Bit>},\n {0x6E, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::MOVBetweenGPR_FPR, OpDispatchBuilder::VectorOpType::SSE>},\n {0x6F, 1, &OpDispatchBuilder::MOVVectorAlignedOp},\n {0x70, 1, &OpDispatchBuilder::PSHUFDOp},\n\n {0x74, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VCMPEQ, OpSize::i8Bit>},\n {0x75, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VCMPEQ, OpSize::i16Bit>},\n {0x76, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VCMPEQ, OpSize::i32Bit>},\n {0x78, 1, nullptr}, // GROUP 17\n {0x79, 1, &OpDispatchBuilder::Extrq},\n {0x7C, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VFADDP, OpSize::i64Bit>},\n {0x7D, 1, &OpDispatchBuilder::HSUBP},\n {0x7E, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::MOVBetweenGPR_FPR, OpDispatchBuilder::VectorOpType::SSE>},\n {0x7F, 1, &OpDispatchBuilder::MOVVectorAlignedOp},\n {0xC2, 1, &OpDispatchBuilder::VFCMPOp},\n {0xC4, 1, &OpDispatchBuilder::PINSROp},\n {0xC5, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PExtrOp, OpSize::i16Bit>},\n {0xC6, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::SHUFOp, OpSize::i64Bit>},\n\n {0xD0, 1, &OpDispatchBuilder::ADDSUBPOp},\n {0xD1, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PSRLDOp, OpSize::i16Bit>},\n {0xD2, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PSRLDOp, OpSize::i32Bit>},\n {0xD3, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PSRLDOp, OpSize::i64Bit>},\n {0xD4, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VADD, OpSize::i64Bit>},\n {0xD5, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VMUL, OpSize::i16Bit>},\n {0xD6, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::MOVQOp, OpDispatchBuilder::VectorOpType::SSE>},\n {0xD7, 1, &OpDispatchBuilder::MOVMSKOpOne}, // PMOVMSKB\n {0xD8, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VUQSUB, OpSize::i8Bit>},\n {0xD9, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VUQSUB, OpSize::i16Bit>},\n {0xDA, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VUMIN, OpSize::i8Bit>},\n {0xDB, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VAND, OpSize::i128Bit>},\n {0xDC, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VUQADD, OpSize::i8Bit>},\n {0xDD, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VUQADD, OpSize::i16Bit>},\n {0xDE, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VUMAX, OpSize::i8Bit>},\n {0xDF, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUROp, IR::OP_VANDN, OpSize::i64Bit>},\n {0xE0, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VURAVG, OpSize::i8Bit>},\n {0xE1, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PSRAOp, OpSize::i16Bit>},\n {0xE2, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PSRAOp, OpSize::i32Bit>},\n {0xE3, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VURAVG, OpSize::i16Bit>},\n {0xE4, 1, &OpDispatchBuilder::PMULHW},\n {0xE5, 1, &OpDispatchBuilder::PMULHW},\n {0xE6, 1, &OpDispatchBuilder::Vector_CVT_Float_To_Int},\n {0xE7, 1, &OpDispatchBuilder::MOVVectorNTOp},\n {0xE8, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VSQSUB, OpSize::i8Bit>},\n {0xE9, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VSQSUB, OpSize::i16Bit>},\n {0xEA, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VSMIN, OpSize::i16Bit>},\n {0xEB, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VOR, OpSize::i128Bit>},\n {0xEC, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VSQADD, OpSize::i8Bit>},\n {0xED, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VSQADD, OpSize::i16Bit>},\n {0xEE, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VSMAX, OpSize::i16Bit>},\n {0xEF, 1, &OpDispatchBuilder::VectorXOROp},\n\n {0xF1, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PSLL, OpSize::i16Bit>},\n {0xF2, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PSLL, OpSize::i32Bit>},\n {0xF3, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PSLL, OpSize::i64Bit>},\n {0xF4, 1, &OpDispatchBuilder::PMULLOp},\n {0xF5, 1, &OpDispatchBuilder::PMADDWD},\n {0xF6, 1, &OpDispatchBuilder::PSADBW},\n {0xF7, 1, &OpDispatchBuilder::MASKMOVOp},\n {0xF8, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VSUB, OpSize::i8Bit>},\n {0xF9, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VSUB, OpSize::i16Bit>},\n {0xFA, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VSUB, OpSize::i32Bit>},\n {0xFB, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VSUB, OpSize::i64Bit>},\n {0xFC, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VADD, OpSize::i8Bit>},\n {0xFD, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VADD, OpSize::i16Bit>},\n {0xFE, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VADD, OpSize::i32Bit>},\n};\n} // namespace FEXCore::IR\n" }, { "path": "FEXCore/Source/Interface/Core/OpcodeDispatcher/VEXTables.h", "content": "// SPDX-License-Identifier: MIT\n#pragma once\n#include \"Interface/Core/OpcodeDispatcher.h\"\n\nnamespace FEXCore::IR {\n#define OPD(map_select, pp, opcode) (((map_select - 1) << 10) | (pp << 8) | (opcode))\nconstexpr DispatchTableEntry OpDispatch_VEXTable[] = {\n {OPD(2, 0b00, 0xF2), 1, &OpDispatchBuilder::ANDNBMIOp}, {OPD(2, 0b00, 0xF5), 1, &OpDispatchBuilder::BZHI},\n {OPD(2, 0b10, 0xF5), 1, &OpDispatchBuilder::PEXT}, {OPD(2, 0b11, 0xF5), 1, &OpDispatchBuilder::PDEP},\n {OPD(2, 0b11, 0xF6), 1, &OpDispatchBuilder::MULX}, {OPD(2, 0b00, 0xF7), 1, &OpDispatchBuilder::BEXTRBMIOp},\n {OPD(2, 0b01, 0xF7), 1, &OpDispatchBuilder::BMI2Shift}, {OPD(2, 0b10, 0xF7), 1, &OpDispatchBuilder::BMI2Shift},\n {OPD(2, 0b11, 0xF7), 1, &OpDispatchBuilder::BMI2Shift},\n\n {OPD(3, 0b11, 0xF0), 1, &OpDispatchBuilder::RORX},\n};\n#undef OPD\n\n#define OPD(group, pp, opcode) (((group - X86Tables::InstType::TYPE_VEX_GROUP_12) << 4) | (pp << 3) | (opcode))\nconstexpr DispatchTableEntry OpDispatch_VEXGroupTable[] = {\n {OPD(X86Tables::InstType::TYPE_VEX_GROUP_17, 0, 0b001), 1, &OpDispatchBuilder::BLSRBMIOp},\n {OPD(X86Tables::InstType::TYPE_VEX_GROUP_17, 0, 0b010), 1, &OpDispatchBuilder::BLSMSKBMIOp},\n {OPD(X86Tables::InstType::TYPE_VEX_GROUP_17, 0, 0b011), 1, &OpDispatchBuilder::BLSIBMIOp},\n};\n#undef OPD\n\n} // namespace FEXCore::IR\n" }, { "path": "FEXCore/Source/Interface/Core/OpcodeDispatcher/Vector.cpp", "content": "// SPDX-License-Identifier: MIT\n/*\n$info$\ntags: frontend|x86-to-ir, opcodes|dispatcher-implementations\ndesc: Handles x86/64 Vector instructions to IR\n$end_info$\n*/\n\n#include \"Interface/Context/Context.h\"\n#include \"Interface/Core/OpcodeDispatcher.h\"\n#include \"Interface/Core/X86Tables/X86Tables.h\"\n#include \"Interface/IR/IR.h\"\n\n#include \n#include \n#include \n#include \n\n#include \n#include \n#include \n#include \n\nnamespace FEXCore::IR {\n#define OpcodeArgs [[maybe_unused]] FEXCore::X86Tables::DecodedOp Op\n\nvoid OpDispatchBuilder::MOVVectorAlignedOp(OpcodeArgs) {\n if (Op->Dest.IsGPR() && Op->Src[0].IsGPR() && Op->Dest.Data.GPR.GPR == Op->Src[0].Data.GPR.GPR) {\n // Nop\n return;\n }\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n StoreResultFPR(Op, Src);\n}\n\nvoid OpDispatchBuilder::MOVVectorUnalignedOp(OpcodeArgs) {\n if (Op->Dest.IsGPR() && Op->Src[0].IsGPR() && Op->Dest.Data.GPR.GPR == Op->Src[0].Data.GPR.GPR) {\n // Nop\n return;\n }\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags, {.Align = OpSize::i8Bit});\n StoreResultFPR(Op, Src, OpSize::i8Bit);\n}\n\nvoid OpDispatchBuilder::MOVVectorNTOp(OpcodeArgs) {\n const auto Size = OpSizeFromDst(Op);\n\n if (Op->Dest.IsGPR() && Size >= OpSize::i128Bit) {\n ///< MOVNTDQA load non-temporal comes from SSE4.1 and is extended by AVX/AVX2.\n Ref SrcAddr = LoadSourceGPR(Op, Op->Src[0], Op->Flags, {.LoadData = false});\n auto Src = _VLoadNonTemporal(Size, SrcAddr, 0);\n\n StoreResultFPR(Op, Src, OpSize::i8Bit, MemoryAccessType::STREAM);\n } else if (Op->Dest.IsGPR()) {\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags, {.Align = OpSize::i8Bit, .AccessType = MemoryAccessType::STREAM});\n StoreResultFPR(Op, Src, OpSize::i8Bit, MemoryAccessType::STREAM);\n } else {\n LOGMAN_THROW_A_FMT(!Op->Dest.IsGPR(), \"Destination can't be GPR for non-temporal stores\");\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags, {.Align = OpSize::i8Bit, .AccessType = MemoryAccessType::STREAM});\n if (Size < OpSize::i128Bit) {\n // Normal streaming store if less than 128-bit\n // XMM Scalar 32-bit and 64-bit comes from SSE4a MOVNTSS, MOVNTSD\n // MMX 64-bit comes from MOVNTQ\n StoreResultFPR(Op, Src, OpSize::i8Bit, MemoryAccessType::STREAM);\n } else {\n Ref Dest = LoadSourceGPR(Op, Op->Dest, Op->Flags, {.LoadData = false});\n\n // Single store non-temporal for larger operations.\n _VStoreNonTemporal(Size, Src, Dest, 0);\n }\n }\n}\n\nvoid OpDispatchBuilder::VMOVAPS_VMOVAPDOp(OpcodeArgs) {\n const auto SrcSize = GetSrcSize(Op);\n const auto Is128Bit = SrcSize == Core::CPUState::XMM_SSE_REG_SIZE;\n\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n\n if (Is128Bit && Op->Dest.IsGPR()) {\n Src = VZeroExtendOperand(OpSize::i128Bit, Op->Src[0], Src);\n }\n StoreResultFPR(Op, Src);\n}\n\nvoid OpDispatchBuilder::VMOVUPS_VMOVUPDOp(OpcodeArgs) {\n const auto SrcSize = GetSrcSize(Op);\n const auto Is128Bit = SrcSize == Core::CPUState::XMM_SSE_REG_SIZE;\n\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags, {.Align = OpSize::i8Bit});\n\n if (Is128Bit && Op->Dest.IsGPR()) {\n Src = VZeroExtendOperand(OpSize::i128Bit, Op->Src[0], Src);\n }\n StoreResultFPR(Op, Src, OpSize::i8Bit);\n}\n\nvoid OpDispatchBuilder::MOVHPDOp(OpcodeArgs) {\n if (Op->Dest.IsGPR()) {\n if (Op->Src[0].IsGPR()) {\n // MOVLHPS between two vector registers.\n Ref Src = LoadSourceGPR(Op, Op->Src[0], Op->Flags);\n Ref Dest = LoadSourceFPR_WithOpSize(Op, Op->Dest, OpSize::i128Bit, Op->Flags);\n auto Result = _VInsElement(OpSize::i128Bit, OpSize::i64Bit, 1, 0, Dest, Src);\n StoreResultFPR(Op, Result);\n } else {\n // If the destination is a GPR then the source is memory\n // xmm1[127:64] = src\n Ref Src = MakeSegmentAddress(Op, Op->Src[0]);\n Ref Dest = LoadSourceFPR_WithOpSize(Op, Op->Dest, OpSize::i128Bit, Op->Flags);\n auto Result = _VLoadVectorElement(OpSize::i128Bit, OpSize::i64Bit, Dest, 1, Src);\n StoreResultFPR(Op, Result);\n }\n } else {\n // In this case memory is the destination and the high bits of the XMM are source\n // Mem64 = xmm1[127:64]\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n Ref Dest = MakeSegmentAddress(Op, Op->Dest);\n _VStoreVectorElement(OpSize::i128Bit, OpSize::i64Bit, Src, 1, Dest);\n }\n}\n\nvoid OpDispatchBuilder::VMOVHPOp(OpcodeArgs) {\n if (Op->Dest.IsGPR()) {\n Ref Src1 = LoadSourceFPR(Op, Op->Src[0], Op->Flags, {.Align = OpSize::i128Bit});\n Ref Src2 = LoadSourceFPR(Op, Op->Src[1], Op->Flags, {.Align = OpSize::i64Bit});\n Ref Result = _VInsElement(OpSize::i128Bit, OpSize::i64Bit, 1, 0, Src1, Src2);\n\n StoreResultFPR(Op, Result);\n } else {\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags, {.Align = OpSize::i128Bit});\n Ref Result = _VInsElement(OpSize::i128Bit, OpSize::i64Bit, 0, 1, Src, Src);\n StoreResultFPR_WithOpSize(Op, Op->Dest, Result, OpSize::i64Bit, OpSize::i64Bit);\n }\n}\n\nvoid OpDispatchBuilder::MOVLPOp(OpcodeArgs) {\n if (Op->Dest.IsGPR()) {\n // xmm, xmm is movhlps special case\n if (Op->Src[0].IsGPR()) {\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags, {.Align = OpSize::i128Bit});\n Ref Dest = LoadSourceFPR(Op, Op->Dest, Op->Flags, {.Align = OpSize::i128Bit});\n auto Result = _VInsElement(OpSize::i128Bit, OpSize::i64Bit, 0, 1, Dest, Src);\n StoreResultFPR_WithOpSize(Op, Op->Dest, Result, OpSize::i128Bit, OpSize::i128Bit);\n } else {\n const auto DstSize = OpSizeFromDst(Op);\n Ref Src = MakeSegmentAddress(Op, Op->Src[0]);\n Ref Dest = LoadSourceFPR_WithOpSize(Op, Op->Dest, DstSize, Op->Flags);\n auto Result = _VLoadVectorElement(OpSize::i128Bit, OpSize::i64Bit, Dest, 0, Src);\n StoreResultFPR(Op, Result);\n }\n } else {\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags, {.Align = OpSize::i64Bit});\n StoreResultFPR_WithOpSize(Op, Op->Dest, Src, OpSize::i64Bit, OpSize::i64Bit);\n }\n}\n\nvoid OpDispatchBuilder::VMOVLPOp(OpcodeArgs) {\n Ref Src1 = LoadSourceFPR(Op, Op->Src[0], Op->Flags, {.Align = OpSize::i128Bit});\n\n if (!Op->Dest.IsGPR()) {\n ///< VMOVLPS/PD mem64, xmm1\n StoreResultFPR_WithOpSize(Op, Op->Dest, Src1, OpSize::i64Bit, OpSize::i64Bit);\n } else if (!Op->Src[1].IsGPR()) {\n ///< VMOVLPS/PD xmm1, xmm2, mem64\n // Bits[63:0] come from Src2[63:0]\n // Bits[127:64] come from Src1[127:64]\n Ref Src2 = LoadSourceFPR(Op, Op->Src[1], Op->Flags, {.Align = OpSize::i64Bit});\n Ref Result = _VInsElement(OpSize::i128Bit, OpSize::i64Bit, 1, 1, Src2, Src1);\n StoreResultFPR(Op, Result);\n } else {\n ///< VMOVHLPS/PD xmm1, xmm2, xmm3\n Ref Src2 = LoadSourceFPR(Op, Op->Src[1], Op->Flags, {.Align = OpSize::i128Bit});\n Ref Result = _VInsElement(OpSize::i128Bit, OpSize::i64Bit, 0, 1, Src1, Src2);\n StoreResultFPR(Op, Result);\n }\n}\n\nvoid OpDispatchBuilder::VMOVSHDUPOp(OpcodeArgs) {\n const auto SrcSize = OpSizeFromSrc(Op);\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n Ref Result = _VTrn2(SrcSize, OpSize::i32Bit, Src, Src);\n StoreResultFPR(Op, Result);\n}\n\nvoid OpDispatchBuilder::VMOVSLDUPOp(OpcodeArgs) {\n const auto SrcSize = OpSizeFromSrc(Op);\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n Ref Result = _VTrn(SrcSize, OpSize::i32Bit, Src, Src);\n StoreResultFPR(Op, Result);\n}\n\nvoid OpDispatchBuilder::MOVScalarOpImpl(OpcodeArgs, IR::OpSize ElementSize) {\n if (Op->Dest.IsGPR() && Op->Src[0].IsGPR()) {\n // MOVSS/SD xmm1, xmm2\n Ref Dest = LoadSourceFPR(Op, Op->Dest, Op->Flags);\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n auto Result = _VInsElement(OpSize::i128Bit, ElementSize, 0, 0, Dest, Src);\n StoreResultFPR(Op, Result);\n } else if (Op->Dest.IsGPR()) {\n // MOVSS/SD xmm1, mem32/mem64\n // xmm1[127:0] <- zext(mem32/mem64)\n Ref Src = LoadSourceFPR_WithOpSize(Op, Op->Src[0], ElementSize, Op->Flags);\n StoreResultFPR(Op, Src);\n } else {\n // MOVSS/SD mem32/mem64, xmm1\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n StoreResultFPR_WithOpSize(Op, Op->Dest, Src, ElementSize);\n }\n}\n\nvoid OpDispatchBuilder::MOVSSOp(OpcodeArgs) {\n MOVScalarOpImpl(Op, OpSize::i32Bit);\n}\n\nvoid OpDispatchBuilder::MOVSDOp(OpcodeArgs) {\n MOVScalarOpImpl(Op, OpSize::i64Bit);\n}\n\nvoid OpDispatchBuilder::VMOVScalarOpImpl(OpcodeArgs, IR::OpSize ElementSize) {\n if (Op->Dest.IsGPR() && Op->Src[0].IsGPR() && Op->Src[1].IsGPR()) {\n // VMOVSS/SD xmm1, xmm2, xmm3\n Ref Src1 = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n Ref Src2 = LoadSourceFPR(Op, Op->Src[1], Op->Flags);\n Ref Result = _VInsElement(OpSize::i128Bit, ElementSize, 0, 0, Src1, Src2);\n StoreResultFPR(Op, Result);\n } else if (Op->Dest.IsGPR()) {\n // VMOVSS/SD xmm1, mem32/mem64\n Ref Src = LoadSourceFPR_WithOpSize(Op, Op->Src[1], ElementSize, Op->Flags);\n StoreResultFPR(Op, Src);\n } else {\n // VMOVSS/SD mem32/mem64, xmm1\n Ref Src = LoadSourceFPR(Op, Op->Src[1], Op->Flags);\n StoreResultFPR_WithOpSize(Op, Op->Dest, Src, ElementSize);\n }\n}\n\nvoid OpDispatchBuilder::VMOVSDOp(OpcodeArgs) {\n VMOVScalarOpImpl(Op, OpSize::i64Bit);\n}\n\nvoid OpDispatchBuilder::VMOVSSOp(OpcodeArgs) {\n VMOVScalarOpImpl(Op, OpSize::i32Bit);\n}\n\nvoid OpDispatchBuilder::VectorALUOp(OpcodeArgs, IROps IROp, IR::OpSize ElementSize) {\n const auto Size = OpSizeFromSrc(Op);\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n Ref Dest = LoadSourceFPR(Op, Op->Dest, Op->Flags);\n\n DeriveOp(ALUOp, IROp, _VAdd(Size, ElementSize, Dest, Src));\n\n StoreResultFPR(Op, ALUOp);\n}\n\nvoid OpDispatchBuilder::VectorXOROp(OpcodeArgs) {\n const auto Size = OpSizeFromSrc(Op);\n\n // Special case for vector xor with itself being the optimal way for x86 to zero vector registers.\n if (Op->Dest.IsGPR() && Op->Src[0].IsGPR() && Op->Dest.Data.GPR.GPR == Op->Src[0].Data.GPR.GPR) {\n const auto ZeroRegister = LoadZeroVector(Size);\n StoreResultFPR(Op, ZeroRegister);\n return;\n }\n\n ///< Regular code path\n VectorALUOp(Op, OP_VXOR, Size);\n}\n\nvoid OpDispatchBuilder::AVXVectorALUOp(OpcodeArgs, IROps IROp, IR::OpSize ElementSize) {\n const auto Size = OpSizeFromSrc(Op);\n\n Ref Src1 = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n Ref Src2 = LoadSourceFPR(Op, Op->Src[1], Op->Flags);\n\n DeriveOp(ALUOp, IROp, _VAdd(Size, ElementSize, Src1, Src2));\n\n StoreResultFPR(Op, ALUOp);\n}\n\nvoid OpDispatchBuilder::AVXVectorXOROp(OpcodeArgs) {\n // Special case for vector xor with itself being the optimal way for x86 to zero vector registers.\n if (Op->Src[0].IsGPR() && Op->Src[1].IsGPR() && Op->Src[0].Data.GPR.GPR == Op->Src[1].Data.GPR.GPR) {\n const auto DstSize = OpSizeFromDst(Op);\n const auto ZeroRegister = LoadZeroVector(DstSize);\n StoreResultFPR(Op, ZeroRegister);\n return;\n }\n\n ///< Regular code path\n AVXVectorALUOp(Op, OP_VXOR, OpSize::i128Bit);\n}\n\nvoid OpDispatchBuilder::VectorALUROp(OpcodeArgs, IROps IROp, IR::OpSize ElementSize) {\n const auto Size = OpSizeFromSrc(Op);\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n Ref Dest = LoadSourceFPR(Op, Op->Dest, Op->Flags);\n\n DeriveOp(ALUOp, IROp, _VAdd(Size, ElementSize, Src, Dest));\n\n StoreResultFPR(Op, ALUOp);\n}\n\nRef OpDispatchBuilder::VectorScalarInsertALUOpImpl(OpcodeArgs, IROps IROp, IR::OpSize DstSize, IR::OpSize ElementSize,\n const X86Tables::DecodedOperand& Src1Op, const X86Tables::DecodedOperand& Src2Op,\n bool ZeroUpperBits) {\n // We load the full vector width when dealing with a source vector,\n // so that we don't do any unnecessary zero extension to the scalar\n // element that we're going to operate on.\n const auto SrcSize = OpSizeFromSrc(Op);\n\n Ref Src1 = LoadSourceFPR_WithOpSize(Op, Src1Op, DstSize, Op->Flags);\n Ref Src2 = LoadSourceFPR_WithOpSize(Op, Src2Op, SrcSize, Op->Flags, {.AllowUpperGarbage = true});\n\n // If OpSize == ElementSize then it only does the lower scalar op\n DeriveOp(ALUOp, IROp, _VFAddScalarInsert(DstSize, ElementSize, Src1, Src2, ZeroUpperBits));\n return ALUOp;\n}\n\ntemplate\nvoid OpDispatchBuilder::VectorScalarInsertALUOp(OpcodeArgs) {\n const auto DstSize = GetGuestVectorLength();\n auto Result = VectorScalarInsertALUOpImpl(Op, IROp, DstSize, ElementSize, Op->Dest, Op->Src[0], false);\n StoreResultFPR_WithOpSize(Op, Op->Dest, Result, DstSize);\n}\n\ntemplate void OpDispatchBuilder::VectorScalarInsertALUOp(OpcodeArgs);\ntemplate void OpDispatchBuilder::VectorScalarInsertALUOp(OpcodeArgs);\ntemplate void OpDispatchBuilder::VectorScalarInsertALUOp(OpcodeArgs);\ntemplate void OpDispatchBuilder::VectorScalarInsertALUOp(OpcodeArgs);\ntemplate void OpDispatchBuilder::VectorScalarInsertALUOp(OpcodeArgs);\ntemplate void OpDispatchBuilder::VectorScalarInsertALUOp(OpcodeArgs);\ntemplate void OpDispatchBuilder::VectorScalarInsertALUOp(OpcodeArgs);\ntemplate void OpDispatchBuilder::VectorScalarInsertALUOp(OpcodeArgs);\ntemplate void OpDispatchBuilder::VectorScalarInsertALUOp(OpcodeArgs);\ntemplate void OpDispatchBuilder::VectorScalarInsertALUOp(OpcodeArgs);\ntemplate void OpDispatchBuilder::VectorScalarInsertALUOp(OpcodeArgs);\ntemplate void OpDispatchBuilder::VectorScalarInsertALUOp(OpcodeArgs);\n\ntemplate\nvoid OpDispatchBuilder::AVXVectorScalarInsertALUOp(OpcodeArgs) {\n const auto DstSize = GetGuestVectorLength();\n auto Result = VectorScalarInsertALUOpImpl(Op, IROp, DstSize, ElementSize, Op->Src[0], Op->Src[1], true);\n StoreResultFPR_WithOpSize(Op, Op->Dest, Result, DstSize);\n}\n\ntemplate void OpDispatchBuilder::AVXVectorScalarInsertALUOp(OpcodeArgs);\ntemplate void OpDispatchBuilder::AVXVectorScalarInsertALUOp(OpcodeArgs);\ntemplate void OpDispatchBuilder::AVXVectorScalarInsertALUOp(OpcodeArgs);\ntemplate void OpDispatchBuilder::AVXVectorScalarInsertALUOp(OpcodeArgs);\ntemplate void OpDispatchBuilder::AVXVectorScalarInsertALUOp(OpcodeArgs);\ntemplate void OpDispatchBuilder::AVXVectorScalarInsertALUOp(OpcodeArgs);\ntemplate void OpDispatchBuilder::AVXVectorScalarInsertALUOp(OpcodeArgs);\ntemplate void OpDispatchBuilder::AVXVectorScalarInsertALUOp(OpcodeArgs);\ntemplate void OpDispatchBuilder::AVXVectorScalarInsertALUOp(OpcodeArgs);\ntemplate void OpDispatchBuilder::AVXVectorScalarInsertALUOp(OpcodeArgs);\ntemplate void OpDispatchBuilder::AVXVectorScalarInsertALUOp(OpcodeArgs);\ntemplate void OpDispatchBuilder::AVXVectorScalarInsertALUOp(OpcodeArgs);\n\nRef OpDispatchBuilder::VectorScalarUnaryInsertALUOpImpl(OpcodeArgs, IROps IROp, IR::OpSize DstSize, IR::OpSize ElementSize,\n const X86Tables::DecodedOperand& Src1Op, const X86Tables::DecodedOperand& Src2Op,\n bool ZeroUpperBits) {\n // We load the full vector width when dealing with a source vector,\n // so that we don't do any unnecessary zero extension to the scalar\n // element that we're going to operate on.\n const auto SrcSize = OpSizeFromSrc(Op);\n\n Ref Src1 = LoadSourceFPR_WithOpSize(Op, Src1Op, DstSize, Op->Flags);\n Ref Src2 = LoadSourceFPR_WithOpSize(Op, Src2Op, SrcSize, Op->Flags, {.AllowUpperGarbage = true});\n\n // If OpSize == ElementSize then it only does the lower scalar op\n DeriveOp(ALUOp, IROp, _VFSqrtScalarInsert(DstSize, ElementSize, Src1, Src2, ZeroUpperBits));\n return ALUOp;\n}\n\ntemplate\nvoid OpDispatchBuilder::VectorScalarUnaryInsertALUOp(OpcodeArgs) {\n const auto DstSize = GetGuestVectorLength();\n auto Result = VectorScalarInsertALUOpImpl(Op, IROp, DstSize, ElementSize, Op->Dest, Op->Src[0], false);\n StoreResultFPR_WithOpSize(Op, Op->Dest, Result, DstSize);\n}\n\ntemplate void OpDispatchBuilder::VectorScalarUnaryInsertALUOp(OpcodeArgs);\ntemplate void OpDispatchBuilder::VectorScalarUnaryInsertALUOp(OpcodeArgs);\n\ntemplate void OpDispatchBuilder::VectorScalarUnaryInsertALUOp(OpcodeArgs);\ntemplate void OpDispatchBuilder::VectorScalarUnaryInsertALUOp(OpcodeArgs);\n\ntemplate void OpDispatchBuilder::VectorScalarUnaryInsertALUOp(OpcodeArgs);\ntemplate void OpDispatchBuilder::VectorScalarUnaryInsertALUOp(OpcodeArgs);\n\ntemplate\nvoid OpDispatchBuilder::AVXVectorScalarUnaryInsertALUOp(OpcodeArgs) {\n const auto DstSize = GetGuestVectorLength();\n auto Result = VectorScalarInsertALUOpImpl(Op, IROp, DstSize, ElementSize, Op->Src[0], Op->Src[1], true);\n StoreResultFPR_WithOpSize(Op, Op->Dest, Result, DstSize);\n}\n\ntemplate void OpDispatchBuilder::AVXVectorScalarUnaryInsertALUOp(OpcodeArgs);\ntemplate void OpDispatchBuilder::AVXVectorScalarUnaryInsertALUOp(OpcodeArgs);\n\ntemplate void OpDispatchBuilder::AVXVectorScalarUnaryInsertALUOp(OpcodeArgs);\ntemplate void OpDispatchBuilder::AVXVectorScalarUnaryInsertALUOp(OpcodeArgs);\n\ntemplate void OpDispatchBuilder::AVXVectorScalarUnaryInsertALUOp(OpcodeArgs);\ntemplate void OpDispatchBuilder::AVXVectorScalarUnaryInsertALUOp(OpcodeArgs);\n\nvoid OpDispatchBuilder::InsertMMX_To_XMM_Vector_CVT_Int_To_Float(OpcodeArgs) {\n // We load the full vector width when dealing with a source vector,\n // so that we don't do any unnecessary zero extension to the scalar\n // element that we're going to operate on.\n const auto DstSize = GetGuestVectorLength();\n const auto SrcSize = Op->Src[0].IsGPR() ? OpSize::i64Bit : OpSizeFromSrc(Op);\n\n Ref Dest = LoadSourceFPR_WithOpSize(Op, Op->Dest, DstSize, Op->Flags);\n Ref Src = LoadSourceFPR_WithOpSize(Op, Op->Src[0], SrcSize, Op->Flags);\n\n // Always 32-bit.\n const auto ElementSize = OpSize::i32Bit;\n // Always signed\n Dest = _VSToFVectorInsert(DstSize, ElementSize, ElementSize, Dest, Src, true, false);\n\n StoreResultFPR_WithOpSize(Op, Op->Dest, Dest, DstSize);\n}\n\nRef OpDispatchBuilder::InsertCVTGPR_To_FPRImpl(OpcodeArgs, IR::OpSize DstSize, IR::OpSize DstElementSize, const X86Tables::DecodedOperand& Src1Op,\n const X86Tables::DecodedOperand& Src2Op, bool ZeroUpperBits) {\n // We load the full vector width when dealing with a source vector,\n // so that we don't do any unnecessary zero extension to the scalar\n // element that we're going to operate on.\n const auto SrcSize = OpSizeFromSrc(Op);\n\n Ref Src1 = LoadSourceFPR_WithOpSize(Op, Src1Op, DstSize, Op->Flags);\n\n if (Src2Op.IsGPR()) {\n // If the source is a GPR then convert directly from the GPR.\n auto Src2 = LoadSourceGPR_WithOpSize(Op, Src2Op, GetGPROpSize(), Op->Flags);\n return _VSToFGPRInsert(DstSize, DstElementSize, SrcSize, Src1, Src2, ZeroUpperBits);\n } else if (SrcSize != DstElementSize) {\n // If the source is from memory but the Source size and destination size aren't the same,\n // then it is more optimal to load in to a GPR and convert between GPR->FPR.\n // ARM GPR->FPR conversion supports different size source and destinations while FPR->FPR doesn't.\n auto Src2 = LoadSourceGPR(Op, Src2Op, Op->Flags);\n return _VSToFGPRInsert(DstSize, DstElementSize, SrcSize, Src1, Src2, ZeroUpperBits);\n }\n\n // In the case of cvtsi2s{s,d} where the source and destination are the same size,\n // then it is more optimal to load in to the FPR register directly and convert there.\n auto Src2 = LoadSourceFPR(Op, Src2Op, Op->Flags);\n // Always signed\n return _VSToFVectorInsert(DstSize, DstElementSize, DstElementSize, Src1, Src2, false, ZeroUpperBits);\n}\n\ntemplate\nvoid OpDispatchBuilder::InsertCVTGPR_To_FPR(OpcodeArgs) {\n const auto DstSize = GetGuestVectorLength();\n auto Result = InsertCVTGPR_To_FPRImpl(Op, DstSize, DstElementSize, Op->Dest, Op->Src[0], false);\n StoreResultFPR_WithOpSize(Op, Op->Dest, Result, DstSize);\n}\n\ntemplate void OpDispatchBuilder::InsertCVTGPR_To_FPR(OpcodeArgs);\ntemplate void OpDispatchBuilder::InsertCVTGPR_To_FPR(OpcodeArgs);\n\ntemplate\nvoid OpDispatchBuilder::AVXInsertCVTGPR_To_FPR(OpcodeArgs) {\n const auto DstSize = GetGuestVectorLength();\n Ref Result = InsertCVTGPR_To_FPRImpl(Op, DstSize, DstElementSize, Op->Src[0], Op->Src[1], true);\n StoreResultFPR_WithOpSize(Op, Op->Dest, Result, DstSize);\n}\ntemplate void OpDispatchBuilder::AVXInsertCVTGPR_To_FPR(OpcodeArgs);\ntemplate void OpDispatchBuilder::AVXInsertCVTGPR_To_FPR(OpcodeArgs);\n\nRef OpDispatchBuilder::InsertScalar_CVT_Float_To_FloatImpl(OpcodeArgs, IR::OpSize DstSize, IR::OpSize DstElementSize,\n IR::OpSize SrcElementSize, const X86Tables::DecodedOperand& Src1Op,\n const X86Tables::DecodedOperand& Src2Op, bool ZeroUpperBits) {\n\n // We load the full vector width when dealing with a source vector,\n // so that we don't do any unnecessary zero extension to the scalar\n // element that we're going to operate on.\n const auto SrcSize = Src2Op.IsGPR() ? OpSize::i128Bit : SrcElementSize;\n\n Ref Src1 = LoadSourceFPR_WithOpSize(Op, Src1Op, DstSize, Op->Flags);\n Ref Src2 = LoadSourceFPR_WithOpSize(Op, Src2Op, SrcSize, Op->Flags, {.AllowUpperGarbage = true});\n\n return _VFToFScalarInsert(DstSize, DstElementSize, SrcElementSize, Src1, Src2, ZeroUpperBits);\n}\n\ntemplate\nvoid OpDispatchBuilder::InsertScalar_CVT_Float_To_Float(OpcodeArgs) {\n const auto DstSize = GetGuestVectorLength();\n Ref Result = InsertScalar_CVT_Float_To_FloatImpl(Op, DstSize, DstElementSize, SrcElementSize, Op->Dest, Op->Src[0], false);\n StoreResultFPR_WithOpSize(Op, Op->Dest, Result, DstSize);\n}\n\ntemplate void OpDispatchBuilder::InsertScalar_CVT_Float_To_Float(OpcodeArgs);\ntemplate void OpDispatchBuilder::InsertScalar_CVT_Float_To_Float(OpcodeArgs);\n\ntemplate\nvoid OpDispatchBuilder::AVXInsertScalar_CVT_Float_To_Float(OpcodeArgs) {\n const auto DstSize = GetGuestVectorLength();\n Ref Result = InsertScalar_CVT_Float_To_FloatImpl(Op, DstSize, DstElementSize, SrcElementSize, Op->Src[0], Op->Src[1], true);\n StoreResultFPR_WithOpSize(Op, Op->Dest, Result, DstSize);\n}\n\ntemplate void OpDispatchBuilder::AVXInsertScalar_CVT_Float_To_Float(OpcodeArgs);\ntemplate void OpDispatchBuilder::AVXInsertScalar_CVT_Float_To_Float(OpcodeArgs);\n\nRoundMode OpDispatchBuilder::TranslateRoundType(uint8_t Mode) {\n const uint64_t RoundControlSource = (Mode >> 2) & 1;\n uint64_t RoundControl = Mode & 0b11;\n\n static constexpr std::array SourceModes = {\n RoundMode::Nearest,\n RoundMode::NegInfinity,\n RoundMode::PosInfinity,\n RoundMode::TowardsZero,\n };\n\n return RoundControlSource ? RoundMode::Host : SourceModes[RoundControl];\n}\n\nRef OpDispatchBuilder::InsertScalarRoundImpl(OpcodeArgs, IR::OpSize DstSize, IR::OpSize ElementSize, const X86Tables::DecodedOperand& Src1Op,\n const X86Tables::DecodedOperand& Src2Op, uint64_t Mode, bool ZeroUpperBits) {\n // We load the full vector width when dealing with a source vector,\n // so that we don't do any unnecessary zero extension to the scalar\n // element that we're going to operate on.\n const auto SrcSize = OpSizeFromSrc(Op);\n\n Ref Src1 = LoadSourceFPR_WithOpSize(Op, Src1Op, DstSize, Op->Flags);\n Ref Src2 = LoadSourceFPR_WithOpSize(Op, Src2Op, SrcSize, Op->Flags, {.AllowUpperGarbage = true});\n\n const auto SourceMode = TranslateRoundType(Mode);\n auto ALUOp = _VFToIScalarInsert(DstSize, ElementSize, Src1, Src2, SourceMode, ZeroUpperBits);\n\n return ALUOp;\n}\n\ntemplate\nvoid OpDispatchBuilder::InsertScalarRound(OpcodeArgs) {\n const uint64_t Mode = Op->Src[1].Literal();\n const auto DstSize = GetGuestVectorLength();\n\n Ref Result = InsertScalarRoundImpl(Op, DstSize, ElementSize, Op->Dest, Op->Src[0], Mode, false);\n StoreResultFPR_WithOpSize(Op, Op->Dest, Result, DstSize);\n}\n\ntemplate void OpDispatchBuilder::InsertScalarRound(OpcodeArgs);\ntemplate void OpDispatchBuilder::InsertScalarRound(OpcodeArgs);\n\ntemplate\nvoid OpDispatchBuilder::AVXInsertScalarRound(OpcodeArgs) {\n const uint64_t Mode = Op->Src[2].Literal();\n const auto DstSize = GetGuestVectorLength();\n\n Ref Result = InsertScalarRoundImpl(Op, DstSize, ElementSize, Op->Src[0], Op->Src[1], Mode, true);\n StoreResultFPR_WithOpSize(Op, Op->Dest, Result, DstSize);\n}\n\ntemplate void OpDispatchBuilder::AVXInsertScalarRound(OpcodeArgs);\ntemplate void OpDispatchBuilder::AVXInsertScalarRound(OpcodeArgs);\n\n\nRef OpDispatchBuilder::InsertScalarFCMPOpImpl(OpSize Size, IR::OpSize OpDstSize, IR::OpSize ElementSize, Ref Src1, Ref Src2,\n uint8_t CompType, bool ZeroUpperBits) {\n switch (static_cast(CompType)) {\n case VectorCompareType::EQ_OQ:\n case VectorCompareType::EQ_OS: return _VFCMPScalarInsert(Size, ElementSize, Src1, Src2, FloatCompareOp::EQ, ZeroUpperBits);\n case VectorCompareType::LT_OS: // GT(Swapped operand)\n case VectorCompareType::LT_OQ: return _VFCMPScalarInsert(Size, ElementSize, Src1, Src2, FloatCompareOp::LT, ZeroUpperBits);\n case VectorCompareType::LE_OS: // GE(Swapped operand)\n case VectorCompareType::LE_OQ: return _VFCMPScalarInsert(Size, ElementSize, Src1, Src2, FloatCompareOp::LE, ZeroUpperBits);\n case VectorCompareType::UNORD_Q:\n case VectorCompareType::UNORD_S: return _VFCMPScalarInsert(Size, ElementSize, Src1, Src2, FloatCompareOp::UNO, ZeroUpperBits);\n case VectorCompareType::NEQ_UQ:\n case VectorCompareType::NEQ_US: return _VFCMPScalarInsert(Size, ElementSize, Src1, Src2, FloatCompareOp::NEQ, ZeroUpperBits);\n case VectorCompareType::NLT_US: // NGT(Swapped operand)\n case VectorCompareType::NLT_UQ: {\n Ref Result = _VFCMPLT(ElementSize, ElementSize, Src1, Src2);\n Result = _VNot(ElementSize, ElementSize, Result);\n // Insert the lower bits\n return _VInsElement(OpDstSize, ElementSize, 0, 0, Src1, Result);\n }\n case VectorCompareType::NLE_US: // NGE(Swapped operand)\n case VectorCompareType::NLE_UQ: {\n Ref Result = _VFCMPLE(ElementSize, ElementSize, Src1, Src2);\n Result = _VNot(ElementSize, ElementSize, Result);\n // Insert the lower bits\n return _VInsElement(OpDstSize, ElementSize, 0, 0, Src1, Result);\n }\n case VectorCompareType::ORD_Q:\n case VectorCompareType::ORD_S: return _VFCMPScalarInsert(Size, ElementSize, Src1, Src2, FloatCompareOp::ORD, ZeroUpperBits);\n case VectorCompareType::NGT_UQ:\n case VectorCompareType::NGT_US: {\n Ref Result = _VFCMPLT(ElementSize, ElementSize, Src2, Src1);\n Result = _VNot(ElementSize, ElementSize, Result);\n // Insert the lower bits\n return _VInsElement(OpDstSize, ElementSize, 0, 0, Src1, Result);\n }\n case VectorCompareType::NGE_UQ:\n case VectorCompareType::NGE_US: {\n Ref Result = _VFCMPLE(ElementSize, ElementSize, Src2, Src1);\n Result = _VNot(ElementSize, ElementSize, Result);\n // Insert the lower bits\n return _VInsElement(OpDstSize, ElementSize, 0, 0, Src1, Result);\n }\n case VectorCompareType::GT_OQ:\n case VectorCompareType::GT_OS: {\n Ref Result = _VFCMPLT(ElementSize, ElementSize, Src2, Src1);\n // Insert the lower bits\n return _VInsElement(OpDstSize, ElementSize, 0, 0, Src1, Result);\n }\n case VectorCompareType::GE_OQ:\n case VectorCompareType::GE_OS: {\n Ref Result = _VFCMPLE(ElementSize, ElementSize, Src2, Src1);\n // Insert the lower bits\n return _VInsElement(OpDstSize, ElementSize, 0, 0, Src1, Result);\n }\n case VectorCompareType::EQ_UQ:\n case VectorCompareType::EQ_US: {\n // If either of the sources are unordered, then returns true.\n Ref Src1_U = _VFCMPEQ(Size, ElementSize, Src1, Src1);\n Ref Src2_U = _VFCMPEQ(Size, ElementSize, Src2, Src2);\n auto Ordered = _VAnd(Size, ElementSize, Src1_U, Src2_U);\n\n Ref Compare_Ordered = _VFCMPEQ(Size, ElementSize, Src1, Src2);\n Ref Result = _VOrn(Size, ElementSize, Compare_Ordered, Ordered);\n\n // Insert the lower bits\n return _VInsElement(OpDstSize, ElementSize, 0, 0, Src1, Result);\n }\n case VectorCompareType::NEQ_OQ:\n case VectorCompareType::NEQ_OS: {\n // If either of the sources are unordered, then returns false.\n Ref Src1_U = _VFCMPEQ(Size, ElementSize, Src1, Src1);\n Ref Src2_U = _VFCMPEQ(Size, ElementSize, Src2, Src2);\n\n Ref Compare_Ordered = _VFCMPEQ(Size, ElementSize, Src1, Src2);\n Ref Result = _VAndn(Size, ElementSize, Src1_U, Compare_Ordered);\n Result = _VAnd(Size, ElementSize, Result, Src2_U);\n\n // Insert the lower bits\n return _VInsElement(OpDstSize, ElementSize, 0, 0, Src1, Result);\n }\n case VectorCompareType::FALSE_OQ:\n case VectorCompareType::FALSE_OS: return _VInsElement(OpDstSize, ElementSize, 0, 0, Src1, LoadZeroVector(OpSize::i128Bit));\n case VectorCompareType::TRUE_UQ:\n case VectorCompareType::TRUE_US:\n return _VInsElement(OpDstSize, ElementSize, 0, 0, Src1, _VectorImm(OpSize::i128Bit, OpSize::i8Bit, -1, 0));\n }\n FEX_UNREACHABLE;\n}\n\ntemplate\nvoid OpDispatchBuilder::InsertScalarFCMPOp(OpcodeArgs) {\n const uint8_t CompType = Op->Src[1].Literal();\n const auto DstSize = GetGuestVectorLength();\n const auto SrcSize = OpSizeFromSrc(Op);\n\n Ref Src1 = LoadSourceFPR_WithOpSize(Op, Op->Dest, DstSize, Op->Flags);\n Ref Src2 = LoadSourceFPR_WithOpSize(Op, Op->Src[0], SrcSize, Op->Flags, {.AllowUpperGarbage = true});\n\n Ref Result = InsertScalarFCMPOpImpl(DstSize, OpSizeFromDst(Op), ElementSize, Src1, Src2, CompType & 0b111, false);\n StoreResultFPR_WithOpSize(Op, Op->Dest, Result, DstSize);\n}\n\ntemplate void OpDispatchBuilder::InsertScalarFCMPOp(OpcodeArgs);\ntemplate void OpDispatchBuilder::InsertScalarFCMPOp(OpcodeArgs);\n\ntemplate\nvoid OpDispatchBuilder::AVXInsertScalarFCMPOp(OpcodeArgs) {\n const uint8_t CompType = Op->Src[2].Literal();\n const auto DstSize = GetGuestVectorLength();\n const auto SrcSize = OpSizeFromSrc(Op);\n\n // We load the full vector width when dealing with a source vector,\n // so that we don't do any unnecessary zero extension to the scalar\n // element that we're going to operate on.\n Ref Src1 = LoadSourceFPR_WithOpSize(Op, Op->Src[0], DstSize, Op->Flags);\n Ref Src2 = LoadSourceFPR_WithOpSize(Op, Op->Src[1], SrcSize, Op->Flags, {.AllowUpperGarbage = true});\n\n Ref Result = InsertScalarFCMPOpImpl(DstSize, OpSizeFromDst(Op), ElementSize, Src1, Src2, CompType & 0b11111, true);\n StoreResultFPR_WithOpSize(Op, Op->Dest, Result, DstSize);\n}\n\ntemplate void OpDispatchBuilder::AVXInsertScalarFCMPOp(OpcodeArgs);\ntemplate void OpDispatchBuilder::AVXInsertScalarFCMPOp(OpcodeArgs);\n\nvoid OpDispatchBuilder::RSqrt3DNowOp(OpcodeArgs, bool Duplicate) {\n const auto Size = OpSizeFromSrc(Op);\n const auto ElementSize = OpSize::i32Bit;\n\n Ref Src = LoadSourceFPR_WithOpSize(Op, Op->Src[0], Size, Op->Flags);\n\n // For the sqrt reciprocal in 3DNow!, if the source is negative,\n // then the result has the same sign as the source but the result is always calculated\n // as if the source was positive.\n Ref AbsSrc = _VFAbs(Size, ElementSize, Src);\n Ref PosRSqrt = _VFRSqrtPrecision(Size, ElementSize, AbsSrc);\n Ref Result = _VFCopySign(Size, ElementSize, PosRSqrt, Src);\n\n if (Duplicate) {\n Result = _VDupElement(Size, ElementSize, Result, 0);\n }\n\n StoreResultFPR(Op, Result);\n}\n\nvoid OpDispatchBuilder::VectorUnaryOp(OpcodeArgs, IROps IROp, IR::OpSize ElementSize) {\n // In the event of a scalar operation and a vector source, then\n // we can specify the entire vector length in order to avoid\n // unnecessary sign extension on the element to be operated on.\n // In the event of a memory operand, we load the exact element size.\n const auto Size = OpSizeFromSrc(Op);\n\n Ref Src = LoadSourceFPR_WithOpSize(Op, Op->Src[0], Size, Op->Flags);\n\n DeriveOp(ALUOp, IROp, _VFSqrt(Size, ElementSize, Src));\n StoreResultFPR(Op, ALUOp);\n}\n\nvoid OpDispatchBuilder::AVXVectorUnaryOp(OpcodeArgs, IROps IROp, IR::OpSize ElementSize) {\n // In the event of a scalar operation and a vector source, then\n // we can specify the entire vector length in order to avoid\n // unnecessary sign extension on the element to be operated on.\n // In the event of a memory operand, we load the exact element size.\n const auto SrcSize = OpSizeFromSrc(Op);\n\n Ref Src = LoadSourceFPR_WithOpSize(Op, Op->Src[0], SrcSize, Op->Flags);\n\n DeriveOp(ALUOp, IROp, _VFSqrt(SrcSize, ElementSize, Src));\n\n // NOTE: We don't need to clear the upper lanes here, since the\n // IR ops make use of 128-bit AdvSimd for 128-bit cases,\n // which, on hardware with SVE, zero-extends as part of\n // storing into the destination.\n\n StoreResultFPR(Op, ALUOp);\n}\n\nvoid OpDispatchBuilder::VectorUnaryDuplicateOpImpl(OpcodeArgs, IROps IROp, IR::OpSize ElementSize) {\n const auto Size = OpSizeFromSrc(Op);\n\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n\n DeriveOp(ALUOp, IROp, _VFSqrt(ElementSize, ElementSize, Src));\n\n // Duplicate the lower bits\n auto Result = _VDupElement(Size, ElementSize, ALUOp, 0);\n StoreResultFPR(Op, Result);\n}\n\ntemplate\nvoid OpDispatchBuilder::VectorUnaryDuplicateOp(OpcodeArgs) {\n VectorUnaryDuplicateOpImpl(Op, IROp, ElementSize);\n}\n\n// TODO: there's only one instantiation of this template. Lets remove it.\ntemplate void OpDispatchBuilder::VectorUnaryDuplicateOp(OpcodeArgs);\n\nvoid OpDispatchBuilder::MOVQOp(OpcodeArgs, VectorOpType VectorType) {\n const auto SrcSize = Op->Src[0].IsGPR() ? OpSize::i128Bit : OpSizeFromSrc(Op);\n Ref Src = LoadSourceFPR_WithOpSize(Op, Op->Src[0], SrcSize, Op->Flags);\n // This instruction is a bit special that if the destination is a register then it'll ZEXT the 64bit source to 128bit\n if (Op->Dest.IsGPR()) {\n const auto gpr = Op->Dest.Data.GPR.GPR;\n const auto gprIndex = gpr - X86State::REG_XMM_0;\n\n auto Reg = VZeroExtendOperand(OpSize::i64Bit, Op->Src[0], Src);\n StoreXMMRegister_WithAVXInsert(VectorType, gprIndex, Reg);\n } else {\n // This is simple, just store the result\n StoreResultFPR(Op, Src);\n }\n}\n\nvoid OpDispatchBuilder::MOVQMMXOp(OpcodeArgs) {\n // Partial store into bottom 64-bits, leave the upper bits unaffected.\n if (MMXState == MMXState_X87) {\n ChgStateX87_MMX();\n }\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags, {.Align = OpSize::i8Bit});\n StoreResultFPR(Op, Src, OpSize::i8Bit);\n}\n\nvoid OpDispatchBuilder::MOVMSKOp(OpcodeArgs, IR::OpSize ElementSize) {\n const auto Size = OpSizeFromSrc(Op);\n const auto NumElements = IR::NumElements(Size, ElementSize);\n\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n\n if (Size == OpSize::i128Bit && ElementSize == OpSize::i64Bit) {\n // UnZip2 the 64-bit elements as 32-bit to get the sign bits closer.\n // Sign bits are now in bit positions 31 and 63 after this.\n Src = _VUnZip2(Size, OpSize::i32Bit, Src, Src);\n\n // Extract the low 64-bits to GPR in one move.\n Ref GPR = _VExtractToGPR(Size, OpSize::i64Bit, Src, 0);\n // BFI the sign bit in 31 in to 62.\n // Inserting the full lower 32-bits offset 31 so the sign bit ends up at offset 63.\n GPR = _Bfi(OpSize::i64Bit, 32, 31, GPR, GPR);\n // Shift right to only get the two sign bits we care about.\n GPR = _Lshr(OpSize::i64Bit, GPR, Constant(62));\n StoreResultGPR_WithOpSize(Op, Op->Dest, GPR, GetGPROpSize());\n } else if (Size == OpSize::i128Bit && ElementSize == OpSize::i32Bit) {\n // Shift all the sign bits to the bottom of their respective elements.\n Src = _VUShrI(Size, OpSize::i32Bit, Src, 31);\n // Load the specific 128-bit movmskps shift elements operator.\n auto ConstantUSHL = LoadAndCacheNamedVectorConstant(Size, NAMED_VECTOR_MOVMSKPS_SHIFT);\n // Shift the sign bits in to specific locations.\n Src = _VUShl(Size, OpSize::i32Bit, Src, ConstantUSHL, false);\n // Add across the vector so the sign bits will end up in bits [3:0]\n Src = _VAddV(Size, OpSize::i32Bit, Src);\n // Extract to a GPR.\n Ref GPR = _VExtractToGPR(Size, OpSize::i32Bit, Src, 0);\n StoreResultGPR_WithOpSize(Op, Op->Dest, GPR, GetGPROpSize());\n } else {\n Ref CurrentVal = Constant(0);\n\n for (unsigned i = 0; i < NumElements; ++i) {\n // Extract the top bit of the element\n Ref Tmp = _VExtractToGPR(Size, ElementSize, Src, i);\n Tmp = _Bfe(ElementSize, 1, IR::OpSizeAsBits(ElementSize) - 1, Tmp);\n\n // Shift it to the correct location and or it with the current value\n if (i != 0) {\n CurrentVal = _Orlshl(OpSize::i64Bit, CurrentVal, Tmp, i);\n } else {\n CurrentVal = Tmp;\n }\n }\n StoreResultGPR(Op, CurrentVal);\n }\n}\n\nvoid OpDispatchBuilder::MOVMSKOpOne(OpcodeArgs) {\n const auto SrcSize = OpSizeFromSrc(Op);\n const auto Is256Bit = SrcSize == OpSize::i256Bit;\n const auto ExtractSize = Is256Bit ? OpSize::i32Bit : OpSize::i16Bit;\n\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n Ref VMask = LoadAndCacheNamedVectorConstant(SrcSize, NAMED_VECTOR_MOVMASKB);\n\n auto VCMP = _VCMPLTZ(SrcSize, OpSize::i8Bit, Src);\n auto VAnd = _VAnd(SrcSize, OpSize::i8Bit, VCMP, VMask);\n\n // Since we also handle the MM MOVMSKB here too,\n // we need to clamp the lower bound.\n const auto VAdd1Size = std::max(SrcSize, OpSize::i128Bit);\n const auto VAdd2Size = std::max(SrcSize >> 1, OpSize::i64Bit);\n\n auto VAdd1 = _VAddP(VAdd1Size, OpSize::i8Bit, VAnd, VAnd);\n auto VAdd2 = _VAddP(VAdd2Size, OpSize::i8Bit, VAdd1, VAdd1);\n auto VAdd3 = _VAddP(OpSize::i64Bit, OpSize::i8Bit, VAdd2, VAdd2);\n\n auto Result = _VExtractToGPR(SrcSize, ExtractSize, VAdd3, 0);\n\n StoreResultGPR(Op, Result);\n}\n\nvoid OpDispatchBuilder::PUNPCKLOp(OpcodeArgs, IR::OpSize ElementSize) {\n const auto Size = OpSizeFromSrc(Op);\n\n Ref Dest = LoadSourceFPR(Op, Op->Dest, Op->Flags);\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n\n auto ALUOp = _VZip(Size, ElementSize, Dest, Src);\n StoreResultFPR(Op, ALUOp);\n}\n\nvoid OpDispatchBuilder::VPUNPCKLOp(OpcodeArgs, IR::OpSize ElementSize) {\n const auto SrcSize = OpSizeFromSrc(Op);\n const auto Is128Bit = SrcSize == OpSize::i128Bit;\n\n Ref Src1 = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n Ref Src2 = LoadSourceFPR(Op, Op->Src[1], Op->Flags);\n\n Ref Result {};\n if (Is128Bit) {\n Result = _VZip(SrcSize, ElementSize, Src1, Src2);\n } else {\n Ref ZipLo = _VZip(SrcSize, ElementSize, Src1, Src2);\n Ref ZipHi = _VZip2(SrcSize, ElementSize, Src1, Src2);\n\n Result = _VInsElement(SrcSize, OpSize::i128Bit, 1, 0, ZipLo, ZipHi);\n }\n\n StoreResultFPR(Op, Result);\n}\n\nvoid OpDispatchBuilder::PUNPCKHOp(OpcodeArgs, IR::OpSize ElementSize) {\n const auto Size = OpSizeFromSrc(Op);\n Ref Dest = LoadSourceFPR(Op, Op->Dest, Op->Flags);\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n\n auto ALUOp = _VZip2(Size, ElementSize, Dest, Src);\n StoreResultFPR(Op, ALUOp);\n}\n\nvoid OpDispatchBuilder::VPUNPCKHOp(OpcodeArgs, IR::OpSize ElementSize) {\n const auto SrcSize = OpSizeFromSrc(Op);\n const auto Is128Bit = SrcSize == OpSize::i128Bit;\n Ref Src1 = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n Ref Src2 = LoadSourceFPR(Op, Op->Src[1], Op->Flags);\n\n Ref Result {};\n if (Is128Bit) {\n Result = _VZip2(SrcSize, ElementSize, Src1, Src2);\n } else {\n Ref ZipLo = _VZip(SrcSize, ElementSize, Src1, Src2);\n Ref ZipHi = _VZip2(SrcSize, ElementSize, Src1, Src2);\n\n Result = _VInsElement(SrcSize, OpSize::i128Bit, 0, 1, ZipHi, ZipLo);\n }\n\n StoreResultFPR(Op, Result);\n}\n\nRef OpDispatchBuilder::GeneratePSHUFBMask(IR::OpSize SrcSize) {\n // PSHUFB doesn't 100% match VTBL behaviour\n // VTBL will set the element zero if the index is greater than\n // the number of elements in the array\n //\n // Bit 7 is the only bit that is supposed to set elements to zero with PSHUFB\n // Mask the selection bits and top bit correctly\n // Bits [6:4] is reserved for 128-bit/256-bit\n // Bits [6:3] is reserved for 64-bit\n const uint8_t MaskImm = SrcSize == OpSize::i64Bit ? 0b1000'0111 : 0b1000'1111;\n\n return _VectorImm(SrcSize, OpSize::i8Bit, MaskImm);\n}\n\nRef OpDispatchBuilder::PSHUFBOpImpl(IR::OpSize SrcSize, Ref Src1, Ref Src2, Ref MaskVector) {\n const auto Is256Bit = SrcSize == OpSize::i256Bit;\n\n // We perform the 256-bit version as two 128-bit operations due to\n // the lane splitting behavior, so cap the maximum size at 16.\n const auto SanitizedSrcSize = std::min(SrcSize, OpSize::i128Bit);\n\n Ref MaskedIndices = _VAnd(SrcSize, SrcSize, Src2, MaskVector);\n\n Ref Low = _VTBL1(SanitizedSrcSize, Src1, MaskedIndices);\n if (!Is256Bit) {\n return Low;\n }\n\n Ref HighSrc1 = _VInsElement(SrcSize, OpSize::i128Bit, 0, 1, Src1, Src1);\n Ref High = _VTBL1(SanitizedSrcSize, HighSrc1, MaskedIndices);\n return _VInsElement(SrcSize, OpSize::i128Bit, 1, 0, Low, High);\n}\n\nvoid OpDispatchBuilder::PSHUFBOp(OpcodeArgs) {\n const auto SrcSize = OpSizeFromSrc(Op);\n Ref Src1 = LoadSourceFPR(Op, Op->Dest, Op->Flags);\n Ref Src2 = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n\n Ref Result = PSHUFBOpImpl(SrcSize, Src1, Src2, GeneratePSHUFBMask(SrcSize));\n StoreResultFPR(Op, Result);\n}\n\nvoid OpDispatchBuilder::VPSHUFBOp(OpcodeArgs) {\n const auto SrcSize = OpSizeFromSrc(Op);\n Ref Src1 = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n Ref Src2 = LoadSourceFPR(Op, Op->Src[1], Op->Flags);\n\n Ref Result = PSHUFBOpImpl(SrcSize, Src1, Src2, GeneratePSHUFBMask(SrcSize));\n StoreResultFPR(Op, Result);\n}\n\nRef OpDispatchBuilder::PShufWLane(IR::OpSize Size, FEXCore::IR::IndexNamedVectorConstant IndexConstant, bool LowLane, Ref IncomingLane,\n uint8_t Shuffle) {\n constexpr auto IdentityCopy = 0b11'10'01'00;\n\n const bool Is128BitLane = Size == OpSize::i128Bit;\n const auto NumElements = IR::NumElements(Size, IR::OpSize::i16Bit);\n const auto HalfNumElements = NumElements >> 1;\n\n // TODO: There can be more optimized copies here.\n switch (Shuffle) {\n case IdentityCopy: {\n // Special case identity copy.\n return IncomingLane;\n }\n case 0b00'00'00'00:\n case 0b01'01'01'01:\n case 0b10'10'10'10:\n case 0b11'11'11'11: {\n // Special case element duplicate and broadcast to low or high 64-bits.\n Ref Dup = _VDupElement(Size, OpSize::i16Bit, IncomingLane, (LowLane ? 0 : HalfNumElements) + (Shuffle & 0b11));\n if (Is128BitLane) {\n if (LowLane) {\n // DUP goes low.\n // Source goes high.\n Dup = _VTrn2(Size, OpSize::i64Bit, Dup, IncomingLane);\n } else {\n // DUP goes high.\n // Source goes low.\n Dup = _VTrn(Size, OpSize::i64Bit, IncomingLane, Dup);\n }\n }\n\n return Dup;\n }\n default: {\n // PSHUFLW needs to scale index by 16.\n // PSHUFHW needs to scale index by 16.\n // PSHUFW (mmx) also needs to scale by 16 to get correct low element.\n auto LookupIndexes = LoadAndCacheIndexedNamedVectorConstant(Size, IndexConstant, Shuffle * 16);\n return _VTBL1(Size, IncomingLane, LookupIndexes);\n }\n }\n}\n\nvoid OpDispatchBuilder::PSHUFW8ByteOp(OpcodeArgs) {\n uint16_t Shuffle = Op->Src[1].Data.Literal.Value;\n const auto Size = OpSizeFromSrc(Op);\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n Ref Dest = PShufWLane(Size, FEXCore::IR::INDEXED_NAMED_VECTOR_PSHUFLW, true, Src, Shuffle);\n StoreResultFPR(Op, Dest);\n}\n\nvoid OpDispatchBuilder::PSHUFWOp(OpcodeArgs, bool Low) {\n uint16_t Shuffle = Op->Src[1].Data.Literal.Value;\n const auto Size = OpSizeFromSrc(Op);\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n const auto IndexedVectorConstant = Low ? FEXCore::IR::IndexNamedVectorConstant::INDEXED_NAMED_VECTOR_PSHUFLW :\n FEXCore::IR::IndexNamedVectorConstant::INDEXED_NAMED_VECTOR_PSHUFHW;\n\n Ref Dest = PShufWLane(Size, IndexedVectorConstant, Low, Src, Shuffle);\n\n StoreResultFPR(Op, Dest);\n}\n\nRef OpDispatchBuilder::Single128Bit4ByteVectorShuffle(Ref Src, uint8_t Shuffle) {\n constexpr auto IdentityCopy = 0b11'10'01'00;\n\n // TODO: There can be more optimized copies here.\n switch (Shuffle) {\n case IdentityCopy: {\n // Special case identity copy.\n return Src;\n }\n case 0b00'00'00'00:\n case 0b01'01'01'01:\n case 0b10'10'10'10:\n case 0b11'11'11'11: {\n // Special case element duplicate and broadcast to low or high 64-bits.\n return _VDupElement(OpSize::i128Bit, OpSize::i32Bit, Src, Shuffle & 0b11);\n }\n case 0b00'00'10'10: {\n // Weird reverse low elements and broadcast to each half of the register\n Ref Tmp = _VUnZip(OpSize::i128Bit, OpSize::i32Bit, Src, Src);\n Tmp = _VRev64(OpSize::i128Bit, OpSize::i32Bit, Tmp);\n return _VZip(OpSize::i128Bit, OpSize::i32Bit, Tmp, Tmp);\n }\n case 0b00'00'11'10: {\n // First element duplicated and shifted in to the top.\n auto Dup = _VDupElement(OpSize::i128Bit, OpSize::i32Bit, Src, 0);\n return _VExtr(OpSize::i128Bit, OpSize::i32Bit, Dup, Src, 2);\n }\n case 0b00'01'00'01: {\n ///< Weird reversed low elements and broadcast\n Ref Tmp = _VRev64(OpSize::i128Bit, OpSize::i32Bit, Src);\n return _VZip(OpSize::i128Bit, OpSize::i64Bit, Tmp, Tmp);\n }\n case 0b00'01'01'00: {\n ///< Weird reverse low two elements in to high half\n Ref Tmp = _VZip(OpSize::i128Bit, OpSize::i32Bit, Src, Src);\n return _VExtr(OpSize::i128Bit, OpSize::i8Bit, Tmp, Tmp, 4);\n }\n case 0b00'01'10'11: {\n // Inverse elements\n Ref Tmp = _VRev64(OpSize::i128Bit, OpSize::i32Bit, Src);\n return _VExtr(OpSize::i128Bit, OpSize::i32Bit, Tmp, Tmp, 2);\n }\n case 0b00'10'00'10: {\n ///< Weird reversed even elements and broadcast\n Ref Tmp = _VUnZip(OpSize::i128Bit, OpSize::i32Bit, Src, Src);\n return _VExtr(OpSize::i128Bit, OpSize::i8Bit, Tmp, Tmp, 4);\n }\n case 0b00'10'10'00: {\n // Weird reversed low elements in upper half of the register\n Ref Tmp = _VUnZip(OpSize::i128Bit, OpSize::i32Bit, Src, Src);\n Tmp = _VZip(OpSize::i128Bit, OpSize::i32Bit, Tmp, Tmp);\n return _VExtr(OpSize::i128Bit, OpSize::i8Bit, Tmp, Tmp, 4);\n }\n case 0b00'11'00'11: {\n ///< Weird Low plus high element reversed and broadcast\n Ref Tmp = _VExtr(OpSize::i128Bit, OpSize::i8Bit, Src, Src, 4);\n return _VZip2(OpSize::i128Bit, OpSize::i64Bit, Tmp, Tmp);\n }\n case 0b00'11'10'01:\n ///< Vector rotate - One element\n return _VExtr(OpSize::i128Bit, OpSize::i8Bit, Src, Src, 4);\n case 0b00'11'11'00: {\n // Weird reversed low and high elements in upper half of the register\n Ref Tmp = _VExtr(OpSize::i128Bit, OpSize::i8Bit, Src, Src, 4);\n Tmp = _VZip2(OpSize::i128Bit, OpSize::i32Bit, Tmp, Tmp);\n return _VExtr(OpSize::i128Bit, OpSize::i8Bit, Tmp, Tmp, 12);\n }\n case 0b01'00'00'01: {\n ///< Weird duplicate bottom two elements, then rotate in the low half\n Ref Tmp = _VZip(OpSize::i128Bit, OpSize::i32Bit, Src, Src);\n return _VExtr(OpSize::i128Bit, OpSize::i8Bit, Tmp, Tmp, 12);\n }\n case 0b01'00'01'00:\n ///< Duplicate bottom 64-bits\n return _VDupElement(OpSize::i128Bit, OpSize::i64Bit, Src, 0);\n case 0b01'00'11'10:\n ///< Vector rotate - Two elements\n return _VExtr(OpSize::i128Bit, OpSize::i8Bit, Src, Src, 8);\n case 0b01'01'00'00: {\n // Zip with self.\n // Dest[0] = Src[0]\n // Dest[1] = Src[0]\n // Dest[2] = Src[1]\n // Dest[3] = Src[1]\n return _VZip(OpSize::i128Bit, OpSize::i32Bit, Src, Src);\n }\n case 0b01'01'10'10: {\n ///< Weird reverse middle elements and broadcast to each half of the register\n Ref Tmp = _VExtr(OpSize::i128Bit, OpSize::i8Bit, Src, Src, 4);\n Tmp = _VRev64(OpSize::i128Bit, OpSize::i32Bit, Tmp);\n return _VZip(OpSize::i128Bit, OpSize::i32Bit, Tmp, Tmp);\n }\n case 0b01'01'11'11: {\n ///< Weird reverse odd elements and broadcast to each half of the register\n Ref Tmp = _VUnZip2(OpSize::i128Bit, OpSize::i32Bit, Src, Src);\n Tmp = _VRev64(OpSize::i128Bit, OpSize::i32Bit, Tmp);\n return _VZip(OpSize::i128Bit, OpSize::i32Bit, Tmp, Tmp);\n }\n case 0b01'10'01'10: {\n ///< Weird middle elements swizzle plus broadcast\n Ref Tmp = _VExtr(OpSize::i128Bit, OpSize::i8Bit, Src, Src, 4);\n Tmp = _VRev64(OpSize::i128Bit, OpSize::i32Bit, Tmp);\n return _VZip(OpSize::i128Bit, OpSize::i64Bit, Tmp, Tmp);\n }\n case 0b01'10'10'01: {\n ///< Weird middle elements swizzle plus broadcast and reverse\n Ref Tmp = _VExtr(OpSize::i128Bit, OpSize::i8Bit, Src, Src, 4);\n Tmp = _VZip(OpSize::i128Bit, OpSize::i32Bit, Tmp, Tmp);\n return _VExtr(OpSize::i128Bit, OpSize::i8Bit, Tmp, Tmp, 4);\n }\n case 0b01'11'01'11: {\n ///< Weird reversed odd elements and broadcast\n Ref Tmp = _VUnZip2(OpSize::i128Bit, OpSize::i32Bit, Src, Src);\n return _VExtr(OpSize::i128Bit, OpSize::i8Bit, Tmp, Tmp, 4);\n }\n case 0b01'11'11'01: {\n ///< Weird odd elements swizzle plus broadcast and reverse\n Ref Tmp = _VUnZip2(OpSize::i128Bit, OpSize::i32Bit, Src, Src);\n Tmp = _VZip(OpSize::i128Bit, OpSize::i32Bit, Tmp, Tmp);\n return _VExtr(OpSize::i128Bit, OpSize::i8Bit, Tmp, Tmp, 4);\n }\n case 0b10'00'00'10: {\n ///< Weird even elements swizzle plus broadcast and reverse\n Ref Tmp = _VUnZip(OpSize::i128Bit, OpSize::i32Bit, Src, Src);\n Tmp = _VZip(OpSize::i128Bit, OpSize::i32Bit, Tmp, Tmp);\n return _VExtr(OpSize::i128Bit, OpSize::i8Bit, Tmp, Tmp, 12);\n }\n case 0b10'00'10'00:\n ///< Even elements broadcast\n return _VUnZip(OpSize::i128Bit, OpSize::i32Bit, Src, Src);\n case 0b10'01'00'11:\n ///< Vector rotate - Three elements\n return _VExtr(OpSize::i128Bit, OpSize::i8Bit, Src, Src, 12);\n\n case 0b10'01'01'10: {\n ///< Weird odd elements swizzle plus broadcast and reverse\n Ref Tmp = _VExtr(OpSize::i128Bit, OpSize::i8Bit, Src, Src, 4);\n Tmp = _VZip(OpSize::i128Bit, OpSize::i32Bit, Tmp, Tmp);\n return _VExtr(OpSize::i128Bit, OpSize::i8Bit, Tmp, Tmp, 12);\n }\n case 0b10'01'10'01: {\n ///< Middle two elements broadcast\n Ref Tmp = _VExtr(OpSize::i128Bit, OpSize::i8Bit, Src, Src, 4);\n return _VZip(OpSize::i128Bit, OpSize::i64Bit, Tmp, Tmp);\n }\n case 0b10'10'00'00: {\n ///< Broadcast even elements to each half of the register\n Ref Tmp = _VUnZip(OpSize::i128Bit, OpSize::i32Bit, Src, Src);\n return _VZip(OpSize::i128Bit, OpSize::i32Bit, Tmp, Tmp);\n }\n case 0b10'10'01'01: {\n ///< Broadcast middle elements to each half of the register\n Ref Tmp = _VExtr(OpSize::i128Bit, OpSize::i8Bit, Src, Src, 4);\n return _VZip(OpSize::i128Bit, OpSize::i32Bit, Tmp, Tmp);\n }\n case 0b10'10'11'11: {\n ///< Reverse top two elements and broadcast to each half of the register\n Ref Tmp = _VZip2(OpSize::i128Bit, OpSize::i32Bit, Src, Src);\n return _VExtr(OpSize::i128Bit, OpSize::i8Bit, Tmp, Tmp, 8);\n }\n case 0b10'11'00'01: {\n // Reverse each 64-bit lane.\n return _VRev64(OpSize::i128Bit, OpSize::i32Bit, Src);\n }\n case 0b10'11'10'11: {\n ///< Weird top two elements reverse and broadcast\n Ref Tmp = _VZip2(OpSize::i128Bit, OpSize::i64Bit, Src, Src);\n return _VExtr(OpSize::i128Bit, OpSize::i8Bit, Tmp, Tmp, 4);\n }\n case 0b10'11'11'10: {\n ///< Weird move top two elements to bottom and reverse in the top half\n Ref Tmp = _VZip2(OpSize::i128Bit, OpSize::i32Bit, Src, Src);\n return _VExtr(OpSize::i128Bit, OpSize::i8Bit, Tmp, Tmp, 4);\n }\n case 0b11'00'00'11: {\n ///< Weird low plus high elements swizzle plus broadcast and reverse\n Ref Tmp = _VExtr(OpSize::i128Bit, OpSize::i8Bit, Src, Src, 4);\n Tmp = _VZip2(OpSize::i128Bit, OpSize::i32Bit, Tmp, Tmp);\n return _VExtr(OpSize::i128Bit, OpSize::i8Bit, Tmp, Tmp, 4);\n }\n case 0b11'00'11'00: {\n ///< Weird low plus high element broadcast\n Ref Tmp = _VExtr(OpSize::i128Bit, OpSize::i8Bit, Src, Src, 4);\n Tmp = _VZip2(OpSize::i128Bit, OpSize::i64Bit, Tmp, Tmp);\n return _VExtr(OpSize::i128Bit, OpSize::i8Bit, Tmp, Tmp, 4);\n }\n case 0b11'01'01'11: {\n ///< Weird odd elements swizzle plus broadcast and reverse\n Ref Tmp = _VUnZip2(OpSize::i128Bit, OpSize::i32Bit, Src, Src);\n Tmp = _VZip(OpSize::i128Bit, OpSize::i32Bit, Tmp, Tmp);\n return _VExtr(OpSize::i128Bit, OpSize::i8Bit, Tmp, Tmp, 12);\n }\n case 0b11'01'11'01:\n ///< Odd elements broadcast\n return _VUnZip2(OpSize::i128Bit, OpSize::i32Bit, Src, Src);\n case 0b11'10'10'11: {\n ///< Rotate top two elements in to bottom half of the register\n Ref Tmp = _VZip2(OpSize::i128Bit, OpSize::i32Bit, Src, Src);\n return _VExtr(OpSize::i128Bit, OpSize::i8Bit, Tmp, Tmp, 12);\n }\n case 0b11'10'11'10:\n ///< Duplicate Top 64-bits\n return _VDupElement(OpSize::i128Bit, OpSize::i64Bit, Src, 1);\n case 0b11'11'00'00: {\n ///< Weird Broadcast bottom and top element to each half of the register\n Ref Tmp = _VExtr(OpSize::i128Bit, OpSize::i8Bit, Src, Src, 12);\n Tmp = _VRev64(OpSize::i128Bit, OpSize::i32Bit, Tmp);\n return _VZip(OpSize::i128Bit, OpSize::i32Bit, Tmp, Tmp);\n }\n case 0b11'11'01'01: {\n ///< Broadcast odd elements to each half of the register\n Ref Tmp = _VUnZip2(OpSize::i128Bit, OpSize::i32Bit, Src, Src);\n return _VZip(OpSize::i128Bit, OpSize::i32Bit, Tmp, Tmp);\n }\n case 0b11'11'10'10:\n ///< Broadcast top two elements to each half of the register\n return _VZip2(OpSize::i128Bit, OpSize::i32Bit, Src, Src);\n default: {\n // PSHUFD needs to scale index by 16.\n auto LookupIndexes =\n LoadAndCacheIndexedNamedVectorConstant(OpSize::i128Bit, FEXCore::IR::IndexNamedVectorConstant::INDEXED_NAMED_VECTOR_PSHUFD, Shuffle * 16);\n return _VTBL1(OpSize::i128Bit, Src, LookupIndexes);\n }\n }\n}\n\nvoid OpDispatchBuilder::PSHUFDOp(OpcodeArgs) {\n uint16_t Shuffle = Op->Src[1].Data.Literal.Value;\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n StoreResultFPR(Op, Single128Bit4ByteVectorShuffle(Src, Shuffle));\n}\n\nvoid OpDispatchBuilder::VPSHUFWOp(OpcodeArgs, IR::OpSize ElementSize, bool Low) {\n const auto SrcSize = OpSizeFromSrc(Op);\n const auto Is256Bit = SrcSize == OpSize::i256Bit;\n auto Shuffle = Op->Src[1].Literal();\n\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n\n // Note/TODO: With better immediate facilities or vector loading in our IR\n // much of this can be reduced to setting up a table index register\n // and then using TBL\n //\n // SVE has the INDEX instruction that works essentially like\n // std::iota (setting a range to an initial value and progressively\n // incrementing each successive element), so it's well suited for this.\n // It's just a matter of exposing these facilities in a way that works\n // well together.\n //\n // Should be much nicer than doing repeated inserts in any case.\n\n const size_t BaseElement = Low ? 0 : 4;\n Ref Result = Src;\n if (Is256Bit) {\n for (size_t i = 0; i < 4; i++) {\n const auto Index = Shuffle & 0b11;\n const auto UpperLaneOffset = IR::NumElements(OpSize::i128Bit, ElementSize);\n\n const auto LowDstIndex = BaseElement + i;\n const auto LowSrcIndex = BaseElement + Index;\n\n const auto HighDstIndex = BaseElement + UpperLaneOffset + i;\n const auto HighSrcIndex = BaseElement + UpperLaneOffset + Index;\n\n // Take care of both lanes per iteration\n Result = _VInsElement(SrcSize, ElementSize, LowDstIndex, LowSrcIndex, Result, Src);\n Result = _VInsElement(SrcSize, ElementSize, HighDstIndex, HighSrcIndex, Result, Src);\n\n Shuffle >>= 2;\n }\n } else {\n for (size_t i = 0; i < 4; i++) {\n const auto Index = Shuffle & 0b11;\n Result = _VInsElement(SrcSize, ElementSize, BaseElement + i, BaseElement + Index, Result, Src);\n Shuffle >>= 2;\n }\n }\n\n StoreResultFPR(Op, Result);\n}\n\nRef OpDispatchBuilder::SHUFOpImpl(OpcodeArgs, IR::OpSize DstSize, IR::OpSize ElementSize, Ref Src1, Ref Src2, uint8_t Shuffle) {\n // Since 256-bit variants and up don't lane cross, we can construct\n // everything in terms of the 128-variant, as each lane is essentially\n // its own 128-bit segment.\n const uint8_t NumElements = IR::NumElements(OpSize::i128Bit, ElementSize);\n const uint8_t HalfNumElements = NumElements >> 1;\n\n const bool Is256Bit = DstSize == OpSize::i256Bit;\n\n std::array Srcs {};\n for (size_t i = 0; i < HalfNumElements; ++i) {\n Srcs[i] = Src1;\n }\n for (size_t i = HalfNumElements; i < NumElements; ++i) {\n Srcs[i] = Src2;\n }\n\n Ref Dest = Src1;\n const uint8_t SelectionMask = NumElements - 1;\n const uint8_t ShiftAmount = std::popcount(SelectionMask);\n\n if (Is256Bit) {\n for (uint8_t Element = 0; Element < NumElements; ++Element) {\n const auto SrcIndex1 = Shuffle & SelectionMask;\n\n // AVX differs the behavior of VSHUFPD and VSHUFPS.\n // The same immediate bits are used for both lanes with VSHUFPS,\n // but VSHUFPD uses different immediate bits for each lane.\n const auto SrcIndex2 = ElementSize == OpSize::i32Bit ? SrcIndex1 : ((Shuffle >> 2) & SelectionMask);\n\n Ref Insert = _VInsElement(DstSize, ElementSize, Element, SrcIndex1, Dest, Srcs[Element]);\n Dest = _VInsElement(DstSize, ElementSize, Element + NumElements, SrcIndex2 + NumElements, Insert, Srcs[Element]);\n\n Shuffle >>= ShiftAmount;\n }\n } else {\n if (ElementSize == OpSize::i32Bit) {\n // We can shuffle optimally in a lot of cases.\n // TODO: We can optimize more of these cases.\n switch (Shuffle) {\n case 0b01'00'01'00:\n // Combining of low 64-bits.\n // Dest[63:0] = Src1[63:0]\n // Dest[127:64] = Src2[63:0]\n return _VZip(DstSize, OpSize::i64Bit, Src1, Src2);\n case 0b11'10'11'10:\n // Combining of high 64-bits.\n // Dest[63:0] = Src1[127:64]\n // Dest[127:64] = Src2[127:64]\n return _VZip2(DstSize, OpSize::i64Bit, Src1, Src2);\n case 0b11'10'01'00:\n // Mixing Low and high elements\n // Dest[63:0] = Src1[63:0]\n // Dest[127:64] = Src2[127:64]\n return _VInsElement(DstSize, OpSize::i64Bit, 1, 1, Src1, Src2);\n case 0b01'00'11'10:\n // Mixing Low and high elements, inverse of above\n // Dest[63:0] = Src1[127:64]\n // Dest[127:64] = Src2[63:0]\n return _VExtr(DstSize, OpSize::i8Bit, Src2, Src1, 8);\n case 0b10'00'10'00:\n // Mixing even elements.\n // Dest[31:0] = Src1[31:0]\n // Dest[63:32] = Src1[95:64]\n // Dest[95:64] = Src2[31:0]\n // Dest[127:96] = Src2[95:64]\n return _VUnZip(DstSize, ElementSize, Src1, Src2);\n case 0b11'01'11'01:\n // Mixing odd elements.\n // Dest[31:0] = Src1[63:32]\n // Dest[63:32] = Src1[127:96]\n // Dest[95:64] = Src2[63:32]\n // Dest[127:96] = Src2[127:96]\n return _VUnZip2(DstSize, ElementSize, Src1, Src2);\n case 0b11'10'00'00:\n case 0b11'10'01'01:\n case 0b11'10'10'10:\n case 0b11'10'11'11: {\n // Bottom elements duplicated, Top 64-bits inserted\n auto DupSrc1 = _VDupElement(DstSize, ElementSize, Src1, Shuffle & 0b11);\n return _VZip2(DstSize, OpSize::i64Bit, DupSrc1, Src2);\n }\n case 0b01'00'00'00:\n case 0b01'00'01'01:\n case 0b01'00'10'10:\n case 0b01'00'11'11: {\n // Bottom elements duplicated, Bottom 64-bits inserted\n auto DupSrc1 = _VDupElement(DstSize, ElementSize, Src1, Shuffle & 0b11);\n return _VZip(DstSize, OpSize::i64Bit, DupSrc1, Src2);\n }\n case 0b00'00'01'00:\n case 0b01'01'01'00:\n case 0b10'10'01'00:\n case 0b11'11'01'00: {\n // Top elements duplicated, Bottom 64-bits inserted\n auto DupSrc2 = _VDupElement(DstSize, ElementSize, Src2, (Shuffle >> 4) & 0b11);\n return _VZip(DstSize, OpSize::i64Bit, Src1, DupSrc2);\n }\n case 0b00'00'11'10:\n case 0b01'01'11'10:\n case 0b10'10'11'10:\n case 0b11'11'11'10: {\n // Top elements duplicated, Top 64-bits inserted\n auto DupSrc2 = _VDupElement(DstSize, ElementSize, Src2, (Shuffle >> 4) & 0b11);\n return _VZip2(DstSize, OpSize::i64Bit, Src1, DupSrc2);\n }\n case 0b01'00'01'11: {\n // TODO: This doesn't generate optimal code.\n // RA doesn't understand that Src1 is dead after VInsElement due to SRA class differences.\n // With RA fixes this would be 2 instructions.\n // Odd elements inverted, Low 64-bits inserted\n Src1 = _VInsElement(DstSize, OpSize::i32Bit, 0, 3, Src1, Src1);\n return _VZip(DstSize, OpSize::i64Bit, Src1, Src2);\n }\n case 0b11'10'01'11: {\n // TODO: This doesn't generate optimal code.\n // RA doesn't understand that Src1 is dead after VInsElement due to SRA class differences.\n // With RA fixes this would be 2 instructions.\n // Odd elements inverted, Top 64-bits inserted\n Src1 = _VInsElement(DstSize, OpSize::i32Bit, 0, 3, Src1, Src1);\n return _VInsElement(DstSize, OpSize::i64Bit, 1, 1, Src1, Src2);\n }\n case 0b01'00'00'01: {\n // Lower 32-bit elements inverted, low 64-bits inserted\n Src1 = _VRev64(DstSize, OpSize::i32Bit, Src1);\n return _VZip(DstSize, OpSize::i64Bit, Src1, Src2);\n }\n case 0b11'10'00'01: {\n // TODO: This doesn't generate optimal code.\n // RA doesn't understand that Src1 is dead after VInsElement due to SRA class differences.\n // With RA fixes this would be 2 instructions.\n // Lower 32-bit elements inverted, Top 64-bits inserted\n Src1 = _VRev64(DstSize, OpSize::i32Bit, Src1);\n return _VInsElement(DstSize, OpSize::i64Bit, 1, 1, Src1, Src2);\n }\n case 0b00'00'00'00:\n case 0b00'00'01'01:\n case 0b00'00'10'10:\n case 0b00'00'11'11:\n case 0b01'01'00'00:\n case 0b01'01'01'01:\n case 0b01'01'10'10:\n case 0b01'01'11'11:\n case 0b10'10'00'00:\n case 0b10'10'01'01:\n case 0b10'10'10'10:\n case 0b10'10'11'11:\n case 0b11'11'00'00:\n case 0b11'11'01'01:\n case 0b11'11'10'10:\n case 0b11'11'11'11: {\n // Duplicate element in upper and lower across each 64-bit segment.\n auto DupSrc1 = _VDupElement(DstSize, ElementSize, Src1, Shuffle & 0b11);\n auto DupSrc2 = _VDupElement(DstSize, ElementSize, Src2, (Shuffle >> 4) & 0b11);\n return _VZip(DstSize, OpSize::i64Bit, DupSrc1, DupSrc2);\n }\n default:\n // Use a TBL2 operation to handle this implementation.\n auto LookupIndexes =\n LoadAndCacheIndexedNamedVectorConstant(DstSize, FEXCore::IR::IndexNamedVectorConstant::INDEXED_NAMED_VECTOR_SHUFPS, Shuffle * 16);\n return _VTBL2(DstSize, Src1, Src2, LookupIndexes);\n }\n } else {\n switch (Shuffle & 0b11) {\n case 0b00:\n // Low 64-bits of each source interleaved.\n return _VZip(DstSize, ElementSize, Src1, Src2);\n case 0b01:\n // Upper 64-bits of Src1 in lower bits\n // Lower 64-bits of Src2 in upper bits.\n return _VExtr(DstSize, OpSize::i8Bit, Src2, Src1, 8);\n case 0b10:\n // Lower 32-bits of Src1 in lower bits.\n // Upper 64-bits of Src2 in upper bits.\n return _VInsElement(DstSize, ElementSize, 1, 1, Src1, Src2);\n case 0b11:\n // Upper 64-bits of each source interleaved.\n return _VZip2(DstSize, ElementSize, Src1, Src2);\n }\n }\n\n for (uint8_t Element = 0; Element < NumElements; ++Element) {\n const auto SrcIndex = Shuffle & SelectionMask;\n Dest = _VInsElement(DstSize, ElementSize, Element, SrcIndex, Dest, Srcs[Element]);\n Shuffle >>= ShiftAmount;\n }\n }\n\n return Dest;\n}\n\nvoid OpDispatchBuilder::SHUFOp(OpcodeArgs, IR::OpSize ElementSize) {\n Ref Src1Node = LoadSourceFPR(Op, Op->Dest, Op->Flags);\n Ref Src2Node = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n uint8_t Shuffle = Op->Src[1].Literal();\n\n Ref Result = SHUFOpImpl(Op, OpSizeFromDst(Op), ElementSize, Src1Node, Src2Node, Shuffle);\n StoreResultFPR(Op, Result);\n}\n\nvoid OpDispatchBuilder::VSHUFOp(OpcodeArgs, IR::OpSize ElementSize) {\n Ref Src1Node = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n Ref Src2Node = LoadSourceFPR(Op, Op->Src[1], Op->Flags);\n uint8_t Shuffle = Op->Src[2].Literal();\n\n Ref Result = SHUFOpImpl(Op, OpSizeFromDst(Op), ElementSize, Src1Node, Src2Node, Shuffle);\n StoreResultFPR(Op, Result);\n}\n\nvoid OpDispatchBuilder::VANDNOp(OpcodeArgs) {\n const auto SrcSize = OpSizeFromSrc(Op);\n\n Ref Src1 = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n Ref Src2 = LoadSourceFPR(Op, Op->Src[1], Op->Flags);\n Ref Dest = _VAndn(SrcSize, SrcSize, Src2, Src1);\n\n StoreResultFPR(Op, Dest);\n}\n\ntemplate\nvoid OpDispatchBuilder::VHADDPOp(OpcodeArgs) {\n const auto SrcSize = OpSizeFromSrc(Op);\n const auto Is256Bit = SrcSize == OpSize::i256Bit;\n\n Ref Src1 = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n Ref Src2 = LoadSourceFPR(Op, Op->Src[1], Op->Flags);\n\n DeriveOp(Res, IROp, _VFAddP(SrcSize, ElementSize, Src1, Src2));\n\n Ref Dest = Res;\n if (Is256Bit) {\n Dest = _VInsElement(SrcSize, OpSize::i64Bit, 1, 2, Res, Res);\n Dest = _VInsElement(SrcSize, OpSize::i64Bit, 2, 1, Dest, Res);\n }\n\n StoreResultFPR(Op, Dest);\n}\n\ntemplate void OpDispatchBuilder::VHADDPOp(OpcodeArgs);\ntemplate void OpDispatchBuilder::VHADDPOp(OpcodeArgs);\ntemplate void OpDispatchBuilder::VHADDPOp(OpcodeArgs);\ntemplate void OpDispatchBuilder::VHADDPOp(OpcodeArgs);\n\nvoid OpDispatchBuilder::VBROADCASTOp(OpcodeArgs, IR::OpSize ElementSize) {\n const auto DstSize = OpSizeFromDst(Op);\n Ref Result {};\n\n if (Op->Src[0].IsGPR()) {\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n Result = _VDupElement(DstSize, ElementSize, Src, 0);\n } else {\n // Get the address to broadcast from into a GPR.\n Ref Address = MakeSegmentAddress(Op, Op->Src[0], GetGPROpSize());\n Result = _VBroadcastFromMem(DstSize, ElementSize, Address);\n }\n\n // No need to zero-extend result, since implementations\n // use zero extending AdvSIMD or zeroing SVE loads internally.\n\n StoreResultFPR(Op, Result);\n}\n\nRef OpDispatchBuilder::PINSROpImpl(OpcodeArgs, IR::OpSize ElementSize, const X86Tables::DecodedOperand& Src1Op,\n const X86Tables::DecodedOperand& Src2Op, const X86Tables::DecodedOperand& Imm) {\n const auto Size = OpSizeFromDst(Op);\n const auto NumElements = IR::NumElements(Size, ElementSize);\n const uint64_t Index = Imm.Literal() & (NumElements - 1);\n Ref Src1 = LoadSourceFPR_WithOpSize(Op, Src1Op, Size, Op->Flags);\n\n if (Src2Op.IsGPR()) {\n // If the source is a GPR then convert directly from the GPR.\n auto Src2 = LoadSourceGPR_WithOpSize(Op, Src2Op, GetGPROpSize(), Op->Flags);\n return _VInsGPR(Size, ElementSize, Index, Src1, Src2);\n }\n\n // If loading from memory then we only load the element size\n Ref Src2 = MakeSegmentAddress(Op, Src2Op);\n return _VLoadVectorElement(Size, ElementSize, Src1, Index, Src2);\n}\n\ntemplate\nvoid OpDispatchBuilder::PINSROp(OpcodeArgs) {\n Ref Result = PINSROpImpl(Op, ElementSize, Op->Dest, Op->Src[0], Op->Src[1]);\n StoreResultFPR(Op, Result);\n}\n\ntemplate void OpDispatchBuilder::PINSROp(OpcodeArgs);\ntemplate void OpDispatchBuilder::PINSROp(OpcodeArgs);\ntemplate void OpDispatchBuilder::PINSROp(OpcodeArgs);\ntemplate void OpDispatchBuilder::PINSROp(OpcodeArgs);\n\nvoid OpDispatchBuilder::VPINSRBOp(OpcodeArgs) {\n Ref Result = PINSROpImpl(Op, OpSize::i8Bit, Op->Src[0], Op->Src[1], Op->Src[2]);\n if (Op->Dest.Data.GPR.GPR == Op->Src[0].Data.GPR.GPR) {\n Result = _VMov(OpSize::i128Bit, Result);\n }\n StoreResultFPR(Op, Result);\n}\n\nvoid OpDispatchBuilder::VPINSRDQOp(OpcodeArgs) {\n const auto SrcSize = OpSizeFromSrc(Op);\n Ref Result = PINSROpImpl(Op, SrcSize, Op->Src[0], Op->Src[1], Op->Src[2]);\n if (Op->Dest.Data.GPR.GPR == Op->Src[0].Data.GPR.GPR) {\n Result = _VMov(OpSize::i128Bit, Result);\n }\n StoreResultFPR(Op, Result);\n}\n\nvoid OpDispatchBuilder::VPINSRWOp(OpcodeArgs) {\n Ref Result = PINSROpImpl(Op, OpSize::i16Bit, Op->Src[0], Op->Src[1], Op->Src[2]);\n if (Op->Dest.Data.GPR.GPR == Op->Src[0].Data.GPR.GPR) {\n Result = _VMov(OpSize::i128Bit, Result);\n }\n StoreResultFPR(Op, Result);\n}\n\nRef OpDispatchBuilder::InsertPSOpImpl(OpcodeArgs, const X86Tables::DecodedOperand& Src1, const X86Tables::DecodedOperand& Src2,\n const X86Tables::DecodedOperand& Imm) {\n const uint8_t ImmValue = Imm.Literal();\n uint8_t CountS = (ImmValue >> 6);\n uint8_t CountD = (ImmValue >> 4) & 0b11;\n const uint8_t ZMask = ImmValue & 0xF;\n\n const auto DstSize = OpSizeFromDst(Op);\n\n Ref Dest {};\n if (ZMask != 0xF) {\n // Only need to load destination if it isn't a full zero\n Dest = LoadSourceFPR_WithOpSize(Op, Src1, DstSize, Op->Flags);\n }\n\n if ((ZMask & (1 << CountD)) == 0) {\n // In the case that ZMask overwrites the destination element, then don't even insert\n Ref Src {};\n if (Src2.IsGPR()) {\n Src = LoadSourceFPR(Op, Src2, Op->Flags);\n } else {\n // If loading from memory then CountS is forced to zero\n CountS = 0;\n Src = LoadSourceFPR_WithOpSize(Op, Src2, OpSize::i32Bit, Op->Flags);\n }\n\n Dest = _VInsElement(DstSize, OpSize::i32Bit, CountD, CountS, Dest, Src);\n }\n\n // ZMask happens after insert\n if (ZMask == 0xF) {\n return LoadZeroVector(DstSize);\n }\n\n if (ZMask) {\n auto Zero = LoadZeroVector(DstSize);\n for (size_t i = 0; i < 4; ++i) {\n if ((ZMask & (1 << i)) != 0) {\n Dest = _VInsElement(DstSize, OpSize::i32Bit, i, 0, Dest, Zero);\n }\n }\n }\n\n return Dest;\n}\n\nvoid OpDispatchBuilder::InsertPSOp(OpcodeArgs) {\n Ref Result = InsertPSOpImpl(Op, Op->Dest, Op->Src[0], Op->Src[1]);\n StoreResultFPR(Op, Result);\n}\n\nvoid OpDispatchBuilder::VINSERTPSOp(OpcodeArgs) {\n Ref Result = InsertPSOpImpl(Op, Op->Src[0], Op->Src[1], Op->Src[2]);\n StoreResultFPR(Op, Result);\n}\n\nvoid OpDispatchBuilder::PExtrOp(OpcodeArgs, IR::OpSize ElementSize) {\n const auto DstSize = OpSizeFromDst(Op);\n\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n uint64_t Index = Op->Src[1].Literal();\n\n // Fixup of 32-bit element size.\n // When the element size is 32-bit then it can be overriden as 64-bit because the encoding of PEXTRD/PEXTRQ\n // is the same except that REX.W or VEX.W is set to 1. Incredibly frustrating.\n // Use the destination size as the element size in this case.\n auto OverridenElementSize = ElementSize;\n if (ElementSize == OpSize::i32Bit) {\n OverridenElementSize = DstSize;\n }\n\n // AVX version only operates on 128-bit.\n const uint8_t NumElements = IR::NumElements(std::min(OpSizeFromSrc(Op), OpSize::i128Bit), OverridenElementSize);\n Index &= NumElements - 1;\n\n if (Op->Dest.IsGPR()) {\n const auto GPRSize = GetGPROpSize();\n // Extract already zero extends the result.\n Ref Result = _VExtractToGPR(OpSize::i128Bit, OverridenElementSize, Src, Index);\n StoreResultGPR_WithOpSize(Op, Op->Dest, Result, GPRSize);\n return;\n }\n\n // If we are storing to memory then we store the size of the element extracted\n Ref Dest = MakeSegmentAddress(Op, Op->Dest);\n _VStoreVectorElement(OpSize::i128Bit, OverridenElementSize, Src, Index, Dest);\n}\n\nvoid OpDispatchBuilder::VEXTRACT128Op(OpcodeArgs) {\n const auto DstIsXMM = Op->Dest.IsGPR();\n const auto StoreSize = DstIsXMM ? OpSize::i256Bit : OpSize::i128Bit;\n const auto Selector = Op->Src[1].Literal() & 0b1;\n\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n\n // A selector of zero is the same as doing a 128-bit vector move.\n if (Selector == 0) {\n Ref Result = DstIsXMM ? _VMov(OpSize::i128Bit, Src) : Src;\n StoreResultFPR_WithOpSize(Op, Op->Dest, Result, StoreSize);\n return;\n }\n\n // Otherwise replicate the element and only store the first 128-bits.\n Ref Result = _VDupElement(OpSize::i256Bit, OpSize::i128Bit, Src, Selector);\n if (DstIsXMM) {\n Result = _VMov(OpSize::i128Bit, Result);\n }\n StoreResultFPR_WithOpSize(Op, Op->Dest, Result, StoreSize);\n}\n\nRef OpDispatchBuilder::PSIGNImpl(OpcodeArgs, IR::OpSize ElementSize, Ref Src1, Ref Src2) {\n const auto Size = OpSizeFromSrc(Op);\n\n Ref Control = _VSQSHL(Size, ElementSize, Src2, IR::OpSizeAsBits(ElementSize) - 1);\n Control = _VSRSHR(Size, ElementSize, Control, IR::OpSizeAsBits(ElementSize) - 1);\n return _VMul(Size, ElementSize, Src1, Control);\n}\n\ntemplate\nvoid OpDispatchBuilder::PSIGN(OpcodeArgs) {\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n Ref Dest = LoadSourceFPR(Op, Op->Dest, Op->Flags);\n Ref Res = PSIGNImpl(Op, ElementSize, Dest, Src);\n\n StoreResultFPR(Op, Res);\n}\n\ntemplate void OpDispatchBuilder::PSIGN(OpcodeArgs);\ntemplate void OpDispatchBuilder::PSIGN(OpcodeArgs);\ntemplate void OpDispatchBuilder::PSIGN(OpcodeArgs);\n\ntemplate\nvoid OpDispatchBuilder::VPSIGN(OpcodeArgs) {\n Ref Src1 = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n Ref Src2 = LoadSourceFPR(Op, Op->Src[1], Op->Flags);\n Ref Res = PSIGNImpl(Op, ElementSize, Src1, Src2);\n\n StoreResultFPR(Op, Res);\n}\n\ntemplate void OpDispatchBuilder::VPSIGN(OpcodeArgs);\ntemplate void OpDispatchBuilder::VPSIGN(OpcodeArgs);\ntemplate void OpDispatchBuilder::VPSIGN(OpcodeArgs);\n\nRef OpDispatchBuilder::PSRLDOpImpl(OpcodeArgs, IR::OpSize ElementSize, Ref Src, Ref ShiftVec) {\n const auto Size = OpSizeFromSrc(Op);\n\n // Incoming element size for the shift source is always 8\n return _VUShrSWide(Size, ElementSize, Src, ShiftVec);\n}\n\nvoid OpDispatchBuilder::PSRLDOp(OpcodeArgs, IR::OpSize ElementSize) {\n Ref Dest = LoadSourceFPR(Op, Op->Dest, Op->Flags);\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n Ref Result = PSRLDOpImpl(Op, ElementSize, Dest, Src);\n\n StoreResultFPR(Op, Result);\n}\n\nvoid OpDispatchBuilder::VPSRLDOp(OpcodeArgs, IR::OpSize ElementSize) {\n const auto DstSize = GetDstSize(Op);\n const auto Is128Bit = DstSize == Core::CPUState::XMM_SSE_REG_SIZE;\n\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n Ref Shift = LoadSourceFPR(Op, Op->Src[1], Op->Flags);\n Ref Result = PSRLDOpImpl(Op, ElementSize, Src, Shift);\n\n if (Is128Bit) {\n Result = _VMov(OpSize::i128Bit, Result);\n }\n StoreResultFPR(Op, Result);\n}\n\nvoid OpDispatchBuilder::PSRLI(OpcodeArgs, IR::OpSize ElementSize) {\n const uint64_t ShiftConstant = Op->Src[1].Literal();\n if (ShiftConstant == 0) [[unlikely]] {\n // Nothing to do, value is already in Dest.\n return;\n }\n\n const auto Size = OpSizeFromSrc(Op);\n\n Ref Dest = LoadSourceFPR(Op, Op->Dest, Op->Flags);\n Ref Shift = _VUShrI(Size, ElementSize, Dest, ShiftConstant);\n StoreResultFPR(Op, Shift);\n}\n\nvoid OpDispatchBuilder::VPSRLIOp(OpcodeArgs, IR::OpSize ElementSize) {\n const auto Size = OpSizeFromSrc(Op);\n const auto Is128Bit = Size == OpSize::i128Bit;\n const uint64_t ShiftConstant = Op->Src[1].Literal();\n\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n Ref Result = Src;\n\n if (ShiftConstant != 0) [[likely]] {\n Result = _VUShrI(Size, ElementSize, Src, ShiftConstant);\n } else {\n if (Is128Bit) {\n Result = _VMov(OpSize::i128Bit, Result);\n }\n }\n\n StoreResultFPR(Op, Result);\n}\n\nRef OpDispatchBuilder::PSLLIImpl(OpcodeArgs, IR::OpSize ElementSize, Ref Src, uint64_t Shift) {\n if (Shift == 0) [[unlikely]] {\n // If zero-shift then just return the source.\n return Src;\n }\n const auto Size = OpSizeFromSrc(Op);\n return _VShlI(Size, ElementSize, Src, Shift);\n}\n\nvoid OpDispatchBuilder::PSLLI(OpcodeArgs, IR::OpSize ElementSize) {\n const uint64_t ShiftConstant = Op->Src[1].Literal();\n if (ShiftConstant == 0) [[unlikely]] {\n // Nothing to do, value is already in Dest.\n return;\n }\n\n Ref Dest = LoadSourceFPR(Op, Op->Dest, Op->Flags);\n Ref Result = PSLLIImpl(Op, ElementSize, Dest, ShiftConstant);\n\n StoreResultFPR(Op, Result);\n}\n\nvoid OpDispatchBuilder::VPSLLIOp(OpcodeArgs, IR::OpSize ElementSize) {\n const uint64_t ShiftConstant = Op->Src[1].Literal();\n const auto DstSize = GetDstSize(Op);\n const auto Is128Bit = DstSize == Core::CPUState::XMM_SSE_REG_SIZE;\n\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n Ref Result = PSLLIImpl(Op, ElementSize, Src, ShiftConstant);\n if (ShiftConstant == 0 && Is128Bit) {\n Result = _VMov(OpSize::i128Bit, Result);\n }\n\n StoreResultFPR(Op, Result);\n}\n\nRef OpDispatchBuilder::PSLLImpl(OpcodeArgs, IR::OpSize ElementSize, Ref Src, Ref ShiftVec) {\n const auto Size = OpSizeFromDst(Op);\n\n // Incoming element size for the shift source is always 8\n return _VUShlSWide(Size, ElementSize, Src, ShiftVec);\n}\n\nvoid OpDispatchBuilder::PSLL(OpcodeArgs, IR::OpSize ElementSize) {\n Ref Dest = LoadSourceFPR(Op, Op->Dest, Op->Flags);\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n Ref Result = PSLLImpl(Op, ElementSize, Dest, Src);\n\n StoreResultFPR(Op, Result);\n}\n\nvoid OpDispatchBuilder::VPSLLOp(OpcodeArgs, IR::OpSize ElementSize) {\n const auto DstSize = GetDstSize(Op);\n const auto Is128Bit = DstSize == Core::CPUState::XMM_SSE_REG_SIZE;\n\n Ref Src1 = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n Ref Src2 = LoadSourceFPR_WithOpSize(Op, Op->Src[1], OpSize::i128Bit, Op->Flags);\n Ref Result = PSLLImpl(Op, ElementSize, Src1, Src2);\n\n if (Is128Bit) {\n Result = _VMov(OpSize::i128Bit, Result);\n }\n StoreResultFPR(Op, Result);\n}\n\nRef OpDispatchBuilder::PSRAOpImpl(OpcodeArgs, IR::OpSize ElementSize, Ref Src, Ref ShiftVec) {\n const auto Size = OpSizeFromDst(Op);\n\n // Incoming element size for the shift source is always 8\n return _VSShrSWide(Size, ElementSize, Src, ShiftVec);\n}\n\nvoid OpDispatchBuilder::PSRAOp(OpcodeArgs, IR::OpSize ElementSize) {\n Ref Dest = LoadSourceFPR(Op, Op->Dest, Op->Flags);\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n Ref Result = PSRAOpImpl(Op, ElementSize, Dest, Src);\n\n StoreResultFPR(Op, Result);\n}\n\nvoid OpDispatchBuilder::VPSRAOp(OpcodeArgs, IR::OpSize ElementSize) {\n const auto DstSize = GetDstSize(Op);\n const auto Is128Bit = DstSize == Core::CPUState::XMM_SSE_REG_SIZE;\n\n Ref Src1 = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n Ref Src2 = LoadSourceFPR(Op, Op->Src[1], Op->Flags);\n Ref Result = PSRAOpImpl(Op, ElementSize, Src1, Src2);\n\n if (Is128Bit) {\n Result = _VMov(OpSize::i128Bit, Result);\n }\n StoreResultFPR(Op, Result);\n}\n\nvoid OpDispatchBuilder::PSRLDQ(OpcodeArgs) {\n const uint64_t Shift = Op->Src[1].Literal();\n if (Shift == 0) [[unlikely]] {\n // Nothing to do, value is already in Dest.\n return;\n }\n\n const auto Size = OpSizeFromDst(Op);\n\n Ref Dest = LoadSourceFPR(Op, Op->Dest, Op->Flags);\n Ref Result = LoadZeroVector(Size);\n\n if (Shift < IR::OpSizeToSize(Size)) {\n Result = _VExtr(Size, OpSize::i8Bit, Result, Dest, Shift);\n }\n StoreResultFPR(Op, Result);\n}\n\nvoid OpDispatchBuilder::VPSRLDQOp(OpcodeArgs) {\n const auto DstSize = OpSizeFromDst(Op);\n const auto Is128Bit = DstSize == OpSize::i128Bit;\n const uint64_t Shift = Op->Src[1].Literal();\n\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n\n Ref Result {};\n if (Shift == 0) [[unlikely]] {\n if (Is128Bit) {\n Result = _VMov(OpSize::i128Bit, Src);\n } else {\n Result = Src;\n }\n } else {\n Result = LoadZeroVector(DstSize);\n\n if (Is128Bit) {\n if (Shift < IR::OpSizeToSize(DstSize)) {\n Result = _VExtr(DstSize, OpSize::i8Bit, Result, Src, Shift);\n }\n } else {\n if (Shift < Core::CPUState::XMM_SSE_REG_SIZE) {\n Ref ResultBottom = _VExtr(OpSize::i128Bit, OpSize::i8Bit, Result, Src, Shift);\n Ref ResultTop = _VExtr(DstSize, OpSize::i8Bit, Result, Src, 16 + Shift);\n\n Result = _VInsElement(DstSize, OpSize::i128Bit, 1, 0, ResultBottom, ResultTop);\n }\n }\n }\n\n StoreResultFPR(Op, Result);\n}\n\nvoid OpDispatchBuilder::PSLLDQ(OpcodeArgs) {\n const uint64_t Shift = Op->Src[1].Literal();\n if (Shift == 0) [[unlikely]] {\n // Nothing to do, value is already in Dest.\n return;\n }\n\n const auto Size = OpSizeFromDst(Op);\n\n Ref Dest = LoadSourceFPR(Op, Op->Dest, Op->Flags);\n Ref Result = LoadZeroVector(Size);\n if (Shift < IR::OpSizeToSize(Size)) {\n Result = _VExtr(Size, OpSize::i8Bit, Dest, Result, IR::OpSizeToSize(Size) - Shift);\n }\n\n StoreResultFPR(Op, Result);\n}\n\nvoid OpDispatchBuilder::VPSLLDQOp(OpcodeArgs) {\n const auto DstSize = OpSizeFromDst(Op);\n const auto DstSizeInt = IR::OpSizeToSize(DstSize);\n const auto Is128Bit = DstSize == OpSize::i128Bit;\n const uint64_t Shift = Op->Src[1].Literal();\n\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n\n Ref Result = Src;\n\n if (Shift == 0) {\n if (Is128Bit) {\n Result = _VMov(OpSize::i128Bit, Result);\n }\n } else {\n Result = LoadZeroVector(DstSize);\n if (Is128Bit) {\n if (Shift < DstSizeInt) {\n Result = _VExtr(DstSize, OpSize::i8Bit, Src, Result, DstSizeInt - Shift);\n }\n } else {\n if (Shift < Core::CPUState::XMM_SSE_REG_SIZE) {\n Ref ResultBottom = _VExtr(OpSize::i128Bit, OpSize::i8Bit, Src, Result, 16 - Shift);\n Ref ResultTop = _VExtr(DstSize, OpSize::i8Bit, Src, Result, DstSizeInt - Shift);\n\n Result = _VInsElement(DstSize, OpSize::i128Bit, 1, 0, ResultBottom, ResultTop);\n }\n }\n }\n StoreResultFPR(Op, Result);\n}\n\nvoid OpDispatchBuilder::PSRAIOp(OpcodeArgs, IR::OpSize ElementSize) {\n const uint64_t Shift = Op->Src[1].Literal();\n if (Shift == 0) [[unlikely]] {\n // Nothing to do, value is already in Dest.\n return;\n }\n\n const auto Size = OpSizeFromDst(Op);\n\n Ref Dest = LoadSourceFPR(Op, Op->Dest, Op->Flags);\n Ref Result = _VSShrI(Size, ElementSize, Dest, Shift);\n StoreResultFPR(Op, Result);\n}\n\nvoid OpDispatchBuilder::VPSRAIOp(OpcodeArgs, IR::OpSize ElementSize) {\n const uint64_t Shift = Op->Src[1].Literal();\n const auto Size = OpSizeFromDst(Op);\n const auto Is128Bit = Size == OpSize::i128Bit;\n\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n Ref Result = Src;\n\n if (Shift != 0) [[likely]] {\n Result = _VSShrI(Size, ElementSize, Src, Shift);\n } else {\n if (Is128Bit) {\n Result = _VMov(OpSize::i128Bit, Result);\n }\n }\n\n StoreResultFPR(Op, Result);\n}\n\nvoid OpDispatchBuilder::AVXVariableShiftImpl(OpcodeArgs, IROps IROp) {\n const auto DstSize = OpSizeFromDst(Op);\n const auto SrcSize = OpSizeFromSrc(Op);\n\n Ref Vector = LoadSourceFPR_WithOpSize(Op, Op->Src[0], DstSize, Op->Flags);\n Ref ShiftVector = LoadSourceFPR_WithOpSize(Op, Op->Src[1], DstSize, Op->Flags);\n\n DeriveOp(Shift, IROp, _VUShr(DstSize, SrcSize, Vector, ShiftVector, true));\n\n StoreResultFPR(Op, Shift);\n}\n\nvoid OpDispatchBuilder::VPSLLVOp(OpcodeArgs) {\n AVXVariableShiftImpl(Op, IROps::OP_VUSHL);\n}\n\nvoid OpDispatchBuilder::VPSRAVDOp(OpcodeArgs) {\n AVXVariableShiftImpl(Op, IROps::OP_VSSHR);\n}\n\nvoid OpDispatchBuilder::VPSRLVOp(OpcodeArgs) {\n AVXVariableShiftImpl(Op, IROps::OP_VUSHR);\n}\n\nvoid OpDispatchBuilder::MOVDDUPOp(OpcodeArgs) {\n // If loading a vector, use the full size, so we don't\n // unnecessarily zero extend the vector. Otherwise, if\n // memory, then we want to load the element size exactly.\n const auto SrcSize = Op->Src[0].IsGPR() ? OpSize::i128Bit : OpSizeFromSrc(Op);\n Ref Src = LoadSourceFPR_WithOpSize(Op, Op->Src[0], SrcSize, Op->Flags);\n Ref Res = _VDupElement(OpSize::i128Bit, OpSizeFromSrc(Op), Src, 0);\n\n StoreResultFPR(Op, Res);\n}\n\nvoid OpDispatchBuilder::VMOVDDUPOp(OpcodeArgs) {\n const auto SrcSize = OpSizeFromSrc(Op);\n const auto IsSrcGPR = Op->Src[0].IsGPR();\n const auto Is256Bit = SrcSize == OpSize::i256Bit;\n const auto MemSize = Is256Bit ? OpSize::i256Bit : OpSize::i64Bit;\n\n const auto LoadSize = IsSrcGPR ? SrcSize : MemSize;\n Ref Src = LoadSourceFPR_WithOpSize(Op, Op->Src[0], LoadSize, Op->Flags);\n\n Ref Res {};\n if (Is256Bit) {\n Res = _VTrn(SrcSize, OpSize::i64Bit, Src, Src);\n } else {\n Res = _VDupElement(SrcSize, OpSize::i64Bit, Src, 0);\n }\n\n StoreResultFPR(Op, Res);\n}\n\nRef OpDispatchBuilder::CVTGPR_To_FPRImpl(OpcodeArgs, IR::OpSize DstElementSize, const X86Tables::DecodedOperand& Src1Op,\n const X86Tables::DecodedOperand& Src2Op) {\n const auto SrcSize = OpSizeFromSrc(Op);\n\n Ref Src1 = LoadSourceFPR_WithOpSize(Op, Src1Op, OpSize::i128Bit, Op->Flags);\n Ref Converted {};\n if (Src2Op.IsGPR()) {\n // If the source is a GPR then convert directly from the GPR.\n auto Src2 = LoadSourceGPR_WithOpSize(Op, Src2Op, GetGPROpSize(), Op->Flags);\n Converted = _Float_FromGPR_S(DstElementSize, SrcSize, Src2);\n } else if (SrcSize != DstElementSize) {\n // If the source is from memory but the Source size and destination size aren't the same,\n // then it is more optimal to load in to a GPR and convert between GPR->FPR.\n // ARM GPR->FPR conversion supports different size source and destinations while FPR->FPR doesn't.\n auto Src2 = LoadSourceGPR(Op, Src2Op, Op->Flags);\n Converted = _Float_FromGPR_S(DstElementSize, SrcSize, Src2);\n } else {\n // In the case of cvtsi2s{s,d} where the source and destination are the same size,\n // then it is more optimal to load in to the FPR register directly and convert there.\n auto Src2 = LoadSourceFPR(Op, Src2Op, Op->Flags);\n Converted = _Vector_SToF(SrcSize, SrcSize, Src2);\n }\n\n return _VInsElement(OpSize::i128Bit, DstElementSize, 0, 0, Src1, Converted);\n}\n\ntemplate\nvoid OpDispatchBuilder::CVTGPR_To_FPR(OpcodeArgs) {\n Ref Result = CVTGPR_To_FPRImpl(Op, DstElementSize, Op->Dest, Op->Src[0]);\n StoreResultFPR(Op, Result);\n}\n\ntemplate void OpDispatchBuilder::CVTGPR_To_FPR(OpcodeArgs);\ntemplate void OpDispatchBuilder::CVTGPR_To_FPR(OpcodeArgs);\n\ntemplate\nvoid OpDispatchBuilder::AVXCVTGPR_To_FPR(OpcodeArgs) {\n Ref Result = CVTGPR_To_FPRImpl(Op, DstElementSize, Op->Src[0], Op->Src[1]);\n StoreResultFPR(Op, Result);\n}\ntemplate void OpDispatchBuilder::AVXCVTGPR_To_FPR(OpcodeArgs);\ntemplate void OpDispatchBuilder::AVXCVTGPR_To_FPR(OpcodeArgs);\n\nRef OpDispatchBuilder::CVTFPR_To_GPRImpl(OpcodeArgs, Ref Src, IR::OpSize SrcElementSize, bool HostRoundingMode) {\n // GPR size is determined by REX.W\n // Source Element size is determined by instruction\n const auto GPRSize = OpSizeFromDst(Op);\n\n if (CTX->HostFeatures.SupportsFRINTTS) {\n // When we have FRINTTS, this is a two-step process. First, we round to the\n // right integer (where _Vector_FToISized matches x86 semantics), then just\n // convert that to a GPR.\n Src = _Vector_FToISized(SrcElementSize, SrcElementSize, Src, HostRoundingMode, GPRSize);\n return _Float_ToGPR_ZS(GPRSize, SrcElementSize, Src);\n } else {\n // When we lack hardware support, we need a bit of a convoluted sequence of\n // fixups before before and after conversion to emulate x86 semantics.\n if (HostRoundingMode) {\n Src = _Vector_FToI(SrcElementSize, SrcElementSize, Src, RoundMode::Host);\n }\n\n Ref Converted = _Float_ToGPR_ZS(GPRSize, SrcElementSize, Src);\n\n bool Dst32 = GPRSize == OpSize::i32Bit;\n Ref MaxI = Dst32 ? Constant(0x80000000) : Constant(0x8000000000000000);\n Ref MaxF = LoadAndCacheNamedVectorConstant(SrcElementSize, (SrcElementSize == OpSize::i32Bit) ?\n (Dst32 ? NAMED_VECTOR_CVTMAX_F32_I32 : NAMED_VECTOR_CVTMAX_F32_I64) :\n (Dst32 ? NAMED_VECTOR_CVTMAX_F64_I32 : NAMED_VECTOR_CVTMAX_F64_I64));\n return _Select(GPRSize, SrcElementSize, CondClass::FGT, MaxF, Src, Converted, MaxI);\n }\n}\n\ntemplate\nvoid OpDispatchBuilder::CVTFPR_To_GPR(OpcodeArgs) {\n // If loading a vector, use the full size, so we don't\n // unnecessarily zero extend the vector. Otherwise, if\n // memory, then we want to load the element size exactly.\n const auto SrcSize = Op->Src[0].IsGPR() ? OpSize::i128Bit : SrcElementSize;\n Ref Src = LoadSourceFPR_WithOpSize(Op, Op->Src[0], SrcSize, Op->Flags);\n Ref Result = CVTFPR_To_GPRImpl(Op, Src, SrcElementSize, HostRoundingMode);\n StoreResultGPR(Op, Result);\n}\n\ntemplate void OpDispatchBuilder::CVTFPR_To_GPR(OpcodeArgs);\ntemplate void OpDispatchBuilder::CVTFPR_To_GPR(OpcodeArgs);\n\ntemplate void OpDispatchBuilder::CVTFPR_To_GPR(OpcodeArgs);\ntemplate void OpDispatchBuilder::CVTFPR_To_GPR(OpcodeArgs);\n\nRef OpDispatchBuilder::Vector_CVT_Int_To_FloatImpl(OpcodeArgs, IR::OpSize SrcElementSize, bool Widen) {\n const auto Size = OpSizeFromDst(Op);\n\n Ref Src = [&] {\n if (Widen) {\n // If loading a vector, use the full size, so we don't\n // unnecessarily zero extend the vector. Otherwise, if\n // memory, then we want to load the element size exactly.\n const auto LoadSize = Op->Src[0].IsGPR() ? OpSize::i128Bit : IR::SizeToOpSize(8 * (IR::OpSizeToSize(Size) / 16));\n return LoadSourceFPR_WithOpSize(Op, Op->Src[0], LoadSize, Op->Flags);\n } else {\n return LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n }\n }();\n\n auto ElementSize = SrcElementSize;\n if (Widen) {\n Src = _VSXTL(Size, ElementSize, Src);\n ElementSize = ElementSize << 1;\n }\n\n return _Vector_SToF(Size, ElementSize, Src);\n}\n\ntemplate\nvoid OpDispatchBuilder::Vector_CVT_Int_To_Float(OpcodeArgs) {\n Ref Result = Vector_CVT_Int_To_FloatImpl(Op, SrcElementSize, Widen);\n StoreResultFPR(Op, Result);\n}\n\ntemplate void OpDispatchBuilder::Vector_CVT_Int_To_Float(OpcodeArgs);\ntemplate void OpDispatchBuilder::Vector_CVT_Int_To_Float(OpcodeArgs);\n\nRef OpDispatchBuilder::Vector_CVT_Float_To_Int32Impl(OpcodeArgs, IR::OpSize DstSize, Ref Src, IR::OpSize SrcSize, IR::OpSize SrcElementSize,\n bool HostRoundingMode, bool ZeroUpperHalf) {\n if (CTX->HostFeatures.SupportsFRINTTS && SrcSize != OpSize::i256Bit) {\n // If we have FRINTS, this is the usual 2-step\n Src = _Vector_FToISized(SrcSize, SrcElementSize, Src, HostRoundingMode, OpSize::i32Bit);\n Ref Dst = _Vector_FToZS(SrcSize, SrcElementSize, Src);\n if (SrcElementSize == OpSize::i32Bit) {\n // Return 32-bit result as-is\n return Dst;\n } else {\n // Down step from 64-bit ints to 32-bit ints\n return _VUShrNI(DstSize, SrcElementSize, Dst, 0);\n }\n } else {\n // Otherwise, we have to do all the fixups, but vectorized.\n if (HostRoundingMode) {\n Src = _Vector_FToI(SrcSize, SrcElementSize, Src, RoundMode::Host);\n }\n\n OpSize OverflowConstSize = ZeroUpperHalf && SrcElementSize == OpSize::i64Bit ? DstSize / 2 : DstSize;\n Ref MaxI = LoadAndCacheNamedVectorConstant(OverflowConstSize, NAMED_VECTOR_CVTMAX_I32);\n Ref Converted {}, Cmp {};\n if (SrcElementSize == OpSize::i64Bit) {\n Ref MaxF = LoadAndCacheNamedVectorConstant(SrcSize, NAMED_VECTOR_CVTMAX_F64_I32);\n Converted = _Vector_F64ToI32(DstSize, Src, RoundMode::TowardsZero, ZeroUpperHalf);\n\n Cmp = _VFCMPGT(SrcSize, OpSize::i64Bit, MaxF, Src);\n Cmp = _VUShrNI(DstSize, OpSize::i64Bit, Cmp, 32);\n } else {\n Ref MaxF = LoadAndCacheNamedVectorConstant(DstSize, NAMED_VECTOR_CVTMAX_F32_I32);\n Converted = _Vector_FToZS(DstSize, OpSize::i32Bit, Src);\n Cmp = _VFCMPGT(DstSize, OpSize::i32Bit, MaxF, Src);\n }\n return _VBSL(DstSize, Cmp, Converted, MaxI);\n }\n}\n\ntemplate\nvoid OpDispatchBuilder::Vector_CVT_Float_To_Int(OpcodeArgs) {\n const auto DstSize = OpSizeFromDst(Op);\n\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n Ref Result = Vector_CVT_Float_To_Int32Impl(Op, DstSize, Src, OpSizeFromSrc(Op), SrcElementSize, HostRoundingMode, true);\n StoreResultFPR_WithOpSize(Op, Op->Dest, Result, DstSize);\n}\n\ntemplate void OpDispatchBuilder::Vector_CVT_Float_To_Int(OpcodeArgs);\ntemplate void OpDispatchBuilder::Vector_CVT_Float_To_Int(OpcodeArgs);\n\ntemplate void OpDispatchBuilder::Vector_CVT_Float_To_Int(OpcodeArgs);\ntemplate void OpDispatchBuilder::Vector_CVT_Float_To_Int(OpcodeArgs);\n\nRef OpDispatchBuilder::Scalar_CVT_Float_To_FloatImpl(OpcodeArgs, IR::OpSize DstElementSize, IR::OpSize SrcElementSize,\n const X86Tables::DecodedOperand& Src1Op, const X86Tables::DecodedOperand& Src2Op) {\n // In the case of vectors, we can just specify the full vector length,\n // so that we don't unnecessarily zero-extend the entire vector.\n // Otherwise, if it's a memory load, then we only want to load its exact size.\n const auto Src2Size = Src2Op.IsGPR() ? OpSize::i128Bit : SrcElementSize;\n\n Ref Src1 = LoadSourceFPR_WithOpSize(Op, Src1Op, OpSize::i128Bit, Op->Flags);\n Ref Src2 = LoadSourceFPR_WithOpSize(Op, Src2Op, Src2Size, Op->Flags);\n\n Ref Converted = _Float_FToF(DstElementSize, SrcElementSize, Src2);\n\n return _VInsElement(OpSize::i128Bit, DstElementSize, 0, 0, Src1, Converted);\n}\n\ntemplate\nvoid OpDispatchBuilder::Scalar_CVT_Float_To_Float(OpcodeArgs) {\n Ref Result = Scalar_CVT_Float_To_FloatImpl(Op, DstElementSize, SrcElementSize, Op->Dest, Op->Src[0]);\n StoreResultFPR(Op, Result);\n}\n\ntemplate void OpDispatchBuilder::Scalar_CVT_Float_To_Float(OpcodeArgs);\ntemplate void OpDispatchBuilder::Scalar_CVT_Float_To_Float(OpcodeArgs);\n\ntemplate\nvoid OpDispatchBuilder::AVXScalar_CVT_Float_To_Float(OpcodeArgs) {\n Ref Result = Scalar_CVT_Float_To_FloatImpl(Op, DstElementSize, SrcElementSize, Op->Src[0], Op->Src[1]);\n StoreResultFPR(Op, Result);\n}\n\ntemplate void OpDispatchBuilder::AVXScalar_CVT_Float_To_Float(OpcodeArgs);\ntemplate void OpDispatchBuilder::AVXScalar_CVT_Float_To_Float(OpcodeArgs);\n\nvoid OpDispatchBuilder::Vector_CVT_Float_To_Float(OpcodeArgs, IR::OpSize DstElementSize, IR::OpSize SrcElementSize, bool IsAVX) {\n const auto SrcSize = OpSizeFromSrc(Op);\n\n const auto IsFloatSrc = SrcElementSize == OpSize::i32Bit;\n const auto Is128Bit = SrcSize == OpSize::i128Bit;\n\n const auto LoadSize = IsFloatSrc && !Op->Src[0].IsGPR() ? (SrcSize >> 1) : SrcSize;\n\n Ref Src = LoadSourceFPR_WithOpSize(Op, Op->Src[0], LoadSize, Op->Flags);\n\n Ref Result {};\n if (DstElementSize > SrcElementSize) {\n Result = _Vector_FToF(SrcSize, SrcElementSize << 1, Src, SrcElementSize);\n } else {\n Result = _Vector_FToF(SrcSize, SrcElementSize >> 1, Src, SrcElementSize);\n }\n\n if (IsAVX) {\n if (!IsFloatSrc && !Is128Bit) {\n // VCVTPD2PS path\n Result = _VMov(OpSize::i128Bit, Result);\n } else if (IsFloatSrc && Is128Bit) {\n // VCVTPS2PD path\n Result = _VMov(OpSize::i128Bit, Result);\n }\n }\n StoreResultFPR(Op, Result);\n}\n\nvoid OpDispatchBuilder::MMX_To_XMM_Vector_CVT_Int_To_Float(OpcodeArgs) {\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n\n // Always 32-bit.\n auto ElementSize = OpSize::i32Bit;\n const auto DstSize = OpSizeFromDst(Op);\n\n Src = _VSXTL(DstSize, ElementSize, Src);\n ElementSize = ElementSize << 1;\n\n // Always signed\n Src = _Vector_SToF(DstSize, ElementSize, Src);\n\n StoreResultFPR(Op, Src);\n}\n\ntemplate\nvoid OpDispatchBuilder::XMM_To_MMX_Vector_CVT_Float_To_Int(OpcodeArgs) {\n // This function causes a change in MMX state from X87 to MMX\n if (MMXState == MMXState_X87) {\n ChgStateX87_MMX();\n }\n\n // If loading a vector, use the full size, so we don't\n // unnecessarily zero extend the vector. Otherwise, if\n // memory, then we want to load the element size exactly.\n const auto SrcSize = Op->Src[0].IsGPR() ? OpSize::i128Bit : OpSizeFromSrc(Op);\n const auto DstSize = OpSizeFromDst(Op);\n Ref Src = LoadSourceFPR_WithOpSize(Op, Op->Src[0], SrcSize, Op->Flags);\n Ref Result = Vector_CVT_Float_To_Int32Impl(Op, DstSize, Src, SrcSize, SrcElementSize, HostRoundingMode, false /* TODO? */);\n StoreResultFPR_WithOpSize(Op, Op->Dest, Result, DstSize);\n}\n\ntemplate void OpDispatchBuilder::XMM_To_MMX_Vector_CVT_Float_To_Int(OpcodeArgs);\ntemplate void OpDispatchBuilder::XMM_To_MMX_Vector_CVT_Float_To_Int(OpcodeArgs);\ntemplate void OpDispatchBuilder::XMM_To_MMX_Vector_CVT_Float_To_Int(OpcodeArgs);\ntemplate void OpDispatchBuilder::XMM_To_MMX_Vector_CVT_Float_To_Int(OpcodeArgs);\n\nvoid OpDispatchBuilder::MASKMOVOp(OpcodeArgs) {\n const auto Size = OpSizeFromSrc(Op);\n\n Ref MaskSrc = LoadSourceGPR(Op, Op->Src[0], Op->Flags);\n // Mask only cares about the top bit of each byte\n MaskSrc = _VCMPLTZ(Size, OpSize::i8Bit, MaskSrc);\n\n // Vector that will overwrite byte elements.\n Ref VectorSrc = LoadSourceGPR(Op, Op->Dest, Op->Flags);\n\n // RDI source (DS prefix by default)\n auto MemDest = MakeSegmentAddress(X86State::REG_RDI, Op->Flags, X86Tables::DecodeFlags::FLAG_DS_PREFIX);\n\n Ref XMMReg = _LoadMemFPR(Size, MemDest, OpSize::i8Bit);\n\n // If the Mask element high bit is set then overwrite the element with the source, else keep the memory variant\n XMMReg = _VBSL(Size, MaskSrc, VectorSrc, XMMReg);\n _StoreMemFPR(Size, MemDest, XMMReg, OpSize::i8Bit);\n}\n\nvoid OpDispatchBuilder::VMASKMOVOpImpl(OpcodeArgs, IR::OpSize ElementSize, IR::OpSize DataSize, bool IsStore,\n const X86Tables::DecodedOperand& MaskOp, const X86Tables::DecodedOperand& DataOp) {\n\n const auto MakeAddress = [this, Op](const X86Tables::DecodedOperand& Data) {\n return MakeSegmentAddress(Op, Data, GetGPROpSize());\n };\n\n Ref Mask = LoadSourceFPR_WithOpSize(Op, MaskOp, DataSize, Op->Flags);\n\n if (IsStore) {\n Ref Data = LoadSourceFPR_WithOpSize(Op, DataOp, DataSize, Op->Flags);\n Ref Address = MakeAddress(Op->Dest);\n _VStoreVectorMasked(DataSize, ElementSize, Mask, Data, Address, Invalid(), MemOffsetType::SXTX, 1);\n } else {\n const auto Is128Bit = GetDstSize(Op) == Core::CPUState::XMM_SSE_REG_SIZE;\n\n Ref Address = MakeAddress(DataOp);\n Ref Result = _VLoadVectorMasked(DataSize, ElementSize, Mask, Address, Invalid(), MemOffsetType::SXTX, 1);\n\n if (Is128Bit) {\n Result = _VMov(OpSize::i128Bit, Result);\n }\n StoreResultFPR(Op, Result);\n }\n}\n\ntemplate\nvoid OpDispatchBuilder::VMASKMOVOp(OpcodeArgs) {\n VMASKMOVOpImpl(Op, ElementSize, OpSizeFromDst(Op), IsStore, Op->Src[0], Op->Src[1]);\n}\ntemplate void OpDispatchBuilder::VMASKMOVOp(OpcodeArgs);\ntemplate void OpDispatchBuilder::VMASKMOVOp(OpcodeArgs);\ntemplate void OpDispatchBuilder::VMASKMOVOp(OpcodeArgs);\ntemplate void OpDispatchBuilder::VMASKMOVOp(OpcodeArgs);\n\ntemplate\nvoid OpDispatchBuilder::VPMASKMOVOp(OpcodeArgs) {\n VMASKMOVOpImpl(Op, OpSizeFromSrc(Op), OpSizeFromDst(Op), IsStore, Op->Src[0], Op->Src[1]);\n}\ntemplate void OpDispatchBuilder::VPMASKMOVOp(OpcodeArgs);\ntemplate void OpDispatchBuilder::VPMASKMOVOp(OpcodeArgs);\n\nvoid OpDispatchBuilder::MOVBetweenGPR_FPR(OpcodeArgs, VectorOpType VectorType) {\n if (Op->Dest.IsGPR() && Op->Dest.Data.GPR.GPR >= FEXCore::X86State::REG_XMM_0) {\n Ref Result {};\n if (Op->Src[0].IsGPR()) {\n // Loading from GPR and moving to Vector.\n Ref Src = LoadSourceFPR_WithOpSize(Op, Op->Src[0], GetGPROpSize(), Op->Flags);\n // zext to 128bit\n Result = _VCastFromGPR(OpSize::i128Bit, OpSizeFromSrc(Op), Src);\n } else {\n // Loading from Memory as a scalar. Zero extend\n Result = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n }\n\n StoreResult_WithAVXInsert(VectorType, RegClass::FPR, Op, Result);\n } else {\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n\n if (Op->Dest.IsGPR()) {\n const auto ElementSize = OpSizeFromDst(Op);\n // Extract element from GPR. Zero extending in the process.\n Src = _VExtractToGPR(OpSizeFromSrc(Op), ElementSize, Src, 0);\n StoreResultGPR(Op, Op->Dest, Src);\n } else {\n // Storing first element to memory.\n Ref Dest = LoadSourceGPR(Op, Op->Dest, Op->Flags, {.LoadData = false});\n _StoreMemFPR(OpSizeFromDst(Op), Dest, Src, OpSize::i8Bit);\n }\n }\n}\n\nRef OpDispatchBuilder::VFCMPOpImpl(OpSize Size, IR::OpSize ElementSize, Ref Src1, Ref Src2, uint8_t CompType) {\n switch (static_cast(CompType)) {\n case VectorCompareType::EQ_OQ:\n case VectorCompareType::EQ_OS: return _VFCMPEQ(Size, ElementSize, Src1, Src2);\n case VectorCompareType::LT_OS: // GT(Swapped operand)\n case VectorCompareType::LT_OQ: return _VFCMPLT(Size, ElementSize, Src1, Src2);\n case VectorCompareType::LE_OS: // GE(Swapped operand)\n case VectorCompareType::LE_OQ: return _VFCMPLE(Size, ElementSize, Src1, Src2);\n case VectorCompareType::UNORD_Q:\n case VectorCompareType::UNORD_S: return _VFCMPUNO(Size, ElementSize, Src1, Src2);\n case VectorCompareType::NEQ_UQ:\n case VectorCompareType::NEQ_US: return _VFCMPNEQ(Size, ElementSize, Src1, Src2);\n case VectorCompareType::NLT_US: // NGT(Swapped operand)\n case VectorCompareType::NLT_UQ: {\n Ref Result = _VFCMPLT(Size, ElementSize, Src1, Src2);\n return _VNot(Size, ElementSize, Result);\n }\n case VectorCompareType::NLE_US: // NGE(Swapped operand)\n case VectorCompareType::NLE_UQ: {\n Ref Result = _VFCMPLE(Size, ElementSize, Src1, Src2);\n return _VNot(Size, ElementSize, Result);\n }\n case VectorCompareType::ORD_Q:\n case VectorCompareType::ORD_S: return _VFCMPORD(Size, ElementSize, Src1, Src2);\n case VectorCompareType::NGT_UQ:\n case VectorCompareType::NGT_US: {\n Ref Result = _VFCMPLT(Size, ElementSize, Src2, Src1);\n return _VNot(Size, ElementSize, Result);\n }\n case VectorCompareType::NGE_UQ:\n case VectorCompareType::NGE_US: {\n Ref Result = _VFCMPLE(Size, ElementSize, Src2, Src1);\n return _VNot(Size, ElementSize, Result);\n }\n case VectorCompareType::GT_OQ:\n case VectorCompareType::GT_OS: return _VFCMPLT(Size, ElementSize, Src2, Src1);\n case VectorCompareType::GE_OQ:\n case VectorCompareType::GE_OS: return _VFCMPLE(Size, ElementSize, Src2, Src1);\n case VectorCompareType::EQ_UQ:\n case VectorCompareType::EQ_US: {\n // If either of the sources are unordered, then returns true.\n Ref Src1_U = _VFCMPEQ(Size, ElementSize, Src1, Src1);\n Ref Src2_U = _VFCMPEQ(Size, ElementSize, Src2, Src2);\n auto Ordered = _VAnd(Size, ElementSize, Src1_U, Src2_U);\n\n Ref Compare_Ordered = _VFCMPEQ(Size, ElementSize, Src1, Src2);\n return _VOrn(Size, ElementSize, Compare_Ordered, Ordered);\n }\n case VectorCompareType::NEQ_OQ:\n case VectorCompareType::NEQ_OS: {\n // If either of the sources are unordered, then returns false.\n Ref Src1_U = _VFCMPEQ(Size, ElementSize, Src1, Src1);\n Ref Src2_U = _VFCMPEQ(Size, ElementSize, Src2, Src2);\n\n Ref Compare_Ordered = _VFCMPEQ(Size, ElementSize, Src1, Src2);\n Ref Result = _VAndn(Size, ElementSize, Src1_U, Compare_Ordered);\n return _VAnd(Size, ElementSize, Result, Src2_U);\n }\n case VectorCompareType::FALSE_OQ:\n case VectorCompareType::FALSE_OS: return LoadZeroVector(Size);\n case VectorCompareType::TRUE_UQ:\n case VectorCompareType::TRUE_US: return _VectorImm(Size, OpSize::i8Bit, -1, 0);\n }\n FEX_UNREACHABLE;\n}\n\ntemplate\nvoid OpDispatchBuilder::VFCMPOp(OpcodeArgs) {\n // No need for zero-extending in the scalar case, since\n // all we need is an insert at the end of the operation.\n const auto SrcSize = OpSizeFromSrc(Op);\n const auto DstSize = OpSizeFromDst(Op);\n\n Ref Src = LoadSourceFPR_WithOpSize(Op, Op->Src[0], SrcSize, Op->Flags);\n Ref Dest = LoadSourceFPR_WithOpSize(Op, Op->Dest, DstSize, Op->Flags);\n const uint8_t CompType = Op->Src[1].Data.Literal.Value;\n\n Ref Result = VFCMPOpImpl(OpSizeFromSrc(Op), ElementSize, Dest, Src, CompType & 0b111);\n\n StoreResultFPR(Op, Result);\n}\n\ntemplate void OpDispatchBuilder::VFCMPOp(OpcodeArgs);\ntemplate void OpDispatchBuilder::VFCMPOp(OpcodeArgs);\n\ntemplate\nvoid OpDispatchBuilder::AVXVFCMPOp(OpcodeArgs) {\n // No need for zero-extending in the scalar case, since\n // all we need is an insert at the end of the operation.\n const auto SrcSize = OpSizeFromSrc(Op);\n const auto DstSize = OpSizeFromDst(Op);\n const uint8_t CompType = Op->Src[2].Literal();\n\n Ref Src1 = LoadSourceFPR_WithOpSize(Op, Op->Src[0], DstSize, Op->Flags);\n Ref Src2 = LoadSourceFPR_WithOpSize(Op, Op->Src[1], SrcSize, Op->Flags);\n Ref Result = VFCMPOpImpl(OpSizeFromSrc(Op), ElementSize, Src1, Src2, CompType & 0b11111);\n\n StoreResultFPR(Op, Result);\n}\n\ntemplate void OpDispatchBuilder::AVXVFCMPOp(OpcodeArgs);\ntemplate void OpDispatchBuilder::AVXVFCMPOp(OpcodeArgs);\n\nvoid OpDispatchBuilder::FXSaveOp(OpcodeArgs) {\n Ref Mem = MakeSegmentAddress(Op, Op->Dest);\n\n SaveX87State(Op, Mem);\n SaveSSEState(Mem);\n SaveMXCSRState(Mem);\n}\n\nvoid OpDispatchBuilder::XSaveOp(OpcodeArgs) {\n XSaveOpImpl(Op);\n}\n\nRef OpDispatchBuilder::XSaveBase(X86Tables::DecodedOp Op) {\n return MakeSegmentAddress(Op, Op->Dest);\n}\n\nvoid OpDispatchBuilder::XSaveOpImpl(OpcodeArgs) {\n // NOTE: Mask should be EAX and EDX concatenated, but we only need to test\n // for features that are in the lower 32 bits, so EAX only is sufficient.\n const auto OpSize = GetGPROpSize();\n\n const auto StoreIfFlagSet = [this, OpSize](uint32_t BitIndex, auto fn, uint32_t FieldSize = 1) {\n Ref Mask = LoadGPRRegister(X86State::REG_RAX);\n Ref BitFlag = _Bfe(OpSize, FieldSize, BitIndex, Mask);\n auto CondJump_ = CondJump(BitFlag, CondClass::NEQ);\n\n auto StoreBlock = CreateNewCodeBlockAfter(GetCurrentBlock());\n SetTrueJumpTarget(CondJump_, StoreBlock);\n SetCurrentCodeBlock(StoreBlock);\n StartNewBlock();\n { fn(); }\n auto Jump_ = Jump();\n auto NextJumpTarget = CreateNewCodeBlockAfter(StoreBlock);\n SetJumpTarget(Jump_, NextJumpTarget);\n SetFalseJumpTarget(CondJump_, NextJumpTarget);\n SetCurrentCodeBlock(NextJumpTarget);\n StartNewBlock();\n };\n\n // x87\n {\n StoreIfFlagSet(0, [this, Op] { SaveX87State(Op, XSaveBase(Op)); });\n }\n // SSE\n {\n StoreIfFlagSet(1, [this, Op] { SaveSSEState(XSaveBase(Op)); });\n }\n // AVX\n if (CTX->HostFeatures.SupportsAVX) {\n StoreIfFlagSet(2, [this, Op] { std::invoke(SaveAVXStateFunc, this, XSaveBase(Op)); });\n }\n\n // We need to save MXCSR and MXCSR_MASK if either SSE or AVX are requested to be saved\n {\n StoreIfFlagSet(1, [this, Op] { SaveMXCSRState(XSaveBase(Op)); }, 2);\n }\n\n // Update XSTATE_BV region of the XSAVE header\n {\n Ref Base = XSaveBase(Op);\n\n // NOTE: We currently only support the first 3 bits (x87, SSE, and AVX)\n Ref Mask = LoadGPRRegister(X86State::REG_RAX);\n Ref RequestedFeatures = _Bfe(OpSize, 3, 0, Mask);\n\n // XSTATE_BV section of the header is 8 bytes in size, but we only really\n // care about setting at most 3 bits in the first byte. We zero out the rest.\n _StoreMemGPR(OpSize::i64Bit, RequestedFeatures, Base, Constant(512), OpSize::i8Bit, MemOffsetType::SXTX, 1);\n }\n}\n\nvoid OpDispatchBuilder::SaveX87State(OpcodeArgs, Ref MemBase) {\n _SyncStackToSlow();\n\n // Saves 512bytes to the memory location provided\n // Header changes depending on if REX.W is set or not\n if (Op->Flags & X86Tables::DecodeFlags::FLAG_REX_WIDENING) {\n // BYTE | 0 1 | 2 3 | 4 | 5 | 6 7 | 8 9 | a b | c d | e f |\n // ------------------------------------------\n // 00 | FCW | FSW | FTW | | FOP | FIP |\n // 16 | FDP | MXCSR | MXCSR_MASK|\n } else {\n // BYTE | 0 1 | 2 3 | 4 | 5 | 6 7 | 8 9 | a b | c d | e f |\n // ------------------------------------------\n // 00 | FCW | FSW | FTW | | FOP | FIP[31:0] | FCS | |\n // 16 | FDP[31:0] | FDS | | MXCSR | MXCSR_MASK|\n }\n\n {\n auto FCW = _LoadContextGPR(OpSize::i16Bit, offsetof(FEXCore::Core::CPUState, FCW));\n _StoreMemGPR(OpSize::i16Bit, MemBase, FCW, OpSize::i16Bit);\n }\n\n { _StoreMemGPR(OpSize::i16Bit, ReconstructFSW_Helper(), MemBase, Constant(2), OpSize::i16Bit, MemOffsetType::SXTX, 1); }\n\n {\n // Abridged FTW\n auto FTW = _LoadContextGPR(OpSize::i8Bit, offsetof(FEXCore::Core::CPUState, AbridgedFTW));\n _StoreMemGPR(OpSize::i8Bit, FTW, MemBase, Constant(4), OpSize::i8Bit, MemOffsetType::SXTX, 1);\n }\n\n // BYTE | 0 1 | 2 3 | 4 | 5 | 6 7 | 8 9 | a b | c d | e f |\n // ------------------------------------------\n // 32 | ST0/MM0 | \n // 48 | ST1/MM1 | \n // 64 | ST2/MM2 | \n // 80 | ST3/MM3 | \n // 96 | ST4/MM4 | \n // 112 | ST5/MM5 | \n // 128 | ST6/MM6 | \n // 144 | ST7/MM7 | \n // 160 | XMM0\n // 173 | XMM1\n // 192 | XMM2\n // 208 | XMM3\n // 224 | XMM4\n // 240 | XMM5\n // 256 | XMM6\n // 272 | XMM7\n // 288 | 64BitMode ? : XMM8\n // 304 | 64BitMode ? : XMM9\n // 320 | 64BitMode ? : XMM10\n // 336 | 64BitMode ? : XMM11\n // 352 | 64BitMode ? : XMM12\n // 368 | 64BitMode ? : XMM13\n // 384 | 64BitMode ? : XMM14\n // 400 | 64BitMode ? : XMM15\n // 416 | \n // 432 | \n // 448 | \n // 464 | Available\n // 480 | Available\n // 496 | Available\n // FCW: x87 FPU control word\n // FSW: x87 FPU status word\n // FTW: x87 FPU Tag word (Abridged)\n // FOP: x87 FPU opcode. Lower 11 bits of the opcode\n // FIP: x87 FPU instructyion pointer offset\n // FCS: x87 FPU instruction pointer selector. If CPUID_0000_0007_0000_00000:EBX[bit 13] = 1 then this is deprecated and stores as 0\n // FDP: x87 FPU instruction operand (data) pointer offset\n // FDS: x87 FPU instruction operand (data) pointer selector. Same deprecation as FCS\n // MXCSR: If OSFXSR bit in CR4 is not set then this may not be saved\n // MXCSR_MASK: Mask for writes to the MXCSR register\n // If OSFXSR bit in CR4 is not set than FXSAVE /may/ not save the XMM registers\n // This is implementation dependent\n //\n // x87 registers are stored rotated depending on the current TOP.\n Ref Top = GetX87Top();\n auto SevenConst = Constant(7);\n const auto LoadSize = ReducedPrecisionMode ? OpSize::i64Bit : OpSize::i128Bit;\n\n for (uint32_t i = 0; i < Core::CPUState::NUM_MMS; ++i) {\n Ref data = _LoadContextFPRIndexed(Top, LoadSize, MMBaseOffset(), IR::OpSizeToSize(OpSize::i128Bit));\n if (ReducedPrecisionMode) {\n data = _F80CVTTo(data, OpSize::i64Bit);\n }\n _StoreMemFPR(OpSize::i128Bit, data, MemBase, Constant(16 * i + 32), OpSize::i8Bit, MemOffsetType::SXTX, 1);\n Top = _And(OpSize::i32Bit, Add(OpSize::i32Bit, Top, 1), SevenConst);\n }\n}\n\nvoid OpDispatchBuilder::SaveSSEState(Ref MemBase) {\n const auto NumRegs = Is64BitMode ? 16U : 8U;\n\n for (uint32_t i = 0; i < NumRegs; i += 2) {\n _StoreMemPairFPR(OpSize::i128Bit, LoadXMMRegister(i), LoadXMMRegister(i + 1), MemBase, i * 16 + 160);\n }\n}\n\nvoid OpDispatchBuilder::SaveMXCSRState(Ref MemBase) {\n // Store MXCSR and the mask for all bits.\n _StoreMemPairGPR(OpSize::i32Bit, GetMXCSR(), Constant(0xFFFF), MemBase, 24);\n}\n\nvoid OpDispatchBuilder::SaveAVXState(Ref MemBase) {\n const auto NumRegs = Is64BitMode ? 16U : 8U;\n\n for (uint32_t i = 0; i < NumRegs; i += 2) {\n Ref Upper0 = _VDupElement(OpSize::i256Bit, OpSize::i128Bit, LoadXMMRegister(i + 0), 1);\n Ref Upper1 = _VDupElement(OpSize::i256Bit, OpSize::i128Bit, LoadXMMRegister(i + 1), 1);\n\n _StoreMemPairFPR(OpSize::i128Bit, Upper0, Upper1, MemBase, i * 16 + 576);\n }\n}\n\nRef OpDispatchBuilder::GetMXCSR() {\n Ref MXCSR = _LoadContextGPR(OpSize::i32Bit, offsetof(FEXCore::Core::CPUState, mxcsr));\n // Mask out unsupported bits\n // Keeps FZ, RC, exception masks, and DAZ\n MXCSR = _And(OpSize::i32Bit, MXCSR, Constant(0xFFC0));\n return MXCSR;\n}\n\nvoid OpDispatchBuilder::FXRStoreOp(OpcodeArgs) {\n Ref Mem = MakeSegmentAddress(Op, Op->Src[0]);\n\n RestoreX87State(Mem);\n RestoreSSEState(Mem);\n\n Ref MXCSR = _LoadMemGPR(OpSize::i32Bit, Mem, Constant(24), OpSize::i32Bit, MemOffsetType::SXTX, 1);\n RestoreMXCSRState(MXCSR);\n}\n\nvoid OpDispatchBuilder::XRstorOpImpl(OpcodeArgs) {\n const auto OpSize = GetGPROpSize();\n\n // If a bit in our XSTATE_BV is set, then we restore from that region of the XSAVE area,\n // otherwise, if not set, then we need to set the relevant data the bit corresponds to\n // to it's defined initial configuration.\n const auto RestoreIfFlagSetOrDefault = [this, Op, OpSize](uint32_t BitIndex, auto restore_fn, auto default_fn, uint32_t FieldSize = 1) {\n // Set up base address for the XSAVE region to restore from, and also read\n // the XSTATE_BV bit flags out of the XSTATE header.\n //\n // Note: we rematerialize Base/Mask in each block to avoid crossblock\n // liveness.\n Ref Base = XSaveBase(Op);\n Ref Mask = _LoadMemGPR(OpSize::i64Bit, Base, Constant(512), OpSize::i64Bit, MemOffsetType::SXTX, 1);\n\n Ref BitFlag = _Bfe(OpSize, FieldSize, BitIndex, Mask);\n auto CondJump_ = CondJump(BitFlag, CondClass::NEQ);\n\n auto RestoreBlock = CreateNewCodeBlockAfter(GetCurrentBlock());\n SetTrueJumpTarget(CondJump_, RestoreBlock);\n SetCurrentCodeBlock(RestoreBlock);\n StartNewBlock();\n { restore_fn(); }\n auto RestoreExitJump = Jump();\n auto DefaultBlock = CreateNewCodeBlockAfter(RestoreBlock);\n auto ExitBlock = CreateNewCodeBlockAfter(DefaultBlock);\n SetJumpTarget(RestoreExitJump, ExitBlock);\n SetFalseJumpTarget(CondJump_, DefaultBlock);\n SetCurrentCodeBlock(DefaultBlock);\n StartNewBlock();\n { default_fn(); }\n auto DefaultExitJump = Jump();\n SetJumpTarget(DefaultExitJump, ExitBlock);\n SetCurrentCodeBlock(ExitBlock);\n StartNewBlock();\n };\n\n // x87\n {\n RestoreIfFlagSetOrDefault(0, [this, Op] { RestoreX87State(XSaveBase(Op)); }, [this, Op] { DefaultX87State(Op); });\n }\n // SSE\n {\n RestoreIfFlagSetOrDefault(1, [this, Op] { RestoreSSEState(XSaveBase(Op)); }, [this] { DefaultSSEState(); });\n }\n // AVX\n if (CTX->HostFeatures.SupportsAVX) {\n RestoreIfFlagSetOrDefault(\n 2, [this, Op] { std::invoke(RestoreAVXStateFunc, this, XSaveBase(Op)); }, [this] { std::invoke(DefaultAVXStateFunc, this); });\n }\n\n {\n // We need to restore the MXCSR if either SSE or AVX are requested to be saved\n RestoreIfFlagSetOrDefault(\n 1,\n [this, Op] {\n Ref Base = XSaveBase(Op);\n Ref MXCSR = _LoadMemGPR(OpSize::i32Bit, Base, Constant(24), OpSize::i32Bit, MemOffsetType::SXTX, 1);\n RestoreMXCSRState(MXCSR);\n },\n [] { /* Intentionally do nothing*/ }, 2);\n }\n}\n\nvoid OpDispatchBuilder::RestoreX87State(Ref MemBase) {\n _StackForceSlow();\n\n auto NewFCW = _LoadMemGPR(OpSize::i16Bit, MemBase, OpSize::i16Bit);\n _StoreContextGPR(OpSize::i16Bit, NewFCW, offsetof(FEXCore::Core::CPUState, FCW));\n\n {\n auto NewFSW = _LoadMemGPR(OpSize::i16Bit, MemBase, Constant(2), OpSize::i16Bit, MemOffsetType::SXTX, 1);\n ReconstructX87StateFromFSW_Helper(NewFSW);\n }\n\n {\n // Abridged FTW\n auto NewFTW = _LoadMemGPR(OpSize::i8Bit, MemBase, Constant(4), OpSize::i8Bit, MemOffsetType::SXTX, 1);\n _StoreContextGPR(OpSize::i8Bit, NewFTW, offsetof(FEXCore::Core::CPUState, AbridgedFTW));\n }\n\n for (uint32_t i = 0; i < Core::CPUState::NUM_MMS; i += 2) {\n auto MMRegs = LoadMemPairFPR(OpSize::i128Bit, MemBase, i * 16 + 32);\n _StoreContextFPR(OpSize::i128Bit, MMRegs.Low, MMBaseOffset() + i * 16);\n _StoreContextFPR(OpSize::i128Bit, MMRegs.High, MMBaseOffset() + (i + 1) * 16);\n }\n}\n\nvoid OpDispatchBuilder::RestoreSSEState(Ref MemBase) {\n const auto NumRegs = Is64BitMode ? 16U : 8U;\n\n for (uint32_t i = 0; i < NumRegs; i += 2) {\n auto XMMRegs = LoadMemPairFPR(OpSize::i128Bit, MemBase, i * 16 + 160);\n\n StoreXMMRegister(i, XMMRegs.Low);\n StoreXMMRegister(i + 1, XMMRegs.High);\n }\n}\n\nvoid OpDispatchBuilder::RestoreMXCSRState(Ref MXCSR) {\n // Mask out unsupported bits\n MXCSR = _And(OpSize::i32Bit, MXCSR, Constant(0xFFC0));\n\n _StoreContextGPR(OpSize::i32Bit, MXCSR, offsetof(FEXCore::Core::CPUState, mxcsr));\n // We only support the rounding mode and FTZ bit being set\n Ref RoundingMode = _Bfe(OpSize::i32Bit, 3, 13, MXCSR);\n _SetRoundingMode(RoundingMode, true, MXCSR);\n}\n\nvoid OpDispatchBuilder::RestoreAVXState(Ref MemBase) {\n const auto NumRegs = Is64BitMode ? 16U : 8U;\n\n for (uint32_t i = 0; i < NumRegs; i += 2) {\n Ref XMMReg0 = LoadXMMRegister(i + 0);\n Ref XMMReg1 = LoadXMMRegister(i + 1);\n auto YMMHRegs = LoadMemPairFPR(OpSize::i128Bit, MemBase, i * 16 + 576);\n StoreXMMRegister(i + 0, _VInsElement(OpSize::i256Bit, OpSize::i128Bit, 1, 0, XMMReg0, YMMHRegs.Low));\n StoreXMMRegister(i + 1, _VInsElement(OpSize::i256Bit, OpSize::i128Bit, 1, 0, XMMReg1, YMMHRegs.High));\n }\n}\n\nvoid OpDispatchBuilder::DefaultX87State(OpcodeArgs) {\n // We can piggy-back on FNINIT's implementation, since\n // it performs the same behavior as required by XRSTOR for resetting flags\n FNINIT(Op);\n\n // On top of resetting the flags to a default state, we also need to clear\n // all of the ST0-7/MM0-7 registers to zero.\n Ref ZeroVector = LoadZeroVector(OpSize::i64Bit);\n for (uint32_t i = 0; i < Core::CPUState::NUM_MMS; ++i) {\n _StoreContextFPR(OpSize::i128Bit, ZeroVector, MMBaseOffset() + i * 16);\n }\n}\n\nvoid OpDispatchBuilder::DefaultSSEState() {\n const auto NumRegs = Is64BitMode ? 16U : 8U;\n\n Ref ZeroVector = LoadZeroVector(OpSize::i128Bit);\n for (uint32_t i = 0; i < NumRegs; ++i) {\n StoreXMMRegister(i, ZeroVector);\n }\n}\n\nvoid OpDispatchBuilder::DefaultAVXState() {\n const auto NumRegs = Is64BitMode ? 16U : 8U;\n\n for (uint32_t i = 0; i < NumRegs; i++) {\n Ref Reg = LoadXMMRegister(i);\n Ref Dst = _VMov(OpSize::i128Bit, Reg);\n StoreXMMRegister(i, Dst);\n }\n}\n\nRef OpDispatchBuilder::PALIGNROpImpl(OpcodeArgs, const X86Tables::DecodedOperand& Src1, const X86Tables::DecodedOperand& Src2,\n const X86Tables::DecodedOperand& Imm, bool IsAVX) {\n // For the 256-bit case we handle it as pairs of 128-bit halves.\n const auto DstSize = OpSizeFromDst(Op);\n const auto SanitizedDstSize = std::min(DstSize, OpSize::i128Bit);\n\n const auto Is256Bit = DstSize == OpSize::i256Bit;\n const auto Index = Imm.Literal();\n\n Ref Src2Node = LoadSourceFPR(Op, Src2, Op->Flags);\n if (Index == 0) {\n if (IsAVX && !Is256Bit) {\n // 128-bit AVX needs to zero the upper bits.\n return _VMov(OpSize::i128Bit, Src2Node);\n } else {\n return Src2Node;\n }\n }\n Ref Src1Node = LoadSourceFPR(Op, Src1, Op->Flags);\n\n if (Index >= (IR::OpSizeToSize(SanitizedDstSize) * 2)) {\n // If the immediate is greater than both vectors combined then it zeroes the vector\n return LoadZeroVector(DstSize);\n }\n\n Ref Low = _VExtr(SanitizedDstSize, OpSize::i8Bit, Src1Node, Src2Node, Index);\n if (!Is256Bit) {\n return Low;\n }\n\n Ref HighSrc1 = _VInsElement(DstSize, OpSize::i128Bit, 0, 1, Src1Node, Src1Node);\n Ref HighSrc2 = _VInsElement(DstSize, OpSize::i128Bit, 0, 1, Src2Node, Src2Node);\n Ref High = _VExtr(SanitizedDstSize, OpSize::i8Bit, HighSrc1, HighSrc2, Index);\n return _VInsElement(DstSize, OpSize::i128Bit, 1, 0, Low, High);\n}\n\nvoid OpDispatchBuilder::PAlignrOp(OpcodeArgs) {\n Ref Result = PALIGNROpImpl(Op, Op->Dest, Op->Src[0], Op->Src[1], false);\n StoreResultFPR(Op, Result);\n}\n\nvoid OpDispatchBuilder::VPALIGNROp(OpcodeArgs) {\n Ref Result = PALIGNROpImpl(Op, Op->Src[0], Op->Src[1], Op->Src[2], true);\n StoreResultFPR(Op, Result);\n}\n\ntemplate\nvoid OpDispatchBuilder::UCOMISxOp(OpcodeArgs) {\n const auto SrcSize = Op->Src[0].IsGPR() ? GetGuestVectorLength() : ElementSize;\n Ref Src1 = LoadSourceFPR_WithOpSize(Op, Op->Dest, GetGuestVectorLength(), Op->Flags);\n Ref Src2 = LoadSourceFPR_WithOpSize(Op, Op->Src[0], SrcSize, Op->Flags);\n\n Comiss(ElementSize, Src1, Src2);\n}\n\ntemplate void OpDispatchBuilder::UCOMISxOp(OpcodeArgs);\ntemplate void OpDispatchBuilder::UCOMISxOp(OpcodeArgs);\n\nvoid OpDispatchBuilder::LDMXCSR(OpcodeArgs) {\n Ref Dest = LoadSourceGPR_WithOpSize(Op, Op->Dest, OpSize::i32Bit, Op->Flags);\n RestoreMXCSRState(Dest);\n}\n\nvoid OpDispatchBuilder::STMXCSR(OpcodeArgs) {\n StoreResultGPR_WithOpSize(Op, Op->Dest, GetMXCSR(), OpSize::i32Bit);\n}\n\ntemplate\nvoid OpDispatchBuilder::PACKUSOp(OpcodeArgs) {\n Ref Dest = LoadSourceFPR(Op, Op->Dest, Op->Flags);\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n Ref Result = _VSQXTUNPair(OpSizeFromSrc(Op), ElementSize, Dest, Src);\n\n StoreResultFPR(Op, Result);\n}\n\ntemplate void OpDispatchBuilder::PACKUSOp(OpcodeArgs);\ntemplate void OpDispatchBuilder::PACKUSOp(OpcodeArgs);\n\nvoid OpDispatchBuilder::VPACKUSOp(OpcodeArgs, IR::OpSize ElementSize) {\n const auto DstSize = OpSizeFromDst(Op);\n const auto Is256Bit = DstSize == OpSize::i256Bit;\n\n Ref Src1 = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n Ref Src2 = LoadSourceFPR(Op, Op->Src[1], Op->Flags);\n Ref Result = _VSQXTUNPair(OpSizeFromSrc(Op), ElementSize, Src1, Src2);\n\n if (Is256Bit) {\n // We do a little cheeky 64-bit swapping to interleave the result.\n Ref Swapped = _VInsElement(DstSize, OpSize::i64Bit, 2, 1, Result, Result);\n Result = _VInsElement(DstSize, OpSize::i64Bit, 1, 2, Swapped, Result);\n }\n StoreResultFPR(Op, Result);\n}\n\ntemplate\nvoid OpDispatchBuilder::PACKSSOp(OpcodeArgs) {\n Ref Dest = LoadSourceFPR(Op, Op->Dest, Op->Flags);\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n Ref Result = _VSQXTNPair(OpSizeFromSrc(Op), ElementSize, Dest, Src);\n\n StoreResultFPR(Op, Result);\n}\n\ntemplate void OpDispatchBuilder::PACKSSOp(OpcodeArgs);\ntemplate void OpDispatchBuilder::PACKSSOp(OpcodeArgs);\n\nvoid OpDispatchBuilder::VPACKSSOp(OpcodeArgs, IR::OpSize ElementSize) {\n const auto DstSize = OpSizeFromDst(Op);\n const auto Is256Bit = DstSize == OpSize::i256Bit;\n\n Ref Src1 = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n Ref Src2 = LoadSourceFPR(Op, Op->Src[1], Op->Flags);\n Ref Result = _VSQXTNPair(OpSizeFromSrc(Op), ElementSize, Src1, Src2);\n\n if (Is256Bit) {\n // We do a little cheeky 64-bit swapping to interleave the result.\n Ref Swapped = _VInsElement(DstSize, OpSize::i64Bit, 2, 1, Result, Result);\n Result = _VInsElement(DstSize, OpSize::i64Bit, 1, 2, Swapped, Result);\n }\n StoreResultFPR(Op, Result);\n}\n\nRef OpDispatchBuilder::PMULLOpImpl(OpSize Size, IR::OpSize ElementSize, bool Signed, Ref Src1, Ref Src2) {\n if (Size == OpSize::i64Bit) {\n if (Signed) {\n return _VSMull(OpSize::i128Bit, ElementSize, Src1, Src2);\n } else {\n return _VUMull(OpSize::i128Bit, ElementSize, Src1, Src2);\n }\n } else {\n auto InsSrc1 = _VUnZip(Size, ElementSize, Src1, Src1);\n auto InsSrc2 = _VUnZip(Size, ElementSize, Src2, Src2);\n\n if (Signed) {\n return _VSMull(Size, ElementSize, InsSrc1, InsSrc2);\n } else {\n return _VUMull(Size, ElementSize, InsSrc1, InsSrc2);\n }\n }\n}\n\ntemplate\nvoid OpDispatchBuilder::PMULLOp(OpcodeArgs) {\n static_assert(ElementSize == OpSize::i32Bit, \"Currently only handles 32-bit -> 64-bit\");\n\n Ref Src1 = LoadSourceFPR(Op, Op->Dest, Op->Flags);\n Ref Src2 = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n Ref Res = PMULLOpImpl(OpSizeFromSrc(Op), ElementSize, Signed, Src1, Src2);\n\n StoreResultFPR(Op, Res);\n}\n\ntemplate void OpDispatchBuilder::PMULLOp(OpcodeArgs);\ntemplate void OpDispatchBuilder::PMULLOp(OpcodeArgs);\n\ntemplate\nvoid OpDispatchBuilder::VPMULLOp(OpcodeArgs) {\n static_assert(ElementSize == OpSize::i32Bit, \"Currently only handles 32-bit -> 64-bit\");\n\n Ref Src1 = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n Ref Src2 = LoadSourceFPR(Op, Op->Src[1], Op->Flags);\n Ref Result = PMULLOpImpl(OpSizeFromSrc(Op), ElementSize, Signed, Src1, Src2);\n\n StoreResultFPR(Op, Result);\n}\n\ntemplate void OpDispatchBuilder::VPMULLOp(OpcodeArgs);\ntemplate void OpDispatchBuilder::VPMULLOp(OpcodeArgs);\n\ntemplate\nvoid OpDispatchBuilder::MOVQ2DQ(OpcodeArgs) {\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n\n // This instruction is a bit special in that if the source is MMX then it zexts to 128bit\n if constexpr (ToXMM) {\n const auto Index = Op->Dest.Data.GPR.GPR - FEXCore::X86State::REG_XMM_0;\n\n Src = VZeroExtendOperand(OpSize::i128Bit, Op->Src[0], Src);\n StoreXMMRegister(Index, Src);\n } else {\n // This is simple, just store the result\n StoreResultFPR(Op, Src);\n }\n}\n\ntemplate void OpDispatchBuilder::MOVQ2DQ(OpcodeArgs);\ntemplate void OpDispatchBuilder::MOVQ2DQ(OpcodeArgs);\n\nRef OpDispatchBuilder::ADDSUBPOpImpl(OpSize Size, IR::OpSize ElementSize, Ref Src1, Ref Src2) {\n if (CTX->HostFeatures.SupportsFCMA) {\n if (ElementSize == OpSize::i32Bit) {\n auto Swizzle = _VRev64(Size, OpSize::i32Bit, Src2);\n return _VFCADD(Size, ElementSize, Src1, Swizzle, 90);\n } else {\n auto Swizzle = _VExtr(Size, OpSize::i8Bit, Src2, Src2, 8);\n return _VFCADD(Size, ElementSize, Src1, Swizzle, 90);\n }\n } else {\n auto ConstantEOR =\n LoadAndCacheNamedVectorConstant(Size, ElementSize == OpSize::i32Bit ? NAMED_VECTOR_PADDSUBPS_INVERT : NAMED_VECTOR_PADDSUBPD_INVERT);\n auto InvertedSource = _VXor(Size, ElementSize, Src2, ConstantEOR);\n return _VFAdd(Size, ElementSize, Src1, InvertedSource);\n }\n}\n\ntemplate\nvoid OpDispatchBuilder::ADDSUBPOp(OpcodeArgs) {\n Ref Dest = LoadSourceFPR(Op, Op->Dest, Op->Flags);\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n Ref Result = ADDSUBPOpImpl(OpSizeFromSrc(Op), ElementSize, Dest, Src);\n\n StoreResultFPR(Op, Result);\n}\n\ntemplate void OpDispatchBuilder::ADDSUBPOp(OpcodeArgs);\ntemplate void OpDispatchBuilder::ADDSUBPOp(OpcodeArgs);\n\ntemplate\nvoid OpDispatchBuilder::VADDSUBPOp(OpcodeArgs) {\n Ref Src1 = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n Ref Src2 = LoadSourceFPR(Op, Op->Src[1], Op->Flags);\n Ref Result = ADDSUBPOpImpl(OpSizeFromSrc(Op), ElementSize, Src1, Src2);\n\n StoreResultFPR(Op, Result);\n}\n\ntemplate void OpDispatchBuilder::VADDSUBPOp(OpcodeArgs);\ntemplate void OpDispatchBuilder::VADDSUBPOp(OpcodeArgs);\n\nvoid OpDispatchBuilder::PFNACCOp(OpcodeArgs) {\n const auto Size = OpSizeFromSrc(Op);\n\n Ref Dest = LoadSourceFPR(Op, Op->Dest, Op->Flags);\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n\n auto DestUnzip = _VUnZip(Size, OpSize::i32Bit, Dest, Src);\n auto SrcUnzip = _VUnZip2(Size, OpSize::i32Bit, Dest, Src);\n auto Result = _VFSub(Size, OpSize::i32Bit, DestUnzip, SrcUnzip);\n\n StoreResultFPR(Op, Result);\n}\n\nvoid OpDispatchBuilder::PFPNACCOp(OpcodeArgs) {\n const auto Size = OpSizeFromSrc(Op);\n\n Ref Dest = LoadSourceFPR(Op, Op->Dest, Op->Flags);\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n\n Ref ResAdd {};\n Ref ResSub {};\n auto UpperSubDest = _VDupElement(Size, OpSize::i32Bit, Dest, 1);\n\n ResSub = _VFSub(OpSize::i32Bit, OpSize::i32Bit, Dest, UpperSubDest);\n ResAdd = _VFAddP(Size, OpSize::i32Bit, Src, Src);\n\n auto Result = _VInsElement(OpSize::i64Bit, OpSize::i32Bit, 1, 0, ResSub, ResAdd);\n\n StoreResultFPR(Op, Result);\n}\n\nvoid OpDispatchBuilder::PSWAPDOp(OpcodeArgs) {\n const auto Size = OpSizeFromSrc(Op);\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n\n auto Result = _VRev64(Size, OpSize::i32Bit, Src);\n StoreResultFPR(Op, Result);\n}\n\nvoid OpDispatchBuilder::PI2FWOp(OpcodeArgs) {\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n\n const auto Size = OpSizeFromDst(Op);\n\n // We now need to transpose the lower 16-bits of each element together\n // Only needing to move the upper element down in this case\n Src = _VUnZip(Size, OpSize::i16Bit, Src, Src);\n\n // Now we need to sign extend the 16bit value to 32-bit\n Src = _VSXTL(Size, OpSize::i16Bit, Src);\n\n // int32_t to float\n Src = _Vector_SToF(Size, OpSize::i32Bit, Src);\n\n StoreResultFPR_WithOpSize(Op, Op->Dest, Src, Size);\n}\n\nvoid OpDispatchBuilder::PF2IWOp(OpcodeArgs) {\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n\n const auto Size = OpSizeFromDst(Op);\n\n // Float to int32_t\n Src = _Vector_FToZS(Size, OpSize::i32Bit, Src);\n\n // We now need to transpose the lower 16-bits of each element together\n // Only needing to move the upper element down in this case\n Src = _VUnZip(Size, OpSize::i16Bit, Src, Src);\n\n // Now we need to sign extend the 16bit value to 32-bit\n Src = _VSXTL(Size, OpSize::i16Bit, Src);\n StoreResultFPR_WithOpSize(Op, Op->Dest, Src, Size);\n}\n\nvoid OpDispatchBuilder::PMULHRWOp(OpcodeArgs) {\n const auto Size = OpSizeFromSrc(Op);\n\n Ref Dest = LoadSourceFPR(Op, Op->Dest, Op->Flags);\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n\n Ref Res {};\n\n // Implementation is more efficient for 8byte registers\n // Multiplies 4 16bit values in to 4 32bit values\n Res = _VSMull(Size << 1, OpSize::i16Bit, Dest, Src);\n\n // Load 0x0000_8000 in to each 32-bit element.\n Ref VConstant = _VectorImm(OpSize::i128Bit, OpSize::i32Bit, 0x80, 8);\n\n Res = _VAdd(Size << 1, OpSize::i32Bit, Res, VConstant);\n\n // Now shift and narrow to convert 32-bit values to 16bit, storing the top 16bits\n Res = _VUShrNI(Size << 1, OpSize::i32Bit, Res, 16);\n\n StoreResultFPR(Op, Res);\n}\n\ntemplate\nvoid OpDispatchBuilder::VPFCMPOp(OpcodeArgs) {\n const auto Size = OpSizeFromSrc(Op);\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n Ref Dest = LoadSourceFPR_WithOpSize(Op, Op->Dest, OpSizeFromDst(Op), Op->Flags);\n\n Ref Result {};\n // This maps 1:1 to an AArch64 NEON Op\n // auto ALUOp = _VCMPGT(Size, 4, Dest, Src);\n switch (CompType) {\n case 0x00: // EQ\n Result = _VFCMPEQ(Size, OpSize::i32Bit, Dest, Src);\n break;\n case 0x01: // GE(Swapped operand)\n Result = _VFCMPLE(Size, OpSize::i32Bit, Src, Dest);\n break;\n case 0x02: // GT\n Result = _VFCMPGT(Size, OpSize::i32Bit, Dest, Src);\n break;\n default: LOGMAN_MSG_A_FMT(\"Unknown Comparison type: {}\", CompType); break;\n }\n\n StoreResultFPR(Op, Result);\n}\n\ntemplate void OpDispatchBuilder::VPFCMPOp<0>(OpcodeArgs);\ntemplate void OpDispatchBuilder::VPFCMPOp<1>(OpcodeArgs);\ntemplate void OpDispatchBuilder::VPFCMPOp<2>(OpcodeArgs);\n\nRef OpDispatchBuilder::PMADDWDOpImpl(IR::OpSize Size, Ref Src1, Ref Src2) {\n // This is a pretty curious operation\n // Does two MADD operations across 4 16bit signed integers and accumulates to 32bit integers in the destination\n //\n // x86 PMADDWD: xmm1, xmm2\n // xmm1[31:0] = (xmm1[15:0] * xmm2[15:0]) + (xmm1[31:16] * xmm2[31:16])\n // xmm1[63:32] = (xmm1[47:32] * xmm2[47:32]) + (xmm1[63:48] * xmm2[63:48])\n // etc.. for larger registers\n\n if (Size == OpSize::i64Bit) {\n // MMX implementation can be slightly more optimal\n Size = Size >> 1;\n auto MullResult = _VSMull(Size, OpSize::i16Bit, Src1, Src2);\n return _VAddP(Size, OpSize::i32Bit, MullResult, MullResult);\n }\n\n auto Lower = _VSMull(Size, OpSize::i16Bit, Src1, Src2);\n auto Upper = _VSMull2(Size, OpSize::i16Bit, Src1, Src2);\n\n // [15:0 ] + [31:16], [32:47 ] + [63:48 ], [79:64] + [95:80], [111:96] + [127:112]\n return _VAddP(Size, OpSize::i32Bit, Lower, Upper);\n}\n\nvoid OpDispatchBuilder::PMADDWD(OpcodeArgs) {\n const auto Size = OpSizeFromSrc(Op);\n\n Ref Src1 = LoadSourceFPR(Op, Op->Dest, Op->Flags);\n Ref Src2 = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n\n Ref Result = PMADDWDOpImpl(Size, Src1, Src2);\n StoreResultFPR(Op, Result);\n}\n\nvoid OpDispatchBuilder::VPMADDWDOp(OpcodeArgs) {\n const auto Size = OpSizeFromSrc(Op);\n\n Ref Src1 = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n Ref Src2 = LoadSourceFPR(Op, Op->Src[1], Op->Flags);\n\n Ref Result = PMADDWDOpImpl(Size, Src1, Src2);\n StoreResultFPR(Op, Result);\n}\n\nRef OpDispatchBuilder::PMADDUBSWOpImpl(IR::OpSize Size, Ref Src1, Ref Src2) {\n if (Size == OpSize::i64Bit) {\n const auto MultSize = Size << 1;\n // 64bit is more efficient\n\n // Src1 is unsigned\n auto Src1_16b = _VUXTL(MultSize, OpSize::i8Bit, Src1); // [7:0 ], [15:8], [23:16], [31:24], [39:32], [47:40], [55:48], [63:56]\n\n // Src2 is signed\n auto Src2_16b = _VSXTL(MultSize, OpSize::i8Bit, Src2); // [7:0 ], [15:8], [23:16], [31:24], [39:32], [47:40], [55:48], [63:56]\n\n auto ResMul_L = _VSMull(MultSize, OpSize::i16Bit, Src1_16b, Src2_16b);\n auto ResMul_H = _VSMull2(MultSize, OpSize::i16Bit, Src1_16b, Src2_16b);\n\n // Now add pairwise across the vector\n auto ResAdd = _VAddP(MultSize, OpSize::i32Bit, ResMul_L, ResMul_H);\n\n // Add saturate back down to 16bit\n return _VSQXTN(MultSize, OpSize::i32Bit, ResAdd);\n }\n\n // V{U,S}XTL{,2}/ and VUnZip{,2} can be optimized in this solution to save about one instruction.\n // We can up-front zero extend and sign extend the elements in-place.\n // This means extracting even and odd elements up-front so the unzips aren't required.\n // Requires implementing IR ops for BIC (vector, immediate) although.\n\n // Src1 is unsigned\n auto Src1_16b_L = _VUXTL(Size, OpSize::i8Bit, Src1); // [7:0 ], [15:8], [23:16], [31:24], [39:32], [47:40], [55:48], [63:56]\n auto Src2_16b_L = _VSXTL(Size, OpSize::i8Bit, Src2); // [7:0 ], [15:8], [23:16], [31:24], [39:32], [47:40], [55:48], [63:56]\n auto ResMul_L = _VMul(Size, OpSize::i16Bit, Src1_16b_L, Src2_16b_L);\n\n // Src2 is signed\n auto Src1_16b_H = _VUXTL2(Size, OpSize::i8Bit, Src1); // Offset to +64bits [7:0 ], [15:8], [23:16], [31:24], [39:32], [47:40], [55:48], [63:56]\n auto Src2_16b_H = _VSXTL2(Size, OpSize::i8Bit, Src2); // Offset to +64bits [7:0 ], [15:8], [23:16], [31:24], [39:32], [47:40], [55:48], [63:56]\n auto ResMul_L_H = _VMul(Size, OpSize::i16Bit, Src1_16b_H, Src2_16b_H);\n\n auto TmpZip1 = _VUnZip(Size, OpSize::i16Bit, ResMul_L, ResMul_L_H);\n auto TmpZip2 = _VUnZip2(Size, OpSize::i16Bit, ResMul_L, ResMul_L_H);\n\n return _VSQAdd(Size, OpSize::i16Bit, TmpZip1, TmpZip2);\n}\n\nvoid OpDispatchBuilder::PMADDUBSW(OpcodeArgs) {\n const auto Size = OpSizeFromSrc(Op);\n\n Ref Src1 = LoadSourceFPR(Op, Op->Dest, Op->Flags);\n Ref Src2 = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n\n Ref Result = PMADDUBSWOpImpl(Size, Src1, Src2);\n StoreResultFPR(Op, Result);\n}\n\nvoid OpDispatchBuilder::VPMADDUBSWOp(OpcodeArgs) {\n const auto Size = OpSizeFromSrc(Op);\n\n Ref Src1 = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n Ref Src2 = LoadSourceFPR(Op, Op->Src[1], Op->Flags);\n\n Ref Result = PMADDUBSWOpImpl(Size, Src1, Src2);\n StoreResultFPR(Op, Result);\n}\n\nRef OpDispatchBuilder::PMULHWOpImpl(OpcodeArgs, bool Signed, Ref Src1, Ref Src2) {\n const auto Size = OpSizeFromSrc(Op);\n if (Signed) {\n return _VSMulH(Size, OpSize::i16Bit, Src1, Src2);\n } else {\n return _VUMulH(Size, OpSize::i16Bit, Src1, Src2);\n }\n}\n\ntemplate\nvoid OpDispatchBuilder::PMULHW(OpcodeArgs) {\n Ref Dest = LoadSourceFPR(Op, Op->Dest, Op->Flags);\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n Ref Result = PMULHWOpImpl(Op, Signed, Dest, Src);\n\n StoreResultFPR(Op, Result);\n}\n\ntemplate void OpDispatchBuilder::PMULHW(OpcodeArgs);\ntemplate void OpDispatchBuilder::PMULHW(OpcodeArgs);\n\ntemplate\nvoid OpDispatchBuilder::VPMULHWOp(OpcodeArgs) {\n const auto DstSize = GetDstSize(Op);\n const auto Is128Bit = DstSize == Core::CPUState::XMM_SSE_REG_SIZE;\n\n Ref Dest = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n Ref Src = LoadSourceFPR(Op, Op->Src[1], Op->Flags);\n Ref Result = PMULHWOpImpl(Op, Signed, Dest, Src);\n\n if (Is128Bit) {\n Result = _VMov(OpSize::i128Bit, Result);\n }\n StoreResultFPR(Op, Result);\n}\n\ntemplate void OpDispatchBuilder::VPMULHWOp(OpcodeArgs);\ntemplate void OpDispatchBuilder::VPMULHWOp(OpcodeArgs);\n\nRef OpDispatchBuilder::PMULHRSWOpImpl(OpSize Size, Ref Src1, Ref Src2) {\n Ref Res {};\n if (Size == OpSize::i64Bit) {\n // Implementation is more efficient for 8byte registers\n Res = _VSMull(Size << 1, OpSize::i16Bit, Src1, Src2);\n Res = _VSShrI(Size << 1, OpSize::i32Bit, Res, 14);\n auto OneVector = _VectorImm(Size << 1, OpSize::i32Bit, 1);\n Res = _VAdd(Size << 1, OpSize::i32Bit, Res, OneVector);\n return _VUShrNI(Size << 1, OpSize::i32Bit, Res, 1);\n } else {\n // 128-bit and 256-bit are less efficient\n Ref ResultLow;\n Ref ResultHigh;\n\n ResultLow = _VSMull(Size, OpSize::i16Bit, Src1, Src2);\n ResultHigh = _VSMull2(Size, OpSize::i16Bit, Src1, Src2);\n\n ResultLow = _VSShrI(Size, OpSize::i32Bit, ResultLow, 14);\n ResultHigh = _VSShrI(Size, OpSize::i32Bit, ResultHigh, 14);\n auto OneVector = _VectorImm(Size, OpSize::i32Bit, 1);\n\n ResultLow = _VAdd(Size, OpSize::i32Bit, ResultLow, OneVector);\n ResultHigh = _VAdd(Size, OpSize::i32Bit, ResultHigh, OneVector);\n\n // Combine the results\n Res = _VUShrNI(Size, OpSize::i32Bit, ResultLow, 1);\n return _VUShrNI2(Size, OpSize::i32Bit, Res, ResultHigh, 1);\n }\n}\n\nvoid OpDispatchBuilder::PMULHRSW(OpcodeArgs) {\n Ref Dest = LoadSourceFPR(Op, Op->Dest, Op->Flags);\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n Ref Result = PMULHRSWOpImpl(OpSizeFromSrc(Op), Dest, Src);\n\n StoreResultFPR(Op, Result);\n}\n\nvoid OpDispatchBuilder::VPMULHRSWOp(OpcodeArgs) {\n Ref Dest = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n Ref Src = LoadSourceFPR(Op, Op->Src[1], Op->Flags);\n Ref Result = PMULHRSWOpImpl(OpSizeFromSrc(Op), Dest, Src);\n\n StoreResultFPR(Op, Result);\n}\n\nRef OpDispatchBuilder::HSUBPOpImpl(OpSize SrcSize, IR::OpSize ElementSize, Ref Src1, Ref Src2) {\n auto Even = _VUnZip(SrcSize, ElementSize, Src1, Src2);\n auto Odd = _VUnZip2(SrcSize, ElementSize, Src1, Src2);\n return _VFSub(SrcSize, ElementSize, Even, Odd);\n}\n\ntemplate\nvoid OpDispatchBuilder::HSUBP(OpcodeArgs) {\n Ref Src1 = LoadSourceFPR(Op, Op->Dest, Op->Flags);\n Ref Src2 = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n Ref Result = HSUBPOpImpl(OpSizeFromSrc(Op), ElementSize, Src1, Src2);\n StoreResultFPR(Op, Result);\n}\n\ntemplate void OpDispatchBuilder::HSUBP(OpcodeArgs);\ntemplate void OpDispatchBuilder::HSUBP(OpcodeArgs);\n\nvoid OpDispatchBuilder::VHSUBPOp(OpcodeArgs, IR::OpSize ElementSize) {\n const auto DstSize = OpSizeFromDst(Op);\n const auto Is256Bit = DstSize == OpSize::i256Bit;\n\n Ref Src1 = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n Ref Src2 = LoadSourceFPR(Op, Op->Src[1], Op->Flags);\n\n Ref Result = HSUBPOpImpl(OpSizeFromSrc(Op), ElementSize, Src1, Src2);\n Ref Dest = Result;\n if (Is256Bit) {\n Dest = _VInsElement(DstSize, OpSize::i64Bit, 1, 2, Result, Result);\n Dest = _VInsElement(DstSize, OpSize::i64Bit, 2, 1, Dest, Result);\n }\n\n StoreResultFPR(Op, Dest);\n}\n\nRef OpDispatchBuilder::PHSUBOpImpl(OpSize Size, Ref Src1, Ref Src2, IR::OpSize ElementSize) {\n auto Even = _VUnZip(Size, ElementSize, Src1, Src2);\n auto Odd = _VUnZip2(Size, ElementSize, Src1, Src2);\n return _VSub(Size, ElementSize, Even, Odd);\n}\n\ntemplate\nvoid OpDispatchBuilder::PHSUB(OpcodeArgs) {\n Ref Src1 = LoadSourceFPR(Op, Op->Dest, Op->Flags);\n Ref Src2 = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n Ref Result = PHSUBOpImpl(OpSizeFromSrc(Op), Src1, Src2, ElementSize);\n StoreResultFPR(Op, Result);\n}\n\ntemplate void OpDispatchBuilder::PHSUB(OpcodeArgs);\ntemplate void OpDispatchBuilder::PHSUB(OpcodeArgs);\n\nvoid OpDispatchBuilder::VPHSUBOp(OpcodeArgs, IR::OpSize ElementSize) {\n const auto DstSize = OpSizeFromDst(Op);\n const auto Is256Bit = DstSize == OpSize::i256Bit;\n\n Ref Src1 = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n Ref Src2 = LoadSourceFPR(Op, Op->Src[1], Op->Flags);\n Ref Result = PHSUBOpImpl(OpSizeFromSrc(Op), Src1, Src2, ElementSize);\n if (Is256Bit) {\n Ref Inserted = _VInsElement(DstSize, OpSize::i64Bit, 1, 2, Result, Result);\n Result = _VInsElement(DstSize, OpSize::i64Bit, 2, 1, Inserted, Result);\n }\n StoreResultFPR(Op, Result);\n}\n\nRef OpDispatchBuilder::PHADDSOpImpl(OpSize Size, Ref Src1, Ref Src2) {\n const auto ElementSize = OpSize::i16Bit;\n\n auto Even = _VUnZip(Size, ElementSize, Src1, Src2);\n auto Odd = _VUnZip2(Size, ElementSize, Src1, Src2);\n\n // Saturate back down to the result\n return _VSQAdd(Size, ElementSize, Even, Odd);\n}\n\nvoid OpDispatchBuilder::PHADDS(OpcodeArgs) {\n Ref Src1 = LoadSourceFPR(Op, Op->Dest, Op->Flags);\n Ref Src2 = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n\n Ref Result = PHADDSOpImpl(OpSizeFromSrc(Op), Src1, Src2);\n StoreResultFPR(Op, Result);\n}\n\nvoid OpDispatchBuilder::VPHADDSWOp(OpcodeArgs) {\n const auto SrcSize = OpSizeFromSrc(Op);\n const auto Is256Bit = SrcSize == OpSize::i256Bit;\n\n Ref Src1 = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n Ref Src2 = LoadSourceFPR(Op, Op->Src[1], Op->Flags);\n\n Ref Result = PHADDSOpImpl(OpSizeFromSrc(Op), Src1, Src2);\n Ref Dest = Result;\n\n if (Is256Bit) {\n Dest = _VInsElement(SrcSize, OpSize::i64Bit, 1, 2, Result, Result);\n Dest = _VInsElement(SrcSize, OpSize::i64Bit, 2, 1, Dest, Result);\n }\n\n StoreResultFPR(Op, Dest);\n}\n\nRef OpDispatchBuilder::PHSUBSOpImpl(OpSize Size, Ref Src1, Ref Src2) {\n const auto ElementSize = OpSize::i16Bit;\n\n auto Even = _VUnZip(Size, ElementSize, Src1, Src2);\n auto Odd = _VUnZip2(Size, ElementSize, Src1, Src2);\n\n // Saturate back down to the result\n return _VSQSub(Size, ElementSize, Even, Odd);\n}\n\nvoid OpDispatchBuilder::PHSUBS(OpcodeArgs) {\n Ref Src1 = LoadSourceFPR(Op, Op->Dest, Op->Flags);\n Ref Src2 = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n Ref Result = PHSUBSOpImpl(OpSizeFromSrc(Op), Src1, Src2);\n StoreResultFPR(Op, Result);\n}\n\nvoid OpDispatchBuilder::VPHSUBSWOp(OpcodeArgs) {\n const auto DstSize = OpSizeFromDst(Op);\n const auto Is256Bit = DstSize == OpSize::i256Bit;\n\n Ref Src1 = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n Ref Src2 = LoadSourceFPR(Op, Op->Src[1], Op->Flags);\n Ref Result = PHSUBSOpImpl(OpSizeFromSrc(Op), Src1, Src2);\n\n Ref Dest = Result;\n if (Is256Bit) {\n Dest = _VInsElement(DstSize, OpSize::i64Bit, 1, 2, Result, Result);\n Dest = _VInsElement(DstSize, OpSize::i64Bit, 2, 1, Dest, Result);\n }\n\n StoreResultFPR(Op, Dest);\n}\n\nRef OpDispatchBuilder::PSADBWOpImpl(IR::OpSize Size, Ref Src1, Ref Src2) {\n // The documentation is actually incorrect in how this instruction operates\n // It strongly implies that the `abs(dest[i] - src[i])` operates in 8bit space\n // but it actually operates in more than 8bit space\n // This can be seen with `abs(0 - 0xFF)` returning a different result depending\n // on bit length\n const auto Is128Bit = Size == OpSize::i128Bit;\n\n if (Size == OpSize::i64Bit) {\n auto AbsResult = _VUABDL(Size << 1, OpSize::i8Bit, Src1, Src2);\n\n // Now vector-wide add the results for each\n return _VAddV(Size << 1, OpSize::i16Bit, AbsResult);\n }\n\n auto AbsResult_Low = _VUABDL(Size, OpSize::i8Bit, Src1, Src2);\n auto AbsResult_High = _VUABDL2(Size, OpSize::i8Bit, Src1, Src2);\n\n Ref Result_Low = _VAddV(OpSize::i128Bit, OpSize::i16Bit, AbsResult_Low);\n Ref Result_High = _VAddV(OpSize::i128Bit, OpSize::i16Bit, AbsResult_High);\n auto Low = _VZip(Size, OpSize::i64Bit, Result_Low, Result_High);\n\n if (Is128Bit) {\n return Low;\n }\n\n Ref HighSrc1 = _VDupElement(Size, OpSize::i128Bit, AbsResult_Low, 1);\n Ref HighSrc2 = _VDupElement(Size, OpSize::i128Bit, AbsResult_High, 1);\n\n Ref HighResult_Low = _VAddV(OpSize::i128Bit, OpSize::i16Bit, HighSrc1);\n Ref HighResult_High = _VAddV(OpSize::i128Bit, OpSize::i16Bit, HighSrc2);\n\n Ref High = _VInsElement(Size, OpSize::i64Bit, 1, 0, HighResult_Low, HighResult_High);\n Ref Full = _VInsElement(Size, OpSize::i128Bit, 1, 0, Low, High);\n\n Ref Tmp = _VInsElement(Size, OpSize::i64Bit, 2, 1, Full, Full);\n return _VInsElement(Size, OpSize::i64Bit, 1, 2, Tmp, Full);\n}\n\nvoid OpDispatchBuilder::PSADBW(OpcodeArgs) {\n const auto Size = OpSizeFromSrc(Op);\n\n Ref Src1 = LoadSourceFPR(Op, Op->Dest, Op->Flags);\n Ref Src2 = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n\n Ref Result = PSADBWOpImpl(Size, Src1, Src2);\n StoreResultFPR(Op, Result);\n}\n\nvoid OpDispatchBuilder::VPSADBWOp(OpcodeArgs) {\n const auto Size = OpSizeFromSrc(Op);\n\n Ref Src1 = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n Ref Src2 = LoadSourceFPR(Op, Op->Src[1], Op->Flags);\n\n Ref Result = PSADBWOpImpl(Size, Src1, Src2);\n StoreResultFPR(Op, Result);\n}\n\nRef OpDispatchBuilder::ExtendVectorElementsImpl(OpcodeArgs, IR::OpSize ElementSize, IR::OpSize DstElementSize, bool Signed) {\n const auto DstSize = OpSizeFromDst(Op);\n\n const auto GetSrc = [&] {\n if (Op->Src[0].IsGPR()) {\n return LoadSourceFPR_WithOpSize(Op, Op->Src[0], DstSize, Op->Flags);\n } else {\n // For memory operands the 256-bit variant loads twice the size specified in the table.\n const auto Is256Bit = DstSize == OpSize::i256Bit;\n const auto SrcSize = OpSizeFromSrc(Op);\n const auto LoadSize = Is256Bit ? IR::SizeToOpSize(IR::OpSizeToSize(SrcSize) * 2) : SrcSize;\n\n return LoadSourceFPR_WithOpSize(Op, Op->Src[0], LoadSize, Op->Flags);\n }\n };\n\n Ref Src = GetSrc();\n Ref Result {Src};\n\n for (auto CurrentElementSize = ElementSize; CurrentElementSize != DstElementSize; CurrentElementSize = CurrentElementSize << 1) {\n if (Signed) {\n Result = _VSXTL(DstSize, CurrentElementSize, Result);\n } else {\n Result = _VUXTL(DstSize, CurrentElementSize, Result);\n }\n }\n\n return Result;\n}\n\ntemplate\nvoid OpDispatchBuilder::ExtendVectorElements(OpcodeArgs) {\n Ref Result = ExtendVectorElementsImpl(Op, ElementSize, DstElementSize, Signed);\n StoreResultFPR(Op, Result);\n}\n\ntemplate void OpDispatchBuilder::ExtendVectorElements(OpcodeArgs);\ntemplate void OpDispatchBuilder::ExtendVectorElements(OpcodeArgs);\ntemplate void OpDispatchBuilder::ExtendVectorElements(OpcodeArgs);\ntemplate void OpDispatchBuilder::ExtendVectorElements(OpcodeArgs);\ntemplate void OpDispatchBuilder::ExtendVectorElements(OpcodeArgs);\ntemplate void OpDispatchBuilder::ExtendVectorElements(OpcodeArgs);\n\ntemplate void OpDispatchBuilder::ExtendVectorElements(OpcodeArgs);\ntemplate void OpDispatchBuilder::ExtendVectorElements(OpcodeArgs);\ntemplate void OpDispatchBuilder::ExtendVectorElements(OpcodeArgs);\ntemplate void OpDispatchBuilder::ExtendVectorElements(OpcodeArgs);\ntemplate void OpDispatchBuilder::ExtendVectorElements(OpcodeArgs);\ntemplate void OpDispatchBuilder::ExtendVectorElements(OpcodeArgs);\n\nRef OpDispatchBuilder::VectorRoundImpl(OpSize Size, IR::OpSize ElementSize, Ref Src, uint64_t Mode) {\n return _Vector_FToI(Size, ElementSize, Src, TranslateRoundType(Mode));\n}\n\ntemplate\nvoid OpDispatchBuilder::VectorRound(OpcodeArgs) {\n // No need to zero extend the vector in the event we have a\n // scalar source, especially since it's only inserted into another vector.\n const auto SrcSize = OpSizeFromSrc(Op);\n Ref Src = LoadSourceFPR_WithOpSize(Op, Op->Src[0], SrcSize, Op->Flags);\n\n const uint64_t Mode = Op->Src[1].Literal();\n Src = VectorRoundImpl(OpSizeFromDst(Op), ElementSize, Src, Mode);\n\n StoreResultFPR(Op, Src);\n}\n\ntemplate void OpDispatchBuilder::VectorRound(OpcodeArgs);\ntemplate void OpDispatchBuilder::VectorRound(OpcodeArgs);\n\ntemplate\nvoid OpDispatchBuilder::AVXVectorRound(OpcodeArgs) {\n const auto Mode = Op->Src[1].Literal();\n\n // No need to zero extend the vector in the event we have a\n // scalar source, especially since it's only inserted into another vector.\n const auto SrcSize = OpSizeFromSrc(Op);\n\n Ref Src = LoadSourceFPR_WithOpSize(Op, Op->Src[0], SrcSize, Op->Flags);\n Ref Result = VectorRoundImpl(OpSizeFromDst(Op), ElementSize, Src, Mode);\n\n StoreResultFPR(Op, Result);\n}\n\ntemplate void OpDispatchBuilder::AVXVectorRound(OpcodeArgs);\ntemplate void OpDispatchBuilder::AVXVectorRound(OpcodeArgs);\n\nRef OpDispatchBuilder::VectorBlend(OpSize Size, IR::OpSize ElementSize, Ref Src1, Ref Src2, uint8_t Selector) {\n if (ElementSize == OpSize::i32Bit) {\n Selector &= 0b1111;\n switch (Selector) {\n case 0b0000:\n // Dest[31:0] = Src1[31:0]\n // Dest[63:32] = Src1[63:32]\n // Dest[95:64] = Src1[95:64]\n // Dest[127:96] = Src1[127:96]\n // Copy\n return Src1;\n case 0b0001:\n // Dest[31:0] = Src2[31:0]\n // Dest[63:32] = Src1[63:32]\n // Dest[95:64] = Src1[95:64]\n // Dest[127:96] = Src1[127:96]\n return _VInsElement(Size, ElementSize, 0, 0, Src1, Src2);\n case 0b0010:\n // Dest[31:0] = Src1[31:0]\n // Dest[63:32] = Src2[63:32]\n // Dest[95:64] = Src1[95:64]\n // Dest[127:96] = Src1[127:96]\n return _VInsElement(Size, ElementSize, 1, 1, Src1, Src2);\n case 0b0011:\n // Dest[31:0] = Src2[31:0]\n // Dest[63:32] = Src2[63:32]\n // Dest[95:64] = Src1[95:64]\n // Dest[127:96] = Src1[127:96]\n return _VInsElement(Size, OpSize::i64Bit, 0, 0, Src1, Src2);\n case 0b0100:\n // Dest[31:0] = Src1[31:0]\n // Dest[63:32] = Src1[63:32]\n // Dest[95:64] = Src2[95:64]\n // Dest[127:96] = Src1[127:96]\n return _VInsElement(Size, ElementSize, 2, 2, Src1, Src2);\n case 0b0101: {\n // Dest[31:0] = Src2[31:0]\n // Dest[63:32] = Src1[63:32]\n // Dest[95:64] = Src2[95:64]\n // Dest[127:96] = Src1[127:96]\n // Rotate the elements of the incoming source so they end up in the correct location.\n // Then trn2 keeps the destination results in the expected location.\n auto Temp = _VRev64(Size, OpSize::i32Bit, Src2);\n return _VTrn2(Size, ElementSize, Temp, Src1);\n }\n case 0b0110: {\n // Dest[31:0] = Src1[31:0]\n // Dest[63:32] = Src2[63:32]\n // Dest[95:64] = Src2[95:64]\n // Dest[127:96] = Src1[127:96]\n auto ConstantSwizzle = LoadAndCacheNamedVectorConstant(Size, FEXCore::IR::NamedVectorConstant::NAMED_VECTOR_BLENDPS_0110B);\n return _VTBX1(Size, Src1, Src2, ConstantSwizzle);\n }\n case 0b0111: {\n // Dest[31:0] = Src2[31:0]\n // Dest[63:32] = Src2[63:32]\n // Dest[95:64] = Src2[95:64]\n // Dest[127:96] = Src1[127:96]\n auto ConstantSwizzle = LoadAndCacheNamedVectorConstant(Size, FEXCore::IR::NamedVectorConstant::NAMED_VECTOR_BLENDPS_0111B);\n return _VTBX1(Size, Src1, Src2, ConstantSwizzle);\n }\n case 0b1000:\n // Dest[31:0] = Src1[31:0]\n // Dest[63:32] = Src1[63:32]\n // Dest[95:64] = Src1[95:64]\n // Dest[127:96] = Src2[127:96]\n return _VInsElement(Size, ElementSize, 3, 3, Src1, Src2);\n case 0b1001: {\n // Dest[31:0] = Src2[31:0]\n // Dest[63:32] = Src1[63:32]\n // Dest[95:64] = Src1[95:64]\n // Dest[127:96] = Src2[127:96]\n auto ConstantSwizzle = LoadAndCacheNamedVectorConstant(Size, FEXCore::IR::NamedVectorConstant::NAMED_VECTOR_BLENDPS_1001B);\n return _VTBX1(Size, Src1, Src2, ConstantSwizzle);\n }\n case 0b1010: {\n // Dest[31:0] = Src1[31:0]\n // Dest[63:32] = Src2[63:32]\n // Dest[95:64] = Src1[95:64]\n // Dest[127:96] = Src2[127:96]\n // Rotate the elements of the incoming destination so they end up in the correct location.\n // Then trn2 keeps the source results in the expected location.\n auto Temp = _VRev64(Size, OpSize::i32Bit, Src1);\n return _VTrn2(Size, ElementSize, Temp, Src2);\n }\n case 0b1011: {\n // Dest[31:0] = Src2[31:0]\n // Dest[63:32] = Src2[63:32]\n // Dest[95:64] = Src1[95:64]\n // Dest[127:96] = Src2[127:96]\n auto ConstantSwizzle = LoadAndCacheNamedVectorConstant(Size, FEXCore::IR::NamedVectorConstant::NAMED_VECTOR_BLENDPS_1011B);\n return _VTBX1(Size, Src1, Src2, ConstantSwizzle);\n }\n case 0b1100:\n // Dest[31:0] = Src1[31:0]\n // Dest[63:32] = Src1[63:32]\n // Dest[95:64] = Src2[95:64]\n // Dest[127:96] = Src2[127:96]\n return _VInsElement(Size, OpSize::i64Bit, 1, 1, Src1, Src2);\n case 0b1101: {\n // Dest[31:0] = Src2[31:0]\n // Dest[63:32] = Src1[63:32]\n // Dest[95:64] = Src2[95:64]\n // Dest[127:96] = Src2[127:96]\n auto ConstantSwizzle = LoadAndCacheNamedVectorConstant(Size, FEXCore::IR::NamedVectorConstant::NAMED_VECTOR_BLENDPS_1101B);\n return _VTBX1(Size, Src1, Src2, ConstantSwizzle);\n }\n case 0b1110: {\n // Dest[31:0] = Src1[31:0]\n // Dest[63:32] = Src2[63:32]\n // Dest[95:64] = Src2[95:64]\n // Dest[127:96] = Src2[127:96]\n auto ConstantSwizzle = LoadAndCacheNamedVectorConstant(Size, FEXCore::IR::NamedVectorConstant::NAMED_VECTOR_BLENDPS_1110B);\n return _VTBX1(Size, Src1, Src2, ConstantSwizzle);\n }\n case 0b1111:\n // Dest[31:0] = Src2[31:0]\n // Dest[63:32] = Src2[63:32]\n // Dest[95:64] = Src2[95:64]\n // Dest[127:96] = Src2[127:96]\n // Copy\n return Src2;\n default: break;\n }\n } else if (ElementSize == OpSize::i64Bit) {\n Selector &= 0b11;\n switch (Selector) {\n case 0b00:\n // No-op\n return Src1;\n case 0b01:\n // Dest[63:0] = Src2[63:0]\n // Dest[127:64] = Src1[127:64]\n return _VInsElement(Size, ElementSize, 0, 0, Src1, Src2);\n case 0b10:\n // Dest[63:0] = Src1[63:0]\n // Dest[127:64] = Src2[127:64]\n return _VInsElement(Size, ElementSize, 1, 1, Src1, Src2);\n case 0b11:\n // Copy\n return Src2;\n }\n } else {\n ///< Zero instruction copies\n switch (Selector) {\n case 0b0000'0000: return Src1;\n case 0b1111'1111: return Src2;\n default: break;\n }\n\n ///< Single instruction implementation\n switch (Selector) {\n case 0b0000'0001:\n case 0b0000'0010:\n case 0b0000'0100:\n case 0b0000'1000:\n case 0b0001'0000:\n case 0b0010'0000:\n case 0b0100'0000:\n case 0b1000'0000: {\n // Single 16-bit element insert.\n const auto Element = FEXCore::ilog2(Selector);\n return _VInsElement(Size, ElementSize, Element, Element, Src1, Src2);\n }\n case 0b1111'1110:\n case 0b1111'1101:\n case 0b1111'1011:\n case 0b1111'0111:\n case 0b1110'1111:\n case 0b1101'1111:\n case 0b1011'1111:\n case 0b0111'1111: {\n // Single 16-bit element insert, inverted\n uint8_t SelectorInvert = ~Selector;\n const auto Element = FEXCore::ilog2(SelectorInvert);\n return _VInsElement(Size, ElementSize, Element, Element, Src2, Src1);\n }\n case 0b0000'0011:\n case 0b0000'1100:\n case 0b0011'0000:\n case 0b1100'0000: {\n // Single 32-bit element insert.\n const auto Element = std::countr_zero(Selector) / 2;\n return _VInsElement(Size, OpSize::i32Bit, Element, Element, Src1, Src2);\n }\n case 0b1111'1100:\n case 0b1111'0011:\n case 0b1100'1111:\n case 0b0011'1111: {\n // Single 32-bit element insert, inverted\n uint8_t SelectorInvert = ~Selector;\n const auto Element = std::countr_zero(SelectorInvert) / 2;\n return _VInsElement(Size, OpSize::i32Bit, Element, Element, Src2, Src1);\n }\n case 0b0000'1111:\n case 0b1111'0000: {\n // Single 64-bit element insert.\n const auto Element = std::countr_zero(Selector) / 4;\n return _VInsElement(Size, OpSize::i64Bit, Element, Element, Src1, Src2);\n }\n default: break;\n }\n\n ///< Two instruction implementation\n switch (Selector) {\n ///< Fancy double VExtr\n case 0b0'0'0'0'0'1'1'1: {\n auto Tmp = _VExtr(OpSize::i128Bit, OpSize::i8Bit, Src2, Src1, 6);\n return _VExtr(OpSize::i128Bit, OpSize::i8Bit, Tmp, Tmp, 10);\n }\n case 0b0'0'0'1'1'1'1'1: {\n auto Tmp = _VExtr(OpSize::i128Bit, OpSize::i8Bit, Src2, Src1, 10);\n return _VExtr(OpSize::i128Bit, OpSize::i8Bit, Tmp, Tmp, 6);\n }\n case 0b1'1'1'0'0'0'0'0: {\n auto Tmp = _VExtr(OpSize::i128Bit, OpSize::i8Bit, Src1, Src2, 10);\n return _VExtr(OpSize::i128Bit, OpSize::i8Bit, Tmp, Tmp, 6);\n }\n case 0b1'1'1'1'1'0'0'0: {\n auto Tmp = _VExtr(OpSize::i128Bit, OpSize::i8Bit, Src1, Src2, 6);\n return _VExtr(OpSize::i128Bit, OpSize::i8Bit, Tmp, Tmp, 10);\n }\n default: break;\n }\n\n // TODO: There are some of these swizzles that can be more optimal.\n // NamedConstant + VTBX1 is quite quick already.\n // Implement more if it becomes relevant.\n auto ConstantSwizzle =\n LoadAndCacheIndexedNamedVectorConstant(Size, FEXCore::IR::IndexNamedVectorConstant::INDEXED_NAMED_VECTOR_PBLENDW, Selector * 16);\n return _VTBX1(Size, Src1, Src2, ConstantSwizzle);\n }\n\n FEX_UNREACHABLE;\n}\n\ntemplate\nvoid OpDispatchBuilder::VectorBlend(OpcodeArgs) {\n uint8_t Select = Op->Src[1].Literal();\n\n Ref Dest = LoadSourceFPR(Op, Op->Dest, Op->Flags);\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n Dest = VectorBlend(OpSize::i128Bit, ElementSize, Dest, Src, Select);\n StoreResultFPR(Op, Dest);\n}\n\ntemplate void OpDispatchBuilder::VectorBlend(OpcodeArgs);\ntemplate void OpDispatchBuilder::VectorBlend(OpcodeArgs);\ntemplate void OpDispatchBuilder::VectorBlend(OpcodeArgs);\n\nvoid OpDispatchBuilder::VectorVariableBlend(OpcodeArgs, IR::OpSize ElementSize) {\n const auto Size = OpSizeFromSrc(Op);\n\n Ref Dest = LoadSourceFPR(Op, Op->Dest, Op->Flags);\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n\n auto Mask = LoadXMMRegister(0);\n\n // Each element is selected by the high bit of that element size\n // Dest[ElementIdx] = Xmm0[ElementIndex][HighBit] ? Src : Dest;\n //\n // To emulate this on AArch64\n // Arithmetic shift right by the element size, then use BSL to select the registers\n Mask = _VSShrI(Size, ElementSize, Mask, IR::OpSizeAsBits(ElementSize) - 1);\n\n auto Result = _VBSL(Size, Mask, Src, Dest);\n\n StoreResultFPR(Op, Result);\n}\n\nvoid OpDispatchBuilder::AVXVectorVariableBlend(OpcodeArgs, IR::OpSize ElementSize) {\n const auto SrcSize = OpSizeFromSrc(Op);\n const auto ElementSizeBits = IR::OpSizeAsBits(ElementSize);\n\n Ref Src1 = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n Ref Src2 = LoadSourceFPR(Op, Op->Src[1], Op->Flags);\n\n // Mask register is encoded within bits [7:4] of the selector\n const auto Src3Selector = Op->Src[2].Literal();\n Ref Mask = LoadXMMRegister((Src3Selector >> 4) & 0b1111);\n\n Ref Shifted = _VSShrI(SrcSize, ElementSize, Mask, ElementSizeBits - 1);\n Ref Result = _VBSL(SrcSize, Shifted, Src2, Src1);\n StoreResultFPR(Op, Result);\n}\n\nvoid OpDispatchBuilder::PTestOpImpl(OpSize Size, Ref Dest, Ref Src) {\n Ref Test1 = _VAnd(Size, OpSize::i8Bit, Dest, Src);\n Ref Test2 = _VAndn(Size, OpSize::i8Bit, Src, Dest);\n\n // Element size must be less than 32-bit for the sign bit tricks.\n Test1 = _VUMaxV(Size, OpSize::i16Bit, Test1);\n Test2 = _VUMaxV(Size, OpSize::i16Bit, Test2);\n\n Test1 = _VExtractToGPR(Size, OpSize::i16Bit, Test1, 0);\n Test2 = _VExtractToGPR(Size, OpSize::i16Bit, Test2, 0);\n\n Test2 = To01(OpSize::i64Bit, Test2);\n\n // Careful, these flags are different between {V,}PTEST and VTESTP{S,D}\n // Set ZF according to Test1. SF will be zeroed since we do a 32-bit test on\n // the results of a 16-bit value from the UMaxV, so the 32-bit sign bit is\n // cleared even if the 16-bit scalars were negative.\n SetNZ_ZeroCV(OpSize::i32Bit, Test1);\n SetCFInverted(Test2);\n ZeroPF_AF();\n}\n\nvoid OpDispatchBuilder::PTestOp(OpcodeArgs) {\n Ref Dest = LoadSourceFPR(Op, Op->Dest, Op->Flags);\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n\n PTestOpImpl(OpSizeFromSrc(Op), Dest, Src);\n}\n\nvoid OpDispatchBuilder::VTESTOpImpl(OpSize SrcSize, IR::OpSize ElementSize, Ref Src1, Ref Src2) {\n LOGMAN_THROW_A_FMT(ElementSize >= IR::OpSize::i8Bit && ElementSize <= IR::OpSize::i64Bit, \"Invalid size\");\n const auto ElementSizeInBits = IR::OpSizeAsBits(ElementSize);\n const auto MaskConstant = uint64_t {1} << (ElementSizeInBits - 1);\n\n Ref Mask = _VDupFromGPR(SrcSize, ElementSize, Constant(MaskConstant));\n\n Ref AndTest = _VAnd(SrcSize, OpSize::i8Bit, Src2, Src1);\n Ref AndNotTest = _VAndn(SrcSize, OpSize::i8Bit, Src2, Src1);\n\n Ref MaskedAnd = _VAnd(SrcSize, OpSize::i8Bit, AndTest, Mask);\n Ref MaskedAndNot = _VAnd(SrcSize, OpSize::i8Bit, AndNotTest, Mask);\n\n Ref MaxAnd = _VUMaxV(SrcSize, OpSize::i16Bit, MaskedAnd);\n Ref MaxAndNot = _VUMaxV(SrcSize, OpSize::i16Bit, MaskedAndNot);\n\n Ref AndGPR = _VExtractToGPR(SrcSize, OpSize::i16Bit, MaxAnd, 0);\n Ref AndNotGPR = _VExtractToGPR(SrcSize, OpSize::i16Bit, MaxAndNot, 0);\n\n Ref CFInv = To01(OpSize::i64Bit, AndNotGPR);\n\n // As in PTest, this sets Z appropriately while zeroing the rest of NZCV.\n SetNZ_ZeroCV(OpSize::i32Bit, AndGPR);\n SetCFInverted(CFInv);\n ZeroPF_AF();\n}\n\ntemplate\nvoid OpDispatchBuilder::VTESTPOp(OpcodeArgs) {\n Ref Src1 = LoadSourceFPR(Op, Op->Dest, Op->Flags);\n Ref Src2 = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n\n VTESTOpImpl(OpSizeFromSrc(Op), ElementSize, Src1, Src2);\n}\ntemplate void OpDispatchBuilder::VTESTPOp(OpcodeArgs);\ntemplate void OpDispatchBuilder::VTESTPOp(OpcodeArgs);\n\nRef OpDispatchBuilder::PHMINPOSUWOpImpl(OpcodeArgs) {\n const auto Size = OpSizeFromSrc(Op);\n\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n\n // Setup a vector swizzle\n // Initially load a 64-bit mask of immediates\n // Then zero-extend that to 128-bit mask with the immediates in the lower 16-bits of each element\n auto ConstantSwizzle = LoadAndCacheNamedVectorConstant(Size, FEXCore::IR::NamedVectorConstant::NAMED_VECTOR_INCREMENTAL_U16_INDEX);\n\n // We now need to zip the vector sources together to become two uint32x4_t vectors\n // Upper:\n // [127:96]: ([127:112] << 16) | (7)\n // [95:64] : ([111:96] << 16) | (6)\n // [63:32] : ([95:80] << 16) | (5)\n // [31:0] : ([79:64] << 16) | (4)\n\n // Lower:\n // [127:96]: ([63:48] << 16) | (3)\n // [95:64] : ([47:32] << 16) | (2)\n // [63:32] : ([31:16] << 16) | (1)\n // [31:0] : ([15:0] << 16) | (0)\n\n auto ZipLower = _VZip(Size, OpSize::i16Bit, ConstantSwizzle, Src);\n auto ZipUpper = _VZip2(Size, OpSize::i16Bit, ConstantSwizzle, Src);\n // The elements are now 32-bit between two vectors.\n auto MinBetween = _VUMin(Size, OpSize::i32Bit, ZipLower, ZipUpper);\n\n // Now do a horizontal vector minimum\n auto Min = _VUMinV(Size, OpSize::i32Bit, MinBetween);\n\n // We now have a value in the bottom 32-bits in the order of:\n // [31:0]: (Src[] << 16) | \n // This instruction wants it in the form of:\n // [31:0]: ( << 16) | Src[]\n // Rev32 does this for us\n return _VRev32(Size, OpSize::i16Bit, Min);\n}\n\nvoid OpDispatchBuilder::PHMINPOSUWOp(OpcodeArgs) {\n Ref Result = PHMINPOSUWOpImpl(Op);\n StoreResultFPR(Op, Result);\n}\n\nRef OpDispatchBuilder::DPPOpImpl(IR::OpSize DstSize, Ref Src1, Ref Src2, uint8_t Mask, IR::OpSize ElementSize) {\n const auto SizeMask = [ElementSize]() {\n if (ElementSize == OpSize::i32Bit) {\n return 0b1111;\n }\n return 0b11;\n }();\n\n const uint8_t SrcMask = (Mask >> 4) & SizeMask;\n const uint8_t DstMask = Mask & SizeMask;\n\n const auto NamedIndexMask = [ElementSize]() {\n if (ElementSize == OpSize::i32Bit) {\n return FEXCore::IR::IndexNamedVectorConstant::INDEXED_NAMED_VECTOR_DPPS_MASK;\n }\n\n return FEXCore::IR::IndexNamedVectorConstant::INDEXED_NAMED_VECTOR_DPPD_MASK;\n }();\n\n Ref ZeroVec = LoadZeroVector(DstSize);\n if (SrcMask == 0 || DstMask == 0) {\n // What are you even doing here? Go away.\n return ZeroVec;\n }\n\n // First step is to do an FMUL\n Ref Temp = _VFMul(DstSize, ElementSize, Src1, Src2);\n\n // Now mask results based on IndexMask.\n if (SrcMask != SizeMask) {\n auto InputMask = LoadAndCacheIndexedNamedVectorConstant(DstSize, NamedIndexMask, SrcMask * 16);\n Temp = _VAnd(DstSize, ElementSize, Temp, InputMask);\n }\n\n // Now due a float reduction\n Temp = _VFAddV(DstSize, ElementSize, Temp);\n\n // Now using the destination mask we choose where the result ends up\n // It can duplicate and zero results\n if (ElementSize == OpSize::i64Bit) {\n switch (DstMask) {\n case 0b01:\n // Dest[63:0] = Result\n // Dest[127:64] = Zero\n return _VZip(DstSize, ElementSize, Temp, ZeroVec);\n case 0b10:\n // Dest[63:0] = Zero\n // Dest[127:64] = Result\n return _VZip(DstSize, ElementSize, ZeroVec, Temp);\n case 0b11:\n // Broadcast\n // Dest[63:0] = Result\n // Dest[127:64] = Result\n return _VDupElement(DstSize, ElementSize, Temp, 0);\n case 0:\n default: LOGMAN_MSG_A_FMT(\"Unsupported\");\n }\n } else {\n auto BadPath = [&]() {\n Ref Result = ZeroVec;\n\n for (size_t i = 0; i < IR::NumElements(DstSize, ElementSize); ++i) {\n const auto Bit = 1U << (i % 4);\n\n if ((DstMask & Bit) != 0) {\n Result = _VInsElement(DstSize, ElementSize, i, 0, Result, Temp);\n }\n }\n\n return Result;\n };\n switch (DstMask) {\n case 0b0001:\n // Dest[31:0] = Result\n // Dest[63:32] = Zero\n // Dest[95:64] = Zero\n // Dest[127:96] = Zero\n return _VZip(DstSize, ElementSize, Temp, ZeroVec);\n case 0b0010:\n // Dest[31:0] = Zero\n // Dest[63:32] = Result\n // Dest[95:64] = Zero\n // Dest[127:96] = Zero\n return _VZip(DstSize >> 1, ElementSize, ZeroVec, Temp);\n case 0b0011:\n // Dest[31:0] = Result\n // Dest[63:32] = Result\n // Dest[95:64] = Zero\n // Dest[127:96] = Zero\n return _VDupElement(DstSize >> 1, ElementSize, Temp, 0);\n case 0b0100:\n // Dest[31:0] = Zero\n // Dest[63:32] = Zero\n // Dest[95:64] = Result\n // Dest[127:96] = Zero\n return _VZip(DstSize, OpSize::i64Bit, ZeroVec, Temp);\n case 0b0101:\n // Dest[31:0] = Result\n // Dest[63:32] = Zero\n // Dest[95:64] = Result\n // Dest[127:96] = Zero\n return _VZip(DstSize, OpSize::i64Bit, Temp, Temp);\n case 0b0110:\n // Dest[31:0] = Zero\n // Dest[63:32] = Result\n // Dest[95:64] = Result\n // Dest[127:96] = Zero\n return BadPath();\n case 0b0111:\n // Dest[31:0] = Result\n // Dest[63:32] = Result\n // Dest[95:64] = Result\n // Dest[127:96] = Zero\n Temp = _VDupElement(DstSize, ElementSize, Temp, 0);\n return _VInsElement(DstSize, ElementSize, 3, 0, Temp, ZeroVec);\n case 0b1000:\n // Dest[31:0] = Zero\n // Dest[63:32] = Zero\n // Dest[95:64] = Zero\n // Dest[127:96] = Result\n return _VExtr(DstSize, OpSize::i8Bit, Temp, ZeroVec, 4);\n case 0b1001:\n // Dest[31:0] = Result\n // Dest[63:32] = Zero\n // Dest[95:64] = Zero\n // Dest[127:96] = Result\n return BadPath();\n case 0b1010:\n // Dest[31:0] = Zero\n // Dest[63:32] = Result\n // Dest[95:64] = Zero\n // Dest[127:96] = Result\n Temp = _VDupElement(DstSize, ElementSize, Temp, 0);\n return _VZip(DstSize, OpSize::i32Bit, ZeroVec, Temp);\n case 0b1011:\n // Dest[31:0] = Result\n // Dest[63:32] = Result\n // Dest[95:64] = Zero\n // Dest[127:96] = Result\n Temp = _VDupElement(DstSize, ElementSize, Temp, 0);\n return _VInsElement(DstSize, ElementSize, 2, 0, Temp, ZeroVec);\n case 0b1100:\n // Dest[31:0] = Zero\n // Dest[63:32] = Zero\n // Dest[95:64] = Result\n // Dest[127:96] = Result\n Temp = _VDupElement(DstSize, ElementSize, Temp, 0);\n return _VZip(DstSize, OpSize::i64Bit, ZeroVec, Temp);\n case 0b1101:\n // Dest[31:0] = Result\n // Dest[63:32] = Zero\n // Dest[95:64] = Result\n // Dest[127:96] = Result\n Temp = _VDupElement(DstSize, ElementSize, Temp, 0);\n return _VInsElement(DstSize, ElementSize, 1, 0, Temp, ZeroVec);\n case 0b1110:\n // Dest[31:0] = Zero\n // Dest[63:32] = Result\n // Dest[95:64] = Result\n // Dest[127:96] = Result\n Temp = _VDupElement(DstSize, ElementSize, Temp, 0);\n return _VInsElement(DstSize, ElementSize, 0, 0, Temp, ZeroVec);\n case 0b1111:\n // Broadcast\n // Dest[31:0] = Result\n // Dest[63:32] = Zero\n // Dest[95:64] = Zero\n // Dest[127:96] = Zero\n return _VDupElement(DstSize, ElementSize, Temp, 0);\n case 0:\n default: LOGMAN_MSG_A_FMT(\"Unsupported\");\n }\n }\n FEX_UNREACHABLE;\n}\n\ntemplate\nvoid OpDispatchBuilder::DPPOp(OpcodeArgs) {\n\n Ref Dest = LoadSourceFPR(Op, Op->Dest, Op->Flags);\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n\n Ref Result = DPPOpImpl(OpSizeFromDst(Op), Dest, Src, Op->Src[1].Literal(), ElementSize);\n StoreResultFPR(Op, Result);\n}\n\ntemplate void OpDispatchBuilder::DPPOp(OpcodeArgs);\ntemplate void OpDispatchBuilder::DPPOp(OpcodeArgs);\n\nRef OpDispatchBuilder::VDPPSOpImpl(OpcodeArgs, const X86Tables::DecodedOperand& Src1, const X86Tables::DecodedOperand& Src2,\n const X86Tables::DecodedOperand& Imm) {\n constexpr auto ElementSize = OpSize::i32Bit;\n const uint8_t Mask = Imm.Literal();\n const uint8_t SrcMask = Mask >> 4;\n const uint8_t DstMask = Mask & 0xF;\n\n const auto DstSize = OpSizeFromDst(Op);\n\n Ref Src1V = LoadSourceFPR(Op, Src1, Op->Flags);\n Ref Src2V = LoadSourceFPR(Op, Src2, Op->Flags);\n\n Ref ZeroVec = LoadZeroVector(DstSize);\n\n // First step is to do an FMUL\n Ref Temp = _VFMul(DstSize, ElementSize, Src1V, Src2V);\n\n // Now we zero out elements based on src mask\n for (size_t i = 0; i < IR::NumElements(DstSize, ElementSize); ++i) {\n const auto Bit = 1U << (i % 4);\n\n if ((SrcMask & Bit) == 0) {\n Temp = _VInsElement(DstSize, ElementSize, i, 0, Temp, ZeroVec);\n }\n }\n\n // Now we need to do a horizontal add of the elements\n // We only have pairwise float add so this needs to be done in steps\n Temp = _VFAddP(DstSize, ElementSize, Temp, ZeroVec);\n\n if (ElementSize == OpSize::i32Bit) {\n // For 32-bit float we need one more step to add all four results together\n Temp = _VFAddP(DstSize, ElementSize, Temp, ZeroVec);\n }\n\n // Now using the destination mask we choose where the result ends up\n // It can duplicate and zero results\n Ref Result = ZeroVec;\n\n for (size_t i = 0; i < IR::NumElements(DstSize, ElementSize); ++i) {\n const auto Bit = 1U << (i % 4);\n\n if ((DstMask & Bit) != 0) {\n Result = _VInsElement(DstSize, ElementSize, i, 0, Result, Temp);\n }\n }\n\n return Result;\n}\n\ntemplate\nvoid OpDispatchBuilder::VDPPOp(OpcodeArgs) {\n const auto DstSize = OpSizeFromDst(Op);\n\n Ref Result {};\n if (ElementSize == OpSize::i32Bit && DstSize == OpSize::i256Bit) {\n // 256-bit DPPS isn't handled by the 128-bit solution.\n Result = VDPPSOpImpl(Op, Op->Src[0], Op->Src[1], Op->Src[2]);\n } else {\n Ref Src1 = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n Ref Src2 = LoadSourceFPR(Op, Op->Src[1], Op->Flags);\n\n Result = DPPOpImpl(DstSize, Src1, Src2, Op->Src[2].Literal(), ElementSize);\n }\n\n // We don't need to emit a _VMov to clear the upper lane, since DPPOpImpl uses a zero vector\n // to construct the results, so the upper lane will always be cleared for the 128-bit version.\n StoreResultFPR(Op, Result);\n}\n\ntemplate void OpDispatchBuilder::VDPPOp(OpcodeArgs);\ntemplate void OpDispatchBuilder::VDPPOp(OpcodeArgs);\n\nRef OpDispatchBuilder::MPSADBWOpImpl(IR::OpSize SrcSize, Ref Src1, Ref Src2, uint8_t Select) {\n const auto LaneHelper = [&, this](uint32_t Selector_Src1, uint32_t Selector_Src2, Ref Src1, Ref Src2) {\n // Src2 will grab a 32bit element and duplicate it across the 128bits\n Ref DupSrc = _VDupElement(OpSize::i128Bit, OpSize::i32Bit, Src2, Selector_Src2);\n\n // Src1/Dest needs a bunch of magic\n\n // Shift right by selected bytes\n // This will give us Dest[15:0], and Dest[79:64]\n Ref Dest1 = _VExtr(OpSize::i128Bit, OpSize::i8Bit, Src1, Src1, Selector_Src1 + 0);\n // This will give us Dest[31:16], and Dest[95:80]\n Ref Dest2 = _VExtr(OpSize::i128Bit, OpSize::i8Bit, Src1, Src1, Selector_Src1 + 1);\n // This will give us Dest[47:32], and Dest[111:96]\n Ref Dest3 = _VExtr(OpSize::i128Bit, OpSize::i8Bit, Src1, Src1, Selector_Src1 + 2);\n // This will give us Dest[63:48], and Dest[127:112]\n Ref Dest4 = _VExtr(OpSize::i128Bit, OpSize::i8Bit, Src1, Src1, Selector_Src1 + 3);\n\n // For each shifted section, we now have two 32-bit values per vector that can be used\n // Dest1.S[0] and Dest1.S[1] = Bytes - 0,1,2,3:4,5,6,7\n // Dest2.S[0] and Dest2.S[1] = Bytes - 1,2,3,4:5,6,7,8\n // Dest3.S[0] and Dest3.S[1] = Bytes - 2,3,4,5:6,7,8,9\n // Dest4.S[0] and Dest4.S[1] = Bytes - 3,4,5,6:7,8,9,10\n Dest1 = _VUABDL(OpSize::i128Bit, OpSize::i8Bit, Dest1, DupSrc);\n Dest2 = _VUABDL(OpSize::i128Bit, OpSize::i8Bit, Dest2, DupSrc);\n Dest3 = _VUABDL(OpSize::i128Bit, OpSize::i8Bit, Dest3, DupSrc);\n Dest4 = _VUABDL(OpSize::i128Bit, OpSize::i8Bit, Dest4, DupSrc);\n\n // Dest[1,2,3,4] Now contains the data prior to combining\n // Temp[0,1,2,3] for each step\n\n // Each destination now has 16bit x 8 elements in it that were the absolute difference for each byte\n // Needs each to be 16bit to store the next step\n // Next stage is to sum pairwise\n // Dest1:\n // ADDP Dest3, Dest1: TmpCombine1\n // ADDP Dest4, Dest2: TmpCombine2\n // TmpCombine1.8H[0] = Dest1.8H[0] + Dest1.8H[1];\n // TmpCombine1.8H[1] = Dest1.8H[2] + Dest1.8H[3];\n // TmpCombine1.8H[2] = Dest1.8H[4] + Dest1.8H[5];\n // TmpCombine1.8H[3] = Dest1.8H[6] + Dest1.8H[7];\n // TmpCombine1.8H[4] = Dest3.8H[0] + Dest3.8H[1];\n // TmpCombine1.8H[5] = Dest3.8H[2] + Dest3.8H[3];\n // TmpCombine1.8H[6] = Dest3.8H[4] + Dest3.8H[5];\n // TmpCombine1.8H[7] = Dest3.8H[6] + Dest3.8H[7];\n // \n auto TmpCombine1 = _VAddP(OpSize::i128Bit, OpSize::i16Bit, Dest1, Dest3);\n auto TmpCombine2 = _VAddP(OpSize::i128Bit, OpSize::i16Bit, Dest2, Dest4);\n\n // TmpTranspose1:\n // VTrn TmpCombine1, TmpCombine2: TmpTranspose1\n // Transposes Even and odd elements so we can use vaddp for final results.\n auto TmpTranspose1 = _VTrn(OpSize::i128Bit, OpSize::i32Bit, TmpCombine1, TmpCombine2);\n auto TmpTranspose2 = _VTrn2(OpSize::i128Bit, OpSize::i32Bit, TmpCombine1, TmpCombine2);\n\n // ADDP TmpTranspose1, TmpTranspose2: FinalCombine\n // FinalCombine.8H[0] = TmpTranspose1.8H[0] + TmpTranspose1.8H[1]\n // FinalCombine.8H[1] = TmpTranspose1.8H[2] + TmpTranspose1.8H[3]\n // FinalCombine.8H[2] = TmpTranspose1.8H[4] + TmpTranspose1.8H[5]\n // FinalCombine.8H[3] = TmpTranspose1.8H[6] + TmpTranspose1.8H[7]\n // FinalCombine.8H[4] = TmpTranspose2.8H[0] + TmpTranspose2.8H[1]\n // FinalCombine.8H[5] = TmpTranspose2.8H[2] + TmpTranspose2.8H[3]\n // FinalCombine.8H[6] = TmpTranspose2.8H[4] + TmpTranspose2.8H[5]\n // FinalCombine.8H[7] = TmpTranspose2.8H[6] + TmpTranspose2.8H[7]\n\n return _VAddP(OpSize::i128Bit, OpSize::i16Bit, TmpTranspose1, TmpTranspose2);\n };\n\n const auto Is128Bit = SrcSize == OpSize::i128Bit;\n\n // Src1 needs to be in byte offset\n const uint8_t Select_Src1_Low = ((Select & 0b100) >> 2) * 32 / 8;\n const uint8_t Select_Src2_Low = Select & 0b11;\n\n Ref Lower = LaneHelper(Select_Src1_Low, Select_Src2_Low, Src1, Src2);\n if (Is128Bit) {\n return Lower;\n }\n\n const uint8_t Select_Src1_High = ((Select & 0b100000) >> 5) * 32 / 8;\n const uint8_t Select_Src2_High = (Select & 0b11000) >> 3;\n\n Ref UpperSrc1 = _VDupElement(OpSize::i256Bit, OpSize::i128Bit, Src1, 1);\n Ref UpperSrc2 = _VDupElement(OpSize::i256Bit, OpSize::i128Bit, Src2, 1);\n Ref Upper = LaneHelper(Select_Src1_High, Select_Src2_High, UpperSrc1, UpperSrc2);\n return _VInsElement(OpSize::i256Bit, OpSize::i128Bit, 1, 0, Lower, Upper);\n}\n\nvoid OpDispatchBuilder::MPSADBWOp(OpcodeArgs) {\n const uint8_t Select = Op->Src[1].Literal();\n const auto SrcSize = OpSizeFromSrc(Op);\n Ref Src1 = LoadSourceFPR(Op, Op->Dest, Op->Flags);\n Ref Src2 = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n\n Ref Result = MPSADBWOpImpl(SrcSize, Src1, Src2, Select);\n StoreResultFPR(Op, Result);\n}\n\nvoid OpDispatchBuilder::VMPSADBWOp(OpcodeArgs) {\n const uint8_t Select = Op->Src[2].Literal();\n const auto SrcSize = OpSizeFromSrc(Op);\n Ref Src1 = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n Ref Src2 = LoadSourceFPR(Op, Op->Src[1], Op->Flags);\n\n Ref Result = MPSADBWOpImpl(SrcSize, Src1, Src2, Select);\n StoreResultFPR(Op, Result);\n}\n\nvoid OpDispatchBuilder::VINSERTOp(OpcodeArgs) {\n const auto DstSize = OpSizeFromDst(Op);\n Ref Src1 = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n Ref Src2 = LoadSourceFPR_WithOpSize(Op, Op->Src[1], OpSize::i128Bit, Op->Flags);\n\n const auto Selector = Op->Src[2].Literal() & 1;\n Ref Result = _VInsElement(DstSize, OpSize::i128Bit, Selector, 0, Src1, Src2);\n\n StoreResultFPR(Op, Result);\n}\n\nvoid OpDispatchBuilder::VCVTPH2PSOp(OpcodeArgs) {\n // In the event that a memory operand is used as the source operand,\n // the access width will always be half the size of the destination vector width\n // (i.e. 128-bit vector -> 64-bit mem, 256-bit vector -> 128-bit mem)\n const auto DstSize = OpSizeFromDst(Op);\n const auto SrcLoadSize = Op->Src[0].IsGPR() ? DstSize : IR::SizeToOpSize(IR::OpSizeToSize(DstSize) / 2);\n\n Ref Src = LoadSourceFPR_WithOpSize(Op, Op->Src[0], SrcLoadSize, Op->Flags);\n Ref Result = _Vector_FToF(DstSize, OpSize::i32Bit, Src, OpSize::i16Bit);\n\n StoreResultFPR(Op, Result);\n}\n\nvoid OpDispatchBuilder::VCVTPS2PHOp(OpcodeArgs) {\n const auto SrcSize = OpSizeFromSrc(Op);\n const auto StoreSize = Op->Dest.IsGPR() ? OpSize::i128Bit : IR::SizeToOpSize(IR::OpSizeToSize(SrcSize) / 2);\n\n const auto Imm8 = Op->Src[1].Literal();\n const auto UseMXCSR = (Imm8 & 0b100) != 0;\n\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n\n Ref Result = nullptr;\n if (UseMXCSR) {\n Result = _Vector_FToF(SrcSize, OpSize::i16Bit, Src, OpSize::i32Bit);\n } else {\n // No ARM float conversion instructions allow passing in\n // a rounding mode as an immediate. All of them depend on\n // the RM field in the FPCR. And so! We have to do some ugly\n // rounding mode shuffling.\n const auto NewRMode = Imm8 & 0b11;\n Ref SavedFPCR = _PushRoundingMode(NewRMode);\n\n Result = _Vector_FToF(SrcSize, OpSize::i16Bit, Src, OpSize::i32Bit);\n _PopRoundingMode(SavedFPCR);\n }\n\n // We need to eliminate upper junk if we're storing into a register with\n // a 256-bit source (VCVTPS2PH's destination for registers is an XMM).\n if (Op->Src[0].IsGPR() && SrcSize == OpSize::i256Bit) {\n Result = _VMov(OpSize::i128Bit, Result);\n }\n\n StoreResultFPR_WithOpSize(Op, Op->Dest, Result, StoreSize);\n}\n\nvoid OpDispatchBuilder::VPERM2Op(OpcodeArgs) {\n const auto DstSize = OpSizeFromDst(Op);\n Ref Src1 = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n Ref Src2 = LoadSourceFPR(Op, Op->Src[1], Op->Flags);\n\n const auto Selector = Op->Src[2].Literal();\n Ref Result = LoadZeroVector(DstSize);\n\n const auto SelectElement = [&](uint64_t Index, uint64_t SelectorIdx) {\n switch (SelectorIdx) {\n case 0:\n case 1: return _VInsElement(DstSize, OpSize::i128Bit, Index, SelectorIdx, Result, Src1);\n case 2:\n case 3:\n default: return _VInsElement(DstSize, OpSize::i128Bit, Index, SelectorIdx - 2, Result, Src2);\n }\n };\n\n if ((Selector & 0b00001000) == 0) {\n Result = SelectElement(0, Selector & 0b11);\n }\n if ((Selector & 0b10000000) == 0) {\n Result = SelectElement(1, (Selector >> 4) & 0b11);\n }\n\n StoreResultFPR(Op, Result);\n}\n\nRef OpDispatchBuilder::VPERMDIndices(OpSize DstSize, Ref Indices, Ref IndexMask, Ref Repeating3210) {\n // Get rid of any junk unrelated to the relevant selector index bits (bits [2:0])\n Ref SanitizedIndices = _VAnd(DstSize, OpSize::i8Bit, Indices, IndexMask);\n\n // Build up the broadcasted index mask. e.g. On x86-64, the selector index\n // is always in the lower 3 bits of a 32-bit element. However, in order to\n // build up a vector we can use with the ARMv8 TBL instruction, we need the\n // selector index for each particular element to be within each byte of the\n // 32-bit element.\n //\n // We can do this by TRN-ing the selector index vector twice. Once using byte elements\n // then once more using half-word elements.\n //\n // The first pass creates the half-word elements, and then the second pass uses those\n // halfword elements to place the indices in the top part of the 32-bit element.\n //\n // e.g. Consider a selector vector with indices in 32-bit elements like:\n //\n // ╔═══════════╗╔═══════════╗╔═══════════╗╔═══════════╗╔═══════════╗╔═══════════╗╔═══════════╗╔═══════════╗\n // ║ 4 ║║ 1 ║║ 2 ║║ 6 ║║ 7 ║║ 0 ║║ 3 ║║ 5 ║\n // ╚═══════════╝╚═══════════╝╚═══════════╝╚═══════════╝╚═══════════╝╚═══════════╝╚═══════════╝╚═══════════╝\n //\n // TRNing once using byte elements by itself will create a vector with 8-bit elements like:\n // ╔═══╗╔═══╗╔═══╗╔═══╗╔═══╗╔═══╗╔═══╗╔═══╗╔═══╗╔═══╗╔═══╗╔═══╗╔═══╗╔═══╗╔═══╗╔═══╗╔═══╗╔═══╗╔═══╗╔═══╗╔═══╗╔═══╗╔═══╗╔═══╗╔═══╗╔═══╗╔═══╗╔═══╗╔═══╗╔═══╗╔═══╗╔═══╗\n // ║ 0 ║║ 0 ║║ 4 ║║ 4 ║║ 0 ║║ 0 ║║ 1 ║║ 1 ║║ 0 ║║ 0 ║║ 2 ║║ 2 ║║ 0 ║║ 0 ║║ 6 ║║ 6 ║║ 0 ║║ 0 ║║ 7 ║║ 7 ║║ 0 ║║ 0 ║║ 0 ║║ 0 ║║ 0 ║║ 0 ║║ 3 ║║ 3 ║║ 0 ║║ 0 ║║ 5 ║║ 5 ║\n // ╚═══╝╚═══╝╚═══╝╚═══╝╚═══╝╚═══╝╚═══╝╚═══╝╚═══╝╚═══╝╚═══╝╚═══╝╚═══╝╚═══╝╚═══╝╚═══╝╚═══╝╚═══╝╚═══╝╚═══╝╚═══╝╚═══╝╚═══╝╚═══╝╚═══╝╚═══╝╚═══╝╚═══╝╚═══╝╚═══╝╚═══╝╚═══╝\n //\n // TRNing once using half-word elements by itself will then transform the vector into:\n // ╔═══╗╔═══╗╔═══╗╔═══╗╔═══╗╔═══╗╔═══╗╔═══╗╔═══╗╔═══╗╔═══╗╔═══╗╔═══╗╔═══╗╔═══╗╔═══╗╔═══╗╔═══╗╔═══╗╔═══╗╔═══╗╔═══╗╔═══╗╔═══╗╔═══╗╔═══╗╔═══╗╔═══╗╔═══╗╔═══╗╔═══╗╔═══╗\n // ║ 4 ║║ 4 ║║ 4 ║║ 4 ║║ 1 ║║ 1 ║║ 1 ║║ 1 ║║ 2 ║║ 2 ║║ 2 ║║ 2 ║║ 6 ║║ 6 ║║ 6 ║║ 6 ║║ 7 ║║ 7 ║║ 7 ║║ 7 ║║ 0 ║║ 0 ║║ 0 ║║ 0 ║║ 3 ║║ 3 ║║ 3 ║║ 3 ║║ 5 ║║ 5 ║║ 5 ║║ 5 ║\n // ╚═══╝╚═══╝╚═══╝╚═══╝╚═══╝╚═══╝╚═══╝╚═══╝╚═══╝╚═══╝╚═══╝╚═══╝╚═══╝╚═══╝╚═══╝╚═══╝╚═══╝╚═══╝╚═══╝╚═══╝╚═══╝╚═══╝╚═══╝╚═══╝╚═══╝╚═══╝╚═══╝╚═══╝╚═══╝╚═══╝╚═══╝╚═══╝\n //\n // Cool! We now have everything we need to take this further.\n\n Ref IndexTrn1 = _VTrn(DstSize, OpSize::i8Bit, SanitizedIndices, SanitizedIndices);\n Ref IndexTrn2 = _VTrn(DstSize, OpSize::i16Bit, IndexTrn1, IndexTrn1);\n\n // Now that we have the indices set up, now we need to multiply each\n // element by 4 to convert the elements into byte indices rather than\n // 32-bit word indices.\n //\n // e.g. We turn our vector into:\n // ╔════╗╔════╗╔════╗╔════╗╔════╗╔════╗╔════╗╔════╗╔════╗╔════╗╔════╗╔════╗╔════╗╔════╗╔════╗╔════╗╔════╗╔════╗╔════╗╔════╗╔════╗╔════╗╔════╗╔════╗╔════╗╔════╗╔════╗╔════╗╔════╗╔════╗╔════╗╔════╗\n // ║ 16 ║║ 16 ║║ 16 ║║ 16 ║║ 4 ║║ 4 ║║ 4 ║║ 4 ║║ 8 ║║ 8 ║║ 8 ║║ 8 ║║ 24 ║║ 24 ║║ 24 ║║ 24 ║║ 28 ║║ 28 ║║ 28 ║║ 28 ║║ 0 ║║ 0 ║║ 00 ║║ 0 ║║ 12 ║║ 12 ║║ 12 ║║ 12 ║║ 20 ║║ 20 ║║ 20 ║║ 20 ║\n // ╚════╝╚════╝╚════╝╚════╝╚════╝╚════╝╚════╝╚════╝╚════╝╚════╝╚════╝╚════╝╚════╝╚════╝╚════╝╚════╝╚════╝╚════╝╚════╝╚════╝╚════╝╚════╝╚════╝╚════╝╚════╝╚════╝╚════╝╚════╝╚════╝╚════╝╚════╝╚════╝\n //\n Ref ShiftedIndices = _VShlI(DstSize, OpSize::i8Bit, IndexTrn2, 2);\n\n // Now we need to add a byte vector containing [3, 2, 1, 0] repeating for the\n // entire length of it, to the index register, so that we specify the bytes\n // that make up the entire word in the source register.\n //\n // e.g. Our vector finally looks like so:\n //\n // ╔════╗╔════╗╔════╗╔════╗╔════╗╔════╗╔════╗╔════╗╔════╗╔════╗╔════╗╔════╗╔════╗╔════╗╔════╗╔════╗╔════╗╔════╗╔════╗╔════╗╔════╗╔════╗╔════╗╔════╗╔════╗╔════╗╔════╗╔════╗╔════╗╔════╗╔════╗╔════╗\n // ║ 19 ║║ 18 ║║ 17 ║║ 16 ║║ 7 ║║ 6 ║║ 5 ║║ 4 ║║ 11 ║║ 10 ║║ 9 ║║ 8 ║║ 27 ║║ 26 ║║ 25 ║║ 24 ║║ 31 ║║ 30 ║║ 29 ║║ 28 ║║ 3 ║║ 2 ║║ 01 ║║ 0 ║║ 15 ║║ 14 ║║ 13 ║║ 12 ║║ 23 ║║ 22 ║║ 21 ║║ 20 ║\n // ╚════╝╚════╝╚════╝╚════╝╚════╝╚════╝╚════╝╚════╝╚════╝╚════╝╚════╝╚════╝╚════╝╚════╝╚════╝╚════╝╚════╝╚════╝╚════╝╚════╝╚════╝╚════╝╚════╝╚════╝╚════╝╚════╝╚════╝╚════╝╚════╝╚════╝╚════╝╚════╝\n //\n // Which finally lets us permute the source vector and be done with everything.\n return _VAdd(DstSize, OpSize::i8Bit, ShiftedIndices, Repeating3210);\n}\n\nvoid OpDispatchBuilder::VPERMDOp(OpcodeArgs) {\n const auto DstSize = OpSizeFromDst(Op);\n\n Ref Indices = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n Ref Src = LoadSourceFPR(Op, Op->Src[1], Op->Flags);\n\n // Get rid of any junk unrelated to the relevant selector index bits (bits [2:0])\n Ref IndexMask = _VectorImm(DstSize, OpSize::i32Bit, 0b111);\n\n Ref AddConst = Constant(0x03020100);\n Ref Repeating3210 = _VDupFromGPR(DstSize, OpSize::i32Bit, AddConst);\n Ref FinalIndices = VPERMDIndices(OpSizeFromDst(Op), Indices, IndexMask, Repeating3210);\n\n // Now lets finally shuffle this bad boy around.\n Ref Result = _VTBL1(DstSize, Src, FinalIndices);\n\n StoreResultFPR(Op, Result);\n}\n\nvoid OpDispatchBuilder::VPERMQOp(OpcodeArgs) {\n const auto DstSize = OpSizeFromDst(Op);\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n\n const auto Selector = Op->Src[1].Literal();\n Ref Result {};\n\n // If we're just broadcasting one element in particular across the vector\n // then this can be done fairly simply without any individual inserts.\n if (Selector == 0x00 || Selector == 0x55 || Selector == 0xAA || Selector == 0xFF) {\n const auto Index = Selector & 0b11;\n Result = _VDupElement(DstSize, OpSize::i64Bit, Src, Index);\n } else {\n Result = LoadZeroVector(DstSize);\n for (size_t i = 0; i < IR::NumElements(DstSize, IR::OpSize::i64Bit); i++) {\n const auto SrcIndex = (Selector >> (i * 2)) & 0b11;\n Result = _VInsElement(DstSize, OpSize::i64Bit, i, SrcIndex, Result, Src);\n }\n }\n StoreResultFPR(Op, Result);\n}\n\nRef OpDispatchBuilder::VBLENDOpImpl(IR::OpSize VecSize, IR::OpSize ElementSize, Ref Src1, Ref Src2, Ref ZeroRegister, uint64_t Selector) {\n const std::array Sources {Src1, Src2};\n\n Ref Result = ZeroRegister;\n const auto NumElements = IR::NumElements(VecSize, ElementSize);\n for (int i = 0; i < NumElements; i++) {\n const auto SelectorIndex = (Selector >> i) & 1;\n\n Result = _VInsElement(VecSize, ElementSize, i, i, Result, Sources[SelectorIndex]);\n }\n\n return Result;\n}\n\nvoid OpDispatchBuilder::VBLENDPDOp(OpcodeArgs) {\n const auto DstSize = OpSizeFromDst(Op);\n const auto Is256Bit = DstSize == OpSize::i256Bit;\n const auto Selector = Op->Src[2].Literal();\n\n Ref Src1 = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n Ref Src2 = LoadSourceFPR(Op, Op->Src[1], Op->Flags);\n\n if (Selector == 0) {\n Ref Result = Is256Bit ? Src1 : _VMov(OpSize::i128Bit, Src1);\n StoreResultFPR(Op, Result);\n return;\n }\n // Only the first four bits of the 8-bit immediate are used, so only check them.\n if (((Selector & 0b11) == 0b11 && !Is256Bit) || (Selector & 0b1111) == 0b1111) {\n Ref Result = Is256Bit ? Src2 : _VMov(OpSize::i128Bit, Src2);\n StoreResultFPR(Op, Result);\n return;\n }\n\n const auto ZeroRegister = LoadZeroVector(DstSize);\n Ref Result = VBLENDOpImpl(DstSize, OpSize::i64Bit, Src1, Src2, ZeroRegister, Selector);\n StoreResultFPR(Op, Result);\n}\n\nvoid OpDispatchBuilder::VPBLENDDOp(OpcodeArgs) {\n const auto DstSize = OpSizeFromDst(Op);\n const auto Is256Bit = DstSize == OpSize::i256Bit;\n const auto Selector = Op->Src[2].Literal();\n\n Ref Src1 = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n Ref Src2 = LoadSourceFPR(Op, Op->Src[1], Op->Flags);\n\n // Each bit in the selector chooses between Src1 and Src2.\n // If a bit is set, then we select it's corresponding 32-bit element from Src2\n // If a bit is not set, then we select it's corresponding 32-bit element from Src1\n\n // Cases where we can exit out early, since the selector is indicating a copy\n // of an entire input vector. Unlikely to occur, since it's slower than\n // just an equivalent vector move instruction. but just in case something\n // silly is happening, we have your back.\n\n if (Selector == 0) {\n Ref Result = Is256Bit ? Src1 : _VMov(OpSize::i128Bit, Src1);\n StoreResultFPR(Op, Result);\n return;\n }\n if (Selector == 0xFF && Is256Bit) {\n StoreResultFPR(Op, Src2);\n return;\n }\n // The only bits we care about from the 8-bit immediate for 128-bit operations\n // are the first four bits. We do a bitwise check here to catch cases where\n // silliness is going on and the upper bits are being set even when they'll\n // be ignored\n if ((Selector & 0xF) == 0xF && !Is256Bit) {\n StoreResultFPR(Op, _VMov(OpSize::i128Bit, Src2));\n return;\n }\n\n const auto ZeroRegister = LoadZeroVector(DstSize);\n Ref Result = VBLENDOpImpl(DstSize, OpSize::i32Bit, Src1, Src2, ZeroRegister, Selector);\n if (!Is256Bit) {\n Result = _VMov(OpSize::i128Bit, Result);\n }\n StoreResultFPR(Op, Result);\n}\n\nvoid OpDispatchBuilder::VPBLENDWOp(OpcodeArgs) {\n const auto DstSize = OpSizeFromDst(Op);\n const auto Is128Bit = DstSize == OpSize::i128Bit;\n const auto Selector = Op->Src[2].Literal();\n\n Ref Src1 = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n Ref Src2 = LoadSourceFPR(Op, Op->Src[1], Op->Flags);\n\n if (Selector == 0) {\n Ref Result = Is128Bit ? _VMov(OpSize::i128Bit, Src1) : Src1;\n StoreResultFPR(Op, Result);\n return;\n }\n if (Selector == 0xFF) {\n Ref Result = Is128Bit ? _VMov(OpSize::i128Bit, Src2) : Src2;\n StoreResultFPR(Op, Result);\n return;\n }\n\n // 256-bit VPBLENDW acts as if the 8-bit selector values were also applied\n // to the upper bits, so we can just replicate the bits by forming a 16-bit\n // imm for the helper function to use.\n const auto NewSelector = Selector << 8 | Selector;\n\n const auto ZeroRegister = LoadZeroVector(DstSize);\n Ref Result = VBLENDOpImpl(DstSize, OpSize::i16Bit, Src1, Src2, ZeroRegister, NewSelector);\n if (Is128Bit) {\n Result = _VMov(OpSize::i128Bit, Result);\n }\n StoreResultFPR(Op, Result);\n}\n\nvoid OpDispatchBuilder::VZEROOp(OpcodeArgs) {\n const auto DstSize = OpSizeFromDst(Op);\n const auto IsVZEROALL = DstSize == OpSize::i256Bit;\n const auto NumRegs = Is64BitMode ? 16U : 8U;\n\n if (IsVZEROALL) {\n // NOTE: Despite the name being VZEROALL, this will still only ever\n // zero out up to the first 16 registers (even on AVX-512, where we have 32 registers)\n\n for (uint32_t i = 0; i < NumRegs; i++) {\n // Explicitly not caching named vector zero. This ensures that every register gets movi #0.0 directly.\n Ref ZeroVector = LoadUncachedZeroVector(DstSize);\n StoreXMMRegister(i, ZeroVector);\n }\n } else {\n // Likewise, VZEROUPPER will only ever zero only up to the first 16 registers\n\n for (uint32_t i = 0; i < NumRegs; i++) {\n Ref Reg = LoadXMMRegister(i);\n Ref Dst = _VMov(OpSize::i128Bit, Reg);\n StoreXMMRegister(i, Dst);\n }\n }\n}\n\nvoid OpDispatchBuilder::VPERMILImmOp(OpcodeArgs, IR::OpSize ElementSize) {\n const auto DstSize = OpSizeFromDst(Op);\n const auto Is256Bit = DstSize == OpSize::i256Bit;\n const auto Selector = Op->Src[1].Literal() & 0xFF;\n\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n Ref Result = LoadZeroVector(DstSize);\n\n if (ElementSize == OpSize::i64Bit) {\n Result = _VInsElement(DstSize, ElementSize, 0, Selector & 0b0001, Result, Src);\n Result = _VInsElement(DstSize, ElementSize, 1, (Selector & 0b0010) >> 1, Result, Src);\n\n if (Is256Bit) {\n Result = _VInsElement(DstSize, ElementSize, 2, ((Selector & 0b0100) >> 2) + 2, Result, Src);\n Result = _VInsElement(DstSize, ElementSize, 3, ((Selector & 0b1000) >> 3) + 2, Result, Src);\n }\n } else {\n Result = _VInsElement(DstSize, ElementSize, 0, Selector & 0b00000011, Result, Src);\n Result = _VInsElement(DstSize, ElementSize, 1, (Selector & 0b00001100) >> 2, Result, Src);\n Result = _VInsElement(DstSize, ElementSize, 2, (Selector & 0b00110000) >> 4, Result, Src);\n Result = _VInsElement(DstSize, ElementSize, 3, (Selector & 0b11000000) >> 6, Result, Src);\n\n if (Is256Bit) {\n Result = _VInsElement(DstSize, ElementSize, 4, (Selector & 0b00000011) + 4, Result, Src);\n Result = _VInsElement(DstSize, ElementSize, 5, ((Selector & 0b00001100) >> 2) + 4, Result, Src);\n Result = _VInsElement(DstSize, ElementSize, 6, ((Selector & 0b00110000) >> 4) + 4, Result, Src);\n Result = _VInsElement(DstSize, ElementSize, 7, ((Selector & 0b11000000) >> 6) + 4, Result, Src);\n }\n }\n\n StoreResultFPR(Op, Result);\n}\n\nRef OpDispatchBuilder::VPERMILRegOpImpl(OpSize DstSize, IR::OpSize ElementSize, Ref Src, Ref Indices) {\n // NOTE: See implementation of VPERMD for the gist of what we do to make this work.\n //\n // The only difference here is that we need to add 16 to the upper lane\n // before doing the final addition to build up the indices for TBL.\n\n const auto Is256Bit = DstSize == OpSize::i256Bit;\n auto IsPD = ElementSize == OpSize::i64Bit;\n\n if (IsPD) {\n // VPERMILPD stores the selector in the second bit, rather than the\n // first bit of each element in the index vector. So move it over by one.\n Indices = _VUShrI(DstSize, ElementSize, Indices, 1);\n }\n\n // Sanitize indices first\n const auto ShiftAmount = 0b11 >> static_cast(IsPD);\n Ref IndexMask = _VectorImm(DstSize, ElementSize, ShiftAmount);\n Ref SanitizedIndices = _VAnd(DstSize, OpSize::i8Bit, Indices, IndexMask);\n\n Ref IndexTrn1 = _VTrn(DstSize, OpSize::i8Bit, SanitizedIndices, SanitizedIndices);\n Ref IndexTrn2 = _VTrn(DstSize, OpSize::i16Bit, IndexTrn1, IndexTrn1);\n Ref IndexTrn3 = IndexTrn2;\n if (IsPD) {\n IndexTrn3 = _VTrn(DstSize, OpSize::i32Bit, IndexTrn2, IndexTrn2);\n }\n\n auto IndexShift = IsPD ? 3 : 2;\n Ref ShiftedIndices = _VShlI(DstSize, OpSize::i8Bit, IndexTrn3, IndexShift);\n\n uint64_t VConstant = IsPD ? 0x0706050403020100 : 0x03020100;\n Ref VectorConst = _VDupFromGPR(DstSize, ElementSize, Constant(VConstant));\n Ref FinalIndices {};\n\n if (Is256Bit) {\n const auto ZeroRegister = LoadZeroVector(DstSize);\n Ref Vector16 = _VInsElement(DstSize, OpSize::i128Bit, 1, 0, ZeroRegister, _VectorImm(DstSize, OpSize::i8Bit, 16));\n Ref IndexOffsets = _VAdd(DstSize, OpSize::i8Bit, VectorConst, Vector16);\n\n FinalIndices = _VAdd(DstSize, OpSize::i8Bit, IndexOffsets, ShiftedIndices);\n } else {\n FinalIndices = _VAdd(DstSize, OpSize::i8Bit, VectorConst, ShiftedIndices);\n }\n\n return _VTBL1(DstSize, Src, FinalIndices);\n}\n\ntemplate\nvoid OpDispatchBuilder::VPERMILRegOp(OpcodeArgs) {\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n Ref Indices = LoadSourceFPR(Op, Op->Src[1], Op->Flags);\n\n Ref Result = VPERMILRegOpImpl(OpSizeFromDst(Op), ElementSize, Src, Indices);\n StoreResultFPR(Op, Result);\n}\n\ntemplate void OpDispatchBuilder::VPERMILRegOp(OpcodeArgs);\ntemplate void OpDispatchBuilder::VPERMILRegOp(OpcodeArgs);\n\nvoid OpDispatchBuilder::PCMPXSTRXOpImpl(OpcodeArgs, bool IsExplicit, bool IsMask) {\n const uint16_t Control = Op->Src[1].Literal();\n\n // NOTE: Unlike most other SSE/AVX instructions, the SSE4.2 string and text\n // instructions do *not* require memory operands to be aligned on a 16 byte\n // boundary (see \"Other Exceptions\" descriptions for the relevant\n // instructions in the Intel Software Development Manual).\n //\n // So, we specify Src2 as having an alignment of 1 to indicate this.\n Ref Src1 = LoadSourceFPR_WithOpSize(Op, Op->Dest, OpSize::i128Bit, Op->Flags);\n Ref Src2 = LoadSourceFPR_WithOpSize(Op, Op->Src[0], OpSize::i128Bit, Op->Flags, {.Align = OpSize::i8Bit});\n\n Ref IntermediateResult {};\n if (IsExplicit) {\n // Will be 4 in the absence of a REX.W bit and 8 in the presence of a REX.W bit.\n //\n // While the control bit immediate for the instruction itself is only ever 8 bits\n // in size, we use it as a 16-bit value so that we can use the 8th bit to signify\n // whether or not RAX and RDX should be interpreted as a 64-bit value.\n const auto SrcSize = OpSizeFromSrc(Op);\n const auto Is64Bit = SrcSize == OpSize::i64Bit;\n const auto NewControl = uint16_t(Control | (uint16_t(Is64Bit) << 8));\n\n Ref SrcRAX = LoadGPRRegister(X86State::REG_RAX);\n Ref SrcRDX = LoadGPRRegister(X86State::REG_RDX);\n\n IntermediateResult = _VPCMPESTRX(Src1, Src2, SrcRAX, SrcRDX, NewControl);\n } else {\n IntermediateResult = _VPCMPISTRX(Src1, Src2, Control);\n }\n\n Ref ZeroConst = Constant(0);\n\n if (IsMask) {\n // For the masked variant of the instructions, if control[6] is set, then we\n // need to expand the intermediate result into a byte or word mask (depending\n // on data size specified in control[1]) along the entire length of XMM0,\n // where set bits in the intermediate result set the corresponding entry\n // in XMM0 to all 1s and unset bits set the corresponding entry to all 0s.\n //\n // If control[6] is not set, then we just store the intermediate result as-is\n // into the least significant bits of XMM0 and zero extend it.\n const auto IsExpandedMask = (Control & 0b0100'0000) != 0;\n\n if (IsExpandedMask) {\n // We need to iterate over the intermediate result and\n // expand the mask into XMM0 elements.\n const auto ElementSize = 1U << (Control & 1);\n const auto NumElements = 16U >> (Control & 1);\n\n Ref Result = LoadZeroVector(OpSize::i128Bit);\n for (uint32_t i = 0; i < NumElements; i++) {\n Ref SignBit = _Sbfe(OpSize::i64Bit, 1, i, IntermediateResult);\n Result = _VInsGPR(OpSize::i128Bit, IR::SizeToOpSize(ElementSize), i, Result, SignBit);\n }\n StoreXMMRegister(0, Result);\n } else {\n // We insert the intermediate result as-is.\n StoreXMMRegister(0, _VCastFromGPR(OpSize::i128Bit, OpSize::i16Bit, IntermediateResult));\n }\n } else {\n // For the indexed variant of the instructions, if control[6] is set, then we\n // store the index of the most significant bit into ECX. If it's not set,\n // then we store the least significant bit.\n const auto UseMSBIndex = (Control & 0b0100'0000) != 0;\n\n Ref ResultNoFlags = _Bfe(OpSize::i32Bit, 16, 0, IntermediateResult);\n\n Ref IfZero = Constant(16 >> (Control & 1));\n Ref IfNotZero = UseMSBIndex ? _FindMSB(IR::OpSize::i32Bit, ResultNoFlags) : _FindLSB(IR::OpSize::i32Bit, ResultNoFlags);\n Ref Result = _Select(OpSize::i64Bit, OpSize::i64Bit, CondClass::EQ, ResultNoFlags, ZeroConst, IfZero, IfNotZero);\n\n // Store the result, it is already zero-extended to 64-bit implicitly.\n StoreGPRRegister(X86State::REG_RCX, Result);\n }\n\n // Set all of the necessary flags. NZCV stored in bits 28...31 like the hw op.\n SetNZCV(IntermediateResult);\n CFInverted = false;\n ZeroPF_AF();\n}\n\nvoid OpDispatchBuilder::VPCMPESTRIOp(OpcodeArgs) {\n PCMPXSTRXOpImpl(Op, true, false);\n}\nvoid OpDispatchBuilder::VPCMPESTRMOp(OpcodeArgs) {\n PCMPXSTRXOpImpl(Op, true, true);\n}\nvoid OpDispatchBuilder::VPCMPISTRIOp(OpcodeArgs) {\n PCMPXSTRXOpImpl(Op, false, false);\n}\nvoid OpDispatchBuilder::VPCMPISTRMOp(OpcodeArgs) {\n PCMPXSTRXOpImpl(Op, false, true);\n}\n\nvoid OpDispatchBuilder::VFMAImpl(OpcodeArgs, IROps IROp, bool Scalar, uint8_t Src1Idx, uint8_t Src2Idx, uint8_t AddendIdx) {\n const auto Size = OpSizeFromDst(Op);\n const auto Is256Bit = Size == OpSize::i256Bit;\n\n const OpSize ElementSize = Op->Flags & X86Tables::DecodeFlags::FLAG_OPTION_AVX_W ? OpSize::i64Bit : OpSize::i32Bit;\n\n Ref Dest = LoadSourceFPR_WithOpSize(Op, Op->Dest, Size, Op->Flags);\n Ref Src1 = LoadSourceFPR_WithOpSize(Op, Op->Src[0], Size, Op->Flags);\n Ref Src2 {};\n if (Op->Src[1].IsGPR()) {\n Src2 = LoadSourceFPR_WithOpSize(Op, Op->Src[1], Size, Op->Flags);\n } else {\n Src2 = LoadSourceFPR(Op, Op->Src[1], Op->Flags);\n }\n\n Ref Sources[3] = {\n Dest,\n Src1,\n Src2,\n };\n\n DeriveOp(FMAResult, IROp, _VFMLA(Size, ElementSize, Sources[Src1Idx - 1], Sources[Src2Idx - 1], Sources[AddendIdx - 1]));\n Ref Result = FMAResult;\n if (Scalar) {\n // Special case, scalar inserts in to the low bits of the destination.\n Result = _VInsElement(OpSize::i128Bit, ElementSize, 0, 0, Dest, Result);\n }\n\n if (!Is256Bit) {\n Result = _VMov(OpSize::i128Bit, Result);\n }\n StoreResultFPR(Op, Result);\n}\n\nvoid OpDispatchBuilder::VFMAddSubImpl(OpcodeArgs, bool AddSub, uint8_t Src1Idx, uint8_t Src2Idx, uint8_t AddendIdx) {\n const auto Size = OpSizeFromDst(Op);\n const auto Is256Bit = Size == OpSize::i256Bit;\n\n const OpSize ElementSize = Op->Flags & X86Tables::DecodeFlags::FLAG_OPTION_AVX_W ? OpSize::i64Bit : OpSize::i32Bit;\n\n Ref Dest = LoadSourceFPR(Op, Op->Dest, Op->Flags);\n Ref Src1 = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n Ref Src2 = LoadSourceFPR(Op, Op->Src[1], Op->Flags);\n\n Ref Sources[3] = {\n Dest,\n Src1,\n Src2,\n };\n\n Ref ConstantEOR {};\n if (AddSub) {\n ConstantEOR =\n LoadAndCacheNamedVectorConstant(Size, ElementSize == OpSize::i32Bit ? NAMED_VECTOR_PADDSUBPS_INVERT : NAMED_VECTOR_PADDSUBPD_INVERT);\n } else {\n ConstantEOR =\n LoadAndCacheNamedVectorConstant(Size, ElementSize == OpSize::i32Bit ? NAMED_VECTOR_PSUBADDPS_INVERT : NAMED_VECTOR_PSUBADDPD_INVERT);\n }\n\n auto InvertedSourc = _VXor(Size, ElementSize, Sources[AddendIdx - 1], ConstantEOR);\n\n Ref Result = _VFMLA(Size, ElementSize, Sources[Src1Idx - 1], Sources[Src2Idx - 1], InvertedSourc);\n if (!Is256Bit) {\n Result = _VMov(OpSize::i128Bit, Result);\n }\n StoreResultFPR(Op, Result);\n}\n\nOpDispatchBuilder::RefVSIB OpDispatchBuilder::LoadVSIB(const X86Tables::DecodedOp& Op, const X86Tables::DecodedOperand& Operand, uint32_t Flags) {\n const bool IsVSIB = (Op->Flags & X86Tables::DecodeFlags::FLAG_VSIB_BYTE) != 0;\n LOGMAN_THROW_A_FMT((Operand.IsSIB() || Operand.IsSIBRelocation()) && IsVSIB, \"Trying to load VSIB for something that isn't the correct \"\n \"type!\");\n\n // VSIB is a very special case which has a ton of encoded data.\n // Get it in a format we can reason about.\n\n const auto Index_gpr = Operand.Data.SIB.Index;\n const auto Base_gpr = Operand.Data.SIB.Base;\n LOGMAN_THROW_A_FMT(Index_gpr >= FEXCore::X86State::REG_XMM_0 && Index_gpr <= FEXCore::X86State::REG_XMM_15, \"must be AVX reg\");\n LOGMAN_THROW_A_FMT(Base_gpr == FEXCore::X86State::REG_INVALID || (Base_gpr >= FEXCore::X86State::REG_RAX && Base_gpr <= FEXCore::X86State::REG_R15),\n \"Base must be a GPR.\");\n const auto Index_XMM_gpr = Index_gpr - X86State::REG_XMM_0;\n\n OpDispatchBuilder::RefVSIB A {\n .Low = LoadXMMRegister(Index_XMM_gpr),\n .BaseAddr = Base_gpr != FEXCore::X86State::REG_INVALID ? LoadGPRRegister(Base_gpr, OpSize::i64Bit, 0, false) : nullptr,\n .Scale = Operand.Data.SIB.Scale,\n };\n\n if (Operand.IsSIBRelocation()) {\n auto EPOffset = _EntrypointOffset(OpSize::i64Bit, Operand.Data.SIB.Offset);\n if (A.BaseAddr) {\n A.BaseAddr = Add(OpSize::i64Bit, EPOffset, A.BaseAddr);\n } else {\n A.BaseAddr = EPOffset;\n }\n } else {\n A.Displacement = static_cast(Operand.Data.SIB.Offset);\n }\n\n return A;\n}\n\ntemplate\nvoid OpDispatchBuilder::VPGATHER(OpcodeArgs) {\n LOGMAN_THROW_A_FMT(AddrElementSize == OpSize::i32Bit || AddrElementSize == OpSize::i64Bit, \"Unknown address element size\");\n\n const auto Size = OpSizeFromDst(Op);\n const auto Is128Bit = Size == OpSize::i128Bit;\n const auto GPRSize = GetGPROpSize();\n auto AddrSize = (Op->Flags & X86Tables::DecodeFlags::FLAG_ADDRESS_SIZE) != 0 ? (GPRSize >> 1) : GPRSize;\n\n ///< Element size is determined by W flag.\n const OpSize ElementLoadSize = Op->Flags & X86Tables::DecodeFlags::FLAG_OPTION_AVX_W ? OpSize::i64Bit : OpSize::i32Bit;\n\n // We only need the high address register if the number of data elements is more than what the low half can consume.\n // But also the number of address elements is clamped by the destination size as well.\n const size_t NumDataElements = IR::NumElements(Size, ElementLoadSize);\n const size_t NumAddrElementBytes = std::min(IR::OpSizeToSize(Size), (NumDataElements * IR::OpSizeToSize(AddrElementSize)));\n const bool Needs128BitHighAddrBytes = NumAddrElementBytes > IR::OpSizeToSize(OpSize::i128Bit);\n\n auto VSIB = LoadVSIB(Op, Op->Src[0], Op->Flags);\n\n const bool SupportsSVELoad = (VSIB.Scale == 1 || VSIB.Scale == IR::OpSizeToSize(AddrElementSize)) && (AddrElementSize == ElementLoadSize);\n\n Ref Dest = LoadSourceFPR(Op, Op->Dest, Op->Flags);\n Ref Mask = LoadSourceFPR(Op, Op->Src[1], Op->Flags);\n\n Ref Result {};\n if (!SupportsSVELoad) {\n // We need to go down the fallback path in the case that we don't hit the backend's SVE mode.\n RefPair Dest128 {\n .Low = Dest,\n .High = _VDupElement(OpSize::i256Bit, OpSize::i128Bit, Dest, 1),\n };\n\n RefPair Mask128 {\n .Low = Mask,\n .High = _VDupElement(OpSize::i256Bit, OpSize::i128Bit, Mask, 1),\n };\n\n RefVSIB VSIB128 = VSIB;\n VSIB128.High = Invalid();\n\n if (Needs128BitHighAddrBytes) {\n if (Is128Bit) {\n ///< A bit careful for the VSIB index register duplicating.\n VSIB128.High = VSIB128.Low;\n } else {\n VSIB128.High = _VDupElement(OpSize::i256Bit, OpSize::i128Bit, VSIB128.Low, 1);\n }\n }\n\n auto Result128 = AVX128_VPGatherImpl(Op, Size, ElementLoadSize, AddrElementSize, Dest128, Mask128, VSIB128);\n // The registers are current split, need to merge them.\n Result = _VInsElement(OpSize::i256Bit, OpSize::i128Bit, 1, 0, Result128.Low, Result128.High);\n } else {\n ///< Calculate the full operation.\n ///< BaseAddr doesn't need to exist, calculate that here.\n Ref BaseAddr = VSIB.BaseAddr;\n if (BaseAddr && VSIB.Displacement) {\n BaseAddr = Add(OpSize::i64Bit, BaseAddr, VSIB.Displacement);\n } else if (VSIB.Displacement) {\n BaseAddr = Constant(VSIB.Displacement);\n } else if (!BaseAddr) {\n BaseAddr = Invalid();\n }\n\n Result =\n _VLoadVectorGatherMasked(Size, ElementLoadSize, Dest, Mask, BaseAddr, VSIB.Low, Invalid(), AddrElementSize, VSIB.Scale, 0, 0, AddrSize);\n }\n\n if (Is128Bit) {\n if (AddrElementSize == OpSize::i64Bit && ElementLoadSize == OpSize::i32Bit) {\n // Special case for the 128-bit gather load using 64-bit address indexes with 32-bit results.\n // Only loads two 32-bit elements in to the lower 64-bits of the first destination.\n // Bits [255:65] all become zero.\n Result = _VMov(OpSize::i64Bit, Result);\n } else {\n Result = _VMov(OpSize::i128Bit, Result);\n }\n } else {\n if (AddrElementSize == OpSize::i64Bit && ElementLoadSize == OpSize::i32Bit) {\n // If we only fetched 128-bits worth of data then the upper-result is all zero.\n Result = _VMov(OpSize::i128Bit, Result);\n }\n }\n\n StoreResultFPR(Op, Result);\n\n ///< Assume non-faulting behaviour and clear the mask register.\n auto Zero = LoadZeroVector(Size);\n StoreResultFPR_WithOpSize(Op, Op->Src[1], Zero, Size);\n}\n\ntemplate void OpDispatchBuilder::VPGATHER(OpcodeArgs);\ntemplate void OpDispatchBuilder::VPGATHER(OpcodeArgs);\n\nvoid OpDispatchBuilder::Extrq_imm(OpcodeArgs) {\n const uint8_t MaskWidth = Op->Src[1].Literal() & 0x3F;\n const uint8_t Shift = (Op->Src[1].Literal() >> 8) & 0x3F;\n\n Ref Dest = LoadSourceFPR(Op, Op->Dest, Op->Flags);\n Ref Result = Dest;\n if (Shift > 0) {\n Result = _VUShrI(OpSize::i64Bit, OpSize::i64Bit, Dest, Shift);\n }\n\n const uint64_t Mask = ~0ULL >> (MaskWidth == 0 ? 0 : (64 - MaskWidth));\n const Ref MaskVector = _VCastFromGPR(OpSize::i128Bit, OpSize::i64Bit, _Constant(Mask));\n Result = _VAnd(OpSize::i128Bit, OpSize::i64Bit, Result, MaskVector);\n\n StoreResultFPR(Op, Result);\n}\n\nvoid OpDispatchBuilder::Insertq_imm(OpcodeArgs) {\n const uint8_t MaskWidth = Op->Src[1].Literal() & 0x3F;\n const uint8_t Shift = (Op->Src[1].Literal() >> 8) & 0x3F;\n\n Ref Dest = LoadSourceFPR(Op, Op->Dest, Op->Flags);\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n\n const uint64_t Mask = ~0ULL >> (MaskWidth == 0 ? 0 : (64 - MaskWidth));\n Ref MaskVector = _VCastFromGPR(OpSize::i128Bit, OpSize::i64Bit, _Constant(Mask));\n\n // Mask incoming source.\n Src = _VAnd(OpSize::i64Bit, OpSize::i64Bit, Src, MaskVector);\n\n // If shifting then shift source and mask in to the correct location.\n if (Shift) {\n Src = _VShlI(OpSize::i64Bit, OpSize::i64Bit, Src, Shift);\n MaskVector = _VShlI(OpSize::i128Bit, OpSize::i64Bit, MaskVector, Shift);\n }\n\n // Negate the mask.\n MaskVector = _VNot(OpSize::i64Bit, OpSize::i64Bit, MaskVector);\n\n Dest = _VAnd(OpSize::i64Bit, OpSize::i64Bit, Dest, MaskVector);\n const Ref Result = _VOr(OpSize::i64Bit, OpSize::i64Bit, Dest, Src);\n\n StoreResultFPR(Op, Result);\n}\n\nvoid OpDispatchBuilder::Extrq(OpcodeArgs) {\n Ref Dest = LoadSourceFPR(Op, Op->Dest, Op->Flags);\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n\n const Ref ElementMask = _VectorImm(OpSize::i64Bit, OpSize::i64Bit, 0x3F);\n\n auto GenerateMask = [this](Ref VectorWidthInBits) -> Ref {\n const Ref VectorWidth = _VExtractToGPR(OpSize::i64Bit, OpSize::i64Bit, VectorWidthInBits, 0);\n return _VCastFromGPR(OpSize::i128Bit, OpSize::i64Bit, _MaskGenerateFromBitWidth(VectorWidth));\n };\n\n // Bits[5:0] = Mask width in bits\n const Ref MaskWidthBits = _VAnd(OpSize::i64Bit, OpSize::i64Bit, Src, ElementMask);\n\n // Bits[13:8] = Shift right in bits\n const Ref ShiftBits = _VAnd(OpSize::i64Bit, OpSize::i64Bit, _VUShrI(OpSize::i64Bit, OpSize::i64Bit, Src, 8), ElementMask);\n\n // First shift in to the correct position.\n Ref Result = _VUShr(OpSize::i64Bit, OpSize::i64Bit, Dest, ShiftBits, false);\n\n Result = _VAnd(OpSize::i128Bit, OpSize::i64Bit, Result, GenerateMask(MaskWidthBits));\n\n StoreResultFPR(Op, Result);\n}\n\nvoid OpDispatchBuilder::Insertq(OpcodeArgs) {\n Ref Dest = LoadSourceFPR(Op, Op->Dest, Op->Flags);\n Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n\n auto SelectorBits = _VDupElement(OpSize::i128Bit, OpSize::i64Bit, Src, 1);\n\n const Ref ElementMask = _VectorImm(OpSize::i64Bit, OpSize::i64Bit, 0x3F);\n\n auto GenerateMask = [this](Ref VectorWidthInBits) -> Ref {\n const Ref VectorWidth = _VExtractToGPR(OpSize::i64Bit, OpSize::i64Bit, VectorWidthInBits, 0);\n return _VCastFromGPR(OpSize::i128Bit, OpSize::i64Bit, _MaskGenerateFromBitWidth(VectorWidth));\n };\n\n // Bits[5:0] = Mask width in bits\n const Ref MaskWidthBits = _VAnd(OpSize::i64Bit, OpSize::i64Bit, SelectorBits, ElementMask);\n\n // Bits[13:8] = Shift right in bits\n const Ref ShiftBits = _VAnd(OpSize::i64Bit, OpSize::i64Bit, _VUShrI(OpSize::i64Bit, OpSize::i64Bit, SelectorBits, 8), ElementMask);\n\n // Extract the source data and put in to the correct location\n const Ref SrcMask = GenerateMask(MaskWidthBits);\n Ref SrcData = _VAnd(OpSize::i128Bit, OpSize::i64Bit, Src, SrcMask);\n SrcData = _VUShl(OpSize::i128Bit, OpSize::i64Bit, SrcData, ShiftBits, false);\n\n // Generate a destination mask\n const Ref DstMask = _VNot(OpSize::i64Bit, OpSize::i64Bit, _VUShl(OpSize::i128Bit, OpSize::i64Bit, SrcMask, ShiftBits, false));\n\n Ref Result = _VAnd(OpSize::i64Bit, OpSize::i64Bit, Dest, DstMask);\n Result = _VOr(OpSize::i64Bit, OpSize::i64Bit, Result, SrcData);\n StoreResultFPR(Op, Result);\n}\n\n} // namespace FEXCore::IR\n" }, { "path": "FEXCore/Source/Interface/Core/OpcodeDispatcher/X87.cpp", "content": "// SPDX-License-Identifier: MIT\n/*\n$info$\ntags: frontend|x86-to-ir, opcodes|dispatcher-implementations\ndesc: Handles x86/64 x87 to IR\n$end_info$\n*/\n\n#include \"Interface/Core/OpcodeDispatcher.h\"\n#include \"Interface/Core/X86Tables/X86Tables.h\"\n#include \"Interface/IR/IR.h\"\n#include \"Interface/Core/Addressing.h\"\n\n#include \n#include \n#include \n#include \n#include \n\n#include \n#include \n\nnamespace FEXCore::IR {\nclass OrderedNode;\n#define OpcodeArgs [[maybe_unused]] FEXCore::X86Tables::DecodedOp Op\n\nRef OpDispatchBuilder::GetX87Top() {\n // Yes, we are storing 3 bits in a single flag register.\n // Deal with it\n return _LoadContextGPR(OpSize::i8Bit, offsetof(FEXCore::Core::CPUState, flags) + FEXCore::X86State::X87FLAG_TOP_LOC);\n}\n\nvoid OpDispatchBuilder::SetX87FTW(Ref FTW) {\n _StackForceSlow(); // Invalidate x87 FTW register cache\n\n // For the output, we want a 1-bit for each pair not equal to 11 (Empty).\n static_assert(static_cast(FPState::X87Tag::Empty) == 0b11);\n\n // Make even bits 1 if the pair is equal to 11, and 0 otherwise.\n FTW = _AndShift(OpSize::i32Bit, FTW, FTW, ShiftType::LSR, 1);\n\n // Invert FTW and clear the odd bits. Even bits are 1 if the pair\n // is not equal to 11, and odd bits are 0.\n FTW = _Andn(OpSize::i32Bit, Constant(0x55555555), FTW);\n\n // All that's left is to compact away the odd bits. That is a Morton\n // deinterleave operation, which has a standard solution. See\n // https://stackoverflow.com/questions/3137266/how-to-de-interleave-bits-unmortonizing\n FTW = _And(OpSize::i32Bit, _Orlshr(OpSize::i32Bit, FTW, FTW, 1), Constant(0x33333333));\n FTW = _And(OpSize::i32Bit, _Orlshr(OpSize::i32Bit, FTW, FTW, 2), Constant(0x0f0f0f0f));\n FTW = _Orlshr(OpSize::i32Bit, FTW, FTW, 4);\n\n // ...and that's it. StoreContext implicitly does the final masking.\n _StoreContextGPR(OpSize::i8Bit, FTW, offsetof(FEXCore::Core::CPUState, AbridgedFTW));\n}\n\nvoid OpDispatchBuilder::SetX87Top(Ref Value) {\n _StoreContextGPR(OpSize::i8Bit, Value, offsetof(FEXCore::Core::CPUState, flags) + FEXCore::X86State::X87FLAG_TOP_LOC);\n}\n\n// Float LoaD operation with memory operand\nvoid OpDispatchBuilder::FLD(OpcodeArgs, IR::OpSize Width) {\n Ref Data = LoadSourceFPR_WithOpSize(Op, Op->Src[0], Width, Op->Flags);\n Ref ConvertedData = Data;\n // Convert to 80bit float\n if (Width == OpSize::i32Bit || Width == OpSize::i64Bit) {\n ConvertedData = _F80CVTTo(Data, Width);\n }\n _PushStack(ConvertedData, Data, Width);\n}\n\n// Float LoaD operation with memory operand\nvoid OpDispatchBuilder::FLDFromStack(OpcodeArgs) {\n _CopyPushStack(Op->OP & 7);\n}\n\nvoid OpDispatchBuilder::FBLD(OpcodeArgs) {\n // Read from memory\n Ref Data = LoadSourceFPR_WithOpSize(Op, Op->Src[0], OpSize::f80Bit, Op->Flags);\n Ref ConvertedData = _F80BCDLoad(Data);\n _PushStack(ConvertedData, Invalid(), OpSize::iInvalid);\n}\n\nvoid OpDispatchBuilder::FBSTP(OpcodeArgs) {\n Ref converted = _F80BCDStore(_ReadStackValue(0));\n StoreResultFPR_WithOpSize(Op, Op->Dest, converted, OpSize::f80Bit, OpSize::i8Bit);\n _PopStackDestroy();\n}\n\nvoid OpDispatchBuilder::FLD_Const(OpcodeArgs, NamedVectorConstant K) {\n // Update TOP\n Ref Data = LoadAndCacheNamedVectorConstant(OpSize::i128Bit, K);\n _PushStack(Data, Data, OpSize::f80Bit);\n}\n\nvoid OpDispatchBuilder::FILD(OpcodeArgs) {\n const auto ReadWidth = OpSizeFromSrc(Op);\n // Read from memory\n Ref Data = LoadSourceGPR_WithOpSize(Op, Op->Src[0], ReadWidth, Op->Flags);\n\n // Sign extend to 64bits\n if (ReadWidth != OpSize::i64Bit) {\n Data = _Sbfe(OpSize::i64Bit, IR::OpSizeAsBits(ReadWidth), 0, Data);\n }\n\n // We're about to clobber flags to grab the sign, so save NZCV.\n SaveNZCV();\n\n // Extract sign and make integer absolute\n auto zero = Constant(0);\n _SubNZCV(OpSize::i64Bit, Data, zero);\n auto sign = _NZCVSelect(OpSize::i64Bit, CondClass::SLT, Constant(0x8000), zero);\n auto absolute = _Neg(OpSize::i64Bit, Data, CondClass::MI);\n\n // left justify the absolute integer\n auto shift = Sub(OpSize::i64Bit, Constant(63), _FindMSB(IR::OpSize::i64Bit, absolute));\n auto shifted = _Lshl(OpSize::i64Bit, absolute, shift);\n\n auto adjusted_exponent = Sub(OpSize::i64Bit, Constant(0x3fff + 63), shift);\n auto zeroed_exponent = _Select(OpSize::i64Bit, OpSize::i64Bit, CondClass::EQ, absolute, zero, zero, adjusted_exponent);\n auto upper = _Or(OpSize::i64Bit, sign, zeroed_exponent);\n\n Ref ConvertedData = _VLoadTwoGPRs(shifted, upper);\n _PushStack(ConvertedData, Invalid(), OpSize::iInvalid);\n}\n\nvoid OpDispatchBuilder::FST(OpcodeArgs, IR::OpSize Width) {\n LOGMAN_THROW_A_FMT(Width == OpSize::i32Bit || Width == OpSize::i64Bit || Width == OpSize::f80Bit, \"Invalid store width for FST\");\n const auto SourceSize = ReducedPrecisionMode ? OpSize::i64Bit : OpSize::f80Bit;\n AddressMode A = DecodeAddress(Op, Op->Dest, MemoryAccessType::DEFAULT, false);\n\n A = SelectAddressMode(this, A, GetGPROpSize(), CTX->HostFeatures.SupportsTSOImm9, false, false, Width);\n _StoreStackMem(SourceSize, Width, A.Base, A.Index, OpSize::iInvalid, A.IndexType, A.IndexScale);\n\n if (Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) {\n _PopStackDestroy();\n }\n}\n\nvoid OpDispatchBuilder::FSTToStack(OpcodeArgs) {\n const uint8_t Offset = Op->OP & 7;\n if (Offset != 0) {\n _StoreStackToStack(Offset);\n }\n\n if (Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) {\n _PopStackDestroy();\n }\n}\n\n// Store integer to memory (possibly with truncation)\nvoid OpDispatchBuilder::FIST(OpcodeArgs, bool Truncate) {\n const auto Size = OpSizeFromSrc(Op);\n Ref Data = _ReadStackValue(0);\n\n // For 16-bit integers, we need to manually check for overflow\n // since _F80CVTInt doesn't handle 16-bit overflow detection properly\n if (Size == OpSize::i16Bit) {\n // Extract the 80-bit float value to check for special cases\n // Get the upper 64 bits which contain sign and exponent and then the exponent from upper.\n Ref Upper = _VExtractToGPR(OpSize::i128Bit, OpSize::i64Bit, Data, 1);\n Ref Exponent = _And(OpSize::i64Bit, Upper, Constant(0x7fff));\n\n // Check for NaN/Infinity: exponent = 0x7fff\n SaveNZCV();\n _TestNZ(OpSize::i64Bit, Exponent, Constant(0x7fff));\n Ref IsSpecial = _NZCVSelect01(CondClass::EQ);\n\n // For overflow detection, check if exponent indicates a value >= 2^15\n // Biased exponent for 2^15 is 0x3fff + 15 = 0x400e\n SubWithFlags(OpSize::i64Bit, Exponent, 0x400e);\n Ref IsOverflow = _NZCVSelect01(CondClass::UGE);\n\n // Set Invalid Operation flag if overflow or special value\n Ref InvalidFlag = _Or(OpSize::i64Bit, IsSpecial, IsOverflow);\n SetRFLAG(InvalidFlag);\n }\n\n Data = _F80CVTInt(Size, Data, Truncate);\n\n StoreResultGPR_WithOpSize(Op, Op->Dest, Data, Size, OpSize::i8Bit);\n\n if ((Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) != 0) {\n _PopStackDestroy();\n }\n}\n\nvoid OpDispatchBuilder::FADD(OpcodeArgs, IR::OpSize Width, bool Integer, OpDispatchBuilder::OpResult ResInST0) {\n if (Op->Src[0].IsNone()) { // Implicit argument case\n auto Offset = Op->OP & 7;\n auto St0 = 0;\n if (ResInST0 == OpResult::RES_STI) {\n _F80AddStack(Offset, St0);\n } else {\n _F80AddStack(St0, Offset);\n }\n if (Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) {\n _PopStackDestroy();\n }\n return;\n }\n\n LOGMAN_THROW_A_FMT(Width != OpSize::f80Bit, \"No 80-bit floats from memory\");\n // We have one memory argument\n Ref Arg {};\n if (Integer) {\n Arg = LoadSourceGPR(Op, Op->Src[0], Op->Flags);\n Arg = _F80CVTToInt(Arg, Width);\n } else {\n Arg = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n Arg = _F80CVTTo(Arg, Width);\n }\n\n // top of stack is at offset zero\n _F80AddValue(0, Arg);\n}\n\nvoid OpDispatchBuilder::FMUL(OpcodeArgs, IR::OpSize Width, bool Integer, OpDispatchBuilder::OpResult ResInST0) {\n if (Op->Src[0].IsNone()) { // Implicit argument case\n auto offset = Op->OP & 7;\n auto st0 = 0;\n if (ResInST0 == OpResult::RES_STI) {\n _F80MulStack(offset, st0);\n } else {\n _F80MulStack(st0, offset);\n }\n if (Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) {\n _PopStackDestroy();\n }\n return;\n }\n\n LOGMAN_THROW_A_FMT(Width != OpSize::f80Bit, \"No 80-bit floats from memory\");\n // We have one memory argument\n Ref arg {};\n if (Integer) {\n arg = LoadSourceGPR(Op, Op->Src[0], Op->Flags);\n arg = _F80CVTToInt(arg, Width);\n } else {\n arg = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n arg = _F80CVTTo(arg, Width);\n }\n\n // top of stack is at offset zero\n _F80MulValue(0, arg);\n\n if (Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) {\n _PopStackDestroy();\n }\n}\n\nvoid OpDispatchBuilder::FDIV(OpcodeArgs, IR::OpSize Width, bool Integer, bool Reverse, OpDispatchBuilder::OpResult ResInST0) {\n if (Op->Src[0].IsNone()) {\n const uint8_t Offset = Op->OP & 7;\n const uint8_t St0 = 0;\n const uint8_t Result = (ResInST0 == OpResult::RES_STI) ? Offset : St0;\n\n if (Reverse ^ (ResInST0 == OpResult::RES_STI)) {\n _F80DivStack(Result, Offset, St0);\n } else {\n _F80DivStack(Result, St0, Offset);\n }\n\n if (Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) {\n _PopStackDestroy();\n }\n return;\n }\n\n LOGMAN_THROW_A_FMT(Width != OpSize::f80Bit, \"No 80-bit floats from memory\");\n // We have one memory argument\n Ref arg {};\n if (Integer) {\n arg = LoadSourceGPR(Op, Op->Src[0], Op->Flags);\n arg = _F80CVTToInt(arg, Width);\n } else {\n arg = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n arg = _F80CVTTo(arg, Width);\n }\n\n // top of stack is at offset zero\n if (Reverse) {\n _F80DivRValue(arg, 0);\n } else {\n _F80DivValue(0, arg);\n }\n\n if (Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) {\n _PopStackDestroy();\n }\n}\n\nvoid OpDispatchBuilder::FSUB(OpcodeArgs, IR::OpSize Width, bool Integer, bool Reverse, OpDispatchBuilder::OpResult ResInST0) {\n if (Op->Src[0].IsNone()) {\n const auto Offset = Op->OP & 7;\n const auto St0 = 0;\n const auto Result = (ResInST0 == OpResult::RES_STI) ? Offset : St0;\n\n if (Reverse ^ (ResInST0 == OpResult::RES_STI)) {\n _F80SubStack(Result, Offset, St0);\n } else {\n _F80SubStack(Result, St0, Offset);\n }\n\n if (Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) {\n _PopStackDestroy();\n }\n return;\n }\n\n LOGMAN_THROW_A_FMT(Width != OpSize::f80Bit, \"No 80-bit floats from memory\");\n // We have one memory argument\n Ref Arg {};\n if (Integer) {\n Arg = LoadSourceGPR(Op, Op->Src[0], Op->Flags);\n Arg = _F80CVTToInt(Arg, Width);\n } else {\n Arg = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n Arg = _F80CVTTo(Arg, Width);\n }\n\n // top of stack is at offset zero\n if (Reverse) {\n _F80SubRValue(Arg, 0);\n } else {\n _F80SubValue(0, Arg);\n }\n\n if (Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) {\n _PopStackDestroy();\n }\n}\n\nRef OpDispatchBuilder::GetX87FTW_Helper() {\n // AbridgedFTWIndex has 1-bit per slot (8 slots). Duplicate each bit to get\n // 2-bits per slot (16-bit result). Duplicating bits is equivalent to\n // Morton interleaving a number with itself. To interleave efficiently two\n // bytes, we use the well-known bit twiddling algorithm:\n //\n // https://graphics.stanford.edu/~seander/bithacks.html#InterleaveBMN\n Ref X = _LoadContextGPR(OpSize::i8Bit, offsetof(FEXCore::Core::CPUState, AbridgedFTW));\n X = _Orlshl(OpSize::i32Bit, X, X, 4);\n X = _And(OpSize::i32Bit, X, Constant(0x0f0f0f0f));\n X = _Orlshl(OpSize::i32Bit, X, X, 2);\n X = _And(OpSize::i32Bit, X, Constant(0x33333333));\n X = _Orlshl(OpSize::i32Bit, X, X, 1);\n X = _And(OpSize::i32Bit, X, Constant(0x55555555));\n X = _Orlshl(OpSize::i32Bit, X, X, 1);\n\n // The above sequence sets valid to 11 and empty to 00, so invert to finalize.\n static_assert(static_cast(FPState::X87Tag::Valid) == 0b00);\n static_assert(static_cast(FPState::X87Tag::Empty) == 0b11);\n return _Xor(OpSize::i32Bit, X, Constant(0xffff));\n}\n\nvoid OpDispatchBuilder::X87FNSTENV(OpcodeArgs) {\n\n\n // 14 bytes for 16bit\n // 2 Bytes : FCW\n // 2 Bytes : FSW\n // 2 bytes : FTW\n // 2 bytes : Instruction offset\n // 2 bytes : Instruction CS selector\n // 2 bytes : Data offset\n // 2 bytes : Data selector\n\n // 28 bytes for 32bit\n // 4 bytes : FCW\n // 4 bytes : FSW\n // 4 bytes : FTW\n // 4 bytes : Instruction pointer\n // 2 bytes : Instruction pointer selector\n // 2 bytes : Opcode\n // 4 bytes : data pointer offset\n // 4 bytes : data pointer selector\n\n // Before we store anything we need to sync our stack to the registers.\n _SyncStackToSlow();\n\n const auto Size = OpSizeFromSrc(Op);\n Ref Mem = LoadSourceGPR(Op, Op->Dest, Op->Flags, {.LoadData = false});\n Mem = AppendSegmentOffset(Mem, Op->Flags);\n\n {\n auto FCW = _LoadContextGPR(OpSize::i16Bit, offsetof(FEXCore::Core::CPUState, FCW));\n _StoreMemGPR(Size, Mem, FCW, Size);\n }\n\n { _StoreMemGPR(Size, ReconstructFSW_Helper(), Mem, Constant(IR::OpSizeToSize(Size) * 1), Size, MemOffsetType::SXTX, 1); }\n\n auto ZeroConst = Constant(0);\n\n {\n // FTW\n _StoreMemGPR(Size, GetX87FTW_Helper(), Mem, Constant(IR::OpSizeToSize(Size) * 2), Size, MemOffsetType::SXTX, 1);\n }\n\n {\n // Instruction Offset\n _StoreMemGPR(Size, ZeroConst, Mem, Constant(IR::OpSizeToSize(Size) * 3), Size, MemOffsetType::SXTX, 1);\n }\n\n {\n // Instruction CS selector (+ Opcode)\n _StoreMemGPR(Size, ZeroConst, Mem, Constant(IR::OpSizeToSize(Size) * 4), Size, MemOffsetType::SXTX, 1);\n }\n\n {\n // Data pointer offset\n _StoreMemGPR(Size, ZeroConst, Mem, Constant(IR::OpSizeToSize(Size) * 5), Size, MemOffsetType::SXTX, 1);\n }\n\n {\n // Data pointer selector\n _StoreMemGPR(Size, ZeroConst, Mem, Constant(IR::OpSizeToSize(Size) * 6), Size, MemOffsetType::SXTX, 1);\n }\n}\n\nRef OpDispatchBuilder::ReconstructX87StateFromFSW_Helper(Ref FSW) {\n auto Top = _Bfe(OpSize::i32Bit, 3, 11, FSW);\n SetX87Top(Top);\n\n auto C0 = _Bfe(OpSize::i32Bit, 1, 8, FSW);\n auto C1 = _Bfe(OpSize::i32Bit, 1, 9, FSW);\n auto C2 = _Bfe(OpSize::i32Bit, 1, 10, FSW);\n auto C3 = _Bfe(OpSize::i32Bit, 1, 14, FSW);\n auto IE = _Bfe(OpSize::i32Bit, 1, 0, FSW);\n\n SetRFLAG(C0);\n SetRFLAG(C1);\n SetRFLAG(C2);\n SetRFLAG(C3);\n SetRFLAG(IE);\n return Top;\n}\n\nvoid OpDispatchBuilder::X87LDENV(OpcodeArgs) {\n _StackForceSlow();\n\n const auto Size = OpSizeFromSrc(Op);\n Ref Mem = LoadSourceGPR(Op, Op->Src[0], Op->Flags, {.LoadData = false});\n Mem = AppendSegmentOffset(Mem, Op->Flags);\n\n auto NewFCW = _LoadMemGPR(OpSize::i16Bit, Mem, OpSize::i16Bit);\n _StoreContextGPR(OpSize::i16Bit, NewFCW, offsetof(FEXCore::Core::CPUState, FCW));\n\n Ref MemLocation = Add(OpSize::i64Bit, Mem, IR::OpSizeToSize(Size) * 1);\n auto NewFSW = _LoadMemGPR(Size, MemLocation, Size);\n ReconstructX87StateFromFSW_Helper(NewFSW);\n\n {\n // FTW\n Ref MemLocation = Add(OpSize::i64Bit, Mem, IR::OpSizeToSize(Size) * 2);\n SetX87FTW(_LoadMemGPR(Size, MemLocation, Size));\n }\n}\n\nvoid OpDispatchBuilder::X87FNSAVE(OpcodeArgs) {\n _SyncStackToSlow();\n\n // 14 bytes for 16bit\n // 2 Bytes : FCW\n // 2 Bytes : FSW\n // 2 bytes : FTW\n // 2 bytes : Instruction offset\n // 2 bytes : Instruction CS selector\n // 2 bytes : Data offset\n // 2 bytes : Data selector\n\n // 28 bytes for 32bit\n // 4 bytes : FCW\n // 4 bytes : FSW\n // 4 bytes : FTW\n // 4 bytes : Instruction pointer\n // 2 bytes : instruction pointer selector\n // 2 bytes : Opcode\n // 4 bytes : data pointer offset\n // 4 bytes : data pointer selector\n const auto Size = OpSizeFromDst(Op);\n Ref Mem = MakeSegmentAddress(Op, Op->Dest);\n Ref Top = GetX87Top();\n {\n auto FCW = _LoadContextGPR(OpSize::i16Bit, offsetof(FEXCore::Core::CPUState, FCW));\n _StoreMemGPR(Size, Mem, FCW, Size);\n }\n\n { _StoreMemGPR(Size, ReconstructFSW_Helper(), Mem, Constant(IR::OpSizeToSize(Size) * 1), Size, MemOffsetType::SXTX, 1); }\n\n auto ZeroConst = Constant(0);\n\n {\n // FTW\n _StoreMemGPR(Size, GetX87FTW_Helper(), Mem, Constant(IR::OpSizeToSize(Size) * 2), Size, MemOffsetType::SXTX, 1);\n }\n\n {\n // Instruction Offset\n _StoreMemGPR(Size, ZeroConst, Mem, Constant(IR::OpSizeToSize(Size) * 3), Size, MemOffsetType::SXTX, 1);\n }\n\n {\n // Instruction CS selector (+ Opcode)\n _StoreMemGPR(Size, ZeroConst, Mem, Constant(IR::OpSizeToSize(Size) * 4), Size, MemOffsetType::SXTX, 1);\n }\n\n {\n // Data pointer offset\n _StoreMemGPR(Size, ZeroConst, Mem, Constant(IR::OpSizeToSize(Size) * 5), Size, MemOffsetType::SXTX, 1);\n }\n\n {\n // Data pointer selector\n _StoreMemGPR(Size, ZeroConst, Mem, Constant(IR::OpSizeToSize(Size) * 6), Size, MemOffsetType::SXTX, 1);\n }\n\n auto SevenConst = Constant(7);\n const auto LoadSize = ReducedPrecisionMode ? OpSize::i64Bit : OpSize::i128Bit;\n for (int i = 0; i < 7; ++i) {\n Ref data = _LoadContextFPRIndexed(Top, LoadSize, MMBaseOffset(), IR::OpSizeToSize(OpSize::i128Bit));\n if (ReducedPrecisionMode) {\n data = _F80CVTTo(data, OpSize::i64Bit);\n }\n _StoreMemFPR(OpSize::i128Bit, data, Mem, Constant((IR::OpSizeToSize(Size) * 7) + (10 * i)), OpSize::i8Bit, MemOffsetType::SXTX, 1);\n Top = _And(OpSize::i32Bit, Add(OpSize::i32Bit, Top, 1), SevenConst);\n }\n\n // The final st(7) needs a bit of special handling here\n Ref data = _LoadContextFPRIndexed(Top, LoadSize, MMBaseOffset(), IR::OpSizeToSize(OpSize::i128Bit));\n if (ReducedPrecisionMode) {\n data = _F80CVTTo(data, OpSize::i64Bit);\n }\n // ST7 broken in to two parts\n // Lower 64bits [63:0]\n // upper 16 bits [79:64]\n _StoreMemFPR(OpSize::i64Bit, data, Mem, Constant((IR::OpSizeToSize(Size) * 7) + (7 * 10)), OpSize::i8Bit, MemOffsetType::SXTX, 1);\n auto topBytes = _VDupElement(OpSize::i128Bit, OpSize::i16Bit, data, 4);\n _StoreMemFPR(OpSize::i16Bit, topBytes, Mem, Constant((IR::OpSizeToSize(Size) * 7) + (7 * 10) + 8), OpSize::i8Bit, MemOffsetType::SXTX, 1);\n\n // reset to default\n FNINIT(Op);\n}\n\nvoid OpDispatchBuilder::X87FRSTOR(OpcodeArgs) {\n _StackForceSlow();\n const auto Size = OpSizeFromSrc(Op);\n Ref Mem = MakeSegmentAddress(Op, Op->Src[0]);\n\n auto NewFCW = _LoadMemGPR(OpSize::i16Bit, Mem, OpSize::i16Bit);\n _StoreContextGPR(OpSize::i16Bit, NewFCW, offsetof(FEXCore::Core::CPUState, FCW));\n if (ReducedPrecisionMode) {\n // ignore the rounding precision, we're always 64-bit in F64.\n // extract rounding mode\n Ref roundingMode = NewFCW;\n auto roundShift = Constant(10);\n auto roundMask = Constant(3);\n roundingMode = _Lshr(OpSize::i32Bit, roundingMode, roundShift);\n roundingMode = _And(OpSize::i32Bit, roundingMode, roundMask);\n _SetRoundingMode(roundingMode, false, roundingMode);\n }\n\n auto NewFSW = _LoadMemGPR(Size, Mem, Constant(IR::OpSizeToSize(Size) * 1), Size, MemOffsetType::SXTX, 1);\n Ref Top = ReconstructX87StateFromFSW_Helper(NewFSW);\n {\n // FTW\n SetX87FTW(_LoadMemGPR(Size, Mem, Constant(IR::OpSizeToSize(Size) * 2), Size, MemOffsetType::SXTX, 1));\n }\n\n auto SevenConst = Constant(7);\n auto low = Constant(~0ULL);\n auto high = Constant(0xFFFF);\n Ref Mask = _VLoadTwoGPRs(low, high);\n const auto StoreSize = ReducedPrecisionMode ? OpSize::i64Bit : OpSize::i128Bit;\n for (int i = 0; i < 7; ++i) {\n Ref Reg = _LoadMemFPR(OpSize::i128Bit, Mem, Constant((IR::OpSizeToSize(Size) * 7) + (10 * i)), OpSize::i8Bit, MemOffsetType::SXTX, 1);\n // Mask off the top bits\n Reg = _VAnd(OpSize::i128Bit, OpSize::i128Bit, Reg, Mask);\n if (ReducedPrecisionMode) {\n // Convert to double precision\n Reg = _F80CVT(OpSize::i64Bit, Reg);\n }\n _StoreContextFPRIndexed(Reg, Top, StoreSize, MMBaseOffset(), IR::OpSizeToSize(OpSize::i128Bit));\n\n Top = _And(OpSize::i32Bit, Add(OpSize::i32Bit, Top, 1), SevenConst);\n }\n\n // The final st(7) needs a bit of special handling here\n // ST7 broken in to two parts\n // Lower 64bits [63:0]\n // upper 16 bits [79:64]\n Ref Reg = _LoadMemFPR(OpSize::i64Bit, Mem, Constant((IR::OpSizeToSize(Size) * 7) + (10 * 7)), OpSize::i8Bit, MemOffsetType::SXTX, 1);\n Ref RegHigh = _LoadMemFPR(OpSize::i16Bit, Mem, Constant((IR::OpSizeToSize(Size) * 7) + (10 * 7) + 8), OpSize::i8Bit, MemOffsetType::SXTX, 1);\n Reg = _VInsElement(OpSize::i128Bit, OpSize::i16Bit, 4, 0, Reg, RegHigh);\n if (ReducedPrecisionMode) {\n Reg = _F80CVT(OpSize::i64Bit, Reg); // Convert to double precision\n }\n _StoreContextFPRIndexed(Reg, Top, StoreSize, MMBaseOffset(), IR::OpSizeToSize(OpSize::i128Bit));\n}\n\n// Load / Store Control Word\nvoid OpDispatchBuilder::X87FSTCW(OpcodeArgs) {\n auto FCW = _LoadContextGPR(OpSize::i16Bit, offsetof(FEXCore::Core::CPUState, FCW));\n StoreResultGPR(Op, FCW);\n}\n\nvoid OpDispatchBuilder::X87FLDCW(OpcodeArgs) {\n // FIXME: Because loading control flags will affect several instructions in fast path, we might have\n // to switch for now to slow mode whenever these are manually changed.\n // Remove the next line and try DF_04.asm in fast path.\n _StackForceSlow();\n Ref NewFCW = LoadSourceGPR(Op, Op->Src[0], Op->Flags);\n _StoreContextGPR(OpSize::i16Bit, NewFCW, offsetof(FEXCore::Core::CPUState, FCW));\n}\n\nvoid OpDispatchBuilder::FXCH(OpcodeArgs) {\n uint8_t Offset = Op->OP & 7;\n // fxch st0, st0 is for us essentially a nop\n if (Offset != 0) {\n _F80StackXchange(Offset);\n }\n SetRFLAG(Constant(0));\n}\n\nvoid OpDispatchBuilder::X87FYL2X(OpcodeArgs, bool IsFYL2XP1) {\n if (IsFYL2XP1) {\n // create an add between top of stack and 1.\n Ref One = ReducedPrecisionMode ? _VCastFromGPR(OpSize::i64Bit, OpSize::i64Bit, Constant(0x3FF0000000000000)) :\n LoadAndCacheNamedVectorConstant(OpSize::i128Bit, NamedVectorConstant::NAMED_VECTOR_X87_ONE);\n _F80AddValue(0, One);\n }\n\n _F80FYL2XStack();\n}\n\nvoid OpDispatchBuilder::FCOMI(OpcodeArgs, IR::OpSize Width, bool Integer, OpDispatchBuilder::FCOMIFlags WhichFlags, bool PopTwice) {\n Ref arg {};\n Ref b {};\n\n Ref Res {};\n if (Op->Src[0].IsNone()) {\n // Implicit arg\n uint8_t Offset = Op->OP & 7;\n Res = _F80CmpStack(Offset);\n } else {\n if (Width == OpSize::i16Bit || Width == OpSize::i32Bit || Width == OpSize::i64Bit) {\n // Memory arg\n if (Integer) {\n arg = LoadSourceGPR(Op, Op->Src[0], Op->Flags);\n b = _F80CVTToInt(arg, Width);\n } else {\n arg = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n b = _F80CVTTo(arg, Width);\n }\n } else {\n FEX_UNREACHABLE;\n }\n Res = _F80CmpValue(b);\n }\n\n Ref HostFlag_CF = _Bfe(OpSize::i64Bit, 1, FCMP_FLAG_LT, Res);\n Ref HostFlag_ZF = _Bfe(OpSize::i64Bit, 1, FCMP_FLAG_EQ, Res);\n Ref HostFlag_Unordered = _Bfe(OpSize::i64Bit, 1, FCMP_FLAG_UNORDERED, Res);\n HostFlag_CF = _Or(OpSize::i32Bit, HostFlag_CF, HostFlag_Unordered);\n HostFlag_ZF = _Or(OpSize::i32Bit, HostFlag_ZF, HostFlag_Unordered);\n\n if (WhichFlags == FCOMIFlags::FLAGS_X87) {\n SetRFLAG(HostFlag_CF);\n SetRFLAG(Constant(0));\n SetRFLAG(HostFlag_Unordered);\n SetRFLAG(HostFlag_ZF);\n } else {\n // OF, SF, AF, PF all undefined\n SetCFDirect(HostFlag_CF);\n SetRFLAG(HostFlag_ZF);\n\n // PF is stored inverted, so invert from the host flag.\n // TODO: This could perhaps be optimized?\n auto PF = _Xor(OpSize::i32Bit, HostFlag_Unordered, Constant(1));\n SetRFLAG(PF);\n }\n\n // Set Invalid Operation flag when unordered (NaN comparison)\n SetRFLAG(HostFlag_Unordered);\n\n if (PopTwice) {\n _PopStackDestroy();\n _PopStackDestroy();\n } else if ((Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) != 0) {\n _PopStackDestroy();\n }\n}\n\nvoid OpDispatchBuilder::FTST(OpcodeArgs) {\n Ref Res = _F80StackTest(0);\n\n Ref HostFlag_CF = _Bfe(OpSize::i64Bit, 1, FCMP_FLAG_LT, Res);\n Ref HostFlag_ZF = _Bfe(OpSize::i64Bit, 1, FCMP_FLAG_EQ, Res);\n Ref HostFlag_Unordered = _Bfe(OpSize::i64Bit, 1, FCMP_FLAG_UNORDERED, Res);\n HostFlag_CF = _Or(OpSize::i32Bit, HostFlag_CF, HostFlag_Unordered);\n HostFlag_ZF = _Or(OpSize::i32Bit, HostFlag_ZF, HostFlag_Unordered);\n\n SetRFLAG(HostFlag_CF);\n SetRFLAG(Constant(0));\n SetRFLAG(HostFlag_Unordered);\n SetRFLAG(HostFlag_ZF);\n\n // Set Invalid Operation flag when unordered (NaN comparison)\n SetRFLAG(HostFlag_Unordered);\n}\n\nvoid OpDispatchBuilder::X87OpHelper(OpcodeArgs, FEXCore::IR::IROps IROp, bool ZeroC2) {\n DeriveOp(Result, IROp, _F80SCALEStack());\n if (ZeroC2) {\n SetRFLAG(Constant(0));\n }\n}\n\nvoid OpDispatchBuilder::X87ModifySTP(OpcodeArgs, bool Inc) {\n if (Inc) {\n _IncStackTop();\n } else {\n _DecStackTop();\n }\n}\n\n// Operations dealing with loading and storing environment pieces\n\n// Reconstruct as a constant the Status Word of the FPU.\n// We only track stack top and each of the code conditions (C flags)\n// Top is 3 bits at bit 11.\n// C0 is 1 bit at bit 8.\n// C1 is 1 bit at bit 9.\n// C2 is 1 bit at bit 10.\n// C3 is 1 bit at bit 14.\n// Optionally we can pass a pre calculated value for Top, otherwise we calculate it\n// during the function runtime.\nRef OpDispatchBuilder::ReconstructFSW_Helper(Ref T) {\n // Start with the top value\n auto Top = T ? T : GetX87Top();\n Ref FSW = _Lshl(OpSize::i64Bit, Top, Constant(11));\n\n // We must construct the FSW from our various bits\n auto C0 = GetRFLAG(FEXCore::X86State::X87FLAG_C0_LOC);\n FSW = _Orlshl(OpSize::i64Bit, FSW, C0, 8);\n\n auto C1 = GetRFLAG(FEXCore::X86State::X87FLAG_C1_LOC);\n FSW = _Orlshl(OpSize::i64Bit, FSW, C1, 9);\n\n auto C2 = GetRFLAG(FEXCore::X86State::X87FLAG_C2_LOC);\n FSW = _Orlshl(OpSize::i64Bit, FSW, C2, 10);\n\n auto C3 = GetRFLAG(FEXCore::X86State::X87FLAG_C3_LOC);\n FSW = _Orlshl(OpSize::i64Bit, FSW, C3, 14);\n\n auto IE = GetRFLAG(FEXCore::X86State::X87FLAG_IE_LOC);\n FSW = _Or(OpSize::i64Bit, FSW, IE);\n\n return FSW;\n}\n\n// Store Status Word\n// There's no load Status Word instruction but you can load it through frstor\n// or fldenv.\nvoid OpDispatchBuilder::X87FNSTSW(OpcodeArgs) {\n Ref TopValue = _SyncStackToSlow();\n Ref StatusWord = ReconstructFSW_Helper(TopValue);\n StoreResultGPR(Op, StatusWord);\n}\n\nvoid OpDispatchBuilder::FNCLEX(OpcodeArgs) {\n // Clear the exception flag bit\n SetRFLAG(_Constant(0));\n}\n\nvoid OpDispatchBuilder::FNINIT(OpcodeArgs) {\n _SyncStackToSlow(); // Invalidate x87 register caches\n\n auto Zero = Constant(0);\n\n if (ReducedPrecisionMode) {\n _SetRoundingMode(Zero, false, Zero);\n }\n\n // Init FCW to 0x037F\n auto NewFCW = Constant(0x037F);\n _StoreContextGPR(OpSize::i16Bit, NewFCW, offsetof(FEXCore::Core::CPUState, FCW));\n\n // Set top to zero\n SetX87Top(Zero);\n // Tags all get marked as invalid\n _StoreContextGPR(OpSize::i8Bit, Zero, offsetof(FEXCore::Core::CPUState, AbridgedFTW));\n\n // Reinits the simulated stack\n _InitStack();\n\n SetRFLAG(Zero);\n SetRFLAG(Zero);\n SetRFLAG(Zero);\n SetRFLAG(Zero);\n SetRFLAG(Zero);\n}\n\nvoid OpDispatchBuilder::X87FFREE(OpcodeArgs) {\n _InvalidateStack(Op->OP & 7);\n}\n\nvoid OpDispatchBuilder::X87EMMS(OpcodeArgs) {\n // Tags all get set to 0b11\n _InvalidateStack(0xff);\n}\n\nvoid OpDispatchBuilder::X87FCMOV(OpcodeArgs) {\n CalculateDeferredFlags();\n\n uint16_t Opcode = Op->OP & 0b1111'1111'1000;\n uint8_t CC = 0;\n\n switch (Opcode) {\n case 0x3'C0:\n CC = 0x3; // JNC\n break;\n case 0x2'C0:\n CC = 0x2; // JC\n break;\n case 0x2'C8:\n CC = 0x4; // JE\n break;\n case 0x3'C8:\n CC = 0x5; // JNE\n break;\n case 0x2'D0:\n CC = 0x6; // JNA\n break;\n case 0x3'D0:\n CC = 0x7; // JA\n break;\n case 0x2'D8:\n CC = 0xA; // JP\n break;\n case 0x3'D8:\n CC = 0xB; // JNP\n break;\n default: LOGMAN_MSG_A_FMT(\"Unhandled FCMOV op: 0x{:x}\", Opcode); break;\n }\n\n Ref VecCond = _VDupFromGPR(OpSize::i128Bit, OpSize::i64Bit, SelectCC0All1(CC));\n _F80VBSLStack(OpSize::i128Bit, VecCond, Op->OP & 7, 0);\n}\n\nvoid OpDispatchBuilder::X87FXAM(OpcodeArgs) {\n auto a = _ReadStackValue(0);\n Ref Result =\n ReducedPrecisionMode ? _VExtractToGPR(OpSize::i64Bit, OpSize::i64Bit, a, 0) : _VExtractToGPR(OpSize::i128Bit, OpSize::i64Bit, a, 1);\n\n // Extract the sign bit\n Result = ReducedPrecisionMode ? _Bfe(OpSize::i64Bit, 1, 63, Result) : _Bfe(OpSize::i64Bit, 1, 15, Result);\n SetRFLAG(Result);\n\n // Claim this is a normal number\n // We don't support anything else\n auto TopValid = _StackValidTag(0);\n\n // In the case of top being invalid then C3:C2:C0 is 0b101\n auto C3 = Select01(OpSize::i32Bit, CondClass::NEQ, TopValid, Constant(1));\n\n auto C2 = TopValid;\n auto C0 = C3; // Mirror C3 until something other than zero is supported\n SetRFLAG(C0);\n SetRFLAG(C2);\n SetRFLAG(C3);\n}\n\nvoid OpDispatchBuilder::X87FXTRACT(OpcodeArgs) {\n auto Top = _ReadStackValue(0);\n\n _PopStackDestroy();\n auto Exp = _F80XTRACT_EXP(Top);\n auto Sig = _F80XTRACT_SIG(Top);\n _PushStack(Exp, Invalid(), OpSize::iInvalid);\n _PushStack(Sig, Invalid(), OpSize::iInvalid);\n}\n\n} // namespace FEXCore::IR\n" }, { "path": "FEXCore/Source/Interface/Core/OpcodeDispatcher/X87F64.cpp", "content": "// SPDX-License-Identifier: MIT\n/*\n$info$\ntags: frontend|x86-to-ir, opcodes|dispatcher-implementations\ndesc: Handles x86/64 x87 to IR\n$end_info$\n*/\n\n#include \"Interface/Core/OpcodeDispatcher.h\"\n#include \"Interface/Core/X86Tables/X86Tables.h\"\n#include \"Interface/IR/IR.h\"\n\n#include \n#include \n#include \n#include \n\n#include \n#include \n\nnamespace FEXCore::IR {\nclass OrderedNode;\n\n#define OpcodeArgs [[maybe_unused]] FEXCore::X86Tables::DecodedOp Op\n\nvoid OpDispatchBuilder::X87LDENVF64(OpcodeArgs) {\n _StackForceSlow();\n\n const auto Size = OpSizeFromSrc(Op);\n Ref Mem = MakeSegmentAddress(Op, Op->Src[0]);\n\n auto NewFCW = _LoadMemGPR(OpSize::i16Bit, Mem, OpSize::i16Bit);\n // ignore the rounding precision, we're always 64-bit in F64.\n // extract rounding mode\n Ref roundingMode = _Bfe(OpSize::i32Bit, 3, 10, NewFCW);\n _SetRoundingMode(roundingMode, false, roundingMode);\n _StoreContextGPR(OpSize::i16Bit, NewFCW, offsetof(FEXCore::Core::CPUState, FCW));\n\n auto NewFSW = _LoadMemGPR(Size, Mem, Constant(IR::OpSizeToSize(Size)), Size, MemOffsetType::SXTX, 1);\n ReconstructX87StateFromFSW_Helper(NewFSW);\n\n {\n // FTW\n SetX87FTW(_LoadMemGPR(Size, Mem, Constant(IR::OpSizeToSize(Size) * 2), Size, MemOffsetType::SXTX, 1));\n }\n}\n\nvoid OpDispatchBuilder::X87FLDCWF64(OpcodeArgs) {\n _StackForceSlow();\n\n Ref NewFCW = LoadSourceGPR(Op, Op->Src[0], Op->Flags);\n // ignore the rounding precision, we're always 64-bit in F64.\n // extract rounding mode\n Ref roundingMode = _Bfe(OpSize::i32Bit, 3, 10, NewFCW);\n _SetRoundingMode(roundingMode, false, roundingMode);\n _StoreContextGPR(OpSize::i16Bit, NewFCW, offsetof(FEXCore::Core::CPUState, FCW));\n}\n\n// F64 ops\n// Float load op with memory operand\nvoid OpDispatchBuilder::FLDF64(OpcodeArgs, IR::OpSize Width) {\n Ref Data = LoadSourceFPR_WithOpSize(Op, Op->Src[0], Width, Op->Flags);\n // Convert to 64bit float\n Ref ConvertedData = Data;\n if (Width == OpSize::i32Bit) {\n ConvertedData = _Float_FToF(OpSize::i64Bit, OpSize::i32Bit, Data);\n } else if (Width == OpSize::f80Bit) {\n ConvertedData = _F80CVT(OpSize::i64Bit, Data);\n }\n _PushStack(ConvertedData, Data, Width);\n}\n\nvoid OpDispatchBuilder::FBLDF64(OpcodeArgs) {\n // Read from memory\n Ref Data = LoadSourceFPR_WithOpSize(Op, Op->Src[0], OpSize::f80Bit, Op->Flags);\n Ref ConvertedData = _F80BCDLoad(Data);\n ConvertedData = _F80CVT(OpSize::i64Bit, ConvertedData);\n _PushStack(ConvertedData, Invalid(), OpSize::iInvalid);\n}\n\nvoid OpDispatchBuilder::FBSTPF64(OpcodeArgs) {\n Ref converted = _F80CVTTo(_ReadStackValue(0), OpSize::i64Bit);\n converted = _F80BCDStore(converted);\n StoreResultFPR_WithOpSize(Op, Op->Dest, converted, OpSize::f80Bit, OpSize::i8Bit);\n _PopStackDestroy();\n}\n\nvoid OpDispatchBuilder::FLDF64_Const(OpcodeArgs, uint64_t Num) {\n auto Data = _VCastFromGPR(OpSize::i64Bit, OpSize::i64Bit, Constant(Num));\n _PushStack(Data, Data, OpSize::i64Bit);\n}\n\nvoid OpDispatchBuilder::FILDF64(OpcodeArgs) {\n const auto ReadWidth = OpSizeFromSrc(Op);\n\n // Read from memory\n Ref Data = LoadSourceGPR_WithOpSize(Op, Op->Src[0], ReadWidth, Op->Flags);\n if (ReadWidth == OpSize::i16Bit) {\n Data = _Sbfe(OpSize::i64Bit, IR::OpSizeAsBits(ReadWidth), 0, Data);\n }\n auto ConvertedData = _Float_FromGPR_S(OpSize::i64Bit, ReadWidth == OpSize::i32Bit ? OpSize::i32Bit : OpSize::i64Bit, Data);\n _PushStack(ConvertedData, Invalid(), OpSize::iInvalid);\n}\n\nvoid OpDispatchBuilder::FISTF64(OpcodeArgs, bool Truncate) {\n const auto Size = OpSizeFromSrc(Op);\n\n Ref data = _ReadStackValue(0);\n if (Truncate) {\n data = _Float_ToGPR_ZS(Size == OpSize::i32Bit ? OpSize::i32Bit : OpSize::i64Bit, OpSize::i64Bit, data);\n } else {\n data = _Float_ToGPR_S(Size == OpSize::i32Bit ? OpSize::i32Bit : OpSize::i64Bit, OpSize::i64Bit, data);\n }\n StoreResultGPR_WithOpSize(Op, Op->Dest, data, Size, OpSize::i8Bit);\n\n if ((Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) != 0) {\n _PopStackDestroy();\n }\n}\n\nvoid OpDispatchBuilder::FADDF64(OpcodeArgs, IR::OpSize Width, bool Integer, OpDispatchBuilder::OpResult ResInST0) {\n if (Op->Src[0].IsNone()) { // Implicit argument case\n auto Offset = Op->OP & 7;\n auto St0 = 0;\n if (ResInST0 == OpResult::RES_STI) {\n _F80AddStack(Offset, St0);\n } else {\n _F80AddStack(St0, Offset);\n }\n if (Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) {\n _PopStackDestroy();\n }\n return;\n }\n\n // We have one memory argument\n Ref arg {};\n\n if (Integer) {\n arg = LoadSourceGPR(Op, Op->Src[0], Op->Flags);\n if (Width == OpSize::i16Bit) {\n arg = _Sbfe(OpSize::i64Bit, 16, 0, arg);\n }\n arg = _Float_FromGPR_S(OpSize::i64Bit, Width == OpSize::i64Bit ? OpSize::i64Bit : OpSize::i32Bit, arg);\n } else if (Width == OpSize::i32Bit) {\n arg = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n arg = _Float_FToF(OpSize::i64Bit, OpSize::i32Bit, arg);\n } else if (Width == OpSize::i64Bit) {\n arg = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n } else {\n FEX_UNREACHABLE;\n }\n\n // top of stack is at offset zero\n _F80AddValue(0, arg);\n}\n\n// FIXME: following is very similar to FADDF64\nvoid OpDispatchBuilder::FMULF64(OpcodeArgs, IR::OpSize Width, bool Integer, OpDispatchBuilder::OpResult ResInST0) {\n if (Op->Src[0].IsNone()) { // Implicit argument case\n auto offset = Op->OP & 7;\n auto st0 = 0;\n if (ResInST0 == OpResult::RES_STI) {\n _F80MulStack(offset, st0);\n } else {\n _F80MulStack(st0, offset);\n }\n if (Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) {\n _PopStackDestroy();\n }\n return;\n }\n\n // We have one memory argument\n Ref arg {};\n\n if (Integer) {\n arg = LoadSourceGPR(Op, Op->Src[0], Op->Flags);\n if (Width == OpSize::i16Bit) {\n arg = _Sbfe(OpSize::i64Bit, 16, 0, arg);\n }\n arg = _Float_FromGPR_S(OpSize::i64Bit, Width == OpSize::i64Bit ? OpSize::i64Bit : OpSize::i32Bit, arg);\n } else if (Width == OpSize::i32Bit) {\n arg = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n arg = _Float_FToF(OpSize::i64Bit, OpSize::i32Bit, arg);\n } else if (Width == OpSize::i64Bit) {\n arg = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n } else {\n FEX_UNREACHABLE;\n }\n\n // top of stack is at offset zero\n _F80MulValue(0, arg);\n\n if (Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) {\n _PopStackDestroy();\n }\n}\n\nvoid OpDispatchBuilder::FDIVF64(OpcodeArgs, IR::OpSize Width, bool Integer, bool Reverse, OpDispatchBuilder::OpResult ResInST0) {\n if (Op->Src[0].IsNone()) {\n const auto offset = Op->OP & 7;\n const auto st0 = 0;\n\n if (Reverse) {\n if (ResInST0 == OpResult::RES_STI) {\n _F80DivStack(offset, st0, offset);\n } else {\n _F80DivStack(st0, offset, st0);\n }\n } else {\n if (ResInST0 == OpResult::RES_STI) {\n _F80DivStack(offset, offset, st0);\n } else {\n _F80DivStack(st0, st0, offset);\n }\n }\n\n if (Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) {\n _PopStackDestroy();\n }\n return;\n }\n\n // We have one memory argument\n Ref Arg {};\n\n if (Width == OpSize::i16Bit || Width == OpSize::i32Bit || Width == OpSize::i64Bit) {\n if (Integer) {\n Arg = LoadSourceGPR(Op, Op->Src[0], Op->Flags);\n if (Width == OpSize::i16Bit) {\n Arg = _Sbfe(OpSize::i64Bit, 16, 0, Arg);\n }\n Arg = _Float_FromGPR_S(OpSize::i64Bit, Width == OpSize::i64Bit ? OpSize::i64Bit : OpSize::i32Bit, Arg);\n } else if (Width == OpSize::i32Bit) {\n Arg = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n Arg = _Float_FToF(OpSize::i64Bit, OpSize::i32Bit, Arg);\n } else if (Width == OpSize::i64Bit) {\n Arg = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n }\n } else {\n FEX_UNREACHABLE;\n }\n\n // top of stack is at offset zero\n if (Reverse) {\n _F80DivRValue(Arg, 0);\n } else {\n _F80DivValue(0, Arg);\n }\n\n if (Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) {\n _PopStackDestroy();\n }\n}\n\nvoid OpDispatchBuilder::FSUBF64(OpcodeArgs, IR::OpSize Width, bool Integer, bool Reverse, OpDispatchBuilder::OpResult ResInST0) {\n if (Op->Src[0].IsNone()) {\n const auto Offset = Op->OP & 7;\n const auto St0 = 0;\n\n if (Reverse) {\n if (ResInST0 == OpResult::RES_STI) {\n _F80SubStack(Offset, St0, Offset);\n } else {\n _F80SubStack(St0, Offset, St0);\n }\n } else {\n if (ResInST0 == OpResult::RES_STI) {\n _F80SubStack(Offset, Offset, St0);\n } else {\n _F80SubStack(St0, St0, Offset);\n }\n }\n\n if (Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) {\n _PopStackDestroy();\n }\n return;\n }\n\n // We have one memory argument\n Ref arg {};\n\n if (Width == OpSize::i16Bit || Width == OpSize::i32Bit || Width == OpSize::i64Bit) {\n if (Integer) {\n arg = LoadSourceGPR(Op, Op->Src[0], Op->Flags);\n if (Width == OpSize::i16Bit) {\n arg = _Sbfe(OpSize::i64Bit, 16, 0, arg);\n }\n arg = _Float_FromGPR_S(OpSize::i64Bit, Width == OpSize::i64Bit ? OpSize::i64Bit : OpSize::i32Bit, arg);\n } else if (Width == OpSize::i32Bit) {\n arg = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n arg = _Float_FToF(OpSize::i64Bit, OpSize::i32Bit, arg);\n } else if (Width == OpSize::i64Bit) {\n arg = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n }\n } else {\n FEX_UNREACHABLE;\n }\n\n // top of stack is at offset zero\n if (Reverse) {\n _F80SubRValue(arg, 0);\n } else {\n _F80SubValue(0, arg);\n }\n\n if (Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) {\n _PopStackDestroy();\n }\n}\n\nvoid OpDispatchBuilder::FTSTF64(OpcodeArgs) {\n // We are going to clobber NZCV, make sure it's in a GPR first.\n SaveNZCV();\n\n // Now we do our comparison.\n _F80StackTest(0);\n ConvertNZCVToX87();\n}\n\nvoid OpDispatchBuilder::FCOMIF64(OpcodeArgs, IR::OpSize Width, bool Integer, OpDispatchBuilder::FCOMIFlags WhichFlags, bool PopTwice) {\n Ref arg {};\n Ref b {};\n\n if (Op->Src[0].IsNone()) {\n // Implicit arg\n uint8_t offset = Op->OP & 7;\n b = _ReadStackValue(offset);\n } else if (Width == OpSize::i16Bit || Width == OpSize::i32Bit || Width == OpSize::i64Bit) {\n // Memory arg\n if (Integer) {\n arg = LoadSourceGPR(Op, Op->Src[0], Op->Flags);\n if (Width == OpSize::i16Bit) {\n arg = _Sbfe(OpSize::i64Bit, 16, 0, arg);\n }\n b = _Float_FromGPR_S(OpSize::i64Bit, Width == OpSize::i64Bit ? OpSize::i64Bit : OpSize::i32Bit, arg);\n } else if (Width == OpSize::i32Bit) {\n arg = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n b = _Float_FToF(OpSize::i64Bit, OpSize::i32Bit, arg);\n } else if (Width == OpSize::i64Bit) {\n b = LoadSourceFPR(Op, Op->Src[0], Op->Flags);\n }\n } else {\n FEX_UNREACHABLE;\n }\n\n if (WhichFlags == FCOMIFlags::FLAGS_X87) {\n // We are going to clobber NZCV, make sure it's in a GPR first.\n SaveNZCV();\n\n _F80CmpValue(b);\n ConvertNZCVToX87();\n } else {\n HandleNZCVWrite();\n _F80CmpValue(b);\n ComissFlags(true /* InvalidateAF */);\n }\n\n if (PopTwice) {\n _PopStackDestroy();\n _PopStackDestroy();\n } else if ((Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) != 0) {\n _PopStackDestroy();\n }\n}\n\nvoid OpDispatchBuilder::X87FXTRACTF64(OpcodeArgs) {\n // Split node into SIG and EXP while handling the special zero case.\n // i.e. if val == 0.0, then sig = 0.0, exp = -inf\n // if val == -0.0, then sig = -0.0, exp = -inf\n // otherwise we just extract the 64-bit sig and exp as normal.\n Ref Node = _ReadStackValue(0);\n\n Ref Gpr = _VExtractToGPR(OpSize::i64Bit, OpSize::i64Bit, Node, 0);\n\n // zero case\n Ref ExpZV = _VCastFromGPR(OpSize::i64Bit, OpSize::i64Bit, Constant(0xfff0'0000'0000'0000UL));\n Ref SigZV = Node;\n\n // non zero case\n Ref ExpNZ = _Bfe(OpSize::i64Bit, 11, 52, Gpr);\n ExpNZ = Sub(OpSize::i64Bit, ExpNZ, Constant(1023));\n Ref ExpNZV = _Float_FromGPR_S(OpSize::i64Bit, OpSize::i64Bit, ExpNZ);\n\n Ref SigNZ = _And(OpSize::i64Bit, Gpr, Constant(0x800f'ffff'ffff'ffffLL));\n SigNZ = _Or(OpSize::i64Bit, SigNZ, Constant(0x3ff0'0000'0000'0000LL));\n Ref SigNZV = _VCastFromGPR(OpSize::i64Bit, OpSize::i64Bit, SigNZ);\n\n // Comparison and select to push onto stack\n SaveNZCV();\n _TestNZ(OpSize::i64Bit, Gpr, Constant(0x7fff'ffff'ffff'ffffUL));\n\n Ref Sig = _NZCVSelectV(OpSize::i64Bit, CondClass::EQ, SigZV, SigNZV);\n Ref Exp = _NZCVSelectV(OpSize::i64Bit, CondClass::EQ, ExpZV, ExpNZV);\n\n _PopStackDestroy();\n _PushStack(Exp, Invalid(), OpSize::iInvalid);\n _PushStack(Sig, Invalid(), OpSize::iInvalid);\n}\n} // namespace FEXCore::IR\n" }, { "path": "FEXCore/Source/Interface/Core/OpcodeDispatcher.cpp", "content": "// SPDX-License-Identifier: MIT\n/*\n$info$\ntags: frontend|x86-to-ir, opcodes|dispatcher-implementations\ndesc: Handles x86/64 ops to IR, no-pf opt, local-flags opt\n$end_info$\n*/\n\n#include \"FEXCore/Core/HostFeatures.h\"\n#include \"FEXCore/Utils/Telemetry.h\"\n#include \"Interface/Context/Context.h\"\n#include \"Interface/Core/OpcodeDispatcher.h\"\n#include \"Interface/Core/X86Tables/X86Tables.h\"\n#include \"Interface/IR/IR.h\"\n#include \"Interface/IR/IREmitter.h\"\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n#include \n\n#include \n#include \n#include \n\nnamespace FEXCore::IR {\n\nusing X86Tables::OpToIndex;\n\n#define OpcodeArgs [[maybe_unused]] FEXCore::X86Tables::DecodedOp Op\n\nvoid OpDispatchBuilder::SyscallOp(OpcodeArgs, bool IsSyscallInst) {\n constexpr size_t SyscallArgs = 7;\n using SyscallArray = std::array;\n\n size_t NumArguments {};\n const SyscallArray* GPRIndexes {};\n static constexpr SyscallArray GPRIndexes_64 = {\n FEXCore::X86State::REG_RAX, FEXCore::X86State::REG_RDI, FEXCore::X86State::REG_RSI, FEXCore::X86State::REG_RDX,\n FEXCore::X86State::REG_R10, FEXCore::X86State::REG_R8, FEXCore::X86State::REG_R9,\n };\n static constexpr SyscallArray GPRIndexes_32 = {\n FEXCore::X86State::REG_RAX, FEXCore::X86State::REG_RBX, FEXCore::X86State::REG_RCX, FEXCore::X86State::REG_RDX,\n FEXCore::X86State::REG_RSI, FEXCore::X86State::REG_RDI, FEXCore::X86State::REG_RBP,\n };\n\n const auto OSABI = CTX->SyscallHandler->GetOSABI();\n if (OSABI == FEXCore::HLE::SyscallOSABI::OS_LINUX64) {\n NumArguments = GPRIndexes_64.size();\n GPRIndexes = &GPRIndexes_64;\n } else if (OSABI == FEXCore::HLE::SyscallOSABI::OS_LINUX32) {\n NumArguments = GPRIndexes_32.size();\n GPRIndexes = &GPRIndexes_32;\n } else if (OSABI == FEXCore::HLE::SyscallOSABI::OS_GENERIC) {\n // All registers will be spilled before the syscall and filled afterwards so no JIT-side argument handling is necessary.\n NumArguments = 0;\n GPRIndexes = nullptr;\n } else {\n ERROR_AND_DIE_FMT(\"Unhandled OSABI syscall\");\n }\n\n // Calculate flags early.\n CalculateDeferredFlags();\n\n const auto GPRSize = GetGPROpSize();\n auto NewRIP = GetRelocatedPC(Op, -Op->InstSize);\n _StoreContextGPR(GPRSize, NewRIP, offsetof(FEXCore::Core::CPUState, rip));\n\n Ref Arguments[SyscallArgs] {\n InvalidNode, InvalidNode, InvalidNode, InvalidNode, InvalidNode, InvalidNode, InvalidNode,\n };\n for (size_t i = 0; i < NumArguments; ++i) {\n Arguments[i] = LoadGPRRegister(GPRIndexes->at(i));\n }\n\n if (IsSyscallInst) {\n // If this is the `Syscall` instruction rather than `int 0x80` then we need to do some additional work.\n // RCX = RIP after this instruction\n // R11 = EFlags\n // Calculate flags.\n CalculateDeferredFlags();\n\n auto RFLAG = GetPackedRFLAG();\n StoreGPRRegister(X86State::REG_R11, RFLAG, OpSize::i64Bit);\n\n auto RIPAfterInst = GetRelocatedPC(Op);\n StoreGPRRegister(X86State::REG_RCX, RIPAfterInst, OpSize::i64Bit);\n }\n\n FlushRegisterCache();\n auto SyscallOp = _Syscall(Arguments[0], Arguments[1], Arguments[2], Arguments[3], Arguments[4], Arguments[5], Arguments[6]);\n\n // Generic ABI doesn't store result in RAX.\n if (OSABI != FEXCore::HLE::SyscallOSABI::OS_GENERIC) {\n StoreGPRRegister(X86State::REG_RAX, SyscallOp);\n }\n\n if (Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_BLOCK_END) {\n // RIP could have been updated after coming back from the Syscall.\n NewRIP = _LoadContextGPR(GPRSize, offsetof(FEXCore::Core::CPUState, rip));\n ExitFunction(NewRIP);\n }\n}\n\nvoid OpDispatchBuilder::ThunkOp(OpcodeArgs) {\n const auto GPRSize = GetGPROpSize();\n uint8_t* sha256 = (uint8_t*)(Op->PC + 2);\n\n if (Is64BitMode) {\n // x86-64 ABI puts the function argument in RDI\n Thunk(LoadGPRRegister(X86State::REG_RDI), *reinterpret_cast(sha256));\n } else {\n // x86 fastcall ABI puts the function argument in ECX\n Thunk(LoadGPRRegister(X86State::REG_RCX), *reinterpret_cast(sha256));\n }\n\n auto NewRIP = Pop(GPRSize);\n\n // Store the new RIP\n ExitFunction(NewRIP, BranchHint::Return);\n BlockSetRIP = true;\n}\n\nvoid OpDispatchBuilder::LEAOp(OpcodeArgs) {\n // LEA specifically ignores segment prefixes\n const auto SrcSize = OpSizeFromSrc(Op);\n const auto OpAddr = X86Tables::DecodeFlags::GetOpAddr(Op->Flags, 0);\n OpSize DstSize {};\n\n if (Is64BitMode) {\n DstSize = OpAddr == X86Tables::DecodeFlags::FLAG_OPERAND_SIZE_LAST ? OpSize::i16Bit :\n OpAddr == X86Tables::DecodeFlags::FLAG_WIDENING_SIZE_LAST ? OpSize::i64Bit :\n OpSize::i32Bit;\n } else {\n DstSize = OpAddr == X86Tables::DecodeFlags::FLAG_OPERAND_SIZE_LAST ? OpSize::i16Bit : OpSize::i32Bit;\n }\n\n auto Src = LoadSourceGPR_WithOpSize(Op, Op->Src[0], SrcSize, Op->Flags, {.LoadData = false, .AllowUpperGarbage = SrcSize > DstSize});\n StoreResultGPR_WithOpSize(Op, Op->Dest, Src, DstSize);\n}\n\nvoid OpDispatchBuilder::NOPOp(OpcodeArgs) {}\n\nvoid OpDispatchBuilder::RETOp(OpcodeArgs) {\n const auto GPRSize = GetGPROpSize();\n\n Ref SP = _RMWHandle(LoadGPRRegister(X86State::REG_RSP));\n Ref NewRIP = Pop(GPRSize, SP);\n\n if (Op->OP == 0xC2) {\n auto Offset = LoadSourceGPR(Op, Op->Src[0], Op->Flags);\n SP = Add(GPRSize, SP, Offset);\n }\n\n // Store the new stack pointer\n StoreGPRRegister(X86State::REG_RSP, SP);\n\n // Store the new RIP\n ExitFunction(NewRIP, BranchHint::Return);\n BlockSetRIP = true;\n}\n\n/*\nstack contains:\nSize of each member is 64-bit, 32-bit, or 16-bit depending on operating size\nRIP\nCS\nEFLAGS\nRSP\nSS\n*/\nvoid OpDispatchBuilder::IRETOp(OpcodeArgs) {\n // Operand Size override unsupported!\n if ((Op->Flags & X86Tables::DecodeFlags::FLAG_OPERAND_SIZE) != 0) {\n LogMan::Msg::EFmt(\"IRET only implemented for 64bit and 32bit sizes\");\n DecodeFailure = true;\n return;\n }\n\n const auto GPRSize = GetGPROpSize();\n\n Ref SP = _RMWHandle(LoadGPRRegister(X86State::REG_RSP));\n\n // RIP (64/32/16 bits)\n auto NewRIP = Pop(GPRSize, SP);\n // CS (lower 16 used)\n auto NewSegmentCS = Pop(GPRSize, SP);\n _StoreContextGPR(OpSize::i16Bit, NewSegmentCS, offsetof(FEXCore::Core::CPUState, cs_idx));\n UpdatePrefixFromSegment(NewSegmentCS, FEXCore::X86Tables::DecodeFlags::FLAG_CS_PREFIX);\n\n // eflags (lower 16 used)\n SetPackedRFLAG(false, Pop(GPRSize, SP));\n\n if (Is64BitMode) {\n // RSP and SS only happen in 64-bit mode or if this is a CPL mode jump!\n // FEX doesn't support a CPL mode switch, so don't need to worry about this on 32-bit\n StoreGPRRegister(X86State::REG_RSP, Pop(GPRSize, SP));\n\n // ss\n auto NewSegmentSS = Pop(GPRSize, SP);\n _StoreContextGPR(OpSize::i16Bit, NewSegmentSS, offsetof(FEXCore::Core::CPUState, ss_idx));\n UpdatePrefixFromSegment(NewSegmentSS, FEXCore::X86Tables::DecodeFlags::FLAG_SS_PREFIX);\n } else {\n // Store the stack in 32-bit mode\n StoreGPRRegister(X86State::REG_RSP, SP);\n }\n\n ExitFunction(NewRIP);\n BlockSetRIP = true;\n}\n\nvoid OpDispatchBuilder::CallbackReturnOp(OpcodeArgs) {\n const auto GPRSize = GetGPROpSize();\n // Store the new RIP\n _CallbackReturn();\n auto NewRIP = _LoadContextGPR(GPRSize, offsetof(FEXCore::Core::CPUState, rip));\n // This ExitFunction won't actually get hit but needs to exist\n ExitFunction(NewRIP);\n BlockSetRIP = true;\n}\n\nvoid OpDispatchBuilder::SecondaryALUOp(OpcodeArgs) {\n FEXCore::IR::IROps IROp, AtomicIROp;\n#define OPD(group, prefix, Reg) (((group - FEXCore::X86Tables::TYPE_GROUP_1) << 6) | (prefix) << 3 | (Reg))\n switch (Op->OP) {\n case OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x80), 0):\n case OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x81), 0):\n case OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x83), 0):\n IROp = FEXCore::IR::IROps::OP_ADD;\n AtomicIROp = FEXCore::IR::IROps::OP_ATOMICFETCHADD;\n break;\n case OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x80), 1):\n case OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x81), 1):\n case OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x83), 1):\n IROp = FEXCore::IR::IROps::OP_OR;\n AtomicIROp = FEXCore::IR::IROps::OP_ATOMICFETCHOR;\n break;\n case OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x80), 4):\n case OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x81), 4):\n case OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x83), 4):\n IROp = FEXCore::IR::IROps::OP_ANDWITHFLAGS;\n AtomicIROp = FEXCore::IR::IROps::OP_ATOMICFETCHAND;\n break;\n case OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x80), 5):\n case OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x81), 5):\n case OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x83), 5):\n IROp = FEXCore::IR::IROps::OP_SUB;\n AtomicIROp = FEXCore::IR::IROps::OP_ATOMICFETCHSUB;\n break;\n case OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x80), 6):\n case OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x81), 6):\n case OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x83), 6):\n IROp = FEXCore::IR::IROps::OP_XOR;\n AtomicIROp = FEXCore::IR::IROps::OP_ATOMICFETCHXOR;\n break;\n default:\n IROp = FEXCore::IR::IROps::OP_LAST;\n AtomicIROp = FEXCore::IR::IROps::OP_LAST;\n LogMan::Msg::EFmt(\"Unknown ALU Op: 0x{:x}\", Op->OP);\n DecodeFailure = true;\n return;\n };\n#undef OPD\n\n ALUOp(Op, IROp, AtomicIROp, 1);\n}\n\nvoid OpDispatchBuilder::ADCOp(OpcodeArgs, uint32_t SrcIndex) {\n // Calculate flags early.\n CalculateDeferredFlags();\n\n Ref Src = LoadSourceGPR(Op, Op->Src[SrcIndex], Op->Flags, {.AllowUpperGarbage = true});\n const auto Size = OpSizeFromDst(Op);\n const auto OpSize = std::max(OpSize::i32Bit, Size);\n\n Ref Before {};\n if (DestIsLockedMem(Op)) {\n auto ALUOp = IncrementByCarry(OpSize, Src);\n HandledLock = true;\n\n Ref DestMem = MakeSegmentAddress(Op, Op->Dest);\n Before = _AtomicFetchAdd(Size, ALUOp, DestMem);\n } else {\n Before = LoadSourceGPR(Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = true});\n }\n\n Ref Result;\n if (!DestIsLockedMem(Op) && Op->Src[SrcIndex].IsLiteral() && Op->Src[SrcIndex].Literal() == 0 && Size >= OpSize::i32Bit) {\n HandleNZCV_RMW();\n RectifyCarryInvert(true);\n Result = _AdcZeroWithFlags(OpSize, Before);\n SetRFLAG(Before);\n CalculatePF(Result);\n CFInverted = false;\n } else {\n Result = CalculateFlags_ADC(Size, Before, Src);\n }\n\n if (!DestIsLockedMem(Op)) {\n StoreResultGPR(Op, Result);\n }\n}\n\nvoid OpDispatchBuilder::SBBOp(OpcodeArgs, uint32_t SrcIndex) {\n // Calculate flags early.\n CalculateDeferredFlags();\n\n Ref Src = LoadSourceGPR(Op, Op->Src[SrcIndex], Op->Flags, {.AllowUpperGarbage = true});\n const auto Size = OpSizeFromDst(Op);\n const auto OpSize = std::max(OpSize::i32Bit, Size);\n\n Ref Result {};\n Ref Before {};\n if (DestIsLockedMem(Op)) {\n HandledLock = true;\n\n Ref DestMem = MakeSegmentAddress(Op, Op->Dest);\n auto SrcPlusCF = IncrementByCarry(OpSize, Src);\n Before = _AtomicFetchSub(Size, SrcPlusCF, DestMem);\n } else {\n Before = LoadSourceGPR(Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = true});\n }\n\n Result = CalculateFlags_SBB(Size, Before, Src);\n\n if (!DestIsLockedMem(Op)) {\n StoreResultGPR(Op, Result);\n }\n}\n\nvoid OpDispatchBuilder::SALCOp(OpcodeArgs) {\n CalculateDeferredFlags();\n\n auto Result = NZCVSelect(OpSize::i32Bit, CondClass::UGE /* CF = 1 */, _InlineConstant(0xffffffff), _InlineConstant(0));\n\n StoreResultGPR(Op, Result);\n}\n\nvoid OpDispatchBuilder::PUSHOp(OpcodeArgs) {\n const auto Size = OpSizeFromSrc(Op);\n\n Push(Size, LoadSourceGPR(Op, Op->Src[0], Op->Flags));\n}\n\nvoid OpDispatchBuilder::PUSHREGOp(OpcodeArgs) {\n const auto Size = OpSizeFromSrc(Op);\n\n Push(Size, LoadSourceGPR(Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = true}));\n}\n\nvoid OpDispatchBuilder::PUSHAOp(OpcodeArgs) {\n // 32bit only\n const auto Size = OpSizeFromSrc(Op);\n\n Ref OldSP = _Copy(LoadGPRRegister(X86State::REG_RSP));\n\n Push(Size, LoadGPRRegister(X86State::REG_RAX));\n Push(Size, LoadGPRRegister(X86State::REG_RCX));\n Push(Size, LoadGPRRegister(X86State::REG_RDX));\n Push(Size, LoadGPRRegister(X86State::REG_RBX));\n Push(Size, OldSP);\n Push(Size, LoadGPRRegister(X86State::REG_RBP));\n Push(Size, LoadGPRRegister(X86State::REG_RSI));\n Push(Size, LoadGPRRegister(X86State::REG_RDI));\n}\n\nvoid OpDispatchBuilder::PUSHSegmentOp(OpcodeArgs, uint32_t SegmentReg) {\n const auto SrcSize = OpSizeFromSrc(Op);\n const auto DstSize = OpSizeFromDst(Op);\n\n Ref Src {};\n if (!Is64BitMode) {\n switch (SegmentReg) {\n case FEXCore::X86Tables::DecodeFlags::FLAG_ES_PREFIX: {\n Src = _LoadContextGPR(SrcSize, offsetof(FEXCore::Core::CPUState, es_idx));\n break;\n }\n case FEXCore::X86Tables::DecodeFlags::FLAG_CS_PREFIX: {\n Src = _LoadContextGPR(SrcSize, offsetof(FEXCore::Core::CPUState, cs_idx));\n break;\n }\n case FEXCore::X86Tables::DecodeFlags::FLAG_SS_PREFIX: {\n Src = _LoadContextGPR(SrcSize, offsetof(FEXCore::Core::CPUState, ss_idx));\n break;\n }\n case FEXCore::X86Tables::DecodeFlags::FLAG_DS_PREFIX: {\n Src = _LoadContextGPR(SrcSize, offsetof(FEXCore::Core::CPUState, ds_idx));\n break;\n }\n case FEXCore::X86Tables::DecodeFlags::FLAG_FS_PREFIX: {\n Src = _LoadContextGPR(SrcSize, offsetof(FEXCore::Core::CPUState, fs_idx));\n break;\n }\n case FEXCore::X86Tables::DecodeFlags::FLAG_GS_PREFIX: {\n Src = _LoadContextGPR(SrcSize, offsetof(FEXCore::Core::CPUState, gs_idx));\n break;\n }\n default: FEX_UNREACHABLE;\n }\n } else {\n switch (SegmentReg) {\n case FEXCore::X86Tables::DecodeFlags::FLAG_ES_PREFIX:\n Src = _LoadContextGPR(SrcSize, offsetof(FEXCore::Core::CPUState, es_cached));\n break;\n case FEXCore::X86Tables::DecodeFlags::FLAG_CS_PREFIX:\n Src = _LoadContextGPR(SrcSize, offsetof(FEXCore::Core::CPUState, cs_cached));\n break;\n case FEXCore::X86Tables::DecodeFlags::FLAG_SS_PREFIX:\n Src = _LoadContextGPR(SrcSize, offsetof(FEXCore::Core::CPUState, ss_cached));\n break;\n case FEXCore::X86Tables::DecodeFlags::FLAG_DS_PREFIX:\n Src = _LoadContextGPR(SrcSize, offsetof(FEXCore::Core::CPUState, ds_cached));\n break;\n case FEXCore::X86Tables::DecodeFlags::FLAG_FS_PREFIX:\n Src = _LoadContextGPR(SrcSize, offsetof(FEXCore::Core::CPUState, fs_cached));\n break;\n case FEXCore::X86Tables::DecodeFlags::FLAG_GS_PREFIX:\n Src = _LoadContextGPR(SrcSize, offsetof(FEXCore::Core::CPUState, gs_cached));\n break;\n default: FEX_UNREACHABLE;\n }\n }\n\n // Store our value to the new stack location\n // AMD hardware zexts segment selector to 32bit\n // Intel hardware inserts segment selector\n Push(DstSize, Src);\n}\n\nvoid OpDispatchBuilder::POPOp(OpcodeArgs) {\n Ref Value = Pop(OpSizeFromSrc(Op));\n StoreResultGPR(Op, Value);\n}\n\nvoid OpDispatchBuilder::POPAOp(OpcodeArgs) {\n // 32bit only\n const auto Size = OpSizeFromSrc(Op);\n\n Ref SP = _RMWHandle(LoadGPRRegister(X86State::REG_RSP));\n\n StoreGPRRegister(X86State::REG_RDI, Pop(Size, SP), Size);\n StoreGPRRegister(X86State::REG_RSI, Pop(Size, SP), Size);\n StoreGPRRegister(X86State::REG_RBP, Pop(Size, SP), Size);\n\n // Skip loading RSP because it'll be correct at the end\n SP = _RMWHandle(Add(OpSize::i64Bit, SP, IR::OpSizeToSize(Size)));\n\n StoreGPRRegister(X86State::REG_RBX, Pop(Size, SP), Size);\n StoreGPRRegister(X86State::REG_RDX, Pop(Size, SP), Size);\n StoreGPRRegister(X86State::REG_RCX, Pop(Size, SP), Size);\n StoreGPRRegister(X86State::REG_RAX, Pop(Size, SP), Size);\n\n // Store the new stack pointer\n StoreGPRRegister(X86State::REG_RSP, SP);\n}\n\nvoid OpDispatchBuilder::POPSegmentOp(OpcodeArgs, uint32_t SegmentReg) {\n const auto SrcSize = OpSizeFromSrc(Op);\n const auto DstSize = OpSizeFromDst(Op);\n\n auto NewSegment = Pop(SrcSize);\n\n switch (SegmentReg) {\n case FEXCore::X86Tables::DecodeFlags::FLAG_ES_PREFIX:\n _StoreContextGPR(DstSize, NewSegment, offsetof(FEXCore::Core::CPUState, es_idx));\n break;\n case FEXCore::X86Tables::DecodeFlags::FLAG_CS_PREFIX:\n _StoreContextGPR(DstSize, NewSegment, offsetof(FEXCore::Core::CPUState, cs_idx));\n break;\n case FEXCore::X86Tables::DecodeFlags::FLAG_SS_PREFIX:\n // Unset the 'active' bit in the packed TF, skipping the single step exception after this instruction\n SetRFLAG(_And(OpSize::i32Bit, GetRFLAG(FEXCore::X86State::RFLAG_TF_RAW_LOC), Constant(1)));\n _StoreContextGPR(DstSize, NewSegment, offsetof(FEXCore::Core::CPUState, ss_idx));\n break;\n case FEXCore::X86Tables::DecodeFlags::FLAG_DS_PREFIX:\n _StoreContextGPR(DstSize, NewSegment, offsetof(FEXCore::Core::CPUState, ds_idx));\n break;\n case FEXCore::X86Tables::DecodeFlags::FLAG_FS_PREFIX:\n _StoreContextGPR(DstSize, NewSegment, offsetof(FEXCore::Core::CPUState, fs_idx));\n break;\n case FEXCore::X86Tables::DecodeFlags::FLAG_GS_PREFIX:\n _StoreContextGPR(DstSize, NewSegment, offsetof(FEXCore::Core::CPUState, gs_idx));\n break;\n default: break; // Do nothing\n }\n\n UpdatePrefixFromSegment(NewSegment, SegmentReg);\n}\n\nvoid OpDispatchBuilder::LEAVEOp(OpcodeArgs) {\n const auto GPRSize = GetGPROpSize();\n const auto OperandSize = (Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_OPERAND_SIZE) ? OpSize::i16Bit : GPRSize;\n\n // First we move RBP in to RSP and then behave effectively like a pop\n auto SP = _RMWHandle(LoadGPRRegister(X86State::REG_RBP));\n auto NewGPR = Pop(OperandSize, SP);\n\n // Store the new stack pointer\n StoreGPRRegister(X86State::REG_RSP, SP, OperandSize);\n\n // Store what we loaded to RBP\n StoreGPRRegister(X86State::REG_RBP, NewGPR, OperandSize);\n}\n\nvoid OpDispatchBuilder::CALLOp(OpcodeArgs) {\n const auto GPRSize = GetGPROpSize();\n\n BlockSetRIP = true;\n\n // Call instruction only uses up to 32-bit signed displacement\n const int64_t TargetOffset = Op->Src[0].Literal();\n\n const auto ConstantPC = GetRelocatedPC(Op);\n\n // Push the return address.\n Push(GPRSize, ConstantPC);\n\n if (TargetOffset != 0) {\n // Store the RIP\n const uint64_t NextRIP = Op->PC + Op->InstSize;\n\n ExitRelocatedPC(Op, TargetOffset, BranchHint::Call, ConstantPC, [&]() {\n auto CallReturnJumpTarget = JumpTargets.find(NextRIP);\n if (CallReturnJumpTarget != JumpTargets.end() && CallReturnJumpTarget->second.IsEntryPoint) {\n return CallReturnJumpTarget->second.BlockEntry;\n }\n return InvalidNode;\n }());\n } else {\n NeedsBlockEnd = true;\n }\n}\n\nvoid OpDispatchBuilder::CALLAbsoluteOp(OpcodeArgs) {\n BlockSetRIP = true;\n\n const auto Size = OpSizeFromSrc(Op);\n Ref JMPPCOffset = LoadSourceGPR(Op, Op->Src[0], Op->Flags);\n\n // Push the return address.\n auto ConstantPC = GetRelocatedPC(Op);\n Push(Size, ConstantPC);\n\n // Store the RIP\n const uint64_t NextRIP = Op->PC + Op->InstSize;\n ExitFunction(JMPPCOffset, BranchHint::Call, ConstantPC, [&]() {\n auto CallReturnJumpTarget = JumpTargets.find(NextRIP);\n if (CallReturnJumpTarget != JumpTargets.end() && CallReturnJumpTarget->second.IsEntryPoint) {\n return CallReturnJumpTarget->second.BlockEntry;\n }\n return InvalidNode;\n }());\n}\n\nstd::optional OpDispatchBuilder::DecodeNZCVCondition(uint8_t OP) {\n switch (OP) {\n case 0x0: { // JO - Jump if OF == 1\n return CondClass::FU;\n }\n case 0x1: { // JNO - Jump if OF == 0\n return CondClass::FNU;\n }\n case 0x2: { // JC - Jump if CF == 1\n return CFInverted ? CondClass::ULT : CondClass::UGE;\n }\n case 0x3: { // JNC - Jump if CF == 0\n return CFInverted ? CondClass::UGE : CondClass::ULT;\n }\n case 0x4: { // JE - Jump if ZF == 1\n return CondClass::EQ;\n }\n case 0x5: { // JNE - Jump if ZF == 0\n return CondClass::NEQ;\n }\n case 0x6: { // JNA - Jump if CF == 1 || ZF == 1\n // With CF, we want (C == 0 || Z == 1). By De Morgan's, that's\n // equivalent to !(C == 1 && Z == 0). That's .ls\n RectifyCarryInvert(true);\n return CondClass::ULE;\n }\n case 0x7: { // JA - Jump if CF == 0 && ZF == 0\n // With CF inverted, we want (C == 1 && Z == 0). That's .hi\n RectifyCarryInvert(true);\n return CondClass::UGT;\n }\n case 0x8: { // JS - Jump if SF == 1\n return CondClass::MI;\n }\n case 0x9: { // JNS - Jump if SF == 0\n return CondClass::PL;\n }\n case 0xC: { // SF <> OF\n return CondClass::SLT;\n }\n case 0xD: { // SF = OF\n return CondClass::SGE;\n }\n case 0xE: { // ZF = 1 || SF <> OF\n return CondClass::SLE;\n }\n case 0xF: { // ZF = 0 && SF = OF\n return CondClass::SGT;\n }\n default:\n // Other conditions do not map directly, caller gets to deal with it.\n return std::nullopt;\n }\n}\n\nstatic bool ParityJumpIsJP(uint8_t OP) {\n LOGMAN_THROW_A_FMT(OP == 0xA || OP == 0xB, \"JP or JNP\");\n return OP == 0xA;\n}\n\nRef OpDispatchBuilder::SelectCC0All1(uint8_t OP) {\n if (auto Cond = DecodeNZCVCondition(OP); Cond) {\n // Use raw select since DecodeNZCVCondition handles the carry invert\n return _NZCVSelect(OpSize::i64Bit, *Cond, _InlineConstant(~0ULL), _InlineConstant(0));\n } else {\n // Raw value contains inverted PF in bottom bit\n return _Sbfe(OpSize::i64Bit, 1, 0, LoadPFRaw(false, ParityJumpIsJP(OP)));\n }\n}\n\nvoid OpDispatchBuilder::SETccOp(OpcodeArgs) {\n CalculateDeferredFlags();\n\n Ref SrcCond;\n if (auto Cond = DecodeNZCVCondition(Op->OP & 0xf); Cond) {\n // Use raw select since DecodeNZCVCondition handles the carry invert\n SrcCond = _NZCVSelect01(*Cond);\n } else {\n SrcCond = LoadPFRaw(true, ParityJumpIsJP(Op->OP & 0xf));\n }\n\n StoreResultGPR(Op, SrcCond);\n}\n\nvoid OpDispatchBuilder::CMOVOp(OpcodeArgs) {\n const auto GPRSize = GetGPROpSize();\n const auto OP = Op->OP & 0xF;\n const auto ResultSize = std::max(OpSize::i32Bit, OpSizeFromSrc(Op));\n\n CalculateDeferredFlags();\n\n // Destination is always a GPR.\n Ref Dest = LoadSourceGPR_WithOpSize(Op, Op->Dest, GPRSize, Op->Flags);\n Ref Src {}, SrcCond {};\n if (Op->Src[0].IsGPR()) {\n Src = LoadSourceGPR_WithOpSize(Op, Op->Src[0], GPRSize, Op->Flags);\n } else {\n Src = LoadSourceGPR(Op, Op->Src[0], Op->Flags);\n }\n\n if (auto Cond = DecodeNZCVCondition(OP); Cond) {\n // Use raw select since DecodeNZCVCondition handles the carry invert\n SrcCond = _NZCVSelect(ResultSize, *Cond, Src, Dest);\n } else {\n // Raw value contains inverted PF in bottom bit\n Ref Cmp = LoadPFRaw(false, ParityJumpIsJP(OP));\n SaveNZCV();\n\n // Because we're only clobbering NZCV internally, we ignore all carry flag\n // shenanigans and just use the raw test and raw select.\n _TestNZ(OpSize::i32Bit, Cmp, _InlineConstant(1));\n SrcCond = _NZCVSelect(ResultSize, CondClass::NEQ, Src, Dest);\n }\n\n StoreResultGPR(Op, SrcCond);\n}\n\nvoid OpDispatchBuilder::CondJUMPOp(OpcodeArgs) {\n // Calculate flags early.\n CalculateDeferredFlags();\n\n BlockSetRIP = true;\n\n // Jump instruction only uses up to 32-bit signed displacement\n int64_t TargetOffset = Op->Src[0].Literal();\n uint64_t InstRIP = Op->PC + Op->InstSize;\n uint64_t Target = InstRIP + TargetOffset;\n\n if (GetGPROpSize() == OpSize::i32Bit) {\n // If the GPRSize is 4 then we need to be careful about PC wrapping\n if (TargetOffset < 0 && -TargetOffset > InstRIP) {\n // Invert the signed value if we are underflowing\n TargetOffset = 0x1'0000'0000ULL + TargetOffset;\n } else if (TargetOffset >= 0 && Target >= 0x1'0000'0000ULL) {\n // We are overflowing, wrap around\n TargetOffset = TargetOffset - 0x1'0000'0000ULL;\n }\n Target &= 0xFFFFFFFFU;\n }\n\n FlushRegisterCache();\n auto TrueBlock = JumpTargets.find(Target);\n auto FalseBlock = JumpTargets.find(Op->PC + Op->InstSize);\n\n auto CurrentBlock = GetCurrentBlock();\n\n {\n IRPair CondJump_;\n auto OP = Op->OP & 0xF;\n auto Cond = DecodeNZCVCondition(OP);\n if (Cond) {\n CondJump_ = CondJumpNZCV(*Cond);\n } else {\n LOGMAN_THROW_A_FMT(OP == 0xA || OP == 0xB, \"only PF left\");\n CondJump_ = CondJumpBit(LoadPFRaw(false, false), 0, OP == 0xB);\n }\n\n // Taking branch block\n if (TrueBlock != JumpTargets.end()) {\n SetTrueJumpTarget(CondJump_, TrueBlock->second.BlockEntry);\n } else {\n // Make sure to start a new block after ending this one\n auto JumpTarget = CreateNewCodeBlockAtEnd();\n SetTrueJumpTarget(CondJump_, JumpTarget);\n SetCurrentCodeBlock(JumpTarget);\n StartNewBlock();\n\n // Store the new RIP\n ExitRelocatedPC(Op, TargetOffset);\n }\n\n // Failure to take branch\n if (FalseBlock != JumpTargets.end()) {\n SetFalseJumpTarget(CondJump_, FalseBlock->second.BlockEntry);\n } else {\n // Make sure to start a new block after ending this one\n // Place it after this block for fallthrough optimization\n auto JumpTarget = CreateNewCodeBlockAfter(CurrentBlock);\n SetFalseJumpTarget(CondJump_, JumpTarget);\n SetCurrentCodeBlock(JumpTarget);\n StartNewBlock();\n\n // Leave block & store the new RIP\n ExitRelocatedPC(Op);\n }\n }\n}\n\nvoid OpDispatchBuilder::CondJUMPRCXOp(OpcodeArgs) {\n // Calculate flags early.\n CalculateDeferredFlags();\n\n BlockSetRIP = true;\n auto JcxGPRSize = GetGPROpSize();\n JcxGPRSize = (Op->Flags & X86Tables::DecodeFlags::FLAG_ADDRESS_SIZE) ? (JcxGPRSize >> 1) : JcxGPRSize;\n\n uint64_t Target = Op->PC + Op->InstSize + Op->Src[0].Literal();\n\n Ref CondReg = LoadGPRRegister(X86State::REG_RCX, JcxGPRSize);\n\n auto TrueBlock = JumpTargets.find(Target);\n auto FalseBlock = JumpTargets.find(Op->PC + Op->InstSize);\n\n auto CurrentBlock = GetCurrentBlock();\n\n {\n auto CondJump_ = CondJump(CondReg, CondClass::EQ);\n\n // Taking branch block\n if (TrueBlock != JumpTargets.end()) {\n SetTrueJumpTarget(CondJump_, TrueBlock->second.BlockEntry);\n } else {\n // Make sure to start a new block after ending this one\n auto JumpTarget = CreateNewCodeBlockAtEnd();\n SetTrueJumpTarget(CondJump_, JumpTarget);\n SetCurrentCodeBlock(JumpTarget);\n StartNewBlock();\n\n // Store the new RIP\n ExitRelocatedPC(Op, Op->Src[0].Literal());\n }\n\n // Failure to take branch\n if (FalseBlock != JumpTargets.end()) {\n SetFalseJumpTarget(CondJump_, FalseBlock->second.BlockEntry);\n } else {\n // Make sure to start a new block after ending this one\n // Place it after the current block for fallthrough behavior\n auto JumpTarget = CreateNewCodeBlockAfter(CurrentBlock);\n SetFalseJumpTarget(CondJump_, JumpTarget);\n SetCurrentCodeBlock(JumpTarget);\n StartNewBlock();\n\n // Leave block & store the new RIP\n ExitRelocatedPC(Op);\n }\n }\n}\n\nvoid OpDispatchBuilder::LoopOp(OpcodeArgs) {\n // Calculate flags early.\n CalculateDeferredFlags();\n\n bool CheckZF = Op->OP != 0xE2;\n bool ZFTrue = Op->OP == 0xE1;\n\n BlockSetRIP = true;\n auto SrcSize = (Op->Flags & X86Tables::DecodeFlags::FLAG_ADDRESS_SIZE) ? OpSize::i32Bit : OpSize::i64Bit;\n auto OpSize = SrcSize == OpSize::i64Bit ? OpSize::i64Bit : OpSize::i32Bit;\n\n if (!Is64BitMode) {\n // RCX size is 32-bit or 16-bit when executing in 32-bit mode.\n SrcSize = IR::SizeToOpSize(IR::OpSizeToSize(SrcSize) >> 1);\n OpSize = OpSize::i32Bit;\n }\n\n uint64_t Target = Op->PC + Op->InstSize + Op->Src[1].Literal();\n\n Ref CondReg = LoadSourceGPR_WithOpSize(Op, Op->Src[0], SrcSize, Op->Flags);\n CondReg = Sub(OpSize, CondReg, 1);\n StoreResultGPR(Op, Op->Src[0], CondReg);\n\n // If LOOPE then jumps to target if RCX != 0 && ZF == 1\n // If LOOPNE then jumps to target if RCX != 0 && ZF == 0\n //\n // To handle efficiently, smash RCX to zero if ZF is wrong (1 csel).\n if (CheckZF) {\n const auto cond = ZFTrue ? CondClass::EQ : CondClass::NEQ;\n CondReg = NZCVSelect(OpSize, cond, CondReg, _InlineConstant(0));\n }\n\n CalculateDeferredFlags();\n auto TrueBlock = JumpTargets.find(Target);\n auto FalseBlock = JumpTargets.find(Op->PC + Op->InstSize);\n\n {\n auto CondJump_ = CondJump(CondReg);\n\n // Taking branch block\n if (TrueBlock != JumpTargets.end()) {\n SetTrueJumpTarget(CondJump_, TrueBlock->second.BlockEntry);\n } else {\n // Make sure to start a new block after ending this one\n auto JumpTarget = CreateNewCodeBlockAtEnd();\n SetTrueJumpTarget(CondJump_, JumpTarget);\n SetCurrentCodeBlock(JumpTarget);\n StartNewBlock();\n\n // Store the new RIP\n ExitRelocatedPC(Op, Op->Src[1].Literal());\n }\n\n // Failure to take branch\n if (FalseBlock != JumpTargets.end()) {\n SetFalseJumpTarget(CondJump_, FalseBlock->second.BlockEntry);\n } else {\n // Make sure to start a new block after ending this one\n // Place after this block for fallthrough behavior\n auto JumpTarget = CreateNewCodeBlockAfter(GetCurrentBlock());\n SetFalseJumpTarget(CondJump_, JumpTarget);\n SetCurrentCodeBlock(JumpTarget);\n StartNewBlock();\n\n // Leave block & store the new RIP\n ExitRelocatedPC(Op);\n }\n }\n}\n\nvoid OpDispatchBuilder::JUMPOp(OpcodeArgs) {\n // Calculate flags early.\n CalculateDeferredFlags();\n\n BlockSetRIP = true;\n\n // Jump instruction only uses up to 32-bit signed displacement\n int64_t TargetOffset = Op->Src[0].Literal();\n uint64_t InstRIP = Op->PC + Op->InstSize;\n uint64_t TargetRIP = InstRIP + TargetOffset;\n\n if (GetGPROpSize() == OpSize::i32Bit) {\n // If the GPRSize is 4 then we need to be careful about PC wrapping\n if (TargetOffset < 0 && -TargetOffset > InstRIP) {\n // Invert the signed value if we are underflowing\n TargetOffset = 0x1'0000'0000ULL + TargetOffset;\n } else if (TargetOffset >= 0 && TargetRIP >= 0x1'0000'0000ULL) {\n // We are overflowing, wrap around\n TargetOffset = TargetOffset - 0x1'0000'0000ULL;\n }\n\n TargetRIP &= 0xFFFFFFFFU;\n }\n\n CalculateDeferredFlags();\n // This is just an unconditional relative literal jump\n if (Multiblock) {\n auto JumpBlock = JumpTargets.find(TargetRIP);\n if (JumpBlock != JumpTargets.end()) {\n Jump(GetNewJumpBlock(TargetRIP));\n } else {\n // If the block isn't a jump target then we need to create an exit block\n auto Jump_ = Jump();\n\n // Place after this block for fallthrough behavior\n auto JumpTarget = CreateNewCodeBlockAfter(GetCurrentBlock());\n SetJumpTarget(Jump_, JumpTarget);\n SetCurrentCodeBlock(JumpTarget);\n StartNewBlock();\n ExitRelocatedPC(Op, TargetOffset);\n }\n } else {\n ExitRelocatedPC(Op, TargetOffset);\n }\n}\n\nvoid OpDispatchBuilder::JUMPAbsoluteOp(OpcodeArgs) {\n // Calculate flags early.\n CalculateDeferredFlags();\n\n BlockSetRIP = true;\n // This is just an unconditional jump\n // This uses ModRM to determine its location\n // No way to use this effectively in multiblock\n auto RIPOffset = LoadSourceGPR(Op, Op->Src[0], Op->Flags);\n\n // Store the new RIP\n ExitFunction(RIPOffset);\n}\n\nvoid OpDispatchBuilder::JUMPFARIndirectOp(OpcodeArgs) {\n // Calculate flags early.\n CalculateDeferredFlags();\n\n BlockSetRIP = true;\n // This is just an unconditional jump\n // This uses ModRM to determine its location\n // No way to use this effectively in multiblock\n Ref Src = MakeSegmentAddress(Op, Op->Dest);\n AddressMode SrcCS = {.Base = Src, .Offset = 4, .AddrSize = OpSize::i64Bit};\n auto RIPOffset = _LoadMemGPRAutoTSO(OpSize::i32Bit, Src, OpSize::i8Bit);\n auto NewSegmentCS = _LoadMemGPRAutoTSO(OpSize::i16Bit, SrcCS, OpSize::i8Bit);\n\n // Set up the new CSSegment.\n _StoreContextGPR(OpSize::i16Bit, NewSegmentCS, offsetof(FEXCore::Core::CPUState, cs_idx));\n UpdatePrefixFromSegment(NewSegmentCS, FEXCore::X86Tables::DecodeFlags::FLAG_CS_PREFIX);\n\n // Store the new RIP\n ExitFunction(RIPOffset);\n}\n\nvoid OpDispatchBuilder::CALLFARIndirectOp(OpcodeArgs) {\n const auto SrcSize = Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_REX_WIDENING ? OpSize::i64Bit : OpSize::i32Bit;\n\n // Calculate flags early.\n CalculateDeferredFlags();\n\n BlockSetRIP = true;\n\n Ref Src = MakeSegmentAddress(Op, Op->Dest);\n AddressMode SrcCS = {.Base = Src, .Offset = 4, .AddrSize = OpSize::i64Bit};\n auto RIPOffset = _LoadMemGPRAutoTSO(OpSize::i32Bit, Src, OpSize::i8Bit);\n auto NewSegmentCS = _LoadMemGPRAutoTSO(OpSize::i16Bit, SrcCS, OpSize::i8Bit);\n auto CurrentCS = _LoadContextGPR(OpSize::i16Bit, offsetof(FEXCore::Core::CPUState, cs_idx));\n\n auto NewRIP = GetRelocatedPC(Op);\n\n // Push the current CS\n Push(SrcSize, CurrentCS);\n\n // Push the return address.\n Push(SrcSize, NewRIP);\n\n // Set up the new CSSegment.\n _StoreContextGPR(OpSize::i16Bit, NewSegmentCS, offsetof(FEXCore::Core::CPUState, cs_idx));\n UpdatePrefixFromSegment(NewSegmentCS, FEXCore::X86Tables::DecodeFlags::FLAG_CS_PREFIX);\n\n // Store the new RIP\n ExitFunction(RIPOffset);\n}\n\nvoid OpDispatchBuilder::RETFARIndirectOp(OpcodeArgs) {\n const auto GPRSize = GetGPROpSize();\n const auto SrcSize = Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_REX_WIDENING ? OpSize::i64Bit : OpSize::i32Bit;\n\n Ref SP = _RMWHandle(LoadGPRRegister(X86State::REG_RSP));\n Ref NewRIP = Pop(SrcSize, SP);\n Ref NewSegmentCS = Pop(SrcSize, SP);\n\n // Optional SP offset.\n if (Op->Src[0].IsLiteral()) {\n SP = Add(GPRSize, SP, Op->Src[0].Literal());\n }\n\n // Store the new stack pointer\n StoreGPRRegister(X86State::REG_RSP, SP);\n\n _StoreContextGPR(OpSize::i16Bit, NewSegmentCS, offsetof(FEXCore::Core::CPUState, cs_idx));\n UpdatePrefixFromSegment(NewSegmentCS, FEXCore::X86Tables::DecodeFlags::FLAG_CS_PREFIX);\n\n // Store the new RIP\n ExitFunction(NewRIP);\n BlockSetRIP = true;\n}\n\nvoid OpDispatchBuilder::TESTOp(OpcodeArgs, uint32_t SrcIndex) {\n // TEST is an instruction that does an AND between the sources\n // Result isn't stored in result, only writes to flags\n Ref Src = LoadSourceGPR(Op, Op->Src[SrcIndex], Op->Flags, {.AllowUpperGarbage = true});\n Ref Dest = LoadSourceGPR(Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = true});\n\n const auto Size = OpSizeFromDst(Op);\n LOGMAN_THROW_A_FMT(Size >= IR::OpSize::i8Bit && Size <= IR::OpSize::i64Bit, \"Invalid size\");\n\n uint64_t Const;\n bool AlwaysNonnegative = false;\n if (IsValueConstant(WrapNode(Src), &Const)) {\n // Optimize out masking constants\n if (Const == (Size == OpSize::i64Bit ? ~0ULL : ((1ull << IR::OpSizeAsBits(Size)) - 1))) {\n Src = Dest;\n }\n\n // Optimize test with non-sign bits\n AlwaysNonnegative = (Const & (1ull << (IR::OpSizeAsBits(Size) - 1))) == 0;\n }\n\n if (Dest == Src) {\n // Optimize out the AND.\n SetNZP_ZeroCV(Size, Src);\n } else if (Size < OpSize::i32Bit && AlwaysNonnegative) {\n // If we know the result is always nonnegative, we can use a 32-bit test.\n auto Res = _And(OpSize::i32Bit, Dest, Src);\n CalculatePF(Res);\n SetNZ_ZeroCV(OpSize::i32Bit, Res);\n } else {\n HandleNZ00Write();\n CalculatePF(_AndWithFlags(Size, Dest, Src));\n }\n\n InvalidateAF();\n}\n\nvoid OpDispatchBuilder::ARPLOp(OpcodeArgs) {\n // ARPL r/m16, r16\n // If the RPL field in the destination selector is less privileged than the\n // RPL field in the source selector, then adjust destination RPL to match\n // source RPL and set ZF=1. Otherwise ZF=0 and destination is unchanged.\n //\n // Only ZF is modified by ARPL.\n constexpr auto Size = OpSize::i16Bit;\n\n Ref Dest = LoadSourceGPR_WithOpSize(Op, Op->Dest, Size, Op->Flags, {.AllowUpperGarbage = true});\n Ref Src = LoadSourceGPR_WithOpSize(Op, Op->Src[0], Size, Op->Flags, {.AllowUpperGarbage = true});\n\n // RPL is the low two bits of the selector.\n Ref DestRPL = _Bfe(OpSize::i32Bit, 2, 0, Dest);\n Ref SrcRPL = _Bfe(OpSize::i32Bit, 2, 0, Src);\n\n // NeedUpdate is 1 when DestRPL < SrcRPL, else 0.\n Ref NeedUpdate = _Select(OpSize::i32Bit, OpSize::i32Bit, CondClass::ULT, DestRPL, SrcRPL, Constant(1), Constant(0));\n SetRFLAG(NeedUpdate);\n\n // Compute adjusted destination selector: (Dest & ~3) | SrcRPL.\n auto NewDest = _Bfxil(OpSize::i32Bit, 2, 0, Dest, SrcRPL);\n\n // Conditionally select updated selector based on NeedUpdate.\n Ref FinalDest = _Select(OpSize::i32Bit, OpSize::i32Bit, CondClass::NEQ, NeedUpdate, Constant(0), NewDest, Dest);\n StoreResultGPR_WithOpSize(Op, Op->Dest, FinalDest, Size);\n}\n\nvoid OpDispatchBuilder::MOVSXDOp(OpcodeArgs) {\n // This instruction is a bit special\n // if SrcSize == 2\n // Then lower 16 bits of destination is written without changing the upper 48 bits\n // else /* Size == 4 */\n // if REX_WIDENING:\n // Sext(32, Src)\n // else\n // Zext(32, Src)\n //\n auto Size = std::min(OpSize::i32Bit, OpSizeFromSrc(Op));\n bool Sext = (Size != OpSize::i16Bit) && Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_REX_WIDENING;\n\n Ref Src = LoadSourceGPR_WithOpSize(Op, Op->Src[0], Size, Op->Flags, {.AllowUpperGarbage = Sext});\n if (Size == OpSize::i16Bit) {\n // This'll make sure to insert in to the lower 16bits without modifying upper bits\n StoreResultGPR_WithOpSize(Op, Op->Dest, Src, Size);\n } else if (Sext) {\n // With REX.W then Sext\n Src = _Sbfe(OpSize::i64Bit, IR::OpSizeAsBits(Size), 0, Src);\n StoreResultGPR(Op, Src);\n } else {\n // Without REX.W then Zext (store result implicitly zero extends)\n StoreResultGPR(Op, Src);\n }\n}\n\nvoid OpDispatchBuilder::MOVSXOp(OpcodeArgs) {\n // Load garbage in upper bits, since we're sign extending anyway\n const auto Size = OpSizeFromSrc(Op);\n Ref Src = LoadSourceGPR(Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true});\n\n // Sign-extend to DstSize and zero-extend to the register size, using a fast\n // path for 32-bit dests where the native 32-bit Sbfe zero extends the top.\n const auto DstSize = OpSizeFromDst(Op);\n Src = _Sbfe(DstSize == OpSize::i64Bit ? OpSize::i64Bit : OpSize::i32Bit, IR::OpSizeAsBits(Size), 0, Src);\n StoreResultGPR(Op, Op->Dest, Src);\n}\n\nvoid OpDispatchBuilder::MOVZXOp(OpcodeArgs) {\n Ref Src = LoadSourceGPR(Op, Op->Src[0], Op->Flags);\n // Store result implicitly zero extends\n StoreResultGPR(Op, Src);\n}\n\nvoid OpDispatchBuilder::CMPOp(OpcodeArgs, uint32_t SrcIndex) {\n // CMP is an instruction that does a SUB between the sources\n // Result isn't stored in result, only writes to flags\n Ref Src = LoadSourceGPR(Op, Op->Src[SrcIndex], Op->Flags, {.AllowUpperGarbage = true});\n Ref Dest = LoadSourceGPR(Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = true});\n CalculateFlags_SUB(OpSizeFromSrc(Op), Dest, Src);\n}\n\nvoid OpDispatchBuilder::CQOOp(OpcodeArgs) {\n Ref Src = LoadSourceGPR(Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true});\n auto Size = OpSizeFromSrc(Op);\n Ref Upper = _Sbfe(std::max(OpSize::i32Bit, Size), 1, GetSrcBitSize(Op) - 1, Src);\n\n StoreResultGPR(Op, Upper);\n}\n\nvoid OpDispatchBuilder::XCHGOp(OpcodeArgs) {\n // Load both the source and the destination\n if (Op->OP == 0x90 && Op->Src[0].IsGPR() && Op->Src[0].Data.GPR.GPR == FEXCore::X86State::REG_RAX && Op->Dest.IsGPR() &&\n Op->Dest.Data.GPR.GPR == FEXCore::X86State::REG_RAX) {\n // This is one heck of a sucky special case\n // If we are the 0x90 XCHG opcode (Meaning source is GPR RAX)\n // and destination register is ALSO RAX\n // and in this very specific case we are 32bit or above\n // Then this is a no-op\n // This is because 0x90 without a prefix is technically `xchg eax, eax`\n // But this would result in a zext on 64bit, which would ruin the no-op nature of the instruction\n // So x86-64 spec mandates this special case that even though it is a 32bit instruction and\n // is supposed to zext the result, it is a true no-op\n //\n // x86 spec text here:\n //\n // XCHG (E)AX, (E)AX (encoded instruction byte is 90H) is an alias for\n // NOP regardless of data size prefixes, including REX.W.\n //\n // Note that also includes 16-bit so we don't gate this on size. The\n // sequence (66 90) is a valid two-byte nop that we also ignore.\n if (Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_REP_PREFIX) {\n // If this instruction has a REP prefix then this is architecturally\n // defined to be a `PAUSE` instruction. On older processors this ends up\n // being a true `REP NOP` which is why they stuck this here.\n _Yield();\n }\n return;\n }\n\n // AllowUpperGarbage: OK to allow as it will be overwritten by StoreResult.\n Ref Src = LoadSourceGPR(Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true});\n if (DestIsMem(Op)) {\n HandledLock = (Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_LOCK) != 0;\n\n Ref Dest = MakeSegmentAddress(Op, Op->Dest);\n if (IsMonoBackpatcherBlock) {\n _MonoBackpatcherWrite(OpSizeFromSrc(Op), Src, Dest);\n } else {\n auto Result = _AtomicSwap(OpSizeFromSrc(Op), Src, Dest);\n StoreResultGPR(Op, Op->Src[0], Result);\n }\n } else {\n // AllowUpperGarbage: OK to allow as it will be overwritten by StoreResult.\n Ref Dest = LoadSourceGPR(Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = true});\n\n // Swap the contents\n // Order matters here since we don't want to swap context contents for one that effects the other\n StoreResultGPR(Op, Op->Dest, Src);\n StoreResultGPR(Op, Op->Src[0], Dest);\n }\n}\n\nvoid OpDispatchBuilder::CDQOp(OpcodeArgs) {\n const auto DstSize = OpSizeFromDst(Op);\n const auto SrcSize = DstSize / 2;\n Ref Src = LoadGPRRegister(X86State::REG_RAX, SrcSize, 0, true);\n\n Src = _Sbfe(DstSize <= OpSize::i32Bit ? OpSize::i32Bit : OpSize::i64Bit, IR::OpSizeAsBits(SrcSize), 0, Src);\n\n StoreResultGPR_WithOpSize(Op, Op->Dest, Src, DstSize);\n}\n\nvoid OpDispatchBuilder::SAHFOp(OpcodeArgs) {\n // Extract AH\n Ref Src = LoadGPRRegister(X86State::REG_RAX, OpSize::i8Bit, 8);\n\n // Clear bits that aren't supposed to be set\n Src = _Andn(OpSize::i64Bit, Src, Constant(0b101000));\n\n // Set the bit that is always set here\n Src = _Or(OpSize::i64Bit, Src, _InlineConstant(0b10));\n\n // Store the lower 8 bits in to RFLAGS\n SetPackedRFLAG(true, Src);\n}\nvoid OpDispatchBuilder::LAHFOp(OpcodeArgs) {\n // Load the lower 8 bits of the Rflags register\n auto RFLAG = GetPackedRFLAG(0xFF);\n\n // Store the lower 8 bits of the rflags register in to AH\n StoreGPRRegister(X86State::REG_RAX, RFLAG, OpSize::i8Bit, 8);\n}\n\nvoid OpDispatchBuilder::FLAGControlOp(OpcodeArgs) {\n // Calculate flags early.\n CalculateDeferredFlags();\n\n switch (Op->OP) {\n case 0xF5: // CMC\n CarryInvert();\n break;\n case 0xF8: // CLC\n SetCFInverted(Constant(1));\n break;\n case 0xF9: // STC\n SetCFInverted(Constant(0));\n break;\n case 0xFC: // CLD\n // Transformed\n StoreDF(Constant(1));\n break;\n case 0xFD: // STD\n StoreDF(Constant(-1));\n break;\n }\n}\n\n\nvoid OpDispatchBuilder::MOVSegOp(OpcodeArgs, bool ToSeg) {\n // In x86-64 mode the accesses to the segment registers end up being constant zero moves\n // Aside from FS/GS\n // In x86-64 mode the accesses to segment registers can actually still touch the segments\n // These write to the selector portion of the register\n //\n // FS and GS are specially handled here though\n // AMD documentation is /wrong/ in this regard\n // AMD documentation claims that the MOV to SReg and POP SReg registers will load a 32bit\n // value in to the HIDDEN portions of the FS and GS registers /OR/ ignored if a null selector is\n // selected for the registers\n // This statement is actually untrue, the instructions will /actually/ load 16bits in to the selector portion of the register!\n // Tested on a Zen+ CPU, the selector is the portion that is modified!\n // We don't currently support FS/GS selector modifying, so this needs to be asserted out\n // The loads here also load the selector, NOT the base\n\n if (ToSeg) {\n Ref Src = LoadSourceGPR_WithOpSize(Op, Op->Src[0], OpSize::i16Bit, Op->Flags);\n\n switch (Op->Dest.Data.GPR.GPR) {\n case FEXCore::X86State::REG_RAX: // ES\n case FEXCore::X86State::REG_R8: // ES\n _StoreContextGPR(OpSize::i16Bit, Src, offsetof(FEXCore::Core::CPUState, es_idx));\n UpdatePrefixFromSegment(Src, FEXCore::X86Tables::DecodeFlags::FLAG_ES_PREFIX);\n break;\n case FEXCore::X86State::REG_RBX: // DS\n case FEXCore::X86State::REG_R11: // DS\n _StoreContextGPR(OpSize::i16Bit, Src, offsetof(FEXCore::Core::CPUState, ds_idx));\n UpdatePrefixFromSegment(Src, FEXCore::X86Tables::DecodeFlags::FLAG_DS_PREFIX);\n break;\n case FEXCore::X86State::REG_RCX: // CS\n case FEXCore::X86State::REG_R9: // CS\n // CPL3 can't write to this\n Break(FEXCore::IR::BreakDefinition {\n .ErrorRegister = 0,\n .Signal = SIGILL,\n .TrapNumber = 0,\n .si_code = 0,\n });\n break;\n case FEXCore::X86State::REG_RDX: // SS\n case FEXCore::X86State::REG_R10: // SS\n _StoreContextGPR(OpSize::i16Bit, Src, offsetof(FEXCore::Core::CPUState, ss_idx));\n UpdatePrefixFromSegment(Src, FEXCore::X86Tables::DecodeFlags::FLAG_SS_PREFIX);\n break;\n case FEXCore::X86State::REG_RBP: // GS\n case FEXCore::X86State::REG_R13: // GS\n if (!Is64BitMode) {\n _StoreContextGPR(OpSize::i16Bit, Src, offsetof(FEXCore::Core::CPUState, gs_idx));\n UpdatePrefixFromSegment(Src, FEXCore::X86Tables::DecodeFlags::FLAG_GS_PREFIX);\n } else {\n LogMan::Msg::EFmt(\"We don't support modifying GS selector in 64bit mode!\");\n DecodeFailure = true;\n }\n break;\n case FEXCore::X86State::REG_RSP: // FS\n case FEXCore::X86State::REG_R12: // FS\n if (!Is64BitMode) {\n _StoreContextGPR(OpSize::i16Bit, Src, offsetof(FEXCore::Core::CPUState, fs_idx));\n UpdatePrefixFromSegment(Src, FEXCore::X86Tables::DecodeFlags::FLAG_FS_PREFIX);\n } else {\n LogMan::Msg::EFmt(\"We don't support modifying FS selector in 64bit mode!\");\n DecodeFailure = true;\n }\n break;\n default: UnimplementedOp(Op); return;\n }\n } else {\n Ref Segment {};\n\n switch (Op->Src[0].Data.GPR.GPR) {\n case FEXCore::X86State::REG_RAX: // ES\n case FEXCore::X86State::REG_R8: // ES\n Segment = _LoadContextGPR(OpSize::i16Bit, offsetof(FEXCore::Core::CPUState, es_idx));\n break;\n case FEXCore::X86State::REG_RBX: // DS\n case FEXCore::X86State::REG_R11: // DS\n Segment = _LoadContextGPR(OpSize::i16Bit, offsetof(FEXCore::Core::CPUState, ds_idx));\n break;\n case FEXCore::X86State::REG_RCX: // CS\n case FEXCore::X86State::REG_R9: // CS\n Segment = _LoadContextGPR(OpSize::i16Bit, offsetof(FEXCore::Core::CPUState, cs_idx));\n break;\n case FEXCore::X86State::REG_RDX: // SS\n case FEXCore::X86State::REG_R10: // SS\n Segment = _LoadContextGPR(OpSize::i16Bit, offsetof(FEXCore::Core::CPUState, ss_idx));\n break;\n case FEXCore::X86State::REG_RBP: // GS\n case FEXCore::X86State::REG_R13: // GS\n if (Is64BitMode) {\n Segment = Constant(0);\n } else {\n Segment = _LoadContextGPR(OpSize::i16Bit, offsetof(FEXCore::Core::CPUState, gs_idx));\n }\n break;\n case FEXCore::X86State::REG_RSP: // FS\n case FEXCore::X86State::REG_R12: // FS\n if (Is64BitMode) {\n Segment = Constant(0);\n } else {\n Segment = _LoadContextGPR(OpSize::i16Bit, offsetof(FEXCore::Core::CPUState, fs_idx));\n }\n break;\n default: UnimplementedOp(Op); return;\n }\n if (DestIsMem(Op)) {\n // If the destination is memory then we always store 16-bits only\n StoreResultGPR_WithOpSize(Op, Op->Dest, Segment, OpSize::i16Bit);\n } else {\n // If the destination is a GPR then we follow register storing rules\n StoreResultGPR(Op, Segment);\n }\n }\n}\n\nvoid OpDispatchBuilder::MOVOffsetOp(OpcodeArgs) {\n switch (Op->OP) {\n case 0xA0:\n case 0xA1: {\n // Source is memory(literal)\n // Dest is GPR\n auto Src = LoadSourceGPR(Op, Op->Src[0], Op->Flags, {.ForceLoad = true});\n StoreResultGPR(Op, Op->Dest, Src);\n break;\n }\n case 0xA2:\n case 0xA3: {\n // Source is GPR\n // Dest is memory(literal)\n Ref Src = LoadSourceGPR(Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true});\n\n // This one is a bit special since the destination is a literal\n // So the destination gets stored in Src[1]\n StoreResultGPR(Op, Op->Src[1], Src);\n break;\n }\n }\n}\n\nvoid OpDispatchBuilder::CPUIDOp(OpcodeArgs) {\n const auto GPRSize = GetGPROpSize();\n\n Ref Src = LoadSourceGPR_WithOpSize(Op, Op->Src[0], GPRSize, Op->Flags);\n Ref Leaf = LoadGPRRegister(X86State::REG_RCX);\n\n Ref RAX = _AllocateGPR(false);\n Ref RBX = _AllocateGPR(false);\n Ref RCX = _AllocateGPR(false);\n Ref RDX = _AllocateGPR(false);\n\n _CPUID(Src, Leaf, RAX, RBX, RCX, RDX);\n\n StoreGPRRegister(X86State::REG_RAX, RAX);\n StoreGPRRegister(X86State::REG_RBX, RBX);\n StoreGPRRegister(X86State::REG_RCX, RCX);\n StoreGPRRegister(X86State::REG_RDX, RDX);\n}\n\nuint32_t OpDispatchBuilder::GetConstantShift(X86Tables::DecodedOp Op, bool Is1Bit) {\n if (Is1Bit) {\n return 1;\n } else {\n // x86 masks the shift by 0x3F or 0x1F depending on size of op\n const auto Size = OpSizeFromSrc(Op);\n uint64_t Mask = Size == OpSize::i64Bit ? 0x3F : 0x1F;\n\n return Op->Src[1].Literal() & Mask;\n }\n}\n\nvoid OpDispatchBuilder::XGetBVOp(OpcodeArgs) {\n Ref Function = LoadGPRRegister(X86State::REG_RCX);\n\n auto RAX = _AllocateGPR(false);\n auto RDX = _AllocateGPR(false);\n _XGetBV(Function, RAX, RDX);\n\n StoreGPRRegister(X86State::REG_RAX, RAX);\n StoreGPRRegister(X86State::REG_RDX, RDX);\n}\n\nvoid OpDispatchBuilder::SHLOp(OpcodeArgs) {\n const auto Size = OpSizeFromSrc(Op);\n auto Dest = LoadSourceGPR(Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = true});\n auto Src = LoadSourceGPR(Op, Op->Src[1], Op->Flags, {.AllowUpperGarbage = true});\n\n Ref Result = _Lshl(Size == OpSize::i64Bit ? OpSize::i64Bit : OpSize::i32Bit, Dest, Src);\n HandleShift(Op, Result, Dest, ShiftType::LSL, Src);\n}\n\nvoid OpDispatchBuilder::SHLImmediateOp(OpcodeArgs, bool SHL1Bit) {\n Ref Dest = LoadSourceGPR(Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = true});\n\n uint64_t Shift = GetConstantShift(Op, SHL1Bit);\n const auto Size = GetSrcBitSize(Op);\n\n Ref Result = _Lshl(Size == 64 ? OpSize::i64Bit : OpSize::i32Bit, Dest, Constant(Shift));\n\n CalculateFlags_ShiftLeftImmediate(OpSizeFromSrc(Op), Result, Dest, Shift);\n CalculateDeferredFlags();\n StoreResultGPR(Op, Result);\n}\n\nvoid OpDispatchBuilder::SHROp(OpcodeArgs) {\n const auto Size = OpSizeFromSrc(Op);\n auto Dest = LoadSourceGPR(Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = Size >= OpSize::i32Bit});\n auto Src = LoadSourceGPR(Op, Op->Src[1], Op->Flags, {.AllowUpperGarbage = true});\n\n auto ALUOp = _Lshr(std::max(OpSize::i32Bit, Size), Dest, Src);\n HandleShift(Op, ALUOp, Dest, ShiftType::LSR, Src);\n}\n\nvoid OpDispatchBuilder::SHRImmediateOp(OpcodeArgs, bool SHR1Bit) {\n const auto Size = GetSrcBitSize(Op);\n auto Dest = LoadSourceGPR(Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = Size >= 32});\n\n uint64_t Shift = GetConstantShift(Op, SHR1Bit);\n auto ALUOp = _Lshr(Size == 64 ? OpSize::i64Bit : OpSize::i32Bit, Dest, Constant(Shift));\n\n CalculateFlags_ShiftRightImmediate(OpSizeFromSrc(Op), ALUOp, Dest, Shift);\n CalculateDeferredFlags();\n StoreResultGPR(Op, ALUOp);\n}\n\nvoid OpDispatchBuilder::SHLDOp(OpcodeArgs) {\n // Calculate flags early.\n CalculateDeferredFlags();\n\n const auto Size = GetSrcBitSize(Op);\n\n // Allow garbage on the Src if it will be ignored by the Lshr below\n Ref Src = LoadSourceGPR(Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = Size >= 32});\n Ref Dest = LoadSourceGPR(Op, Op->Dest, Op->Flags);\n\n // Allow garbage on the shift, we're masking it anyway.\n Ref Shift = LoadSourceGPR(Op, Op->Src[1], Op->Flags, {.AllowUpperGarbage = true});\n\n // x86 masks the shift by 0x3F or 0x1F depending on size of op.\n if (Size == 64) {\n Shift = _And(OpSize::i64Bit, Shift, _InlineConstant(0x3F));\n } else {\n Shift = _And(OpSize::i64Bit, Shift, _InlineConstant(0x1F));\n }\n\n // a64 masks the bottom bits, so if we're using a native 32/64-bit shift, we\n // can negate to do the subtract (it's congruent), which saves a constant.\n auto ShiftRight = Size >= 32 ? _Neg(OpSize::i64Bit, Shift) : Sub(OpSize::i64Bit, Constant(Size), Shift);\n\n auto Tmp1 = _Lshl(OpSize::i64Bit, Dest, Shift);\n auto Tmp2 = _Lshr(Size == 64 ? OpSize::i64Bit : OpSize::i32Bit, Src, ShiftRight);\n\n Ref Res = _Or(OpSize::i64Bit, Tmp1, Tmp2);\n\n // If shift count was zero then output doesn't change\n // Needs to be checked for the 32bit operand case\n // where shift = 0 and the source register still gets Zext\n //\n // TODO: With a backwards pass ahead-of-time, we could stick this in the\n // if(shift) used for flags.\n //\n // TODO: This whole function wants to be wrapped in the if. Maybe b/w pass is\n // a good idea after all.\n Res = _Select(OpSize::i64Bit, OpSize::i64Bit, CondClass::EQ, Shift, Constant(0), Dest, Res);\n\n HandleShift(Op, Res, Dest, ShiftType::LSL, Shift);\n}\n\nvoid OpDispatchBuilder::SHLDImmediateOp(OpcodeArgs) {\n uint64_t Shift = GetConstantShift(Op, false);\n const auto Size = GetSrcBitSize(Op);\n\n Ref Src = LoadSourceGPR(Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = Size >= 32});\n Ref Dest = LoadSourceGPR(Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = Size >= 32});\n\n if (Shift != 0) {\n Ref Res {};\n if (Size < 32) {\n Ref ShiftLeft = Constant(Shift);\n auto ShiftRight = Size - Shift;\n\n auto Tmp1 = _Lshl(OpSize::i64Bit, Dest, ShiftLeft);\n Ref Tmp2 = ShiftRight ? _Lshr(OpSize::i32Bit, Src, Constant(ShiftRight)) : Src;\n\n Res = _Or(OpSize::i64Bit, Tmp1, Tmp2);\n } else {\n // 32-bit and 64-bit SHLD behaves like an EXTR where the lower bits are filled from the source.\n Res = _Extr(OpSizeFromSrc(Op), Dest, Src, Size - Shift);\n }\n\n CalculateFlags_ShiftLeftImmediate(OpSizeFromSrc(Op), Res, Dest, Shift);\n CalculateDeferredFlags();\n StoreResultGPR(Op, Res);\n } else if (Shift == 0 && Size == 32) {\n // Ensure Zext still occurs\n StoreResultGPR(Op, Dest);\n }\n}\n\nvoid OpDispatchBuilder::SHRDOp(OpcodeArgs) {\n // Calculate flags early.\n // This instruction conditionally generates flags so we need to insure sane state going in.\n CalculateDeferredFlags();\n\n Ref Src = LoadSourceGPR(Op, Op->Src[0], Op->Flags);\n Ref Dest = LoadSourceGPR(Op, Op->Dest, Op->Flags);\n\n Ref Shift = LoadGPRRegister(X86State::REG_RCX);\n\n const auto Size = GetDstBitSize(Op);\n\n // x86 masks the shift by 0x3F or 0x1F depending on size of op\n if (Size == 64) {\n Shift = _And(OpSize::i64Bit, Shift, _InlineConstant(0x3F));\n } else {\n Shift = _And(OpSize::i64Bit, Shift, _InlineConstant(0x1F));\n }\n\n auto ShiftLeft = Sub(OpSize::i64Bit, Constant(Size), Shift);\n\n auto Tmp1 = _Lshr(Size == 64 ? OpSize::i64Bit : OpSize::i32Bit, Dest, Shift);\n auto Tmp2 = _Lshl(OpSize::i64Bit, Src, ShiftLeft);\n\n Ref Res = _Or(OpSize::i64Bit, Tmp1, Tmp2);\n\n // If shift count was zero then output doesn't change\n // Needs to be checked for the 32bit operand case\n // where shift = 0 and the source register still gets Zext\n Res = _Select(OpSize::i64Bit, OpSize::i64Bit, CondClass::EQ, Shift, Constant(0), Dest, Res);\n\n HandleShift(Op, Res, Dest, ShiftType::LSR, Shift);\n}\n\nvoid OpDispatchBuilder::SHRDImmediateOp(OpcodeArgs) {\n Ref Src = LoadSourceGPR(Op, Op->Src[0], Op->Flags);\n Ref Dest = LoadSourceGPR(Op, Op->Dest, Op->Flags);\n\n uint64_t Shift = GetConstantShift(Op, false);\n const auto Size = GetSrcBitSize(Op);\n\n if (Shift != 0) {\n Ref Res {};\n if (Size < 32) {\n Ref ShiftRight = Constant(Shift);\n auto ShiftLeft = Constant(Size - Shift);\n\n auto Tmp1 = _Lshr(OpSize::i32Bit, Dest, ShiftRight);\n auto Tmp2 = _Lshl(OpSize::i64Bit, Src, ShiftLeft);\n\n Res = _Or(OpSize::i64Bit, Tmp1, Tmp2);\n } else {\n // 32-bit and 64-bit SHRD behaves like an EXTR where the upper bits are filled from the source.\n Res = _Extr(OpSizeFromSrc(Op), Src, Dest, Shift);\n }\n\n StoreResultGPR(Op, Res);\n CalculateFlags_ShiftRightDoubleImmediate(OpSizeFromSrc(Op), Res, Dest, Shift);\n } else if (Shift == 0 && Size == 32) {\n // Ensure Zext still occurs\n StoreResultGPR(Op, Dest);\n }\n}\n\nvoid OpDispatchBuilder::ASHROp(OpcodeArgs, bool Immediate, bool SHR1Bit) {\n const auto Size = OpSizeFromSrc(Op);\n const auto OpSize = std::max(OpSize::i32Bit, OpSizeFromDst(Op));\n\n // If Size < 4, then we Sbfe the Dest so we can have garbage.\n // Otherwise, if Size = Opsize, then both are 4 or 8 and match the a64\n // semantics directly, so again we can have garbage. The only case where we\n // need zero-extension here is when the sizes mismatch.\n auto Dest = LoadSourceGPR(Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = (OpSize == Size) || (Size < OpSize::i32Bit)});\n\n if (Size < OpSize::i32Bit) {\n Dest = _Sbfe(OpSize::i64Bit, IR::OpSizeAsBits(Size), 0, Dest);\n }\n\n if (Immediate) {\n uint64_t Shift = GetConstantShift(Op, SHR1Bit);\n Ref Result = _Ashr(OpSize, Dest, Constant(Shift));\n\n CalculateFlags_SignShiftRightImmediate(OpSizeFromSrc(Op), Result, Dest, Shift);\n CalculateDeferredFlags();\n StoreResultGPR(Op, Result);\n } else {\n auto Src = LoadSourceGPR(Op, Op->Src[1], Op->Flags, {.AllowUpperGarbage = true});\n Ref Result = _Ashr(OpSize, Dest, Src);\n\n HandleShift(Op, Result, Dest, ShiftType::ASR, Src);\n }\n}\n\nvoid OpDispatchBuilder::RotateOp(OpcodeArgs, bool Left, bool IsImmediate, bool Is1Bit) {\n CalculateDeferredFlags();\n\n const uint32_t Size = GetSrcBitSize(Op);\n const auto OpSize = Size == 64 ? OpSize::i64Bit : OpSize::i32Bit;\n uint64_t UnmaskedConst {};\n\n // x86 masks the shift by 0x3F or 0x1F depending on size of op. But it's\n // equivalent to mask to the actual size of the op, that way we can bound\n // things tighter for 8-bit later in the function.\n uint64_t Mask = Size == 8 ? 7 : (Size == 64 ? 0x3F : 0x1F);\n\n ArithRef UnmaskedSrc;\n if (Is1Bit || IsImmediate) {\n UnmaskedConst = GetConstantShift(Op, Is1Bit);\n UnmaskedSrc = ARef(UnmaskedConst);\n } else {\n UnmaskedSrc = ARef(LoadSourceGPR(Op, Op->Src[1], Op->Flags, {.AllowUpperGarbage = true}));\n }\n auto Src = UnmaskedSrc.And(Mask);\n\n // We fill the upper bits so we allow garbage on load.\n auto Dest = LoadSourceGPR(Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = true});\n\n if (Size < 32) {\n // ARM doesn't support 8/16bit rotates. Emulate with an insert\n // StoreResult truncates back to a 8/16 bit value\n Dest = _Bfi(OpSize, Size, Left ? (32 - Size) : Size, Dest, Dest);\n }\n\n // To rotate 64-bits left, right-rotate by (64 - Shift) = -Shift mod 64.\n auto Res = _Ror(OpSize, Dest, (Left ? Src.Neg() : Src).Ref());\n StoreResultGPR(Op, Res);\n\n if (Is1Bit || IsImmediate) {\n if (UnmaskedSrc.C) {\n // Extract the last bit shifted in to CF\n SetCFDirect(Res, Left ? 0 : Size - 1, true);\n\n // For ROR, OF is the XOR of the new CF bit and the most significant bit of the result.\n // For ROL, OF is the LSB and MSB XOR'd together.\n // OF is architecturally only defined for 1-bit rotate.\n if (UnmaskedSrc.C == 1) {\n auto NewOF = _XorShift(OpSize, Res, Res, ShiftType::LSR, Left ? Size - 1 : 1);\n SetRFLAG(NewOF, Left ? 0 : Size - 2, true);\n }\n }\n } else {\n HandleNZCVWrite();\n RectifyCarryInvert(true);\n\n // We deferred the masking for 8-bit to the flag section, do it here.\n if (Size == 8) {\n Src = UnmaskedSrc.And(0x1F);\n }\n\n _RotateFlags(OpSizeFromSrc(Op), Res, Src.Ref(), Left);\n }\n}\n\nvoid OpDispatchBuilder::ANDNBMIOp(OpcodeArgs) {\n auto* Src1 = LoadSourceGPR(Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true});\n auto* Src2 = LoadSourceGPR(Op, Op->Src[1], Op->Flags, {.AllowUpperGarbage = true});\n\n auto Dest = _Andn(OpSizeFromSrc(Op), Src2, Src1);\n\n StoreResultGPR(Op, Dest);\n CalculateFlags_Logical(OpSizeFromSrc(Op), Dest);\n}\n\nvoid OpDispatchBuilder::BEXTRBMIOp(OpcodeArgs) {\n // Essentially (Src1 >> Start) & ((1 << Length) - 1)\n // along with some edge-case handling and flag setting.\n\n LOGMAN_THROW_A_FMT(Op->InstSize >= 4, \"No masking needed\");\n auto* Src1 = LoadSourceGPR(Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true});\n auto* Src2 = LoadSourceGPR(Op, Op->Src[1], Op->Flags, {.AllowUpperGarbage = true});\n\n const auto Size = OpSizeFromSrc(Op);\n const auto SrcSize = IR::OpSizeAsBits(Size);\n const auto MaxSrcBit = SrcSize - 1;\n auto MaxSrcBitOp = Constant(MaxSrcBit);\n\n // Shift the operand down to the starting bit\n auto Start = _Bfe(OpSizeFromSrc(Op), 8, 0, Src2);\n auto Shifted = _Lshr(Size, Src1, Start);\n\n // Shifts larger than operand size need to be set to zero.\n auto SanitizedShifted = _Select(Size, Size, CondClass::ULE, Start, MaxSrcBitOp, Shifted, Constant(0));\n\n // Now handle the length specifier.\n auto Length = _Bfe(Size, 8, 8, Src2);\n\n // Now build up the mask\n // (1 << Length) - 1 = ~(~0 << Length)\n auto AllOnes = Constant(~0ull);\n auto InvertedMask = _Lshl(Size, AllOnes, Length);\n\n // Now put it all together and make the result.\n auto Masked = _Andn(Size, SanitizedShifted, InvertedMask);\n\n // Sanitize the length. If it is above the max, we don't do the masking.\n auto Dest = _Select(Size, Size, CondClass::ULE, Length, MaxSrcBitOp, Masked, SanitizedShifted);\n\n // Finally store the result.\n StoreResultGPR(Op, Dest);\n\n // ZF is set properly. CF and OF are defined as being set to zero. SF, PF, and\n // AF are undefined.\n SetNZ_ZeroCV(GetOpSize(Dest), Dest);\n InvalidatePF_AF();\n}\n\nvoid OpDispatchBuilder::BLSIBMIOp(OpcodeArgs) {\n // Equivalent to performing: SRC & -SRC\n LOGMAN_THROW_A_FMT(Op->InstSize >= 4, \"No masking needed\");\n const auto Size = OpSizeFromSrc(Op);\n\n auto* Src = LoadSourceGPR(Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true});\n auto NegatedSrc = _Neg(Size, Src);\n auto Result = _And(Size, Src, NegatedSrc);\n\n StoreResultGPR(Op, Result);\n\n // CF is cleared if Src is zero, otherwise it's set. However, Src is zero iff\n // Result is zero, so we can test the result instead. So, CF is just the\n // inverted ZF.\n //\n // ZF/SF/OF set as usual.\n SetNZ_ZeroCV(Size, Result);\n InvalidatePF_AF();\n SetCFInverted(GetRFLAG(X86State::RFLAG_ZF_RAW_LOC));\n}\n\nvoid OpDispatchBuilder::BLSMSKBMIOp(OpcodeArgs) {\n // Equivalent to: (Src - 1) ^ Src\n LOGMAN_THROW_A_FMT(Op->InstSize >= 4, \"No masking needed\");\n const auto Size = OpSizeFromSrc(Op);\n\n auto* Src = LoadSourceGPR(Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true});\n auto Result = _Xor(Size, Sub(Size, Src, 1), Src);\n\n StoreResultGPR(Op, Result);\n InvalidatePF_AF();\n\n // CF set according to the Src\n auto CFInv = To01(OpSize::i64Bit, Src);\n\n // The output of BLSMSK is always nonzero, so TST will clear Z (along with C\n // and O) while setting S.\n SetNZ_ZeroCV(Size, Result);\n SetCFInverted(CFInv);\n}\n\nvoid OpDispatchBuilder::BLSRBMIOp(OpcodeArgs) {\n // Equivalent to: (Src - 1) & Src\n LOGMAN_THROW_A_FMT(Op->InstSize >= 4, \"No masking needed\");\n const auto Size = OpSizeFromSrc(Op);\n\n auto* Src = LoadSourceGPR(Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true});\n auto Result = _And(Size, Sub(Size, Src, 1), Src);\n\n StoreResultGPR(Op, Result);\n\n auto CFInv = To01(OpSize::i64Bit, Src);\n\n SetNZ_ZeroCV(Size, Result);\n SetCFInverted(CFInv);\n InvalidatePF_AF();\n}\n\n// Handles SARX, SHLX, and SHRX\nvoid OpDispatchBuilder::BMI2Shift(OpcodeArgs) {\n // In the event the source is a memory operand, use the\n // exact width instead of the GPR size.\n const auto GPRSize = GetGPROpSize();\n const auto Size = OpSizeFromSrc(Op);\n const auto SrcSize = Op->Src[0].IsGPR() ? GPRSize : Size;\n\n auto* Src = LoadSourceGPR_WithOpSize(Op, Op->Src[0], SrcSize, Op->Flags);\n auto* Shift = LoadSourceGPR_WithOpSize(Op, Op->Src[1], GPRSize, Op->Flags, {.AllowUpperGarbage = true});\n\n Ref Result;\n if (Op->OP == 0x6F7) {\n // SARX\n Result = _Ashr(Size, Src, Shift);\n } else if (Op->OP == 0x5F7) {\n // SHLX\n Result = _Lshl(Size, Src, Shift);\n } else {\n // SHRX\n Result = _Lshr(Size, Src, Shift);\n }\n\n StoreResultGPR(Op, Result);\n}\n\nvoid OpDispatchBuilder::BZHI(OpcodeArgs) {\n const auto Size = OpSizeFromSrc(Op);\n const auto OperandSize = IR::OpSizeAsBits(Size);\n\n // In 32-bit mode we only look at bottom 32-bit, no 8 or 16-bit BZHI so no\n // need to zero-extend sources\n auto* Src = LoadSourceGPR(Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true});\n\n auto* Index = LoadSourceGPR(Op, Op->Src[1], Op->Flags, {.AllowUpperGarbage = true});\n\n // Clear the high bits specified by the index. A64 only considers bottom bits\n // of the shift, so we don't need to mask bottom 8-bits ourselves.\n // Out-of-bounds results ignored after.\n auto Mask = _Lshl(Size, Constant(-1), Index);\n auto MaskResult = _Andn(Size, Src, Mask);\n\n // If the index is above OperandSize, we don't clear anything. BZHI only\n // considers the bottom 8-bits, so we really want to know if the bottom 8-bits\n // have their top bits set. Test exactly that.\n //\n // Because we're clobbering flags internally we ignore all carry invert\n // shenanigans and use the raw versions here.\n _TestNZ(OpSize::i64Bit, Index, Constant(0xFF & ~(OperandSize - 1)));\n auto Result = _NZCVSelect(Size, CondClass::NEQ, Src, MaskResult);\n StoreResultGPR(Op, Result);\n\n auto CFInv = _NZCVSelect01(CondClass::EQ);\n\n InvalidatePF_AF();\n SetNZ_ZeroCV(Size, Result);\n SetCFInverted(CFInv);\n}\n\nvoid OpDispatchBuilder::RORX(OpcodeArgs) {\n const auto SrcSize = OpSizeFromSrc(Op);\n const auto SrcSizeBits = IR::OpSizeAsBits(SrcSize);\n const auto Amount = Op->Src[1].Literal() & (SrcSizeBits - 1);\n const auto GPRSize = GetGPROpSize();\n\n const auto DoRotation = Amount != 0 && Amount < SrcSizeBits;\n const auto IsSameGPR = Op->Src[0].IsGPR() && Op->Dest.IsGPR() && Op->Src[0].Data.GPR.GPR == Op->Dest.Data.GPR.GPR;\n const auto SrcSizeIsGPRSize = SrcSize == GPRSize;\n\n // If we don't need to rotate and our source is the same as the destination\n // then we don't need to do anything at all. We still need to be careful,\n // since 32-bit operations on 64-bit mode still need to zero-extend the\n // destination register. So also compare source size and GPR size.\n //\n // Very unlikely, but hey, we can do nothing faster.\n if (!DoRotation && IsSameGPR && SrcSizeIsGPRSize) [[unlikely]] {\n return;\n }\n\n auto* Src = LoadSourceGPR(Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true});\n auto* Result = Src;\n if (DoRotation) [[likely]] {\n Result = _Ror(OpSizeFromSrc(Op), Src, _InlineConstant(Amount));\n }\n\n StoreResultGPR(Op, Result);\n}\n\nvoid OpDispatchBuilder::MULX(OpcodeArgs) {\n // RDX is the implied source operand in the instruction\n const auto OpSize = OpSizeFromSrc(Op);\n\n // Src1 can be a memory operand, so ensure we constrain to the\n // absolute width of the access in that scenario.\n const auto GPRSize = GetGPROpSize();\n const auto Src1Size = Op->Src[1].IsGPR() ? GPRSize : OpSize;\n\n Ref Src1 = LoadSourceGPR_WithOpSize(Op, Op->Src[1], Src1Size, Op->Flags);\n Ref Src2 = LoadGPRRegister(X86State::REG_RDX, GPRSize);\n\n // As per the Intel Software Development Manual, if the destination and\n // first operand correspond to the same register, then the result\n // will be the high half of the multiplication result.\n if (Op->Dest.Data.GPR.GPR == Op->Src[0].Data.GPR.GPR) {\n Ref ResultHi = _UMulH(OpSize, Src1, Src2);\n StoreResultGPR(Op, Op->Dest, ResultHi);\n } else {\n Ref ResultLo = _UMul(OpSize, Src1, Src2);\n Ref ResultHi = _UMulH(OpSize, Src1, Src2);\n\n StoreResultGPR(Op, Op->Src[0], ResultLo);\n StoreResultGPR(Op, Op->Dest, ResultHi);\n }\n}\n\nvoid OpDispatchBuilder::PDEP(OpcodeArgs) {\n LOGMAN_THROW_A_FMT(Op->InstSize >= 4, \"No masking needed\");\n auto* Input = LoadSourceGPR(Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true});\n auto* Mask = LoadSourceGPR(Op, Op->Src[1], Op->Flags, {.AllowUpperGarbage = true});\n auto Result = _PDep(OpSizeFromSrc(Op), Input, Mask);\n\n StoreResultGPR(Op, Op->Dest, Result);\n}\n\nvoid OpDispatchBuilder::PEXT(OpcodeArgs) {\n LOGMAN_THROW_A_FMT(Op->InstSize >= 4, \"No masking needed\");\n auto* Input = LoadSourceGPR(Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true});\n auto* Mask = LoadSourceGPR(Op, Op->Src[1], Op->Flags, {.AllowUpperGarbage = true});\n auto Result = _PExt(OpSizeFromSrc(Op), Input, Mask);\n\n StoreResultGPR(Op, Op->Dest, Result);\n}\n\nvoid OpDispatchBuilder::ADXOp(OpcodeArgs) {\n const auto OpSize = OpSizeFromSrc(Op);\n\n // Only 32/64-bit anyway so allow garbage, we use 32-bit ops.\n auto* Src = LoadSourceGPR(Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true});\n auto* Before = LoadSourceGPR(Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = true});\n\n // Handles ADCX and ADOX\n const bool IsADCX = Op->OP == 0x1F6;\n auto Zero = Constant(0);\n\n // Before we go trashing NZCV, save the current NZCV state.\n Ref OldNZCV = GetNZCV();\n\n // We want to use arm64 adc. For ADOX, copy the overflow flag into CF. For\n // ADCX, we just rectify the carry.\n if (IsADCX) {\n RectifyCarryInvert(false);\n } else {\n // If overflow, 0 - 0 sets carry. Else, forces carry to 0.\n _CondSubNZCV(OpSize::i32Bit, Zero, Zero, CondClass::FU, 0x0 /* nzcv */);\n }\n\n // Do the actual add.\n HandleNZCV_RMW();\n auto Result = _AdcWithFlags(OpSize, Src, Before);\n StoreResultGPR(Op, Result);\n\n // Now restore all flags except the one we're updating.\n if (CTX->HostFeatures.SupportsFlagM) {\n // For ADOX, we need to copy the new carry into the overflow flag. If carry is clear (ULT with uninverted\n // carry), 0 - 0 clears overflow. Else, force overflow on.\n if (!IsADCX) {\n _CondSubNZCV(OpSize::i32Bit, Zero, Zero, CondClass::ULT, 0x1 /* nzcV */);\n }\n\n _RmifNZCV(OldNZCV, 28, IsADCX ? 0xd /* NzcV */ : 0xe /* NZCv */);\n } else {\n // For either operation, insert the new flag into the old NZCV.\n bool SavedCFInvert = CFInverted;\n CFInverted = false;\n Ref OutputCF = GetRFLAG(X86State::RFLAG_CF_RAW_LOC, IsADCX);\n CFInverted = IsADCX ? true : SavedCFInvert;\n\n Ref NewNZCV = _Bfi(OpSize::i32Bit, 1, IsADCX ? 29 : 28, OldNZCV, OutputCF);\n SetNZCV(NewNZCV);\n }\n}\n\nvoid OpDispatchBuilder::RCROp1Bit(OpcodeArgs) {\n // Calculate flags early.\n CalculateDeferredFlags();\n\n // We expliclty mask for <32-bit so allow garbage\n Ref Dest = LoadSourceGPR(Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = true});\n const auto Size = GetSrcBitSize(Op);\n auto CF = GetRFLAG(FEXCore::X86State::RFLAG_CF_RAW_LOC);\n Ref Res;\n\n // Our new CF will be bit 0 of the source. Set upfront to avoid a move.\n SetCFDirect(Dest, 0, true);\n\n uint32_t Shift = 1;\n\n if (Size == 32 || Size == 64) {\n // Rotate and insert CF in the upper bit\n Res = _Extr(OpSizeFromSrc(Op), CF, Dest, Shift);\n } else {\n // Res = Src >> Shift\n Res = _Bfe(OpSize::i32Bit, Size - Shift, Shift, Dest);\n\n // inject the CF\n Res = _Orlshl(OpSize::i32Bit, Res, CF, Size - Shift);\n }\n\n StoreResultGPR(Op, Res);\n\n // OF is the top two MSBs XOR'd together\n // Only when Shift == 1, it is undefined otherwise\n SetRFLAG(_XorShift(OpSize::i64Bit, Res, Res, ShiftType::LSR, 1), Size - 2, true);\n}\n\nvoid OpDispatchBuilder::RCROp8x1Bit(OpcodeArgs) {\n // Calculate flags early.\n CalculateDeferredFlags();\n\n Ref Dest = LoadSourceGPR(Op, Op->Dest, Op->Flags);\n const auto SizeBit = GetSrcBitSize(Op);\n auto CF = GetRFLAG(FEXCore::X86State::RFLAG_CF_RAW_LOC);\n\n // Our new CF will be bit (Shift - 1) of the source\n SetCFDirect(Dest, 0, true);\n\n // Rotate and insert CF in the upper bit\n Ref Res = _Bfe(OpSize::i32Bit, 7, 1, Dest);\n Res = _Bfi(OpSize::i32Bit, 1, 7, Res, CF);\n\n StoreResultGPR(Op, Res);\n\n // OF is the top two MSBs XOR'd together\n SetRFLAG(_XorShift(OpSize::i32Bit, Res, Res, ShiftType::LSR, 1), SizeBit - 2, true);\n}\n\nvoid OpDispatchBuilder::RCROp(OpcodeArgs) {\n const auto Size = GetSrcBitSize(Op);\n\n if (Size == 8 || Size == 16) {\n RCRSmallerOp(Op);\n return;\n }\n\n const auto Mask = (Size == 64) ? 0x3F : 0x1F;\n\n // Calculate flags early.\n CalculateDeferredFlags();\n const auto OpSize = OpSizeFromSrc(Op);\n\n Ref Src = LoadSourceGPR(Op, Op->Src[1], Op->Flags, {.AllowUpperGarbage = true});\n uint64_t Const;\n if (IsValueConstant(WrapNode(Src), &Const)) {\n Const &= Mask;\n if (!Const) {\n ZeroShiftResult(Op);\n return;\n }\n\n Ref Dest = LoadSourceGPR(Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = true});\n\n // Res = Src >> Shift\n Ref Res = _Lshr(OpSize, Dest, Src);\n auto CF = GetRFLAG(FEXCore::X86State::RFLAG_CF_RAW_LOC);\n\n // Constant folded version of the above, with fused shifts.\n if (Const > 1) {\n Res = _Orlshl(OpSize, Res, Dest, Size + 1 - Const);\n }\n\n // Our new CF will be bit (Shift - 1) of the source.\n SetCFDirect(Dest, Const - 1, true);\n\n // Since shift != 0 we can inject the CF\n Res = _Orlshl(OpSize, Res, CF, Size - Const);\n\n // OF is the top two MSBs XOR'd together\n // Only when Shift == 1, it is undefined otherwise\n if (Const == 1) {\n auto Xor = _XorShift(OpSize, Res, Res, ShiftType::LSR, 1);\n SetRFLAG(Xor, Size - 2, true);\n }\n\n StoreResultGPR(Op, Res);\n return;\n }\n\n Ref SrcMasked = _And(OpSize, Src, _InlineConstant(Mask));\n Calculate_ShiftVariable(\n Op, SrcMasked,\n [this, Op, Size, OpSize]() {\n // Rematerialize loads to avoid crossblock liveness\n Ref Src = LoadSourceGPR(Op, Op->Src[1], Op->Flags, {.AllowUpperGarbage = true});\n Ref Dest = LoadSourceGPR(Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = true});\n\n // Res = Src >> Shift\n Ref Res = _Lshr(OpSize, Dest, Src);\n auto CF = GetRFLAG(FEXCore::X86State::RFLAG_CF_RAW_LOC);\n\n // Res |= (Dest << (Size - Shift + 1));\n // Expressed as Res | ((Src << (Size - Shift)) << 1) to get correct\n // behaviour for Shift without clobbering NZCV. Then observe that modulo\n // Size, Size - Shift = -Shift so we can use a simple Neg.\n //\n // The masking of Lshl means we don't need mask the source, since:\n //\n // -(x & Mask) & Mask = (-x) & Mask\n Ref NegSrc = _Neg(OpSize, Src);\n Res = _Orlshl(OpSize, Res, _Lshl(OpSize, Dest, NegSrc), 1);\n\n // Our new CF will be bit (Shift - 1) of the source. this is hoisted up to\n // avoid the need to copy the source. Again, the Lshr absorbs the masking.\n auto NewCF = _Lshr(OpSize, Dest, Sub(OpSize, Src, 1));\n SetCFDirect(NewCF, 0, true);\n\n // Since shift != 0 we can inject the CF\n Res = _Or(OpSize, Res, _Lshl(OpSize, CF, NegSrc));\n\n // OF is the top two MSBs XOR'd together\n // Only when Shift == 1, it is undefined otherwise\n auto Xor = _XorShift(OpSize, Res, Res, ShiftType::LSR, 1);\n SetRFLAG(Xor, Size - 2, true);\n\n StoreResultGPR(Op, Res);\n },\n OpSizeFromSrc(Op) == OpSize::i32Bit ? std::make_optional(&OpDispatchBuilder::ZeroShiftResult) : std::nullopt);\n}\n\nvoid OpDispatchBuilder::RCRSmallerOp(OpcodeArgs) {\n CalculateDeferredFlags();\n\n const auto Size = GetSrcBitSize(Op);\n\n // x86 masks the shift by 0x3F or 0x1F depending on size of op\n auto Src = ARef(LoadSourceGPR(Op, Op->Src[1], Op->Flags, {.AllowUpperGarbage = true}));\n Src = Src.And(0x1F);\n\n // CF only changes if we actually shifted. OF undefined if we didn't shift.\n // The result is unchanged if we didn't shift. So branch over the whole thing.\n Calculate_ShiftVariable(Op, Src.Ref(), [this, Op, Size]() {\n // Rematerialized to avoid crossblock liveness\n auto Src = ARef(LoadSourceGPR(Op, Op->Src[1], Op->Flags, {.AllowUpperGarbage = true}));\n Src = Src.And(0x1F);\n\n auto CF = GetRFLAG(FEXCore::X86State::RFLAG_CF_RAW_LOC);\n\n Ref Dest = LoadSourceGPR(Op, Op->Dest, Op->Flags);\n Ref Tmp {};\n\n // Insert the incoming value across the temporary 64bit source\n // Make sure to insert at + 1 offsets\n // We need to cover 32bits plus the amount that could rotate in\n\n if (Size == 8) {\n // 8-bit optimal cascade\n // Cascade: 0\n // Data: -> [7:0]\n // CF: -> [8:8]\n // Cascade: 1\n // Data: -> [16:9]\n // CF: -> [17:17]\n // Cascade: 2\n // Data: -> [25:18]\n // CF: -> [26:26]\n // Cascade: 3\n // Data: -> [34:27]\n // CF: -> [35:35]\n // Cascade: 4\n // Data: -> [43:36]\n // CF: -> [44:44]\n\n // Insert CF, Destination already at [7:0]\n Tmp = _Bfi(OpSize::i64Bit, 1, 8, Dest, CF);\n\n // First Cascade, copies 9 bits from itself.\n Tmp = _Bfi(OpSize::i64Bit, 9, 9, Tmp, Tmp);\n\n // Second cascade, copies 18 bits from itself.\n Tmp = _Bfi(OpSize::i64Bit, 18, 18, Tmp, Tmp);\n\n // Final cascade, copies 9 bits again from itself.\n Tmp = _Bfi(OpSize::i64Bit, 9, 36, Tmp, Tmp);\n } else {\n // 16-bit optimal cascade\n // Cascade: 0\n // Data: -> [15:0]\n // CF: -> [16:16]\n // Cascade: 1\n // Data: -> [32:17]\n // CF: -> [33:33]\n // Cascade: 2\n // Data: -> [49:34]\n // CF: -> [50:50]\n\n // Insert CF, Destination already at [15:0]\n Tmp = _Bfi(OpSize::i64Bit, 1, 16, Dest, CF);\n\n // First Cascade, copies 17 bits from itself.\n Tmp = _Bfi(OpSize::i64Bit, 17, 17, Tmp, Tmp);\n\n // Final Cascade, copies 17 bits from itself again.\n Tmp = _Bfi(OpSize::i64Bit, 17, 34, Tmp, Tmp);\n }\n\n // Entire bitfield has been setup. Just extract the 8 or 16bits we need.\n // 64-bit shift used because we want to rotate in our cascaded upper bits\n // rather than zeroes.\n Ref Res = _Lshr(OpSize::i64Bit, Tmp, Src.Ref());\n\n StoreResultGPR(Op, Res);\n\n // Our new CF will be bit (Shift - 1) of the source. 32-bit Lshr masks the\n // same as x86, but if we constant fold we must mask ourselves.\n if (Src.IsConstant) {\n SetCFDirect(Tmp, (Src.C & 0x1f) - 1, true);\n } else {\n auto NewCF = _Lshr(OpSize::i32Bit, Tmp, Sub(OpSize::i32Bit, Src.Ref(), 1));\n SetCFDirect(NewCF, 0, true);\n }\n\n // OF is the top two MSBs XOR'd together\n // Only when Shift == 1, it is undefined otherwise\n if (!Src.IsConstant || Src.C == 1) {\n auto NewOF = _XorShift(OpSize::i32Bit, Res, Res, ShiftType::LSR, 1);\n SetRFLAG(NewOF, Size - 2, true);\n }\n });\n}\n\nvoid OpDispatchBuilder::RCLOp1Bit(OpcodeArgs) {\n // Calculate flags early.\n CalculateDeferredFlags();\n\n Ref Dest = LoadSourceGPR(Op, Op->Dest, Op->Flags);\n const auto Size = GetSrcBitSize(Op);\n const auto OpSize = Size == 64 ? OpSize::i64Bit : OpSize::i32Bit;\n auto CF = GetRFLAG(FEXCore::X86State::RFLAG_CF_RAW_LOC);\n\n // Rotate left and insert CF in to lowest bit\n // TODO: Use `adc Res, xzr, Dest, lsl 1` to save an instruction\n Ref Res = _Orlshl(OpSize, CF, Dest, 1);\n\n // Our new CF will be the top bit of the source\n SetCFDirect(Dest, Size - 1, true);\n\n // OF is the top two MSBs XOR'd together\n // Top two MSBs is CF and top bit of result\n SetRFLAG(_Xor(OpSize, Res, Dest), Size - 1, true);\n\n StoreResultGPR(Op, Res);\n}\n\nvoid OpDispatchBuilder::RCLOp(OpcodeArgs) {\n const auto Size = GetSrcBitSize(Op);\n\n if (Size == 8 || Size == 16) {\n RCLSmallerOp(Op);\n return;\n }\n\n const auto Mask = (Size == 64) ? 0x3F : 0x1F;\n\n // Calculate flags early.\n CalculateDeferredFlags();\n\n Ref Src = LoadSourceGPR(Op, Op->Src[1], Op->Flags, {.AllowUpperGarbage = true});\n const auto OpSize = OpSizeFromSrc(Op);\n\n uint64_t Const;\n if (IsValueConstant(WrapNode(Src), &Const)) {\n Const &= Mask;\n if (!Const) {\n ZeroShiftResult(Op);\n return;\n }\n\n // Res = Src << Shift\n Ref Dest = LoadSourceGPR(Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = true});\n Ref Res = _Lshl(OpSize, Dest, Src);\n auto CF = GetRFLAG(FEXCore::X86State::RFLAG_CF_RAW_LOC);\n\n // Res |= (Src << (Size - Shift + 1));\n if (Const > 1) {\n Res = _Orlshr(OpSize, Res, Dest, Size + 1 - Const);\n }\n\n // Our new CF will be bit (Shift - 1) of the source\n SetCFDirect(Dest, Size - Const, true);\n\n // Since Shift != 0 we can inject the CF\n Res = _Orlshl(OpSize, Res, CF, Const - 1);\n\n // OF is the top two MSBs XOR'd together\n // Only when Shift == 1, it is undefined otherwise\n if (Const == 1) {\n auto NewOF = _Xor(OpSize, Res, Dest);\n SetRFLAG(NewOF, Size - 1, true);\n }\n\n StoreResultGPR(Op, Res);\n return;\n }\n\n Ref SrcMasked = _And(OpSize, Src, _InlineConstant(Mask));\n Calculate_ShiftVariable(\n Op, SrcMasked,\n [this, Op, Size, OpSize]() {\n // Rematerialized to avoid crossblock liveness\n Ref Src = LoadSourceGPR(Op, Op->Src[1], Op->Flags, {.AllowUpperGarbage = true});\n\n // Res = Src << Shift\n Ref Dest = LoadSourceGPR(Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = true});\n Ref Res = _Lshl(OpSize, Dest, Src);\n auto CF = GetRFLAG(FEXCore::X86State::RFLAG_CF_RAW_LOC);\n\n // Res |= (Dest >> (Size - Shift + 1)), expressed as\n // Res | ((Dest >> (-Shift)) >> 1), since Size - Shift = -Shift mod\n // Size. The shift aborbs the masking.\n auto NegSrc = _Neg(OpSize, Src);\n Res = _Orlshr(OpSize, Res, _Lshr(OpSize, Dest, NegSrc), 1);\n\n // Our new CF will be bit (Shift - 1) of the source\n auto NewCF = _Lshr(OpSize, Dest, NegSrc);\n SetCFDirect(NewCF, 0, true);\n\n // Since Shift != 0 we can inject the CF. Shift absorbs the masking.\n Ref CFShl = Sub(OpSize, Src, 1);\n auto TmpCF = _Lshl(OpSize, CF, CFShl);\n Res = _Or(OpSize, Res, TmpCF);\n\n // OF is the top two MSBs XOR'd together\n // Only when Shift == 1, it is undefined otherwise\n //\n // Note that NewCF has garbage in the upper bits, but we ignore them here\n // and mask as part of the set after.\n auto NewOF = _XorShift(OpSize, Res, NewCF, ShiftType::LSL, Size - 1);\n SetRFLAG(NewOF, Size - 1, true);\n\n StoreResultGPR(Op, Res);\n },\n OpSizeFromSrc(Op) == OpSize::i32Bit ? std::make_optional(&OpDispatchBuilder::ZeroShiftResult) : std::nullopt);\n}\n\nvoid OpDispatchBuilder::RCLSmallerOp(OpcodeArgs) {\n CalculateDeferredFlags();\n\n const auto Size = GetSrcBitSize(Op);\n\n // x86 masks the shift by 0x3F or 0x1F depending on size of op\n auto Src = ARef(LoadSourceGPR(Op, Op->Src[1], Op->Flags, {.AllowUpperGarbage = true}));\n Src = Src.And(0x1F);\n\n // CF only changes if we actually shifted. OF undefined if we didn't shift.\n // The result is unchanged if we didn't shift. So branch over the whole thing.\n Calculate_ShiftVariable(Op, Src.Ref(), [this, Op, Size]() {\n // Rematerialized to avoid crossblock liveness\n auto Src = ARef(LoadSourceGPR(Op, Op->Src[1], Op->Flags, {.AllowUpperGarbage = true}));\n Src = Src.And(0x1F);\n Ref Dest = LoadSourceGPR(Op, Op->Dest, Op->Flags);\n\n auto CF = GetRFLAG(FEXCore::X86State::RFLAG_CF_RAW_LOC);\n\n Ref Tmp = Constant(0);\n\n for (size_t i = 0; i < (32 + Size + 1); i += (Size + 1)) {\n // Insert incoming value\n Tmp = _Bfi(OpSize::i64Bit, Size, 63 - i - Size, Tmp, Dest);\n\n // Insert CF\n Tmp = _Bfi(OpSize::i64Bit, 1, 63 - i, Tmp, CF);\n }\n\n // Insert incoming value\n Tmp = _Bfi(OpSize::i64Bit, Size, 0, Tmp, Dest);\n\n // The data is now set up like this\n // [Data][CF]:[Data][CF]:[Data][CF]:[Data][CF]\n // Shift 1 more bit that expected to get our result\n // Shifting to the right will now behave like a rotate to the left\n // Which we emulate with a _Ror\n Ref Res = _Ror(OpSize::i64Bit, Tmp, Src.Neg().Ref());\n\n StoreResultGPR(Op, Res);\n\n // Our new CF is now at the bit position that we are shifting\n // Either 0 if CF hasn't changed (CF is living in bit 0)\n // or higher\n auto NewCF = _Ror(OpSize::i64Bit, Tmp, Src.Presub(63).Ref());\n SetCFDirect(NewCF, 0, true);\n\n // OF is the XOR of the NewCF and the MSB of the result\n // Only defined for 1-bit rotates.\n if (!Src.IsConstant || Src.C == 1) {\n auto NewOF = _XorShift(OpSize::i64Bit, NewCF, Res, ShiftType::LSR, Size - 1);\n SetRFLAG(NewOF, 0, true);\n }\n });\n}\n\nvoid OpDispatchBuilder::BTOp(OpcodeArgs, uint32_t SrcIndex, BTAction Action) {\n Ref Value;\n ArithRef Src;\n bool IsNonconstant = Op->Src[SrcIndex].IsGPR();\n\n const uint32_t Size = GetDstBitSize(Op);\n const uint32_t Mask = Size - 1;\n\n if (IsNonconstant) {\n // Because we mask explicitly with And/Bfe/Sbfe after, we can allow garbage here.\n Src = ARef(LoadSourceGPR(Op, Op->Src[SrcIndex], Op->Flags, {.AllowUpperGarbage = true}));\n } else {\n // Can only be an immediate\n // Masked by operand size\n Src = ARef(Op->Src[SrcIndex].Literal() & Mask);\n }\n\n if (Op->Dest.IsGPR()) {\n // When the destination is a GPR, we don't care about garbage in the upper bits.\n // Load the full register.\n auto Dest = LoadSourceGPR_WithOpSize(Op, Op->Dest, GetGPROpSize(), Op->Flags);\n Value = Dest;\n\n // Get the bit selection from the src. We need to mask for 8/16-bit, but\n // rely on the implicit masking of Lshr for native sizes.\n unsigned LshrSize = std::max(IR::OpSizeToSize(OpSize::i32Bit), Size / 8);\n auto BitSelect = (Size == (LshrSize * 8)) ? Src : Src.And(Mask);\n auto LshrOpSize = IR::SizeToOpSize(LshrSize);\n\n // OF/SF/AF/PF undefined. ZF must be preserved. We choose to preserve OF/SF\n // too since we just use an rmif to insert into CF directly. We could\n // optimize perhaps.\n //\n // Set CF before the action to save a move, except for complements where we\n // can reuse the invert.\n if (Action != BTAction::BTComplement) {\n if (IsNonconstant) {\n Value = _Lshr(IR::SizeToOpSize(LshrSize), Value, BitSelect.Ref());\n }\n\n SetRFLAG(Value, X86State::RFLAG_CF_RAW_LOC, Src.IsConstant ? Src.C : 0, true);\n CFInverted = false;\n }\n\n switch (Action) {\n case BTAction::BTNone: {\n /* Nothing to do */\n break;\n }\n\n case BTAction::BTClear: {\n Dest = _Andn(LshrOpSize, Dest, BitSelect.MaskBit(LshrOpSize).Ref());\n StoreResultGPR(Op, Dest);\n break;\n }\n\n case BTAction::BTSet: {\n Dest = _Or(LshrOpSize, Dest, BitSelect.MaskBit(LshrOpSize).Ref());\n StoreResultGPR(Op, Dest);\n break;\n }\n\n case BTAction::BTComplement: {\n Dest = _Xor(LshrOpSize, Dest, BitSelect.MaskBit(LshrOpSize).Ref());\n\n if (IsNonconstant) {\n Value = _Lshr(LshrOpSize, Dest, BitSelect.Ref());\n } else {\n Value = Dest;\n }\n\n SetRFLAG(Value, X86State::RFLAG_CF_RAW_LOC, Src.IsConstant ? Src.C : 0, true);\n CFInverted = true;\n\n StoreResultGPR(Op, Dest);\n break;\n }\n }\n } else {\n // Load the address to the memory location\n Ref Dest = MakeSegmentAddress(Op, Op->Dest);\n // Get the bit selection from the src\n auto BitSelect = Src.Bfe(0, 3);\n\n // Address is provided as bits we want BYTE offsets\n // Extract Signed offset\n Src = Src.Sbfe(3, Size - 3);\n\n // Get the address offset by shifting out the size of the op (To shift out the bit selection)\n // Then use that to index in to the memory location by size of op\n AddressMode Address = {.Base = Dest, .Index = Src.Ref(), .AddrSize = OpSize::i64Bit};\n\n switch (Action) {\n case BTAction::BTNone: {\n Value = _LoadMemGPRAutoTSO(OpSize::i8Bit, Address, OpSize::i8Bit);\n break;\n }\n\n case BTAction::BTClear: {\n Ref BitMask = BitSelect.MaskBit(OpSize::i64Bit).Ref();\n\n if (DestIsLockedMem(Op)) {\n HandledLock = true;\n Value = _AtomicFetchCLR(OpSize::i8Bit, BitMask, LoadEffectiveAddress(this, Address, GetGPROpSize(), true));\n } else {\n Value = _LoadMemGPRAutoTSO(OpSize::i8Bit, Address, OpSize::i8Bit);\n\n auto Modified = _Andn(OpSize::i64Bit, Value, BitMask);\n _StoreMemGPRAutoTSO(OpSize::i8Bit, Address, Modified, OpSize::i8Bit);\n }\n break;\n }\n\n case BTAction::BTSet: {\n Ref BitMask = BitSelect.MaskBit(OpSize::i64Bit).Ref();\n\n if (DestIsLockedMem(Op)) {\n HandledLock = true;\n Value = _AtomicFetchOr(OpSize::i8Bit, BitMask, LoadEffectiveAddress(this, Address, GetGPROpSize(), true));\n } else {\n Value = _LoadMemGPRAutoTSO(OpSize::i8Bit, Address, OpSize::i8Bit);\n\n auto Modified = _Or(OpSize::i64Bit, Value, BitMask);\n _StoreMemGPRAutoTSO(OpSize::i8Bit, Address, Modified, OpSize::i8Bit);\n }\n break;\n }\n\n case BTAction::BTComplement: {\n Ref BitMask = BitSelect.MaskBit(OpSize::i64Bit).Ref();\n\n if (DestIsLockedMem(Op)) {\n HandledLock = true;\n Value = _AtomicFetchXor(OpSize::i8Bit, BitMask, LoadEffectiveAddress(this, Address, GetGPROpSize(), true));\n } else {\n Value = _LoadMemGPRAutoTSO(OpSize::i8Bit, Address, OpSize::i8Bit);\n\n auto Modified = _Xor(OpSize::i64Bit, Value, BitMask);\n _StoreMemGPRAutoTSO(OpSize::i8Bit, Address, Modified, OpSize::i8Bit);\n }\n break;\n }\n }\n\n // Now shift in to the correct bit location\n if (!BitSelect.IsDefinitelyZero()) {\n Value = _Lshr(std::max(OpSize::i32Bit, GetOpSize(Value)), Value, BitSelect.Ref());\n }\n\n // OF/SF/ZF/AF/PF undefined.\n SetCFDirect(Value, 0, true);\n }\n}\n\nvoid OpDispatchBuilder::IMUL1SrcOp(OpcodeArgs) {\n /* We're just going to sign-extend the non-garbage anyway.. */\n Ref Src1 = LoadSourceGPR(Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = true});\n Ref Src2 = LoadSourceGPR(Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true});\n\n const auto Size = OpSizeFromSrc(Op);\n const auto SizeBits = IR::OpSizeAsBits(Size);\n\n Ref Dest {};\n Ref ResultHigh {};\n switch (Size) {\n case OpSize::i8Bit:\n case OpSize::i16Bit: {\n Src1 = _Sbfe(OpSize::i64Bit, SizeBits, 0, Src1);\n Src2 = _Sbfe(OpSize::i64Bit, SizeBits, 0, Src2);\n Dest = _Mul(OpSize::i64Bit, Src1, Src2);\n ResultHigh = _Sbfe(OpSize::i64Bit, SizeBits, SizeBits, Dest);\n break;\n }\n case OpSize::i32Bit: {\n ResultHigh = _SMull(Src1, Src2);\n ResultHigh = _Sbfe(OpSize::i64Bit, SizeBits, SizeBits, ResultHigh);\n // Flipped order to save a move\n Dest = _Mul(OpSize::i32Bit, Src1, Src2);\n break;\n }\n case OpSize::i64Bit: {\n ResultHigh = _MulH(OpSize::i64Bit, Src1, Src2);\n // Flipped order to save a move\n Dest = _Mul(OpSize::i64Bit, Src1, Src2);\n break;\n }\n default: FEX_UNREACHABLE;\n }\n\n StoreResultGPR(Op, Dest);\n CalculateFlags_MUL(Size, Dest, ResultHigh);\n}\n\nvoid OpDispatchBuilder::IMUL2SrcOp(OpcodeArgs) {\n Ref Src1 = LoadSourceGPR(Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true});\n Ref Src2 = LoadSourceGPR(Op, Op->Src[1], Op->Flags, {.AllowUpperGarbage = true});\n\n const auto Size = OpSizeFromSrc(Op);\n const auto SizeBits = IR::OpSizeAsBits(Size);\n\n Ref Dest {};\n Ref ResultHigh {};\n\n switch (Size) {\n case OpSize::i8Bit:\n case OpSize::i16Bit: {\n Src1 = _Sbfe(OpSize::i64Bit, SizeBits, 0, Src1);\n Src2 = ARef(Src2).Sbfe(0, SizeBits).Ref();\n Dest = _Mul(OpSize::i64Bit, Src1, Src2);\n ResultHigh = _Sbfe(OpSize::i64Bit, SizeBits, SizeBits, Dest);\n break;\n }\n case OpSize::i32Bit: {\n ResultHigh = _SMull(Src1, Src2);\n ResultHigh = _Sbfe(OpSize::i64Bit, SizeBits, SizeBits, ResultHigh);\n // Flipped order to save a move\n Dest = _Mul(OpSize::i32Bit, Src1, Src2);\n break;\n }\n case OpSize::i64Bit: {\n ResultHigh = _MulH(OpSize::i64Bit, Src1, Src2);\n // Flipped order to save a move\n Dest = _Mul(OpSize::i64Bit, Src1, Src2);\n break;\n }\n default: FEX_UNREACHABLE;\n }\n\n StoreResultGPR(Op, Dest);\n CalculateFlags_MUL(Size, Dest, ResultHigh);\n}\n\nvoid OpDispatchBuilder::IMULOp(OpcodeArgs) {\n const auto Size = OpSizeFromSrc(Op);\n const auto SizeBits = IR::OpSizeAsBits(Size);\n\n Ref Src1 = LoadSourceGPR(Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = true});\n Ref Src2 = LoadGPRRegister(X86State::REG_RAX);\n\n if (Size != OpSize::i64Bit) {\n Src1 = _Sbfe(OpSize::i64Bit, SizeBits, 0, Src1);\n Src2 = _Sbfe(OpSize::i64Bit, SizeBits, 0, Src2);\n }\n\n // 64-bit special cased to save a move\n Ref Result {};\n if (Size < OpSize::i64Bit) {\n Result = _Mul(OpSize::i64Bit, Src1, Src2);\n }\n Ref ResultHigh {};\n if (Size == OpSize::i8Bit) {\n // Result is stored in AX\n StoreGPRRegister(X86State::REG_RAX, Result, OpSize::i16Bit);\n ResultHigh = _Sbfe(OpSize::i64Bit, 8, 8, Result);\n } else if (Size == OpSize::i16Bit) {\n // 16bits stored in AX\n // 16bits stored in DX\n StoreGPRRegister(X86State::REG_RAX, Result, Size);\n ResultHigh = _Sbfe(OpSize::i64Bit, 16, 16, Result);\n StoreGPRRegister(X86State::REG_RDX, ResultHigh, Size);\n } else if (Size == OpSize::i32Bit) {\n // 32bits stored in EAX\n // 32bits stored in EDX\n // Make sure they get Zext correctly\n auto LocalResult = _Bfe(OpSize::i64Bit, 32, 0, Result);\n auto LocalResultHigh = _Bfe(OpSize::i64Bit, 32, 32, Result);\n ResultHigh = _Sbfe(OpSize::i64Bit, 32, 32, Result);\n Result = _Sbfe(OpSize::i64Bit, 32, 0, Result);\n StoreGPRRegister(X86State::REG_RAX, LocalResult);\n StoreGPRRegister(X86State::REG_RDX, LocalResultHigh);\n } else if (Size == OpSize::i64Bit) {\n if (!Is64BitMode) {\n LogMan::Msg::EFmt(\"Doesn't exist in 32bit mode\");\n DecodeFailure = true;\n return;\n }\n // 64bits stored in RAX\n // 64bits stored in RDX\n ResultHigh = _MulH(OpSize::i64Bit, Src1, Src2);\n Result = _Mul(OpSize::i64Bit, Src1, Src2);\n StoreGPRRegister(X86State::REG_RAX, Result);\n StoreGPRRegister(X86State::REG_RDX, ResultHigh);\n }\n\n CalculateFlags_MUL(Size, Result, ResultHigh);\n}\n\nvoid OpDispatchBuilder::MULOp(OpcodeArgs) {\n const auto Size = OpSizeFromSrc(Op);\n const auto SizeBits = IR::OpSizeAsBits(Size);\n\n Ref Src1 = LoadSourceGPR(Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = true});\n Ref Src2 = LoadGPRRegister(X86State::REG_RAX);\n Ref Result {};\n\n if (Size != OpSize::i64Bit) {\n Src1 = _Bfe(OpSize::i64Bit, SizeBits, 0, Src1);\n Src2 = _Bfe(OpSize::i64Bit, SizeBits, 0, Src2);\n Result = _UMul(OpSize::i64Bit, Src1, Src2);\n }\n Ref ResultHigh {};\n\n if (Size == OpSize::i8Bit) {\n // Result is stored in AX\n StoreGPRRegister(X86State::REG_RAX, Result, OpSize::i16Bit);\n ResultHigh = _Bfe(OpSize::i64Bit, 8, 8, Result);\n } else if (Size == OpSize::i16Bit) {\n // 16bits stored in AX\n // 16bits stored in DX\n StoreGPRRegister(X86State::REG_RAX, Result, Size);\n ResultHigh = _Bfe(OpSize::i64Bit, 16, 16, Result);\n StoreGPRRegister(X86State::REG_RDX, ResultHigh, Size);\n } else if (Size == OpSize::i32Bit) {\n // 32bits stored in EAX\n // 32bits stored in EDX\n Ref ResultLow = _Bfe(OpSize::i64Bit, 32, 0, Result);\n ResultHigh = _Bfe(OpSize::i64Bit, 32, 32, Result);\n StoreGPRRegister(X86State::REG_RAX, ResultLow);\n StoreGPRRegister(X86State::REG_RDX, ResultHigh);\n } else if (Size == OpSize::i64Bit) {\n if (!Is64BitMode) {\n LogMan::Msg::EFmt(\"Doesn't exist in 32bit mode\");\n DecodeFailure = true;\n return;\n }\n // 64bits stored in RAX\n // 64bits stored in RDX\n //\n // Calculate high first to allow better RA.\n ResultHigh = _UMulH(OpSize::i64Bit, Src1, Src2);\n Result = _UMul(OpSize::i64Bit, Src1, Src2);\n StoreGPRRegister(X86State::REG_RAX, Result);\n StoreGPRRegister(X86State::REG_RDX, ResultHigh);\n }\n\n CalculateFlags_UMUL(ResultHigh);\n}\n\nvoid OpDispatchBuilder::NOTOp(OpcodeArgs) {\n const auto Size = OpSizeFromSrc(Op);\n const auto SizeBits = IR::OpSizeAsBits(Size);\n LOGMAN_THROW_A_FMT(Size >= IR::OpSize::i8Bit && Size <= IR::OpSize::i64Bit, \"Invalid size\");\n\n Ref MaskConst {};\n if (Size == OpSize::i64Bit) {\n MaskConst = Constant(~0ULL);\n } else {\n MaskConst = Constant((1ULL << SizeBits) - 1);\n }\n\n if (DestIsLockedMem(Op)) {\n HandledLock = true;\n Ref DestMem = MakeSegmentAddress(Op, Op->Dest);\n // Result unused\n _AtomicFetchXor(Size, MaskConst, DestMem);\n } else if (!Op->Dest.IsGPR()) {\n // GPR version plays fast and loose with sizes, be safe for memory tho.\n Ref Src = LoadSourceGPR(Op, Op->Dest, Op->Flags);\n Src = _Xor(OpSize::i64Bit, Src, MaskConst);\n StoreResultGPR(Op, Src);\n } else {\n // Specially handle high bits so we can invert in place with the correct\n // mask and a larger type.\n auto Dest = Op->Dest;\n if (Dest.Data.GPR.HighBits) {\n LOGMAN_THROW_A_FMT(Size == OpSize::i8Bit, \"Only 8-bit GPRs get high bits\");\n MaskConst = Constant(0xFF00);\n Dest.Data.GPR.HighBits = false;\n }\n\n // Always load full size, we explicitly want the upper bits to get the\n // insert behaviour for free/implicitly.\n const auto GPRSize = GetGPROpSize();\n Ref Src = LoadSourceGPR_WithOpSize(Op, Dest, GPRSize, Op->Flags);\n\n // For 8/16-bit, use 64-bit invert so we invert in place, while getting\n // insert behaviour. For 32-bit, use 32-bit invert to zero the upper bits.\n const auto EffectiveSize = Size == OpSize::i32Bit ? OpSize::i32Bit : GPRSize;\n\n // If we're inverting the whole thing, use Not instead of Xor to save a constant.\n if (Size >= OpSize::i32Bit) {\n Src = _Not(EffectiveSize, Src);\n } else {\n Src = _Xor(EffectiveSize, Src, MaskConst);\n }\n\n // Always store 64-bit, the Not/Xor correctly handle the upper bits and this\n // way we can delete the store.\n StoreResultGPR_WithOpSize(Op, Dest, Src, GPRSize);\n }\n}\n\nvoid OpDispatchBuilder::XADDOp(OpcodeArgs) {\n Ref Dest = LoadSourceGPR(Op, Op->Dest, Op->Flags, {.LoadData = false});\n Ref Src = LoadSourceGPR(Op, Op->Src[0], Op->Flags);\n Ref Result;\n\n if (Op->Dest.IsGPR()) {\n // If this is a GPR then we can just do an Add\n Result = CalculateFlags_ADD(OpSizeFromSrc(Op), Dest, Src);\n\n // Previous value in dest gets stored in src\n StoreResultGPR(Op, Op->Src[0], Dest);\n\n // Calculated value gets stored in dst (order is important if dst is same as src)\n StoreResultGPR(Op, Result);\n } else {\n HandledLock = Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_LOCK;\n Dest = AppendSegmentOffset(Dest, Op->Flags);\n auto Before = _AtomicFetchAdd(OpSizeFromSrc(Op), Src, Dest);\n CalculateFlags_ADD(OpSizeFromSrc(Op), Before, Src);\n StoreResultGPR(Op, Op->Src[0], Before);\n }\n}\n\nvoid OpDispatchBuilder::PopcountOp(OpcodeArgs) {\n Ref Src = LoadSourceGPR(Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = CTX->HostFeatures.SupportsCSSC || GetSrcSize(Op) >= 4});\n Src = _Popcount(OpSizeFromSrc(Op), Src);\n StoreResultGPR(Op, Src);\n\n // We need to set ZF while clearing the rest of NZCV. The result of a popcount\n // is in the range [0, 63]. In particular, it is always positive. So a\n // combined NZ test will correctly zero SF/CF/OF while setting ZF.\n SetNZ_ZeroCV(OpSize::i32Bit, Src);\n ZeroPF_AF();\n}\n\nRef OpDispatchBuilder::CalculateAFForDecimal(Ref A) {\n auto Nibble = _And(OpSize::i64Bit, A, Constant(0xF));\n auto Greater = Select01(OpSize::i64Bit, CondClass::UGT, Nibble, Constant(9));\n\n return _Or(OpSize::i64Bit, LoadAF(), Greater);\n}\n\nvoid OpDispatchBuilder::DAAOp(OpcodeArgs) {\n CalculateDeferredFlags();\n auto AL = LoadGPRRegister(X86State::REG_RAX, OpSize::i8Bit);\n auto CFInv = GetRFLAG(FEXCore::X86State::RFLAG_CF_RAW_LOC, true);\n auto AF = CalculateAFForDecimal(AL);\n\n // CF |= (AL > 0x99);\n CFInv = _And(OpSize::i64Bit, CFInv, Select01(OpSize::i64Bit, CondClass::ULE, AL, Constant(0x99)));\n\n // AL = AF ? (AL + 0x6) : AL;\n AL = _Select(OpSize::i64Bit, OpSize::i64Bit, CondClass::NEQ, AF, Constant(0), Add(OpSize::i64Bit, AL, 0x6), AL);\n\n // AL = CF ? (AL + 0x60) : AL;\n AL = _Select(OpSize::i64Bit, OpSize::i64Bit, CondClass::EQ, CFInv, Constant(0), Add(OpSize::i64Bit, AL, 0x60), AL);\n\n // SF, ZF, PF set according to result. CF set per above. OF undefined.\n StoreGPRRegister(X86State::REG_RAX, AL, OpSize::i8Bit);\n SetNZ_ZeroCV(OpSize::i8Bit, AL);\n SetCFInverted(CFInv);\n CalculatePF(AL);\n SetAFAndFixup(AF);\n}\n\nvoid OpDispatchBuilder::DASOp(OpcodeArgs) {\n CalculateDeferredFlags();\n auto AL = LoadGPRRegister(X86State::REG_RAX, OpSize::i8Bit);\n auto CF = GetRFLAG(FEXCore::X86State::RFLAG_CF_RAW_LOC);\n auto AF = CalculateAFForDecimal(AL);\n\n // CF |= (AL > 0x99);\n CF = _Or(OpSize::i64Bit, CF, Select01(OpSize::i64Bit, CondClass::UGT, AL, Constant(0x99)));\n\n // NewCF = CF | (AF && (Borrow from AL - 6))\n auto NewCF = _Or(OpSize::i32Bit, CF, _Select(OpSize::i64Bit, OpSize::i64Bit, CondClass::ULT, AL, Constant(6), AF, CF));\n\n // AL = AF ? (AL - 0x6) : AL;\n AL = _Select(OpSize::i64Bit, OpSize::i64Bit, CondClass::NEQ, AF, Constant(0), Sub(OpSize::i64Bit, AL, 0x6), AL);\n\n // AL = CF ? (AL - 0x60) : AL;\n AL = _Select(OpSize::i64Bit, OpSize::i64Bit, CondClass::NEQ, CF, Constant(0), Sub(OpSize::i64Bit, AL, 0x60), AL);\n\n // SF, ZF, PF set according to result. CF set per above. OF undefined.\n StoreGPRRegister(X86State::REG_RAX, AL, OpSize::i8Bit);\n SetNZ_ZeroCV(OpSize::i8Bit, AL);\n SetCFDirect(NewCF);\n CalculatePF(AL);\n SetAFAndFixup(AF);\n}\n\nvoid OpDispatchBuilder::AAAOp(OpcodeArgs) {\n auto A = LoadGPRRegister(X86State::REG_RAX);\n auto AF = CalculateAFForDecimal(A);\n\n // CF = AF, OF/SF/ZF/PF undefined\n SetCFDirect_InvalidateNZV(AF);\n SetAFAndFixup(AF);\n CalculateDeferredFlags();\n\n // AX = CF ? (AX + 0x106) : 0\n A = NZCVSelect(OpSize::i32Bit, CondClass::UGE /* CF = 1 */, Add(OpSize::i32Bit, A, 0x106), A);\n\n // AL = AL & 0x0F\n A = _And(OpSize::i32Bit, A, Constant(0xFF0F));\n StoreGPRRegister(X86State::REG_RAX, A, OpSize::i16Bit);\n}\n\nvoid OpDispatchBuilder::AASOp(OpcodeArgs) {\n auto A = LoadGPRRegister(X86State::REG_RAX);\n auto AF = CalculateAFForDecimal(A);\n\n // CF = AF, OF/SF/ZF/PF undefined\n SetCFDirect_InvalidateNZV(AF);\n SetAFAndFixup(AF);\n CalculateDeferredFlags();\n\n // AX = CF ? (AX - 0x106) : 0\n A = NZCVSelect(OpSize::i32Bit, CondClass::UGE /* CF = 1 */, Sub(OpSize::i32Bit, A, 0x106), A);\n\n // AL = AL & 0x0F\n A = _And(OpSize::i32Bit, A, Constant(0xFF0F));\n StoreGPRRegister(X86State::REG_RAX, A, OpSize::i16Bit);\n}\n\nvoid OpDispatchBuilder::AAMOp(OpcodeArgs) {\n auto AL = LoadGPRRegister(X86State::REG_RAX, OpSize::i8Bit);\n auto Imm8 = Constant(Op->Src[0].Literal() & 0xFF);\n Ref Quotient = _AllocateGPR(true);\n Ref Remainder = _AllocateGPR(true);\n _UDiv(OpSize::i64Bit, AL, Invalid(), Imm8, Quotient, Remainder);\n auto Res = _AddShift(OpSize::i64Bit, Remainder, Quotient, ShiftType::LSL, 8);\n StoreGPRRegister(X86State::REG_RAX, Res, OpSize::i16Bit);\n\n SetNZ_ZeroCV(OpSize::i8Bit, Res);\n CalculatePF(Res);\n InvalidateAF();\n}\n\nvoid OpDispatchBuilder::AADOp(OpcodeArgs) {\n auto A = LoadGPRRegister(X86State::REG_RAX);\n auto AH = _Lshr(OpSize::i32Bit, A, Constant(8));\n auto Imm8 = Constant(Op->Src[0].Literal() & 0xFF);\n auto NewAL = Add(OpSize::i64Bit, A, _Mul(OpSize::i64Bit, AH, Imm8));\n auto Result = _And(OpSize::i64Bit, NewAL, Constant(0xFF));\n StoreGPRRegister(X86State::REG_RAX, Result, OpSize::i16Bit);\n\n SetNZ_ZeroCV(OpSize::i8Bit, Result);\n CalculatePF(Result);\n InvalidateAF();\n}\n\nvoid OpDispatchBuilder::XLATOp(OpcodeArgs) {\n Ref Src = MakeSegmentAddress(X86State::REG_RBX, Op->Flags, X86Tables::DecodeFlags::FLAG_DS_PREFIX);\n Ref Offset = LoadGPRRegister(X86State::REG_RAX, OpSize::i8Bit);\n\n AddressMode A = {.Base = Src, .Index = Offset, .AddrSize = OpSize::i64Bit};\n auto Res = _LoadMemGPRAutoTSO(OpSize::i8Bit, A, OpSize::i8Bit);\n\n StoreGPRRegister(X86State::REG_RAX, Res, OpSize::i8Bit);\n}\n\nvoid OpDispatchBuilder::ReadSegmentReg(OpcodeArgs, OpDispatchBuilder::Segment Seg) {\n // 64-bit only\n // Doesn't hit the segment register optimization\n const auto Size = OpSizeFromSrc(Op);\n Ref Src {};\n if (Seg == Segment::FS) {\n Src = _LoadContextGPR(Size, offsetof(FEXCore::Core::CPUState, fs_cached));\n } else {\n Src = _LoadContextGPR(Size, offsetof(FEXCore::Core::CPUState, gs_cached));\n }\n\n StoreResultGPR(Op, Src);\n}\n\nvoid OpDispatchBuilder::WriteSegmentReg(OpcodeArgs, OpDispatchBuilder::Segment Seg) {\n // Documentation claims that the 32-bit version of this instruction inserts in to the lower 32-bits of the segment\n // This is incorrect and it instead zero extends the 32-bit value to 64-bit\n const auto Size = OpSizeFromDst(Op);\n Ref Src = LoadSourceGPR(Op, Op->Src[0], Op->Flags);\n if (Seg == Segment::FS) {\n _StoreContextGPR(Size, Src, offsetof(FEXCore::Core::CPUState, fs_cached));\n } else {\n _StoreContextGPR(Size, Src, offsetof(FEXCore::Core::CPUState, gs_cached));\n }\n}\n\nvoid OpDispatchBuilder::EnterOp(OpcodeArgs) {\n const auto GPRSize = GetGPROpSize();\n const auto OperandSize = (Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_OPERAND_SIZE) ? OpSize::i16Bit : GPRSize;\n const uint64_t Value = Op->Src[0].Literal();\n\n const uint16_t AllocSpace = Value & 0xFFFF;\n const uint8_t Level = (Value >> 16) & 0x1F;\n\n const auto PushValue = [&](IR::OpSize Size, Ref Src) -> Ref {\n auto OldSP = LoadGPRRegister(X86State::REG_RSP);\n auto NewSP = _Push(GPRSize, Size, Src, OldSP);\n\n // Store the new stack pointer\n StoreGPRRegister(X86State::REG_RSP, NewSP);\n return NewSP;\n };\n\n auto OldBP = LoadGPRRegister(X86State::REG_RBP);\n auto NewSP = PushValue(OperandSize, OldBP);\n auto temp_RBP = NewSP;\n\n if (Level > 0) {\n for (uint8_t i = 1; i < Level; ++i) {\n auto MemLoc = Sub(GPRSize, OldBP, i * IR::OpSizeToSize(OperandSize));\n auto Mem = _LoadMemGPR(OperandSize, MemLoc, OperandSize);\n NewSP = PushValue(OperandSize, Mem);\n }\n NewSP = PushValue(OperandSize, temp_RBP);\n }\n NewSP = Sub(GPRSize, NewSP, AllocSpace);\n StoreGPRRegister(X86State::REG_RSP, NewSP);\n StoreGPRRegister(X86State::REG_RBP, temp_RBP);\n}\n\nvoid OpDispatchBuilder::SGDTOp(OpcodeArgs) {\n auto DestAddress = LoadSourceGPR(Op, Op->Dest, Op->Flags, {.LoadData = false});\n\n // Store an emulated value in the format of:\n // uint16_t Limit;\n // {uint32_t,uint64_t} Base;\n //\n // Limit is always 0\n // Base is always in kernel space at: 0xFFFFFFFFFFFE0000ULL\n //\n // Operand size prefix is ignored on this instruction, size purely depends on operating mode.\n uint64_t GDTAddress = 0xFFFFFFFFFFFE0000ULL;\n auto GDTStoreSize = OpSize::i64Bit;\n if (!Is64BitMode) {\n // Mask off upper bits if 32-bit result.\n GDTAddress &= ~0U;\n GDTStoreSize = OpSize::i32Bit;\n }\n\n _StoreMemGPRAutoTSO(OpSize::i16Bit, DestAddress, Constant(0));\n _StoreMemGPRAutoTSO(GDTStoreSize, AddressMode {.Base = DestAddress, .Offset = 2, .AddrSize = OpSize::i64Bit}, Constant(GDTAddress));\n}\n\nvoid OpDispatchBuilder::SIDTOp(OpcodeArgs) {\n auto DestAddress = LoadSourceGPR(Op, Op->Dest, Op->Flags, {.LoadData = false});\n\n // See SGDTOp, matches Linux in reported values\n uint64_t IDTAddress = 0xFFFFFE0000000000ULL;\n auto IDTStoreSize = OpSize::i64Bit;\n if (!Is64BitMode) {\n // Mask off upper bits if 32-bit result.\n IDTAddress &= ~0U;\n IDTStoreSize = OpSize::i32Bit;\n }\n\n _StoreMemGPRAutoTSO(OpSize::i16Bit, DestAddress, Constant(0xfff));\n _StoreMemGPRAutoTSO(IDTStoreSize, AddressMode {.Base = DestAddress, .Offset = 2, .AddrSize = OpSize::i64Bit}, Constant(IDTAddress));\n}\n\nvoid OpDispatchBuilder::SMSWOp(OpcodeArgs) {\n const bool IsMemDst = DestIsMem(Op);\n\n IR::OpSize DstSize {OpSize::iInvalid};\n Ref Const = Constant((1U << 31) | ///< PG - Paging\n (0U << 30) | ///< CD - Cache Disable\n (0U << 29) | ///< NW - Not Writethrough (Legacy, now ignored)\n ///< [28:19] - Reserved\n (1U << 18) | ///< AM - Alignment Mask\n ///< 17 - Reserved\n (1U << 16) | ///< WP - Write Protect\n ///< [15:6] - Reserved\n (1U << 5) | ///< NE - Numeric Error\n (1U << 4) | ///< ET - Extension Type (Legacy, now reserved and 1)\n (0U << 3) | ///< TS - Task Switched\n (0U << 2) | ///< EM - Emulation\n (1U << 1) | ///< MP - Monitor Coprocessor\n (1U << 0)); ///< PE - Protection Enabled\n const auto OpAddr = X86Tables::DecodeFlags::GetOpAddr(Op->Flags, 0);\n if (Is64BitMode) {\n DstSize = OpAddr == X86Tables::DecodeFlags::FLAG_OPERAND_SIZE_LAST ? OpSize::i16Bit :\n OpAddr == X86Tables::DecodeFlags::FLAG_WIDENING_SIZE_LAST ? OpSize::i64Bit :\n OpSize::i32Bit;\n\n if (!IsMemDst && DstSize == OpSize::i32Bit) {\n // Special-case version of `smsw ebx`. This instruction does an insert in to the lower 32-bits on 64-bit hosts.\n // Override and insert.\n auto Dest = LoadSourceGPR_WithOpSize(Op, Op->Dest, GetGPROpSize(), Op->Flags);\n Const = _Bfi(OpSize::i64Bit, 32, 0, Dest, Const);\n DstSize = OpSize::i64Bit;\n }\n } else {\n DstSize = OpAddr == X86Tables::DecodeFlags::FLAG_OPERAND_SIZE_LAST ? OpSize::i16Bit : OpSize::i32Bit;\n }\n\n if (IsMemDst) {\n // Memory destinatino always writes only 16-bits.\n DstSize = OpSize::i16Bit;\n }\n\n StoreResultGPR_WithOpSize(Op, Op->Dest, Const, DstSize);\n}\n\nOpDispatchBuilder::CycleCounterPair OpDispatchBuilder::CycleCounter(bool SelfSynchronizingLoads) {\n Ref CounterLow {};\n Ref CounterHigh {};\n auto Counter = _CycleCounter(SelfSynchronizingLoads);\n if (CTX->Config.TSCScale) {\n CounterLow = _Lshl(OpSize::i32Bit, Counter, Constant(CTX->Config.TSCScale));\n CounterHigh = _Lshr(OpSize::i64Bit, Counter, Constant(32 - CTX->Config.TSCScale));\n } else {\n CounterLow = _Bfe(OpSize::i64Bit, 32, 0, Counter);\n CounterHigh = _Bfe(OpSize::i64Bit, 32, 32, Counter);\n }\n\n return {\n .CounterLow = CounterLow,\n .CounterHigh = CounterHigh,\n };\n}\n\nvoid OpDispatchBuilder::RDTSCOp(OpcodeArgs) {\n auto Counter = CycleCounter(false);\n StoreGPRRegister(X86State::REG_RAX, Counter.CounterLow);\n StoreGPRRegister(X86State::REG_RDX, Counter.CounterHigh);\n}\n\nvoid OpDispatchBuilder::INCOp(OpcodeArgs) {\n Ref Dest;\n Ref Result;\n const auto Size = GetSrcBitSize(Op);\n const bool IsLocked = DestIsLockedMem(Op);\n\n if (IsLocked) {\n HandledLock = true;\n\n Ref DestAddress = MakeSegmentAddress(Op, Op->Dest);\n Dest = _AtomicFetchAdd(OpSizeFromSrc(Op), Constant(1), DestAddress);\n } else {\n Dest = LoadSourceGPR(Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = Size >= 32});\n }\n\n CalculateDeferredFlags();\n\n if (Size < 32 && CTX->HostFeatures.SupportsFlagM) {\n // Addition producing upper garbage\n Result = Add(OpSize::i32Bit, Dest, 1);\n CalculatePF(Result);\n CalculateAF(Dest, Constant(1));\n\n // Correctly set NZ flags, preserving C\n HandleNZCV_RMW();\n _SetSmallNZV(OpSizeFromSrc(Op), Result);\n\n // Fix up V flag. INC overflows only when incrementing a positive and\n // getting a negative. So compare the sign bits to calculate V.\n _RmifNZCV(_Andn(OpSize::i32Bit, Result, Dest), Size - 1, 1);\n } else {\n Result = CalculateFlags_ADD(OpSizeFromSrc(Op), Dest, Constant(1), false);\n }\n\n if (!IsLocked) {\n StoreResultGPR(Op, Result);\n }\n}\n\nvoid OpDispatchBuilder::DECOp(OpcodeArgs) {\n Ref Dest;\n Ref Result;\n const auto Size = GetSrcBitSize(Op);\n const bool IsLocked = DestIsLockedMem(Op);\n\n if (IsLocked) {\n HandledLock = true;\n\n Ref DestAddress = MakeSegmentAddress(Op, Op->Dest);\n\n // Use Add instead of Sub to avoid a NEG\n Dest = _AtomicFetchAdd(OpSizeFromSrc(Op), Constant(Size == 64 ? -1 : ((1ULL << Size) - 1)), DestAddress);\n } else {\n Dest = LoadSourceGPR(Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = Size >= 32});\n }\n\n CalculateDeferredFlags();\n\n if (Size < 32 && CTX->HostFeatures.SupportsFlagM) {\n // Subtraction producing upper garbage\n Result = Sub(OpSize::i32Bit, Dest, 1);\n CalculatePF(Result);\n CalculateAF(Dest, Constant(1));\n\n // Correctly set NZ flags, preserving C\n HandleNZCV_RMW();\n _SetSmallNZV(OpSizeFromSrc(Op), Result);\n\n // Fix up V flag. DEC overflows only when decrementing a negative and\n // getting a positive. So compare the sign bits to calculate V.\n _RmifNZCV(_Andn(OpSize::i32Bit, Dest, Result), Size - 1, 1);\n } else {\n Result = CalculateFlags_SUB(OpSizeFromSrc(Op), Dest, Constant(1), false);\n }\n\n if (!IsLocked) {\n StoreResultGPR(Op, Result);\n }\n}\n\nvoid OpDispatchBuilder::STOSOp(OpcodeArgs) {\n if (Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_ADDRESS_SIZE) {\n LogMan::Msg::EFmt(\"STOSOp: Can't handle address size override (OP: 0x{:04X}, Flags: 0x{:08X})\", Op->OP, Op->Flags);\n DecodeFailure = true;\n return;\n }\n\n const auto Size = OpSizeFromSrc(Op);\n const bool Repeat = (Op->Flags & (FEXCore::X86Tables::DecodeFlags::FLAG_REP_PREFIX | FEXCore::X86Tables::DecodeFlags::FLAG_REPNE_PREFIX)) != 0;\n\n if (!Repeat) {\n // Src is used only for a store of the same size so allow garbage\n Ref Src = LoadSourceGPR(Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true});\n\n // Only ES prefix\n Ref Dest = MakeSegmentAddress(X86State::REG_RDI, 0, X86Tables::DecodeFlags::FLAG_ES_PREFIX, true);\n\n // Store to memory where RDI points\n if (CTX->IsMemcpyAtomicTSOEnabled()) {\n _StoreMemGPRAutoTSO(Size, Dest, Src, Size);\n } else {\n _StoreMem(RegClass::GPR, Size, Src, Dest, Invalid(), OpSize::i8Bit, MemOffsetType::SXTX, 1);\n }\n\n // Offset the pointer\n Ref TailDest = LoadGPRRegister(X86State::REG_RDI);\n StoreGPRRegister(X86State::REG_RDI, OffsetByDir(TailDest, IR::OpSizeToSize(Size)));\n } else {\n // FEX doesn't support partial faulting REP instructions.\n // Converting this to a `MemSet` IR op optimizes this quite significantly in our codegen.\n // If FEX is to gain support for faulting REP instructions, then this implementation needs to change significantly.\n Ref Src = LoadSourceGPR(Op, Op->Src[0], Op->Flags);\n Ref Dest = LoadGPRRegister(X86State::REG_RDI);\n\n // Only ES prefix\n auto Segment = GetSegment(0, FEXCore::X86Tables::DecodeFlags::FLAG_ES_PREFIX, true);\n\n Ref Counter = LoadGPRRegister(X86State::REG_RCX);\n\n auto Result = _MemSet(CTX->IsAtomicTSOEnabled(), Size, Segment ?: InvalidNode, Dest, Src, Counter, LoadDir(1));\n StoreGPRRegister(X86State::REG_RCX, Constant(0));\n StoreGPRRegister(X86State::REG_RDI, Result);\n }\n}\n\nvoid OpDispatchBuilder::MOVSOp(OpcodeArgs) {\n if (Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_ADDRESS_SIZE) {\n LogMan::Msg::EFmt(\"MOVSOp: Can't handle address size override (OP: 0x{:04X}, Flags: 0x{:08X})\", Op->OP, Op->Flags);\n DecodeFailure = true;\n return;\n }\n\n // RA now can handle these to be here, to avoid DF accesses\n const auto Size = OpSizeFromSrc(Op);\n\n if (Op->Flags & (FEXCore::X86Tables::DecodeFlags::FLAG_REP_PREFIX | FEXCore::X86Tables::DecodeFlags::FLAG_REPNE_PREFIX)) {\n auto SrcAddr = LoadGPRRegister(X86State::REG_RSI);\n auto DstAddr = LoadGPRRegister(X86State::REG_RDI);\n auto Counter = LoadGPRRegister(X86State::REG_RCX);\n\n auto DstSegment = GetSegment(0, FEXCore::X86Tables::DecodeFlags::FLAG_ES_PREFIX, true);\n auto SrcSegment = GetSegment(Op->Flags, FEXCore::X86Tables::DecodeFlags::FLAG_DS_PREFIX);\n\n if (DstSegment) {\n DstAddr = Add(OpSize::i64Bit, DstAddr, DstSegment);\n }\n\n if (SrcSegment) {\n SrcAddr = Add(OpSize::i64Bit, SrcAddr, SrcSegment);\n }\n\n Ref Result_Src = _AllocateGPR(false);\n Ref Result_Dst = _AllocateGPR(false);\n _MemCpy(CTX->IsAtomicTSOEnabled(), Size, DstAddr, SrcAddr, Counter, LoadDir(1), Result_Dst, Result_Src);\n\n if (DstSegment) {\n Result_Dst = Sub(OpSize::i64Bit, Result_Dst, DstSegment);\n }\n\n if (SrcSegment) {\n Result_Src = Sub(OpSize::i64Bit, Result_Src, SrcSegment);\n }\n\n StoreGPRRegister(X86State::REG_RCX, Constant(0));\n StoreGPRRegister(X86State::REG_RDI, Result_Dst);\n StoreGPRRegister(X86State::REG_RSI, Result_Src);\n } else {\n Ref RSI = MakeSegmentAddress(X86State::REG_RSI, Op->Flags, X86Tables::DecodeFlags::FLAG_DS_PREFIX);\n Ref RDI = MakeSegmentAddress(X86State::REG_RDI, 0, X86Tables::DecodeFlags::FLAG_ES_PREFIX, true);\n\n if (CTX->IsMemcpyAtomicTSOEnabled()) {\n auto Src = _LoadMemGPRAutoTSO(Size, RSI, Size);\n\n // Store to memory where RDI points\n _StoreMemGPRAutoTSO(Size, RDI, Src, Size);\n } else {\n auto Src = _LoadMem(RegClass::GPR, Size, RSI, Invalid(), OpSize::i8Bit, MemOffsetType::SXTX, 1);\n _StoreMem(RegClass::GPR, Size, Src, RDI, Invalid(), OpSize::i8Bit, MemOffsetType::SXTX, 1);\n }\n\n RSI = OffsetByDir(RSI, IR::OpSizeToSize(Size));\n RDI = OffsetByDir(RDI, IR::OpSizeToSize(Size));\n\n StoreGPRRegister(X86State::REG_RSI, RSI);\n StoreGPRRegister(X86State::REG_RDI, RDI);\n }\n}\n\nIR::OpSize OpDispatchBuilder::GetStringOpSize(X86Tables::DecodedOp Op) const {\n LOGMAN_THROW_A_FMT(Is64BitMode || !(Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_ADDRESS_SIZE), \"Invalid modifier on 32bit address\");\n return !Is64BitMode || (Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_ADDRESS_SIZE) ? OpSize::i32Bit : OpSize::i64Bit;\n}\n\nvoid OpDispatchBuilder::CMPSOp(OpcodeArgs) {\n if (!Is64BitMode && (Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_ADDRESS_SIZE)) {\n LogMan::Msg::EFmt(\"CMPSOp: Address size override (0x67) not supported in 32-bit mode (OP: 0x{:04X}).\", Op->OP);\n DecodeFailure = true;\n return;\n }\n\n const auto Size = OpSizeFromSrc(Op);\n OpSize AddrSize = GetStringOpSize(Op);\n\n bool Repeat = Op->Flags & (FEXCore::X86Tables::DecodeFlags::FLAG_REPNE_PREFIX | FEXCore::X86Tables::DecodeFlags::FLAG_REP_PREFIX);\n if (!Repeat) {\n Ref Src_RSI = LoadGPRRegister(X86State::REG_RSI, AddrSize);\n Ref Src_RDI = LoadGPRRegister(X86State::REG_RDI, AddrSize);\n\n Ref Dest_RSI = AppendSegmentOffset(Src_RSI, Op->Flags, X86Tables::DecodeFlags::FLAG_DS_PREFIX);\n Ref Dest_RDI = AppendSegmentOffset(Src_RDI, 0, X86Tables::DecodeFlags::FLAG_ES_PREFIX, true);\n\n auto Src1 = _LoadMemGPRAutoTSO(Size, Dest_RDI, Size);\n auto Src2 = _LoadMemGPRAutoTSO(Size, Dest_RSI, Size);\n\n CalculateFlags_SUB(OpSizeFromSrc(Op), Src2, Src1);\n\n Dest_RDI = OffsetByDir(Src_RDI, IR::OpSizeToSize(Size));\n if (Is64BitMode && AddrSize == OpSize::i32Bit) {\n Dest_RDI = _Bfe(OpSize::i64Bit, 32, 0, Dest_RDI);\n StoreGPRRegister(X86State::REG_RDI, Dest_RDI);\n } else {\n StoreGPRRegister(X86State::REG_RDI, Dest_RDI, AddrSize);\n }\n\n Dest_RSI = OffsetByDir(Src_RSI, IR::OpSizeToSize(Size));\n if (Is64BitMode && AddrSize == OpSize::i32Bit) {\n Dest_RSI = _Bfe(OpSize::i64Bit, 32, 0, Dest_RSI);\n StoreGPRRegister(X86State::REG_RSI, Dest_RSI);\n } else {\n StoreGPRRegister(X86State::REG_RSI, Dest_RSI, AddrSize);\n }\n } else {\n // Calculate flags early.\n CalculateDeferredFlags();\n\n bool REPE = Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_REP_PREFIX;\n\n // If rcx = 0, skip the whole loop.\n Ref Counter = LoadGPRRegister(X86State::REG_RCX);\n auto OuterJump = CondJump(Counter, CondClass::EQ);\n\n auto BeforeLoop = CreateNewCodeBlockAfter(GetCurrentBlock());\n SetFalseJumpTarget(OuterJump, BeforeLoop);\n SetCurrentCodeBlock(BeforeLoop);\n StartNewBlock();\n\n ForeachDirection([this, Op, Size, AddrSize, REPE](int32_t PtrDir) {\n IRPair InnerJump;\n auto JumpIntoLoop = Jump();\n\n // Setup for the loop\n auto LoopHeader = CreateNewCodeBlockAfter(GetCurrentBlock());\n SetCurrentCodeBlock(LoopHeader);\n StartNewBlock();\n SetJumpTarget(JumpIntoLoop, LoopHeader);\n\n // Working loop\n {\n Ref Src_RSI = LoadGPRRegister(X86State::REG_RSI, AddrSize);\n Ref Src_RDI = LoadGPRRegister(X86State::REG_RDI, AddrSize);\n\n Ref Dest_RSI = AppendSegmentOffset(Src_RSI, Op->Flags, X86Tables::DecodeFlags::FLAG_DS_PREFIX);\n Ref Dest_RDI = AppendSegmentOffset(Src_RDI, 0, X86Tables::DecodeFlags::FLAG_ES_PREFIX, true);\n\n auto Src1 = _LoadMemGPRAutoTSO(Size, Dest_RDI, Size);\n auto Src2 = _LoadMemGPR(Size, Dest_RSI, Size);\n\n // We'll calculate PF/AF after the loop, so use them as temporaries here.\n StoreRegister(Core::CPUState::PF_AS_GREG, false, Src1);\n StoreRegister(Core::CPUState::AF_AS_GREG, false, Src2);\n\n Ref TailCounter = LoadGPRRegister(X86State::REG_RCX);\n\n // Decrement counter\n TailCounter = SubWithFlags(OpSize::i64Bit, TailCounter, 1);\n\n // Store the counter since we don't have phis\n StoreGPRRegister(X86State::REG_RCX, TailCounter);\n\n Dest_RDI = Add(AddrSize, Src_RDI, PtrDir * static_cast(IR::OpSizeToSize(Size)));\n if (Is64BitMode && AddrSize == OpSize::i32Bit) {\n Dest_RDI = _Bfe(OpSize::i64Bit, 32, 0, Dest_RDI);\n StoreGPRRegister(X86State::REG_RDI, Dest_RDI);\n } else {\n StoreGPRRegister(X86State::REG_RDI, Dest_RDI, AddrSize);\n }\n\n Dest_RSI = Add(AddrSize, Src_RSI, PtrDir * static_cast(IR::OpSizeToSize(Size)));\n if (Is64BitMode && AddrSize == OpSize::i32Bit) {\n Dest_RSI = _Bfe(OpSize::i64Bit, 32, 0, Dest_RSI);\n StoreGPRRegister(X86State::REG_RSI, Dest_RSI);\n } else {\n StoreGPRRegister(X86State::REG_RSI, Dest_RSI, AddrSize);\n }\n\n // If TailCounter != 0, compare sources.\n // If TailCounter == 0, set ZF iff that would break.\n _CondSubNZCV(OpSize::i64Bit, Src2, Src1, CondClass::NEQ, REPE ? 0 : (1 << 2) /* Z */);\n CachedNZCV = nullptr;\n NZCVDirty = false;\n InnerJump = CondJumpNZCV(REPE ? CondClass::EQ : CondClass::NEQ);\n\n // Jump back to the start if we have more work to do\n SetTrueJumpTarget(InnerJump, LoopHeader);\n }\n\n // Make sure to start a new block after ending this one\n auto LoopEnd = CreateNewCodeBlockAfter(GetCurrentBlock());\n SetFalseJumpTarget(InnerJump, LoopEnd);\n SetCurrentCodeBlock(LoopEnd);\n StartNewBlock();\n });\n\n // Make sure to start a new block after ending this one\n {\n // Grab the sources from the last iteration so we can set flags.\n auto Src1 = LoadGPR(Core::CPUState::PF_AS_GREG);\n auto Src2 = LoadGPR(Core::CPUState::AF_AS_GREG);\n CalculateFlags_SUB(OpSizeFromSrc(Op), Src2, Src1);\n }\n auto Jump_ = Jump();\n\n auto Exit = CreateNewCodeBlockAfter(GetCurrentBlock());\n SetJumpTarget(Jump_, Exit);\n SetTrueJumpTarget(OuterJump, Exit);\n SetCurrentCodeBlock(Exit);\n StartNewBlock();\n }\n}\n\nvoid OpDispatchBuilder::LODSOp(OpcodeArgs) {\n if (Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_ADDRESS_SIZE) {\n LogMan::Msg::EFmt(\"LODSOp: Can't handle address size override (OP: 0x{:04X}, Flags: 0x{:08X})\", Op->OP, Op->Flags);\n DecodeFailure = true;\n return;\n }\n\n const auto Size = OpSizeFromSrc(Op);\n const bool Repeat = (Op->Flags & (FEXCore::X86Tables::DecodeFlags::FLAG_REP_PREFIX | FEXCore::X86Tables::DecodeFlags::FLAG_REPNE_PREFIX)) != 0;\n\n if (!Repeat) {\n Ref Dest_RSI = MakeSegmentAddress(X86State::REG_RSI, Op->Flags, X86Tables::DecodeFlags::FLAG_DS_PREFIX);\n\n auto Src = _LoadMemGPRAutoTSO(Size, Dest_RSI, Size);\n\n StoreResultGPR(Op, Src);\n\n // Offset the pointer\n Ref TailDest_RSI = LoadGPRRegister(X86State::REG_RSI);\n StoreGPRRegister(X86State::REG_RSI, OffsetByDir(TailDest_RSI, IR::OpSizeToSize(Size)));\n } else {\n // Calculate flags early. because end of block\n CalculateDeferredFlags();\n\n ForeachDirection([this, Op, Size](int32_t PtrDir) {\n // XXX: Theoretically LODS could be optimized to\n // RSI += {-}(RCX * Size)\n // RAX = [RSI - Size]\n // But this might violate the case of an application scanning pages for read permission and catching the fault\n // May or may not matter\n\n auto JumpStart = Jump();\n // Make sure to start a new block after ending this one\n auto LoopStart = CreateNewCodeBlockAfter(GetCurrentBlock());\n SetJumpTarget(JumpStart, LoopStart);\n SetCurrentCodeBlock(LoopStart);\n StartNewBlock();\n\n Ref Counter = LoadGPRRegister(X86State::REG_RCX);\n\n // Can we end the block?\n\n // We leave if RCX = 0\n auto CondJump_ = CondJump(Counter, CondClass::EQ);\n\n auto LoopTail = CreateNewCodeBlockAfter(LoopStart);\n SetFalseJumpTarget(CondJump_, LoopTail);\n SetCurrentCodeBlock(LoopTail);\n StartNewBlock();\n\n // Working loop\n {\n Ref Dest_RSI = MakeSegmentAddress(X86State::REG_RSI, Op->Flags, X86Tables::DecodeFlags::FLAG_DS_PREFIX);\n\n auto Src = _LoadMemGPRAutoTSO(Size, Dest_RSI, Size);\n\n StoreResultGPR(Op, Src);\n\n Ref TailCounter = LoadGPRRegister(X86State::REG_RCX);\n Ref TailDest_RSI = LoadGPRRegister(X86State::REG_RSI);\n\n // Decrement counter\n TailCounter = Sub(OpSize::i64Bit, TailCounter, 1);\n\n // Store the counter since we don't have phis\n StoreGPRRegister(X86State::REG_RCX, TailCounter);\n\n // Offset the pointer\n TailDest_RSI = Add(OpSize::i64Bit, TailDest_RSI, PtrDir * static_cast(IR::OpSizeToSize(Size)));\n StoreGPRRegister(X86State::REG_RSI, TailDest_RSI);\n\n // Jump back to the start, we have more work to do\n Jump(LoopStart);\n }\n // Make sure to start a new block after ending this one\n auto LoopEnd = CreateNewCodeBlockAfter(LoopTail);\n SetTrueJumpTarget(CondJump_, LoopEnd);\n SetCurrentCodeBlock(LoopEnd);\n StartNewBlock();\n });\n }\n}\n\nvoid OpDispatchBuilder::SCASOp(OpcodeArgs) {\n if (!Is64BitMode && (Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_ADDRESS_SIZE)) {\n LogMan::Msg::EFmt(\"SCASOp: Address size override (0x67) not supported in 32-bit mode (OP: 0x{:04X}).\", Op->OP);\n DecodeFailure = true;\n return;\n }\n\n const auto Size = OpSizeFromSrc(Op);\n OpSize AddrSize = GetStringOpSize(Op);\n const bool Repeat = (Op->Flags & (FEXCore::X86Tables::DecodeFlags::FLAG_REPNE_PREFIX | FEXCore::X86Tables::DecodeFlags::FLAG_REP_PREFIX)) != 0;\n\n if (!Repeat) {\n Ref Src_RDI = LoadGPRRegister(X86State::REG_RDI, AddrSize);\n Ref Dest_RDI = AppendSegmentOffset(Src_RDI, 0, X86Tables::DecodeFlags::FLAG_ES_PREFIX, true);\n\n auto Src1 = LoadSourceGPR(Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true});\n auto Src2 = _LoadMemGPRAutoTSO(Size, Dest_RDI, Size);\n\n CalculateFlags_SUB(OpSizeFromSrc(Op), Src1, Src2);\n\n Ref TailDest_RDI = OffsetByDir(Src_RDI, IR::OpSizeToSize(Size));\n if (Is64BitMode && AddrSize == OpSize::i32Bit) {\n TailDest_RDI = _Bfe(OpSize::i64Bit, 32, 0, TailDest_RDI);\n StoreGPRRegister(X86State::REG_RDI, TailDest_RDI);\n } else {\n StoreGPRRegister(X86State::REG_RDI, TailDest_RDI, AddrSize);\n }\n } else {\n // Calculate flags early. because end of block\n CalculateDeferredFlags();\n\n ForeachDirection([this, Op, Size, AddrSize](int32_t Dir) {\n bool REPE = Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_REP_PREFIX;\n\n auto JumpStart = Jump();\n // Make sure to start a new block after ending this one\n auto LoopStart = CreateNewCodeBlockAfter(GetCurrentBlock());\n SetJumpTarget(JumpStart, LoopStart);\n SetCurrentCodeBlock(LoopStart);\n StartNewBlock();\n\n Ref Counter = LoadGPRRegister(X86State::REG_RCX);\n\n // Can we end the block?\n // We leave if RCX = 0\n auto CondJump_ = CondJump(Counter, CondClass::EQ);\n IRPair InternalCondJump;\n\n auto LoopTail = CreateNewCodeBlockAfter(LoopStart);\n SetFalseJumpTarget(CondJump_, LoopTail);\n SetCurrentCodeBlock(LoopTail);\n StartNewBlock();\n\n // Working loop\n {\n Ref Src_RDI = LoadGPRRegister(X86State::REG_RDI, AddrSize);\n Ref Dest_RDI = AppendSegmentOffset(Src_RDI, 0, X86Tables::DecodeFlags::FLAG_ES_PREFIX, true);\n\n auto Src1 = LoadSourceGPR(Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true});\n auto Src2 = _LoadMemGPRAutoTSO(Size, Dest_RDI, Size);\n\n CalculateFlags_SUB(OpSizeFromSrc(Op), Src1, Src2);\n\n // Calculate flags early.\n CalculateDeferredFlags();\n\n Ref TailCounter = LoadGPRRegister(X86State::REG_RCX);\n Ref Src_RDI_Tail = LoadGPRRegister(X86State::REG_RDI, AddrSize);\n\n // Decrement counter\n TailCounter = Sub(OpSize::i64Bit, TailCounter, 1);\n\n // Store the counter since we don't have phis\n StoreGPRRegister(X86State::REG_RCX, TailCounter);\n\n Ref TailDest_RDI = Add(AddrSize, Src_RDI_Tail, Dir * static_cast(IR::OpSizeToSize(Size)));\n if (Is64BitMode && AddrSize == OpSize::i32Bit) {\n TailDest_RDI = _Bfe(OpSize::i64Bit, 32, 0, TailDest_RDI);\n StoreGPRRegister(X86State::REG_RDI, TailDest_RDI);\n } else {\n StoreGPRRegister(X86State::REG_RDI, TailDest_RDI, AddrSize);\n }\n\n CalculateDeferredFlags();\n InternalCondJump = CondJumpNZCV(REPE ? CondClass::EQ : CondClass::NEQ);\n\n // Jump back to the start if we have more work to do\n SetTrueJumpTarget(InternalCondJump, LoopStart);\n }\n // Make sure to start a new block after ending this one\n auto LoopEnd = CreateNewCodeBlockAfter(LoopTail);\n SetTrueJumpTarget(CondJump_, LoopEnd);\n\n SetFalseJumpTarget(InternalCondJump, LoopEnd);\n\n SetCurrentCodeBlock(LoopEnd);\n StartNewBlock();\n });\n }\n}\n\nvoid OpDispatchBuilder::BSWAPOp(OpcodeArgs) {\n Ref Dest;\n const auto Size = OpSizeFromSrc(Op);\n if (Size == OpSize::i16Bit) {\n // BSWAP of 16bit is undef. ZEN+ causes the lower 16bits to get zero'd\n Dest = Constant(0);\n } else {\n Dest = LoadSourceGPR_WithOpSize(Op, Op->Dest, GetGPROpSize(), Op->Flags);\n Dest = _Rev(Size, Dest);\n }\n StoreResultGPR(Op, Dest);\n}\n\nvoid OpDispatchBuilder::PUSHFOp(OpcodeArgs) {\n const auto Size = OpSizeFromSrc(Op);\n\n Push(Size, GetPackedRFLAG());\n}\n\nvoid OpDispatchBuilder::POPFOp(OpcodeArgs) {\n const auto Size = OpSizeFromSrc(Op);\n Ref Src = Pop(Size);\n\n // Add back our flag constants\n // Bit 1 is always 1\n // Bit 9 is always 1 because we always have interrupts enabled\n\n Src = _Or(OpSize::i64Bit, Src, Constant(0x202));\n\n SetPackedRFLAG(false, Src);\n\n auto NewRIP = GetRelocatedPC(Op);\n ExitFunction(NewRIP, BranchHint::CheckTF);\n BlockSetRIP = true;\n}\n\nvoid OpDispatchBuilder::NEGOp(OpcodeArgs) {\n HandledLock = (Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_LOCK) != 0;\n\n const auto Size = OpSizeFromSrc(Op);\n auto ZeroConst = Constant(0);\n\n if (DestIsLockedMem(Op)) {\n Ref DestMem = MakeSegmentAddress(Op, Op->Dest);\n Ref Dest = _AtomicFetchNeg(Size, DestMem);\n CalculateFlags_SUB(Size, ZeroConst, Dest);\n } else {\n Ref Dest = LoadSourceGPR(Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = true});\n Ref Result = CalculateFlags_SUB(Size, ZeroConst, Dest);\n\n StoreResultGPR(Op, Result);\n }\n}\n\nvoid OpDispatchBuilder::DIVOp(OpcodeArgs) {\n const auto GPRSize = GetGPROpSize();\n auto Size = OpSizeFromSrc(Op);\n\n // This loads the divisor. 32-bit/64-bit paths mask inside the JIT, 8/16 do not.\n Ref Divisor = LoadSourceGPR(Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = Size >= OpSize::i32Bit});\n\n if (Size == OpSize::i64Bit && !Is64BitMode) {\n LogMan::Msg::EFmt(\"Doesn't exist in 32bit mode\");\n DecodeFailure = true;\n return;\n }\n\n Ref Quotient = _AllocateGPR(true);\n Ref Remainder = _AllocateGPR(true);\n\n if (Size == OpSize::i8Bit) {\n Ref Src1 = LoadGPRRegister(X86State::REG_RAX, OpSize::i16Bit);\n\n _UDiv(OpSize::i16Bit, Src1, Invalid(), Divisor, Quotient, Remainder);\n\n // AX[15:0] = concat\n auto ResultAX = _Bfi(GPRSize, 8, 8, Quotient, Remainder);\n StoreGPRRegister(X86State::REG_RAX, ResultAX, OpSize::i16Bit);\n } else {\n Ref Src1 = LoadGPRRegister(X86State::REG_RAX);\n Ref Src2 = LoadGPRRegister(X86State::REG_RDX);\n\n _UDiv(Size, Src1, Src2, Divisor, Quotient, Remainder);\n\n if (Size == OpSize::i32Bit) {\n Quotient = _Bfe(OpSize::i32Bit, IR::OpSizeAsBits(Size), 0, Quotient);\n Remainder = _Bfe(OpSize::i32Bit, IR::OpSizeAsBits(Size), 0, Remainder);\n Size = OpSize::iInvalid;\n }\n\n StoreGPRRegister(X86State::REG_RAX, Quotient, Size);\n StoreGPRRegister(X86State::REG_RDX, Remainder, Size);\n }\n}\n\nvoid OpDispatchBuilder::IDIVOp(OpcodeArgs) {\n // This loads the divisor\n Ref Divisor = LoadSourceGPR(Op, Op->Dest, Op->Flags);\n\n const auto GPRSize = GetGPROpSize();\n auto Size = OpSizeFromSrc(Op);\n\n if (Size == OpSize::i64Bit && !Is64BitMode) {\n LogMan::Msg::EFmt(\"Doesn't exist in 32bit mode\");\n DecodeFailure = true;\n return;\n }\n\n Ref Quotient = _AllocateGPR(true);\n Ref Remainder = _AllocateGPR(true);\n\n if (Size == OpSize::i8Bit) {\n Ref Src1 = LoadGPRRegister(X86State::REG_RAX);\n Src1 = _Sbfe(OpSize::i64Bit, 16, 0, Src1);\n Divisor = _Sbfe(OpSize::i64Bit, 8, 0, Divisor);\n\n _Div(OpSize::i64Bit, Src1, Invalid(), Divisor, Quotient, Remainder);\n\n // AX[15:0] = concat\n auto ResultAX = _Bfi(GPRSize, 8, 8, Quotient, Remainder);\n StoreGPRRegister(X86State::REG_RAX, ResultAX, OpSize::i16Bit);\n } else {\n Ref Src1 = LoadGPRRegister(X86State::REG_RAX);\n Ref Src2 = LoadGPRRegister(X86State::REG_RDX);\n\n _Div(Size, Src1, Src2, Divisor, Quotient, Remainder);\n\n if (Size == OpSize::i32Bit) {\n Quotient = _Bfe(OpSize::i32Bit, IR::OpSizeAsBits(Size), 0, Quotient);\n Remainder = _Bfe(OpSize::i32Bit, IR::OpSizeAsBits(Size), 0, Remainder);\n Size = OpSize::iInvalid;\n }\n\n StoreGPRRegister(X86State::REG_RAX, Quotient, Size);\n StoreGPRRegister(X86State::REG_RDX, Remainder, Size);\n }\n}\n\nvoid OpDispatchBuilder::BSFOp(OpcodeArgs) {\n const auto GPRSize = GetGPROpSize();\n const auto DstSize = OpSizeFromDst(Op) == OpSize::i16Bit ? OpSize::i16Bit : GPRSize;\n Ref Dest = LoadSourceGPR_WithOpSize(Op, Op->Dest, DstSize, Op->Flags, {.AllowUpperGarbage = true});\n Ref Src = LoadSourceGPR(Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true});\n\n // Find the LSB of this source\n auto Result = _FindLSB(OpSizeFromSrc(Op), Src);\n\n // OF, SF, AF, PF, CF all undefined\n // ZF is set to 1 if the source was zero\n SetZ_InvalidateNCV(OpSizeFromSrc(Op), Src);\n\n // If Src was zero then the destination doesn't get modified.\n //\n // Although Intel does not guarantee that semantic, AMD does and Intel\n // hardware satisfies it. We provide the stronger AMD behaviour as\n // applications might rely on that in the wild.\n auto SelectOp = NZCVSelect(GPRSize, CondClass::EQ, Dest, Result);\n StoreResultGPR_WithOpSize(Op, Op->Dest, SelectOp, DstSize);\n}\n\nvoid OpDispatchBuilder::BSROp(OpcodeArgs) {\n const auto GPRSize = GetGPROpSize();\n const auto DstSize = OpSizeFromDst(Op) == OpSize::i16Bit ? OpSize::i16Bit : GPRSize;\n Ref Dest = LoadSourceGPR_WithOpSize(Op, Op->Dest, DstSize, Op->Flags, {.AllowUpperGarbage = true});\n Ref Src = LoadSourceGPR(Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true});\n\n // Find the MSB of this source\n auto Result = _FindMSB(OpSizeFromSrc(Op), Src);\n\n // OF, SF, AF, PF, CF all undefined\n // ZF is set to 1 if the source was zero\n SetZ_InvalidateNCV(OpSizeFromSrc(Op), Src);\n\n // If Src was zero then the destination doesn't get modified\n auto SelectOp = NZCVSelect(GPRSize, CondClass::EQ, Dest, Result);\n StoreResultGPR_WithOpSize(Op, Op->Dest, SelectOp, DstSize);\n}\n\nvoid OpDispatchBuilder::CMPXCHGOp(OpcodeArgs) {\n // CMPXCHG ModRM, reg, {RAX}\n // MemData = *ModRM.dest\n // if (RAX == MemData)\n // modRM.dest = reg;\n // ZF = 1\n // else\n // ZF = 0\n // RAX = MemData\n //\n // CASL Xs, Xt, Xn\n // MemData = *Xn\n // if (MemData == Xs)\n // *Xn = Xt\n // Xs = MemData\n\n const auto GPRSize = GetGPROpSize();\n auto Size = OpSizeFromSrc(Op);\n\n if (Op->Dest.IsGPR()) {\n // This is our source register\n Ref Src2 = LoadSourceGPR(Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true});\n Ref Src3 = LoadGPRRegister(X86State::REG_RAX);\n\n // If the destination is also the accumulator, we get some algebraic\n // simplifications. Not sure if this is actually hit but it's in\n // InstCountCI.\n bool Trivial = Op->Dest.Data.GPR.GPR == X86State::REG_RAX && !Op->Dest.IsGPRDirect() && !Op->Dest.Data.GPR.HighBits;\n\n Ref Src1 {};\n Ref Src1Lower {};\n\n if (GPRSize == OpSize::i64Bit && Size == OpSize::i32Bit) {\n Src1 = LoadSourceGPR_WithOpSize(Op, Op->Dest, GPRSize, Op->Flags, {.AllowUpperGarbage = true});\n Src1Lower = Trivial ? Src1 : _Bfe(GPRSize, IR::OpSizeAsBits(Size), 0, Src1);\n } else {\n Src1 = LoadSourceGPR_WithOpSize(Op, Op->Dest, Size, Op->Flags, {.AllowUpperGarbage = true});\n Src1Lower = Src1;\n }\n\n // Compare RAX with the destination, setting flags accordingly.\n CalculateFlags_SUB(OpSizeFromSrc(Op), Src3, Src1Lower);\n CalculateDeferredFlags();\n\n if (!Trivial) {\n if (GPRSize == OpSize::i64Bit && Size == OpSize::i32Bit) {\n // This allows us to only hit the ZEXT case on failure\n Ref RAXResult = NZCVSelect(OpSize::i64Bit, CondClass::EQ, Src3, Src1Lower);\n\n // When the size is 4 we need to make sure not zext the GPR when the comparison fails\n StoreGPRRegister(X86State::REG_RAX, RAXResult);\n } else {\n StoreGPRRegister(X86State::REG_RAX, Src1Lower, Size);\n }\n }\n\n // Op1 = RAX == Op1 ? Op2 : Op1\n // If they match then set the rm operand to the input\n // else don't set the rm operand\n Ref Src2Lower = Src2;\n if (GPRSize == OpSize::i64Bit && Size == OpSize::i32Bit) {\n Src2Lower = _Bfe(GPRSize, IR::OpSizeAsBits(Size), 0, Src2);\n }\n Ref DestResult = Trivial ? Src2 : NZCVSelect(OpSize::i64Bit, CondClass::EQ, Src2Lower, Src1);\n\n // Store in to GPR Dest\n if (GPRSize == OpSize::i64Bit && Size == OpSize::i32Bit) {\n StoreResultGPR_WithOpSize(Op, Op->Dest, DestResult, GPRSize);\n } else {\n StoreResultGPR(Op, DestResult);\n }\n } else {\n Ref Src2 = LoadSourceGPR(Op, Op->Src[0], Op->Flags);\n HandledLock = Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_LOCK;\n\n auto Src3 = LoadGPRRegister(X86State::REG_RAX);\n auto Src3Lower = _Bfe(OpSize::i64Bit, OpSizeAsBits(Size), 0, Src3);\n\n // If this is a memory location then we want the pointer to it\n Ref Src1 = MakeSegmentAddress(Op, Op->Dest);\n\n // DataSrc = *Src1\n // if (DataSrc == Src3) { *Src1 == Src2; } Src2 = DataSrc\n // This will write to memory! Careful!\n // Third operand must be a calculated guest memory address\n Ref CASResult = _CAS(Size, Src3, Src2, Src1);\n Ref RAXResult = CASResult;\n\n CalculateFlags_SUB(OpSizeFromSrc(Op), Src3Lower, CASResult);\n CalculateDeferredFlags();\n\n if (GPRSize == OpSize::i64Bit && Size == OpSize::i32Bit) {\n // This allows us to only hit the ZEXT case on failure\n RAXResult = _NZCVSelect(OpSize::i64Bit, CondClass::EQ, Src3, CASResult);\n Size = OpSize::i64Bit;\n }\n\n // RAX gets the result of the CAS op\n StoreGPRRegister(X86State::REG_RAX, RAXResult, Size);\n }\n}\n\nvoid OpDispatchBuilder::CMPXCHGPairOp(OpcodeArgs) {\n // Calculate flags early.\n CalculateDeferredFlags();\n\n // REX.W used to determine if it is 16byte or 8byte\n // Unlike CMPXCHG, the destination can only be a memory location\n const auto Size = Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_REX_WIDENING ? OpSize::i64Bit : OpSize::i32Bit;\n\n HandledLock = (Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_LOCK) != 0;\n\n // If this is a memory location then we want the pointer to it\n Ref Src1 = MakeSegmentAddress(Op, Op->Dest);\n\n // Load the full 64-bit registers, all the users ignore the upper 32-bits for\n // 32-bit only cmpxchg. This saves some zero extension.\n Ref Expected_Lower = LoadGPRRegister(X86State::REG_RAX);\n Ref Expected_Upper = LoadGPRRegister(X86State::REG_RDX);\n Ref Desired_Lower = LoadGPRRegister(X86State::REG_RBX);\n Ref Desired_Upper = LoadGPRRegister(X86State::REG_RCX);\n\n // ssa0 = Expected\n // ssa1 = Desired\n // ssa2 = MemoryLocation\n\n // DataSrc = *MemSrc\n // if (DataSrc == Expected) { *MemSrc == Desired; } Expected = DataSrc\n // This will write to memory! Careful!\n // Third operand must be a calculated guest memory address\n\n Ref Result_Lower = _AllocateGPR(true);\n Ref Result_Upper = _AllocateGPRAfter(Result_Lower);\n _CASPair(Size, Expected_Lower, Expected_Upper, Desired_Lower, Desired_Upper, Src1, Result_Lower, Result_Upper);\n\n HandleNZCV_RMW();\n _CmpPairZ(Size, Result_Lower, Result_Upper, Expected_Lower, Expected_Upper);\n CalculateDeferredFlags();\n\n auto UpdateIfNotZF = [this](auto Reg, auto Value) {\n // Always use 64-bit csel to preserve existing upper bits. If we have a\n // 32-bit cmpxchg in a 64-bit context, Value will be zeroed in upper bits.\n StoreGPRRegister(Reg, NZCVSelect(OpSize::i64Bit, CondClass::NEQ, Value, LoadGPRRegister(Reg)));\n };\n\n UpdateIfNotZF(X86State::REG_RAX, Result_Lower);\n UpdateIfNotZF(X86State::REG_RDX, Result_Upper);\n}\n\nvoid OpDispatchBuilder::CreateJumpBlocks(const fextl::vector* Blocks) {\n Ref PrevCodeBlock {};\n for (auto& Target : *Blocks) {\n auto CodeNode = CreateCodeNode(Target.IsEntryPoint, Target.Entry - Entry);\n\n JumpTargets.try_emplace(Target.Entry, JumpTargetInfo {CodeNode, false, Target.IsEntryPoint});\n\n if (PrevCodeBlock) {\n LinkCodeBlocks(PrevCodeBlock, CodeNode);\n }\n\n PrevCodeBlock = CodeNode;\n }\n}\n\nvoid OpDispatchBuilder::BeginFunction(uint64_t RIP, const fextl::vector* Blocks,\n uint32_t NumInstructions, bool _Is64BitMode, bool MonoBackpatcherBlock) {\n Entry = RIP;\n Is64BitMode = _Is64BitMode;\n LOGMAN_THROW_A_FMT(Is64BitMode == CTX->Config.Is64BitMode, \"Expected operating mode to not change at runtime!\");\n IsMonoBackpatcherBlock = MonoBackpatcherBlock;\n auto IRHeader = _IRHeader(InvalidNode, RIP, 0, NumInstructions, 0, 0);\n CreateJumpBlocks(Blocks);\n\n auto Block = GetNewJumpBlock(RIP);\n SetCurrentCodeBlock(Block);\n IRHeader.first->Blocks = Block->Wrapped(DualListData.ListBegin());\n CurrentHeader = IRHeader.first;\n}\n\nvoid OpDispatchBuilder::Finalize() {\n // This usually doesn't emit any IR but in the case of hitting the block instruction limit it will\n FlushRegisterCache();\n const auto GPRSize = GetGPROpSize();\n\n // Node 0 is invalid node\n Ref RealNode = reinterpret_cast(GetNode(1));\n\n const FEXCore::IR::IROp_Header* IROp = RealNode->Op(DualListData.DataBegin());\n LOGMAN_THROW_A_FMT(IROp->Op == OP_IRHEADER, \"First op in function must be our header\");\n\n // Let's walk the jump blocks and see if we have handled every block target\n for (auto& Handler : JumpTargets) {\n if (Handler.second.HaveEmitted) {\n continue;\n }\n\n // We haven't emitted. Dump out to the dispatcher\n SetCurrentCodeBlock(Handler.second.BlockEntry);\n ExitFunction(_InlineEntrypointOffset(GPRSize, Handler.first - Entry));\n }\n}\n\nuint8_t OpDispatchBuilder::GetDstSize(X86Tables::DecodedOp Op) const {\n const uint32_t DstSizeFlag = X86Tables::DecodeFlags::GetSizeDstFlags(Op->Flags);\n LOGMAN_THROW_A_FMT(DstSizeFlag != 0 && DstSizeFlag != X86Tables::DecodeFlags::SIZE_MASK, \"Invalid destination size for op\");\n return 1u << (DstSizeFlag - 1);\n}\n\nuint8_t OpDispatchBuilder::GetSrcSize(X86Tables::DecodedOp Op) const {\n const uint32_t SrcSizeFlag = X86Tables::DecodeFlags::GetSizeSrcFlags(Op->Flags);\n LOGMAN_THROW_A_FMT(SrcSizeFlag != 0 && SrcSizeFlag != X86Tables::DecodeFlags::SIZE_MASK, \"Invalid destination size for op\");\n return 1u << (SrcSizeFlag - 1);\n}\n\nuint32_t OpDispatchBuilder::GetSrcBitSize(X86Tables::DecodedOp Op) const {\n return GetSrcSize(Op) * 8;\n}\n\nuint32_t OpDispatchBuilder::GetDstBitSize(X86Tables::DecodedOp Op) const {\n return GetDstSize(Op) * 8;\n}\n\nRef OpDispatchBuilder::GetSegment(uint32_t Flags, uint32_t DefaultPrefix, bool Override) {\n const auto GPRSize = GetGPROpSize();\n uint32_t Prefix = Flags & FEXCore::X86Tables::DecodeFlags::FLAG_SEGMENTS;\n\n if (Is64BitMode) {\n if (Prefix == FEXCore::X86Tables::DecodeFlags::FLAG_FS_PREFIX) {\n return _LoadContextGPR(GPRSize, offsetof(FEXCore::Core::CPUState, fs_cached));\n } else if (Prefix == FEXCore::X86Tables::DecodeFlags::FLAG_GS_PREFIX) {\n return _LoadContextGPR(GPRSize, offsetof(FEXCore::Core::CPUState, gs_cached));\n }\n // If there was any other segment in 64bit then it is ignored\n } else {\n if (Prefix == FEXCore::X86Tables::DecodeFlags::FLAG_NO_PREFIX || Override) {\n // If there was no prefix then use the default one if available\n // Or the argument only uses a specific prefix (with override set)\n Prefix = DefaultPrefix;\n }\n // With the segment register optimization we store the GDT bases directly in the segment register to remove indexed loads\n Ref SegmentResult {};\n switch (Prefix) {\n [[likely]] case FEXCore::X86Tables::DecodeFlags::FLAG_NO_PREFIX:\n return nullptr;\n case FEXCore::X86Tables::DecodeFlags::FLAG_ES_PREFIX:\n SegmentResult = _LoadContextGPR(GPRSize, offsetof(FEXCore::Core::CPUState, es_cached));\n break;\n case FEXCore::X86Tables::DecodeFlags::FLAG_CS_PREFIX:\n SegmentResult = _LoadContextGPR(GPRSize, offsetof(FEXCore::Core::CPUState, cs_cached));\n break;\n case FEXCore::X86Tables::DecodeFlags::FLAG_SS_PREFIX:\n SegmentResult = _LoadContextGPR(GPRSize, offsetof(FEXCore::Core::CPUState, ss_cached));\n break;\n case FEXCore::X86Tables::DecodeFlags::FLAG_DS_PREFIX:\n SegmentResult = _LoadContextGPR(GPRSize, offsetof(FEXCore::Core::CPUState, ds_cached));\n break;\n case FEXCore::X86Tables::DecodeFlags::FLAG_FS_PREFIX:\n SegmentResult = _LoadContextGPR(GPRSize, offsetof(FEXCore::Core::CPUState, fs_cached));\n break;\n case FEXCore::X86Tables::DecodeFlags::FLAG_GS_PREFIX:\n SegmentResult = _LoadContextGPR(GPRSize, offsetof(FEXCore::Core::CPUState, gs_cached));\n break;\n default: FEX_UNREACHABLE;\n }\n\n CheckLegacySegmentRead(SegmentResult, Prefix);\n return SegmentResult;\n }\n return nullptr;\n}\n\nRef OpDispatchBuilder::AppendSegmentOffset(Ref Value, uint32_t Flags, uint32_t DefaultPrefix, bool Override) {\n auto Segment = GetSegment(Flags, DefaultPrefix, Override);\n if (Segment) {\n Value = Add(std::max(OpSize::i32Bit, std::max(GetOpSize(Value), GetOpSize(Segment))), Value, Segment);\n }\n\n return Value;\n}\n\n\nvoid OpDispatchBuilder::CheckLegacySegmentRead(Ref NewNode, uint32_t SegmentReg) {\n#ifndef FEX_DISABLE_TELEMETRY\n if (SegmentReg == FEXCore::X86Tables::DecodeFlags::FLAG_FS_PREFIX || SegmentReg == FEXCore::X86Tables::DecodeFlags::FLAG_GS_PREFIX) {\n // FS and GS segments aren't considered legacy.\n return;\n }\n\n if (!(SegmentsNeedReadCheck & SegmentReg)) {\n // If the block has done multiple reads of a segment register then skip redundant read checks.\n // Segment write will cause another read check.\n return;\n }\n\n if (CTX->Config.DisableTelemetry()) {\n // Telemetry disabled at runtime.\n return;\n }\n\n FEXCore::Telemetry::TelemetryType TelemIndex {};\n switch (SegmentReg) {\n case FEXCore::X86Tables::DecodeFlags::FLAG_ES_PREFIX:\n TelemIndex = FEXCore::Telemetry::TelemetryType::TYPE_USES_32BIT_SEGMENT_ES;\n SegmentsNeedReadCheck &= ~FEXCore::X86Tables::DecodeFlags::FLAG_ES_PREFIX;\n break;\n case FEXCore::X86Tables::DecodeFlags::FLAG_CS_PREFIX:\n TelemIndex = FEXCore::Telemetry::TelemetryType::TYPE_USES_32BIT_SEGMENT_CS;\n SegmentsNeedReadCheck &= ~FEXCore::X86Tables::DecodeFlags::FLAG_CS_PREFIX;\n break;\n case FEXCore::X86Tables::DecodeFlags::FLAG_SS_PREFIX:\n TelemIndex = FEXCore::Telemetry::TelemetryType::TYPE_USES_32BIT_SEGMENT_SS;\n SegmentsNeedReadCheck &= ~FEXCore::X86Tables::DecodeFlags::FLAG_SS_PREFIX;\n break;\n case FEXCore::X86Tables::DecodeFlags::FLAG_DS_PREFIX:\n TelemIndex = FEXCore::Telemetry::TelemetryType::TYPE_USES_32BIT_SEGMENT_DS;\n SegmentsNeedReadCheck &= ~FEXCore::X86Tables::DecodeFlags::FLAG_DS_PREFIX;\n break;\n default: FEX_UNREACHABLE;\n }\n\n // Will set the telemetry value if NewNode is != 0\n _TelemetrySetValue(NewNode, TelemIndex);\n // Telemetry will dirty flags, and user code does not expect LoadSource to clobber flags, fix that up here as this is an edge case.\n CalculateDeferredFlags();\n#endif\n}\n\nvoid OpDispatchBuilder::CheckLegacySegmentWrite(Ref NewNode, uint32_t SegmentReg) {\n#ifndef FEX_DISABLE_TELEMETRY\n if (SegmentReg == FEXCore::X86Tables::DecodeFlags::FLAG_FS_PREFIX || SegmentReg == FEXCore::X86Tables::DecodeFlags::FLAG_GS_PREFIX) {\n // FS and GS segments aren't considered legacy.\n return;\n }\n\n if (CTX->Config.DisableTelemetry()) {\n // Telemetry disabled at runtime.\n return;\n }\n\n FEXCore::Telemetry::TelemetryType TelemIndex {};\n switch (SegmentReg) {\n case FEXCore::X86Tables::DecodeFlags::FLAG_ES_PREFIX:\n TelemIndex = FEXCore::Telemetry::TelemetryType::TYPE_WRITES_32BIT_SEGMENT_ES;\n SegmentsNeedReadCheck |= FEXCore::X86Tables::DecodeFlags::FLAG_ES_PREFIX;\n break;\n case FEXCore::X86Tables::DecodeFlags::FLAG_CS_PREFIX:\n TelemIndex = FEXCore::Telemetry::TelemetryType::TYPE_WRITES_32BIT_SEGMENT_CS;\n SegmentsNeedReadCheck |= FEXCore::X86Tables::DecodeFlags::FLAG_CS_PREFIX;\n break;\n case FEXCore::X86Tables::DecodeFlags::FLAG_SS_PREFIX:\n TelemIndex = FEXCore::Telemetry::TelemetryType::TYPE_WRITES_32BIT_SEGMENT_SS;\n SegmentsNeedReadCheck |= FEXCore::X86Tables::DecodeFlags::FLAG_SS_PREFIX;\n break;\n case FEXCore::X86Tables::DecodeFlags::FLAG_DS_PREFIX:\n TelemIndex = FEXCore::Telemetry::TelemetryType::TYPE_WRITES_32BIT_SEGMENT_DS;\n SegmentsNeedReadCheck |= FEXCore::X86Tables::DecodeFlags::FLAG_DS_PREFIX;\n break;\n default: FEX_UNREACHABLE;\n }\n\n // Will set the telemetry value if NewNode is != 0\n _TelemetrySetValue(NewNode, TelemIndex);\n // Telemetry will dirty flags, and user code does not expect LoadSource to clobber flags, fix that up here as this is an edge case.\n CalculateDeferredFlags();\n#endif\n}\n\nvoid OpDispatchBuilder::UpdatePrefixFromSegment(Ref Segment, uint32_t SegmentReg) {\n // Use BFE to extract the selector index in bits [15,3] of the segment register.\n // In some cases the upper 16-bits of the 32-bit GPR contain garbage to ignore.\n auto GDT = _Bfe(OpSize::i32Bit, 1, 2, Segment);\n // Fun quirk, if we mask the selector then it is premultiplied by 8 which we need to do for accessing anyway.\n auto SegmentOffset = _And(OpSize::i32Bit, Segment, _Constant(0xfff8));\n Ref SegmentBase = _LoadContextGPRIndexed(GDT, OpSize::i64Bit, offsetof(FEXCore::Core::CPUState, segment_arrays[0]), 8);\n Ref NewSegment = _LoadMemGPR(OpSize::i64Bit, SegmentBase, SegmentOffset, OpSize::i8Bit, MemOffsetType::UXTW, 1);\n CheckLegacySegmentWrite(NewSegment, SegmentReg);\n\n // Extract the 32-bit base from the GDT segment.\n auto Upper32 = _Lshr(OpSize::i64Bit, NewSegment, _Constant(32));\n auto Masked = _And(OpSize::i32Bit, Upper32, _Constant(0xFF00'0000));\n Ref Merged = _Orlshr(OpSize::i32Bit, Masked, NewSegment, 16);\n NewSegment = _Bfi(OpSize::i32Bit, 8, 16, Merged, Upper32);\n\n switch (SegmentReg) {\n case FEXCore::X86Tables::DecodeFlags::FLAG_ES_PREFIX:\n _StoreContextGPR(OpSize::i32Bit, NewSegment, offsetof(FEXCore::Core::CPUState, es_cached));\n break;\n case FEXCore::X86Tables::DecodeFlags::FLAG_CS_PREFIX:\n _StoreContextGPR(OpSize::i32Bit, NewSegment, offsetof(FEXCore::Core::CPUState, cs_cached));\n break;\n case FEXCore::X86Tables::DecodeFlags::FLAG_SS_PREFIX:\n _StoreContextGPR(OpSize::i32Bit, NewSegment, offsetof(FEXCore::Core::CPUState, ss_cached));\n break;\n case FEXCore::X86Tables::DecodeFlags::FLAG_DS_PREFIX:\n _StoreContextGPR(OpSize::i32Bit, NewSegment, offsetof(FEXCore::Core::CPUState, ds_cached));\n break;\n case FEXCore::X86Tables::DecodeFlags::FLAG_FS_PREFIX:\n _StoreContextGPR(OpSize::i32Bit, NewSegment, offsetof(FEXCore::Core::CPUState, fs_cached));\n break;\n case FEXCore::X86Tables::DecodeFlags::FLAG_GS_PREFIX:\n _StoreContextGPR(OpSize::i32Bit, NewSegment, offsetof(FEXCore::Core::CPUState, gs_cached));\n break;\n default: break; // Do nothing\n }\n}\n\nAddressMode OpDispatchBuilder::DecodeAddress(const X86Tables::DecodedOp& Op, const X86Tables::DecodedOperand& Operand,\n MemoryAccessType AccessType, bool IsLoad) {\n const auto GPRSize = GetGPROpSize();\n\n AddressMode A {};\n A.Segment = GetSegment(Op->Flags);\n A.AddrSize = (Op->Flags & X86Tables::DecodeFlags::FLAG_ADDRESS_SIZE) != 0 ? (GPRSize >> 1) : GPRSize;\n A.NonTSO = AccessType == MemoryAccessType::NONTSO || AccessType == MemoryAccessType::STREAM;\n\n if (Operand.IsLiteral()) {\n A.Offset = Operand.Literal();\n\n if (Operand.Data.Literal.Size != 8 && IsLoad) {\n // zero extend\n uint64_t width = Operand.Data.Literal.Size * 8;\n A.Offset &= ((1ULL << width) - 1);\n }\n } else if (Operand.IsGPR()) {\n // Not an address, let the caller deal with it\n A.AddrSize = GPRSize;\n } else if (Operand.IsGPRDirect()) {\n A.Base = LoadGPRRegister(Operand.Data.GPR.GPR, GPRSize);\n A.NonTSO |= IsNonTSOReg(AccessType, Operand.Data.GPR.GPR);\n } else if (Operand.IsGPRIndirect() || Operand.IsGPRIndirectRelocation()) {\n A.Base = LoadGPRRegister(Operand.Data.GPRIndirect.GPR, GPRSize);\n if (Operand.IsGPRIndirectRelocation()) {\n A.Base = Add(GPRSize, _EntrypointOffset(GPRSize, Operand.Data.GPRIndirect.Displacement), A.Base);\n } else {\n A.Offset = static_cast(Operand.Data.GPRIndirect.Displacement);\n }\n A.NonTSO |= IsNonTSOReg(AccessType, Operand.Data.GPRIndirect.GPR);\n } else if (Operand.IsRIPRelative() || Operand.IsRIPRelativeRelocation()) {\n if (Is64BitMode) {\n A.Base = GetRelocatedPC(Op, static_cast(Operand.Data.RIPLiteral.Value));\n } else {\n // 32bit this isn't RIP relative but instead absolute\n if (Operand.IsRIPRelativeRelocation()) {\n A.Base = _EntrypointOffset(GPRSize, Operand.Data.RIPLiteral.Value);\n } else {\n A.Offset = Operand.Data.RIPLiteral.Value;\n }\n }\n } else if (Operand.IsSIB() || Operand.IsSIBRelocation()) {\n const bool IsVSIB = IsLoad && ((Op->Flags & X86Tables::DecodeFlags::FLAG_VSIB_BYTE) != 0);\n\n if (Operand.Data.SIB.Base != FEXCore::X86State::REG_INVALID) {\n A.Base = LoadGPRRegister(Operand.Data.SIB.Base, GPRSize);\n }\n\n // NOTE: VSIB cannot have the index * scale portion calculated ahead of time,\n // since the index in this case is a vector. So, we can't just apply the scale\n // to it, since this needs to be applied to each element in the index register\n // after said element has been sign extended. So, we pass this through for the\n // instruction implementation to handle.\n //\n // What we do handle though, is the applying the displacement value to\n // the base register (if a base register is provided), since this is a\n // part of the address calculation that can be done ahead of time.\n if (!IsVSIB && Operand.Data.SIB.Index != FEXCore::X86State::REG_INVALID) {\n A.Index = LoadGPRRegister(Operand.Data.SIB.Index, GPRSize);\n A.IndexScale = Operand.Data.SIB.Scale;\n }\n\n if (Operand.IsSIBRelocation()) {\n auto EPOffset = _EntrypointOffset(GPRSize, Operand.Data.SIB.Offset);\n if (A.Base) {\n A.Base = Add(GPRSize, EPOffset, A.Base);\n } else {\n A.Base = EPOffset;\n }\n } else {\n A.Offset = static_cast(Operand.Data.SIB.Offset);\n }\n\n A.NonTSO |= IsNonTSOReg(AccessType, Operand.Data.SIB.Base) || IsNonTSOReg(AccessType, Operand.Data.SIB.Index);\n } else if (Operand.IsLiteralRelocation()) {\n A.Base = _EntrypointOffset(GPRSize, Operand.Data.LiteralRelocation.EntrypointOffset);\n } else {\n LOGMAN_MSG_A_FMT(\"Unknown Src Type: {}\\n\", Operand.Type);\n }\n\n return A;\n}\n\n\nRef OpDispatchBuilder::LoadSource_WithOpSize(RegClass Class, const X86Tables::DecodedOp& Op, const X86Tables::DecodedOperand& Operand,\n IR::OpSize OpSize, uint32_t Flags, const LoadSourceOptions& Options) {\n auto [Align, LoadData, ForceLoad, AccessType, AllowUpperGarbage] = Options;\n AddressMode A = DecodeAddress(Op, Operand, AccessType, true /* IsLoad */);\n\n if (Operand.IsGPR()) {\n const auto gpr = Operand.Data.GPR.GPR;\n const auto highIndex = Operand.Data.GPR.HighBits ? 1 : 0;\n\n if (gpr >= FEXCore::X86State::REG_MM_0) {\n LOGMAN_THROW_A_FMT(OpSize == OpSize::i64Bit, \"full\");\n\n if (MMXState != MMXState_MMX) {\n ChgStateX87_MMX();\n }\n\n A.Base = LoadContext(OpSize::i64Bit, MM0Index + gpr - FEXCore::X86State::REG_MM_0);\n } else if (gpr >= FEXCore::X86State::REG_XMM_0) {\n const auto gprIndex = gpr - X86State::REG_XMM_0;\n\n // Load the full register size if it is a XMM register source.\n A.Base = LoadXMMRegister(gprIndex);\n\n // Now extract the subregister if it was a partial load /smaller/ than SSE size\n // TODO: Instead of doing the VMov implicitly on load, hunt down all use cases that require partial loads and do it after load.\n // We don't have information here to know if the operation needs zero upper bits or can contain data.\n if (!AllowUpperGarbage && OpSize < OpSize::i128Bit) {\n A.Base = _VMov(OpSize, A.Base);\n }\n } else {\n A.Base = LoadGPRRegister(gpr, OpSize, highIndex ? 8 : 0, AllowUpperGarbage);\n }\n }\n\n if ((IsOperandMem(Operand, true) && LoadData) || ForceLoad) {\n if (OpSize == OpSize::f80Bit) {\n Ref MemSrc = LoadEffectiveAddress(this, A, GetGPROpSize(), true);\n if (CTX->HostFeatures.SupportsSVE128 || CTX->HostFeatures.SupportsSVE256) {\n return _LoadMemX87SVEOptPredicate(OpSize::i128Bit, OpSize::i16Bit, MemSrc);\n } else {\n // For X87 extended doubles, Split the load.\n auto Res = _LoadMem(Class, OpSize::i64Bit, MemSrc, Align == OpSize::iInvalid ? OpSize : Align);\n return _VLoadVectorElement(OpSize::i128Bit, OpSize::i16Bit, Res, 4, Add(OpSize::i64Bit, MemSrc, 8));\n }\n }\n\n return _LoadMemAutoTSO(Class, OpSize, A, Align == OpSize::iInvalid ? OpSize : Align);\n } else {\n return LoadEffectiveAddress(this, A, GetGPROpSize(), false, AllowUpperGarbage);\n }\n}\n\nRef OpDispatchBuilder::LoadGPRRegister(uint32_t GPR, IR::OpSize Size, uint8_t Offset, bool AllowUpperGarbage) {\n const auto GPRSize = GetGPROpSize();\n if (Size == OpSize::iInvalid) {\n Size = GPRSize;\n }\n Ref Reg = LoadGPR(GPR);\n\n if ((!AllowUpperGarbage && (Size != GPRSize)) || Offset != 0) {\n // Extract the subregister if requested.\n const auto OpSize = std::max(OpSize::i32Bit, Size);\n if (AllowUpperGarbage) {\n Reg = _Lshr(OpSize, Reg, Constant(Offset));\n } else {\n Reg = _Bfe(OpSize, IR::OpSizeAsBits(Size), Offset, Reg);\n }\n }\n return Reg;\n}\n\nvoid OpDispatchBuilder::StoreGPRRegister(uint32_t GPR, const Ref Src, IR::OpSize Size, uint8_t Offset) {\n const auto GPRSize = GetGPROpSize();\n if (Size == OpSize::iInvalid) {\n Size = GPRSize;\n }\n\n Ref Reg = Src;\n if (Size != GPRSize || Offset != 0) {\n // Need to do an insert if not automatic size or zero offset.\n Reg = ARef(Reg).BfiInto(LoadGPRRegister(GPR), Offset, IR::OpSizeAsBits(Size));\n }\n\n StoreRegister(GPR, false, Reg);\n}\n\nvoid OpDispatchBuilder::StoreXMMRegister(uint32_t XMM, const Ref Src) {\n StoreRegister(XMM, true, Src);\n}\n\nRef OpDispatchBuilder::LoadSource(RegClass Class, const X86Tables::DecodedOp& Op, const X86Tables::DecodedOperand& Operand, uint32_t Flags,\n const LoadSourceOptions& Options) {\n const auto OpSize = OpSizeFromSrc(Op);\n return LoadSource_WithOpSize(Class, Op, Operand, OpSize, Flags, Options);\n}\n\nvoid OpDispatchBuilder::StoreResult_WithOpSize(RegClass Class, FEXCore::X86Tables::DecodedOp Op, const X86Tables::DecodedOperand& Operand,\n Ref Src, IR::OpSize OpSize, IR::OpSize Align, MemoryAccessType AccessType) {\n if (Operand.IsGPR()) {\n // 8Bit and 16bit destination types store their result without effecting the upper bits\n // 32bit ops ZEXT the result to 64bit\n const auto GPRSize = GetGPROpSize();\n\n const auto gpr = Operand.Data.GPR.GPR;\n if (gpr >= FEXCore::X86State::REG_MM_0) {\n LOGMAN_THROW_A_FMT(OpSize == OpSize::i64Bit, \"full\");\n LOGMAN_THROW_A_FMT(Class == RegClass::FPR, \"MMX is floaty\");\n\n if (MMXState != MMXState_MMX) {\n ChgStateX87_MMX();\n }\n\n uint8_t Index = MM0Index + gpr - FEXCore::X86State::REG_MM_0;\n StoreContext(Index, Src);\n RegCache.Partial |= (1ull << (uint64_t)Index);\n } else if (gpr >= FEXCore::X86State::REG_XMM_0) {\n const auto gprIndex = gpr - X86State::REG_XMM_0;\n const auto VectorSize = GetGuestVectorLength();\n\n auto Result = Src;\n if (OpSize != VectorSize) {\n // Partial writes can come from FPRs.\n // TODO: Fix the instructions doing partial writes rather than dealing with it here.\n\n LOGMAN_THROW_A_FMT(Class != RegClass::GPR, \"Partial writes from GPR not allowed. Instruction: {}\", Op->TableInfo->Name);\n\n // XMM-size is handled in implementations.\n if (VectorSize != OpSize::i256Bit || OpSize != OpSize::i128Bit) {\n auto SrcVector = LoadXMMRegister(gprIndex);\n Result = _VInsElement(VectorSize, OpSize, 0, 0, SrcVector, Src);\n }\n }\n\n StoreXMMRegister(gprIndex, Result);\n } else {\n if (GPRSize == OpSize::i64Bit && OpSize == OpSize::i32Bit) {\n // If the Source IR op is 64 bits, we need to zext the upper bits\n // For all other sizes, the upper bits are guaranteed to already be zero\n Ref Value = GetOpSize(Src) == OpSize::i64Bit ? ARef(Src).Bfe(0, 32).Ref() : Src;\n StoreGPRRegister(gpr, Value, GPRSize);\n\n LOGMAN_THROW_A_FMT(!Operand.Data.GPR.HighBits, \"Can't handle 32bit store to high 8bit register\");\n } else {\n LOGMAN_THROW_A_FMT(!(GPRSize == OpSize::i32Bit && OpSize > OpSize::i32Bit), \"Oops had a {} GPR load\", OpSize);\n\n if (GPRSize != OpSize) {\n // if the GPR isn't the full size then we need to insert.\n // eg:\n // mov al, 2 ; Move in to lower 8-bits.\n // mov ah, 2 ; Move in to upper 8-bits of 16-bit reg.\n // mov ax, 2 ; Move in to lower 16-bits of reg.\n StoreGPRRegister(gpr, Src, OpSize, Operand.Data.GPR.HighBits * 8);\n } else {\n StoreGPRRegister(gpr, Src, std::min(GPRSize, OpSize));\n }\n }\n }\n return;\n }\n\n AddressMode A = DecodeAddress(Op, Operand, AccessType, false /* IsLoad */);\n\n if (OpSize == OpSize::f80Bit) {\n Ref MemStoreDst = LoadEffectiveAddress(this, A, GetGPROpSize(), true);\n if (CTX->HostFeatures.SupportsSVE128 || CTX->HostFeatures.SupportsSVE256) {\n _StoreMemX87SVEOptPredicate(OpSize::i128Bit, OpSize::i16Bit, Src, MemStoreDst);\n } else {\n // For X87 extended doubles, split before storing\n _StoreMemFPR(OpSize::i64Bit, MemStoreDst, Src, Align);\n auto Upper = _VExtractToGPR(OpSize::i128Bit, OpSize::i64Bit, Src, 1);\n _StoreMemGPR(OpSize::i16Bit, Upper, MemStoreDst, Constant(8), std::min(Align, OpSize::i64Bit), MemOffsetType::SXTX, 1);\n }\n } else {\n _StoreMemAutoTSO(Class, OpSize, A, Src, Align == OpSize::iInvalid ? OpSize : Align);\n }\n}\n\nvoid OpDispatchBuilder::StoreResult(RegClass Class, X86Tables::DecodedOp Op, const X86Tables::DecodedOperand& Operand, Ref Src,\n IR::OpSize Align, MemoryAccessType AccessType) {\n StoreResult_WithOpSize(Class, Op, Operand, Src, OpSizeFromDst(Op), Align, AccessType);\n}\n\nvoid OpDispatchBuilder::StoreResult(RegClass Class, X86Tables::DecodedOp Op, Ref Src, IR::OpSize Align, MemoryAccessType AccessType) {\n StoreResult(Class, Op, Op->Dest, Src, Align, AccessType);\n}\n\nOpDispatchBuilder::OpDispatchBuilder(FEXCore::Context::ContextImpl* ctx)\n : IREmitter {ctx->OpDispatcherAllocator, ctx->HostFeatures.SupportsTSOImm9}\n , CTX {ctx} {\n if (CTX->HostFeatures.SupportsAVX && CTX->HostFeatures.SupportsSVE256) {\n SaveAVXStateFunc = &OpDispatchBuilder::SaveAVXState;\n RestoreAVXStateFunc = &OpDispatchBuilder::RestoreAVXState;\n DefaultAVXStateFunc = &OpDispatchBuilder::DefaultAVXState;\n } else if (CTX->HostFeatures.SupportsAVX) {\n SaveAVXStateFunc = &OpDispatchBuilder::AVX128_SaveAVXState;\n RestoreAVXStateFunc = &OpDispatchBuilder::AVX128_RestoreAVXState;\n DefaultAVXStateFunc = &OpDispatchBuilder::AVX128_DefaultAVXState;\n }\n}\n\nvoid OpDispatchBuilder::ResetWorkingList() {\n IREmitter::ReownOrClaimBuffer();\n\n JumpTargets.clear();\n BlockSetRIP = false;\n DecodeFailure = false;\n ShouldDump = false;\n CurrentCodeBlock = nullptr;\n RegCache.Written = 0;\n RegCache.Cached = 0;\n}\n\nvoid OpDispatchBuilder::UnhandledOp(OpcodeArgs) {\n DecodeFailure = true;\n}\n\nvoid OpDispatchBuilder::MOVGPROp(OpcodeArgs, uint32_t SrcIndex) {\n // StoreResult will store with the same size as the input, so we allow upper\n // garbage on the input. The zero extension would be pointless.\n Ref Src = LoadSourceGPR(Op, Op->Src[SrcIndex], Op->Flags, {.Align = OpSize::i8Bit, .AllowUpperGarbage = true});\n StoreResultGPR(Op, Src, OpSize::i8Bit);\n}\n\nvoid OpDispatchBuilder::MOVGPRNTOp(OpcodeArgs) {\n Ref Src = LoadSourceGPR(Op, Op->Src[0], Op->Flags, {.Align = OpSize::i8Bit});\n StoreResultGPR(Op, Src, OpSize::i8Bit, MemoryAccessType::STREAM);\n}\n\nvoid OpDispatchBuilder::ALUOp(OpcodeArgs, FEXCore::IR::IROps ALUIROp, FEXCore::IR::IROps AtomicFetchOp, unsigned SrcIdx) {\n // On x86, the canonical way to zero a register is XOR with itself. Detect and\n // emit optimal arm64 assembly.\n if (!DestIsLockedMem(Op) && ALUIROp == FEXCore::IR::IROps::OP_XOR && Op->Dest.IsGPR() && Op->Src[SrcIdx].IsGPR() &&\n Op->Dest.Data.GPR == Op->Src[SrcIdx].Data.GPR) {\n\n // Set flags for zero result with inverted carry. We subtract an arbitrary\n // register from itself to get the zero, since `subs wzr, #0` is not\n // encodable. This is optimal and works regardless of the opsize.\n auto Zero = LoadGPR(Op->Dest.Data.GPR.GPR);\n HandleNZ00Write();\n InvalidateAF();\n CalculatePF(SubWithFlags(OpSize::i32Bit, Zero, Zero));\n CFInverted = true;\n FlushRegisterCache();\n\n // Move 0 into the register\n StoreResultGPR(Op, Constant(0));\n return;\n }\n\n auto Size = OpSizeFromDst(Op);\n auto ResultSize = Size;\n\n auto RoundedSize = Size;\n if (ALUIROp != FEXCore::IR::IROps::OP_ANDWITHFLAGS) {\n RoundedSize = std::max(OpSize::i32Bit, RoundedSize);\n }\n\n // X86 basic ALU ops just do the operation between the destination and a single source\n Ref Src = LoadSourceGPR(Op, Op->Src[SrcIdx], Op->Flags, {.AllowUpperGarbage = true});\n\n // Try to eliminate the masking after 8/16-bit operations with constants, by\n // promoting to a full size operation that preserves the upper bits.\n uint64_t Const;\n bool IsConst = IsValueConstant(WrapNode(Src), &Const);\n if (Size < OpSize::i32Bit && !DestIsLockedMem(Op) && Op->Dest.IsGPR() && !Op->Dest.Data.GPR.HighBits && IsConst &&\n (ALUIROp == IR::IROps::OP_XOR || ALUIROp == IR::IROps::OP_OR || ALUIROp == IR::IROps::OP_ANDWITHFLAGS)) {\n\n RoundedSize = ResultSize = GetGPROpSize();\n LOGMAN_THROW_A_FMT(Const < (1ull << IR::OpSizeAsBits(Size)), \"does not clobber\");\n\n // For AND, we can play the same trick but we instead need the upper bits of\n // the constant to be all-1s instead of all-0s to preserve. We also can't\n // use andwithflags in this case, since we've promoted to 64-bit so the\n // negate flag would be wrong, but using the regular logical operation path\n // instead still ends up a net win for uops.\n //\n // In the common case where the constant is of the form (1 << x) - 1, the\n // adjusted constant here will inline into the arm64 and instruction, so if\n // flags are not needed, we save an instruction overall.\n if (ALUIROp == IR::IROps::OP_ANDWITHFLAGS) {\n Src = Constant(Const | ~((1ull << IR::OpSizeAsBits(Size)) - 1));\n ALUIROp = IR::IROps::OP_AND;\n }\n }\n\n Ref Result {};\n Ref Dest {};\n\n if (DestIsLockedMem(Op)) {\n HandledLock = true;\n Ref DestMem = MakeSegmentAddress(Op, Op->Dest);\n DeriveOp(FetchOp, AtomicFetchOp, _AtomicFetchAdd(Size, Src, DestMem));\n Dest = FetchOp;\n } else {\n Dest = LoadSourceGPR(Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = true});\n }\n\n const auto OpSize = RoundedSize;\n uint64_t Mask = Size == OpSize::i64Bit ? ~0ull : ((1ull << IR::OpSizeAsBits(Size)) - 1);\n if (IsConst && Const == Mask && !DestIsLockedMem(Op) && ALUIROp == IR::IROps::OP_XOR && Size >= OpSize::i32Bit) {\n Result = _Not(OpSize, Dest);\n } else if (IsConst && Const == Mask && !DestIsLockedMem(Op) && ALUIROp == IR::IROps::OP_AND) {\n Result = Dest;\n } else {\n DeriveOp(ALUOp, ALUIROp, _AndWithFlags(OpSize, Dest, Src));\n Result = ALUOp;\n }\n\n // Flags set\n switch (ALUIROp) {\n case FEXCore::IR::IROps::OP_ADD: Result = CalculateFlags_ADD(Size, Dest, Src); break;\n case FEXCore::IR::IROps::OP_SUB: Result = CalculateFlags_SUB(Size, Dest, Src); break;\n case FEXCore::IR::IROps::OP_XOR:\n case FEXCore::IR::IROps::OP_AND:\n case FEXCore::IR::IROps::OP_OR: {\n CalculateFlags_Logical(Size, Result);\n break;\n }\n case FEXCore::IR::IROps::OP_ANDWITHFLAGS: {\n HandleNZ00Write();\n CalculatePF(Result);\n InvalidateAF();\n break;\n }\n default: break;\n }\n\n if (!DestIsLockedMem(Op)) {\n StoreResultGPR_WithOpSize(Op, Op->Dest, Result, ResultSize, OpSize::iInvalid, MemoryAccessType::DEFAULT);\n }\n}\n\nvoid OpDispatchBuilder::LSLOp(OpcodeArgs) {\n // Emulate by always returning failure, this deviates from both Linux and Windows but\n // shouldn't be depended on by anything.\n SetRFLAG(Constant(0));\n}\n\nvoid OpDispatchBuilder::INTOp(OpcodeArgs) {\n IR::BreakDefinition Reason;\n bool SetRIPToNext = false;\n\n switch (Op->OP) {\n case 0xCD: { // INT imm8\n uint8_t Literal = Op->Src[0].Literal();\n\n#ifndef _WIN32\n constexpr uint8_t SYSCALL_LITERAL = 0x80;\n if (Literal == SYSCALL_LITERAL) {\n if (Is64BitMode) [[unlikely]] {\n LogMan::Msg::EFmt(\"[Unsupported] Trying to execute 32-bit syscall from a 64-bit process.\");\n UnhandledOp(Op);\n return;\n }\n // Syscall on linux\n SyscallOp(Op, false);\n return;\n }\n#else\n constexpr uint8_t SYSCALL_LITERAL = 0x2E;\n if (Literal == SYSCALL_LITERAL) {\n // Can be used for both 64-bit and 32-bit syscalls on windows\n SyscallOp(Op, false);\n return;\n }\n#endif\n\n#ifdef ARCHITECTURE_arm64ec\n // This is used when QueryPerformanceCounter is called on recent Windows versions, it causes CNTVCT to be written into RAX.\n constexpr uint8_t GET_CNTVCT_LITERAL = 0x81;\n if (Literal == GET_CNTVCT_LITERAL) {\n StoreGPRRegister(X86State::REG_RAX, _CycleCounter(false));\n return;\n }\n#endif\n\n Reason.ErrorRegister = Literal << 3 | (0b010);\n Reason.Signal = Core::FAULT_SIGSEGV;\n // GP is raised when task-gate isn't setup to be valid\n Reason.TrapNumber = X86State::X86_TRAPNO_GP;\n Reason.si_code = 0x80;\n break;\n }\n case 0xCE: // INTO\n Reason.ErrorRegister = 0;\n Reason.Signal = Core::FAULT_SIGSEGV;\n Reason.TrapNumber = X86State::X86_TRAPNO_OF;\n Reason.si_code = 0x80;\n break;\n case 0xF1: // INT1\n Reason.ErrorRegister = 0;\n Reason.Signal = Core::FAULT_SIGTRAP;\n Reason.TrapNumber = X86State::X86_TRAPNO_DB;\n Reason.si_code = 1;\n SetRIPToNext = true;\n break;\n case 0xF4: { // HLT\n Reason.ErrorRegister = 0;\n Reason.Signal = Core::FAULT_SIGSEGV;\n Reason.TrapNumber = X86State::X86_TRAPNO_GP;\n Reason.si_code = 0x80;\n break;\n }\n case 0x0B: // UD2\n Reason.ErrorRegister = 0;\n Reason.Signal = Core::FAULT_SIGILL;\n Reason.TrapNumber = X86State::X86_TRAPNO_UD;\n Reason.si_code = 2;\n break;\n case 0xCC: // INT3\n Reason.ErrorRegister = 0;\n Reason.Signal = Core::FAULT_SIGTRAP;\n Reason.TrapNumber = X86State::X86_TRAPNO_BP;\n Reason.si_code = 0x80;\n SetRIPToNext = true;\n break;\n default: FEX_UNREACHABLE;\n }\n\n // Calculate flags early.\n FlushRegisterCache();\n\n const auto GPRSize = GetGPROpSize();\n\n if (SetRIPToNext) {\n BlockSetRIP = SetRIPToNext;\n\n // We want to set RIP to the next instruction after INT3/INT1\n auto NewRIP = GetRelocatedPC(Op);\n _StoreContextGPR(GPRSize, NewRIP, offsetof(FEXCore::Core::CPUState, rip));\n } else if (Op->OP != 0xCE) {\n auto NewRIP = GetRelocatedPC(Op, -Op->InstSize);\n _StoreContextGPR(GPRSize, NewRIP, offsetof(FEXCore::Core::CPUState, rip));\n }\n\n if (Op->OP == 0xCE) { // Conditional to only break if Overflow == 1\n CalculateDeferredFlags();\n\n // If condition doesn't hold then keep going\n // CondClass::FNU means OF == 0\n auto CondJump_ = CondJumpNZCV(CondClass::FNU);\n auto FalseBlock = CreateNewCodeBlockAfter(GetCurrentBlock());\n SetFalseJumpTarget(CondJump_, FalseBlock);\n SetCurrentCodeBlock(FalseBlock);\n StartNewBlock();\n\n auto NewRIP = GetRelocatedPC(Op);\n _StoreContextGPR(GPRSize, NewRIP, offsetof(FEXCore::Core::CPUState, rip));\n Break(Reason);\n\n // Make sure to start a new block after ending this one\n auto JumpTarget = CreateNewCodeBlockAfter(FalseBlock);\n SetTrueJumpTarget(CondJump_, JumpTarget);\n SetCurrentCodeBlock(JumpTarget);\n StartNewBlock();\n } else {\n BlockSetRIP = true;\n Break(Reason);\n }\n}\n\nvoid OpDispatchBuilder::TZCNT(OpcodeArgs) {\n // _FindTrailingZeroes ignores upper garbage so we don't need to mask\n Ref Src = LoadSourceGPR(Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true});\n\n Src = _FindTrailingZeroes(OpSizeFromSrc(Op), Src);\n StoreResultGPR(Op, Src);\n\n CalculateFlags_ZCNT(OpSizeFromSrc(Op), Src);\n}\n\nvoid OpDispatchBuilder::LZCNT(OpcodeArgs) {\n // _CountLeadingZeroes clears upper garbage so we don't need to mask\n Ref Src = LoadSourceGPR(Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true});\n\n auto Res = _CountLeadingZeroes(OpSizeFromSrc(Op), Src);\n StoreResultGPR(Op, Res);\n CalculateFlags_ZCNT(OpSizeFromSrc(Op), Res);\n}\n\nvoid OpDispatchBuilder::MOVBEOp(OpcodeArgs) {\n const auto GPRSize = GetGPROpSize();\n const auto SrcSize = OpSizeFromSrc(Op);\n\n Ref Src = LoadSourceGPR(Op, Op->Src[0], Op->Flags, {.Align = OpSize::i8Bit});\n\n if (DestIsMem(Op) || SrcSize != OpSize::i16Bit) {\n Src = _Rev(SrcSize, Src);\n StoreResultGPR(Op, Op->Dest, Src);\n } else {\n Src = _Rev(std::max(OpSize::i32Bit, SrcSize), Src);\n // 16-bit does an insert.\n // Rev of 16-bit value as 32-bit replaces the result in the upper 16-bits of the result.\n // bfxil the 16-bit result in to the GPR.\n Ref Dest = LoadSourceGPR_WithOpSize(Op, Op->Dest, GPRSize, Op->Flags);\n auto Result = _Bfxil(GPRSize, 16, 16, Dest, Src);\n StoreResultGPR_WithOpSize(Op, Op->Dest, Result, GPRSize);\n }\n}\n\nvoid OpDispatchBuilder::CLWBOrTPause(OpcodeArgs) {\n if (DestIsMem(Op)) {\n Ref DestMem = MakeSegmentAddress(Op, Op->Dest);\n _CacheLineClean(DestMem);\n } else {\n if (!CTX->HostFeatures.SupportsWFXT) {\n UnimplementedOp(Op);\n } else {\n auto RAX = LoadGPRRegister(X86State::REG_RAX);\n auto RDX = LoadGPRRegister(X86State::REG_RDX);\n\n // Incoming source register is unused.\n _WFET(RDX, RAX);\n\n // OF, SF, ZF, AF, PF, CF all zero.\n // CF is used if the OS deadline is set, which we don't do anything with.\n ZeroPF_AF();\n ZeroNZCV();\n }\n }\n}\n\nvoid OpDispatchBuilder::CLFLUSHOPT(OpcodeArgs) {\n Ref DestMem = MakeSegmentAddress(Op, Op->Dest);\n _CacheLineClear(DestMem, false);\n}\n\nvoid OpDispatchBuilder::LoadFenceOrXRSTOR(OpcodeArgs) {\n // 0xE8 signifies LFENCE\n if (Op->ModRM == 0xE8) {\n _Fence(FenceType::Load);\n } else {\n XRstorOpImpl(Op);\n }\n}\n\nvoid OpDispatchBuilder::MemFenceOrXSAVEOPT(OpcodeArgs) {\n if (Op->ModRM == 0xF0) {\n // 0xF0 is MFENCE\n _Fence(FenceType::LoadStore);\n } else {\n XSaveOp(Op);\n }\n}\n\nvoid OpDispatchBuilder::StoreFenceOrCLFlush(OpcodeArgs) {\n if (Op->ModRM == 0xF8) {\n // 0xF8 is SFENCE\n _Fence(FenceType::Store);\n } else {\n // This is a CLFlush\n Ref DestMem = MakeSegmentAddress(Op, Op->Dest);\n _CacheLineClear(DestMem, true);\n }\n}\n\nvoid OpDispatchBuilder::UMonitorOrCLRSSBSY(OpcodeArgs) {\n if (DestIsMem(Op) || !CTX->HostFeatures.SupportsWFXT) {\n // CLRSSBSY\n UnimplementedOp(Op);\n } else {\n // Explicit NOP implementation of umonitor.\n }\n}\n\nvoid OpDispatchBuilder::UMWaitOp(OpcodeArgs) {\n if (DestIsMem(Op) || !CTX->HostFeatures.SupportsWFXT) {\n UnimplementedOp(Op);\n } else {\n // Explicit NOP implementation of umwait.\n // Still zero flags.\n //\n // OF, SF, ZF, AF, PF, CF all zero.\n ZeroPF_AF();\n ZeroNZCV();\n }\n}\n\nvoid OpDispatchBuilder::CLZeroOp(OpcodeArgs) {\n if (!CTX->HostFeatures.SupportsCLZERO) {\n UnimplementedOp(Op);\n return;\n }\n Ref DestMem = LoadSourceGPR(Op, Op->Src[0], Op->Flags, {.LoadData = false});\n _CacheLineZero(DestMem);\n}\n\nvoid OpDispatchBuilder::Prefetch(OpcodeArgs, bool ForStore, bool Stream, uint8_t Level) {\n Ref DestMem = LoadSourceGPR(Op, Op->Src[0], Op->Flags, {.LoadData = false});\n _Prefetch(ForStore, Stream, Level, DestMem, Invalid(), MemOffsetType::SXTX, 1);\n}\n\nvoid OpDispatchBuilder::RDTSCPOp(OpcodeArgs) {\n // RDTSCP is slightly different than RDTSC\n // IA32_TSC_AUX is returned in RCX\n // All previous loads are globally visible\n // - Explicitly does not wait for stores to be globally visible\n // - Explicitly use an MFENCE before this instruction if you want this behaviour\n // This instruction is not an execution fence, so subsequent instructions can execute after this\n // - Explicitly use an LFENCE after RDTSCP if you want to block this behaviour\n\n auto Counter = CycleCounter(true);\n\n auto ID = _ProcessorID();\n StoreGPRRegister(X86State::REG_RAX, Counter.CounterLow);\n StoreGPRRegister(X86State::REG_RCX, ID);\n StoreGPRRegister(X86State::REG_RDX, Counter.CounterHigh);\n}\n\nvoid OpDispatchBuilder::RDPIDOp(OpcodeArgs) {\n StoreResultGPR(Op, _ProcessorID());\n}\n\nvoid OpDispatchBuilder::CRC32(OpcodeArgs) {\n if (!CTX->HostFeatures.SupportsCRC) {\n UnimplementedOp(Op);\n return;\n }\n const auto GPRSize = GetGPROpSize();\n\n // Destination GPR size is always 4 or 8 bytes depending on widening\n const auto DstSize = Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_REX_WIDENING ? OpSize::i64Bit : OpSize::i32Bit;\n Ref Dest = LoadSourceGPR_WithOpSize(Op, Op->Dest, GPRSize, Op->Flags);\n\n // Incoming memory is 8, 16, 32, or 64\n Ref Src {};\n if (Op->Src[0].IsGPR()) {\n Src = LoadSourceGPR_WithOpSize(Op, Op->Src[0], GPRSize, Op->Flags);\n } else {\n Src = LoadSourceGPR(Op, Op->Src[0], Op->Flags, {.Align = OpSize::i8Bit});\n }\n auto Result = _CRC32(Dest, Src, OpSizeFromSrc(Op));\n StoreResultGPR_WithOpSize(Op, Op->Dest, Result, DstSize);\n}\n\ntemplate\nvoid OpDispatchBuilder::RDRANDOp(OpcodeArgs) {\n if (!CTX->HostFeatures.SupportsRAND) {\n UnimplementedOp(Op);\n return;\n }\n\n StoreResultGPR(Op, _RDRAND(Reseed));\n\n // If the rng number is valid then NZCV is 0b0000, otherwise NZCV is 0b0100\n auto CF_inv = GetRFLAG(X86State::RFLAG_ZF_RAW_LOC);\n\n // OF, SF, ZF, AF, PF all zero. CF indicates if valid.\n ZeroPF_AF();\n\n if (!CTX->HostFeatures.SupportsFlagM) {\n ZeroNZCV();\n SetCFInverted(CF_inv);\n } else {\n // Accelerated path. Invalid is 0 or 1, so set NZCV with a single rmif.\n HandleNZCVWrite();\n _RmifNZCV(CF_inv, (64 - 1) /* rotate bit 0 into bit 1 = C */, 0xf);\n CFInverted = true;\n }\n}\n\ntemplate void OpDispatchBuilder::RDRANDOp(OpcodeArgs);\ntemplate void OpDispatchBuilder::RDRANDOp(OpcodeArgs);\n\nvoid OpDispatchBuilder::BreakOp(OpcodeArgs, FEXCore::IR::BreakDefinition BreakDefinition) {\n const auto GPRSize = GetGPROpSize();\n\n // We don't actually support this instruction\n // Multiblock may hit it though\n _StoreContextGPR(GPRSize, GetRelocatedPC(Op, -Op->InstSize), offsetof(FEXCore::Core::CPUState, rip));\n Break(BreakDefinition);\n\n\n if (Multiblock) {\n auto NextBlock = CreateNewCodeBlockAfter(GetCurrentBlock());\n SetCurrentCodeBlock(NextBlock);\n StartNewBlock();\n } else {\n BlockSetRIP = true;\n }\n}\n\nvoid OpDispatchBuilder::UnimplementedOp(OpcodeArgs) {\n BreakOp(Op, FEXCore::IR::BreakDefinition {\n .ErrorRegister = 0,\n .Signal = SIGILL,\n .TrapNumber = X86State::X86_TRAPNO_UD,\n .si_code = 2, ///< ILL_ILLOPN\n });\n}\n\nvoid OpDispatchBuilder::PermissionRestrictedOp(OpcodeArgs) {\n BreakOp(Op, FEXCore::IR::BreakDefinition {\n .ErrorRegister = 0,\n .Signal = SIGSEGV,\n .TrapNumber = X86State::X86_TRAPNO_GP,\n .si_code = 0x80,\n });\n}\n\nvoid OpDispatchBuilder::InvalidOp(OpcodeArgs) {\n BreakOp(Op, FEXCore::IR::BreakDefinition {\n .ErrorRegister = 0,\n .Signal = SIGILL,\n .TrapNumber = 0,\n .si_code = 0,\n });\n}\n\nvoid OpDispatchBuilder::NoExecOp(OpcodeArgs) {\n BreakOp(Op, FEXCore::IR::BreakDefinition {\n .ErrorRegister = X86State::X86_PF_PROT | X86State::X86_PF_USER | X86State::X86_PF_INSTR,\n .Signal = Core::FAULT_SIGSEGV,\n .TrapNumber = X86State::X86_TRAPNO_PF,\n .si_code = 2, // SEGV_ACCERR\n });\n}\n\n#undef OpcodeArgs\n} // namespace FEXCore::IR\n" }, { "path": "FEXCore/Source/Interface/Core/OpcodeDispatcher.h", "content": "// SPDX-License-Identifier: MIT\n#pragma once\n\n#include \"Interface/Core/Frontend.h\"\n#include \"Interface/Core/X86Tables/X86Tables.h\"\n#include \"Interface/Core/Addressing.h\"\n#include \"Interface/Context/Context.h\"\n#include \"Interface/IR/IR.h\"\n#include \"Interface/IR/IREmitter.h\"\n#include \"Interface/IR/RegisterAllocationData.h\"\n\n#include \n#include \n#include \n#include \n\n#include \n#include \n#include \n#include \n\n#include \n#include \n#include \n#include \n#include \n#include \n\nnamespace FEXCore::IR {\nenum class VectorCompareType {\n // SSE comparisons.\n EQ_OQ = 0,\n LT_OS = 1,\n LE_OS = 2,\n UNORD_Q = 3,\n NEQ_UQ = 4,\n NLT_US = 5,\n NLE_US = 6,\n ORD_Q = 7,\n // AVX-only comparisons.\n EQ_UQ = 8,\n NGE_US = 9,\n NGT_US = 10,\n FALSE_OQ = 11,\n NEQ_OQ = 12,\n GE_OS = 13,\n GT_OS = 14,\n TRUE_UQ = 15,\n EQ_OS = 16,\n LT_OQ = 17,\n LE_OQ = 18,\n UNORD_S = 19,\n NEQ_US = 20,\n NLT_UQ = 21,\n NLE_UQ = 22,\n ORD_S = 23,\n EQ_US = 24,\n NGE_UQ = 25,\n NGT_UQ = 26,\n FALSE_OS = 27,\n NEQ_OS = 28,\n GE_OQ = 29,\n GT_OQ = 30,\n TRUE_US = 31,\n};\n\nenum class MemoryAccessType {\n // Choose TSO or Non-TSO depending on access type\n DEFAULT,\n // TSO access behaviour\n TSO,\n // Non-TSO access behaviour\n NONTSO,\n // Non-temporal streaming\n STREAM,\n};\n\nenum class BTAction {\n BTNone,\n BTClear,\n BTSet,\n BTComplement,\n};\n\nenum class ForceTSOMode {\n NoOverride,\n ForceDisabled,\n ForceEnabled,\n};\n\nstruct LoadSourceOptions {\n // Alignment of the load in bytes. iInvalid signifies opsize aligned.\n IR::OpSize Align = OpSize::iInvalid;\n\n // Whether or not to load the data if a memory access occurs.\n // If set to false, then the address that would have been loaded from\n // will be returned instead.\n //\n // Note: If returning the address, make sure to apply the segment offset\n // after with AppendSegmentOffset().\n //\n bool LoadData = true;\n\n // Use to force a load even if the underlying type isn't loadable.\n bool ForceLoad = false;\n\n // Specifies the access type of the load.\n MemoryAccessType AccessType = MemoryAccessType::DEFAULT;\n\n // Whether or not a zero extend should clear the upper bits\n // in the register (e.g. an 8-bit load would clear the upper 24 bits\n // or 56 bits depending on the operating mode).\n // If true, no zero-extension occurs.\n bool AllowUpperGarbage = false;\n};\n\nstruct DispatchTableEntry {\n uint16_t Op;\n uint8_t Count;\n X86Tables::OpDispatchPtr Ptr;\n};\n\nclass OpDispatchBuilder final : public IREmitter {\npublic:\n Ref GetNewJumpBlock(uint64_t RIP) {\n auto it = JumpTargets.find(RIP);\n LOGMAN_THROW_A_FMT(it != JumpTargets.end(), \"Couldn't find block generated for 0x{:x}\", RIP);\n return it->second.BlockEntry;\n }\n\n void SetNewBlockIfChanged(uint64_t RIP) {\n auto it = JumpTargets.find(RIP);\n if (it == JumpTargets.end()) {\n return;\n }\n\n it->second.HaveEmitted = true;\n\n if (CurrentCodeBlock->Wrapped(DualListData.ListBegin()).ID() == it->second.BlockEntry->Wrapped(DualListData.ListBegin()).ID()) {\n return;\n }\n\n // We have hit a RIP that is a jump target\n // Thus we need to end up in a new block\n SetCurrentCodeBlock(it->second.BlockEntry);\n }\n\n void StartNewBlock() {\n // If we loaded flags but didn't change them, invalidate the cached copy and move on.\n // Changes get stored out by CalculateDeferredFlags.\n CachedNZCV = nullptr;\n CFInverted = CFInvertedABI;\n\n FlushRegisterCache();\n\n // Start block in X87 state.\n // This is important to ensure that blocks always start with the same state independently of predecessors\n // which allows independent compilation of blocks.\n // Starting in the X87 state is better than starting in MMX state because\n // MMX state is more work to initialize.\n MMXState = MMXState_X87;\n\n // New block needs to reset segment telemetry.\n SegmentsNeedReadCheck = ~0U;\n\n // Need to clear any named constants that were cached.\n ClearCachedNamedConstants();\n }\n\n IRPair Jump() {\n FlushRegisterCache();\n return _Jump();\n }\n IRPair Jump(Ref _TargetBlock) {\n FlushRegisterCache();\n return _Jump(_TargetBlock);\n }\n IRPair CondJump(Ref _Cmp1, Ref _Cmp2, Ref _TrueBlock, Ref _FalseBlock, CondClass _Cond = CondClass::NEQ,\n IR::OpSize _CompareSize = OpSize::iInvalid) {\n FlushRegisterCache();\n return _CondJump(_Cmp1, _Cmp2, _TrueBlock, _FalseBlock, _Cond, _CompareSize);\n }\n IRPair CondJump(Ref ssa0, CondClass cond = CondClass::NEQ) {\n FlushRegisterCache();\n return _CondJump(ssa0, cond);\n }\n IRPair CondJump(Ref ssa0, Ref ssa1, Ref ssa2, CondClass cond = CondClass::NEQ) {\n FlushRegisterCache();\n return _CondJump(ssa0, ssa1, ssa2, cond);\n }\n IRPair CondJumpNZCV(CondClass Cond) {\n FlushRegisterCache();\n return _CondJump(InvalidNode, InvalidNode, InvalidNode, InvalidNode, Cond, OpSize::iInvalid, true);\n }\n IRPair CondJumpBit(Ref Src, unsigned Bit, bool Set) {\n FlushRegisterCache();\n auto InlineConst = _InlineConstant(Bit);\n auto Cond = Set ? CondClass::TSTNZ : CondClass::TSTZ;\n return _CondJump(Src, InlineConst, InvalidNode, InvalidNode, Cond, OpSize::iInvalid, false);\n }\n IRPair ExitFunction(Ref NewRIP, BranchHint Hint = BranchHint::None) {\n FlushRegisterCache();\n return _ExitFunction(GetOpSize(NewRIP), NewRIP, Hint, InvalidNode, InvalidNode);\n }\n IRPair ExitFunction(Ref NewRIP, BranchHint Hint, Ref CallReturnAddress, Ref CallReturnBlock) {\n FlushRegisterCache();\n return _ExitFunction(GetOpSize(NewRIP), NewRIP, Hint, CallReturnAddress, CallReturnBlock);\n }\n IRPair Break(BreakDefinition Reason) {\n FlushRegisterCache();\n return _Break(Reason);\n }\n IRPair Thunk(Ref ArgPtr, SHA256Sum ThunkNameHash) {\n FlushRegisterCache();\n return _Thunk(ArgPtr, ThunkNameHash);\n }\n\n bool FinishOp(uint64_t NextRIP, bool LastOp) {\n // If we are switching to a new block and this current block has yet to set a RIP\n // Then we need to insert an unconditional jump from the current block to the one we are going to\n // This happens most frequently when an instruction jumps backwards to another location\n // eg:\n //\n // nop dword [rax], eax\n // .label:\n // rdi, 0x8\n // cmp qword [rdi-8], 0\n // jne .label\n if (LastOp && !BlockSetRIP) {\n auto it = JumpTargets.find(NextRIP);\n if (it == JumpTargets.end()) {\n\n const auto GPRSize = GetGPROpSize();\n // If we don't have a jump target to a new block then we have to leave\n // Set the RIP to the next instruction and leave\n ExitFunction(_InlineEntrypointOffset(GPRSize, NextRIP - Entry));\n } else if (it != JumpTargets.end()) {\n Jump(it->second.BlockEntry);\n return true;\n }\n }\n\n BlockSetRIP = false;\n\n return false;\n }\n\n static bool CanHaveSideEffects(const FEXCore::X86Tables::X86InstInfo* TableInfo, FEXCore::X86Tables::DecodedOp Op) {\n if (TableInfo) {\n if (TableInfo->Flags & X86Tables::InstFlags::FLAGS_DEBUG_MEM_ACCESS) {\n // If it is marked as having memory access then always say it has a side-effect.\n // Not always true but better to be safe.\n return true;\n }\n\n if (TableInfo->Flags & (X86Tables::InstFlags::FLAGS_SETS_RIP | X86Tables::InstFlags::FLAGS_BLOCK_END)) {\n // Cooperative suspend interrupts can be triggered at any back-edge, the RIP must be reconstructed correctly in such cases\n return true;\n }\n }\n\n auto CanHaveSideEffects = false;\n\n auto HasPotentialMemoryAccess = [](const X86Tables::DecodedOperand& Operand) -> bool {\n if (Operand.IsNone()) {\n return false;\n }\n\n // This isn't guaranteed that all of these types will access memory, but be safe.\n return Operand.IsGPRDirect() || Operand.IsGPRIndirect() || Operand.IsRIPRelative() || Operand.IsSIB();\n };\n\n CanHaveSideEffects |= HasPotentialMemoryAccess(Op->Dest);\n CanHaveSideEffects |= HasPotentialMemoryAccess(Op->Src[0]);\n CanHaveSideEffects |= HasPotentialMemoryAccess(Op->Src[1]);\n CanHaveSideEffects |= HasPotentialMemoryAccess(Op->Src[2]);\n return CanHaveSideEffects;\n }\n\n template\n void ForeachDirection(F&& Routine) {\n // Otherwise, prepare to branch.\n auto Zero = Constant(0);\n\n // If the shift is zero, do not touch the flags.\n auto ForwardBlock = CreateNewCodeBlockAfter(GetCurrentBlock());\n auto BackwardBlock = CreateNewCodeBlockAfter(ForwardBlock);\n auto ExitBlock = CreateNewCodeBlockAfter(BackwardBlock);\n\n auto DF = GetRFLAG(X86State::RFLAG_DF_RAW_LOC);\n CondJump(DF, Zero, ForwardBlock, BackwardBlock, CondClass::EQ);\n\n for (auto D = 0; D < 2; ++D) {\n SetCurrentCodeBlock(D ? BackwardBlock : ForwardBlock);\n StartNewBlock();\n {\n Routine(D ? -1 : 1);\n Jump(ExitBlock);\n }\n }\n\n SetCurrentCodeBlock(ExitBlock);\n StartNewBlock();\n }\n\n OpDispatchBuilder(FEXCore::Context::ContextImpl* ctx);\n\n // Should only be called at the start of IR Emission.\n void ResetWorkingList();\n\n void ResetDecodeFailure() {\n NeedsBlockEnd = DecodeFailure = false;\n }\n bool HadDecodeFailure() const {\n return DecodeFailure;\n }\n bool NeedsBlockEnder() const {\n return NeedsBlockEnd;\n }\n\n void ResetHandledLock() {\n HandledLock = false;\n }\n bool HasHandledLock() const {\n return HandledLock;\n }\n\n void SetForceTSO(ForceTSOMode Mode) {\n ForceTSO = Mode;\n }\n ForceTSOMode GetForceTSO() const {\n return ForceTSO;\n }\n\n void SetDumpIR(bool DumpIR) {\n ShouldDump = DumpIR;\n }\n bool ShouldDumpIR() const {\n return ShouldDump;\n }\n\n void BeginFunction(uint64_t RIP, const fextl::vector* Blocks, uint32_t NumInstructions,\n bool Is64BitMode, bool MonoBackpatcherBlock);\n void Finalize();\n\n // Dispatch builder functions\n#define OpcodeArgs [[maybe_unused]] FEXCore::X86Tables::DecodedOp Op\n\n /**\n * Binds a sequence of compile-time constants as arguments to another member function.\n * This allows to construct OpDispatchPtrs that are specialized for the given set of arguments.\n */\n template\n void Bind(OpcodeArgs) {\n [[clang::noinline]] (this->*Fn)(Op, Args...);\n };\n\n void UnhandledOp(OpcodeArgs);\n void MOVGPROp(OpcodeArgs, uint32_t SrcIndex);\n void MOVGPRNTOp(OpcodeArgs);\n void MOVVectorAlignedOp(OpcodeArgs);\n void MOVVectorUnalignedOp(OpcodeArgs);\n void MOVVectorNTOp(OpcodeArgs);\n void ALUOp(OpcodeArgs, FEXCore::IR::IROps ALUIROp, FEXCore::IR::IROps AtomicFetchOp, unsigned SrcIdx);\n void LSLOp(OpcodeArgs);\n void INTOp(OpcodeArgs);\n void SyscallOp(OpcodeArgs, bool IsSyscallInst);\n void ThunkOp(OpcodeArgs);\n void LEAOp(OpcodeArgs);\n void NOPOp(OpcodeArgs);\n void RETOp(OpcodeArgs);\n void IRETOp(OpcodeArgs);\n void CallbackReturnOp(OpcodeArgs);\n void SecondaryALUOp(OpcodeArgs);\n void ADCOp(OpcodeArgs, uint32_t SrcIndex);\n void SBBOp(OpcodeArgs, uint32_t SrcIndex);\n void SALCOp(OpcodeArgs);\n void PUSHOp(OpcodeArgs);\n void PUSHREGOp(OpcodeArgs);\n void PUSHAOp(OpcodeArgs);\n void PUSHSegmentOp(OpcodeArgs, uint32_t SegmentReg);\n void POPOp(OpcodeArgs);\n void POPAOp(OpcodeArgs);\n void POPSegmentOp(OpcodeArgs, uint32_t SegmentReg);\n void LEAVEOp(OpcodeArgs);\n void CALLOp(OpcodeArgs);\n void CALLAbsoluteOp(OpcodeArgs);\n void CondJUMPOp(OpcodeArgs);\n void CondJUMPRCXOp(OpcodeArgs);\n void LoopOp(OpcodeArgs);\n void JUMPOp(OpcodeArgs);\n void JUMPAbsoluteOp(OpcodeArgs);\n void JUMPFARIndirectOp(OpcodeArgs);\n void CALLFARIndirectOp(OpcodeArgs);\n void RETFARIndirectOp(OpcodeArgs);\n void TESTOp(OpcodeArgs, uint32_t SrcIndex);\n void ARPLOp(OpcodeArgs);\n void MOVSXDOp(OpcodeArgs);\n void MOVSXOp(OpcodeArgs);\n void MOVZXOp(OpcodeArgs);\n void CMPOp(OpcodeArgs, uint32_t SrcIndex);\n void SETccOp(OpcodeArgs);\n void CQOOp(OpcodeArgs);\n void CDQOp(OpcodeArgs);\n void XCHGOp(OpcodeArgs);\n void SAHFOp(OpcodeArgs);\n void LAHFOp(OpcodeArgs);\n void MOVSegOp(OpcodeArgs, bool ToSeg);\n void FLAGControlOp(OpcodeArgs);\n void MOVOffsetOp(OpcodeArgs);\n void CMOVOp(OpcodeArgs);\n void CPUIDOp(OpcodeArgs);\n void XGetBVOp(OpcodeArgs);\n uint32_t GetConstantShift(X86Tables::DecodedOp Op, bool Is1Bit);\n void SHLOp(OpcodeArgs);\n void SHLImmediateOp(OpcodeArgs, bool SHL1Bit);\n void SHROp(OpcodeArgs);\n void SHRImmediateOp(OpcodeArgs, bool SHR1Bit);\n void SHLDOp(OpcodeArgs);\n void SHLDImmediateOp(OpcodeArgs);\n void SHRDOp(OpcodeArgs);\n void SHRDImmediateOp(OpcodeArgs);\n void ASHROp(OpcodeArgs, bool IsImmediate, bool Is1Bit);\n void RotateOp(OpcodeArgs, bool Left, bool IsImmediate, bool Is1Bit);\n void RCROp1Bit(OpcodeArgs);\n void RCROp8x1Bit(OpcodeArgs);\n void RCROp(OpcodeArgs);\n void RCRSmallerOp(OpcodeArgs);\n void RCLOp1Bit(OpcodeArgs);\n void RCLOp(OpcodeArgs);\n void RCLSmallerOp(OpcodeArgs);\n\n void BTOp(OpcodeArgs, uint32_t SrcIndex, enum BTAction Action);\n\n void IMUL1SrcOp(OpcodeArgs);\n void IMUL2SrcOp(OpcodeArgs);\n void IMULOp(OpcodeArgs);\n void STOSOp(OpcodeArgs);\n void MOVSOp(OpcodeArgs);\n void CMPSOp(OpcodeArgs);\n void LODSOp(OpcodeArgs);\n void SCASOp(OpcodeArgs);\n void BSWAPOp(OpcodeArgs);\n void PUSHFOp(OpcodeArgs);\n void POPFOp(OpcodeArgs);\n\n struct CycleCounterPair {\n Ref CounterLow;\n Ref CounterHigh;\n };\n CycleCounterPair CycleCounter(bool SelfSynchronizingLoads);\n void RDTSCOp(OpcodeArgs);\n void INCOp(OpcodeArgs);\n void DECOp(OpcodeArgs);\n void NEGOp(OpcodeArgs);\n void DIVOp(OpcodeArgs);\n void IDIVOp(OpcodeArgs);\n void BSFOp(OpcodeArgs);\n void BSROp(OpcodeArgs);\n void CMPXCHGOp(OpcodeArgs);\n void CMPXCHGPairOp(OpcodeArgs);\n void MULOp(OpcodeArgs);\n void NOTOp(OpcodeArgs);\n void XADDOp(OpcodeArgs);\n void PopcountOp(OpcodeArgs);\n void DAAOp(OpcodeArgs);\n void DASOp(OpcodeArgs);\n void AAAOp(OpcodeArgs);\n void AASOp(OpcodeArgs);\n void AAMOp(OpcodeArgs);\n void AADOp(OpcodeArgs);\n void XLATOp(OpcodeArgs);\n template\n void RDRANDOp(OpcodeArgs);\n\n enum class Segment {\n FS,\n GS,\n };\n void ReadSegmentReg(OpcodeArgs, Segment Seg);\n void WriteSegmentReg(OpcodeArgs, Segment Seg);\n void EnterOp(OpcodeArgs);\n\n void SGDTOp(OpcodeArgs);\n void SIDTOp(OpcodeArgs);\n void SMSWOp(OpcodeArgs);\n\n enum class VectorOpType {\n MMX,\n SSE,\n AVX,\n };\n // SSE\n void MOVLPOp(OpcodeArgs);\n void MOVHPDOp(OpcodeArgs);\n void MOVSDOp(OpcodeArgs);\n void MOVSSOp(OpcodeArgs);\n void VectorALUOp(OpcodeArgs, IROps IROp, IR::OpSize ElementSize);\n void VectorXOROp(OpcodeArgs);\n\n void VectorALUROp(OpcodeArgs, IROps IROp, IR::OpSize ElementSize);\n void VectorUnaryOp(OpcodeArgs, IROps IROp, IR::OpSize ElementSize);\n void RSqrt3DNowOp(OpcodeArgs, bool Duplicate);\n template\n void VectorUnaryDuplicateOp(OpcodeArgs);\n\n void MOVQOp(OpcodeArgs, VectorOpType VectorType);\n void MOVQMMXOp(OpcodeArgs);\n void MOVMSKOp(OpcodeArgs, IR::OpSize ElementSize);\n void MOVMSKOpOne(OpcodeArgs);\n void PUNPCKLOp(OpcodeArgs, IR::OpSize ElementSize);\n void PUNPCKHOp(OpcodeArgs, IR::OpSize ElementSize);\n void PSHUFBOp(OpcodeArgs);\n Ref PShufWLane(IR::OpSize Size, FEXCore::IR::IndexNamedVectorConstant IndexConstant, bool LowLane, Ref IncomingLane, uint8_t Shuffle);\n void PSHUFWOp(OpcodeArgs, bool Low);\n void PSHUFW8ByteOp(OpcodeArgs);\n void PSHUFDOp(OpcodeArgs);\n void PSRLDOp(OpcodeArgs, IR::OpSize ElementSize);\n void PSRLI(OpcodeArgs, IR::OpSize ElementSize);\n void PSLLI(OpcodeArgs, IR::OpSize ElementSize);\n void PSLL(OpcodeArgs, IR::OpSize ElementSize);\n void PSRAOp(OpcodeArgs, IR::OpSize ElementSize);\n void PSRLDQ(OpcodeArgs);\n void PSLLDQ(OpcodeArgs);\n void PSRAIOp(OpcodeArgs, IR::OpSize ElementSize);\n void MOVDDUPOp(OpcodeArgs);\n template\n void CVTGPR_To_FPR(OpcodeArgs);\n template\n void CVTFPR_To_GPR(OpcodeArgs);\n template\n void Vector_CVT_Int_To_Float(OpcodeArgs);\n template\n void Scalar_CVT_Float_To_Float(OpcodeArgs);\n void Vector_CVT_Float_To_Float(OpcodeArgs, IR::OpSize DstElementSize, IR::OpSize SrcElementSize, bool IsAVX);\n template\n void Vector_CVT_Float_To_Int(OpcodeArgs);\n void MMX_To_XMM_Vector_CVT_Int_To_Float(OpcodeArgs);\n template\n void XMM_To_MMX_Vector_CVT_Float_To_Int(OpcodeArgs);\n void MASKMOVOp(OpcodeArgs);\n void MOVBetweenGPR_FPR(OpcodeArgs, VectorOpType VectorType);\n void TZCNT(OpcodeArgs);\n void LZCNT(OpcodeArgs);\n template\n void VFCMPOp(OpcodeArgs);\n void SHUFOp(OpcodeArgs, IR::OpSize ElementSize);\n template\n void PINSROp(OpcodeArgs);\n void InsertPSOp(OpcodeArgs);\n void PExtrOp(OpcodeArgs, IR::OpSize ElementSize);\n\n template\n void PSIGN(OpcodeArgs);\n template\n void VPSIGN(OpcodeArgs);\n\n // BMI1 Ops\n void ANDNBMIOp(OpcodeArgs);\n void BEXTRBMIOp(OpcodeArgs);\n void BLSIBMIOp(OpcodeArgs);\n void BLSMSKBMIOp(OpcodeArgs);\n void BLSRBMIOp(OpcodeArgs);\n\n // BMI2 Ops\n void BMI2Shift(OpcodeArgs);\n void BZHI(OpcodeArgs);\n void MULX(OpcodeArgs);\n void PDEP(OpcodeArgs);\n void PEXT(OpcodeArgs);\n void RORX(OpcodeArgs);\n\n // ADX Ops\n void ADXOp(OpcodeArgs);\n\n // AVX Ops\n void AVXVectorXOROp(OpcodeArgs);\n\n template\n void AVXVectorRound(OpcodeArgs);\n\n template\n void AVXScalar_CVT_Float_To_Float(OpcodeArgs);\n\n template\n void VectorScalarInsertALUOp(OpcodeArgs);\n template\n void AVXVectorScalarInsertALUOp(OpcodeArgs);\n\n template\n void VectorScalarUnaryInsertALUOp(OpcodeArgs);\n template\n void AVXVectorScalarUnaryInsertALUOp(OpcodeArgs);\n\n void InsertMMX_To_XMM_Vector_CVT_Int_To_Float(OpcodeArgs);\n template\n void InsertCVTGPR_To_FPR(OpcodeArgs);\n template\n void AVXInsertCVTGPR_To_FPR(OpcodeArgs);\n\n template\n void InsertScalar_CVT_Float_To_Float(OpcodeArgs);\n template\n void AVXInsertScalar_CVT_Float_To_Float(OpcodeArgs);\n\n RoundMode TranslateRoundType(uint8_t Mode);\n\n template\n void InsertScalarRound(OpcodeArgs);\n template\n void AVXInsertScalarRound(OpcodeArgs);\n\n template\n void InsertScalarFCMPOp(OpcodeArgs);\n template\n void AVXInsertScalarFCMPOp(OpcodeArgs);\n\n template\n void AVXCVTGPR_To_FPR(OpcodeArgs);\n\n template\n void AVXVFCMPOp(OpcodeArgs);\n\n template\n void VADDSUBPOp(OpcodeArgs);\n\n void VAESDecOp(OpcodeArgs);\n void VAESDecLastOp(OpcodeArgs);\n void VAESEncOp(OpcodeArgs);\n void VAESEncLastOp(OpcodeArgs);\n\n void VANDNOp(OpcodeArgs);\n\n Ref VBLENDOpImpl(IR::OpSize VecSize, IR::OpSize ElementSize, Ref Src1, Ref Src2, Ref ZeroRegister, uint64_t Selector);\n void VBLENDPDOp(OpcodeArgs);\n void VPBLENDDOp(OpcodeArgs);\n void VPBLENDWOp(OpcodeArgs);\n\n void VBROADCASTOp(OpcodeArgs, IR::OpSize ElementSize);\n\n template\n void VDPPOp(OpcodeArgs);\n\n void VEXTRACT128Op(OpcodeArgs);\n\n template\n void VHADDPOp(OpcodeArgs);\n void VHSUBPOp(OpcodeArgs, IR::OpSize ElementSize);\n\n void VINSERTOp(OpcodeArgs);\n void VINSERTPSOp(OpcodeArgs);\n\n template\n void VMASKMOVOp(OpcodeArgs);\n\n void VMOVHPOp(OpcodeArgs);\n void VMOVLPOp(OpcodeArgs);\n\n void VMOVDDUPOp(OpcodeArgs);\n void VMOVSHDUPOp(OpcodeArgs);\n void VMOVSLDUPOp(OpcodeArgs);\n\n void VMOVSDOp(OpcodeArgs);\n void VMOVSSOp(OpcodeArgs);\n\n void VMOVAPS_VMOVAPDOp(OpcodeArgs);\n void VMOVUPS_VMOVUPDOp(OpcodeArgs);\n\n void VMPSADBWOp(OpcodeArgs);\n\n void VPACKSSOp(OpcodeArgs, IR::OpSize ElementSize);\n\n void VPACKUSOp(OpcodeArgs, IR::OpSize ElementSize);\n\n void VPALIGNROp(OpcodeArgs);\n\n void VPCMPESTRIOp(OpcodeArgs);\n void VPCMPESTRMOp(OpcodeArgs);\n void VPCMPISTRIOp(OpcodeArgs);\n void VPCMPISTRMOp(OpcodeArgs);\n\n void VCVTPH2PSOp(OpcodeArgs);\n void VCVTPS2PHOp(OpcodeArgs);\n\n Ref VPERMDIndices(OpSize DstSize, Ref Indices, Ref IndexMask, Ref Repeating3210);\n void VPERM2Op(OpcodeArgs);\n void VPERMDOp(OpcodeArgs);\n void VPERMQOp(OpcodeArgs);\n\n void VPERMILImmOp(OpcodeArgs, IR::OpSize ElementSize);\n\n Ref VPERMILRegOpImpl(OpSize DstSize, IR::OpSize ElementSize, Ref Src, Ref Indices);\n template\n void VPERMILRegOp(OpcodeArgs);\n\n void VPHADDSWOp(OpcodeArgs);\n\n void VPHSUBOp(OpcodeArgs, IR::OpSize ElementSize);\n void VPHSUBSWOp(OpcodeArgs);\n\n void VPINSRBOp(OpcodeArgs);\n void VPINSRDQOp(OpcodeArgs);\n void VPINSRWOp(OpcodeArgs);\n\n void VPMADDUBSWOp(OpcodeArgs);\n void VPMADDWDOp(OpcodeArgs);\n\n template\n void VPMASKMOVOp(OpcodeArgs);\n\n void VPMULHRSWOp(OpcodeArgs);\n\n template\n void VPMULHWOp(OpcodeArgs);\n\n template\n void VPMULLOp(OpcodeArgs);\n\n void VPSADBWOp(OpcodeArgs);\n\n void VPSHUFBOp(OpcodeArgs);\n\n void VPSHUFWOp(OpcodeArgs, IR::OpSize ElementSize, bool Low);\n\n void VPSLLOp(OpcodeArgs, IR::OpSize ElementSize);\n void VPSLLDQOp(OpcodeArgs);\n void VPSLLIOp(OpcodeArgs, IR::OpSize ElementSize);\n void VPSLLVOp(OpcodeArgs);\n\n void VPSRAOp(OpcodeArgs, IR::OpSize ElementSize);\n\n void VPSRAIOp(OpcodeArgs, IR::OpSize ElementSize);\n\n void VPSRAVDOp(OpcodeArgs);\n void VPSRLVOp(OpcodeArgs);\n\n void VPSRLDOp(OpcodeArgs, IR::OpSize ElementSize);\n void VPSRLDQOp(OpcodeArgs);\n\n void VPUNPCKHOp(OpcodeArgs, IR::OpSize ElementSize);\n\n void VPUNPCKLOp(OpcodeArgs, IR::OpSize ElementSize);\n\n void VPSRLIOp(OpcodeArgs, IR::OpSize ElementSize);\n\n void VSHUFOp(OpcodeArgs, IR::OpSize ElementSize);\n\n template\n void VTESTPOp(OpcodeArgs);\n\n void VZEROOp(OpcodeArgs);\n\n // X87 Ops\n Ref ReconstructFSW_Helper(Ref T = nullptr);\n // Returns new x87 stack top from FSW.\n Ref ReconstructX87StateFromFSW_Helper(Ref FSW);\n void FLD(OpcodeArgs, IR::OpSize Width);\n void FLDFromStack(OpcodeArgs);\n void FLD_Const(OpcodeArgs, NamedVectorConstant K);\n\n void FBLD(OpcodeArgs);\n void FBSTP(OpcodeArgs);\n\n void FILD(OpcodeArgs);\n\n void FST(OpcodeArgs, IR::OpSize Width);\n void FSTToStack(OpcodeArgs);\n\n void FIST(OpcodeArgs, bool Truncate);\n\n // OpResult is used for Stack operations,\n // describes if the result of the operation is stored in ST(0) or ST(i),\n // where ST(i) is one of the arguments to the operation.\n enum class OpResult {\n RES_ST0,\n RES_STI,\n };\n\n void FADD(OpcodeArgs, IR::OpSize Width, bool Integer, OpResult ResInST0);\n void FDIV(OpcodeArgs, IR::OpSize Width, bool Integer, bool Reverse, OpResult ResInST0);\n void FMUL(OpcodeArgs, IR::OpSize Width, bool Integer, OpResult ResInST0);\n void FNCLEX(OpcodeArgs);\n void FNINIT(OpcodeArgs);\n void FSUB(OpcodeArgs, IR::OpSize Width, bool Integer, bool Reverse, OpResult ResInST0);\n void FTST(OpcodeArgs);\n void FXCH(OpcodeArgs);\n void X87EMMS(OpcodeArgs);\n void X87FCMOV(OpcodeArgs);\n void X87FFREE(OpcodeArgs);\n void X87FLDCW(OpcodeArgs);\n void X87FNSAVE(OpcodeArgs);\n void X87FNSTENV(OpcodeArgs);\n void X87FNSTSW(OpcodeArgs);\n void X87FRSTOR(OpcodeArgs);\n void X87FSTCW(OpcodeArgs);\n void X87FXAM(OpcodeArgs);\n void X87FXTRACT(OpcodeArgs);\n void X87FYL2X(OpcodeArgs, bool IsFYL2XP1);\n void X87LDENV(OpcodeArgs);\n void X87ModifySTP(OpcodeArgs, bool Inc);\n void X87OpHelper(OpcodeArgs, FEXCore::IR::IROps IROp, bool ZeroC2);\n\n enum class FCOMIFlags {\n FLAGS_X87,\n FLAGS_RFLAGS,\n };\n void FCOMI(OpcodeArgs, IR::OpSize Width, bool Integer, FCOMIFlags WhichFlags, bool PopTwice);\n\n // F64 X87 Ops\n void FADDF64(OpcodeArgs, IR::OpSize Width, bool Integer, OpResult ResInST0);\n void FBLDF64(OpcodeArgs);\n void FBSTPF64(OpcodeArgs);\n void FCOMIF64(OpcodeArgs, IR::OpSize width, bool Integer, FCOMIFlags whichflags, bool poptwice);\n void FDIVF64(OpcodeArgs, IR::OpSize Width, bool Integer, bool Reverse, OpResult ResInST0);\n void FILDF64(OpcodeArgs);\n void FISTF64(OpcodeArgs, bool Truncate);\n void FLDF64_Const(OpcodeArgs, uint64_t Num);\n void FLDF64(OpcodeArgs, IR::OpSize Width);\n void FMULF64(OpcodeArgs, IR::OpSize Width, bool Integer, OpResult ResInST0);\n void FSUBF64(OpcodeArgs, IR::OpSize Width, bool Integer, bool Reverse, OpResult ResInST0);\n void FTSTF64(OpcodeArgs);\n void X87FLDCWF64(OpcodeArgs);\n void X87FXTRACTF64(OpcodeArgs);\n void X87LDENVF64(OpcodeArgs);\n\n void FXSaveOp(OpcodeArgs);\n void FXRStoreOp(OpcodeArgs);\n\n Ref XSaveBase(X86Tables::DecodedOp Op);\n void XSaveOp(OpcodeArgs);\n\n void PAlignrOp(OpcodeArgs);\n template\n void UCOMISxOp(OpcodeArgs);\n void LDMXCSR(OpcodeArgs);\n void STMXCSR(OpcodeArgs);\n\n template\n void PACKUSOp(OpcodeArgs);\n\n template\n void PACKSSOp(OpcodeArgs);\n\n template\n void PMULLOp(OpcodeArgs);\n\n template\n void MOVQ2DQ(OpcodeArgs);\n\n template\n void ADDSUBPOp(OpcodeArgs);\n\n void PFNACCOp(OpcodeArgs);\n void PFPNACCOp(OpcodeArgs);\n void PSWAPDOp(OpcodeArgs);\n\n template\n void VPFCMPOp(OpcodeArgs);\n void PI2FWOp(OpcodeArgs);\n void PF2IWOp(OpcodeArgs);\n\n void PMULHRWOp(OpcodeArgs);\n\n void PMADDWD(OpcodeArgs);\n void PMADDUBSW(OpcodeArgs);\n\n template\n void PMULHW(OpcodeArgs);\n\n void PMULHRSW(OpcodeArgs);\n\n void MOVBEOp(OpcodeArgs);\n template\n void HSUBP(OpcodeArgs);\n template\n void PHSUB(OpcodeArgs);\n\n void PHADDS(OpcodeArgs);\n void PHSUBS(OpcodeArgs);\n\n void CLWBOrTPause(OpcodeArgs);\n void CLFLUSHOPT(OpcodeArgs);\n void LoadFenceOrXRSTOR(OpcodeArgs);\n void MemFenceOrXSAVEOPT(OpcodeArgs);\n void StoreFenceOrCLFlush(OpcodeArgs);\n void UMonitorOrCLRSSBSY(OpcodeArgs);\n void UMWaitOp(OpcodeArgs);\n void CLZeroOp(OpcodeArgs);\n void RDTSCPOp(OpcodeArgs);\n void RDPIDOp(OpcodeArgs);\n\n void Prefetch(OpcodeArgs, bool ForStore, bool Stream, uint8_t Level);\n\n void PSADBW(OpcodeArgs);\n\n void SHA1NEXTEOp(OpcodeArgs);\n void SHA1MSG1Op(OpcodeArgs);\n void SHA1MSG2Op(OpcodeArgs);\n void SHA1RNDS4Op(OpcodeArgs);\n\n void SHA256MSG1Op(OpcodeArgs);\n void SHA256MSG2Op(OpcodeArgs);\n void SHA256RNDS2Op(OpcodeArgs);\n\n void AESImcOp(OpcodeArgs);\n void AESEncOp(OpcodeArgs);\n void AESEncLastOp(OpcodeArgs);\n void AESDecOp(OpcodeArgs);\n void AESDecLastOp(OpcodeArgs);\n void AESKeyGenAssist(OpcodeArgs);\n\n void VFMAImpl(OpcodeArgs, IROps IROp, bool Scalar, uint8_t Src1Idx, uint8_t Src2Idx, uint8_t AddendIdx);\n void VFMAddSubImpl(OpcodeArgs, bool AddSub, uint8_t Src1Idx, uint8_t Src2Idx, uint8_t AddendIdx);\n\n struct RefVSIB {\n Ref Low, High;\n Ref BaseAddr;\n int32_t Displacement;\n uint8_t Scale;\n };\n\n RefVSIB LoadVSIB(const X86Tables::DecodedOp& Op, const X86Tables::DecodedOperand& Operand, uint32_t Flags);\n template\n void VPGATHER(OpcodeArgs);\n\n template\n void ExtendVectorElements(OpcodeArgs);\n template\n void VectorRound(OpcodeArgs);\n\n Ref VectorBlend(OpSize Size, IR::OpSize ElementSize, Ref Src1, Ref Src2, uint8_t Selector);\n\n template\n void VectorBlend(OpcodeArgs);\n\n void VectorVariableBlend(OpcodeArgs, IR::OpSize ElementSize);\n void PTestOpImpl(OpSize Size, Ref Dest, Ref Src);\n void PTestOp(OpcodeArgs);\n void PHMINPOSUWOp(OpcodeArgs);\n template\n void DPPOp(OpcodeArgs);\n\n void MPSADBWOp(OpcodeArgs);\n void PCLMULQDQOp(OpcodeArgs);\n void VPCLMULQDQOp(OpcodeArgs);\n\n void CRC32(OpcodeArgs);\n void Extrq_imm(OpcodeArgs);\n void Insertq_imm(OpcodeArgs);\n void Extrq(OpcodeArgs);\n void Insertq(OpcodeArgs);\n\n void BreakOp(OpcodeArgs, FEXCore::IR::BreakDefinition BreakDefinition);\n void UnimplementedOp(OpcodeArgs);\n void PermissionRestrictedOp(OpcodeArgs);\n\n ///< Helper for PSHUD and VPERMILPS(imm) since they are the same instruction\n Ref Single128Bit4ByteVectorShuffle(Ref Src, uint8_t Shuffle);\n // AVX 128-bit operations\n Ref AVX128_LoadXMMRegister(uint32_t XMM, bool High);\n void AVX128_StoreXMMRegister(uint32_t XMM, const Ref Src, bool High);\n\n struct RefPair {\n Ref Low, High;\n };\n\n RefPair AVX128_Zext(Ref R) {\n RefPair Pair;\n Pair.Low = R;\n Pair.High = LoadZeroVector(OpSize::i128Bit);\n return Pair;\n }\n\n Ref SHADataShuffle(Ref Src) {\n // SHA data shuffle matches PSHUFD shuffle where elements are inverted.\n // Because this shuffle mask gets reused multiple times per instruction, it's always a win to load the mask once and reuse it.\n const uint32_t Shuffle = 0b00'01'10'11;\n auto LookupIndexes =\n LoadAndCacheIndexedNamedVectorConstant(OpSize::i128Bit, FEXCore::IR::IndexNamedVectorConstant::INDEXED_NAMED_VECTOR_PSHUFD, Shuffle * 16);\n return _VTBL1(OpSize::i128Bit, Src, LookupIndexes);\n }\n\n RefPair AVX128_LoadSource_WithOpSize(const X86Tables::DecodedOp& Op, const X86Tables::DecodedOperand& Operand, uint32_t Flags,\n bool NeedsHigh, MemoryAccessType AccessType = MemoryAccessType::DEFAULT);\n\n RefVSIB AVX128_LoadVSIB(const X86Tables::DecodedOp& Op, const X86Tables::DecodedOperand& Operand, uint32_t Flags, bool NeedsHigh);\n void AVX128_StoreResult_WithOpSize(FEXCore::X86Tables::DecodedOp Op, const FEXCore::X86Tables::DecodedOperand& Operand, const RefPair Src,\n MemoryAccessType AccessType = MemoryAccessType::DEFAULT);\n void AVX128_VMOVScalarImpl(OpcodeArgs, IR::OpSize ElementSize);\n void AVX128_VectorALU(OpcodeArgs, IROps IROp, IR::OpSize ElementSize);\n void AVX128_VectorUnary(OpcodeArgs, IROps IROp, IR::OpSize ElementSize);\n void AVX128_VectorUnaryImpl(OpcodeArgs, IR::OpSize SrcSize, IR::OpSize ElementSize, std::function Helper);\n void AVX128_VectorBinaryImpl(OpcodeArgs, IR::OpSize SrcSize, IR::OpSize ElementSize,\n std::function Helper);\n void AVX128_VectorShiftWideImpl(OpcodeArgs, IR::OpSize ElementSize, IROps IROp);\n void AVX128_VectorShiftImmImpl(OpcodeArgs, IR::OpSize ElementSize, IROps IROp);\n void AVX128_VectorTrinaryImpl(OpcodeArgs, IR::OpSize SrcSize, IR::OpSize ElementSize, Ref Src3,\n std::function Helper);\n\n enum class ShiftDirection { RIGHT, LEFT };\n void AVX128_ShiftDoubleImm(OpcodeArgs, ShiftDirection Dir);\n\n void AVX128_VMOVAPS(OpcodeArgs);\n void AVX128_VMOVSD(OpcodeArgs);\n void AVX128_VMOVSS(OpcodeArgs);\n\n void AVX128_VectorXOR(OpcodeArgs);\n\n void AVX128_VZERO(OpcodeArgs);\n void AVX128_MOVVectorNT(OpcodeArgs);\n void AVX128_MOVQ(OpcodeArgs);\n void AVX128_VMOVLP(OpcodeArgs);\n void AVX128_VMOVHP(OpcodeArgs);\n void AVX128_VMOVDDUP(OpcodeArgs);\n void AVX128_VMOVSLDUP(OpcodeArgs);\n void AVX128_VMOVSHDUP(OpcodeArgs);\n void AVX128_VBROADCAST(OpcodeArgs, IR::OpSize ElementSize);\n void AVX128_VPUNPCKL(OpcodeArgs, IR::OpSize ElementSize);\n void AVX128_VPUNPCKH(OpcodeArgs, IR::OpSize ElementSize);\n void AVX128_MOVVectorUnaligned(OpcodeArgs);\n void AVX128_InsertCVTGPR_To_FPR(OpcodeArgs, IR::OpSize DstElementSize);\n void AVX128_CVTFPR_To_GPR(OpcodeArgs, IR::OpSize SrcElementSize, bool HostRoundingMode);\n void AVX128_VANDN(OpcodeArgs);\n void AVX128_VPACKSS(OpcodeArgs, IR::OpSize ElementSize);\n void AVX128_VPACKUS(OpcodeArgs, IR::OpSize ElementSize);\n Ref AVX128_PSIGNImpl(IR::OpSize ElementSize, Ref Src1, Ref Src2);\n void AVX128_VPSIGN(OpcodeArgs, IR::OpSize ElementSize);\n void AVX128_UCOMISx(OpcodeArgs, IR::OpSize ElementSize);\n void AVX128_VectorScalarInsertALU(OpcodeArgs, FEXCore::IR::IROps IROp, IR::OpSize ElementSize);\n void AVX128_VFCMP(OpcodeArgs, IR::OpSize ElementSize);\n void AVX128_InsertScalarFCMP(OpcodeArgs, IR::OpSize ElementSize);\n void AVX128_MOVBetweenGPR_FPR(OpcodeArgs);\n void AVX128_PExtr(OpcodeArgs, IR::OpSize ElementSize);\n void AVX128_ExtendVectorElements(OpcodeArgs, IR::OpSize ElementSize, IR::OpSize DstElementSize, bool Signed);\n void AVX128_MOVMSK(OpcodeArgs, IR::OpSize ElementSize);\n void AVX128_MOVMSKB(OpcodeArgs);\n void AVX128_PINSRImpl(OpcodeArgs, IR::OpSize ElementSize, const X86Tables::DecodedOperand& Src1Op,\n const X86Tables::DecodedOperand& Src2Op, const X86Tables::DecodedOperand& Imm);\n void AVX128_VPINSRB(OpcodeArgs);\n void AVX128_VPINSRW(OpcodeArgs);\n void AVX128_VPINSRDQ(OpcodeArgs);\n\n void AVX128_VariableShiftImpl(OpcodeArgs, IROps IROp);\n\n void AVX128_VINSERT(OpcodeArgs);\n void AVX128_VINSERTPS(OpcodeArgs);\n\n void AVX128_VPHSUB(OpcodeArgs, IR::OpSize ElementSize);\n void AVX128_VPHSUBSW(OpcodeArgs);\n\n void AVX128_VADDSUBP(OpcodeArgs, IR::OpSize ElementSize);\n\n void AVX128_VPMULL(OpcodeArgs, IR::OpSize ElementSize, bool Signed);\n\n void AVX128_VPMULHRSW(OpcodeArgs);\n\n void AVX128_VPMULHW(OpcodeArgs, bool Signed);\n\n void AVX128_InsertScalar_CVT_Float_To_Float(OpcodeArgs, IR::OpSize DstElementSize, IR::OpSize SrcElementSize);\n\n void AVX128_Vector_CVT_Float_To_Float(OpcodeArgs, IR::OpSize DstElementSize, IR::OpSize SrcElementSize);\n\n void AVX128_Vector_CVT_Float_To_Int(OpcodeArgs, IR::OpSize SrcElementSize, bool HostRoundingMode);\n\n void AVX128_Vector_CVT_Int_To_Float(OpcodeArgs, IR::OpSize SrcElementSize, bool Widen);\n\n void AVX128_VEXTRACT128(OpcodeArgs);\n void AVX128_VAESImc(OpcodeArgs);\n void AVX128_VAESEnc(OpcodeArgs);\n void AVX128_VAESEncLast(OpcodeArgs);\n void AVX128_VAESDec(OpcodeArgs);\n void AVX128_VAESDecLast(OpcodeArgs);\n void AVX128_VAESKeyGenAssist(OpcodeArgs);\n\n void AVX128_VPCMPESTRI(OpcodeArgs);\n void AVX128_VPCMPESTRM(OpcodeArgs);\n void AVX128_VPCMPISTRI(OpcodeArgs);\n void AVX128_VPCMPISTRM(OpcodeArgs);\n\n void AVX128_PHMINPOSUW(OpcodeArgs);\n\n void AVX128_VectorRound(OpcodeArgs, IR::OpSize ElementSize);\n void AVX128_InsertScalarRound(OpcodeArgs, IR::OpSize ElementSize);\n\n void AVX128_VDPP(OpcodeArgs, IR::OpSize ElementSize);\n void AVX128_VPERMQ(OpcodeArgs);\n\n void AVX128_VPSHUFW(OpcodeArgs, bool Low);\n\n void AVX128_VSHUF(OpcodeArgs, IR::OpSize ElementSize);\n\n void AVX128_VPERMILImm(OpcodeArgs, IR::OpSize ElementSize);\n\n void AVX128_VHADDP(OpcodeArgs, IROps IROp, IR::OpSize ElementSize);\n\n void AVX128_VPHADDSW(OpcodeArgs);\n\n void AVX128_VPMADDUBSW(OpcodeArgs);\n void AVX128_VPMADDWD(OpcodeArgs);\n\n void AVX128_VBLEND(OpcodeArgs, IR::OpSize ElementSize);\n\n void AVX128_VHSUBP(OpcodeArgs, IR::OpSize ElementSize);\n\n void AVX128_VPSHUFB(OpcodeArgs);\n void AVX128_VPSADBW(OpcodeArgs);\n\n void AVX128_VMPSADBW(OpcodeArgs);\n void AVX128_VPALIGNR(OpcodeArgs);\n\n void AVX128_VMASKMOVImpl(OpcodeArgs, IR::OpSize ElementSize, IR::OpSize DstSize, bool IsStore, const X86Tables::DecodedOperand& MaskOp,\n const X86Tables::DecodedOperand& DataOp);\n\n void AVX128_VPMASKMOV(OpcodeArgs, bool IsStore);\n\n void AVX128_VMASKMOV(OpcodeArgs, IR::OpSize ElementSize, bool IsStore);\n\n void AVX128_MASKMOV(OpcodeArgs);\n\n void AVX128_VectorVariableBlend(OpcodeArgs, IR::OpSize ElementSize);\n\n void AVX128_SaveAVXState(Ref MemBase);\n void AVX128_RestoreAVXState(Ref MemBase);\n void AVX128_DefaultAVXState();\n\n void AVX128_VPERM2(OpcodeArgs);\n void AVX128_VTESTP(OpcodeArgs, IR::OpSize ElementSize);\n void AVX128_PTest(OpcodeArgs);\n\n void AVX128_VPERMILReg(OpcodeArgs, IR::OpSize ElementSize);\n\n void AVX128_VPERMD(OpcodeArgs);\n\n void AVX128_VPCLMULQDQ(OpcodeArgs);\n\n void AVX128_VFMAImpl(OpcodeArgs, IROps IROp, uint8_t Src1Idx, uint8_t Src2Idx, uint8_t AddendIdx);\n void AVX128_VFMAScalarImpl(OpcodeArgs, IROps IROp, uint8_t Src1Idx, uint8_t Src2Idx, uint8_t AddendIdx);\n void AVX128_VFMAddSubImpl(OpcodeArgs, bool AddSub, uint8_t Src1Idx, uint8_t Src2Idx, uint8_t AddendIdx);\n\n RefPair AVX128_VPGatherQPSImpl(OpcodeArgs, Ref Dest, Ref Mask, RefVSIB VSIB);\n RefPair AVX128_VPGatherImpl(OpcodeArgs, OpSize Size, OpSize ElementLoadSize, OpSize AddrElementSize, RefPair Dest, RefPair Mask, RefVSIB VSIB);\n\n void AVX128_VPGATHER(OpcodeArgs, OpSize AddrElementSize);\n\n void AVX128_VCVTPH2PS(OpcodeArgs);\n void AVX128_VCVTPS2PH(OpcodeArgs);\n\n // End of AVX 128-bit implementation\n\n // AVX 256-bit operations\n void StoreResult_WithAVXInsert(VectorOpType Type, RegClass Class, FEXCore::X86Tables::DecodedOp Op, Ref Value,\n IR::OpSize Align = IR::OpSize::iInvalid, MemoryAccessType AccessType = MemoryAccessType::DEFAULT) {\n if (Op->Dest.IsGPR() && Op->Dest.Data.GPR.GPR >= X86State::REG_XMM_0 && Op->Dest.Data.GPR.GPR <= X86State::REG_XMM_15 &&\n GetGuestVectorLength() == OpSize::i256Bit && Type == VectorOpType::SSE) {\n const auto gpr = Op->Dest.Data.GPR.GPR;\n const auto gprIndex = gpr - X86State::REG_XMM_0;\n auto DestVector = LoadXMMRegister(gprIndex);\n Value = _VInsElement(GetGuestVectorLength(), OpSize::i128Bit, 0, 0, DestVector, Value);\n StoreXMMRegister(gprIndex, Value);\n return;\n }\n\n StoreResult(Class, Op, Value, Align, AccessType);\n }\n\n void StoreXMMRegister_WithAVXInsert(VectorOpType Type, uint32_t XMM, Ref Value) {\n if (GetGuestVectorLength() == OpSize::i256Bit && Type == VectorOpType::SSE) {\n ///< SSE vector stores need to insert in the low 128-bit lane of the 256-bit register.\n auto DestVector = LoadXMMRegister(XMM);\n Value = _VInsElement(GetGuestVectorLength(), OpSize::i128Bit, 0, 0, DestVector, Value);\n StoreXMMRegister(XMM, Value);\n return;\n }\n StoreXMMRegister(XMM, Value);\n }\n\n void AVXVectorALUOp(OpcodeArgs, IROps IROp, IR::OpSize ElementSize);\n void AVXVectorUnaryOp(OpcodeArgs, IROps IROp, IR::OpSize ElementSize);\n void AVXVectorVariableBlend(OpcodeArgs, IR::OpSize ElementSize);\n\n // End of AVX 256-bit implementation\n\n void InvalidOp(OpcodeArgs);\n void NoExecOp(OpcodeArgs);\n\n void SetPackedRFLAG(bool Lower8, Ref Src);\n Ref GetPackedRFLAG(uint32_t FlagsMask = ~0U);\n\n void SetMultiblock(bool _Multiblock) {\n Multiblock = _Multiblock;\n }\n\n static inline constexpr unsigned IndexNZCV(unsigned BitOffset) {\n switch (BitOffset) {\n case FEXCore::X86State::RFLAG_OF_RAW_LOC: return 28;\n case FEXCore::X86State::RFLAG_CF_RAW_LOC: return 29;\n case FEXCore::X86State::RFLAG_ZF_RAW_LOC: return 30;\n case FEXCore::X86State::RFLAG_SF_RAW_LOC: return 31;\n default: FEX_UNREACHABLE;\n }\n }\n\n void StoreContextHelper(IR::OpSize Size, RegClass Class, Ref Value, uint32_t Offset) {\n // For i128Bit, we won't see a normal Constant to inline, but as a special\n // case we can replace with a 2x64-bit store which can use inline zeroes.\n if (Size == OpSize::i128Bit) {\n auto Header = GetOpHeader(WrapNode(Value));\n const auto MAX_STP_OFFSET = (252 * 4);\n\n if (Offset <= MAX_STP_OFFSET && Header->Op == OP_LOADNAMEDVECTORCONSTANT) {\n auto Const = Header->C();\n\n if (Const->Constant == IR::NamedVectorConstant::NAMED_VECTOR_ZERO) {\n Ref Zero = _Constant(0);\n Ref STP = _StoreContextPair(IR::OpSize::i64Bit, RegClass::GPR, Zero, Zero, Offset);\n\n // XXX: This works around InlineConstant not having an associated\n // register class, else we'd just do InlineConstant above.\n Ref InlineZero = _InlineConstant(0);\n ReplaceNodeArgument(STP, 0, InlineZero);\n ReplaceNodeArgument(STP, 1, InlineZero);\n return;\n }\n }\n }\n\n _StoreContext(Size, Class, Value, Offset);\n }\n\n void FlushRegisterCache(bool SRAOnly = false, bool MMXOnly = false) {\n // At block boundaries, fix up the carry flag.\n if (!SRAOnly) {\n RectifyCarryInvert(CFInvertedABI);\n }\n\n if (!MMXOnly) {\n CalculateDeferredFlags();\n }\n\n const auto GPRSize = GetGPROpSize();\n const auto VectorSize = GetGuestVectorLength();\n\n // Write backwards. This is a heuristic to improve coalescing, since we\n // often copy from (low) fixed GPRs to (high) PF/AF for celebrity\n // instructions like \"add rax, 1\". This hack will go away with clauses.\n uint64_t Bits = RegCache.Written;\n\n // We have an SRA only mode that exists as a hack to make register caching\n // less aggressive. We should get rid of this once RA can take it.\n uint64_t Mask = ~0ULL;\n\n if (SRAOnly) {\n const uint64_t GPRMask = ((1ull << (AFIndex - GPR0Index + 1)) - 1) << GPR0Index;\n const uint64_t FPRMask = ((1ull << (FPR15Index - FPR0Index + 1)) - 1) << FPR0Index;\n\n Mask &= (GPRMask | FPRMask);\n Bits &= Mask;\n }\n\n if (MMXOnly) {\n Mask &= ((1ull << (MM7Index - MM0Index + 1)) - 1) << MM0Index;\n Bits &= Mask;\n }\n\n while (Bits != 0) {\n uint32_t Index = 63 - std::countl_zero(Bits);\n Ref Value = RegCache.Value[Index];\n\n if (Index >= GPR0Index && Index <= GPR15Index) {\n Ref R = _StoreRegister(Value, GPRSize);\n R->Reg = PhysicalRegister(RegClass::GPRFixed, Index - GPR0Index).Raw;\n } else if (Index == PFIndex) {\n _StorePF(Value, GPRSize);\n } else if (Index == AFIndex) {\n _StoreAF(Value, GPRSize);\n } else if (Index >= FPR0Index && Index <= FPR15Index) {\n Ref R = _StoreRegister(Value, VectorSize);\n R->Reg = PhysicalRegister(RegClass::FPRFixed, Index - FPR0Index).Raw;\n } else if (Index == DFIndex) {\n _StoreContextGPR(OpSize::i8Bit, Value, offsetof(Core::CPUState, flags[X86State::RFLAG_DF_RAW_LOC]));\n } else {\n bool Partial = RegCache.Partial & (1ull << Index);\n auto Size = Partial ? OpSize::i64Bit : CacheIndexToOpSize(Index);\n uint64_t NextBit = (1ull << (Index - 1));\n uint32_t Offset = CacheIndexToContextOffset(Index);\n auto Class = CacheIndexClass(Index);\n LOGMAN_THROW_A_FMT(Offset != ~0U, \"Invalid offset\");\n\n // Use stp where possible to store multiple values at a time. This accelerates AVX.\n // TODO: this is all really confusing because of backwards iteration,\n // can we peel back that hack?\n const auto SizeInt = IR::OpSizeToSize(Size);\n if ((Bits & NextBit) && !Partial && Size >= OpSize::i32Bit && CacheIndexToContextOffset(Index - 1) == Offset - SizeInt &&\n (Offset - SizeInt) / SizeInt < 64) {\n LOGMAN_THROW_A_FMT(CacheIndexClass(Index - 1) == Class, \"construction\");\n LOGMAN_THROW_A_FMT((Offset % SizeInt) == 0, \"construction\");\n Ref ValueNext = RegCache.Value[Index - 1];\n\n _StoreContextPair(Size, Class, ValueNext, Value, Offset - SizeInt);\n Bits &= ~NextBit;\n } else {\n StoreContextHelper(Size, Class, Value, Offset);\n // If Partial and MMX register, then we need to store all 1s in bits 64-80\n if (Partial && Index >= MM0Index && Index <= MM7Index) {\n _StoreContextGPR(OpSize::i16Bit, Constant(0xFFFF), Offset + 8);\n }\n }\n }\n\n Bits &= ~(1ull << Index);\n }\n\n RegCache.Written &= ~Mask;\n RegCache.Cached &= ~Mask;\n RegCache.Partial &= ~Mask;\n }\n\n IR::OpSize GetGPROpSize() const {\n return Is64BitMode ? IR::OpSize::i64Bit : IR::OpSize::i32Bit;\n }\n\nprotected:\n void RecordX87Use() override {\n CurrentHeader->HasX87 = true;\n }\n\n void SaveNZCV(IROps Op = OP_DUMMY) override {\n /* Some opcodes are conservatively marked as clobbering flags, but in fact\n * do not clobber flags in certain conditions. Check for that here as an\n * optimization.\n */\n switch (Op) {\n case OP_VFMINSCALARINSERT:\n case OP_VFMAXSCALARINSERT:\n /* On AFP platforms, becomes fmin/fmax and preserves NZCV. Otherwise\n * becomes fcmp and clobbers.\n */\n if (CTX->HostFeatures.SupportsAFP) {\n return;\n }\n break;\n\n case OP_VLOADVECTORMASKED:\n case OP_VLOADVECTORGATHERMASKED:\n case OP_VLOADVECTORGATHERMASKEDQPS:\n case OP_VSTOREVECTORMASKED:\n /* On ASIMD platforms, the emulation happens to preserve NZCV, unlike the\n * more optimal SVE implementation that clobbers.\n */\n if (!CTX->HostFeatures.SupportsSVE128 && !CTX->HostFeatures.SupportsSVE256) {\n return;\n }\n\n break;\n default: break;\n }\n\n // Invariant: When executing instructions that clobber NZCV, the flags must\n // be resident in a GPR, which is equivalent to CachedNZCV != nullptr. Get\n // the NZCV which fills the cache if necessary.\n if (CachedNZCV == nullptr) {\n GetNZCV();\n }\n\n // Assume we'll need a reload.\n NZCVDirty = true;\n }\n\nprivate:\n FEX_CONFIG_OPT(ReducedPrecisionMode, X87REDUCEDPRECISION);\n\n struct JumpTargetInfo {\n Ref BlockEntry;\n bool HaveEmitted;\n bool IsEntryPoint;\n };\n\n FEXCore::Context::ContextImpl* CTX {};\n\n constexpr static unsigned FullNZCVMask = (1U << FEXCore::X86State::RFLAG_CF_RAW_LOC) | (1U << FEXCore::X86State::RFLAG_ZF_RAW_LOC) |\n (1U << FEXCore::X86State::RFLAG_SF_RAW_LOC) | (1U << FEXCore::X86State::RFLAG_OF_RAW_LOC);\n\n static bool ContainsNZCV(unsigned BitMask) {\n return (BitMask & FullNZCVMask) != 0;\n }\n\n static bool IsNZCV(unsigned BitOffset) {\n return BitOffset < 32 && ContainsNZCV(1U << BitOffset);\n }\n\n Ref CachedNZCV {};\n bool NZCVDirty {};\n\n // Set if the host carry is inverted from the guest carry. This is set after\n // subtraction, because arm64 and x86 have inverted borrow flags, but clear\n // after addition.\n //\n // All CF access needs to maintain this flag. cfinv may be inserted at the end\n // of a block to rectify to the FEX convention (current convention: NOT\n // INVERTED).\n bool CFInverted {};\n\n // FEX convention for CF at the end of blocks: INVERTED.\n const bool CFInvertedABI {true};\n\n fextl::map JumpTargets;\n bool HandledLock {false};\n bool DecodeFailure {false};\n bool NeedsBlockEnd {false};\n ForceTSOMode ForceTSO {ForceTSOMode::NoOverride};\n // Used during new op bringup\n bool ShouldDump {false};\n\n using SaveStoreAVXStatePtr = void (OpDispatchBuilder::*)(Ref MemBase);\n using DefaultAVXStatePtr = void (OpDispatchBuilder::*)();\n SaveStoreAVXStatePtr SaveAVXStateFunc {&OpDispatchBuilder::SaveAVXState};\n SaveStoreAVXStatePtr RestoreAVXStateFunc {&OpDispatchBuilder::RestoreAVXState};\n DefaultAVXStatePtr DefaultAVXStateFunc {&OpDispatchBuilder::DefaultAVXState};\n\n // Opcode helpers for generalizing behavior across VEX and non-VEX variants.\n\n Ref ADDSUBPOpImpl(OpSize Size, IR::OpSize ElementSize, Ref Src1, Ref Src2);\n\n void AVXVariableShiftImpl(OpcodeArgs, IROps IROp);\n\n Ref AESKeyGenAssistImpl(OpcodeArgs);\n\n Ref CVTGPR_To_FPRImpl(OpcodeArgs, IR::OpSize DstElementSize, const X86Tables::DecodedOperand& Src1Op, const X86Tables::DecodedOperand& Src2Op);\n\n Ref DPPOpImpl(IR::OpSize DstSize, Ref Src1, Ref Src2, uint8_t Mask, IR::OpSize ElementSize);\n\n Ref VDPPSOpImpl(OpcodeArgs, const X86Tables::DecodedOperand& Src1, const X86Tables::DecodedOperand& Src2, const X86Tables::DecodedOperand& Imm);\n\n Ref ExtendVectorElementsImpl(OpcodeArgs, IR::OpSize ElementSize, IR::OpSize DstElementSize, bool Signed);\n\n Ref HSUBPOpImpl(OpSize Size, IR::OpSize ElementSize, Ref Src1, Ref Src2);\n\n Ref InsertPSOpImpl(OpcodeArgs, const X86Tables::DecodedOperand& Src1, const X86Tables::DecodedOperand& Src2,\n const X86Tables::DecodedOperand& Imm);\n\n Ref MPSADBWOpImpl(IR::OpSize SrcSize, Ref Src1, Ref Src2, uint8_t Select);\n\n Ref PALIGNROpImpl(OpcodeArgs, const X86Tables::DecodedOperand& Src1, const X86Tables::DecodedOperand& Src2,\n const X86Tables::DecodedOperand& Imm, bool IsAVX);\n\n void PCMPXSTRXOpImpl(OpcodeArgs, bool IsExplicit, bool IsMask);\n\n Ref PHADDSOpImpl(OpSize Size, Ref Src1, Ref Src2);\n\n Ref PHMINPOSUWOpImpl(OpcodeArgs);\n\n Ref PHSUBOpImpl(OpSize Size, Ref Src1, Ref Src2, IR::OpSize ElementSize);\n\n Ref PHSUBSOpImpl(OpSize Size, Ref Src1, Ref Src2);\n\n Ref PINSROpImpl(OpcodeArgs, IR::OpSize ElementSize, const X86Tables::DecodedOperand& Src1Op, const X86Tables::DecodedOperand& Src2Op,\n const X86Tables::DecodedOperand& Imm);\n\n Ref PMADDWDOpImpl(IR::OpSize Size, Ref Src1, Ref Src2);\n\n Ref PMADDUBSWOpImpl(IR::OpSize Size, Ref Src1, Ref Src2);\n\n Ref PMULHRSWOpImpl(OpSize Size, Ref Src1, Ref Src2);\n\n Ref PMULHWOpImpl(OpcodeArgs, bool Signed, Ref Src1, Ref Src2);\n\n Ref PMULLOpImpl(OpSize Size, IR::OpSize ElementSize, bool Signed, Ref Src1, Ref Src2);\n\n Ref PSADBWOpImpl(IR::OpSize Size, Ref Src1, Ref Src2);\n\n Ref GeneratePSHUFBMask(IR::OpSize SrcSize);\n Ref PSHUFBOpImpl(IR::OpSize SrcSize, Ref Src1, Ref Src2, Ref MaskVector);\n\n Ref PSIGNImpl(OpcodeArgs, IR::OpSize ElementSize, Ref Src1, Ref Src2);\n\n Ref PSLLIImpl(OpcodeArgs, IR::OpSize ElementSize, Ref Src, uint64_t Shift);\n\n Ref PSLLImpl(OpcodeArgs, IR::OpSize ElementSize, Ref Src, Ref ShiftVec);\n\n Ref PSRAOpImpl(OpcodeArgs, IR::OpSize ElementSize, Ref Src, Ref ShiftVec);\n\n Ref PSRLDOpImpl(OpcodeArgs, IR::OpSize ElementSize, Ref Src, Ref ShiftVec);\n\n Ref SHUFOpImpl(OpcodeArgs, IR::OpSize DstSize, IR::OpSize ElementSize, Ref Src1, Ref Src2, uint8_t Shuffle);\n\n void VMASKMOVOpImpl(OpcodeArgs, IR::OpSize ElementSize, IR::OpSize DataSize, bool IsStore, const X86Tables::DecodedOperand& MaskOp,\n const X86Tables::DecodedOperand& DataOp);\n\n void MOVScalarOpImpl(OpcodeArgs, IR::OpSize ElementSize);\n void VMOVScalarOpImpl(OpcodeArgs, IR::OpSize ElementSize);\n\n Ref VFCMPOpImpl(OpSize Size, IR::OpSize ElementSize, Ref Src1, Ref Src2, uint8_t CompType);\n\n void VTESTOpImpl(OpSize SrcSize, IR::OpSize ElementSize, Ref Src1, Ref Src2);\n\n void VectorUnaryDuplicateOpImpl(OpcodeArgs, IROps IROp, IR::OpSize ElementSize);\n\n // x86 ALU scalar operations operate in three different ways\n // - AVX512: Writemask shenanigans that we don't care about.\n // - AVX/VEX: Two source\n // - Example 32bit VADDSS Dest, Src1, Src2\n // - Dest[31:0] = Src1[31:0] + Src2[31:0]\n // - Dest[127:32] = Src1[127:32]\n // - SSE: Scalar operation inserts in to the low bits, upper bits completely unaffected.\n // - Example 32bit ADDSS Dest, Src\n // - Dest[31:0] = Dest[31:0] + Src[31:0]\n // - Dest[{256,128}:32] = (Unmodified)\n Ref VectorScalarInsertALUOpImpl(OpcodeArgs, IROps IROp, IR::OpSize DstSize, IR::OpSize ElementSize,\n const X86Tables::DecodedOperand& Src1Op, const X86Tables::DecodedOperand& Src2Op, bool ZeroUpperBits);\n\n Ref VectorScalarUnaryInsertALUOpImpl(OpcodeArgs, IROps IROp, IR::OpSize DstSize, IR::OpSize ElementSize,\n const X86Tables::DecodedOperand& Src1Op, const X86Tables::DecodedOperand& Src2Op, bool ZeroUpperBits);\n\n Ref InsertCVTGPR_To_FPRImpl(OpcodeArgs, IR::OpSize DstSize, IR::OpSize DstElementSize, const X86Tables::DecodedOperand& Src1Op,\n const X86Tables::DecodedOperand& Src2Op, bool ZeroUpperBits);\n\n Ref InsertScalar_CVT_Float_To_FloatImpl(OpcodeArgs, IR::OpSize DstSize, IR::OpSize DstElementSize, IR::OpSize SrcElementSize,\n const X86Tables::DecodedOperand& Src1Op, const X86Tables::DecodedOperand& Src2Op, bool ZeroUpperBits);\n Ref InsertScalarRoundImpl(OpcodeArgs, IR::OpSize DstSize, IR::OpSize ElementSize, const X86Tables::DecodedOperand& Src1Op,\n const X86Tables::DecodedOperand& Src2Op, uint64_t Mode, bool ZeroUpperBits);\n\n Ref InsertScalarFCMPOpImpl(OpSize Size, IR::OpSize OpDstSize, IR::OpSize ElementSize, Ref Src1, Ref Src2, uint8_t CompType, bool ZeroUpperBits);\n\n Ref VectorRoundImpl(OpSize Size, IR::OpSize ElementSize, Ref Src, uint64_t Mode);\n\n Ref Scalar_CVT_Float_To_FloatImpl(OpcodeArgs, IR::OpSize DstElementSize, IR::OpSize SrcElementSize,\n const X86Tables::DecodedOperand& Src1Op, const X86Tables::DecodedOperand& Src2Op);\n\n Ref CVTFPR_To_GPRImpl(OpcodeArgs, Ref Src, IR::OpSize SrcElementSize, bool HostRoundingMode);\n\n Ref Vector_CVT_Float_To_Int32Impl(OpcodeArgs, IR::OpSize DstSize, Ref Src, IR::OpSize SrcSize, IR::OpSize SrcElementSize,\n bool HostRoundingMode, bool ZeroUpperHalf);\n\n Ref Vector_CVT_Int_To_FloatImpl(OpcodeArgs, IR::OpSize SrcElementSize, bool Widen);\n\n void XSaveOpImpl(OpcodeArgs);\n void SaveX87State(OpcodeArgs, Ref MemBase);\n void SaveSSEState(Ref MemBase);\n void SaveMXCSRState(Ref MemBase);\n void SaveAVXState(Ref MemBase);\n\n void XRstorOpImpl(OpcodeArgs);\n void RestoreX87State(Ref MemBase);\n void RestoreSSEState(Ref MemBase);\n void RestoreMXCSRState(Ref MXCSR);\n void RestoreAVXState(Ref MemBase);\n void DefaultX87State(OpcodeArgs);\n void DefaultSSEState();\n void DefaultAVXState();\n\n Ref GetMXCSR();\n\n#undef OpcodeArgs\n\n Ref AppendSegmentOffset(Ref Value, uint32_t Flags, uint32_t DefaultPrefix = 0, bool Override = false);\n Ref GetSegment(uint32_t Flags, uint32_t DefaultPrefix = FEXCore::X86Tables::DecodeFlags::FLAG_NO_PREFIX, bool Override = false);\n\n void UpdatePrefixFromSegment(Ref Segment, uint32_t SegmentReg);\n\n Ref LoadGPRRegister(uint32_t GPR, IR::OpSize Size = OpSize::iInvalid, uint8_t Offset = 0, bool AllowUpperGarbage = false);\n void StoreGPRRegister(uint32_t GPR, const Ref Src, IR::OpSize Size = OpSize::iInvalid, uint8_t Offset = 0);\n void StoreXMMRegister(uint32_t XMM, const Ref Src);\n\n Ref _GetRelocatedPC(const FEXCore::X86Tables::DecodedOp& Op, int64_t Offset, bool Inline) {\n const auto GPRSize = GetGPROpSize();\n const auto Offs = Op->PC + Op->InstSize + Offset - Entry;\n return Inline ? _InlineEntrypointOffset(GPRSize, Offs) : _EntrypointOffset(GPRSize, Offs);\n }\n\n Ref GetRelocatedPC(const FEXCore::X86Tables::DecodedOp& Op, int64_t Offset = 0) {\n return _GetRelocatedPC(Op, Offset, false);\n }\n\n void ExitRelocatedPC(const FEXCore::X86Tables::DecodedOp& Op, int64_t Offset = 0) {\n ExitFunction(_GetRelocatedPC(Op, Offset, true /* Inline */));\n }\n\n void ExitRelocatedPC(const FEXCore::X86Tables::DecodedOp& Op, int64_t Offset, BranchHint Hint, Ref CallReturnAddress, Ref CallReturnBlock) {\n ExitFunction(_GetRelocatedPC(Op, Offset, true /* Inline */), Hint, CallReturnAddress, CallReturnBlock);\n }\n\n [[nodiscard]]\n static bool IsOperandMem(const X86Tables::DecodedOperand& Operand, bool Load) {\n // Literals are immediates as sources but memory addresses as destinations.\n return !(Load && (Operand.IsLiteral() || Operand.IsLiteralRelocation())) && !Operand.IsGPR();\n }\n\n [[nodiscard]]\n static bool IsNonTSOReg(MemoryAccessType Access, uint8_t Reg) {\n return Access == MemoryAccessType::DEFAULT && Reg == X86State::REG_RSP;\n }\n\n AddressMode DecodeAddress(const X86Tables::DecodedOp& Op, const X86Tables::DecodedOperand& Operand, MemoryAccessType AccessType, bool IsLoad);\n\n Ref LoadSource(RegClass Class, const X86Tables::DecodedOp& Op, const X86Tables::DecodedOperand& Operand, uint32_t Flags,\n const LoadSourceOptions& Options = {});\n Ref LoadSourceGPR(const X86Tables::DecodedOp& Op, const X86Tables::DecodedOperand& Operand, uint32_t Flags,\n const LoadSourceOptions& Options = {}) {\n return LoadSource(RegClass::GPR, Op, Operand, Flags, Options);\n }\n Ref LoadSourceFPR(const X86Tables::DecodedOp& Op, const X86Tables::DecodedOperand& Operand, uint32_t Flags,\n const LoadSourceOptions& Options = {}) {\n return LoadSource(RegClass::FPR, Op, Operand, Flags, Options);\n }\n\n Ref LoadSource_WithOpSize(RegClass Class, const X86Tables::DecodedOp& Op, const X86Tables::DecodedOperand& Operand, IR::OpSize OpSize,\n uint32_t Flags, const LoadSourceOptions& Options = {});\n Ref LoadSourceGPR_WithOpSize(const X86Tables::DecodedOp& Op, const X86Tables::DecodedOperand& Operand, IR::OpSize OpSize, uint32_t Flags,\n const LoadSourceOptions& Options = {}) {\n return LoadSource_WithOpSize(RegClass::GPR, Op, Operand, OpSize, Flags, Options);\n }\n Ref LoadSourceFPR_WithOpSize(const X86Tables::DecodedOp& Op, const X86Tables::DecodedOperand& Operand, IR::OpSize OpSize, uint32_t Flags,\n const LoadSourceOptions& Options = {}) {\n return LoadSource_WithOpSize(RegClass::FPR, Op, Operand, OpSize, Flags, Options);\n }\n\n void StoreResult_WithOpSize(RegClass Class, X86Tables::DecodedOp Op, const X86Tables::DecodedOperand& Operand, Ref Src, IR::OpSize OpSize,\n IR::OpSize Align, MemoryAccessType AccessType = MemoryAccessType::DEFAULT);\n void StoreResultGPR_WithOpSize(X86Tables::DecodedOp Op, const X86Tables::DecodedOperand& Operand, Ref Src, IR::OpSize OpSize,\n IR::OpSize Align = IR::OpSize::iInvalid, MemoryAccessType AccessType = MemoryAccessType::DEFAULT) {\n StoreResult_WithOpSize(RegClass::GPR, Op, Operand, Src, OpSize, Align, AccessType);\n }\n void StoreResultFPR_WithOpSize(X86Tables::DecodedOp Op, const X86Tables::DecodedOperand& Operand, Ref Src, IR::OpSize OpSize,\n IR::OpSize Align = IR::OpSize::iInvalid, MemoryAccessType AccessType = MemoryAccessType::DEFAULT) {\n StoreResult_WithOpSize(RegClass::FPR, Op, Operand, Src, OpSize, Align, AccessType);\n }\n\n void StoreResult(RegClass Class, X86Tables::DecodedOp Op, const X86Tables::DecodedOperand& Operand, Ref Src, OpSize Align,\n MemoryAccessType AccessType = MemoryAccessType::DEFAULT);\n void StoreResultGPR(X86Tables::DecodedOp Op, const X86Tables::DecodedOperand& Operand, Ref Src, OpSize Align = OpSize::iInvalid,\n MemoryAccessType AccessType = MemoryAccessType::DEFAULT) {\n StoreResult(RegClass::GPR, Op, Operand, Src, Align, AccessType);\n }\n void StoreResultFPR(X86Tables::DecodedOp Op, const X86Tables::DecodedOperand& Operand, Ref Src, OpSize Align = OpSize::iInvalid,\n MemoryAccessType AccessType = MemoryAccessType::DEFAULT) {\n StoreResult(RegClass::FPR, Op, Operand, Src, Align, AccessType);\n }\n\n void StoreResult(RegClass Class, X86Tables::DecodedOp Op, Ref Src, OpSize Align, MemoryAccessType AccessType = MemoryAccessType::DEFAULT);\n void StoreResultGPR(X86Tables::DecodedOp Op, Ref Src, OpSize Align = OpSize::iInvalid, MemoryAccessType AccessType = MemoryAccessType::DEFAULT) {\n StoreResult(RegClass::GPR, Op, Src, Align, AccessType);\n }\n void StoreResultFPR(X86Tables::DecodedOp Op, Ref Src, OpSize Align = OpSize::iInvalid, MemoryAccessType AccessType = MemoryAccessType::DEFAULT) {\n StoreResult(RegClass::FPR, Op, Src, Align, AccessType);\n }\n\n // In several instances, it's desirable to get a base address with the segment offset\n // applied to it. This pulls all the common-case appending into a single set of functions.\n [[nodiscard]]\n Ref MakeSegmentAddress(const X86Tables::DecodedOp& Op, const X86Tables::DecodedOperand& Operand, IR::OpSize OpSize) {\n Ref Mem = LoadSourceGPR_WithOpSize(Op, Operand, OpSize, Op->Flags, {.LoadData = false});\n return AppendSegmentOffset(Mem, Op->Flags);\n }\n [[nodiscard]]\n Ref MakeSegmentAddress(const X86Tables::DecodedOp& Op, const X86Tables::DecodedOperand& Operand) {\n return MakeSegmentAddress(Op, Operand, OpSizeFromSrc(Op));\n }\n [[nodiscard]]\n Ref MakeSegmentAddress(X86State::X86Reg Reg, uint32_t Flags, uint32_t DefaultPrefix = 0, bool Override = false) {\n Ref Address = LoadGPRRegister(Reg);\n return AppendSegmentOffset(Address, Flags, DefaultPrefix, Override);\n }\n\n constexpr OpSize GetGuestVectorLength() const {\n return (CTX->HostFeatures.SupportsSVE256 && CTX->HostFeatures.SupportsAVX) ? OpSize::i256Bit : OpSize::i128Bit;\n }\n\n [[nodiscard]]\n static uint32_t GPROffset(X86State::X86Reg reg) {\n LOGMAN_THROW_A_FMT(reg <= X86State::X86Reg::REG_R15, \"Invalid reg used\");\n return static_cast(ARRAY_OFFSETOF(Core::CPUState, gregs, reg));\n }\n\n [[nodiscard]]\n static uint32_t MMBaseOffset() {\n return static_cast(offsetof(Core::CPUState, mm[0][0]));\n }\n\n [[nodiscard]]\n uint8_t GetDstSize(X86Tables::DecodedOp Op) const;\n [[nodiscard]]\n uint8_t GetSrcSize(X86Tables::DecodedOp Op) const;\n [[nodiscard]]\n uint32_t GetDstBitSize(X86Tables::DecodedOp Op) const;\n [[nodiscard]]\n uint32_t GetSrcBitSize(X86Tables::DecodedOp Op) const;\n [[nodiscard]]\n IR::OpSize OpSizeFromDst(X86Tables::DecodedOp Op) const {\n return IR::SizeToOpSize(GetDstSize(Op));\n }\n [[nodiscard]]\n IR::OpSize OpSizeFromSrc(X86Tables::DecodedOp Op) const {\n return IR::SizeToOpSize(GetSrcSize(Op));\n }\n\n [[nodiscard]]\n IR::OpSize GetStringOpSize(X86Tables::DecodedOp Op) const;\n\n // Set flag tracking to prepare for an operation that directly writes NZCV.\n void HandleNZCVWrite() {\n CachedNZCV = nullptr;\n NZCVDirty = false;\n }\n\n // Set flag tracking to prepare for a read-modify-write operation on NZCV.\n void HandleNZCV_RMW() {\n CalculateDeferredFlags();\n\n if (NZCVDirty && CachedNZCV) {\n _StoreNZCV(CachedNZCV);\n }\n\n HandleNZCVWrite();\n }\n\n // Special case of the above where we are known to zero C/V\n void HandleNZ00Write() {\n HandleNZCVWrite();\n\n // Host carry will be implicitly zeroed, and we want guest carry zeroed as\n // well. So do not invert.\n CFInverted = false;\n }\n\n Ref GetNZCV() {\n if (!CachedNZCV) {\n CachedNZCV = _LoadNZCV();\n }\n\n return CachedNZCV;\n }\n\n void SetNZCV(Ref Value) {\n CachedNZCV = Value;\n NZCVDirty = true;\n }\n\n void ZeroNZCV() {\n CachedNZCV = Constant(0);\n NZCVDirty = true;\n }\n\n void SetNZ_ZeroCV(IR::OpSize SrcSize, Ref Res, bool SetPF = false) {\n HandleNZ00Write();\n\n // x - 0 = x. NZ set according to Res. C always set. V always unset. This\n // matches what we want since we want carry inverted.\n //\n // This is currently worse for 8/16-bit, but that should be optimized. TODO\n if (SrcSize >= OpSize::i32Bit) {\n if (SetPF) {\n CalculatePF(SubWithFlags(SrcSize, Res, (uint64_t)0));\n } else {\n _SubNZCV(SrcSize, Res, Constant(0));\n }\n\n CFInverted = true;\n } else {\n _TestNZ(SrcSize, Res, Res);\n CFInverted = false;\n\n if (SetPF) {\n CalculatePF(Res);\n }\n }\n }\n\n void SetNZP_ZeroCV(IR::OpSize SrcSize, Ref Res) {\n SetNZ_ZeroCV(SrcSize, Res, true);\n }\n\n void InsertNZCV(unsigned BitOffset, Ref Value, signed FlagOffset, bool MustMask) {\n signed Bit = IndexNZCV(BitOffset);\n\n // Heuristic to choose rmif vs msr.\n bool PreferRmif = !NZCVDirty || FlagOffset || MustMask;\n\n if (CTX->HostFeatures.SupportsFlagM && PreferRmif) {\n // Update NZCV\n if (NZCVDirty && CachedNZCV) {\n _StoreNZCV(CachedNZCV);\n }\n\n CachedNZCV = nullptr;\n NZCVDirty = false;\n\n // Insert as NZCV.\n signed RmifBit = Bit - 28;\n _RmifNZCV(Value, (64 + FlagOffset - RmifBit) % 64, 1u << RmifBit);\n CachedNZCV = nullptr;\n } else {\n // Insert as GPR\n if (FlagOffset || MustMask) {\n Value = _Bfe(OpSize::i64Bit, 1, FlagOffset, Value);\n }\n\n SetNZCV(_Bfi(OpSize::i32Bit, 1, Bit, GetNZCV(), Value));\n }\n }\n\n // If we don't care about N/C/V and just need Z, we can test with a simple\n // mask without any shifting.\n void SetZ_InvalidateNCV(IR::OpSize Size, Ref Src) {\n HandleNZCVWrite();\n CFInverted = true;\n\n if (Size < OpSize::i32Bit) {\n _TestNZ(OpSize::i32Bit, Src, _InlineConstant((1u << (IR::OpSizeAsBits(Size))) - 1));\n } else {\n _TestNZ(Size, Src, Src);\n }\n }\n\n // Ensure the carry invert flag matches the desired form. Used before an\n // operation reading carry or at the end of a block.\n void RectifyCarryInvert(bool RequiredInvert) {\n if (CFInverted != RequiredInvert) {\n if (CTX->HostFeatures.SupportsFlagM && !NZCVDirty) {\n // Invert as NZCV.\n _CarryInvert();\n CachedNZCV = nullptr;\n } else {\n // Invert as a GPR\n unsigned Bit = IndexNZCV(FEXCore::X86State::RFLAG_CF_RAW_LOC);\n SetNZCV(_Xor(OpSize::i32Bit, GetNZCV(), Constant(1u << Bit)));\n CalculateDeferredFlags();\n }\n\n CFInverted ^= true;\n }\n\n LOGMAN_THROW_A_FMT(CFInverted == RequiredInvert, \"post condition\");\n }\n\n void CarryInvert() {\n CFInverted ^= true;\n }\n\n template\n void SetRFLAG(Ref Value, unsigned ValueOffset = 0, bool MustMask = false) {\n SetRFLAG(Value, BitOffset, ValueOffset, MustMask);\n }\n\n void SetCFDirect(Ref Value, unsigned ValueOffset = 0, bool MustMask = false) {\n Value = _Xor(OpSize::i64Bit, Value, _InlineConstant(1ull << ValueOffset));\n SetRFLAG(Value, X86State::RFLAG_CF_RAW_LOC, ValueOffset, MustMask);\n CFInverted = true;\n }\n\n // Set CF directly to the given 0/1 value. This needs to respect the\n // invert. We use a subtraction:\n //\n // 0 - x = 0 + (~x) + 1.\n //\n // If x = 0, then 0 + (~0) + 1 = 0x100000000 so hardware C is set.\n // If x = 1, then 0 + (~1) + 1 = 0x0ffffffff so hardware C is not set.\n void SetCFDirect_InvalidateNZV(Ref Value, unsigned ValueOffset = 0, bool MustMask = false) {\n if (ValueOffset || MustMask) {\n Value = _Bfe(OpSize::i64Bit, 1, ValueOffset, Value);\n }\n\n HandleNZCVWrite();\n _SubNZCV(OpSize::i32Bit, Constant(0), Value);\n CFInverted = true;\n }\n\n void SetCFInverted(Ref Value, unsigned ValueOffset = 0, bool MustMask = false) {\n SetRFLAG(Value, X86State::RFLAG_CF_RAW_LOC, ValueOffset, MustMask);\n CFInverted = true;\n }\n\n void SetRFLAG(Ref Value, unsigned BitOffset, unsigned ValueOffset = 0, bool MustMask = false) {\n if (IsNZCV(BitOffset)) {\n InsertNZCV(BitOffset, Value, ValueOffset, MustMask);\n return;\n }\n\n if (ValueOffset || MustMask) {\n Value = _Bfe(OpSize::i32Bit, 1, ValueOffset, Value);\n }\n\n if (BitOffset == FEXCore::X86State::RFLAG_PF_RAW_LOC) {\n StoreRegister(Core::CPUState::PF_AS_GREG, false, Value);\n } else if (BitOffset == FEXCore::X86State::RFLAG_AF_RAW_LOC) {\n StoreRegister(Core::CPUState::AF_AS_GREG, false, Value);\n } else if (BitOffset == FEXCore::X86State::RFLAG_DF_RAW_LOC) {\n // For DF, we need to transform 0/1 into 1/-1\n StoreDF(_SubShift(OpSize::i64Bit, Constant(1), Value, ShiftType::LSL, 1));\n } else if (BitOffset == FEXCore::X86State::RFLAG_TF_RAW_LOC) {\n auto PackedTF = _LoadContextGPR(OpSize::i8Bit, ARRAY_OFFSETOF(FEXCore::Core::CPUState, flags, BitOffset));\n // An exception should still be raised after an instruction that unsets TF, leave the unblocked bit set but unset\n // the TF bit to cause such behaviour. The handling code at the start of the next block will then unset the\n // unblocked bit before raising the exception.\n auto NewPackedTF =\n _Select(OpSize::i64Bit, OpSize::i64Bit, CondClass::EQ, Value, Constant(0), _And(OpSize::i32Bit, PackedTF, Constant(~1)), Constant(1));\n _StoreContextGPR(OpSize::i8Bit, NewPackedTF, ARRAY_OFFSETOF(FEXCore::Core::CPUState, flags, BitOffset));\n } else {\n _StoreContextGPR(OpSize::i8Bit, Value, ARRAY_OFFSETOF(FEXCore::Core::CPUState, flags, BitOffset));\n }\n }\n\n void SetAF(unsigned K) {\n // AF is stored in bit 4 of the AF flag byte, with garbage in the other\n // bits. This allows us to defer the extract in the usual case. When it is\n // read, bit 4 is extracted. In order to write a constant value of AF, that\n // means we need to left-shift here to compensate.\n SetRFLAG(Constant(K << 4));\n }\n\n void ZeroPF_AF();\n\n void InvalidateAF() {\n _InvalidateFlags((1u << X86State::RFLAG_AF_RAW_LOC));\n InvalidateReg(Core::CPUState::AF_AS_GREG);\n }\n\n void InvalidatePF_AF() {\n _InvalidateFlags((1u << X86State::RFLAG_PF_RAW_LOC) | (1u << X86State::RFLAG_AF_RAW_LOC));\n InvalidateReg(Core::CPUState::PF_AS_GREG);\n InvalidateReg(Core::CPUState::AF_AS_GREG);\n }\n\n [[nodiscard]]\n static CondClass CondForNZCVBit(unsigned BitOffset, bool Invert) {\n switch (BitOffset) {\n case X86State::RFLAG_SF_RAW_LOC: return Invert ? CondClass::PL : CondClass::MI;\n case X86State::RFLAG_ZF_RAW_LOC: return Invert ? CondClass::NEQ : CondClass::EQ;\n case X86State::RFLAG_CF_RAW_LOC: return Invert ? CondClass::ULT : CondClass::UGE;\n case X86State::RFLAG_OF_RAW_LOC: return Invert ? CondClass::FNU : CondClass::FU;\n default: FEX_UNREACHABLE;\n }\n }\n\n /* Layout of cache indices. We use a single 64-bit bitmask for the cache */\n static const int GPR0Index = 0;\n static const int GPR15Index = 15;\n static const int PFIndex = 16;\n static const int AFIndex = 17;\n /* Gap 18..19 */\n /* Note this range is only valid if MMXState = MMXState_MMX */\n static const int MM0Index = 20;\n static const int MM7Index = 27;\n /* Gap 28..30 */\n static const int DFIndex = 31;\n static const int FPR0Index = 32;\n static const int FPR15Index = 47;\n static const int AVXHigh0Index = 48;\n static const int AVXHigh15Index = 63;\n\n [[nodiscard]]\n static uint32_t CacheIndexToContextOffset(int Index) {\n switch (Index) {\n case MM0Index ... MM7Index: return ARRAY_OFFSETOF(FEXCore::Core::CPUState, mm, Index - MM0Index);\n case AVXHigh0Index ... AVXHigh15Index: return ARRAY_OFFSETOF(FEXCore::Core::CPUState, avx_high, Index - AVXHigh0Index);\n default: return ~0U;\n }\n }\n\n [[nodiscard]]\n static RegClass CacheIndexClass(int Index) {\n if ((Index >= MM0Index && Index <= MM7Index) || Index >= FPR0Index) {\n return RegClass::FPR;\n } else {\n return RegClass::GPR;\n }\n }\n\n [[nodiscard]]\n static IR::OpSize CacheIndexToOpSize(int Index) {\n // MMX registers are rounded up to 128-bit since they are shared with 80-bit\n // x87 registers, even though MMX is logically only 64-bit.\n if (Index >= AVXHigh0Index || ((Index >= MM0Index && Index <= MM7Index))) {\n return OpSize::i128Bit;\n } else {\n return OpSize::i8Bit;\n }\n }\n\n struct {\n uint64_t Cached;\n uint64_t Written;\n\n // Indicates that Value contains only the lower 64-bit of the full 80-bit\n // register. Used for MMX/x87 optimization.\n uint64_t Partial;\n\n Ref Value[64];\n } RegCache {};\n\n void InvalidateReg(uint8_t Index) {\n uint64_t Bit = (1ull << (uint64_t)Index);\n RegCache.Cached &= ~Bit;\n RegCache.Written &= ~Bit;\n }\n\n Ref LoadRegCache(uint64_t Offset, uint8_t Index, RegClass Class, IR::OpSize Size) {\n LOGMAN_THROW_A_FMT(Index < 64, \"valid index\");\n uint64_t Bit = (1ull << (uint64_t)Index);\n\n if (Size == OpSize::i128Bit && (RegCache.Partial & Bit)) {\n // We need to load the full register extend if we previously did a partial access.\n Ref Value = RegCache.Value[Index];\n Ref Full = _LoadContext(Size, Class, Offset);\n\n // If we did a partial store, we're inserting into the full register\n if (RegCache.Written & Bit) {\n Full = _VInsElement(OpSize::i128Bit, OpSize::i64Bit, 0, 0, Full, Value);\n }\n\n RegCache.Value[Index] = Full;\n }\n\n if (!(RegCache.Cached & Bit)) {\n if (Index == DFIndex) {\n RegCache.Value[Index] = _LoadDF();\n } else if ((Index >= MM0Index && Index <= MM7Index) || Index >= AVXHigh0Index) {\n RegCache.Value[Index] = _LoadContext(Size, Class, Offset);\n\n // We may have done a partial load, this requires special handling.\n if (Size == OpSize::i64Bit) {\n RegCache.Partial |= Bit;\n }\n } else if (Index == PFIndex) {\n RegCache.Value[Index] = _LoadPF(Size);\n } else if (Index == AFIndex) {\n RegCache.Value[Index] = _LoadAF(Size);\n } else {\n RegCache.Value[Index] = _LoadRegister(Offset, Class, Size);\n }\n\n RegCache.Cached |= Bit;\n }\n\n return RegCache.Value[Index];\n }\n\n RefPair AllocatePair(RegClass Class, IR::OpSize Size) {\n if (Class == RegClass::FPR) {\n return {_AllocateFPR(Size, Size), _AllocateFPR(Size, Size)};\n } else {\n return {_AllocateGPR(false), _AllocateGPR(false)};\n }\n }\n\n RefPair LoadContextPair_Uncached(RegClass Class, IR::OpSize Size, unsigned Offset) {\n RefPair Values = AllocatePair(Class, Size);\n _LoadContextPair(Size, Class, Offset, Values.Low, Values.High);\n return Values;\n }\n\n RefPair LoadRegCachePair(uint64_t Offset, uint8_t Index, RegClass Class, IR::OpSize Size) {\n LOGMAN_THROW_A_FMT(Index != DFIndex, \"must be pairable\");\n LOGMAN_THROW_A_FMT(Size != IR::OpSize::iUnsized, \"Invalid size!\");\n\n // Try to load a pair into the cache\n uint64_t Bits = (3ull << (uint64_t)Index);\n const auto SizeInt = IR::OpSizeToSize(Size);\n if (((RegCache.Partial | RegCache.Cached) & Bits) == 0 && ((Offset / SizeInt) < 64)) {\n auto Values = LoadContextPair_Uncached(Class, Size, Offset);\n RegCache.Value[Index] = Values.Low;\n RegCache.Value[Index + 1] = Values.High;\n RegCache.Cached |= Bits;\n if (Size == OpSize::i64Bit) {\n RegCache.Partial |= Bits;\n }\n return Values;\n }\n\n // Fallback on a pair of loads\n return {\n .Low = LoadRegCache(Offset, Index, Class, Size),\n .High = LoadRegCache(Offset + SizeInt, Index + 1, Class, Size),\n };\n }\n\n Ref LoadGPR(uint8_t Reg) {\n return LoadRegCache(Reg, GPR0Index + Reg, RegClass::GPR, GetGPROpSize());\n }\n\n Ref LoadContext(IR::OpSize Size, uint8_t Index) {\n return LoadRegCache(CacheIndexToContextOffset(Index), Index, CacheIndexClass(Index), Size);\n }\n\n RefPair LoadContextPair(IR::OpSize Size, uint8_t Index) {\n return LoadRegCachePair(CacheIndexToContextOffset(Index), Index, CacheIndexClass(Index), Size);\n }\n\n Ref LoadContext(uint8_t Index) {\n return LoadContext(CacheIndexToOpSize(Index), Index);\n }\n\n Ref LoadXMMRegister(uint8_t Reg) {\n return LoadRegCache(Reg, FPR0Index + Reg, RegClass::FPR, GetGuestVectorLength());\n }\n\n Ref LoadDF() {\n return LoadGPR(DFIndex);\n }\n\n void StoreContext(uint8_t Index, Ref Value) {\n LOGMAN_THROW_A_FMT(Index < 64, \"valid index\");\n LOGMAN_THROW_A_FMT(Value != InvalidNode, \"storing valid\");\n\n uint64_t Bit = (1ull << (uint64_t)Index);\n\n RegCache.Value[Index] = Value;\n RegCache.Cached |= Bit;\n RegCache.Written |= Bit;\n }\n\n void InvalidateHighAVXRegisters() {\n for (size_t i = 0; i < 16; ++i) {\n InvalidateReg(AVXHigh0Index + i);\n }\n }\n\n void StoreRegister(uint8_t Reg, bool FPR, Ref Value) {\n StoreContext(Reg + (FPR ? FPR0Index : GPR0Index), Value);\n }\n\n void StoreDF(Ref Value) {\n StoreContext(DFIndex, Value);\n }\n\n Ref GetRFLAG(unsigned BitOffset, bool Invert = false) {\n if (IsNZCV(BitOffset)) {\n // Handle the CFInverted state internally so GetRFLAG is safe regardless\n // of the invert state. This simplifies the call sites.\n if (BitOffset == X86State::RFLAG_CF_RAW_LOC) {\n Invert ^= CFInverted;\n }\n\n if (NZCVDirty) {\n auto Value = _Bfe(OpSize::i32Bit, 1, IndexNZCV(BitOffset), GetNZCV());\n\n if (Invert) {\n return _Xor(OpSize::i32Bit, Value, Constant(1));\n } else {\n return Value;\n }\n } else {\n // Because we explicitly inverted for CF above, we use the unsafe\n // _NZCVSelect rather than the safe CF-aware version.\n return _NZCVSelect01(CondForNZCVBit(BitOffset, Invert));\n }\n } else if (BitOffset == FEXCore::X86State::RFLAG_PF_RAW_LOC) {\n return LoadGPR(Core::CPUState::PF_AS_GREG);\n } else if (BitOffset == FEXCore::X86State::RFLAG_AF_RAW_LOC) {\n return LoadGPR(Core::CPUState::AF_AS_GREG);\n } else if (BitOffset == FEXCore::X86State::RFLAG_DF_RAW_LOC) {\n // Recover the sign bit, it is the logical DF value\n return _Lshr(OpSize::i64Bit, LoadDF(), Constant(63));\n } else {\n return _LoadContextGPR(OpSize::i8Bit, ARRAY_OFFSETOF(Core::CPUState, flags, BitOffset));\n }\n }\n\n // Returns (DF ? -Size : Size)\n Ref LoadDir(const unsigned Size) {\n return ARef(LoadDF()).Lshl(FEXCore::ilog2(Size)).Ref();\n }\n\n // Returns DF ? (X - Size) : (X + Size)\n Ref OffsetByDir(Ref X, const unsigned Size) {\n auto Shift = FEXCore::ilog2(Size);\n\n return _AddShift(OpSize::i64Bit, X, LoadDF(), ShiftType::LSL, Shift);\n }\n\n // Safe version of NZCVSelect that handles inverted carries automatically.\n Ref NZCVSelect(OpSize OpSize, CondClass Cond, Ref TrueV, Ref FalseV, bool CarryIsInverted = false) {\n switch (Cond) {\n case CondClass::UGE: /* cs */\n case CondClass::ULT: /* cc */\n // Invert the condition to match our expectations.\n if (CarryIsInverted != CFInverted) {\n Cond = (Cond == CondClass::UGE) ? CondClass::ULT : CondClass::UGE;\n }\n break;\n\n case CondClass::UGT: /* hi */\n case CondClass::ULE: /* ls */\n // No clever optimization we can do here, rectify carry itself.\n RectifyCarryInvert(CarryIsInverted);\n break;\n\n default:\n // No other condition codes read carry so no need to rectify.\n break;\n }\n\n return _NZCVSelect(OpSize, Cond, TrueV, FalseV);\n }\n\n // Compares two floats and sets flags for a COMISS instruction\n void Comiss(IR::OpSize ElementSize, Ref Src1, Ref Src2, bool InvalidateAF = false) {\n // First, set flags according to Arm FCMP.\n HandleNZCVWrite();\n _FCmp(ElementSize, Src1, Src2);\n CFInverted = false;\n ComissFlags(InvalidateAF);\n }\n\n // Sets flags for a COMISS instruction\n void ComissFlags(bool InvalidateAF = false) {\n LOGMAN_THROW_A_FMT(!NZCVDirty, \"only expected after fcmp\");\n\n // We need to set PF according to the unordered flag. We'd rather do this\n // after axflag, since some impls fuse fcmp+axflag, so we want to do this\n // after. We can recover \"unordered\" after axflag as (Z && !C), but\n // there's no condition code for this so it would take 2 instructions\n // instead of one, which seems worse than doing 1 op before and breaking\n // the fusion.\n //\n // We set PF to unordered (V), but our PF representation is inverted so we\n // actually set to !V. This is one instruction with the VC cond code.\n Ref V_inv = GetRFLAG(FEXCore::X86State::RFLAG_OF_RAW_LOC, true);\n SetRFLAG(V_inv);\n\n if (!InvalidateAF) {\n // Zero AF. Note that the comparison sets the raw PF to 0/1 above, so\n // PF[4] is 0 so the XOR with PF will have no effect, so setting the AF\n // byte to zero will indeed zero AF as intended.\n SetRFLAG(Constant(0));\n }\n\n // Convert NZCV from the Arm representation to an eXternal representation\n // that's totally not a euphemism for x86, nuh-uh. But maps to exactly we\n // need, what a coincidence!\n //\n // Our AXFlag emulation on FlagM2-less systems needs V_inv passed.\n _AXFlag(CTX->HostFeatures.SupportsFlagM2 ? Invalid() : V_inv);\n CFInverted = true;\n }\n\n // Set x87 comparison flags based on the result set by Arm FCMP. Clobbers\n // NZCV on flagm2 platforms.\n void ConvertNZCVToX87() {\n LOGMAN_THROW_A_FMT(NZCVDirty && CachedNZCV, \"NZCV must be saved\");\n\n Ref V = _NZCVSelect01(CondForNZCVBit(FEXCore::X86State::RFLAG_OF_RAW_LOC, false));\n\n if (CTX->HostFeatures.SupportsFlagM2) {\n // Convert to x86 flags, saves us from or'ing after.\n _AXFlag(Invalid());\n }\n\n // CF is inverted after FCMP\n Ref C = _NZCVSelect01(CondForNZCVBit(FEXCore::X86State::RFLAG_CF_RAW_LOC, true));\n Ref Z = _NZCVSelect01(CondForNZCVBit(FEXCore::X86State::RFLAG_ZF_RAW_LOC, false));\n\n if (!CTX->HostFeatures.SupportsFlagM2) {\n C = _Or(OpSize::i32Bit, C, V);\n Z = _Or(OpSize::i32Bit, Z, V);\n }\n\n SetRFLAG(C);\n SetRFLAG(Constant(0));\n SetRFLAG(V);\n SetRFLAG(Z);\n }\n\n // Helper to store a variable shift and calculate its flags for a variable\n // shift, with correct PF handling.\n void HandleShift(X86Tables::DecodedOp Op, Ref Result, Ref Dest, ShiftType Shift, Ref Src) {\n\n auto OldPF = GetRFLAG(X86State::RFLAG_PF_RAW_LOC);\n\n HandleNZCV_RMW();\n CalculatePF(_ShiftFlags(OpSizeFromSrc(Op), Result, Dest, Shift, Src, OldPF, CFInverted));\n StoreResultGPR(Op, Result);\n }\n\n // Helper to derive Dest by a given builder-using Expression with the opcode\n // replaced with NewOp. Useful for generic building code. Not safe in general.\n // but does the right handling of ImplicitFlagClobber at least and must be\n // used instead of raw Op mutation.\n#define DeriveOp(Dest, NewOp, Expr) \\\n if (ImplicitFlagClobber(NewOp)) SaveNZCV(NewOp); \\\n auto Dest = (Expr); \\\n Dest.first->Header.Op = (NewOp)\n\n // Named constant cache for the current block.\n // Different arrays for sizes 1,2,4,8,16,32.\n Ref CachedNamedVectorConstants[FEXCore::IR::NamedVectorConstant::NAMED_VECTOR_MAX][6] {};\n struct IndexNamedVectorMapKey {\n uint32_t Index {};\n FEXCore::IR::IndexNamedVectorConstant NamedIndexedConstant;\n uint8_t log2_size_in_bytes {};\n uint16_t _pad {};\n\n bool operator==(const IndexNamedVectorMapKey&) const = default;\n };\n struct IndexNamedVectorMapKeyHasher {\n std::size_t operator()(const IndexNamedVectorMapKey& k) const noexcept {\n return XXH3_64bits(&k, sizeof(k));\n }\n };\n fextl::unordered_map CachedIndexedNamedVectorConstants;\n\n // Load and cache a named vector constant.\n Ref LoadAndCacheNamedVectorConstant(IR::OpSize Size, FEXCore::IR::NamedVectorConstant NamedConstant) {\n auto log2_size_bytes = FEXCore::ilog2(IR::OpSizeToSize(Size));\n if (CachedNamedVectorConstants[NamedConstant][log2_size_bytes]) {\n return CachedNamedVectorConstants[NamedConstant][log2_size_bytes];\n }\n\n auto K = _LoadNamedVectorConstant(Size, NamedConstant);\n CachedNamedVectorConstants[NamedConstant][log2_size_bytes] = K;\n return K;\n }\n Ref LoadAndCacheIndexedNamedVectorConstant(IR::OpSize Size, FEXCore::IR::IndexNamedVectorConstant NamedIndexedConstant, uint32_t Index) {\n IndexNamedVectorMapKey Key {\n .Index = Index,\n .NamedIndexedConstant = NamedIndexedConstant,\n .log2_size_in_bytes = FEXCore::ilog2(IR::OpSizeToSize(Size)),\n };\n auto it = CachedIndexedNamedVectorConstants.find(Key);\n\n if (it != CachedIndexedNamedVectorConstants.end()) {\n return it->second;\n }\n\n auto K = _LoadNamedVectorIndexedConstant(Size, NamedIndexedConstant, Index);\n CachedIndexedNamedVectorConstants.insert_or_assign(Key, K);\n return K;\n }\n\n Ref LoadUncachedZeroVector(IR::OpSize Size) {\n return _LoadNamedVectorConstant(Size, IR::NamedVectorConstant::NAMED_VECTOR_ZERO);\n }\n\n Ref LoadZeroVector(IR::OpSize Size) {\n return LoadAndCacheNamedVectorConstant(Size, IR::NamedVectorConstant::NAMED_VECTOR_ZERO);\n }\n\n // Reset the named vector constants cache array.\n // These are only cached per block.\n void ClearCachedNamedConstants() {\n memset(CachedNamedVectorConstants, 0, sizeof(CachedNamedVectorConstants));\n CachedIndexedNamedVectorConstants.clear();\n }\n\n std::optional DecodeNZCVCondition(uint8_t OP);\n Ref SelectCC0All1(uint8_t OP);\n\n /**\n * @brief Flushes NZCV. Mostly vestigial.\n */\n void CalculateDeferredFlags();\n\n void ZeroShiftResult(FEXCore::X86Tables::DecodedOp Op) {\n // In the case of zero-rotate, we need to store the destination still to deal with 32-bit semantics.\n const auto Size = OpSizeFromSrc(Op);\n if (Size != OpSize::i32Bit) {\n return;\n }\n auto Dest = LoadSourceGPR(Op, Op->Dest, Op->Flags);\n StoreResultGPR(Op, Dest);\n }\n\n using ZeroShiftFunctionPtr = void (OpDispatchBuilder::*)(FEXCore::X86Tables::DecodedOp Op);\n\n template\n void Calculate_ShiftVariable(FEXCore::X86Tables::DecodedOp Op, Ref Shift, F&& Calculate,\n std::optional ZeroShiftResult = std::nullopt) {\n // RCR can call this with constants, so handle that without branching.\n uint64_t Const;\n if (IsValueConstant(WrapNode(Shift), &Const)) {\n if (Const) {\n Calculate();\n } else if (ZeroShiftResult) {\n (this->*(*ZeroShiftResult))(Op);\n }\n\n return;\n }\n\n // Otherwise, prepare to branch.\n auto Zero = Constant(0);\n\n // If the shift is zero, do not touch the flags.\n auto SetBlock = CreateNewCodeBlockAfter(GetCurrentBlock());\n IRPair NextBlock = SetBlock;\n IRPair ZeroShiftBlock;\n if (ZeroShiftResult) {\n ZeroShiftBlock = CreateNewCodeBlockAfter(NextBlock);\n NextBlock = ZeroShiftBlock;\n }\n auto EndBlock = CreateNewCodeBlockAfter(NextBlock);\n\n ///< Jump to zeroshift block or end block depending on if it was provided.\n IRPair TailHandling = ZeroShiftResult ? ZeroShiftBlock : EndBlock;\n CondJump(Shift, Zero, TailHandling, SetBlock, CondClass::EQ);\n\n SetCurrentCodeBlock(SetBlock);\n StartNewBlock();\n {\n Calculate();\n Jump(EndBlock);\n }\n\n if (ZeroShiftResult) {\n SetCurrentCodeBlock(ZeroShiftBlock);\n StartNewBlock();\n {\n (this->*(*ZeroShiftResult))(Op);\n Jump(EndBlock);\n }\n }\n\n SetCurrentCodeBlock(EndBlock);\n StartNewBlock();\n }\n\n /**\n * @name These functions are used by the deferred flag handling while it is calculating and storing flags in to RFLAGs.\n * @{ */\n Ref LoadPFRaw(bool Mask, bool Invert);\n Ref LoadAF();\n void FixupAF();\n void SetAFAndFixup(Ref AF);\n Ref CalculateAFForDecimal(Ref A);\n void CalculatePF(Ref Res);\n void CalculateAF(Ref Src1, Ref Src2);\n\n Ref IncrementByCarry(OpSize OpSize, Ref Src);\n\n void CalculateOF(IR::OpSize SrcSize, Ref Res, Ref Src1, Ref Src2, bool Sub);\n Ref CalculateFlags_ADC(IR::OpSize SrcSize, Ref Src1, Ref Src2);\n Ref CalculateFlags_SBB(IR::OpSize SrcSize, Ref Src1, Ref Src2);\n Ref CalculateFlags_SUB(IR::OpSize SrcSize, Ref Src1, Ref Src2, bool UpdateCF = true);\n Ref CalculateFlags_ADD(IR::OpSize SrcSize, Ref Src1, Ref Src2, bool UpdateCF = true);\n void CalculateFlags_MUL(IR::OpSize SrcSize, Ref Res, Ref High);\n void CalculateFlags_UMUL(Ref High);\n void CalculateFlags_Logical(IR::OpSize SrcSize, Ref Res);\n void CalculateFlags_ShiftLeftImmediate(IR::OpSize SrcSize, Ref Res, Ref Src1, uint64_t Shift);\n void CalculateFlags_ShiftRightImmediate(IR::OpSize SrcSize, Ref Res, Ref Src1, uint64_t Shift);\n void CalculateFlags_ShiftRightDoubleImmediate(IR::OpSize SrcSize, Ref Res, Ref Src1, uint64_t Shift);\n void CalculateFlags_ShiftRightImmediateCommon(IR::OpSize SrcSize, Ref Res, Ref Src1, uint64_t Shift);\n void CalculateFlags_SignShiftRightImmediate(IR::OpSize SrcSize, Ref Res, Ref Src1, uint64_t Shift);\n void CalculateFlags_ZCNT(IR::OpSize SrcSize, Ref Result);\n /** @} */\n\n Ref GetX87Top();\n void SetX87FTW(Ref FTW);\n Ref GetX87FTW_Helper();\n void SetX87Top(Ref Value);\n\n void ChgStateX87_MMX() override {\n LOGMAN_THROW_A_FMT(MMXState == MMXState_X87, \"Expected state to be x87\");\n _StackForceSlow();\n SetX87Top(Constant(0)); // top reset to zero\n _StoreContextGPR(OpSize::i8Bit, Constant(0xFFFFUL), offsetof(FEXCore::Core::CPUState, AbridgedFTW));\n MMXState = MMXState_MMX;\n }\n\n void ChgStateMMX_X87() override {\n LOGMAN_THROW_A_FMT(MMXState == MMXState_MMX, \"Expected state to be MMX\");\n // The opcode dispatcher register cache is used for MMX, but the x87 pass register cache is used for x87, spill to\n // context to ensure coherence.\n FlushRegisterCache(false, true);\n // We explicitly initialize to x87 state in StartNewBlock.\n // So if we ever change this to do something else, we need to\n // make sure that we consider if we need to explicitly set it there.\n MMXState = MMXState_X87;\n }\n\n bool DestIsLockedMem(FEXCore::X86Tables::DecodedOp Op) const {\n return DestIsMem(Op) && (Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_LOCK) != 0;\n }\n\n bool DestIsMem(FEXCore::X86Tables::DecodedOp Op) const {\n return !Op->Dest.IsGPR();\n }\n\n void CreateJumpBlocks(const fextl::vector* Blocks);\n bool BlockSetRIP {false};\n\n bool Multiblock {};\n bool Is64BitMode {};\n uint64_t Entry {};\n\n // Set if mono hacks are enabled and the current block is the mono callsite backpatcher, in which case the\n // XCHG ops that would patch code are replaced with a hook that performs the write and manually invalidates\n // the target address.\n bool IsMonoBackpatcherBlock {false};\n IROp_IRHeader* CurrentHeader {};\n\n [[nodiscard]]\n bool IsTSOEnabled(RegClass Class) const {\n if (ForceTSO == ForceTSOMode::ForceEnabled) {\n return true;\n } else if (ForceTSO == ForceTSOMode::ForceDisabled) {\n return false;\n } else if (Class == RegClass::FPR) {\n return CTX->IsVectorAtomicTSOEnabled();\n } else {\n return CTX->IsAtomicTSOEnabled();\n }\n }\n\n Ref _StoreMemAutoTSO(RegClass Class, OpSize Size, Ref Addr, Ref Value, OpSize Align = OpSize::i8Bit) {\n if (IsTSOEnabled(Class)) {\n return _StoreMemTSO(Class, Size, Value, Addr, Invalid(), Align, MemOffsetType::SXTX, 1);\n } else {\n return _StoreMem(Class, Size, Value, Addr, Invalid(), Align, MemOffsetType::SXTX, 1);\n }\n }\n Ref _StoreMemGPRAutoTSO(OpSize Size, Ref Addr, Ref Value, OpSize Align = OpSize::i8Bit) {\n return _StoreMemAutoTSO(RegClass::GPR, Size, Addr, Value, Align);\n }\n Ref _StoreMemFPRAutoTSO(OpSize Size, Ref Addr, Ref Value, OpSize Align = OpSize::i8Bit) {\n return _StoreMemAutoTSO(RegClass::FPR, Size, Addr, Value, Align);\n }\n\n Ref _LoadMemAutoTSO(RegClass Class, OpSize Size, Ref ssa0, OpSize Align = OpSize::i8Bit) {\n if (IsTSOEnabled(Class)) {\n return _LoadMemTSO(Class, Size, ssa0, Invalid(), Align, MemOffsetType::SXTX, 1);\n } else {\n return _LoadMem(Class, Size, ssa0, Invalid(), Align, MemOffsetType::SXTX, 1);\n }\n }\n Ref _LoadMemGPRAutoTSO(OpSize Size, Ref ssa0, OpSize Align = OpSize::i8Bit) {\n return _LoadMemAutoTSO(RegClass::GPR, Size, ssa0, Align);\n }\n Ref _LoadMemFPRAutoTSO(OpSize Size, Ref ssa0, OpSize Align = OpSize::i8Bit) {\n return _LoadMemAutoTSO(RegClass::FPR, Size, ssa0, Align);\n }\n\n Ref _LoadMemAutoTSO(RegClass Class, OpSize Size, const AddressMode& A, OpSize Align = OpSize::i8Bit) {\n const bool AtomicTSO = IsTSOEnabled(Class) && !A.NonTSO;\n const auto B = SelectAddressMode(this, A, GetGPROpSize(), CTX->HostFeatures.SupportsTSOImm9, AtomicTSO, Class != RegClass::GPR, Size);\n\n if (AtomicTSO) {\n return _LoadMemTSO(Class, Size, B.Base, B.Index, Align, B.IndexType, B.IndexScale);\n } else {\n return _LoadMem(Class, Size, B.Base, B.Index, Align, B.IndexType, B.IndexScale);\n }\n }\n Ref _LoadMemGPRAutoTSO(OpSize Size, const AddressMode& A, OpSize Align = OpSize::i8Bit) {\n return _LoadMemAutoTSO(RegClass::GPR, Size, A, Align);\n }\n Ref _LoadMemFPRAutoTSO(OpSize Size, const AddressMode& A, OpSize Align = OpSize::i8Bit) {\n return _LoadMemAutoTSO(RegClass::FPR, Size, A, Align);\n }\n\n AddressMode SelectPairAddressMode(AddressMode A, IR::OpSize Size) {\n LOGMAN_THROW_A_FMT(Size != IR::OpSize::iUnsized, \"Invalid size!\");\n const auto SizeInt = IR::OpSizeToSize(Size);\n AddressMode Out {};\n\n signed OffsetEl = A.Offset / SizeInt;\n if ((A.Offset % SizeInt) == 0 && OffsetEl >= -64 && OffsetEl < 64) {\n Out.Offset = A.Offset;\n A.Offset = 0;\n }\n\n Out.Base = LoadEffectiveAddress(this, A, GetGPROpSize(), true, false);\n return Out;\n }\n\n\n RefPair LoadMemPair(RegClass Class, OpSize Size, Ref Base, uint32_t Offset) {\n RefPair Values = AllocatePair(Class, Size);\n _LoadMemPair(Class, Size, Base, Offset, Values.Low, Values.High);\n return Values;\n }\n RefPair LoadMemPairFPR(OpSize Size, Ref Base, uint32_t Offset) {\n return LoadMemPair(RegClass::FPR, Size, Base, Offset);\n }\n\n RefPair _LoadMemPairAutoTSO(RegClass Class, OpSize Size, const AddressMode& A, OpSize Align = OpSize::i8Bit) {\n const bool AtomicTSO = IsTSOEnabled(Class) && !A.NonTSO;\n\n // Use ldp if possible, otherwise fallback on two loads.\n if (!AtomicTSO && !A.Segment && Size >= OpSize::i32Bit && Size <= OpSize::i128Bit) {\n const auto B = SelectPairAddressMode(A, Size);\n return LoadMemPair(Class, Size, B.Base, B.Offset);\n }\n\n AddressMode HighA = A;\n HighA.Offset += 16;\n\n return {\n .Low = _LoadMemAutoTSO(Class, Size, A, Align),\n .High = _LoadMemAutoTSO(Class, Size, HighA, Align),\n };\n }\n RefPair _LoadMemPairFPRAutoTSO(OpSize Size, const AddressMode& A, OpSize Align = OpSize::i8Bit) {\n return _LoadMemPairAutoTSO(RegClass::FPR, Size, A, Align);\n }\n\n Ref _StoreMemAutoTSO(RegClass Class, OpSize Size, const AddressMode& A, Ref Value, OpSize Align = OpSize::i8Bit) {\n const bool AtomicTSO = IsTSOEnabled(Class) && !A.NonTSO;\n const auto B = SelectAddressMode(this, A, GetGPROpSize(), CTX->HostFeatures.SupportsTSOImm9, AtomicTSO, Class != RegClass::GPR, Size);\n\n if (AtomicTSO) {\n return _StoreMemTSO(Class, Size, Value, B.Base, B.Index, Align, B.IndexType, B.IndexScale);\n } else {\n return _StoreMem(Class, Size, Value, B.Base, B.Index, Align, B.IndexType, B.IndexScale);\n }\n }\n Ref _StoreMemGPRAutoTSO(OpSize Size, const AddressMode& A, Ref Value, OpSize Align = OpSize::i8Bit) {\n return _StoreMemAutoTSO(RegClass::GPR, Size, A, Value, Align);\n }\n Ref _StoreMemFPRAutoTSO(OpSize Size, const AddressMode& A, Ref Value, OpSize Align = OpSize::i8Bit) {\n return _StoreMemAutoTSO(RegClass::FPR, Size, A, Value, Align);\n }\n\n void _StoreMemPairAutoTSO(RegClass Class, OpSize Size, const AddressMode& A, Ref Value1, Ref Value2, OpSize Align = OpSize::i8Bit) {\n const auto SizeInt = IR::OpSizeToSize(Size);\n const bool AtomicTSO = IsTSOEnabled(Class) && !A.NonTSO;\n\n // Use stp if possible, otherwise fallback on two stores.\n if (!AtomicTSO && !A.Segment && Size >= OpSize::i32Bit && Size <= OpSize::i128Bit) {\n const auto B = SelectPairAddressMode(A, Size);\n _StoreMemPair(Class, Size, Value1, Value2, B.Base, B.Offset);\n } else {\n auto B = A;\n\n _StoreMemAutoTSO(Class, Size, B, Value1, OpSize::i8Bit);\n B.Offset += SizeInt;\n _StoreMemAutoTSO(Class, Size, B, Value2, OpSize::i8Bit);\n }\n }\n void _StoreMemPairFPRAutoTSO(OpSize Size, const AddressMode& A, Ref Value1, Ref Value2, OpSize Align = OpSize::i8Bit) {\n return _StoreMemPairAutoTSO(RegClass::FPR, Size, A, Value1, Value2, Align);\n }\n\n Ref Pop(IR::OpSize Size, Ref SP_RMW) {\n Ref Value = _AllocateGPR(false);\n _Pop(Size, SP_RMW, Value);\n return Value;\n }\n\n Ref Pop(IR::OpSize Size) {\n Ref SP = _RMWHandle(LoadGPRRegister(X86State::REG_RSP));\n Ref Value = _AllocateGPR(false);\n\n _Pop(Size, SP, Value);\n\n // Store the new stack pointer\n StoreGPRRegister(X86State::REG_RSP, SP);\n return Value;\n }\n\n Ref VZeroExtendOperand(OpSize Size, X86Tables::DecodedOperand Op, Ref Value) {\n bool IsMMX = Op.IsGPR() && Op.Data.GPR.GPR >= X86State::REG_MM_0;\n bool AlreadyExtended = Op.IsGPRDirect() || Op.IsGPRIndirect() || IsMMX;\n\n return AlreadyExtended ? Value : _VMov(Size, Value);\n }\n\n void Push(IR::OpSize Size, Ref Value) {\n auto OldSP = LoadGPRRegister(X86State::REG_RSP);\n auto NewSP = _Push(GetGPROpSize(), Size, Value, OldSP);\n StoreGPRRegister(X86State::REG_RSP, NewSP);\n FlushRegisterCache();\n }\n\n struct ArithRef {\n IREmitter* E {};\n bool IsConstant {};\n union {\n Ref R {};\n uint64_t C;\n };\n\n ArithRef() {}\n\n ArithRef(IREmitter* IREmit, Ref Reference)\n : E(IREmit)\n , IsConstant(false)\n , R(Reference) {}\n\n ArithRef(IREmitter* IREmit, uint64_t K)\n : E(IREmit)\n , IsConstant(true)\n , C(K) {}\n\n ArithRef Neg() {\n return IsConstant ? ArithRef(E, -C) : ArithRef(E, E->_Neg(OpSize::i64Bit, R));\n }\n\n ArithRef And(uint64_t K) {\n return IsConstant ? ArithRef(E, C & K) : ArithRef(E, E->_And(OpSize::i64Bit, R, E->Constant(K)));\n }\n\n ArithRef Presub(uint64_t K) {\n return IsConstant ? ArithRef(E, K - C) : ArithRef(E, E->Sub(OpSize::i64Bit, E->Constant(K), R));\n }\n\n ArithRef Lshl(uint64_t Shift) {\n if (Shift == 0) {\n return *this;\n } else if (IsConstant) {\n return ArithRef(E, C << Shift);\n } else {\n return ArithRef(E, E->_Lshl(OpSize::i64Bit, R, E->Constant(Shift)));\n }\n }\n\n ArithRef Bfe(unsigned Start, unsigned Size) {\n if (IsConstant) {\n return ArithRef(E, (C >> Start) & ((1ull << Size) - 1));\n } else {\n return ArithRef(E, E->_Bfe(OpSize::i64Bit, Size, Start, R));\n }\n }\n\n ArithRef Sbfe(unsigned Start, unsigned Size) {\n if (IsConstant) {\n uint64_t SourceMask = Size == 64 ? ~0ULL : ((1ULL << Size) - 1);\n SourceMask <<= Start;\n\n int64_t NewConstant = (C & SourceMask) >> Start;\n NewConstant <<= 64 - Size;\n NewConstant >>= 64 - Size;\n\n return ArithRef(E, NewConstant);\n } else {\n return ArithRef(E, E->_Sbfe(OpSize::i64Bit, Size, Start, R));\n }\n }\n\n Ref BfiInto(Ref Bitfield, unsigned Start, unsigned Size) {\n if (IsConstant && (Size > 0 && Size < 64)) {\n uint64_t SourceMask = (1ULL << Size) - 1;\n uint64_t SourceMaskShifted = SourceMask << Start;\n\n if (C == 0) {\n return E->_And(OpSize::i64Bit, Bitfield, E->_InlineConstant(~SourceMaskShifted));\n } else if (C == SourceMask) {\n return E->_Or(OpSize::i64Bit, Bitfield, E->_InlineConstant(SourceMaskShifted));\n }\n }\n\n if (IsConstant) {\n return E->_Bfi(OpSize::i64Bit, Size, Start, Bitfield, E->Constant(C));\n } else {\n return E->_Bfi(OpSize::i64Bit, Size, Start, Bitfield, R);\n }\n }\n\n ArithRef MaskBit(OpSize Size) {\n if (IsConstant) {\n uint64_t ShiftMask = Size == OpSize::i64Bit ? 63 : 31;\n uint64_t Result = 1ull << (C & ShiftMask);\n if (ShiftMask == 31) {\n Result &= ((1ull << 32) - 1);\n }\n\n return ArithRef(E, Result);\n } else {\n return ArithRef(E, E->_Lshl(Size, E->Constant(1), R));\n }\n }\n\n Ref Ref() {\n return IsConstant ? E->Constant(C) : R;\n }\n\n bool IsDefinitelyZero() const {\n return IsConstant && C == 0;\n }\n };\n\n ArithRef ARef(Ref R) {\n uint64_t C;\n\n if (IsValueConstant(WrapNode(R), &C)) {\n return ARef(C);\n } else {\n return ArithRef(this, R);\n }\n }\n\n ArithRef ARef(uint64_t K) {\n return ArithRef(this, K);\n }\n\n ///< Segment telemetry tracking\n uint32_t SegmentsNeedReadCheck {~0U};\n void CheckLegacySegmentWrite(Ref NewNode, uint32_t SegmentReg);\n void CheckLegacySegmentRead(Ref NewNode, uint32_t SegmentReg);\n};\n\nconstexpr inline void InstallToTable(auto& FinalTable, const auto& LocalTable) {\n for (const auto& Op : LocalTable) {\n auto OpNum = Op.Op;\n auto Dispatcher = Op.Ptr;\n for (uint8_t i = 0; i < Op.Count; ++i) {\n auto& TableOp = FinalTable[OpNum + i];\n#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED\n if (TableOp.OpcodeDispatcher.OpDispatch) {\n ERROR_AND_DIE_FMT(\"Duplicate Entry {}\", TableOp.Name);\n }\n#endif\n\n TableOp.OpcodeDispatcher.OpDispatch = Dispatcher;\n }\n }\n}\n\n} // namespace FEXCore::IR\n" }, { "path": "FEXCore/Source/Interface/Core/VSyscall/VSyscall.inc", "content": "// SPDX-License-Identifier: MIT\n// This is the vsyscall page for x86_64 guest code\n// This was compiled with nasm with the following source then exported to binary\n\n//BITS 64;\n//\n//align 4096, db 0xcc\n// ; __NR_gettimeofday\n// mov rax, 96\n// syscall\n// ret\n//\n//align 1024, db 0xcc\n// ; __NR_time\n// mov rax, 201\n// syscall\n// ret\n//\n//align 1024, db 0xcc\n// ; __NR_getcpu\n// mov rax, 309\n// syscall\n// ret\n//\n//align 4096, db 0xcc\n\n// We only want one of these pages per FEX process\n// One page\nconst static uint8_t VSyscallData[0x1000] = {\n\t0xB8, 0x60, 0x00, 0x00, 0x00, 0x0F, 0x05, 0xC3, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xB8, 0xC9, 0x00, 0x00, 0x00, 0x0F, 0x05, 0xC3, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xB8, 0x35, 0x01, 0x00, 0x00, 0x0F, 0x05, 0xC3, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC,\n\t0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC\n};\n" }, { "path": "FEXCore/Source/Interface/Core/X86Tables/BaseTables.cpp", "content": "// SPDX-License-Identifier: MIT\n/*\n$info$\ntags: frontend|x86-tables\n$end_info$\n*/\n\n#include \"Interface/Core/X86Tables/X86Tables.h\"\n#include \"Interface/Core/OpcodeDispatcher/BaseTables.h\"\n\n#include \n\n#include \n\nnamespace FEXCore::X86Tables {\nusing namespace InstFlags;\n\nenum Primary_LUT {\n ENTRY_06,\n ENTRY_07,\n ENTRY_0E,\n ENTRY_16,\n ENTRY_17,\n ENTRY_1E,\n ENTRY_1F,\n ENTRY_27,\n ENTRY_2F,\n ENTRY_37,\n ENTRY_3F,\n ENTRY_40,\n ENTRY_48,\n ENTRY_60,\n ENTRY_61,\n ENTRY_63,\n ENTRY_9A,\n ENTRY_A0,\n ENTRY_A1,\n ENTRY_A2,\n ENTRY_A3,\n ENTRY_CE,\n ENTRY_D4,\n ENTRY_D5,\n ENTRY_D6,\n ENTRY_EA,\n ENTRY_MAX,\n};\n\nconstexpr std::array Primary_ArchSelect_LUT = {{\n // ENTRY_06\n {\n {\"PUSH ES\", TYPE_INST, GenFlagsSrcSize(SIZE_16BIT) | FLAGS_DEBUG_MEM_ACCESS, 0, { .OpDispatch = &IR::OpDispatchBuilder::Bind<&IR::OpDispatchBuilder::PUSHSegmentOp, FEXCore::X86Tables::DecodeFlags::FLAG_ES_PREFIX> } },\n {\"\", TYPE_INVALID, FLAGS_NONE, 0, { .OpDispatch = nullptr } },\n },\n // ENTRY_07\n {\n {\"POP ES\", TYPE_INST, GenFlagsSizes(SIZE_16BIT, SIZE_DEF) | FLAGS_DEBUG_MEM_ACCESS, 0, { .OpDispatch = &IR::OpDispatchBuilder::Bind<&IR::OpDispatchBuilder::POPSegmentOp, FEXCore::X86Tables::DecodeFlags::FLAG_ES_PREFIX> } },\n {\"\", TYPE_INVALID, FLAGS_NONE, 0, { .OpDispatch = nullptr } },\n },\n // ENTRY_0E\n {\n {\"PUSH CS\", TYPE_INST, GenFlagsSrcSize(SIZE_16BIT) | FLAGS_DEBUG_MEM_ACCESS, 0, { .OpDispatch = &IR::OpDispatchBuilder::Bind<&IR::OpDispatchBuilder::PUSHSegmentOp, FEXCore::X86Tables::DecodeFlags::FLAG_CS_PREFIX> } },\n {\"\", TYPE_INVALID, FLAGS_NONE, 0, { .OpDispatch = nullptr } },\n },\n // ENTRY_16\n {\n {\"PUSH SS\", TYPE_INST, GenFlagsSrcSize(SIZE_16BIT) | FLAGS_DEBUG_MEM_ACCESS, 0, { .OpDispatch = &IR::OpDispatchBuilder::Bind<&IR::OpDispatchBuilder::PUSHSegmentOp, FEXCore::X86Tables::DecodeFlags::FLAG_SS_PREFIX> } },\n {\"\", TYPE_INVALID, FLAGS_NONE, 0, { .OpDispatch = nullptr } },\n },\n // ENTRY_17\n {\n {\"POP SS\", TYPE_INST, GenFlagsSizes(SIZE_16BIT, SIZE_DEF) | FLAGS_DEBUG_MEM_ACCESS, 0, { .OpDispatch = &IR::OpDispatchBuilder::Bind<&IR::OpDispatchBuilder::POPSegmentOp, FEXCore::X86Tables::DecodeFlags::FLAG_SS_PREFIX> } },\n {\"\", TYPE_INVALID, FLAGS_NONE, 0, { .OpDispatch = nullptr } },\n },\n // ENTRY_1E\n {\n {\"PUSH DS\", TYPE_INST, GenFlagsSrcSize(SIZE_16BIT) | FLAGS_DEBUG_MEM_ACCESS, 0, { .OpDispatch = &IR::OpDispatchBuilder::Bind<&IR::OpDispatchBuilder::PUSHSegmentOp, FEXCore::X86Tables::DecodeFlags::FLAG_DS_PREFIX> } },\n {\"\", TYPE_INVALID, FLAGS_NONE, 0, { .OpDispatch = nullptr } },\n },\n // ENTRY_1F\n {\n {\"POP DS\", TYPE_INST, GenFlagsSizes(SIZE_16BIT, SIZE_DEF) | FLAGS_DEBUG_MEM_ACCESS, 0, { .OpDispatch = &IR::OpDispatchBuilder::Bind<&IR::OpDispatchBuilder::POPSegmentOp, FEXCore::X86Tables::DecodeFlags::FLAG_DS_PREFIX> } },\n {\"\", TYPE_INVALID, FLAGS_NONE, 0, { .OpDispatch = nullptr } },\n },\n // ENTRY_27\n {\n {\"DAA\", TYPE_INST, GenFlagsDstSize(SIZE_8BIT) | FLAGS_SF_DST_RAX, 0, { .OpDispatch = &IR::OpDispatchBuilder::DAAOp } },\n {\"\", TYPE_INVALID, FLAGS_NONE, 0, { .OpDispatch = nullptr } },\n },\n // ENTRY_2F\n {\n {\"DAS\", TYPE_INST, GenFlagsDstSize(SIZE_8BIT) | FLAGS_SF_DST_RAX, 0, { .OpDispatch = &IR::OpDispatchBuilder::DASOp } },\n {\"\", TYPE_INVALID, FLAGS_NONE, 0, { .OpDispatch = nullptr } },\n },\n // ENTRY_37\n {\n {\"AAA\", TYPE_INST, GenFlagsDstSize(SIZE_16BIT) | FLAGS_SF_DST_RAX, 0, { .OpDispatch = &IR::OpDispatchBuilder::AAAOp } },\n {\"\", TYPE_INVALID, FLAGS_NONE, 0, { .OpDispatch = nullptr } },\n },\n // ENTRY_3F\n {\n {\"AAS\", TYPE_INST, GenFlagsDstSize(SIZE_16BIT) | FLAGS_SF_DST_RAX, 0, { .OpDispatch = &IR::OpDispatchBuilder::AASOp } },\n {\"\", TYPE_INVALID, FLAGS_NONE, 0, { .OpDispatch = nullptr } },\n },\n // ENTRY_40\n {\n {\"INC\", TYPE_INST, FLAGS_SF_REX_IN_BYTE, 0, { .OpDispatch = &IR::OpDispatchBuilder::INCOp } },\n // REX\n {\"\", TYPE_REX_PREFIX, FLAGS_NONE, 0},\n },\n // ENTRY_48\n {\n {\"DEC\", TYPE_INST, FLAGS_SF_REX_IN_BYTE, 0, { .OpDispatch = &IR::OpDispatchBuilder::DECOp } },\n {\"\", TYPE_REX_PREFIX, FLAGS_NONE, 0},\n },\n // ENTRY_60\n {\n {\"PUSHA\", TYPE_INST, FLAGS_DEBUG_MEM_ACCESS, 0, { .OpDispatch = &IR::OpDispatchBuilder::PUSHAOp } },\n {\"\", TYPE_INVALID, FLAGS_NONE, 0, { .OpDispatch = nullptr } },\n },\n // ENTRY_61\n {\n {\"POPA\", TYPE_INST, FLAGS_DEBUG_MEM_ACCESS, 0, { .OpDispatch = &IR::OpDispatchBuilder::POPAOp } },\n {\"\", TYPE_INVALID, FLAGS_NONE, 0, { .OpDispatch = nullptr } },\n },\n // ENTRY_63\n {\n {\"ARPL\", TYPE_INST, GenFlagsSameSize(SIZE_16BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST, 0, { .OpDispatch = &IR::OpDispatchBuilder::ARPLOp } },\n {\"MOVSXD\", TYPE_INST, GenFlagsDstSize(SIZE_64BIT) | FLAGS_MODRM, 0, { .OpDispatch = &IR::OpDispatchBuilder::MOVSXDOp } },\n },\n // ENTRY_9A\n {\n {\"CALLF\", TYPE_INST, FLAGS_NONE, 0, { .OpDispatch = nullptr } },\n {\"\", TYPE_INVALID, FLAGS_NONE, 0, { .OpDispatch = nullptr } },\n },\n // ENTRY_A0\n {\n {\"MOV\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_SF_DST_RAX | FLAGS_MEM_OFFSET, 4, { .OpDispatch = &IR::OpDispatchBuilder::MOVOffsetOp } },\n {\"MOV\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_SF_DST_RAX | FLAGS_MEM_OFFSET, 8, { .OpDispatch = &IR::OpDispatchBuilder::MOVOffsetOp } },\n },\n // ENTRY_A1\n {\n {\"MOV\", TYPE_INST, FLAGS_SF_DST_RAX | FLAGS_MEM_OFFSET, 4, { .OpDispatch = &IR::OpDispatchBuilder::MOVOffsetOp } },\n {\"MOV\", TYPE_INST, FLAGS_SF_DST_RAX | FLAGS_MEM_OFFSET, 8, { .OpDispatch = &IR::OpDispatchBuilder::MOVOffsetOp } },\n },\n // ENTRY_A2\n {\n {\"MOV\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_SF_SRC_RAX | FLAGS_MEM_OFFSET, 4, { .OpDispatch = &IR::OpDispatchBuilder::MOVOffsetOp } },\n {\"MOV\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_SF_SRC_RAX | FLAGS_MEM_OFFSET, 8, { .OpDispatch = &IR::OpDispatchBuilder::MOVOffsetOp } },\n },\n // ENTRY_A3\n {\n {\"MOV\", TYPE_INST, FLAGS_SF_SRC_RAX | FLAGS_MEM_OFFSET, 4, { .OpDispatch = &IR::OpDispatchBuilder::MOVOffsetOp } },\n {\"MOV\", TYPE_INST, FLAGS_SF_SRC_RAX | FLAGS_MEM_OFFSET, 8, { .OpDispatch = &IR::OpDispatchBuilder::MOVOffsetOp } },\n },\n // ENTRY_CE\n {\n {\"INTO\", TYPE_INST, FLAGS_NONE, 0, { .OpDispatch = &IR::OpDispatchBuilder::INTOp } },\n {\"\", TYPE_INVALID, FLAGS_NONE, 0, { .OpDispatch = nullptr } },\n },\n // ENTRY_D4\n {\n {\"AAM\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_SF_DST_RAX, 1, { .OpDispatch = &IR::OpDispatchBuilder::AAMOp } },\n {\"\", TYPE_INVALID, FLAGS_NONE, 0, { .OpDispatch = nullptr } },\n },\n // ENTRY_D5\n {\n {\"AAD\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_SF_DST_RAX, 1, { .OpDispatch = &IR::OpDispatchBuilder::AADOp } },\n {\"\", TYPE_INVALID, FLAGS_NONE, 0, { .OpDispatch = nullptr } },\n },\n // ENTRY_D6\n {\n {\"SALC\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_SF_DST_RAX | FLAGS_SF_SRC_RAX, 0, { .OpDispatch = &IR::OpDispatchBuilder::SALCOp } },\n {\"\", TYPE_INVALID, FLAGS_NONE, 0, { .OpDispatch = nullptr } },\n },\n // ENTRY_EA\n {\n {\"JMPF\", TYPE_INST, FLAGS_NONE, 0, { .OpDispatch = nullptr } },\n {\"\", TYPE_INVALID, FLAGS_NONE, 0, { .OpDispatch = nullptr } },\n },\n}};\n\nconst std::array BaseOps = []() consteval {\n std::array Table{};\n\n constexpr U8U8InfoStruct BaseOpTable[] = {\n // Prefixes\n // Operand size overide\n {0x66, 1, X86InstInfo{\"\", TYPE_PREFIX, FLAGS_NONE, 0}},\n // Address size override\n {0x67, 1, X86InstInfo{\"\", TYPE_PREFIX, FLAGS_NONE, 0}},\n {0x26, 1, X86InstInfo{\"ES\", TYPE_LEGACY_PREFIX, FLAGS_NONE, 0}},\n {0x2E, 1, X86InstInfo{\"CS\", TYPE_LEGACY_PREFIX, FLAGS_NONE, 0}},\n {0x36, 1, X86InstInfo{\"SS\", TYPE_LEGACY_PREFIX, FLAGS_NONE, 0}},\n {0x3E, 1, X86InstInfo{\"DS\", TYPE_LEGACY_PREFIX, FLAGS_NONE, 0}},\n // These are still invalid on 64bit\n {0x64, 1, X86InstInfo{\"FS\", TYPE_PREFIX, FLAGS_NONE, 0}},\n {0x65, 1, X86InstInfo{\"GS\", TYPE_PREFIX, FLAGS_NONE, 0}},\n {0xF0, 1, X86InstInfo{\"LOCK\", TYPE_PREFIX, FLAGS_NONE, 0}},\n {0xF2, 1, X86InstInfo{\"REPNE\", TYPE_PREFIX, FLAGS_NONE, 0}},\n {0xF3, 1, X86InstInfo{\"REP\", TYPE_PREFIX, FLAGS_NONE, 0}},\n\n // Instructions\n {0x00, 1, X86InstInfo{\"ADD\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {0x01, 1, X86InstInfo{\"ADD\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_DISPLACE_SIZE_DIV_2, 0}},\n {0x02, 1, X86InstInfo{\"ADD\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM, 0}},\n {0x03, 1, X86InstInfo{\"ADD\", TYPE_INST, FLAGS_MODRM, 0}},\n {0x04, 1, X86InstInfo{\"ADD\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_SF_DST_RAX , 1}},\n {0x05, 1, X86InstInfo{\"ADD\", TYPE_INST, FLAGS_SF_DST_RAX | FLAGS_SRC_SEXT | FLAGS_DISPLACE_SIZE_DIV_2, 4}},\n\n {0x06, 1, X86InstInfo{\"\", TYPE_ARCH_DISPATCHER, FLAGS_NONE, 0, { .Indirect = Primary_ArchSelect_LUT[ENTRY_06] }}},\n {0x07, 1, X86InstInfo{\"\", TYPE_ARCH_DISPATCHER, FLAGS_NONE, 0, { .Indirect = Primary_ArchSelect_LUT[ENTRY_07] }}},\n\n {0x08, 1, X86InstInfo{\"OR\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {0x09, 1, X86InstInfo{\"OR\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {0x0A, 1, X86InstInfo{\"OR\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM, 0}},\n {0x0B, 1, X86InstInfo{\"OR\", TYPE_INST, FLAGS_MODRM, 0}},\n {0x0C, 1, X86InstInfo{\"OR\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_SF_DST_RAX , 1}},\n {0x0D, 1, X86InstInfo{\"OR\", TYPE_INST, FLAGS_SF_DST_RAX | FLAGS_SRC_SEXT | FLAGS_DISPLACE_SIZE_DIV_2, 4}},\n {0x0E, 1, X86InstInfo{\"\", TYPE_ARCH_DISPATCHER, FLAGS_NONE, 0, { .Indirect = Primary_ArchSelect_LUT[ENTRY_0E] }}},\n\n {0x10, 1, X86InstInfo{\"ADC\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {0x11, 1, X86InstInfo{\"ADC\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_DISPLACE_SIZE_DIV_2, 0}},\n {0x12, 1, X86InstInfo{\"ADC\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM, 0}},\n {0x13, 1, X86InstInfo{\"ADC\", TYPE_INST, FLAGS_MODRM, 0}},\n {0x14, 1, X86InstInfo{\"ADC\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_SF_DST_RAX , 1}},\n {0x15, 1, X86InstInfo{\"ADC\", TYPE_INST, FLAGS_SF_DST_RAX | FLAGS_SRC_SEXT | FLAGS_DISPLACE_SIZE_DIV_2, 4}},\n {0x16, 1, X86InstInfo{\"\", TYPE_ARCH_DISPATCHER, FLAGS_NONE, 0, { .Indirect = Primary_ArchSelect_LUT[ENTRY_16] }}},\n {0x17, 1, X86InstInfo{\"\", TYPE_ARCH_DISPATCHER, FLAGS_NONE, 0, { .Indirect = Primary_ArchSelect_LUT[ENTRY_17] }}},\n\n {0x18, 1, X86InstInfo{\"SBB\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {0x19, 1, X86InstInfo{\"SBB\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_DISPLACE_SIZE_DIV_2, 0}},\n {0x1A, 1, X86InstInfo{\"SBB\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM, 0}},\n {0x1B, 1, X86InstInfo{\"SBB\", TYPE_INST, FLAGS_MODRM, 0}},\n {0x1C, 1, X86InstInfo{\"SBB\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_SF_DST_RAX , 1}},\n {0x1D, 1, X86InstInfo{\"SBB\", TYPE_INST, FLAGS_SF_DST_RAX | FLAGS_SRC_SEXT | FLAGS_DISPLACE_SIZE_DIV_2, 4}},\n {0x1E, 1, X86InstInfo{\"\", TYPE_ARCH_DISPATCHER, FLAGS_NONE, 0, { .Indirect = Primary_ArchSelect_LUT[ENTRY_1E] }}},\n {0x1F, 1, X86InstInfo{\"\", TYPE_ARCH_DISPATCHER, FLAGS_NONE, 0, { .Indirect = Primary_ArchSelect_LUT[ENTRY_1F] }}},\n\n {0x20, 1, X86InstInfo{\"AND\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {0x21, 1, X86InstInfo{\"AND\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {0x22, 1, X86InstInfo{\"AND\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM, 0}},\n {0x23, 1, X86InstInfo{\"AND\", TYPE_INST, FLAGS_MODRM, 0}},\n {0x24, 1, X86InstInfo{\"AND\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_SF_DST_RAX , 1}},\n {0x25, 1, X86InstInfo{\"AND\", TYPE_INST, FLAGS_SF_DST_RAX | FLAGS_SRC_SEXT | FLAGS_DISPLACE_SIZE_DIV_2, 4}},\n\n {0x27, 1, X86InstInfo{\"\", TYPE_ARCH_DISPATCHER, FLAGS_NONE, 0, { .Indirect = Primary_ArchSelect_LUT[ENTRY_27] }}},\n {0x28, 1, X86InstInfo{\"SUB\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {0x29, 1, X86InstInfo{\"SUB\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {0x2A, 1, X86InstInfo{\"SUB\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM, 0}},\n {0x2B, 1, X86InstInfo{\"SUB\", TYPE_INST, FLAGS_MODRM, 0}},\n {0x2C, 1, X86InstInfo{\"SUB\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_SF_DST_RAX , 1}},\n {0x2D, 1, X86InstInfo{\"SUB\", TYPE_INST, FLAGS_SF_DST_RAX | FLAGS_SRC_SEXT | FLAGS_DISPLACE_SIZE_DIV_2, 4}},\n {0x2F, 1, X86InstInfo{\"\", TYPE_ARCH_DISPATCHER, FLAGS_NONE, 0, { .Indirect = Primary_ArchSelect_LUT[ENTRY_2F] }}},\n\n {0x30, 1, X86InstInfo{\"XOR\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {0x31, 1, X86InstInfo{\"XOR\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {0x32, 1, X86InstInfo{\"XOR\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM, 0}},\n {0x33, 1, X86InstInfo{\"XOR\", TYPE_INST, FLAGS_MODRM, 0}},\n {0x34, 1, X86InstInfo{\"XOR\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_SF_DST_RAX , 1}},\n {0x35, 1, X86InstInfo{\"XOR\", TYPE_INST, FLAGS_SF_DST_RAX | FLAGS_SRC_SEXT | FLAGS_DISPLACE_SIZE_DIV_2, 4}},\n\n {0x37, 1, X86InstInfo{\"\", TYPE_ARCH_DISPATCHER, FLAGS_NONE, 0, { .Indirect = Primary_ArchSelect_LUT[ENTRY_37] }}},\n {0x38, 1, X86InstInfo{\"CMP\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {0x39, 1, X86InstInfo{\"CMP\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {0x3A, 1, X86InstInfo{\"CMP\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM, 0}},\n {0x3B, 1, X86InstInfo{\"CMP\", TYPE_INST, FLAGS_MODRM, 0}},\n {0x3C, 1, X86InstInfo{\"CMP\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_SF_DST_RAX , 1}},\n {0x3D, 1, X86InstInfo{\"CMP\", TYPE_INST, FLAGS_SF_DST_RAX | FLAGS_SRC_SEXT | FLAGS_DISPLACE_SIZE_DIV_2, 4}},\n {0x3F, 1, X86InstInfo{\"\", TYPE_ARCH_DISPATCHER, FLAGS_NONE, 0, { .Indirect = Primary_ArchSelect_LUT[ENTRY_3F] }}},\n\n {0x40, 8, X86InstInfo{\"\", TYPE_ARCH_DISPATCHER, FLAGS_NONE, 0, { .Indirect = Primary_ArchSelect_LUT[ENTRY_40] }}},\n {0x48, 8, X86InstInfo{\"\", TYPE_ARCH_DISPATCHER, FLAGS_NONE, 0, { .Indirect = Primary_ArchSelect_LUT[ENTRY_48] }}},\n\n {0x50, 8, X86InstInfo{\"PUSH\", TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_SF_REX_IN_BYTE | FLAGS_DEBUG_MEM_ACCESS , 0}},\n {0x58, 8, X86InstInfo{\"POP\", TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_SF_REX_IN_BYTE | FLAGS_DEBUG_MEM_ACCESS , 0}},\n\n\n {0x60, 1, X86InstInfo{\"\", TYPE_ARCH_DISPATCHER, FLAGS_NONE, 0, { .Indirect = Primary_ArchSelect_LUT[ENTRY_60] }}},\n {0x61, 1, X86InstInfo{\"\", TYPE_ARCH_DISPATCHER, FLAGS_NONE, 0, { .Indirect = Primary_ArchSelect_LUT[ENTRY_61] }}},\n {0x62, 1, X86InstInfo{\"\", TYPE_GROUP_EVEX, FLAGS_NONE, 0}},\n {0x63, 1, X86InstInfo{\"\", TYPE_ARCH_DISPATCHER, FLAGS_NONE, 0, { .Indirect = Primary_ArchSelect_LUT[ENTRY_63] }}},\n\n {0x68, 1, X86InstInfo{\"PUSH\", TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_DEBUG_MEM_ACCESS | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_SRC_SEXT, 4}},\n {0x69, 1, X86InstInfo{\"IMUL\", TYPE_INST, FLAGS_MODRM | FLAGS_SRC_SEXT | FLAGS_DISPLACE_SIZE_DIV_2, 4}},\n {0x6A, 1, X86InstInfo{\"PUSH\", TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_DEBUG_MEM_ACCESS | FLAGS_SRC_SEXT , 1}},\n {0x6B, 1, X86InstInfo{\"IMUL\", TYPE_INST, FLAGS_MODRM | FLAGS_SRC_SEXT , 1}},\n\n // This should just throw a GP\n {0x6C, 1, X86InstInfo{\"INSB\", TYPE_INST, FLAGS_BLOCK_END, 0}},\n {0x6D, 1, X86InstInfo{\"INSW\", TYPE_INST, FLAGS_BLOCK_END, 0}},\n {0x6E, 1, X86InstInfo{\"OUTS\", TYPE_INST, FLAGS_BLOCK_END, 0}},\n {0x6F, 1, X86InstInfo{\"OUTS\", TYPE_INST, FLAGS_BLOCK_END, 0}},\n\n {0x70, 1, X86InstInfo{\"JO\", TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_SETS_RIP | FLAGS_SRC_SEXT , 1}},\n {0x71, 1, X86InstInfo{\"JNO\", TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_SETS_RIP | FLAGS_SRC_SEXT , 1}},\n {0x72, 1, X86InstInfo{\"JB\", TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_SETS_RIP | FLAGS_SRC_SEXT , 1}},\n {0x73, 1, X86InstInfo{\"JNB\", TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_SETS_RIP | FLAGS_SRC_SEXT , 1}},\n {0x74, 1, X86InstInfo{\"JZ\", TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_SETS_RIP | FLAGS_SRC_SEXT , 1}},\n {0x75, 1, X86InstInfo{\"JNZ\", TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_SETS_RIP | FLAGS_SRC_SEXT , 1}},\n {0x76, 1, X86InstInfo{\"JBE\", TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_SETS_RIP | FLAGS_SRC_SEXT , 1}},\n {0x77, 1, X86InstInfo{\"JNBE\", TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_SETS_RIP | FLAGS_SRC_SEXT , 1}},\n {0x78, 1, X86InstInfo{\"JS\", TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_SETS_RIP | FLAGS_SRC_SEXT , 1}},\n {0x79, 1, X86InstInfo{\"JNS\", TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_SETS_RIP | FLAGS_SRC_SEXT , 1}},\n {0x7A, 1, X86InstInfo{\"JP\", TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_SETS_RIP | FLAGS_SRC_SEXT , 1}},\n {0x7B, 1, X86InstInfo{\"JNP\", TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_SETS_RIP | FLAGS_SRC_SEXT , 1}},\n {0x7C, 1, X86InstInfo{\"JL\", TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_SETS_RIP | FLAGS_SRC_SEXT , 1}},\n {0x7D, 1, X86InstInfo{\"JNL\", TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_SETS_RIP | FLAGS_SRC_SEXT , 1}},\n {0x7E, 1, X86InstInfo{\"JLE\", TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_SETS_RIP | FLAGS_SRC_SEXT , 1}},\n {0x7F, 1, X86InstInfo{\"JNLE\", TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_SETS_RIP | FLAGS_SRC_SEXT , 1}},\n\n {0x84, 1, X86InstInfo{\"TEST\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {0x85, 1, X86InstInfo{\"TEST\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {0x86, 1, X86InstInfo{\"XCHG\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {0x87, 1, X86InstInfo{\"XCHG\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n\n {0x88, 1, X86InstInfo{\"MOV\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {0x89, 1, X86InstInfo{\"MOV\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {0x8A, 1, X86InstInfo{\"MOV\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM, 0}},\n {0x8B, 1, X86InstInfo{\"MOV\", TYPE_INST, FLAGS_MODRM, 0}},\n {0x8C, 1, X86InstInfo{\"MOV\", TYPE_INST, GenFlagsSrcSize(SIZE_16BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {0x8D, 1, X86InstInfo{\"LEA\", TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_MODRM, 0}},\n {0x8E, 1, X86InstInfo{\"MOV\", TYPE_INST, GenFlagsSameSize(SIZE_16BIT) | FLAGS_MODRM, 0}},\n {0x8F, 1, X86InstInfo{\"POP\", TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_MOD_ZERO_REG | FLAGS_DEBUG_MEM_ACCESS, 0}},\n {0x90, 8, X86InstInfo{\"XCHG\", TYPE_INST, FLAGS_SF_REX_IN_BYTE | FLAGS_SF_SRC_RAX, 0}},\n {0x98, 1, X86InstInfo{\"CDQE\", TYPE_INST, FLAGS_SF_DST_RAX | FLAGS_SF_SRC_RAX, 0}},\n {0x99, 1, X86InstInfo{\"CQO\", TYPE_INST, FLAGS_SF_DST_RDX | FLAGS_SF_SRC_RAX, 0}},\n\n // These three are all X87 instructions\n {0x9B, 1, X86InstInfo{\"FWAIT\", TYPE_INST, FLAGS_NONE, 0}},\n {0x9C, 1, X86InstInfo{\"PUSHF\", TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF), 0}},\n {0x9D, 1, X86InstInfo{\"POPF\", TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_BLOCK_END, 0}},\n\n {0x9E, 1, X86InstInfo{\"SAHF\", TYPE_INST, FLAGS_NONE, 0}},\n {0x9F, 1, X86InstInfo{\"LAHF\", TYPE_INST, FLAGS_NONE, 0}},\n\n {0xA0, 1, X86InstInfo{\"\", TYPE_ARCH_DISPATCHER, FLAGS_NONE, 0, { .Indirect = Primary_ArchSelect_LUT[ENTRY_A0] }}},\n {0xA1, 1, X86InstInfo{\"\", TYPE_ARCH_DISPATCHER, FLAGS_NONE, 0, { .Indirect = Primary_ArchSelect_LUT[ENTRY_A1] }}},\n {0xA2, 1, X86InstInfo{\"\", TYPE_ARCH_DISPATCHER, FLAGS_NONE, 0, { .Indirect = Primary_ArchSelect_LUT[ENTRY_A2] }}},\n {0xA3, 1, X86InstInfo{\"\", TYPE_ARCH_DISPATCHER, FLAGS_NONE, 0, { .Indirect = Primary_ArchSelect_LUT[ENTRY_A3] }}},\n\n {0xA4, 1, X86InstInfo{\"MOVSB\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_DEBUG_MEM_ACCESS, 0}},\n {0xA5, 1, X86InstInfo{\"MOVS\", TYPE_INST, FLAGS_DEBUG_MEM_ACCESS, 0}},\n {0xA6, 1, X86InstInfo{\"CMPSB\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_DEBUG_MEM_ACCESS, 0}},\n {0xA7, 1, X86InstInfo{\"CMPS\", TYPE_INST, FLAGS_DEBUG_MEM_ACCESS, 0}},\n\n {0xA8, 1, X86InstInfo{\"TEST\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_SF_DST_RAX , 1}},\n {0xA9, 1, X86InstInfo{\"TEST\", TYPE_INST, FLAGS_SF_DST_RAX | FLAGS_SRC_SEXT | FLAGS_DISPLACE_SIZE_DIV_2, 4}},\n {0xAA, 1, X86InstInfo{\"STOS\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_DEBUG_MEM_ACCESS | FLAGS_SF_SRC_RAX, 0}},\n {0xAB, 1, X86InstInfo{\"STOS\", TYPE_INST, FLAGS_DEBUG_MEM_ACCESS | FLAGS_SF_SRC_RAX, 0}},\n {0xAC, 1, X86InstInfo{\"LODS\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_SF_DST_RAX | FLAGS_DEBUG_MEM_ACCESS, 0}},\n {0xAD, 1, X86InstInfo{\"LODS\", TYPE_INST, FLAGS_SF_DST_RAX | FLAGS_DEBUG_MEM_ACCESS, 0}},\n {0xAE, 1, X86InstInfo{\"SCAS\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_DEBUG_MEM_ACCESS | FLAGS_SF_SRC_RAX, 0}},\n {0xAF, 1, X86InstInfo{\"SCAS\", TYPE_INST, FLAGS_DEBUG_MEM_ACCESS | FLAGS_SF_SRC_RAX, 0}},\n\n {0xB0, 8, X86InstInfo{\"MOV\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_SF_REX_IN_BYTE , 1}},\n {0xB8, 8, X86InstInfo{\"MOV\", TYPE_INST, FLAGS_SF_REX_IN_BYTE | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_DISPLACE_SIZE_MUL_2, 4}},\n\n {0xC2, 1, X86InstInfo{\"RET\", TYPE_INST, FLAGS_SETS_RIP | FLAGS_BLOCK_END, 2}},\n {0xC3, 1, X86InstInfo{\"RET\", TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_SETS_RIP | FLAGS_BLOCK_END , 0}},\n {0xC8, 1, X86InstInfo{\"ENTER\", TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_DEBUG_MEM_ACCESS , 3}},\n {0xC9, 1, X86InstInfo{\"LEAVE\", TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_DEBUG_MEM_ACCESS , 0}},\n {0xCA, 1, X86InstInfo{\"RETF\", TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_SETS_RIP | FLAGS_BLOCK_END, 2}},\n {0xCB, 1, X86InstInfo{\"RETF\", TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_SETS_RIP | FLAGS_BLOCK_END, 0}},\n {0xCC, 1, X86InstInfo{\"INT3\", TYPE_INST, FLAGS_BLOCK_END, 0}},\n {0xCD, 1, X86InstInfo{\"INT\", TYPE_INST, DEFAULT_SYSCALL_FLAGS, 1}},\n {0xCE, 1, X86InstInfo{\"\", TYPE_ARCH_DISPATCHER, FLAGS_NONE, 0, { .Indirect = Primary_ArchSelect_LUT[ENTRY_CE] }}},\n {0xCF, 1, X86InstInfo{\"IRET\", TYPE_INST, FLAGS_SETS_RIP | FLAGS_BLOCK_END, 0}},\n\n {0xD4, 1, X86InstInfo{\"\", TYPE_ARCH_DISPATCHER, FLAGS_NONE, 0, { .Indirect = Primary_ArchSelect_LUT[ENTRY_D4] }}},\n {0xD5, 1, X86InstInfo{\"\", TYPE_ARCH_DISPATCHER, FLAGS_NONE, 0, { .Indirect = Primary_ArchSelect_LUT[ENTRY_D5] }}},\n {0xD6, 1, X86InstInfo{\"\", TYPE_ARCH_DISPATCHER, FLAGS_NONE, 0, { .Indirect = Primary_ArchSelect_LUT[ENTRY_D6] }}},\n {0xD7, 1, X86InstInfo{\"XLAT\", TYPE_INST, FLAGS_DEBUG_MEM_ACCESS, 0}},\n\n {0xE0, 1, X86InstInfo{\"LOOPNE\", TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_SETS_RIP | FLAGS_SRC_SEXT | FLAGS_SF_SRC_RCX, 1}},\n {0xE1, 1, X86InstInfo{\"LOOPE\", TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_SETS_RIP | FLAGS_SRC_SEXT | FLAGS_SF_SRC_RCX, 1}},\n {0xE2, 1, X86InstInfo{\"LOOP\", TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_SETS_RIP | FLAGS_SRC_SEXT | FLAGS_SF_SRC_RCX, 1}},\n {0xE3, 1, X86InstInfo{\"JrCXZ\", TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_SETS_RIP | FLAGS_SRC_SEXT , 1}},\n\n // Should just throw GP\n {0xE4, 2, X86InstInfo{\"IN\", TYPE_INST, FLAGS_BLOCK_END, 1}},\n {0xE6, 2, X86InstInfo{\"OUT\", TYPE_INST, FLAGS_BLOCK_END, 1}},\n\n {0xE8, 1, X86InstInfo{\"CALL\", TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_SETS_RIP | FLAGS_SRC_SEXT | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_BLOCK_END | FLAGS_CALL , 4}},\n {0xE9, 1, X86InstInfo{\"JMP\", TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_SETS_RIP | FLAGS_SRC_SEXT | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_BLOCK_END , 4}},\n {0xEB, 1, X86InstInfo{\"JMP\", TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_SETS_RIP | FLAGS_SRC_SEXT | FLAGS_BLOCK_END , 1}},\n\n // Should just throw GP\n {0xEC, 2, X86InstInfo{\"IN\", TYPE_INST, FLAGS_BLOCK_END, 0}},\n {0xEE, 2, X86InstInfo{\"OUT\", TYPE_INST, FLAGS_BLOCK_END, 0}},\n\n {0xF1, 1, X86InstInfo{\"INT1\", TYPE_INST, FLAGS_BLOCK_END, 0}},\n {0xF4, 1, X86InstInfo{\"HLT\", TYPE_INST, FLAGS_BLOCK_END, 0}},\n {0xF5, 1, X86InstInfo{\"CMC\", TYPE_INST, FLAGS_NONE, 0}},\n {0xF8, 1, X86InstInfo{\"CLC\", TYPE_INST, FLAGS_NONE, 0}},\n {0xF9, 1, X86InstInfo{\"STC\", TYPE_INST, FLAGS_NONE, 0}},\n {0xFA, 1, X86InstInfo{\"CLI\", TYPE_INST, FLAGS_NONE, 0}},\n {0xFB, 1, X86InstInfo{\"STI\", TYPE_INST, FLAGS_NONE, 0}},\n {0xFC, 1, X86InstInfo{\"CLD\", TYPE_INST, FLAGS_NONE, 0}},\n {0xFD, 1, X86InstInfo{\"STD\", TYPE_INST, FLAGS_NONE, 0}},\n\n // Two Byte table\n {0x0F, 1, X86InstInfo{\"\", TYPE_SECONDARY_TABLE_PREFIX, FLAGS_NONE, 0}},\n\n // x87 table\n {0xD8, 8, X86InstInfo{\"\", TYPE_X87_TABLE_PREFIX, FLAGS_MODRM, 0}},\n\n // ModRM table\n // MoreBytes field repurposed for valid bits mask\n {0x80, 1, X86InstInfo{\"\", TYPE_GROUP_1, FLAGS_MODRM, 0}},\n {0x81, 1, X86InstInfo{\"\", TYPE_GROUP_1, FLAGS_MODRM, 1}},\n {0x82, 1, X86InstInfo{\"\", TYPE_GROUP_1, FLAGS_MODRM, 2}},\n {0x83, 1, X86InstInfo{\"\", TYPE_GROUP_1, FLAGS_MODRM, 3}},\n {0xC0, 1, X86InstInfo{\"\", TYPE_GROUP_2, FLAGS_MODRM, 0}},\n {0xC1, 1, X86InstInfo{\"\", TYPE_GROUP_2, FLAGS_MODRM, 1}},\n {0xD0, 1, X86InstInfo{\"\", TYPE_GROUP_2, FLAGS_MODRM, 2}},\n {0xD1, 1, X86InstInfo{\"\", TYPE_GROUP_2, FLAGS_MODRM, 3}},\n {0xD2, 1, X86InstInfo{\"\", TYPE_GROUP_2, FLAGS_MODRM, 4}},\n {0xD3, 1, X86InstInfo{\"\", TYPE_GROUP_2, FLAGS_MODRM, 5}},\n {0xF6, 1, X86InstInfo{\"\", TYPE_GROUP_3, FLAGS_MODRM, 0}},\n {0xF7, 1, X86InstInfo{\"\", TYPE_GROUP_3, FLAGS_MODRM, 1}},\n {0xFE, 1, X86InstInfo{\"\", TYPE_GROUP_4, FLAGS_MODRM, 0}},\n {0xFF, 1, X86InstInfo{\"\", TYPE_GROUP_5, FLAGS_MODRM, 0}},\n\n // Group 11\n {0xC6, 1, X86InstInfo{\"\", TYPE_GROUP_11, FLAGS_MODRM, 0}},\n {0xC7, 1, X86InstInfo{\"\", TYPE_GROUP_11, FLAGS_MODRM, 1}},\n\n // VEX table\n {0xC4, 2, X86InstInfo{\"\", TYPE_VEX_TABLE_PREFIX, FLAGS_NONE, 0}},\n };\n\n GenerateTable(Table.data(), BaseOpTable, std::size(BaseOpTable));\n IR::InstallToTable(Table, IR::OpDispatch_BaseOpTable);\n\n return Table;\n}();\n\n}\n" }, { "path": "FEXCore/Source/Interface/Core/X86Tables/DDDTables.cpp", "content": "// SPDX-License-Identifier: MIT\n/*\n$info$\ntags: frontend|x86-tables\n$end_info$\n*/\n\n#include \"Interface/Core/X86Tables/X86Tables.h\"\n#include \"Interface/Core/OpcodeDispatcher/DDDTables.h\"\n\n#include \n\nnamespace FEXCore::X86Tables {\nusing namespace InstFlags;\n\nconstexpr std::array DDDNowOps = []() consteval {\n std::array Table{};\n constexpr U8U8InfoStruct DDDNowOpTable[] = {\n {0x0C, 1, X86InstInfo{\"PI2FW\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {0x0D, 1, X86InstInfo{\"PI2FD\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {0x1C, 1, X86InstInfo{\"PF2IW\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {0x1D, 1, X86InstInfo{\"PF2ID\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n\n // Inverse 3DNow! These two instructions are Geode product line specific\n // No CPUID for these, you're expected to read ID_CONFIG_MSR (1250h) bit 1\n {0x86, 1, X86InstInfo{\"PFRCPV\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {0x87, 1, X86InstInfo{\"PFRSQRTV\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n\n {0x8A, 1, X86InstInfo{\"PFNACC\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {0x8E, 1, X86InstInfo{\"PFPNACC\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n\n {0x90, 1, X86InstInfo{\"PFCMPGE\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {0x94, 1, X86InstInfo{\"PFMIN\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {0x96, 1, X86InstInfo{\"PFRCP\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {0x97, 1, X86InstInfo{\"PFRSQRT\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n\n {0x9A, 1, X86InstInfo{\"PFSUB\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {0x9E, 1, X86InstInfo{\"PFADD\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n\n {0xA0, 1, X86InstInfo{\"PFCMPGT\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {0xA4, 1, X86InstInfo{\"PFMAX\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {0xA6, 1, X86InstInfo{\"PFRCPIT1\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {0xA7, 1, X86InstInfo{\"PFRSQIT1\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n\n {0xAA, 1, X86InstInfo{\"PFSUBR\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {0xAE, 1, X86InstInfo{\"PFACC\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n\n {0xB0, 1, X86InstInfo{\"PFCMPEQ\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {0xB4, 1, X86InstInfo{\"PFMUL\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {0xB6, 1, X86InstInfo{\"PFRCPIT2\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {0xB7, 1, X86InstInfo{\"PMULHRW\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n\n {0xBB, 1, X86InstInfo{\"PSWAPD\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {0xBF, 1, X86InstInfo{\"PAVGUSB\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n };\n\n GenerateTable(Table.data(), DDDNowOpTable, std::size(DDDNowOpTable));\n\n IR::InstallToTable(Table, IR::OpDispatch_DDDTable);\n return Table;\n}();\n\n}\n" }, { "path": "FEXCore/Source/Interface/Core/X86Tables/H0F38Tables.cpp", "content": "// SPDX-License-Identifier: MIT\n/*\n$info$\ntags: frontend|x86-tables\n$end_info$\n*/\n\n#include \"Interface/Core/X86Tables/X86Tables.h\"\n#include \"Interface/Core/OpcodeDispatcher/H0F38Tables.h\"\n\n#include \n#include \n\nnamespace FEXCore::X86Tables {\nusing namespace InstFlags;\nconstexpr std::array H0F38TableOps = []() consteval {\n std::array Table{};\n\n#define OPD(prefix, opcode) (((prefix) << 8) | opcode)\n constexpr uint16_t PF_38_NONE = 0;\n constexpr uint16_t PF_38_66 = (1U << 0);\n constexpr uint16_t PF_38_F2 = (1U << 1);\n constexpr uint16_t PF_38_F3 = (1U << 2);\n\n constexpr U16U8InfoStruct H0F38Table[] = {\n {OPD(PF_38_NONE, 0x00), 1, X86InstInfo{\"PSHUFB\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {OPD(PF_38_66, 0x00), 1, X86InstInfo{\"PSHUFB\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {OPD(PF_38_NONE, 0x01), 1, X86InstInfo{\"PHADDW\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {OPD(PF_38_66, 0x01), 1, X86InstInfo{\"PHADDW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {OPD(PF_38_NONE, 0x02), 1, X86InstInfo{\"PHADDD\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {OPD(PF_38_66, 0x02), 1, X86InstInfo{\"PHADDD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {OPD(PF_38_NONE, 0x03), 1, X86InstInfo{\"PHADDSW\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {OPD(PF_38_66, 0x03), 1, X86InstInfo{\"PHADDSW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {OPD(PF_38_NONE, 0x04), 1, X86InstInfo{\"PMADDUBSW\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {OPD(PF_38_66, 0x04), 1, X86InstInfo{\"PMADDUBSW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {OPD(PF_38_NONE, 0x05), 1, X86InstInfo{\"PHSUBW\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {OPD(PF_38_66, 0x05), 1, X86InstInfo{\"PHSUBW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {OPD(PF_38_NONE, 0x06), 1, X86InstInfo{\"PHSUBD\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {OPD(PF_38_66, 0x06), 1, X86InstInfo{\"PHSUBD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {OPD(PF_38_NONE, 0x07), 1, X86InstInfo{\"PHSUBSW\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {OPD(PF_38_66, 0x07), 1, X86InstInfo{\"PHSUBSW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {OPD(PF_38_NONE, 0x08), 1, X86InstInfo{\"PSIGNB\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {OPD(PF_38_66, 0x08), 1, X86InstInfo{\"PSIGNB\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {OPD(PF_38_NONE, 0x09), 1, X86InstInfo{\"PSIGNW\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {OPD(PF_38_66, 0x09), 1, X86InstInfo{\"PSIGNW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {OPD(PF_38_NONE, 0x0A), 1, X86InstInfo{\"PSIGND\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {OPD(PF_38_66, 0x0A), 1, X86InstInfo{\"PSIGND\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {OPD(PF_38_NONE, 0x0B), 1, X86InstInfo{\"PMULHRSW\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {OPD(PF_38_66, 0x0B), 1, X86InstInfo{\"PMULHRSW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n\n {OPD(PF_38_66, 0x10), 1, X86InstInfo{\"PBLENDVB\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {OPD(PF_38_66, 0x14), 1, X86InstInfo{\"BLENDVPS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {OPD(PF_38_66, 0x15), 1, X86InstInfo{\"BLENDVPD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {OPD(PF_38_66, 0x17), 1, X86InstInfo{\"PTEST\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {OPD(PF_38_NONE, 0x1C), 1, X86InstInfo{\"PABSB\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {OPD(PF_38_66, 0x1C), 1, X86InstInfo{\"PABSB\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {OPD(PF_38_NONE, 0x1D), 1, X86InstInfo{\"PABSW\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {OPD(PF_38_66, 0x1D), 1, X86InstInfo{\"PABSW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {OPD(PF_38_NONE, 0x1E), 1, X86InstInfo{\"PABSD\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {OPD(PF_38_66, 0x1E), 1, X86InstInfo{\"PABSD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n\n {OPD(PF_38_66, 0x20), 1, X86InstInfo{\"PMOVSXBW\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {OPD(PF_38_66, 0x21), 1, X86InstInfo{\"PMOVSXBD\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {OPD(PF_38_66, 0x22), 1, X86InstInfo{\"PMOVSXBQ\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_16BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {OPD(PF_38_66, 0x23), 1, X86InstInfo{\"PMOVSXWD\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {OPD(PF_38_66, 0x24), 1, X86InstInfo{\"PMOVSXWQ\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {OPD(PF_38_66, 0x25), 1, X86InstInfo{\"PMOVSXDQ\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {OPD(PF_38_66, 0x28), 1, X86InstInfo{\"PMULDQ\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {OPD(PF_38_66, 0x29), 1, X86InstInfo{\"PCMPEQQ\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {OPD(PF_38_66, 0x2A), 1, X86InstInfo{\"MOVNTDQA\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY | FLAGS_XMM_FLAGS, 0}},\n {OPD(PF_38_66, 0x2B), 1, X86InstInfo{\"PACKUSDW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n\n {OPD(PF_38_66, 0x30), 1, X86InstInfo{\"PMOVZXBW\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {OPD(PF_38_66, 0x31), 1, X86InstInfo{\"PMOVZXBD\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {OPD(PF_38_66, 0x32), 1, X86InstInfo{\"PMOVZXBQ\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_16BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {OPD(PF_38_66, 0x33), 1, X86InstInfo{\"PMOVZXWD\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {OPD(PF_38_66, 0x34), 1, X86InstInfo{\"PMOVZXWQ\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {OPD(PF_38_66, 0x35), 1, X86InstInfo{\"PMOVZXDQ\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {OPD(PF_38_66, 0x37), 1, X86InstInfo{\"PCMPGTQ\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {OPD(PF_38_66, 0x38), 1, X86InstInfo{\"PMINSB\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {OPD(PF_38_66, 0x39), 1, X86InstInfo{\"PMINSD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {OPD(PF_38_66, 0x3A), 1, X86InstInfo{\"PMINUW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {OPD(PF_38_66, 0x3B), 1, X86InstInfo{\"PMINUD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {OPD(PF_38_66, 0x3C), 1, X86InstInfo{\"PMAXSB\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {OPD(PF_38_66, 0x3D), 1, X86InstInfo{\"PMAXSD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {OPD(PF_38_66, 0x3E), 1, X86InstInfo{\"PMAXUW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {OPD(PF_38_66, 0x3F), 1, X86InstInfo{\"PMAXUD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n\n {OPD(PF_38_66, 0x40), 1, X86InstInfo{\"PMULLD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {OPD(PF_38_66, 0x41), 1, X86InstInfo{\"PHMINPOSUW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n\n {OPD(PF_38_NONE, 0xC8), 1, X86InstInfo{\"SHA1NEXTE\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {OPD(PF_38_NONE, 0xC9), 1, X86InstInfo{\"SHA1MSG1\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {OPD(PF_38_NONE, 0xCA), 1, X86InstInfo{\"SHA1MSG2\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n\n {OPD(PF_38_NONE, 0xCB), 1, X86InstInfo{\"SHA256RNDS2\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {OPD(PF_38_NONE, 0xCC), 1, X86InstInfo{\"SHA256MSG1\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {OPD(PF_38_NONE, 0xCD), 1, X86InstInfo{\"SHA256MSG2\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n\n {OPD(PF_38_66, 0xDB), 1, X86InstInfo{\"AESIMC\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {OPD(PF_38_66, 0xDC), 1, X86InstInfo{\"AESENC\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {OPD(PF_38_66, 0xDD), 1, X86InstInfo{\"AESENCLAST\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {OPD(PF_38_66, 0xDE), 1, X86InstInfo{\"AESDEC\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {OPD(PF_38_66, 0xDF), 1, X86InstInfo{\"AESDECLAST\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n\n {OPD(PF_38_NONE, 0xF0), 1, X86InstInfo{\"MOVBE\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY, 0}},\n {OPD(PF_38_NONE, 0xF1), 1, X86InstInfo{\"MOVBE\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_MOD_MEM_ONLY, 0}},\n\n {OPD(PF_38_66, 0xF0), 1, X86InstInfo{\"MOVBE\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY, 0}},\n {OPD(PF_38_66, 0xF1), 1, X86InstInfo{\"MOVBE\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_MOD_MEM_ONLY, 0}},\n\n {OPD(PF_38_F2, 0xF0), 1, X86InstInfo{\"CRC32\", TYPE_INST, GenFlagsSizes(SIZE_DEF, SIZE_8BIT) | FLAGS_MODRM, 0}},\n {OPD(PF_38_F2, 0xF1), 1, X86InstInfo{\"CRC32\", TYPE_INST, FLAGS_MODRM, 0}},\n {OPD(PF_38_66 | PF_38_F2, 0xF0), 1, X86InstInfo{\"CRC32\", TYPE_INST, GenFlagsSizes(SIZE_DEF, SIZE_8BIT) | FLAGS_MODRM, 0}},\n {OPD(PF_38_66 | PF_38_F2, 0xF1), 1, X86InstInfo{\"CRC32\", TYPE_INST, FLAGS_MODRM, 0}},\n\n {OPD(PF_38_66, 0xF6), 1, X86InstInfo{\"ADCX\", TYPE_INST, FLAGS_MODRM | FLAGS_NO_OVERLAY66, 0}},\n {OPD(PF_38_F3, 0xF6), 1, X86InstInfo{\"ADOX\", TYPE_INST, FLAGS_MODRM, 0}},\n };\n#undef OPD\n\n GenerateTable(Table.data(), H0F38Table, std::size(H0F38Table));\n\n IR::InstallToTable(Table, IR::OpDispatch_H0F38Table);\n return Table;\n}();\n\n}\n" }, { "path": "FEXCore/Source/Interface/Core/X86Tables/H0F3ATables.cpp", "content": "// SPDX-License-Identifier: MIT\n/*\n$info$\ntags: frontend|x86-tables\n$end_info$\n*/\n\n#include \"Interface/Core/X86Tables/X86Tables.h\"\n#include \"Interface/Core/OpcodeDispatcher/H0F3ATables.h\"\n\n#include \n\n#include \n#include \n\nnamespace FEXCore::X86Tables {\nusing namespace InstFlags;\n#define OPD(REX, prefix, opcode) ((REX << 9) | (prefix << 8) | opcode)\nconstexpr uint16_t PF_3A_NONE = 0;\nconstexpr uint16_t PF_3A_66 = 1;\n\nenum H0F3A_LUT {\n ENTRY_1_3A_66_16,\n ENTRY_1_3A_66_22,\n ENTRY_MAX,\n};\n\nconstexpr std::array H0F3A_ArchSelect_LUT = {{\n // ENTRY_1_3A_66_16\n {\n {\"\", TYPE_INVALID, FLAGS_NONE, 0, { .OpDispatch = nullptr } },\n {\"PEXTRQ\", TYPE_INST, GenFlagsSizes(SIZE_64BIT, SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_DST_GPR | FLAGS_XMM_FLAGS, 1, { .OpDispatch = &IR::OpDispatchBuilder::Bind<&IR::OpDispatchBuilder::PExtrOp, IR::OpSize::i64Bit> }},\n },\n // ENTRY_1_3A_66_22\n {\n {\"\", TYPE_INVALID, FLAGS_NONE, 0, { .OpDispatch = nullptr } },\n {\"PINSRQ\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_SRC_GPR, 1, { .OpDispatch = &IR::OpDispatchBuilder::PINSROp }},\n },\n}};\n\nconstexpr std::array H0F3ATableOps = []() consteval {\n std::array Table{};\n auto TableGen = []() consteval {\n constexpr U16U8InfoStruct Table[] = {\n {OPD(REX, PF_3A_NONE, 0x0F), 1, X86InstInfo{\"PALIGNR\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 1}},\n {OPD(REX, PF_3A_66, 0x08), 1, X86InstInfo{\"ROUNDPS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 1}},\n {OPD(REX, PF_3A_66, 0x09), 1, X86InstInfo{\"ROUNDPD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 1}},\n {OPD(REX, PF_3A_66, 0x0A), 1, X86InstInfo{\"ROUNDSS\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 1}},\n {OPD(REX, PF_3A_66, 0x0B), 1, X86InstInfo{\"ROUNDSD\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 1}},\n {OPD(REX, PF_3A_66, 0x0C), 1, X86InstInfo{\"BLENDPS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 1}},\n {OPD(REX, PF_3A_66, 0x0D), 1, X86InstInfo{\"BLENDPD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 1}},\n {OPD(REX, PF_3A_66, 0x0E), 1, X86InstInfo{\"PBLENDW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 1}},\n {OPD(REX, PF_3A_66, 0x0F), 1, X86InstInfo{\"PALIGNR\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 1}},\n\n {OPD(REX, PF_3A_66, 0x14), 1, X86InstInfo{\"PEXTRB\", TYPE_INST, GenFlagsSizes(SIZE_32BIT, SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_DST_GPR | FLAGS_XMM_FLAGS, 1}},\n {OPD(REX, PF_3A_66, 0x15), 1, X86InstInfo{\"PEXTRW\", TYPE_INST, GenFlagsSizes(SIZE_16BIT, SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_DST_GPR | FLAGS_XMM_FLAGS, 1}},\n {OPD(REX, PF_3A_66, 0x17), 1, X86InstInfo{\"EXTRACTPS\", TYPE_INST, GenFlagsSizes(SIZE_32BIT, SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_DST_GPR | FLAGS_XMM_FLAGS, 1}},\n\n {OPD(REX, PF_3A_66, 0x20), 1, X86InstInfo{\"PINSRB\", TYPE_INST, GenFlagsDstSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_SRC_GPR, 1}},\n {OPD(REX, PF_3A_66, 0x21), 1, X86InstInfo{\"INSERTPS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 1}},\n {OPD(REX, PF_3A_66, 0x40), 1, X86InstInfo{\"DPPS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 1}},\n {OPD(REX, PF_3A_66, 0x41), 1, X86InstInfo{\"DPPD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 1}},\n {OPD(REX, PF_3A_66, 0x42), 1, X86InstInfo{\"MPSADBW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 1}},\n {OPD(REX, PF_3A_66, 0x44), 1, X86InstInfo{\"PCLMULQDQ\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 1}},\n\n {OPD(REX, PF_3A_66, 0x60), 1, X86InstInfo{\"PCMPESTRM\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 1}},\n {OPD(REX, PF_3A_66, 0x61), 1, X86InstInfo{\"PCMPESTRI\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 1}},\n {OPD(REX, PF_3A_66, 0x62), 1, X86InstInfo{\"PCMPISTRM\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 1}},\n {OPD(REX, PF_3A_66, 0x63), 1, X86InstInfo{\"PCMPISTRI\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 1}},\n\n {OPD(REX, PF_3A_NONE, 0xCC), 1, X86InstInfo{\"SHA1RNDS4\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 1}},\n\n {OPD(REX, PF_3A_66, 0xDF), 1, X86InstInfo{\"AESKEYGENASSIST\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 1}},\n };\n return std::to_array(Table);\n };\n constexpr auto H0F3ATable_IgnoresREX0 = TableGen.template operator()<0>();\n constexpr auto H0F3ATable_IgnoresREX1 = TableGen.template operator()<1>();\n\n GenerateTable(Table.data(), H0F3ATable_IgnoresREX0.data(), H0F3ATable_IgnoresREX0.size());\n GenerateTable(Table.data(), H0F3ATable_IgnoresREX1.data(), H0F3ATable_IgnoresREX1.size());\n\n constexpr U16U8InfoStruct TableNeedsREX0[] = {\n {OPD(0, PF_3A_66, 0x16), 1, X86InstInfo{\"PEXTRD\", TYPE_INST, GenFlagsSizes(SIZE_32BIT, SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_DST_GPR | FLAGS_XMM_FLAGS, 1}},\n {OPD(0, PF_3A_66, 0x22), 1, X86InstInfo{\"PINSRD\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_SRC_GPR, 1}},\n };\n GenerateTable(Table.data(), TableNeedsREX0, std::size(TableNeedsREX0));\n\n constexpr U16U8InfoStruct TableNeedsREX1[] = {\n {OPD(1, PF_3A_66, 0x16), 1, X86InstInfo{\"\", TYPE_ARCH_DISPATCHER, FLAGS_NONE, 0, { .Indirect = H0F3A_ArchSelect_LUT[ENTRY_1_3A_66_16] }}},\n {OPD(1, PF_3A_66, 0x22), 1, X86InstInfo{\"\", TYPE_ARCH_DISPATCHER, FLAGS_NONE, 0, { .Indirect = H0F3A_ArchSelect_LUT[ENTRY_1_3A_66_22] }}},\n };\n GenerateTable(Table.data(), TableNeedsREX1, std::size(TableNeedsREX1));\n\n IR::InstallToTable(Table, IR::OpDispatch_H0F3ATableIgnoreREX);\n IR::InstallToTable(Table, IR::OpDispatch_H0F3ATableNeedsREX0);\n\n return Table;\n}();\n}\n" }, { "path": "FEXCore/Source/Interface/Core/X86Tables/PrimaryGroupTables.cpp", "content": "// SPDX-License-Identifier: MIT\n/*\n$info$\ntags: frontend|x86-tables\n$end_info$\n*/\n\n#include \"Interface/Core/X86Tables/X86Tables.h\"\n#include \"Interface/Core/OpcodeDispatcher/PrimaryGroupTables.h\"\n\n#include \n\n#include \n\nnamespace FEXCore::X86Tables {\nusing namespace InstFlags;\nenum PrimaryGroup_LUT {\n ENTRY_1_82_0,\n ENTRY_1_82_1,\n ENTRY_1_82_2,\n ENTRY_1_82_3,\n ENTRY_1_82_4,\n ENTRY_1_82_5,\n ENTRY_1_82_6,\n ENTRY_1_82_7,\n ENTRY_MAX,\n};\n\nconstexpr std::array PrimaryGroup_ArchSelect_LUT = {{\n {\n {\"ADD\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST, 1, { .OpDispatch = &IR::OpDispatchBuilder::SecondaryALUOp }},\n {\"\", TYPE_INVALID, FLAGS_NONE, 0, { .OpDispatch = nullptr } },\n },\n {\n {\"OR\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST, 1, { .OpDispatch = &IR::OpDispatchBuilder::SecondaryALUOp }},\n {\"\", TYPE_INVALID, FLAGS_NONE, 0, { .OpDispatch = nullptr } },\n },\n {\n {\"ADC\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST, 1, { .OpDispatch = &IR::OpDispatchBuilder::Bind<&IR::OpDispatchBuilder::ADCOp, 1> }},\n {\"\", TYPE_INVALID, FLAGS_NONE, 0, { .OpDispatch = nullptr } },\n },\n {\n {\"SBB\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST, 1, { .OpDispatch = &IR::OpDispatchBuilder::Bind<&IR::OpDispatchBuilder::SBBOp, 1> }},\n {\"\", TYPE_INVALID, FLAGS_NONE, 0, { .OpDispatch = nullptr } },\n },\n {\n {\"AND\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST, 1, { .OpDispatch = &IR::OpDispatchBuilder::SecondaryALUOp }},\n {\"\", TYPE_INVALID, FLAGS_NONE, 0, { .OpDispatch = nullptr } },\n },\n {\n {\"SUB\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST, 1, { .OpDispatch = &IR::OpDispatchBuilder::SecondaryALUOp }},\n {\"\", TYPE_INVALID, FLAGS_NONE, 0, { .OpDispatch = nullptr } },\n },\n {\n {\"XOR\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST, 1, { .OpDispatch = &IR::OpDispatchBuilder::SecondaryALUOp }},\n {\"\", TYPE_INVALID, FLAGS_NONE, 0, { .OpDispatch = nullptr } },\n },\n {\n {\"CMP\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST, 1, { .OpDispatch = &IR::OpDispatchBuilder::Bind<&IR::OpDispatchBuilder::CMPOp, 1> }},\n {\"\", TYPE_INVALID, FLAGS_NONE, 0, { .OpDispatch = nullptr } },\n },\n}};\n\nconstexpr std::array PrimaryInstGroupOps = []() consteval {\n std::array Table{};\n#define OPD(group, prefix, Reg) (((group - FEXCore::X86Tables::TYPE_GROUP_1) << 6) | (prefix) << 3 | (Reg))\n constexpr U16U8InfoStruct PrimaryGroupOpTable[] = {\n // GROUP_1 | 0x80 | reg\n {OPD(TYPE_GROUP_1, OpToIndex(0x80), 0), 1, X86InstInfo{\"ADD\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST, 1}},\n {OPD(TYPE_GROUP_1, OpToIndex(0x80), 1), 1, X86InstInfo{\"OR\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST, 1}},\n {OPD(TYPE_GROUP_1, OpToIndex(0x80), 2), 1, X86InstInfo{\"ADC\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST, 1}},\n {OPD(TYPE_GROUP_1, OpToIndex(0x80), 3), 1, X86InstInfo{\"SBB\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST, 1}},\n {OPD(TYPE_GROUP_1, OpToIndex(0x80), 4), 1, X86InstInfo{\"AND\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST, 1}},\n {OPD(TYPE_GROUP_1, OpToIndex(0x80), 5), 1, X86InstInfo{\"SUB\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST, 1}},\n {OPD(TYPE_GROUP_1, OpToIndex(0x80), 6), 1, X86InstInfo{\"XOR\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST, 1}},\n {OPD(TYPE_GROUP_1, OpToIndex(0x80), 7), 1, X86InstInfo{\"CMP\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST, 1}},\n\n {OPD(TYPE_GROUP_1, OpToIndex(0x81), 0), 1, X86InstInfo{\"ADD\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SRC_SEXT64BIT | FLAGS_DISPLACE_SIZE_DIV_2, 4}},\n {OPD(TYPE_GROUP_1, OpToIndex(0x81), 1), 1, X86InstInfo{\"OR\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SRC_SEXT64BIT | FLAGS_DISPLACE_SIZE_DIV_2, 4}},\n {OPD(TYPE_GROUP_1, OpToIndex(0x81), 2), 1, X86InstInfo{\"ADC\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SRC_SEXT64BIT | FLAGS_DISPLACE_SIZE_DIV_2, 4}},\n {OPD(TYPE_GROUP_1, OpToIndex(0x81), 3), 1, X86InstInfo{\"SBB\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SRC_SEXT64BIT | FLAGS_DISPLACE_SIZE_DIV_2, 4}},\n {OPD(TYPE_GROUP_1, OpToIndex(0x81), 4), 1, X86InstInfo{\"AND\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SRC_SEXT64BIT | FLAGS_DISPLACE_SIZE_DIV_2, 4}},\n {OPD(TYPE_GROUP_1, OpToIndex(0x81), 5), 1, X86InstInfo{\"SUB\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SRC_SEXT64BIT | FLAGS_DISPLACE_SIZE_DIV_2, 4}},\n {OPD(TYPE_GROUP_1, OpToIndex(0x81), 6), 1, X86InstInfo{\"XOR\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SRC_SEXT64BIT | FLAGS_DISPLACE_SIZE_DIV_2, 4}},\n {OPD(TYPE_GROUP_1, OpToIndex(0x81), 7), 1, X86InstInfo{\"CMP\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SRC_SEXT64BIT | FLAGS_DISPLACE_SIZE_DIV_2, 4}},\n\n // Duplicates the 0x80 opcode group\n {OPD(TYPE_GROUP_1, OpToIndex(0x82), 0), 1, X86InstInfo{\"\", TYPE_ARCH_DISPATCHER, FLAGS_NONE, 0, { .Indirect = PrimaryGroup_ArchSelect_LUT[ENTRY_1_82_0] }}},\n {OPD(TYPE_GROUP_1, OpToIndex(0x82), 1), 1, X86InstInfo{\"\", TYPE_ARCH_DISPATCHER, FLAGS_NONE, 0, { .Indirect = PrimaryGroup_ArchSelect_LUT[ENTRY_1_82_1] }}},\n {OPD(TYPE_GROUP_1, OpToIndex(0x82), 2), 1, X86InstInfo{\"\", TYPE_ARCH_DISPATCHER, FLAGS_NONE, 0, { .Indirect = PrimaryGroup_ArchSelect_LUT[ENTRY_1_82_2] }}},\n {OPD(TYPE_GROUP_1, OpToIndex(0x82), 3), 1, X86InstInfo{\"\", TYPE_ARCH_DISPATCHER, FLAGS_NONE, 0, { .Indirect = PrimaryGroup_ArchSelect_LUT[ENTRY_1_82_3] }}},\n {OPD(TYPE_GROUP_1, OpToIndex(0x82), 4), 1, X86InstInfo{\"\", TYPE_ARCH_DISPATCHER, FLAGS_NONE, 0, { .Indirect = PrimaryGroup_ArchSelect_LUT[ENTRY_1_82_4] }}},\n {OPD(TYPE_GROUP_1, OpToIndex(0x82), 5), 1, X86InstInfo{\"\", TYPE_ARCH_DISPATCHER, FLAGS_NONE, 0, { .Indirect = PrimaryGroup_ArchSelect_LUT[ENTRY_1_82_5] }}},\n {OPD(TYPE_GROUP_1, OpToIndex(0x82), 6), 1, X86InstInfo{\"\", TYPE_ARCH_DISPATCHER, FLAGS_NONE, 0, { .Indirect = PrimaryGroup_ArchSelect_LUT[ENTRY_1_82_6] }}},\n {OPD(TYPE_GROUP_1, OpToIndex(0x82), 7), 1, X86InstInfo{\"\", TYPE_ARCH_DISPATCHER, FLAGS_NONE, 0, { .Indirect = PrimaryGroup_ArchSelect_LUT[ENTRY_1_82_7] }}},\n\n {OPD(TYPE_GROUP_1, OpToIndex(0x83), 0), 1, X86InstInfo{\"ADD\", TYPE_INST, FLAGS_SRC_SEXT | FLAGS_MODRM | FLAGS_SF_MOD_DST, 1}},\n {OPD(TYPE_GROUP_1, OpToIndex(0x83), 1), 1, X86InstInfo{\"OR\", TYPE_INST, FLAGS_SRC_SEXT | FLAGS_MODRM | FLAGS_SF_MOD_DST, 1}},\n {OPD(TYPE_GROUP_1, OpToIndex(0x83), 2), 1, X86InstInfo{\"ADC\", TYPE_INST, FLAGS_SRC_SEXT | FLAGS_MODRM | FLAGS_SF_MOD_DST, 1}},\n {OPD(TYPE_GROUP_1, OpToIndex(0x83), 3), 1, X86InstInfo{\"SBB\", TYPE_INST, FLAGS_SRC_SEXT | FLAGS_MODRM | FLAGS_SF_MOD_DST, 1}},\n {OPD(TYPE_GROUP_1, OpToIndex(0x83), 4), 1, X86InstInfo{\"AND\", TYPE_INST, FLAGS_SRC_SEXT | FLAGS_MODRM | FLAGS_SF_MOD_DST, 1}},\n {OPD(TYPE_GROUP_1, OpToIndex(0x83), 5), 1, X86InstInfo{\"SUB\", TYPE_INST, FLAGS_SRC_SEXT | FLAGS_MODRM | FLAGS_SF_MOD_DST, 1}},\n {OPD(TYPE_GROUP_1, OpToIndex(0x83), 6), 1, X86InstInfo{\"XOR\", TYPE_INST, FLAGS_SRC_SEXT | FLAGS_MODRM | FLAGS_SF_MOD_DST, 1}},\n {OPD(TYPE_GROUP_1, OpToIndex(0x83), 7), 1, X86InstInfo{\"CMP\", TYPE_INST, FLAGS_SRC_SEXT | FLAGS_MODRM | FLAGS_SF_MOD_DST, 1}},\n\n // GROUP 2\n {OPD(TYPE_GROUP_2, OpToIndex(0xC0), 0), 1, X86InstInfo{\"ROL\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST, 1}},\n {OPD(TYPE_GROUP_2, OpToIndex(0xC0), 1), 1, X86InstInfo{\"ROR\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST, 1}},\n {OPD(TYPE_GROUP_2, OpToIndex(0xC0), 2), 1, X86InstInfo{\"RCL\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST, 1}},\n {OPD(TYPE_GROUP_2, OpToIndex(0xC0), 3), 1, X86InstInfo{\"RCR\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST, 1}},\n {OPD(TYPE_GROUP_2, OpToIndex(0xC0), 4), 1, X86InstInfo{\"SHL\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST, 1}},\n {OPD(TYPE_GROUP_2, OpToIndex(0xC0), 5), 1, X86InstInfo{\"SHR\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST, 1}},\n {OPD(TYPE_GROUP_2, OpToIndex(0xC0), 6), 1, X86InstInfo{\"SHL\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST, 1}},\n {OPD(TYPE_GROUP_2, OpToIndex(0xC0), 7), 1, X86InstInfo{\"SAR\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST, 1}},\n\n {OPD(TYPE_GROUP_2, OpToIndex(0xC1), 0), 1, X86InstInfo{\"ROL\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 1}},\n {OPD(TYPE_GROUP_2, OpToIndex(0xC1), 1), 1, X86InstInfo{\"ROR\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 1}},\n {OPD(TYPE_GROUP_2, OpToIndex(0xC1), 2), 1, X86InstInfo{\"RCL\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 1}},\n {OPD(TYPE_GROUP_2, OpToIndex(0xC1), 3), 1, X86InstInfo{\"RCR\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 1}},\n {OPD(TYPE_GROUP_2, OpToIndex(0xC1), 4), 1, X86InstInfo{\"SHL\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 1}},\n {OPD(TYPE_GROUP_2, OpToIndex(0xC1), 5), 1, X86InstInfo{\"SHR\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 1}},\n {OPD(TYPE_GROUP_2, OpToIndex(0xC1), 6), 1, X86InstInfo{\"SHL\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 1}},\n {OPD(TYPE_GROUP_2, OpToIndex(0xC1), 7), 1, X86InstInfo{\"SAR\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 1}},\n\n {OPD(TYPE_GROUP_2, OpToIndex(0xD0), 0), 1, X86InstInfo{\"ROL\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {OPD(TYPE_GROUP_2, OpToIndex(0xD0), 1), 1, X86InstInfo{\"ROR\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {OPD(TYPE_GROUP_2, OpToIndex(0xD0), 2), 1, X86InstInfo{\"RCL\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {OPD(TYPE_GROUP_2, OpToIndex(0xD0), 3), 1, X86InstInfo{\"RCR\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {OPD(TYPE_GROUP_2, OpToIndex(0xD0), 4), 1, X86InstInfo{\"SHL\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {OPD(TYPE_GROUP_2, OpToIndex(0xD0), 5), 1, X86InstInfo{\"SHR\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {OPD(TYPE_GROUP_2, OpToIndex(0xD0), 6), 1, X86InstInfo{\"SHL\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {OPD(TYPE_GROUP_2, OpToIndex(0xD0), 7), 1, X86InstInfo{\"SAR\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n\n {OPD(TYPE_GROUP_2, OpToIndex(0xD1), 0), 1, X86InstInfo{\"ROL\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {OPD(TYPE_GROUP_2, OpToIndex(0xD1), 1), 1, X86InstInfo{\"ROR\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {OPD(TYPE_GROUP_2, OpToIndex(0xD1), 2), 1, X86InstInfo{\"RCL\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {OPD(TYPE_GROUP_2, OpToIndex(0xD1), 3), 1, X86InstInfo{\"RCR\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {OPD(TYPE_GROUP_2, OpToIndex(0xD1), 4), 1, X86InstInfo{\"SHL\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {OPD(TYPE_GROUP_2, OpToIndex(0xD1), 5), 1, X86InstInfo{\"SHR\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {OPD(TYPE_GROUP_2, OpToIndex(0xD1), 6), 1, X86InstInfo{\"SHL\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {OPD(TYPE_GROUP_2, OpToIndex(0xD1), 7), 1, X86InstInfo{\"SAR\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n\n {OPD(TYPE_GROUP_2, OpToIndex(0xD2), 0), 1, X86InstInfo{\"ROL\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_SRC_RCX, 0}},\n {OPD(TYPE_GROUP_2, OpToIndex(0xD2), 1), 1, X86InstInfo{\"ROR\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_SRC_RCX, 0}},\n {OPD(TYPE_GROUP_2, OpToIndex(0xD2), 2), 1, X86InstInfo{\"RCL\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_SRC_RCX, 0}},\n {OPD(TYPE_GROUP_2, OpToIndex(0xD2), 3), 1, X86InstInfo{\"RCR\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_SRC_RCX, 0}},\n {OPD(TYPE_GROUP_2, OpToIndex(0xD2), 4), 1, X86InstInfo{\"SHL\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_SRC_RCX, 0}},\n {OPD(TYPE_GROUP_2, OpToIndex(0xD2), 5), 1, X86InstInfo{\"SHR\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_SRC_RCX, 0}},\n {OPD(TYPE_GROUP_2, OpToIndex(0xD2), 6), 1, X86InstInfo{\"SHL\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_SRC_RCX, 0}},\n {OPD(TYPE_GROUP_2, OpToIndex(0xD2), 7), 1, X86InstInfo{\"SAR\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_SRC_RCX, 0}},\n\n {OPD(TYPE_GROUP_2, OpToIndex(0xD3), 0), 1, X86InstInfo{\"ROL\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_SRC_RCX, 0}},\n {OPD(TYPE_GROUP_2, OpToIndex(0xD3), 1), 1, X86InstInfo{\"ROR\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_SRC_RCX, 0}},\n {OPD(TYPE_GROUP_2, OpToIndex(0xD3), 2), 1, X86InstInfo{\"RCL\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_SRC_RCX, 0}},\n {OPD(TYPE_GROUP_2, OpToIndex(0xD3), 3), 1, X86InstInfo{\"RCR\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_SRC_RCX, 0}},\n {OPD(TYPE_GROUP_2, OpToIndex(0xD3), 4), 1, X86InstInfo{\"SHL\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_SRC_RCX, 0}},\n {OPD(TYPE_GROUP_2, OpToIndex(0xD3), 5), 1, X86InstInfo{\"SHR\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_SRC_RCX, 0}},\n {OPD(TYPE_GROUP_2, OpToIndex(0xD3), 6), 1, X86InstInfo{\"SHL\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_SRC_RCX, 0}},\n {OPD(TYPE_GROUP_2, OpToIndex(0xD3), 7), 1, X86InstInfo{\"SAR\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_SRC_RCX, 0}},\n\n // GROUP 3\n {OPD(TYPE_GROUP_3, OpToIndex(0xF6), 0), 1, X86InstInfo{\"TEST\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST, 1}},\n {OPD(TYPE_GROUP_3, OpToIndex(0xF6), 1), 1, X86InstInfo{\"TEST\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST, 1}},\n {OPD(TYPE_GROUP_3, OpToIndex(0xF6), 2), 1, X86InstInfo{\"NOT\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {OPD(TYPE_GROUP_3, OpToIndex(0xF6), 3), 1, X86InstInfo{\"NEG\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {OPD(TYPE_GROUP_3, OpToIndex(0xF6), 4), 1, X86InstInfo{\"MUL\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {OPD(TYPE_GROUP_3, OpToIndex(0xF6), 5), 1, X86InstInfo{\"IMUL\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {OPD(TYPE_GROUP_3, OpToIndex(0xF6), 6), 1, X86InstInfo{\"DIV\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {OPD(TYPE_GROUP_3, OpToIndex(0xF6), 7), 1, X86InstInfo{\"IDIV\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n\n {OPD(TYPE_GROUP_3, OpToIndex(0xF7), 0), 1, X86InstInfo{\"TEST\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SRC_SEXT64BIT | FLAGS_DISPLACE_SIZE_DIV_2, 4}},\n {OPD(TYPE_GROUP_3, OpToIndex(0xF7), 1), 1, X86InstInfo{\"TEST\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SRC_SEXT64BIT | FLAGS_DISPLACE_SIZE_DIV_2, 4}},\n {OPD(TYPE_GROUP_3, OpToIndex(0xF7), 2), 1, X86InstInfo{\"NOT\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {OPD(TYPE_GROUP_3, OpToIndex(0xF7), 3), 1, X86InstInfo{\"NEG\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {OPD(TYPE_GROUP_3, OpToIndex(0xF7), 4), 1, X86InstInfo{\"MUL\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {OPD(TYPE_GROUP_3, OpToIndex(0xF7), 5), 1, X86InstInfo{\"IMUL\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {OPD(TYPE_GROUP_3, OpToIndex(0xF7), 6), 1, X86InstInfo{\"DIV\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {OPD(TYPE_GROUP_3, OpToIndex(0xF7), 7), 1, X86InstInfo{\"IDIV\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n\n // GROUP 4\n {OPD(TYPE_GROUP_4, OpToIndex(0xFE), 0), 1, X86InstInfo{\"INC\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {OPD(TYPE_GROUP_4, OpToIndex(0xFE), 1), 1, X86InstInfo{\"DEC\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {OPD(TYPE_GROUP_4, OpToIndex(0xFE), 2), 6, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n\n // GROUP 5\n {OPD(TYPE_GROUP_5, OpToIndex(0xFF), 0), 1, X86InstInfo{\"INC\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {OPD(TYPE_GROUP_5, OpToIndex(0xFF), 1), 1, X86InstInfo{\"DEC\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {OPD(TYPE_GROUP_5, OpToIndex(0xFF), 2), 1, X86InstInfo{\"CALL\", TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_SETS_RIP | FLAGS_MODRM | FLAGS_BLOCK_END | FLAGS_CALL , 0}},\n {OPD(TYPE_GROUP_5, OpToIndex(0xFF), 3), 1, X86InstInfo{\"CALLF\", TYPE_INST, FLAGS_SETS_RIP | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_MOD_MEM_ONLY | FLAGS_BLOCK_END, 0}},\n {OPD(TYPE_GROUP_5, OpToIndex(0xFF), 4), 1, X86InstInfo{\"JMP\", TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_SETS_RIP | FLAGS_MODRM | FLAGS_BLOCK_END , 0}},\n {OPD(TYPE_GROUP_5, OpToIndex(0xFF), 5), 1, X86InstInfo{\"JMPF\", TYPE_INST, FLAGS_SETS_RIP | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_MOD_MEM_ONLY | FLAGS_BLOCK_END, 0}},\n {OPD(TYPE_GROUP_5, OpToIndex(0xFF), 6), 1, X86InstInfo{\"PUSH\", TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_DEBUG_MEM_ACCESS | FLAGS_MODRM, 0}},\n {OPD(TYPE_GROUP_5, OpToIndex(0xFF), 7), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n\n // GROUP 11\n {OPD(TYPE_GROUP_11, OpToIndex(0xC6), 0), 1, X86InstInfo{\"MOV\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SRC_SEXT, 1}},\n {OPD(TYPE_GROUP_11, OpToIndex(0xC6), 1), 5, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_11, OpToIndex(0xC6), 7), 1, X86InstInfo{\"XABORT\", TYPE_INST, FLAGS_MODRM, 1}},\n {OPD(TYPE_GROUP_11, OpToIndex(0xC7), 0), 1, X86InstInfo{\"MOV\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SRC_SEXT | FLAGS_DISPLACE_SIZE_DIV_2, 4}},\n {OPD(TYPE_GROUP_11, OpToIndex(0xC7), 1), 5, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_11, OpToIndex(0xC7), 7), 1, X86InstInfo{\"XBEGIN\", TYPE_INST, FLAGS_MODRM | FLAGS_SRC_SEXT | FLAGS_SETS_RIP | FLAGS_DISPLACE_SIZE_DIV_2, 4}},\n };\n\n GenerateTable(Table.data(), PrimaryGroupOpTable, std::size(PrimaryGroupOpTable));\n\n IR::InstallToTable(Table, IR::OpDispatch_PrimaryGroupTables);\n return Table;\n}();\n\n}\n" }, { "path": "FEXCore/Source/Interface/Core/X86Tables/SecondaryGroupTables.cpp", "content": "// SPDX-License-Identifier: MIT\n/*\n$info$\ntags: frontend|x86-tables\n$end_info$\n*/\n\n#include \"Interface/Core/X86Tables/X86Tables.h\"\n#include \"Interface/Core/OpcodeDispatcher/SecondaryGroupTables.h\"\n\n#include \n#include \n\nnamespace FEXCore::X86Tables {\nusing namespace InstFlags;\nconstexpr uint16_t PF_NONE = 0;\nconstexpr uint16_t PF_F3 = 1;\nconstexpr uint16_t PF_66 = 2;\nconstexpr uint16_t PF_F2 = 3;\n#define OPD(group, prefix, Reg) (((group - FEXCore::X86Tables::TYPE_GROUP_6) << 5) | (prefix) << 3 | (Reg))\n\nenum SecondGroup_LUT {\n ENTRY_15_F3_0,\n ENTRY_15_F3_1,\n ENTRY_15_F3_2,\n ENTRY_15_F3_3,\n ENTRY_MAX,\n};\n\nconstexpr std::array SecondGroup_ArchSelect_LUT = {{\n // ENTRY_15_F3_0\n {\n {\"\", TYPE_INVALID, FLAGS_NONE, 0, { .OpDispatch = nullptr } },\n {\"RDFSBASE\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_MOD_REG_ONLY, 0, { .OpDispatch = &IR::OpDispatchBuilder::Bind<&IR::OpDispatchBuilder::ReadSegmentReg, IR::OpDispatchBuilder::Segment::FS> } },\n },\n // ENTRY_15_F3_1\n {\n {\"\", TYPE_INVALID, FLAGS_NONE, 0, { .OpDispatch = nullptr } },\n {\"RDGSBASE\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_MOD_REG_ONLY, 0, { .OpDispatch = &IR::OpDispatchBuilder::Bind<&IR::OpDispatchBuilder::ReadSegmentReg, IR::OpDispatchBuilder::Segment::GS> } },\n },\n // ENTRY_15_F3_2\n {\n {\"\", TYPE_INVALID, FLAGS_NONE, 0, { .OpDispatch = nullptr } },\n {\"WRFSBASE\", TYPE_INST, GenFlagsDstSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_SF_MOD_REG_ONLY, 0, { .OpDispatch = &IR::OpDispatchBuilder::Bind<&IR::OpDispatchBuilder::WriteSegmentReg, IR::OpDispatchBuilder::Segment::FS> } },\n },\n // ENTRY_15_F3_3\n {\n {\"\", TYPE_INVALID, FLAGS_NONE, 0, { .OpDispatch = nullptr } },\n {\"WRGSBASE\", TYPE_INST, GenFlagsDstSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_SF_MOD_REG_ONLY, 0, { .OpDispatch = &IR::OpDispatchBuilder::Bind<&IR::OpDispatchBuilder::WriteSegmentReg, IR::OpDispatchBuilder::Segment::GS> } },\n },\n}};\n\nconstexpr std::array SecondInstGroupOps = []() consteval {\n std::array Table{};\n constexpr U16U8InfoStruct SecondaryExtensionOpTable[] = {\n // GROUP 1\n // GROUP 2\n // GROUP 3\n // GROUP 4\n // GROUP 5\n // Pulls from other MODRM table\n\n // GROUP 6\n {OPD(TYPE_GROUP_6, PF_NONE, 0), 1, X86InstInfo{\"SLDT\", TYPE_UNDEC, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {OPD(TYPE_GROUP_6, PF_NONE, 1), 1, X86InstInfo{\"STR\", TYPE_PRIV, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {OPD(TYPE_GROUP_6, PF_NONE, 2), 1, X86InstInfo{\"LLDT\", TYPE_PRIV, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_6, PF_NONE, 3), 1, X86InstInfo{\"LTR\", TYPE_INST, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_6, PF_NONE, 4), 1, X86InstInfo{\"VERR\", TYPE_UNDEC, FLAGS_MODRM, 0}},\n {OPD(TYPE_GROUP_6, PF_NONE, 5), 1, X86InstInfo{\"VERW\", TYPE_UNDEC, FLAGS_MODRM, 0}},\n {OPD(TYPE_GROUP_6, PF_NONE, 6), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_6, PF_NONE, 7), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n\n {OPD(TYPE_GROUP_6, PF_F3, 0), 1, X86InstInfo{\"SLDT\", TYPE_UNDEC, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {OPD(TYPE_GROUP_6, PF_F3, 1), 1, X86InstInfo{\"STR\", TYPE_PRIV, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {OPD(TYPE_GROUP_6, PF_F3, 2), 1, X86InstInfo{\"LLDT\", TYPE_PRIV, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_6, PF_F3, 3), 1, X86InstInfo{\"LTR\", TYPE_INST, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_6, PF_F3, 4), 1, X86InstInfo{\"VERR\", TYPE_UNDEC, FLAGS_MODRM, 0}},\n {OPD(TYPE_GROUP_6, PF_F3, 5), 1, X86InstInfo{\"VERW\", TYPE_UNDEC, FLAGS_MODRM, 0}},\n {OPD(TYPE_GROUP_6, PF_F3, 6), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_6, PF_F3, 7), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n\n {OPD(TYPE_GROUP_6, PF_66, 0), 1, X86InstInfo{\"SLDT\", TYPE_UNDEC, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {OPD(TYPE_GROUP_6, PF_66, 1), 1, X86InstInfo{\"STR\", TYPE_PRIV, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {OPD(TYPE_GROUP_6, PF_66, 2), 1, X86InstInfo{\"LLDT\", TYPE_PRIV, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_6, PF_66, 3), 1, X86InstInfo{\"LTR\", TYPE_INST, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_6, PF_66, 4), 1, X86InstInfo{\"VERR\", TYPE_UNDEC, FLAGS_MODRM, 0}},\n {OPD(TYPE_GROUP_6, PF_66, 5), 1, X86InstInfo{\"VERW\", TYPE_UNDEC, FLAGS_MODRM, 0}},\n {OPD(TYPE_GROUP_6, PF_66, 6), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_6, PF_66, 7), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n\n {OPD(TYPE_GROUP_6, PF_F2, 0), 1, X86InstInfo{\"SLDT\", TYPE_UNDEC, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {OPD(TYPE_GROUP_6, PF_F2, 1), 1, X86InstInfo{\"STR\", TYPE_PRIV, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {OPD(TYPE_GROUP_6, PF_F2, 2), 1, X86InstInfo{\"LLDT\", TYPE_PRIV, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_6, PF_F2, 3), 1, X86InstInfo{\"LTR\", TYPE_INST, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_6, PF_F2, 4), 1, X86InstInfo{\"VERR\", TYPE_UNDEC, FLAGS_MODRM, 0}},\n {OPD(TYPE_GROUP_6, PF_F2, 5), 1, X86InstInfo{\"VERW\", TYPE_UNDEC, FLAGS_MODRM, 0}},\n {OPD(TYPE_GROUP_6, PF_F2, 6), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_6, PF_F2, 7), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n\n // GROUP 7\n {OPD(TYPE_GROUP_7, PF_NONE, 0), 1, X86InstInfo{\"SGDT\", TYPE_SECOND_GROUP_MODRM, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {OPD(TYPE_GROUP_7, PF_NONE, 1), 1, X86InstInfo{\"SIDT\", TYPE_SECOND_GROUP_MODRM, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {OPD(TYPE_GROUP_7, PF_NONE, 2), 1, X86InstInfo{\"\", TYPE_SECOND_GROUP_MODRM, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_7, PF_NONE, 3), 1, X86InstInfo{\"\", TYPE_SECOND_GROUP_MODRM, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_7, PF_NONE, 4), 1, X86InstInfo{\"SMSW\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {OPD(TYPE_GROUP_7, PF_NONE, 5), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_7, PF_NONE, 6), 1, X86InstInfo{\"LMSW\", TYPE_INST, FLAGS_MODRM, 0}},\n {OPD(TYPE_GROUP_7, PF_NONE, 7), 1, X86InstInfo{\"INVLPG\", TYPE_SECOND_GROUP_MODRM, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n\n {OPD(TYPE_GROUP_7, PF_F3, 0), 1, X86InstInfo{\"SGDT\", TYPE_SECOND_GROUP_MODRM, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {OPD(TYPE_GROUP_7, PF_F3, 1), 1, X86InstInfo{\"SIDT\", TYPE_SECOND_GROUP_MODRM, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {OPD(TYPE_GROUP_7, PF_F3, 2), 1, X86InstInfo{\"\", TYPE_SECOND_GROUP_MODRM, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_7, PF_F3, 3), 1, X86InstInfo{\"\", TYPE_SECOND_GROUP_MODRM, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_7, PF_F3, 4), 1, X86InstInfo{\"SMSW\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {OPD(TYPE_GROUP_7, PF_F3, 5), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_7, PF_F3, 6), 1, X86InstInfo{\"LMSW\", TYPE_INST, FLAGS_MODRM, 0}},\n {OPD(TYPE_GROUP_7, PF_F3, 7), 1, X86InstInfo{\"INVLPG\", TYPE_SECOND_GROUP_MODRM, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n\n {OPD(TYPE_GROUP_7, PF_66, 0), 1, X86InstInfo{\"SGDT\", TYPE_SECOND_GROUP_MODRM, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {OPD(TYPE_GROUP_7, PF_66, 1), 1, X86InstInfo{\"SIDT\", TYPE_SECOND_GROUP_MODRM, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {OPD(TYPE_GROUP_7, PF_66, 2), 1, X86InstInfo{\"\", TYPE_SECOND_GROUP_MODRM, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_7, PF_66, 3), 1, X86InstInfo{\"\", TYPE_SECOND_GROUP_MODRM, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_7, PF_66, 4), 1, X86InstInfo{\"SMSW\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {OPD(TYPE_GROUP_7, PF_66, 5), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_7, PF_66, 6), 1, X86InstInfo{\"LMSW\", TYPE_INST, FLAGS_MODRM, 0}},\n {OPD(TYPE_GROUP_7, PF_66, 7), 1, X86InstInfo{\"INVLPG\", TYPE_SECOND_GROUP_MODRM, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n\n {OPD(TYPE_GROUP_7, PF_F2, 0), 1, X86InstInfo{\"SGDT\", TYPE_SECOND_GROUP_MODRM, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {OPD(TYPE_GROUP_7, PF_F2, 1), 1, X86InstInfo{\"SIDT\", TYPE_SECOND_GROUP_MODRM, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {OPD(TYPE_GROUP_7, PF_F2, 2), 1, X86InstInfo{\"\", TYPE_SECOND_GROUP_MODRM, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_7, PF_F2, 3), 1, X86InstInfo{\"\", TYPE_SECOND_GROUP_MODRM, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_7, PF_F2, 4), 1, X86InstInfo{\"SMSW\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {OPD(TYPE_GROUP_7, PF_F2, 5), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_7, PF_F2, 6), 1, X86InstInfo{\"LMSW\", TYPE_INST, FLAGS_MODRM, 0}},\n {OPD(TYPE_GROUP_7, PF_F2, 7), 1, X86InstInfo{\"INVLPG\", TYPE_SECOND_GROUP_MODRM, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n\n // GROUP 8\n {OPD(TYPE_GROUP_8, PF_NONE, 0), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_8, PF_NONE, 1), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_8, PF_NONE, 2), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_8, PF_NONE, 3), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_8, PF_NONE, 4), 1, X86InstInfo{\"BT\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 1}},\n {OPD(TYPE_GROUP_8, PF_NONE, 5), 1, X86InstInfo{\"BTS\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 1}},\n {OPD(TYPE_GROUP_8, PF_NONE, 6), 1, X86InstInfo{\"BTR\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 1}},\n {OPD(TYPE_GROUP_8, PF_NONE, 7), 1, X86InstInfo{\"BTC\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 1}},\n\n {OPD(TYPE_GROUP_8, PF_F3, 0), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_8, PF_F3, 1), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_8, PF_F3, 2), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_8, PF_F3, 3), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_8, PF_F3, 4), 1, X86InstInfo{\"BT\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 1}},\n {OPD(TYPE_GROUP_8, PF_F3, 5), 1, X86InstInfo{\"BTS\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 1}},\n {OPD(TYPE_GROUP_8, PF_F3, 6), 1, X86InstInfo{\"BTR\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 1}},\n {OPD(TYPE_GROUP_8, PF_F3, 7), 1, X86InstInfo{\"BTC\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 1}},\n\n {OPD(TYPE_GROUP_8, PF_66, 0), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_8, PF_66, 1), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_8, PF_66, 2), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_8, PF_66, 3), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_8, PF_66, 4), 1, X86InstInfo{\"BT\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 1}},\n {OPD(TYPE_GROUP_8, PF_66, 5), 1, X86InstInfo{\"BTS\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 1}},\n {OPD(TYPE_GROUP_8, PF_66, 6), 1, X86InstInfo{\"BTR\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 1}},\n {OPD(TYPE_GROUP_8, PF_66, 7), 1, X86InstInfo{\"BTC\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 1}},\n\n {OPD(TYPE_GROUP_8, PF_F2, 0), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_8, PF_F2, 1), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_8, PF_F2, 2), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_8, PF_F2, 3), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_8, PF_F2, 4), 1, X86InstInfo{\"BT\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 1}},\n {OPD(TYPE_GROUP_8, PF_F2, 5), 1, X86InstInfo{\"BTS\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 1}},\n {OPD(TYPE_GROUP_8, PF_F2, 6), 1, X86InstInfo{\"BTR\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 1}},\n {OPD(TYPE_GROUP_8, PF_F2, 7), 1, X86InstInfo{\"BTC\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 1}},\n\n // GROUP 9\n\n // AMD documentation is a bit broken for Group 9\n // Claims the entire group has n/a applied for the prefix (Implies that the prefix is ignored)\n // RDRAND/RDSEED only work with no prefix (Other than 66h)\n // CMPXCHG8B/16B works with all prefixes\n // Tooling fails to decode CMPXCHG with prefix\n {OPD(TYPE_GROUP_9, PF_NONE, 0), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_9, PF_NONE, 1), 1, X86InstInfo{\"CMPXCHG8B/16B\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_MOD_MEM_ONLY, 0}},\n {OPD(TYPE_GROUP_9, PF_NONE, 2), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_9, PF_NONE, 3), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_9, PF_NONE, 4), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_9, PF_NONE, 5), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_9, PF_NONE, 6), 1, X86InstInfo{\"RDRAND\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_MOD_REG_ONLY, 0}},\n {OPD(TYPE_GROUP_9, PF_NONE, 7), 1, X86InstInfo{\"RDSEED\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_MOD_REG_ONLY, 0}},\n\n {OPD(TYPE_GROUP_9, PF_F3, 0), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_9, PF_F3, 1), 1, X86InstInfo{\"CMPXCHG8B/16B\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_MOD_MEM_ONLY, 0}},\n {OPD(TYPE_GROUP_9, PF_F3, 2), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_9, PF_F3, 3), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_9, PF_F3, 4), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_9, PF_F3, 5), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_9, PF_F3, 6), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_9, PF_F3, 7), 1, X86InstInfo{\"RDPID\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_MOD_REG_ONLY, 0}},\n\n {OPD(TYPE_GROUP_9, PF_66, 0), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_9, PF_66, 1), 1, X86InstInfo{\"CMPXCHG8B/16B\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_MOD_MEM_ONLY, 0}},\n {OPD(TYPE_GROUP_9, PF_66, 2), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_9, PF_66, 3), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_9, PF_66, 4), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_9, PF_66, 5), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_9, PF_66, 6), 1, X86InstInfo{\"RDRAND\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_MOD_REG_ONLY, 0}},\n {OPD(TYPE_GROUP_9, PF_66, 7), 1, X86InstInfo{\"RDSEED\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_MOD_REG_ONLY, 0}},\n\n {OPD(TYPE_GROUP_9, PF_F2, 0), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_9, PF_F2, 1), 1, X86InstInfo{\"CMPXCHG8B/16B\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_MOD_MEM_ONLY, 0}},\n {OPD(TYPE_GROUP_9, PF_F2, 2), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_9, PF_F2, 3), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_9, PF_F2, 4), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_9, PF_F2, 5), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_9, PF_F2, 6), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_9, PF_F2, 7), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n\n // GROUP 10\n {OPD(TYPE_GROUP_10, PF_NONE, 0), 1, X86InstInfo{\"UD1\", TYPE_INST, FLAGS_BLOCK_END, 0}},\n {OPD(TYPE_GROUP_10, PF_NONE, 1), 1, X86InstInfo{\"UD1\", TYPE_INST, FLAGS_BLOCK_END, 0}},\n {OPD(TYPE_GROUP_10, PF_NONE, 2), 1, X86InstInfo{\"UD1\", TYPE_INST, FLAGS_BLOCK_END, 0}},\n {OPD(TYPE_GROUP_10, PF_NONE, 3), 1, X86InstInfo{\"UD1\", TYPE_INST, FLAGS_BLOCK_END, 0}},\n {OPD(TYPE_GROUP_10, PF_NONE, 4), 1, X86InstInfo{\"UD1\", TYPE_INST, FLAGS_BLOCK_END, 0}},\n {OPD(TYPE_GROUP_10, PF_NONE, 5), 1, X86InstInfo{\"UD1\", TYPE_INST, FLAGS_BLOCK_END, 0}},\n {OPD(TYPE_GROUP_10, PF_NONE, 6), 1, X86InstInfo{\"UD1\", TYPE_INST, FLAGS_BLOCK_END, 0}},\n {OPD(TYPE_GROUP_10, PF_NONE, 7), 1, X86InstInfo{\"UD1\", TYPE_INST, FLAGS_BLOCK_END, 0}},\n\n {OPD(TYPE_GROUP_10, PF_F3, 0), 1, X86InstInfo{\"UD1\", TYPE_INST, FLAGS_BLOCK_END, 0}},\n {OPD(TYPE_GROUP_10, PF_F3, 1), 1, X86InstInfo{\"UD1\", TYPE_INST, FLAGS_BLOCK_END, 0}},\n {OPD(TYPE_GROUP_10, PF_F3, 2), 1, X86InstInfo{\"UD1\", TYPE_INST, FLAGS_BLOCK_END, 0}},\n {OPD(TYPE_GROUP_10, PF_F3, 3), 1, X86InstInfo{\"UD1\", TYPE_INST, FLAGS_BLOCK_END, 0}},\n {OPD(TYPE_GROUP_10, PF_F3, 4), 1, X86InstInfo{\"UD1\", TYPE_INST, FLAGS_BLOCK_END, 0}},\n {OPD(TYPE_GROUP_10, PF_F3, 5), 1, X86InstInfo{\"UD1\", TYPE_INST, FLAGS_BLOCK_END, 0}},\n {OPD(TYPE_GROUP_10, PF_F3, 6), 1, X86InstInfo{\"UD1\", TYPE_INST, FLAGS_BLOCK_END, 0}},\n {OPD(TYPE_GROUP_10, PF_F3, 7), 1, X86InstInfo{\"UD1\", TYPE_INST, FLAGS_BLOCK_END, 0}},\n\n {OPD(TYPE_GROUP_10, PF_66, 0), 1, X86InstInfo{\"UD1\", TYPE_INST, FLAGS_BLOCK_END, 0}},\n {OPD(TYPE_GROUP_10, PF_66, 1), 1, X86InstInfo{\"UD1\", TYPE_INST, FLAGS_BLOCK_END, 0}},\n {OPD(TYPE_GROUP_10, PF_66, 2), 1, X86InstInfo{\"UD1\", TYPE_INST, FLAGS_BLOCK_END, 0}},\n {OPD(TYPE_GROUP_10, PF_66, 3), 1, X86InstInfo{\"UD1\", TYPE_INST, FLAGS_BLOCK_END, 0}},\n {OPD(TYPE_GROUP_10, PF_66, 4), 1, X86InstInfo{\"UD1\", TYPE_INST, FLAGS_BLOCK_END, 0}},\n {OPD(TYPE_GROUP_10, PF_66, 5), 1, X86InstInfo{\"UD1\", TYPE_INST, FLAGS_BLOCK_END, 0}},\n {OPD(TYPE_GROUP_10, PF_66, 6), 1, X86InstInfo{\"UD1\", TYPE_INST, FLAGS_BLOCK_END, 0}},\n {OPD(TYPE_GROUP_10, PF_66, 7), 1, X86InstInfo{\"UD1\", TYPE_INST, FLAGS_BLOCK_END, 0}},\n\n {OPD(TYPE_GROUP_10, PF_F2, 0), 1, X86InstInfo{\"UD1\", TYPE_INST, FLAGS_BLOCK_END, 0}},\n {OPD(TYPE_GROUP_10, PF_F2, 1), 1, X86InstInfo{\"UD1\", TYPE_INST, FLAGS_BLOCK_END, 0}},\n {OPD(TYPE_GROUP_10, PF_F2, 2), 1, X86InstInfo{\"UD1\", TYPE_INST, FLAGS_BLOCK_END, 0}},\n {OPD(TYPE_GROUP_10, PF_F2, 3), 1, X86InstInfo{\"UD1\", TYPE_INST, FLAGS_BLOCK_END, 0}},\n {OPD(TYPE_GROUP_10, PF_F2, 4), 1, X86InstInfo{\"UD1\", TYPE_INST, FLAGS_BLOCK_END, 0}},\n {OPD(TYPE_GROUP_10, PF_F2, 5), 1, X86InstInfo{\"UD1\", TYPE_INST, FLAGS_BLOCK_END, 0}},\n {OPD(TYPE_GROUP_10, PF_F2, 6), 1, X86InstInfo{\"UD1\", TYPE_INST, FLAGS_BLOCK_END, 0}},\n {OPD(TYPE_GROUP_10, PF_F2, 7), 1, X86InstInfo{\"UD1\", TYPE_INST, FLAGS_BLOCK_END, 0}},\n\n // GROUP 12\n {OPD(TYPE_GROUP_12, PF_NONE, 0), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_12, PF_NONE, 1), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_12, PF_NONE, 2), 1, X86InstInfo{\"PSRLW\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_MOD_REG_ONLY | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 1}},\n {OPD(TYPE_GROUP_12, PF_NONE, 3), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_12, PF_NONE, 4), 1, X86InstInfo{\"PSRAW\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_MOD_REG_ONLY | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 1}},\n {OPD(TYPE_GROUP_12, PF_NONE, 5), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_12, PF_NONE, 6), 1, X86InstInfo{\"PSLLW\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_MOD_REG_ONLY | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 1}},\n {OPD(TYPE_GROUP_12, PF_NONE, 7), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n\n {OPD(TYPE_GROUP_12, PF_66, 0), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_12, PF_66, 1), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_12, PF_66, 2), 1, X86InstInfo{\"PSRLW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_MOD_REG_ONLY | FLAGS_XMM_FLAGS, 1}},\n {OPD(TYPE_GROUP_12, PF_66, 3), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_12, PF_66, 4), 1, X86InstInfo{\"PSRAW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_MOD_REG_ONLY | FLAGS_XMM_FLAGS, 1}},\n {OPD(TYPE_GROUP_12, PF_66, 5), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_12, PF_66, 6), 1, X86InstInfo{\"PSLLW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_MOD_REG_ONLY | FLAGS_XMM_FLAGS, 1}},\n {OPD(TYPE_GROUP_12, PF_66, 7), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n\n {OPD(TYPE_GROUP_12, PF_F3, 0), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_12, PF_F3, 1), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_12, PF_F3, 2), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_12, PF_F3, 3), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_12, PF_F3, 4), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_12, PF_F3, 5), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_12, PF_F3, 6), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_12, PF_F3, 7), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n\n {OPD(TYPE_GROUP_12, PF_F2, 0), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_12, PF_F2, 1), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_12, PF_F2, 2), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_12, PF_F2, 3), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_12, PF_F2, 4), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_12, PF_F2, 5), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_12, PF_F2, 6), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_12, PF_F2, 7), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n\n // GROUP 13\n {OPD(TYPE_GROUP_13, PF_NONE, 0), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_13, PF_NONE, 1), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_13, PF_NONE, 2), 1, X86InstInfo{\"PSRLD\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_MOD_REG_ONLY | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 1}},\n {OPD(TYPE_GROUP_13, PF_NONE, 3), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_13, PF_NONE, 4), 1, X86InstInfo{\"PSRAD\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_MOD_REG_ONLY | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 1}},\n {OPD(TYPE_GROUP_13, PF_NONE, 5), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_13, PF_NONE, 6), 1, X86InstInfo{\"PSLLD\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_MOD_REG_ONLY | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 1}},\n {OPD(TYPE_GROUP_13, PF_NONE, 7), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n\n {OPD(TYPE_GROUP_13, PF_66, 0), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_13, PF_66, 1), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_13, PF_66, 2), 1, X86InstInfo{\"PSRLD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_MOD_REG_ONLY | FLAGS_XMM_FLAGS, 1}},\n {OPD(TYPE_GROUP_13, PF_66, 3), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_13, PF_66, 4), 1, X86InstInfo{\"PSRAD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_MOD_REG_ONLY | FLAGS_XMM_FLAGS, 1}},\n {OPD(TYPE_GROUP_13, PF_66, 5), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_13, PF_66, 6), 1, X86InstInfo{\"PSLLD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_MOD_REG_ONLY | FLAGS_XMM_FLAGS, 1}},\n {OPD(TYPE_GROUP_13, PF_66, 7), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n\n {OPD(TYPE_GROUP_13, PF_F3, 0), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_13, PF_F3, 1), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_13, PF_F3, 2), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_13, PF_F3, 3), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_13, PF_F3, 4), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_13, PF_F3, 5), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_13, PF_F3, 6), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_13, PF_F3, 7), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n\n {OPD(TYPE_GROUP_13, PF_F2, 0), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_13, PF_F2, 1), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_13, PF_F2, 2), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_13, PF_F2, 3), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_13, PF_F2, 4), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_13, PF_F2, 5), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_13, PF_F2, 6), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_13, PF_F2, 7), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n\n // GROUP 14\n {OPD(TYPE_GROUP_14, PF_NONE, 0), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_14, PF_NONE, 1), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_14, PF_NONE, 2), 1, X86InstInfo{\"PSRLQ\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_MOD_REG_ONLY | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 1}},\n {OPD(TYPE_GROUP_14, PF_NONE, 3), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_14, PF_NONE, 4), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_14, PF_NONE, 5), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_14, PF_NONE, 6), 1, X86InstInfo{\"PSLLQ\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_MOD_REG_ONLY | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 1}},\n {OPD(TYPE_GROUP_14, PF_NONE, 7), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n\n {OPD(TYPE_GROUP_14, PF_66, 0), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_14, PF_66, 1), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_14, PF_66, 2), 1, X86InstInfo{\"PSRLQ\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_MOD_REG_ONLY | FLAGS_XMM_FLAGS, 1}},\n {OPD(TYPE_GROUP_14, PF_66, 3), 1, X86InstInfo{\"PSRLDQ\",TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_MOD_REG_ONLY | FLAGS_XMM_FLAGS, 1}},\n {OPD(TYPE_GROUP_14, PF_66, 4), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_14, PF_66, 5), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_14, PF_66, 6), 1, X86InstInfo{\"PSLLQ\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_MOD_REG_ONLY | FLAGS_XMM_FLAGS, 1}},\n {OPD(TYPE_GROUP_14, PF_66, 7), 1, X86InstInfo{\"PSLLDQ\",TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_MOD_REG_ONLY | FLAGS_XMM_FLAGS, 1}},\n\n {OPD(TYPE_GROUP_14, PF_F3, 0), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_14, PF_F3, 1), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_14, PF_F3, 2), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_14, PF_F3, 3), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_14, PF_F3, 4), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_14, PF_F3, 5), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_14, PF_F3, 6), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_14, PF_F3, 7), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n\n {OPD(TYPE_GROUP_14, PF_F2, 0), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_14, PF_F2, 1), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_14, PF_F2, 2), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_14, PF_F2, 3), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_14, PF_F2, 4), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_14, PF_F2, 5), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_14, PF_F2, 6), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_14, PF_F2, 7), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n\n // GROUP 15\n {OPD(TYPE_GROUP_15, PF_NONE, 0), 1, X86InstInfo{\"FXSAVE\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}}, // MMX/x87\n {OPD(TYPE_GROUP_15, PF_NONE, 1), 1, X86InstInfo{\"FXRSTOR\", TYPE_INST, FLAGS_MODRM, 0}}, // MMX/x87\n {OPD(TYPE_GROUP_15, PF_NONE, 2), 1, X86InstInfo{\"LDMXCSR\", TYPE_INST, GenFlagsSameSize(SIZE_32BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_MOD_MEM_ONLY, 0}},\n {OPD(TYPE_GROUP_15, PF_NONE, 3), 1, X86InstInfo{\"STMXCSR\", TYPE_INST, GenFlagsSameSize(SIZE_32BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_MOD_MEM_ONLY, 0}},\n {OPD(TYPE_GROUP_15, PF_NONE, 4), 1, X86InstInfo{\"XSAVE\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {OPD(TYPE_GROUP_15, PF_NONE, 5), 1, X86InstInfo{\"LFENCE/XRSTOR\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {OPD(TYPE_GROUP_15, PF_NONE, 6), 1, X86InstInfo{\"MFENCE/XSAVEOPT\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {OPD(TYPE_GROUP_15, PF_NONE, 7), 1, X86InstInfo{\"SFENCE/CLFLUSH\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n\n {OPD(TYPE_GROUP_15, PF_F3, 0), 1, X86InstInfo{\"\", TYPE_ARCH_DISPATCHER, FLAGS_NONE, 0, { .Indirect = SecondGroup_ArchSelect_LUT[ENTRY_15_F3_0] }}},\n {OPD(TYPE_GROUP_15, PF_F3, 1), 1, X86InstInfo{\"\", TYPE_ARCH_DISPATCHER, FLAGS_NONE, 0, { .Indirect = SecondGroup_ArchSelect_LUT[ENTRY_15_F3_1] }}},\n {OPD(TYPE_GROUP_15, PF_F3, 2), 1, X86InstInfo{\"\", TYPE_ARCH_DISPATCHER, FLAGS_NONE, 0, { .Indirect = SecondGroup_ArchSelect_LUT[ENTRY_15_F3_2] }}},\n {OPD(TYPE_GROUP_15, PF_F3, 3), 1, X86InstInfo{\"\", TYPE_ARCH_DISPATCHER, FLAGS_NONE, 0, { .Indirect = SecondGroup_ArchSelect_LUT[ENTRY_15_F3_3] }}},\n {OPD(TYPE_GROUP_15, PF_F3, 4), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_15, PF_F3, 5), 1, X86InstInfo{\"INCSSPQ\", TYPE_INST, FLAGS_MODRM, 0}},\n {OPD(TYPE_GROUP_15, PF_F3, 6), 1, X86InstInfo{\"UMONITOR/CLRSSBSY\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {OPD(TYPE_GROUP_15, PF_F3, 7), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n\n {OPD(TYPE_GROUP_15, PF_66, 0), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_15, PF_66, 1), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_15, PF_66, 2), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_15, PF_66, 3), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_15, PF_66, 4), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_15, PF_66, 5), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_15, PF_66, 6), 1, X86InstInfo{\"CLWB\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {OPD(TYPE_GROUP_15, PF_66, 7), 1, X86InstInfo{\"CLFLUSHOPT\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n\n {OPD(TYPE_GROUP_15, PF_F2, 0), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_15, PF_F2, 1), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_15, PF_F2, 2), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_15, PF_F2, 3), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_15, PF_F2, 4), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_15, PF_F2, 5), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_15, PF_F2, 6), 1, X86InstInfo{\"UMWAIT\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {OPD(TYPE_GROUP_15, PF_F2, 7), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n\n // GROUP 16\n // AMD documentation claims again that this entire group is n/a to prefix\n // Tooling once again fails to disassemble oens with the prefix. Disable until proven otherwise\n {OPD(TYPE_GROUP_16, PF_NONE, 0), 1, X86InstInfo{\"PREFETCH NTA\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY, 0}},\n {OPD(TYPE_GROUP_16, PF_NONE, 1), 1, X86InstInfo{\"PREFETCH T0\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY, 0}},\n {OPD(TYPE_GROUP_16, PF_NONE, 2), 1, X86InstInfo{\"PREFETCH T1\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY, 0}},\n {OPD(TYPE_GROUP_16, PF_NONE, 3), 1, X86InstInfo{\"PREFETCH T2\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY, 0}},\n {OPD(TYPE_GROUP_16, PF_NONE, 4), 1, X86InstInfo{\"NOP\", TYPE_INST, FLAGS_MODRM, 0}},\n {OPD(TYPE_GROUP_16, PF_NONE, 5), 1, X86InstInfo{\"NOP\", TYPE_INST, FLAGS_MODRM, 0}},\n {OPD(TYPE_GROUP_16, PF_NONE, 6), 1, X86InstInfo{\"NOP\", TYPE_INST, FLAGS_MODRM, 0}},\n {OPD(TYPE_GROUP_16, PF_NONE, 7), 1, X86InstInfo{\"NOP\", TYPE_INST, FLAGS_MODRM, 0}},\n\n {OPD(TYPE_GROUP_16, PF_F3, 0), 1, X86InstInfo{\"PREFETCH NTA\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY, 0}},\n {OPD(TYPE_GROUP_16, PF_F3, 1), 1, X86InstInfo{\"PREFETCH T0\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY, 0}},\n {OPD(TYPE_GROUP_16, PF_F3, 2), 1, X86InstInfo{\"PREFETCH T1\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY, 0}},\n {OPD(TYPE_GROUP_16, PF_F3, 3), 1, X86InstInfo{\"PREFETCH T2\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY, 0}},\n {OPD(TYPE_GROUP_16, PF_F3, 4), 1, X86InstInfo{\"NOP\", TYPE_INST, FLAGS_MODRM, 0}},\n {OPD(TYPE_GROUP_16, PF_F3, 5), 1, X86InstInfo{\"NOP\", TYPE_INST, FLAGS_MODRM, 0}},\n {OPD(TYPE_GROUP_16, PF_F3, 6), 1, X86InstInfo{\"NOP\", TYPE_INST, FLAGS_MODRM, 0}},\n {OPD(TYPE_GROUP_16, PF_F3, 7), 1, X86InstInfo{\"NOP\", TYPE_INST, FLAGS_MODRM, 0}},\n\n {OPD(TYPE_GROUP_16, PF_66, 0), 1, X86InstInfo{\"PREFETCH NTA\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY, 0}},\n {OPD(TYPE_GROUP_16, PF_66, 1), 1, X86InstInfo{\"PREFETCH T0\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY, 0}},\n {OPD(TYPE_GROUP_16, PF_66, 2), 1, X86InstInfo{\"PREFETCH T1\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY, 0}},\n {OPD(TYPE_GROUP_16, PF_66, 3), 1, X86InstInfo{\"PREFETCH T2\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY, 0}},\n {OPD(TYPE_GROUP_16, PF_66, 4), 1, X86InstInfo{\"NOP\", TYPE_INST, FLAGS_MODRM, 0}},\n {OPD(TYPE_GROUP_16, PF_66, 5), 1, X86InstInfo{\"NOP\", TYPE_INST, FLAGS_MODRM, 0}},\n {OPD(TYPE_GROUP_16, PF_66, 6), 1, X86InstInfo{\"NOP\", TYPE_INST, FLAGS_MODRM, 0}},\n {OPD(TYPE_GROUP_16, PF_66, 7), 1, X86InstInfo{\"NOP\", TYPE_INST, FLAGS_MODRM, 0}},\n\n {OPD(TYPE_GROUP_16, PF_F2, 0), 1, X86InstInfo{\"PREFETCH NTA\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY, 0}},\n {OPD(TYPE_GROUP_16, PF_F2, 1), 1, X86InstInfo{\"PREFETCH T0\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY, 0}},\n {OPD(TYPE_GROUP_16, PF_F2, 2), 1, X86InstInfo{\"PREFETCH T1\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY, 0}},\n {OPD(TYPE_GROUP_16, PF_F2, 3), 1, X86InstInfo{\"PREFETCH T2\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY, 0}},\n {OPD(TYPE_GROUP_16, PF_F2, 4), 1, X86InstInfo{\"NOP\", TYPE_INST, FLAGS_MODRM, 0}},\n {OPD(TYPE_GROUP_16, PF_F2, 5), 1, X86InstInfo{\"NOP\", TYPE_INST, FLAGS_MODRM, 0}},\n {OPD(TYPE_GROUP_16, PF_F2, 6), 1, X86InstInfo{\"NOP\", TYPE_INST, FLAGS_MODRM, 0}},\n {OPD(TYPE_GROUP_16, PF_F2, 7), 1, X86InstInfo{\"NOP\", TYPE_INST, FLAGS_MODRM, 0}},\n\n // GROUP 17\n {OPD(TYPE_GROUP_17, PF_NONE, 0), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_17, PF_NONE, 1), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_17, PF_NONE, 2), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_17, PF_NONE, 3), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_17, PF_NONE, 4), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_17, PF_NONE, 5), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_17, PF_NONE, 6), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_17, PF_NONE, 7), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n\n {OPD(TYPE_GROUP_17, PF_F3, 0), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_17, PF_F3, 1), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_17, PF_F3, 2), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_17, PF_F3, 3), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_17, PF_F3, 4), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_17, PF_F3, 5), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_17, PF_F3, 6), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_17, PF_F3, 7), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n\n {OPD(TYPE_GROUP_17, PF_66, 0), 1, X86InstInfo{\"EXTRQ\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_MOD_REG_ONLY | FLAGS_XMM_FLAGS, 2}},\n {OPD(TYPE_GROUP_17, PF_66, 1), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_17, PF_66, 2), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_17, PF_66, 3), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_17, PF_66, 4), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_17, PF_66, 5), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_17, PF_66, 6), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_17, PF_66, 7), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n\n {OPD(TYPE_GROUP_17, PF_F2, 0), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_17, PF_F2, 1), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_17, PF_F2, 2), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_17, PF_F2, 3), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_17, PF_F2, 4), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_17, PF_F2, 5), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_17, PF_F2, 6), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(TYPE_GROUP_17, PF_F2, 7), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n\n // GROUP P\n // AMD documentation claims n/a for all instructions in Group P\n // It also claims that instructions /2, /4, /5, /6, /7 all alias to /0\n // It claims that /3 is still Prefetch Mod\n // Tooling fails to decode past the /2 encoding but runs fine in hardware\n // Hardware also runs all the prefixes correctly\n {OPD(TYPE_GROUP_P, PF_NONE, 0), 1, X86InstInfo{\"PREFETCH Ex\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY, 0}},\n {OPD(TYPE_GROUP_P, PF_NONE, 1), 1, X86InstInfo{\"PREFETCH Mod\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY, 0}},\n {OPD(TYPE_GROUP_P, PF_NONE, 2), 1, X86InstInfo{\"PREFETCH Res\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY, 0}},\n {OPD(TYPE_GROUP_P, PF_NONE, 3), 1, X86InstInfo{\"PREFETCH Mod\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY, 0}},\n {OPD(TYPE_GROUP_P, PF_NONE, 4), 1, X86InstInfo{\"PREFETCH Res\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY, 0}},\n {OPD(TYPE_GROUP_P, PF_NONE, 5), 1, X86InstInfo{\"PREFETCH Res\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY, 0}},\n {OPD(TYPE_GROUP_P, PF_NONE, 6), 1, X86InstInfo{\"PREFETCH Res\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY, 0}},\n {OPD(TYPE_GROUP_P, PF_NONE, 7), 1, X86InstInfo{\"PREFETCH Res\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY, 0}},\n\n {OPD(TYPE_GROUP_P, PF_F3, 0), 1, X86InstInfo{\"PREFETCH Ex\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY, 0}},\n {OPD(TYPE_GROUP_P, PF_F3, 1), 1, X86InstInfo{\"PREFETCH Mod\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY, 0}},\n {OPD(TYPE_GROUP_P, PF_F3, 2), 1, X86InstInfo{\"PREFETCH Res\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY, 0}},\n {OPD(TYPE_GROUP_P, PF_F3, 3), 1, X86InstInfo{\"PREFETCH Mod\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY, 0}},\n {OPD(TYPE_GROUP_P, PF_F3, 4), 1, X86InstInfo{\"PREFETCH Res\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY, 0}},\n {OPD(TYPE_GROUP_P, PF_F3, 5), 1, X86InstInfo{\"PREFETCH Res\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY, 0}},\n {OPD(TYPE_GROUP_P, PF_F3, 6), 1, X86InstInfo{\"PREFETCH Res\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY, 0}},\n {OPD(TYPE_GROUP_P, PF_F3, 7), 1, X86InstInfo{\"PREFETCH Res\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY, 0}},\n\n {OPD(TYPE_GROUP_P, PF_66, 0), 1, X86InstInfo{\"PREFETCH Ex\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY, 0}},\n {OPD(TYPE_GROUP_P, PF_66, 1), 1, X86InstInfo{\"PREFETCH Mod\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY, 0}},\n {OPD(TYPE_GROUP_P, PF_66, 2), 1, X86InstInfo{\"PREFETCH Res\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY, 0}},\n {OPD(TYPE_GROUP_P, PF_66, 3), 1, X86InstInfo{\"PREFETCH Mod\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY, 0}},\n {OPD(TYPE_GROUP_P, PF_66, 4), 1, X86InstInfo{\"PREFETCH Res\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY, 0}},\n {OPD(TYPE_GROUP_P, PF_66, 5), 1, X86InstInfo{\"PREFETCH Res\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY, 0}},\n {OPD(TYPE_GROUP_P, PF_66, 6), 1, X86InstInfo{\"PREFETCH Res\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY, 0}},\n {OPD(TYPE_GROUP_P, PF_66, 7), 1, X86InstInfo{\"PREFETCH Res\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY, 0}},\n\n {OPD(TYPE_GROUP_P, PF_F2, 0), 1, X86InstInfo{\"PREFETCH Ex\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY, 0}},\n {OPD(TYPE_GROUP_P, PF_F2, 1), 1, X86InstInfo{\"PREFETCH Mod\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY, 0}},\n {OPD(TYPE_GROUP_P, PF_F2, 2), 1, X86InstInfo{\"PREFETCH Res\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY, 0}},\n {OPD(TYPE_GROUP_P, PF_F2, 3), 1, X86InstInfo{\"PREFETCH Mod\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY, 0}},\n {OPD(TYPE_GROUP_P, PF_F2, 4), 1, X86InstInfo{\"PREFETCH Res\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY, 0}},\n {OPD(TYPE_GROUP_P, PF_F2, 5), 1, X86InstInfo{\"PREFETCH Res\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY, 0}},\n {OPD(TYPE_GROUP_P, PF_F2, 6), 1, X86InstInfo{\"PREFETCH Res\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY, 0}},\n {OPD(TYPE_GROUP_P, PF_F2, 7), 1, X86InstInfo{\"PREFETCH Res\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY, 0}},\n };\n#undef OPD\n\n GenerateTable(Table.data(), SecondaryExtensionOpTable, std::size(SecondaryExtensionOpTable));\n\n IR::InstallToTable(Table, IR::OpDispatch_SecondaryGroupTables);\n return Table;\n}();\n\n}\n" }, { "path": "FEXCore/Source/Interface/Core/X86Tables/SecondaryModRMTables.cpp", "content": "// SPDX-License-Identifier: MIT\n/*\n$info$\ntags: frontend|x86-tables\n$end_info$\n*/\n\n#include \"Interface/Core/X86Tables/X86Tables.h\"\n#include \"Interface/Core/OpcodeDispatcher/SecondaryModRMTables.h\"\n\n#include \n\nnamespace FEXCore::X86Tables {\nusing namespace InstFlags;\nconstexpr std::array SecondModRMTableOps = []() consteval {\n std::array Table{};\n constexpr U8U8InfoStruct SecondaryModRMExtensionOpTable[] = {\n // REG /1\n {((0 << 3) | 0), 1, X86InstInfo{\"MONITOR\", TYPE_INST, FLAGS_NONE, 0}},\n {((0 << 3) | 1), 1, X86InstInfo{\"MWAIT\", TYPE_INST, FLAGS_NONE, 0}},\n {((0 << 3) | 2), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {((0 << 3) | 3), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {((0 << 3) | 4), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {((0 << 3) | 5), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {((0 << 3) | 6), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {((0 << 3) | 7), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n\n // REG /2\n {((1 << 3) | 0), 1, X86InstInfo{\"XGETBV\", TYPE_INST, FLAGS_NONE, 0}},\n {((1 << 3) | 1), 1, X86InstInfo{\"XSETBV\", TYPE_PRIV, FLAGS_NONE, 0}},\n {((1 << 3) | 2), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {((1 << 3) | 3), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {((1 << 3) | 4), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {((1 << 3) | 5), 1, X86InstInfo{\"XEND\", TYPE_INST, FLAGS_NONE, 0}},\n {((1 << 3) | 6), 1, X86InstInfo{\"XTEST\", TYPE_INST, FLAGS_NONE, 0}},\n {((1 << 3) | 7), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n\n // REG /3\n {((2 << 3) | 0), 1, X86InstInfo{\"VMRUN\", TYPE_PRIV, FLAGS_NONE, 0}},\n {((2 << 3) | 1), 1, X86InstInfo{\"VMMCALL\", TYPE_PRIV, FLAGS_NONE, 0}},\n {((2 << 3) | 2), 1, X86InstInfo{\"VMLOAD\", TYPE_PRIV, FLAGS_NONE, 0}},\n {((2 << 3) | 3), 1, X86InstInfo{\"VMSAVE\", TYPE_PRIV, FLAGS_NONE, 0}},\n {((2 << 3) | 4), 1, X86InstInfo{\"STGI\", TYPE_PRIV, FLAGS_NONE, 0}},\n {((2 << 3) | 5), 1, X86InstInfo{\"CLGI\", TYPE_PRIV, FLAGS_NONE, 0}},\n {((2 << 3) | 6), 1, X86InstInfo{\"SKINIT\", TYPE_PRIV, FLAGS_NONE, 0}},\n {((2 << 3) | 7), 1, X86InstInfo{\"INVLPGA\", TYPE_INST, FLAGS_NONE, 0}},\n\n // REG /7\n {((3 << 3) | 0), 1, X86InstInfo{\"SWAPGS\", TYPE_INST, FLAGS_NONE, 0}},\n {((3 << 3) | 1), 1, X86InstInfo{\"RDTSCP\", TYPE_INST, FLAGS_NONE, 0}},\n {((3 << 3) | 2), 1, X86InstInfo{\"MONITORX\", TYPE_PRIV, FLAGS_NONE, 0}},\n {((3 << 3) | 3), 1, X86InstInfo{\"MWAITX\", TYPE_PRIV, FLAGS_NONE, 0}},\n {((3 << 3) | 4), 1, X86InstInfo{\"CLZERO\", TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_SF_SRC_RAX | FLAGS_DEBUG_MEM_ACCESS, 0}},\n {((3 << 3) | 5), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {((3 << 3) | 6), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {((3 << 3) | 7), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n };\n\n GenerateTable(Table.data(), SecondaryModRMExtensionOpTable, std::size(SecondaryModRMExtensionOpTable));\n\n IR::InstallToTable(Table, IR::OpDispatch_SecondaryModRMTables);\n return Table;\n}();\n\n}\n" }, { "path": "FEXCore/Source/Interface/Core/X86Tables/SecondaryTables.cpp", "content": "// SPDX-License-Identifier: MIT\n/*\n$info$\ntags: frontend|x86-tables\n$end_info$\n*/\n\n#include \"Interface/Core/X86Tables/X86Tables.h\"\n#include \"Interface/Core/OpcodeDispatcher/SecondaryTables.h\"\n\n#include \n\n#include \n\nnamespace FEXCore::X86Tables {\nusing namespace InstFlags;\n\nenum Secondary_LUT {\n ENTRY_05,\n ENTRY_A0,\n ENTRY_A1,\n ENTRY_A8,\n ENTRY_A9,\n ENTRY_MAX,\n};\n\nconstexpr std::array Secondary_ArchSelect_LUT = {{\n {\n {\"SYSCALL\", TYPE_INST, DEFAULT_SYSCALL_FLAGS, 0, { .OpDispatch = &IR::OpDispatchBuilder::NOPOp } },\n {\"SYSCALL\", TYPE_INST, DEFAULT_SYSCALL_FLAGS, 0, { .OpDispatch = &IR::OpDispatchBuilder::Bind<&IR::OpDispatchBuilder::SyscallOp, true> } },\n },\n {\n {\"PUSH FS\", TYPE_INST, GenFlagsSrcSize(SIZE_16BIT) | FLAGS_DEBUG_MEM_ACCESS | FLAGS_NO_OVERLAY, 0, { .OpDispatch = &IR::OpDispatchBuilder::Bind<&IR::OpDispatchBuilder::PUSHSegmentOp, FEXCore::X86Tables::DecodeFlags::FLAG_FS_PREFIX> } },\n {\"PUSH FS\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_DEBUG_MEM_ACCESS | FLAGS_NO_OVERLAY, 0, { .OpDispatch = &IR::OpDispatchBuilder::Bind<&IR::OpDispatchBuilder::PUSHSegmentOp, FEXCore::X86Tables::DecodeFlags::FLAG_FS_PREFIX> } },\n },\n {\n {\"POP FS\", TYPE_INST, GenFlagsSizes(SIZE_16BIT, SIZE_DEF) | FLAGS_DEBUG_MEM_ACCESS | FLAGS_NO_OVERLAY, 0, { .OpDispatch = &IR::OpDispatchBuilder::Bind<&IR::OpDispatchBuilder::POPSegmentOp, FEXCore::X86Tables::DecodeFlags::FLAG_FS_PREFIX> } },\n {\"POP FS\", TYPE_INST, GenFlagsSizes(SIZE_16BIT, SIZE_64BIT) | FLAGS_DEBUG_MEM_ACCESS | FLAGS_NO_OVERLAY, 0, { .OpDispatch = &IR::OpDispatchBuilder::Bind<&IR::OpDispatchBuilder::POPSegmentOp, FEXCore::X86Tables::DecodeFlags::FLAG_FS_PREFIX> } },\n },\n {\n {\"PUSH GS\", TYPE_INST, GenFlagsSrcSize(SIZE_16BIT) | FLAGS_DEBUG_MEM_ACCESS | FLAGS_NO_OVERLAY, 0, { .OpDispatch = &IR::OpDispatchBuilder::Bind<&IR::OpDispatchBuilder::PUSHSegmentOp, FEXCore::X86Tables::DecodeFlags::FLAG_GS_PREFIX> } },\n {\"PUSH GS\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_DEBUG_MEM_ACCESS | FLAGS_NO_OVERLAY, 0, { .OpDispatch = &IR::OpDispatchBuilder::Bind<&IR::OpDispatchBuilder::PUSHSegmentOp, FEXCore::X86Tables::DecodeFlags::FLAG_GS_PREFIX> } },\n },\n {\n {\"POP GS\", TYPE_INST, GenFlagsSizes(SIZE_16BIT, SIZE_DEF) | FLAGS_DEBUG_MEM_ACCESS | FLAGS_NO_OVERLAY, 0, { .OpDispatch = &IR::OpDispatchBuilder::Bind<&IR::OpDispatchBuilder::POPSegmentOp, FEXCore::X86Tables::DecodeFlags::FLAG_GS_PREFIX> } },\n {\"POP GS\", TYPE_INST, GenFlagsSizes(SIZE_16BIT, SIZE_64BIT) | FLAGS_DEBUG_MEM_ACCESS | FLAGS_NO_OVERLAY, 0, { .OpDispatch = &IR::OpDispatchBuilder::Bind<&IR::OpDispatchBuilder::POPSegmentOp, FEXCore::X86Tables::DecodeFlags::FLAG_GS_PREFIX> } },\n },\n}};\n\nconstexpr std::array SecondBaseOps = []() consteval {\n std::array Table{};\n\n constexpr U8U8InfoStruct TwoByteOpTable[] = {\n // Instructions\n {0x00, 1, X86InstInfo{\"\", TYPE_GROUP_6, FLAGS_MODRM | FLAGS_NO_OVERLAY, 0}},\n {0x01, 1, X86InstInfo{\"\", TYPE_GROUP_7, FLAGS_NO_OVERLAY, 0}},\n // These two load segment register data\n {0x02, 1, X86InstInfo{\"LAR\", TYPE_UNDEC, FLAGS_NO_OVERLAY, 0}},\n {0x03, 1, X86InstInfo{\"LSL\", TYPE_INST, GenFlagsSrcSize(SIZE_16BIT) | FLAGS_MODRM | FLAGS_NO_OVERLAY, 0}},\n {0x04, 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NO_OVERLAY, 0}},\n {0x05, 1, X86InstInfo{\"\", TYPE_ARCH_DISPATCHER, FLAGS_NONE, 0, { .Indirect = Secondary_ArchSelect_LUT[ENTRY_05] }}},\n {0x06, 1, X86InstInfo{\"CLTS\", TYPE_INST, FLAGS_NO_OVERLAY, 0}},\n {0x07, 1, X86InstInfo{\"SYSRET\", TYPE_INST, FLAGS_NO_OVERLAY, 0}},\n {0x08, 1, X86InstInfo{\"INVD\", TYPE_PRIV, FLAGS_NO_OVERLAY, 0}},\n {0x09, 1, X86InstInfo{\"WBINVD\", TYPE_PRIV, FLAGS_NO_OVERLAY, 0}},\n {0x0A, 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NO_OVERLAY, 0}},\n {0x0B, 1, X86InstInfo{\"UD2\", TYPE_INST, FLAGS_BLOCK_END | FLAGS_NO_OVERLAY, 0}},\n {0x0C, 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NO_OVERLAY, 0}},\n {0x0D, 1, X86InstInfo{\"\", TYPE_GROUP_P, FLAGS_MODRM | FLAGS_NO_OVERLAY, 0}},\n {0x0E, 1, X86InstInfo{\"FEMMS\", TYPE_INST, FLAGS_NO_OVERLAY, 0}},\n {0x0F, 1, X86InstInfo{\"\", TYPE_3DNOW_TABLE, FLAGS_NO_OVERLAY, 0}},\n\n {0x10, 1, X86InstInfo{\"MOVUPS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x11, 1, X86InstInfo{\"MOVUPS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_XMM_FLAGS, 0}},\n {0x12, 1, X86InstInfo{\"MOVLPS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x13, 1, X86InstInfo{\"MOVLPS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY | FLAGS_SF_MOD_DST | FLAGS_XMM_FLAGS, 0}},\n {0x14, 1, X86InstInfo{\"UNPCKLPS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x15, 1, X86InstInfo{\"UNPCKHPS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x16, 1, X86InstInfo{\"MOVLHPS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x17, 1, X86InstInfo{\"MOVHPS\", TYPE_INST, GenFlagsSizes(SIZE_64BIT, SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_XMM_FLAGS, 0}},\n {0x18, 1, X86InstInfo{\"\", TYPE_GROUP_16, FLAGS_NO_OVERLAY, 0}},\n {0x19, 7, X86InstInfo{\"NOP\", TYPE_INST, FLAGS_MODRM | FLAGS_NO_OVERLAY, 0}},\n\n {0x20, 2, X86InstInfo{\"MOV\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_NO_OVERLAY, 0}},\n {0x22, 2, X86InstInfo{\"MOV\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_NO_OVERLAY, 0}},\n {0x24, 4, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {0x28, 1, X86InstInfo{\"MOVAPS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x29, 1, X86InstInfo{\"MOVAPS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_XMM_FLAGS, 0}},\n {0x2A, 1, X86InstInfo{\"CVTPI2PS\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX_SRC, 0}},\n {0x2B, 1, X86InstInfo{\"MOVNTPS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY | FLAGS_SF_MOD_DST | FLAGS_XMM_FLAGS, 0}},\n {0x2C, 1, X86InstInfo{\"CVTTPS2PI\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX_DST, 0}},\n {0x2D, 1, X86InstInfo{\"CVTPS2PI\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX_DST, 0}},\n {0x2E, 1, X86InstInfo{\"UCOMISS\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x2F, 1, X86InstInfo{\"COMISS\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n\n {0x30, 1, X86InstInfo{\"WRMSR\", TYPE_INST, FLAGS_NO_OVERLAY, 0}},\n {0x31, 1, X86InstInfo{\"RDTSC\", TYPE_INST, FLAGS_NO_OVERLAY, 0}},\n {0x32, 1, X86InstInfo{\"RDMSR\", TYPE_INST, FLAGS_NO_OVERLAY, 0}},\n {0x33, 1, X86InstInfo{\"RDPMC\", TYPE_INST, FLAGS_NO_OVERLAY, 0}},\n {0x34, 1, X86InstInfo{\"SYSENTER\", TYPE_INST, FLAGS_NO_OVERLAY, 0}},\n {0x35, 1, X86InstInfo{\"SYSEXIT\", TYPE_INST, FLAGS_NO_OVERLAY, 0}},\n {0x36, 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NO_OVERLAY, 0}},\n {0x37, 1, X86InstInfo{\"GETSEC\", TYPE_INVALID, FLAGS_NO_OVERLAY, 0}},\n {0x38, 1, X86InstInfo{\"\", TYPE_0F38_TABLE, FLAGS_NO_OVERLAY, 0}},\n {0x39, 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NO_OVERLAY, 0}},\n {0x3A, 1, X86InstInfo{\"\", TYPE_0F3A_TABLE, FLAGS_NO_OVERLAY, 0}},\n {0x3B, 3, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NO_OVERLAY, 0}},\n\n {0x40, 1, X86InstInfo{\"CMOVO\", TYPE_INST, FLAGS_MODRM | FLAGS_NO_OVERLAY, 0}},\n {0x41, 1, X86InstInfo{\"CMOVNO\", TYPE_INST, FLAGS_MODRM | FLAGS_NO_OVERLAY, 0}},\n {0x42, 1, X86InstInfo{\"CMOVB\", TYPE_INST, FLAGS_MODRM | FLAGS_NO_OVERLAY, 0}},\n {0x43, 1, X86InstInfo{\"CMOVNB\", TYPE_INST, FLAGS_MODRM | FLAGS_NO_OVERLAY, 0}},\n {0x44, 1, X86InstInfo{\"CMOVZ\", TYPE_INST, FLAGS_MODRM | FLAGS_NO_OVERLAY, 0}},\n {0x45, 1, X86InstInfo{\"CMOVNZ\", TYPE_INST, FLAGS_MODRM | FLAGS_NO_OVERLAY, 0}},\n {0x46, 1, X86InstInfo{\"CMOVBE\", TYPE_INST, FLAGS_MODRM | FLAGS_NO_OVERLAY, 0}},\n {0x47, 1, X86InstInfo{\"CMOVNBE\", TYPE_INST, FLAGS_MODRM | FLAGS_NO_OVERLAY, 0}},\n {0x48, 1, X86InstInfo{\"CMOVS\", TYPE_INST, FLAGS_MODRM | FLAGS_NO_OVERLAY, 0}},\n {0x49, 1, X86InstInfo{\"CMOVNS\", TYPE_INST, FLAGS_MODRM | FLAGS_NO_OVERLAY, 0}},\n {0x4A, 1, X86InstInfo{\"CMOVP\", TYPE_INST, FLAGS_MODRM | FLAGS_NO_OVERLAY, 0}},\n {0x4B, 1, X86InstInfo{\"CMOVNP\", TYPE_INST, FLAGS_MODRM | FLAGS_NO_OVERLAY, 0}},\n {0x4C, 1, X86InstInfo{\"CMOVL\", TYPE_INST, FLAGS_MODRM | FLAGS_NO_OVERLAY, 0}},\n {0x4D, 1, X86InstInfo{\"CMOVNL\", TYPE_INST, FLAGS_MODRM | FLAGS_NO_OVERLAY, 0}},\n {0x4E, 1, X86InstInfo{\"CMOVLE\", TYPE_INST, FLAGS_MODRM | FLAGS_NO_OVERLAY, 0}},\n {0x4F, 1, X86InstInfo{\"CMOVNLE\", TYPE_INST, FLAGS_MODRM | FLAGS_NO_OVERLAY, 0}},\n\n {0x50, 1, X86InstInfo{\"MOVMSKPS\", TYPE_INST, GenFlagsSizes(SIZE_32BIT, SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_REG_ONLY | FLAGS_XMM_FLAGS | FLAGS_SF_DST_GPR, 0}},\n {0x51, 1, X86InstInfo{\"SQRTPS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x52, 1, X86InstInfo{\"RSQRTPS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x53, 1, X86InstInfo{\"RCPPS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x54, 1, X86InstInfo{\"ANDPS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x55, 1, X86InstInfo{\"ANDNPS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x56, 1, X86InstInfo{\"ORPS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x57, 1, X86InstInfo{\"XORPS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x58, 1, X86InstInfo{\"ADDPS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x59, 1, X86InstInfo{\"MULPS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x5A, 1, X86InstInfo{\"CVTPS2PD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x5B, 1, X86InstInfo{\"CVTDQ2PS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x5C, 1, X86InstInfo{\"SUBPS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x5D, 1, X86InstInfo{\"MINPS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x5E, 1, X86InstInfo{\"DIVPS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x5F, 1, X86InstInfo{\"MAXPS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n\n {0x60, 1, X86InstInfo{\"PUNPCKLBW\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {0x61, 1, X86InstInfo{\"PUNPCKLWD\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {0x62, 1, X86InstInfo{\"PUNPCKLDQ\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {0x63, 1, X86InstInfo{\"PACKSSWB\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {0x64, 1, X86InstInfo{\"PCMPGTB\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {0x65, 1, X86InstInfo{\"PCMPGTW\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {0x66, 1, X86InstInfo{\"PCMPGTD\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {0x67, 1, X86InstInfo{\"PACKUSWB\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {0x68, 1, X86InstInfo{\"PUNPCKHBW\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {0x69, 1, X86InstInfo{\"PUNPCKHWD\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {0x6A, 1, X86InstInfo{\"PUNPCKHDQ\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {0x6B, 1, X86InstInfo{\"PACKSSDW\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {0x6C, 2, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {0x6E, 1, X86InstInfo{\"MOVD\", TYPE_INST, GenFlagsDstSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_SF_SRC_GPR | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {0x6F, 1, X86InstInfo{\"MOVQ\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n\n {0x70, 1, X86InstInfo{\"PSHUFW\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 1}},\n {0x71, 1, X86InstInfo{\"\", TYPE_GROUP_12, FLAGS_NO_OVERLAY, 0}},\n {0x72, 1, X86InstInfo{\"\", TYPE_GROUP_13, FLAGS_NO_OVERLAY, 0}},\n {0x73, 1, X86InstInfo{\"\", TYPE_GROUP_14, FLAGS_NO_OVERLAY, 0}},\n {0x74, 1, X86InstInfo{\"PCMPEQB\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {0x75, 1, X86InstInfo{\"PCMPEQW\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {0x76, 1, X86InstInfo{\"PCMPEQD\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {0x77, 1, X86InstInfo{\"EMMS\", TYPE_INST, FLAGS_NONE, 0}},\n {0x78, 6, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {0x7E, 1, X86InstInfo{\"MOVD\", TYPE_INST, GenFlagsSrcSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_DST_GPR | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {0x7F, 1, X86InstInfo{\"MOVQ\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n\n {0x80, 1, X86InstInfo{\"JO\", TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_SETS_RIP | FLAGS_SRC_SEXT | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_NO_OVERLAY, 4}},\n {0x81, 1, X86InstInfo{\"JNO\", TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_SETS_RIP | FLAGS_SRC_SEXT | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_NO_OVERLAY, 4}},\n {0x82, 1, X86InstInfo{\"JB\", TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_SETS_RIP | FLAGS_SRC_SEXT | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_NO_OVERLAY, 4}},\n {0x83, 1, X86InstInfo{\"JNB\", TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_SETS_RIP | FLAGS_SRC_SEXT | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_NO_OVERLAY, 4}},\n {0x84, 1, X86InstInfo{\"JZ\", TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_SETS_RIP | FLAGS_SRC_SEXT | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_NO_OVERLAY, 4}},\n {0x85, 1, X86InstInfo{\"JNZ\", TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_SETS_RIP | FLAGS_SRC_SEXT | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_NO_OVERLAY, 4}},\n {0x86, 1, X86InstInfo{\"JBE\", TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_SETS_RIP | FLAGS_SRC_SEXT | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_NO_OVERLAY, 4}},\n {0x87, 1, X86InstInfo{\"JNBE\", TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_SETS_RIP | FLAGS_SRC_SEXT | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_NO_OVERLAY, 4}},\n {0x88, 1, X86InstInfo{\"JS\", TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_SETS_RIP | FLAGS_SRC_SEXT | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_NO_OVERLAY, 4}},\n {0x89, 1, X86InstInfo{\"JNS\", TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_SETS_RIP | FLAGS_SRC_SEXT | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_NO_OVERLAY, 4}},\n {0x8A, 1, X86InstInfo{\"JP\", TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_SETS_RIP | FLAGS_SRC_SEXT | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_NO_OVERLAY, 4}},\n {0x8B, 1, X86InstInfo{\"JNP\", TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_SETS_RIP | FLAGS_SRC_SEXT | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_NO_OVERLAY, 4}},\n {0x8C, 1, X86InstInfo{\"JL\", TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_SETS_RIP | FLAGS_SRC_SEXT | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_NO_OVERLAY, 4}},\n {0x8D, 1, X86InstInfo{\"JNL\", TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_SETS_RIP | FLAGS_SRC_SEXT | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_NO_OVERLAY, 4}},\n {0x8E, 1, X86InstInfo{\"JLE\", TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_SETS_RIP | FLAGS_SRC_SEXT | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_NO_OVERLAY, 4}},\n {0x8F, 1, X86InstInfo{\"JNLE\", TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_SETS_RIP | FLAGS_SRC_SEXT | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_NO_OVERLAY, 4}},\n\n {0x90, 1, X86InstInfo{\"SETO\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_NO_OVERLAY, 0}},\n {0x91, 1, X86InstInfo{\"SETNO\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_NO_OVERLAY, 0}},\n {0x92, 1, X86InstInfo{\"SETB\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_NO_OVERLAY, 0}},\n {0x93, 1, X86InstInfo{\"SETNB\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_NO_OVERLAY, 0}},\n {0x94, 1, X86InstInfo{\"SETZ\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_NO_OVERLAY, 0}},\n {0x95, 1, X86InstInfo{\"SETNZ\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_NO_OVERLAY, 0}},\n {0x96, 1, X86InstInfo{\"SETBE\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_NO_OVERLAY, 0}},\n {0x97, 1, X86InstInfo{\"SETNBE\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_NO_OVERLAY, 0}},\n {0x98, 1, X86InstInfo{\"SETS\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_NO_OVERLAY, 0}},\n {0x99, 1, X86InstInfo{\"SETNS\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_NO_OVERLAY, 0}},\n {0x9A, 1, X86InstInfo{\"SETP\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_NO_OVERLAY, 0}},\n {0x9B, 1, X86InstInfo{\"SETNP\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_NO_OVERLAY, 0}},\n {0x9C, 1, X86InstInfo{\"SETL\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_NO_OVERLAY, 0}},\n {0x9D, 1, X86InstInfo{\"SETNL\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_NO_OVERLAY, 0}},\n {0x9E, 1, X86InstInfo{\"SETLE\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_NO_OVERLAY, 0}},\n {0x9F, 1, X86InstInfo{\"SETNLE\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_NO_OVERLAY, 0}},\n\n {0xA0, 1, X86InstInfo{\"\", TYPE_ARCH_DISPATCHER, FLAGS_DEBUG_MEM_ACCESS | FLAGS_NO_OVERLAY, 0, { .Indirect = Secondary_ArchSelect_LUT[ENTRY_A0] }}},\n {0xA1, 1, X86InstInfo{\"\", TYPE_ARCH_DISPATCHER, FLAGS_DEBUG_MEM_ACCESS | FLAGS_NO_OVERLAY, 0, { .Indirect = Secondary_ArchSelect_LUT[ENTRY_A1] }}},\n {0xA2, 1, X86InstInfo{\"CPUID\", TYPE_INST, FLAGS_SF_SRC_RAX | FLAGS_NO_OVERLAY, 0}},\n {0xA3, 1, X86InstInfo{\"BT\", TYPE_INST, FLAGS_DEBUG_MEM_ACCESS | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_NO_OVERLAY, 0}},\n {0xA4, 1, X86InstInfo{\"SHLD\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_NO_OVERLAY, 1}},\n {0xA5, 1, X86InstInfo{\"SHLD\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_SRC_RCX | FLAGS_NO_OVERLAY, 0}},\n {0xA6, 2, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NO_OVERLAY, 0}},\n {0xA8, 1, X86InstInfo{\"\", TYPE_ARCH_DISPATCHER, FLAGS_DEBUG_MEM_ACCESS | FLAGS_NO_OVERLAY, 0, { .Indirect = Secondary_ArchSelect_LUT[ENTRY_A8] }}},\n {0xA9, 1, X86InstInfo{\"\", TYPE_ARCH_DISPATCHER, FLAGS_DEBUG_MEM_ACCESS | FLAGS_NO_OVERLAY, 0, { .Indirect = Secondary_ArchSelect_LUT[ENTRY_A9] }}},\n {0xAA, 1, X86InstInfo{\"RSM\", TYPE_PRIV, FLAGS_NO_OVERLAY, 0}},\n {0xAB, 1, X86InstInfo{\"BTS\", TYPE_INST, FLAGS_DEBUG_MEM_ACCESS | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_NO_OVERLAY, 0}},\n {0xAC, 1, X86InstInfo{\"SHRD\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_NO_OVERLAY, 1}},\n {0xAD, 1, X86InstInfo{\"SHRD\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_SRC_RCX | FLAGS_NO_OVERLAY, 0}},\n {0xAE, 1, X86InstInfo{\"\", TYPE_GROUP_15, FLAGS_NO_OVERLAY, 0}},\n {0xAF, 1, X86InstInfo{\"IMUL\", TYPE_INST, FLAGS_MODRM | FLAGS_NO_OVERLAY, 0}},\n\n {0xB0, 1, X86InstInfo{\"CMPXCHG\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_NO_OVERLAY, 0}},\n {0xB1, 1, X86InstInfo{\"CMPXCHG\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_NO_OVERLAY, 0}},\n {0xB2, 1, X86InstInfo{\"LSS\", TYPE_INVALID, FLAGS_NO_OVERLAY, 0}},\n {0xB3, 1, X86InstInfo{\"BTR\", TYPE_INST, FLAGS_DEBUG_MEM_ACCESS | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_NO_OVERLAY, 0}},\n {0xB4, 1, X86InstInfo{\"LFS\", TYPE_INVALID, FLAGS_NO_OVERLAY, 0}},\n {0xB5, 1, X86InstInfo{\"LGS\", TYPE_INVALID, FLAGS_NO_OVERLAY, 0}},\n {0xB6, 1, X86InstInfo{\"MOVZX\", TYPE_INST, GenFlagsSrcSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_NO_OVERLAY, 0}},\n {0xB7, 1, X86InstInfo{\"MOVZX\", TYPE_INST, GenFlagsSrcSize(SIZE_16BIT) | FLAGS_MODRM | FLAGS_NO_OVERLAY, 0}},\n {0xB8, 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {0xB9, 1, X86InstInfo{\"\", TYPE_GROUP_10, FLAGS_NO_OVERLAY, 0}},\n {0xBA, 1, X86InstInfo{\"\", TYPE_GROUP_8, FLAGS_NO_OVERLAY, 0}},\n {0xBB, 1, X86InstInfo{\"BTC\", TYPE_INST, FLAGS_DEBUG_MEM_ACCESS | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_NO_OVERLAY, 0}},\n {0xBC, 1, X86InstInfo{\"BSF\", TYPE_INST, FLAGS_MODRM | FLAGS_NO_OVERLAY66, 0}},\n {0xBD, 1, X86InstInfo{\"BSR\", TYPE_INST, FLAGS_MODRM | FLAGS_NO_OVERLAY66, 0}},\n {0xBE, 1, X86InstInfo{\"MOVSX\", TYPE_INST, GenFlagsSrcSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_NO_OVERLAY, 0}},\n {0xBF, 1, X86InstInfo{\"MOVSX\", TYPE_INST, GenFlagsSrcSize(SIZE_16BIT) | FLAGS_MODRM | FLAGS_NO_OVERLAY, 0}},\n\n {0xC0, 1, X86InstInfo{\"XADD\", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {0xC1, 1, X86InstInfo{\"XADD\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_NO_OVERLAY, 0}},\n {0xC2, 1, X86InstInfo{\"CMPPS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 1}},\n {0xC3, 1, X86InstInfo{\"MOVNTI\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY | FLAGS_SF_MOD_DST, 0}},\n {0xC4, 1, X86InstInfo{\"PINSRW\", TYPE_INST, GenFlagsSizes(SIZE_64BIT, SIZE_16BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX | FLAGS_SF_SRC_GPR, 1}},\n {0xC5, 1, X86InstInfo{\"PEXTRW\", TYPE_INST, GenFlagsSizes(SIZE_32BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_SF_MOD_REG_ONLY | FLAGS_SF_DST_GPR | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 1}},\n {0xC6, 1, X86InstInfo{\"SHUFPS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 1}},\n {0xC7, 1, X86InstInfo{\"\", TYPE_GROUP_9, FLAGS_NO_OVERLAY, 0}},\n {0xC8, 8, X86InstInfo{\"BSWAP\", TYPE_INST, FLAGS_SF_REX_IN_BYTE | FLAGS_NO_OVERLAY, 0}},\n\n {0xD0, 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {0xD1, 1, X86InstInfo{\"PSRLW\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {0xD2, 1, X86InstInfo{\"PSRLD\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {0xD3, 1, X86InstInfo{\"PSRLQ\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {0xD4, 1, X86InstInfo{\"PADDQ\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {0xD5, 1, X86InstInfo{\"PMULLW\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {0xD6, 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {0xD7, 1, X86InstInfo{\"PMOVMSKB\", TYPE_INST, GenFlagsSizes(SIZE_32BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_SF_MOD_REG_ONLY | FLAGS_XMM_FLAGS | FLAGS_SF_DST_GPR | FLAGS_SF_MMX_SRC, 0}},\n {0xD8, 1, X86InstInfo{\"PSUBUSB\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {0xD9, 1, X86InstInfo{\"PSUBUSW\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {0xDA, 1, X86InstInfo{\"PMINUB\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {0xDB, 1, X86InstInfo{\"PAND\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {0xDC, 1, X86InstInfo{\"PADDUSB\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {0xDD, 1, X86InstInfo{\"PADDUSW\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {0xDE, 1, X86InstInfo{\"PMAXUB\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {0xDF, 1, X86InstInfo{\"PANDN\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n\n {0xE0, 1, X86InstInfo{\"PAVGB\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {0xE1, 1, X86InstInfo{\"PSRAW\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {0xE2, 1, X86InstInfo{\"PSRAD\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {0xE3, 1, X86InstInfo{\"PAVGW\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {0xE4, 1, X86InstInfo{\"PMULHUW\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {0xE5, 1, X86InstInfo{\"PMULHW\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {0xE6, 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {0xE7, 1, X86InstInfo{\"MOVNTQ\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {0xE8, 1, X86InstInfo{\"PSUBSB\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {0xE9, 1, X86InstInfo{\"PSUBSW\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {0xEA, 1, X86InstInfo{\"PMINSW\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {0xEB, 1, X86InstInfo{\"POR\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {0xEC, 1, X86InstInfo{\"PADDSB\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {0xED, 1, X86InstInfo{\"PADDSW\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {0xEE, 1, X86InstInfo{\"PMAXSW\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {0xEF, 1, X86InstInfo{\"PXOR\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n\n {0xF0, 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {0xF1, 1, X86InstInfo{\"PSLLW\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {0xF2, 1, X86InstInfo{\"PSLLD\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {0xF3, 1, X86InstInfo{\"PSLLQ\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {0xF4, 1, X86InstInfo{\"PMULUDQ\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {0xF5, 1, X86InstInfo{\"PMADDWD\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {0xF6, 1, X86InstInfo{\"PSADBW\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {0xF7, 1, X86InstInfo{\"MASKMOVQ\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_SF_MOD_REG_ONLY | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {0xF8, 1, X86InstInfo{\"PSUBB\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {0xF9, 1, X86InstInfo{\"PSUBW\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {0xFA, 1, X86InstInfo{\"PSUBD\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {0xFB, 1, X86InstInfo{\"PSUBQ\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {0xFC, 1, X86InstInfo{\"PADDB\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {0xFD, 1, X86InstInfo{\"PADDW\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {0xFE, 1, X86InstInfo{\"PADDD\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0}},\n {0xFF, 1, X86InstInfo{\"UD0\", TYPE_INST, FLAGS_BLOCK_END, 0}},\n\n#ifndef _WIN32\n // FEX reserved instructions\n // Unused x86 encoding instruction.\n\n {0x3E, 1, X86InstInfo{\"CALLBACKRET\", TYPE_INST, FLAGS_BLOCK_END | FLAGS_NO_OVERLAY | FLAGS_SETS_RIP, 0}},\n\n // This was originally used by VIA to jump to its alternative instruction set. Used for OP_THUNK\n {0x3F, 1, X86InstInfo{\"ALTINST\", TYPE_INST, FLAGS_BLOCK_END | FLAGS_NO_OVERLAY | FLAGS_SETS_RIP, 0}},\n#endif\n };\n\n GenerateTable(Table.data(), TwoByteOpTable, std::size(TwoByteOpTable));\n\n IR::InstallToTable(Table, IR::OpDispatch_TwoByteOpTable);\n\n return Table;\n}();\n\nconstexpr std::array RepModOps = []() consteval {\n std::array Table{};\n\n constexpr U8U8InfoStruct RepModOpTable[] = {\n {0x0, 16, X86InstInfo{\"\", TYPE_COPY_OTHER, FLAGS_NONE, 0}},\n\n {0x10, 1, X86InstInfo{\"MOVSS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x11, 1, X86InstInfo{\"MOVSS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_XMM_FLAGS, 0}},\n {0x12, 1, X86InstInfo{\"MOVSLDUP\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x13, 3, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {0x16, 1, X86InstInfo{\"MOVSHDUP\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x17, 2, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {0x19, 7, X86InstInfo{\"\", TYPE_COPY_OTHER, FLAGS_NONE, 0}},\n\n {0x20, 4, X86InstInfo{\"\", TYPE_COPY_OTHER, FLAGS_NONE, 0}},\n {0x24, 6, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {0x2A, 1, X86InstInfo{\"CVTSI2SS\", TYPE_INST, GenFlagsDstSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_SRC_GPR, 0}},\n {0x2B, 1, X86InstInfo{\"MOVNTSS\", TYPE_INST, GenFlagsSizes(SIZE_32BIT, SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY | FLAGS_SF_MOD_DST | FLAGS_XMM_FLAGS, 0}},\n {0x2C, 1, X86InstInfo{\"CVTTSS2SI\", TYPE_INST, FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_DST_GPR, 0}},\n {0x2D, 1, X86InstInfo{\"CVTSS2SI\", TYPE_INST, FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_DST_GPR, 0}},\n {0x2E, 2, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n\n {0x30, 16, X86InstInfo{\"\", TYPE_COPY_OTHER, FLAGS_NONE, 0}},\n {0x40, 16, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n\n {0x50, 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {0x51, 1, X86InstInfo{\"SQRTSS\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x52, 1, X86InstInfo{\"RSQRTSS\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x53, 1, X86InstInfo{\"RCPSS\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x54, 4, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {0x58, 1, X86InstInfo{\"ADDSS\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x59, 1, X86InstInfo{\"MULSS\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x5A, 1, X86InstInfo{\"CVTSS2SD\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x5B, 1, X86InstInfo{\"CVTTPS2DQ\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x5C, 1, X86InstInfo{\"SUBSS\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x5D, 1, X86InstInfo{\"MINSS\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x5E, 1, X86InstInfo{\"DIVSS\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x5F, 1, X86InstInfo{\"MAXSS\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n\n {0x60, 8, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {0x68, 7, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {0x6F, 1, X86InstInfo{\"MOVDQU\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n\n {0x70, 1, X86InstInfo{\"PSHUFHW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 1}},\n {0x71, 3, X86InstInfo{\"\", TYPE_COPY_OTHER, FLAGS_NONE, 0}},\n {0x74, 4, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {0x78, 6, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {0x7E, 1, X86InstInfo{\"MOVQ\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x7F, 1, X86InstInfo{\"MOVDQU\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_XMM_FLAGS, 0}},\n\n {0x80, 16, X86InstInfo{\"\", TYPE_COPY_OTHER, FLAGS_NONE, 0}},\n {0x90, 16, X86InstInfo{\"\", TYPE_COPY_OTHER, FLAGS_NONE, 0}},\n {0xA0, 16, X86InstInfo{\"\", TYPE_COPY_OTHER, FLAGS_NONE, 0}},\n\n {0xB0, 8, X86InstInfo{\"\", TYPE_COPY_OTHER, FLAGS_NONE, 0}},\n {0xB8, 1, X86InstInfo{\"POPCNT\", TYPE_INST, FLAGS_MODRM, 0}},\n {0xB9, 1, X86InstInfo{\"\", TYPE_COPY_OTHER, FLAGS_NONE, 0}},\n {0xBA, 1, X86InstInfo{\"\", TYPE_COPY_OTHER, FLAGS_NONE, 0}},\n {0xBB, 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {0xBC, 1, X86InstInfo{\"TZCNT\", TYPE_INST, FLAGS_MODRM, 0}},\n {0xBD, 1, X86InstInfo{\"LZCNT\", TYPE_INST, FLAGS_MODRM, 0}},\n {0xBE, 2, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n\n {0xC0, 2, X86InstInfo{\"\", TYPE_COPY_OTHER, FLAGS_NONE, 0}},\n {0xC2, 1, X86InstInfo{\"CMPSS\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 1}},\n {0xC3, 5, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {0xC8, 8, X86InstInfo{\"\", TYPE_COPY_OTHER, FLAGS_NONE, 0}},\n\n {0xD0, 6, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {0xD6, 1, X86InstInfo{\"MOVQ2DQ\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX_SRC, 0}},\n {0xD7, 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {0xD8, 8, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n\n {0xE0, 6, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {0xE6, 1, X86InstInfo{\"CVTDQ2PD\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0xE7, 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {0xE8, 8, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n\n {0xF0, 8, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {0xF8, 7, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {0xFF, 1, X86InstInfo{\"\", TYPE_COPY_OTHER, FLAGS_NONE, 0}},\n };\n\n GenerateTableWithCopy(Table.data(), RepModOpTable, std::size(RepModOpTable), SecondBaseOps.data());\n\n IR::InstallToTable(Table, IR::OpDispatch_SecondaryRepModTables);\n return Table;\n}();\n\nconstexpr std::array RepNEModOps = []() consteval {\n std::array Table{};\n\n constexpr U8U8InfoStruct RepNEModOpTable[] = {\n {0x0, 16, X86InstInfo{\"\", TYPE_COPY_OTHER, FLAGS_NONE, 0}},\n\n {0x10, 1, X86InstInfo{\"MOVSD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x11, 1, X86InstInfo{\"MOVSD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_XMM_FLAGS, 0}},\n {0x12, 1, X86InstInfo{\"MOVDDUP\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x13, 6, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {0x19, 7, X86InstInfo{\"\", TYPE_COPY_OTHER, FLAGS_NONE, 0}},\n\n {0x20, 4, X86InstInfo{\"\", TYPE_COPY_OTHER, FLAGS_NONE, 0}},\n {0x24, 6, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {0x2A, 1, X86InstInfo{\"CVTSI2SD\", TYPE_INST, GenFlagsDstSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_SRC_GPR, 0}},\n {0x2B, 1, X86InstInfo{\"MOVNTSD\", TYPE_INST, GenFlagsSizes(SIZE_64BIT, SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY | FLAGS_SF_MOD_DST | FLAGS_XMM_FLAGS, 0}},\n {0x2C, 1, X86InstInfo{\"CVTTSD2SI\", TYPE_INST, FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_DST_GPR, 0}},\n {0x2D, 1, X86InstInfo{\"CVTSD2SI\", TYPE_INST, FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_DST_GPR, 0}},\n {0x2E, 2, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n\n {0x30, 16, X86InstInfo{\"\", TYPE_COPY_OTHER, FLAGS_NONE, 0}},\n {0x40, 16, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n\n {0x50, 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {0x51, 1, X86InstInfo{\"SQRTSD\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x52, 6, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {0x58, 1, X86InstInfo{\"ADDSD\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x59, 1, X86InstInfo{\"MULSD\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x5A, 1, X86InstInfo{\"CVTSD2SS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x5B, 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {0x5C, 1, X86InstInfo{\"SUBSD\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x5D, 1, X86InstInfo{\"MINSD\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x5E, 1, X86InstInfo{\"DIVSD\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x5F, 1, X86InstInfo{\"MAXSD\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n\n {0x60, 16, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n\n {0x70, 1, X86InstInfo{\"PSHUFLW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 1}},\n {0x71, 3, X86InstInfo{\"\", TYPE_COPY_OTHER, FLAGS_NONE, 0}},\n {0x74, 4, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {0x78, 1, X86InstInfo{\"INSERTQ\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_REG_ONLY | FLAGS_XMM_FLAGS,2}},\n {0x79, 1, X86InstInfo{\"INSERTQ\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_REG_ONLY | FLAGS_XMM_FLAGS, 0}},\n {0x7A, 2, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {0x7C, 1, X86InstInfo{\"HADDPS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x7D, 1, X86InstInfo{\"HSUBPS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x7E, 2, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n\n {0x80, 16, X86InstInfo{\"\", TYPE_COPY_OTHER, FLAGS_NONE, 0}},\n {0x90, 16, X86InstInfo{\"\", TYPE_COPY_OTHER, FLAGS_NONE, 0}},\n {0xA0, 16, X86InstInfo{\"\", TYPE_COPY_OTHER, FLAGS_NONE, 0}},\n {0xB0, 8, X86InstInfo{\"\", TYPE_COPY_OTHER, FLAGS_NONE, 0}},\n {0xB8, 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {0xB9, 1, X86InstInfo{\"\", TYPE_COPY_OTHER, FLAGS_NONE, 0}},\n {0xBA, 1, X86InstInfo{\"\", TYPE_COPY_OTHER, FLAGS_NONE, 0}},\n {0xBB, 5, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {0xC0, 2, X86InstInfo{\"\", TYPE_COPY_OTHER, FLAGS_NONE, 0}},\n {0xC2, 1, X86InstInfo{\"CMPSD\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 1}},\n {0xC3, 5, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {0xC8, 8, X86InstInfo{\"\", TYPE_COPY_OTHER, FLAGS_NONE, 0}},\n\n {0xD0, 1, X86InstInfo{\"ADDSUBPS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0xD1, 5, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {0xD6, 1, X86InstInfo{\"MOVDQ2Q\", TYPE_INST, GenFlagsSizes(SIZE_64BIT, SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX_DST, 0}},\n {0xD7, 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {0xD8, 8, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n\n {0xE0, 6, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {0xE6, 1, X86InstInfo{\"CVTPD2DQ\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0xE7, 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {0xE8, 8, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n\n {0xF0, 1, X86InstInfo{\"LDDQU\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY | FLAGS_XMM_FLAGS,0}},\n {0xF1, 7, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {0xF8, 8, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n };\n\n GenerateTableWithCopy(Table.data(), RepNEModOpTable, std::size(RepNEModOpTable), SecondBaseOps.data());\n\n IR::InstallToTable(Table, IR::OpDispatch_SecondaryRepNEModTables);\n return Table;\n}();\n\nconstexpr std::array OpSizeModOps = []() consteval {\n std::array Table{};\n\n constexpr U8U8InfoStruct OpSizeModOpTable[] = {\n {0x0, 16, X86InstInfo{\"\", TYPE_COPY_OTHER, FLAGS_NONE, 0}},\n\n {0x10, 1, X86InstInfo{\"MOVUPD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x11, 1, X86InstInfo{\"MOVUPD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_XMM_FLAGS, 0}},\n {0x12, 1, X86InstInfo{\"MOVLPD\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY | FLAGS_XMM_FLAGS, 0}},\n {0x13, 1, X86InstInfo{\"MOVLPD\", TYPE_INST, GenFlagsSizes(SIZE_64BIT, SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY | FLAGS_SF_MOD_DST | FLAGS_XMM_FLAGS, 0}},\n {0x14, 1, X86InstInfo{\"UNPCKLPD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x15, 1, X86InstInfo{\"UNPCKHPD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x16, 1, X86InstInfo{\"MOVHPD\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY | FLAGS_XMM_FLAGS, 0}},\n {0x17, 1, X86InstInfo{\"MOVHPD\", TYPE_INST, GenFlagsSizes(SIZE_64BIT, SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY | FLAGS_SF_MOD_DST | FLAGS_XMM_FLAGS, 0}},\n {0x18, 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {0x19, 7, X86InstInfo{\"\", TYPE_COPY_OTHER, FLAGS_NONE, 0}},\n\n {0x20, 4, X86InstInfo{\"\", TYPE_COPY_OTHER, FLAGS_NONE, 0}},\n {0x24, 4, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n\n {0x28, 1, X86InstInfo{\"MOVAPD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x29, 1, X86InstInfo{\"MOVAPD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_XMM_FLAGS, 0}},\n {0x2A, 1, X86InstInfo{\"CVTPI2PD\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX_SRC, 0}},\n {0x2B, 1, X86InstInfo{\"MOVNTPD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY | FLAGS_SF_MOD_DST | FLAGS_XMM_FLAGS, 0}},\n {0x2C, 1, X86InstInfo{\"CVTTPD2PI\", TYPE_INST, GenFlagsSizes(SIZE_64BIT, SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX_DST, 0}},\n {0x2D, 1, X86InstInfo{\"CVTPD2PI\", TYPE_INST, GenFlagsSizes(SIZE_64BIT, SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX_DST, 0}},\n {0x2E, 1, X86InstInfo{\"UCOMISD\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x2F, 1, X86InstInfo{\"COMISD\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n\n {0x30, 16, X86InstInfo{\"\", TYPE_COPY_OTHER, FLAGS_NONE, 0}},\n {0x40, 16, X86InstInfo{\"\", TYPE_COPY_OTHER, FLAGS_NONE, 0}},\n\n {0x50, 1, X86InstInfo{\"MOVMSKPD\", TYPE_INST, GenFlagsSizes(SIZE_32BIT, SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_REG_ONLY | FLAGS_XMM_FLAGS | FLAGS_SF_DST_GPR, 0}},\n {0x51, 1, X86InstInfo{\"SQRTPD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x52, 2, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {0x54, 1, X86InstInfo{\"ANDPD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x55, 1, X86InstInfo{\"ANDNPD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x56, 1, X86InstInfo{\"ORPD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x57, 1, X86InstInfo{\"XORPD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x58, 1, X86InstInfo{\"ADDPD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x59, 1, X86InstInfo{\"MULPD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x5A, 1, X86InstInfo{\"CVTPD2PS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x5B, 1, X86InstInfo{\"CVTPS2DQ\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x5C, 1, X86InstInfo{\"SUBPD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x5D, 1, X86InstInfo{\"MINPD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x5E, 1, X86InstInfo{\"DIVPD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x5F, 1, X86InstInfo{\"MAXPD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n\n {0x60, 1, X86InstInfo{\"PUNPCKLBW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x61, 1, X86InstInfo{\"PUNPCKLWD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x62, 1, X86InstInfo{\"PUNPCKLDQ\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x63, 1, X86InstInfo{\"PACKSSWB\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x64, 1, X86InstInfo{\"PCMPGTB\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x65, 1, X86InstInfo{\"PCMPGTW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x66, 1, X86InstInfo{\"PCMPGTD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x67, 1, X86InstInfo{\"PACKUSWB\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x68, 1, X86InstInfo{\"PUNPCKHBW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x69, 1, X86InstInfo{\"PUNPCKHWD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x6A, 1, X86InstInfo{\"PUNPCKHDQ\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x6B, 1, X86InstInfo{\"PACKSSDW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x6C, 1, X86InstInfo{\"PUNPCKLQDQ\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x6D, 1, X86InstInfo{\"PUNPCKHQDQ\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x6E, 1, X86InstInfo{\"MOVD\", TYPE_INST, GenFlagsDstSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_SRC_GPR | FLAGS_XMM_FLAGS, 0}},\n {0x6F, 1, X86InstInfo{\"MOVDQA\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n\n {0x70, 1, X86InstInfo{\"PSHUFD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 1}},\n {0x71, 3, X86InstInfo{\"\", TYPE_COPY_OTHER, FLAGS_NONE, 0}},\n {0x74, 1, X86InstInfo{\"PCMPEQB\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x75, 1, X86InstInfo{\"PCMPEQW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x76, 1, X86InstInfo{\"PCMPEQD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x77, 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {0x78, 1, X86InstInfo{\"\", TYPE_GROUP_17, FLAGS_NONE, 0}},\n\n {0x79, 1, X86InstInfo{\"EXTRQ\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_REG_ONLY | FLAGS_XMM_FLAGS, 0}},\n {0x7A, 2, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {0x7C, 1, X86InstInfo{\"HADDPD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x7D, 1, X86InstInfo{\"HSUBPD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0x7E, 1, X86InstInfo{\"MOVD\", TYPE_INST, GenFlagsSrcSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_DST_GPR | FLAGS_XMM_FLAGS, 0}},\n {0x7F, 1, X86InstInfo{\"MOVDQA\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_XMM_FLAGS, 0}},\n\n {0x80, 16, X86InstInfo{\"\", TYPE_COPY_OTHER, FLAGS_NONE, 0}},\n {0x90, 16, X86InstInfo{\"\", TYPE_COPY_OTHER, FLAGS_NONE, 0}},\n {0xA0, 16, X86InstInfo{\"\", TYPE_COPY_OTHER, FLAGS_NONE, 0}},\n {0xB0, 8, X86InstInfo{\"\", TYPE_COPY_OTHER, FLAGS_NONE, 0}},\n {0xB8, 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {0xB9, 1, X86InstInfo{\"\", TYPE_COPY_OTHER, FLAGS_NONE, 0}},\n {0xBA, 1, X86InstInfo{\"\", TYPE_COPY_OTHER, FLAGS_NONE, 0}},\n {0xBB, 5, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n\n {0xC0, 2, X86InstInfo{\"\", TYPE_COPY_OTHER, FLAGS_NONE, 0}},\n {0xC2, 1, X86InstInfo{\"CMPPD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 1}},\n {0xC3, 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {0xC4, 1, X86InstInfo{\"PINSRW\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_16BIT) | FLAGS_MODRM | FLAGS_SF_SRC_GPR | FLAGS_XMM_FLAGS, 1}},\n {0xC5, 1, X86InstInfo{\"PEXTRW\", TYPE_INST, GenFlagsSizes(SIZE_32BIT, SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_REG_ONLY | FLAGS_SF_DST_GPR | FLAGS_XMM_FLAGS, 1}},\n {0xC6, 1, X86InstInfo{\"SHUFPD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 1}},\n {0xC7, 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {0xC8, 8, X86InstInfo{\"\", TYPE_COPY_OTHER, FLAGS_NONE, 0}},\n\n {0xD0, 1, X86InstInfo{\"ADDSUBPD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0xD1, 1, X86InstInfo{\"PSRLW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0xD2, 1, X86InstInfo{\"PSRLD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0xD3, 1, X86InstInfo{\"PSRLQ\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0xD4, 1, X86InstInfo{\"PADDQ\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0xD5, 1, X86InstInfo{\"PMULLW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0xD6, 1, X86InstInfo{\"MOVQ\", TYPE_INST, GenFlagsSizes(SIZE_64BIT, SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_XMM_FLAGS, 0}},\n {0xD7, 1, X86InstInfo{\"PMOVMSKB\", TYPE_INST, GenFlagsSizes(SIZE_32BIT, SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_REG_ONLY | FLAGS_XMM_FLAGS | FLAGS_SF_DST_GPR, 0}},\n {0xD8, 1, X86InstInfo{\"PSUBUSB\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0xD9, 1, X86InstInfo{\"PSUBUSW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0xDA, 1, X86InstInfo{\"PMINUB\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0xDB, 1, X86InstInfo{\"PAND\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0xDC, 1, X86InstInfo{\"PADDUSB\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0xDD, 1, X86InstInfo{\"PADDUSW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0xDE, 1, X86InstInfo{\"PMAXUB\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0xDF, 1, X86InstInfo{\"PANDN\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n\n {0xE0, 1, X86InstInfo{\"PAVGB\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0xE1, 1, X86InstInfo{\"PSRAW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0xE2, 1, X86InstInfo{\"PSRAD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0xE3, 1, X86InstInfo{\"PAVGW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0xE4, 1, X86InstInfo{\"PMULHUW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0xE5, 1, X86InstInfo{\"PMULHW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0xE6, 1, X86InstInfo{\"CVTTPD2DQ\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0xE7, 1, X86InstInfo{\"MOVNTDQ\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY | FLAGS_SF_MOD_DST | FLAGS_XMM_FLAGS, 0}},\n {0xE8, 1, X86InstInfo{\"PSUBSB\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0xE9, 1, X86InstInfo{\"PSUBSW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0xEA, 1, X86InstInfo{\"PMINSW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0xEB, 1, X86InstInfo{\"POR\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0xEC, 1, X86InstInfo{\"PADDSB\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0xED, 1, X86InstInfo{\"PADDSW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0xEE, 1, X86InstInfo{\"PMAXSW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0xEF, 1, X86InstInfo{\"PXOR\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n\n {0xF0, 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {0xF1, 1, X86InstInfo{\"PSLLW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0xF2, 1, X86InstInfo{\"PSLLD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0xF3, 1, X86InstInfo{\"PSLLQ\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0xF4, 1, X86InstInfo{\"PMULUDQ\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0xF5, 1, X86InstInfo{\"PMADDWD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0xF6, 1, X86InstInfo{\"PSADBW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0xF7, 1, X86InstInfo{\"MASKMOVDQU\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_REG_ONLY | FLAGS_XMM_FLAGS, 0}},\n {0xF8, 1, X86InstInfo{\"PSUBB\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0xF9, 1, X86InstInfo{\"PSUBW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0xFA, 1, X86InstInfo{\"PSUBD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0xFB, 1, X86InstInfo{\"PSUBQ\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0xFC, 1, X86InstInfo{\"PADDB\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0xFD, 1, X86InstInfo{\"PADDW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0xFE, 1, X86InstInfo{\"PADDD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {0xFF, 1, X86InstInfo{\"\", TYPE_COPY_OTHER, FLAGS_NONE, 0}},\n };\n\n GenerateTableWithCopy(Table.data(), OpSizeModOpTable, std::size(OpSizeModOpTable), SecondBaseOps.data());\n\n IR::InstallToTable(Table, IR::OpDispatch_SecondaryOpSizeModTables);\n return Table;\n}();\n}\n" }, { "path": "FEXCore/Source/Interface/Core/X86Tables/VEXTables.cpp", "content": "// SPDX-License-Identifier: MIT\n/*\n$info$\ntags: frontend|x86-tables\n$end_info$\n*/\n\n#include \"Interface/Core/X86Tables/X86Tables.h\"\n#include \"Interface/Core/OpcodeDispatcher/VEXTables.h\"\n\n#include \n\nnamespace FEXCore::X86Tables {\nusing namespace InstFlags;\n\nnamespace AVX128 {\n using namespace IR;\n#define OPD(map_select, pp, opcode) (((map_select - 1) << 10) | (pp << 8) | (opcode))\n constexpr DispatchTableEntry BaseTable[] = {\n {OPD(1, 0b00, 0x10), 1, &OpDispatchBuilder::AVX128_VMOVAPS},\n {OPD(1, 0b01, 0x10), 1, &OpDispatchBuilder::AVX128_VMOVAPS},\n {OPD(1, 0b10, 0x10), 1, &OpDispatchBuilder::AVX128_VMOVSS},\n {OPD(1, 0b11, 0x10), 1, &OpDispatchBuilder::AVX128_VMOVSD},\n {OPD(1, 0b00, 0x11), 1, &OpDispatchBuilder::AVX128_VMOVAPS},\n {OPD(1, 0b01, 0x11), 1, &OpDispatchBuilder::AVX128_VMOVAPS},\n {OPD(1, 0b10, 0x11), 1, &OpDispatchBuilder::AVX128_VMOVSS},\n {OPD(1, 0b11, 0x11), 1, &OpDispatchBuilder::AVX128_VMOVSD},\n\n {OPD(1, 0b00, 0x12), 1, &OpDispatchBuilder::AVX128_VMOVLP},\n {OPD(1, 0b01, 0x12), 1, &OpDispatchBuilder::AVX128_VMOVLP},\n {OPD(1, 0b10, 0x12), 1, &OpDispatchBuilder::AVX128_VMOVSLDUP},\n {OPD(1, 0b11, 0x12), 1, &OpDispatchBuilder::AVX128_VMOVDDUP},\n {OPD(1, 0b00, 0x13), 1, &OpDispatchBuilder::AVX128_VMOVLP},\n {OPD(1, 0b01, 0x13), 1, &OpDispatchBuilder::AVX128_VMOVLP},\n\n {OPD(1, 0b00, 0x14), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VPUNPCKL, OpSize::i32Bit>},\n {OPD(1, 0b01, 0x14), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VPUNPCKL, OpSize::i64Bit>},\n\n {OPD(1, 0b00, 0x15), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VPUNPCKH, OpSize::i32Bit>},\n {OPD(1, 0b01, 0x15), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VPUNPCKH, OpSize::i64Bit>},\n\n {OPD(1, 0b00, 0x16), 1, &OpDispatchBuilder::AVX128_VMOVHP},\n {OPD(1, 0b01, 0x16), 1, &OpDispatchBuilder::AVX128_VMOVHP},\n {OPD(1, 0b10, 0x16), 1, &OpDispatchBuilder::AVX128_VMOVSHDUP},\n {OPD(1, 0b00, 0x17), 1, &OpDispatchBuilder::AVX128_VMOVHP},\n {OPD(1, 0b01, 0x17), 1, &OpDispatchBuilder::AVX128_VMOVHP},\n\n {OPD(1, 0b00, 0x28), 1, &OpDispatchBuilder::AVX128_VMOVAPS},\n {OPD(1, 0b01, 0x28), 1, &OpDispatchBuilder::AVX128_VMOVAPS},\n {OPD(1, 0b00, 0x29), 1, &OpDispatchBuilder::AVX128_VMOVAPS},\n {OPD(1, 0b01, 0x29), 1, &OpDispatchBuilder::AVX128_VMOVAPS},\n\n {OPD(1, 0b10, 0x2A), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_InsertCVTGPR_To_FPR, OpSize::i32Bit>},\n {OPD(1, 0b11, 0x2A), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_InsertCVTGPR_To_FPR, OpSize::i64Bit>},\n\n {OPD(1, 0b00, 0x2B), 1, &OpDispatchBuilder::AVX128_MOVVectorNT},\n {OPD(1, 0b01, 0x2B), 1, &OpDispatchBuilder::AVX128_MOVVectorNT},\n\n {OPD(1, 0b10, 0x2C), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_CVTFPR_To_GPR, OpSize::i32Bit, false>},\n {OPD(1, 0b11, 0x2C), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_CVTFPR_To_GPR, OpSize::i64Bit, false>},\n\n {OPD(1, 0b10, 0x2D), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_CVTFPR_To_GPR, OpSize::i32Bit, true>},\n {OPD(1, 0b11, 0x2D), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_CVTFPR_To_GPR, OpSize::i64Bit, true>},\n\n {OPD(1, 0b00, 0x2E), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_UCOMISx, OpSize::i32Bit>},\n {OPD(1, 0b01, 0x2E), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_UCOMISx, OpSize::i64Bit>},\n {OPD(1, 0b00, 0x2F), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_UCOMISx, OpSize::i32Bit>},\n {OPD(1, 0b01, 0x2F), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_UCOMISx, OpSize::i64Bit>},\n\n {OPD(1, 0b00, 0x50), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_MOVMSK, OpSize::i32Bit>},\n {OPD(1, 0b01, 0x50), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_MOVMSK, OpSize::i64Bit>},\n\n {OPD(1, 0b00, 0x51), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorUnary, IR::OP_VFSQRT, OpSize::i32Bit>},\n {OPD(1, 0b01, 0x51), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorUnary, IR::OP_VFSQRT, OpSize::i64Bit>},\n {OPD(1, 0b10, 0x51), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorScalarInsertALU, IR::OP_VFSQRTSCALARINSERT, OpSize::i32Bit>},\n {OPD(1, 0b11, 0x51), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorScalarInsertALU, IR::OP_VFSQRTSCALARINSERT, OpSize::i64Bit>},\n\n {OPD(1, 0b00, 0x52), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorUnary, IR::OP_VFRSQRT, OpSize::i32Bit>},\n {OPD(1, 0b10, 0x52), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorScalarInsertALU, IR::OP_VFRSQRTSCALARINSERT, OpSize::i32Bit>},\n\n {OPD(1, 0b00, 0x53), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorUnary, IR::OP_VFRECP, OpSize::i32Bit>},\n {OPD(1, 0b10, 0x53), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorScalarInsertALU, IR::OP_VFRECPSCALARINSERT, OpSize::i32Bit>},\n\n {OPD(1, 0b00, 0x54), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorALU, IR::OP_VAND, OpSize::i128Bit>},\n {OPD(1, 0b01, 0x54), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorALU, IR::OP_VAND, OpSize::i128Bit>},\n\n {OPD(1, 0b00, 0x55), 1, &OpDispatchBuilder::AVX128_VANDN},\n {OPD(1, 0b01, 0x55), 1, &OpDispatchBuilder::AVX128_VANDN},\n\n {OPD(1, 0b00, 0x56), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorALU, IR::OP_VOR, OpSize::i128Bit>},\n {OPD(1, 0b01, 0x56), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorALU, IR::OP_VOR, OpSize::i128Bit>},\n\n {OPD(1, 0b00, 0x57), 1, &OpDispatchBuilder::AVX128_VectorXOR},\n {OPD(1, 0b01, 0x57), 1, &OpDispatchBuilder::AVX128_VectorXOR},\n\n {OPD(1, 0b00, 0x58), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorALU, IR::OP_VFADD, OpSize::i32Bit>},\n {OPD(1, 0b01, 0x58), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorALU, IR::OP_VFADD, OpSize::i64Bit>},\n {OPD(1, 0b10, 0x58), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorScalarInsertALU, IR::OP_VFADDSCALARINSERT, OpSize::i32Bit>},\n {OPD(1, 0b11, 0x58), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorScalarInsertALU, IR::OP_VFADDSCALARINSERT, OpSize::i64Bit>},\n\n {OPD(1, 0b00, 0x59), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorALU, IR::OP_VFMUL, OpSize::i32Bit>},\n {OPD(1, 0b01, 0x59), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorALU, IR::OP_VFMUL, OpSize::i64Bit>},\n {OPD(1, 0b10, 0x59), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorScalarInsertALU, IR::OP_VFMULSCALARINSERT, OpSize::i32Bit>},\n {OPD(1, 0b11, 0x59), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorScalarInsertALU, IR::OP_VFMULSCALARINSERT, OpSize::i64Bit>},\n\n {OPD(1, 0b00, 0x5A), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_Vector_CVT_Float_To_Float, OpSize::i64Bit, OpSize::i32Bit>},\n {OPD(1, 0b01, 0x5A), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_Vector_CVT_Float_To_Float, OpSize::i32Bit, OpSize::i64Bit>},\n {OPD(1, 0b10, 0x5A), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_InsertScalar_CVT_Float_To_Float, OpSize::i64Bit, OpSize::i32Bit>},\n {OPD(1, 0b11, 0x5A), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_InsertScalar_CVT_Float_To_Float, OpSize::i32Bit, OpSize::i64Bit>},\n\n {OPD(1, 0b00, 0x5B), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_Vector_CVT_Int_To_Float, OpSize::i32Bit, false>},\n {OPD(1, 0b01, 0x5B), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_Vector_CVT_Float_To_Int, OpSize::i32Bit, true>},\n {OPD(1, 0b10, 0x5B), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_Vector_CVT_Float_To_Int, OpSize::i32Bit, false>},\n\n {OPD(1, 0b00, 0x5C), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorALU, IR::OP_VFSUB, OpSize::i32Bit>},\n {OPD(1, 0b01, 0x5C), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorALU, IR::OP_VFSUB, OpSize::i64Bit>},\n {OPD(1, 0b10, 0x5C), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorScalarInsertALU, IR::OP_VFSUBSCALARINSERT, OpSize::i32Bit>},\n {OPD(1, 0b11, 0x5C), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorScalarInsertALU, IR::OP_VFSUBSCALARINSERT, OpSize::i64Bit>},\n\n {OPD(1, 0b00, 0x5D), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorALU, IR::OP_VFMIN, OpSize::i32Bit>},\n {OPD(1, 0b01, 0x5D), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorALU, IR::OP_VFMIN, OpSize::i64Bit>},\n {OPD(1, 0b10, 0x5D), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorScalarInsertALU, IR::OP_VFMINSCALARINSERT, OpSize::i32Bit>},\n {OPD(1, 0b11, 0x5D), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorScalarInsertALU, IR::OP_VFMINSCALARINSERT, OpSize::i64Bit>},\n\n {OPD(1, 0b00, 0x5E), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorALU, IR::OP_VFDIV, OpSize::i32Bit>},\n {OPD(1, 0b01, 0x5E), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorALU, IR::OP_VFDIV, OpSize::i64Bit>},\n {OPD(1, 0b10, 0x5E), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorScalarInsertALU, IR::OP_VFDIVSCALARINSERT, OpSize::i32Bit>},\n {OPD(1, 0b11, 0x5E), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorScalarInsertALU, IR::OP_VFDIVSCALARINSERT, OpSize::i64Bit>},\n\n {OPD(1, 0b00, 0x5F), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorALU, IR::OP_VFMAX, OpSize::i32Bit>},\n {OPD(1, 0b01, 0x5F), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorALU, IR::OP_VFMAX, OpSize::i64Bit>},\n {OPD(1, 0b10, 0x5F), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorScalarInsertALU, IR::OP_VFMAXSCALARINSERT, OpSize::i32Bit>},\n {OPD(1, 0b11, 0x5F), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorScalarInsertALU, IR::OP_VFMAXSCALARINSERT, OpSize::i64Bit>},\n\n {OPD(1, 0b01, 0x60), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VPUNPCKL, OpSize::i8Bit>},\n {OPD(1, 0b01, 0x61), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VPUNPCKL, OpSize::i16Bit>},\n {OPD(1, 0b01, 0x62), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VPUNPCKL, OpSize::i32Bit>},\n {OPD(1, 0b01, 0x63), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VPACKSS, OpSize::i16Bit>},\n {OPD(1, 0b01, 0x64), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorALU, IR::OP_VCMPGT, OpSize::i8Bit>},\n {OPD(1, 0b01, 0x65), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorALU, IR::OP_VCMPGT, OpSize::i16Bit>},\n {OPD(1, 0b01, 0x66), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorALU, IR::OP_VCMPGT, OpSize::i32Bit>},\n {OPD(1, 0b01, 0x67), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VPACKUS, OpSize::i16Bit>},\n {OPD(1, 0b01, 0x68), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VPUNPCKH, OpSize::i8Bit>},\n {OPD(1, 0b01, 0x69), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VPUNPCKH, OpSize::i16Bit>},\n {OPD(1, 0b01, 0x6A), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VPUNPCKH, OpSize::i32Bit>},\n {OPD(1, 0b01, 0x6B), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VPACKSS, OpSize::i32Bit>},\n {OPD(1, 0b01, 0x6C), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VPUNPCKL, OpSize::i64Bit>},\n {OPD(1, 0b01, 0x6D), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VPUNPCKH, OpSize::i64Bit>},\n {OPD(1, 0b01, 0x6E), 1, &OpDispatchBuilder::AVX128_MOVBetweenGPR_FPR},\n\n {OPD(1, 0b01, 0x6F), 1, &OpDispatchBuilder::AVX128_VMOVAPS},\n {OPD(1, 0b10, 0x6F), 1, &OpDispatchBuilder::AVX128_VMOVAPS},\n\n {OPD(1, 0b01, 0x70), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VPERMILImm, OpSize::i32Bit>},\n {OPD(1, 0b10, 0x70), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VPSHUFW, false>},\n {OPD(1, 0b11, 0x70), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VPSHUFW, true>},\n\n {OPD(1, 0b01, 0x74), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorALU, IR::OP_VCMPEQ, OpSize::i8Bit>},\n {OPD(1, 0b01, 0x75), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorALU, IR::OP_VCMPEQ, OpSize::i16Bit>},\n {OPD(1, 0b01, 0x76), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorALU, IR::OP_VCMPEQ, OpSize::i32Bit>},\n\n {OPD(1, 0b00, 0x77), 1, &OpDispatchBuilder::AVX128_VZERO},\n\n {OPD(1, 0b01, 0x7C), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VHADDP, IR::OP_VFADDP, OpSize::i64Bit>},\n {OPD(1, 0b11, 0x7C), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VHADDP, IR::OP_VFADDP, OpSize::i32Bit>},\n {OPD(1, 0b01, 0x7D), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VHSUBP, OpSize::i64Bit>},\n {OPD(1, 0b11, 0x7D), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VHSUBP, OpSize::i32Bit>},\n\n {OPD(1, 0b01, 0x7E), 1, &OpDispatchBuilder::AVX128_MOVBetweenGPR_FPR},\n {OPD(1, 0b10, 0x7E), 1, &OpDispatchBuilder::AVX128_MOVQ},\n\n {OPD(1, 0b01, 0x7F), 1, &OpDispatchBuilder::AVX128_VMOVAPS},\n {OPD(1, 0b10, 0x7F), 1, &OpDispatchBuilder::AVX128_VMOVAPS},\n\n {OPD(1, 0b00, 0xC2), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VFCMP, OpSize::i32Bit>},\n {OPD(1, 0b01, 0xC2), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VFCMP, OpSize::i64Bit>},\n {OPD(1, 0b10, 0xC2), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_InsertScalarFCMP, OpSize::i32Bit>},\n {OPD(1, 0b11, 0xC2), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_InsertScalarFCMP, OpSize::i64Bit>},\n\n {OPD(1, 0b01, 0xC4), 1, &OpDispatchBuilder::AVX128_VPINSRW},\n {OPD(1, 0b01, 0xC5), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_PExtr, OpSize::i16Bit>},\n\n {OPD(1, 0b00, 0xC6), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VSHUF, OpSize::i32Bit>},\n {OPD(1, 0b01, 0xC6), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VSHUF, OpSize::i64Bit>},\n\n {OPD(1, 0b01, 0xD0), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VADDSUBP, OpSize::i64Bit>},\n {OPD(1, 0b11, 0xD0), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VADDSUBP, OpSize::i32Bit>},\n\n {OPD(1, 0b01, 0xD1), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorShiftWideImpl, OpSize::i16Bit, IROps::OP_VUSHRSWIDE>}, // VPSRL\n {OPD(1, 0b01, 0xD2), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorShiftWideImpl, OpSize::i32Bit, IROps::OP_VUSHRSWIDE>}, // VPSRL\n {OPD(1, 0b01, 0xD3), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorShiftWideImpl, OpSize::i64Bit, IROps::OP_VUSHRSWIDE>}, // VPSRL\n {OPD(1, 0b01, 0xD4), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorALU, IR::OP_VADD, OpSize::i64Bit>},\n {OPD(1, 0b01, 0xD5), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorALU, IR::OP_VMUL, OpSize::i16Bit>},\n {OPD(1, 0b01, 0xD6), 1, &OpDispatchBuilder::AVX128_MOVQ},\n {OPD(1, 0b01, 0xD7), 1, &OpDispatchBuilder::AVX128_MOVMSKB},\n\n {OPD(1, 0b01, 0xD8), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorALU, IR::OP_VUQSUB, OpSize::i8Bit>},\n {OPD(1, 0b01, 0xD9), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorALU, IR::OP_VUQSUB, OpSize::i16Bit>},\n {OPD(1, 0b01, 0xDA), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorALU, IR::OP_VUMIN, OpSize::i8Bit>},\n {OPD(1, 0b01, 0xDB), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorALU, IR::OP_VAND, OpSize::i128Bit>},\n {OPD(1, 0b01, 0xDC), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorALU, IR::OP_VUQADD, OpSize::i8Bit>},\n {OPD(1, 0b01, 0xDD), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorALU, IR::OP_VUQADD, OpSize::i16Bit>},\n {OPD(1, 0b01, 0xDE), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorALU, IR::OP_VUMAX, OpSize::i8Bit>},\n {OPD(1, 0b01, 0xDF), 1, &OpDispatchBuilder::AVX128_VANDN},\n\n {OPD(1, 0b01, 0xE0), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorALU, IR::OP_VURAVG, OpSize::i8Bit>},\n {OPD(1, 0b01, 0xE1), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorShiftWideImpl, OpSize::i16Bit, IROps::OP_VSSHRSWIDE>}, // VPSRA\n {OPD(1, 0b01, 0xE2), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorShiftWideImpl, OpSize::i32Bit, IROps::OP_VSSHRSWIDE>}, // VPSRA\n {OPD(1, 0b01, 0xE3), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorALU, IR::OP_VURAVG, OpSize::i16Bit>},\n {OPD(1, 0b01, 0xE4), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VPMULHW, false>},\n {OPD(1, 0b01, 0xE5), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VPMULHW, true>},\n\n {OPD(1, 0b01, 0xE6), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_Vector_CVT_Float_To_Int, OpSize::i64Bit, false>},\n {OPD(1, 0b10, 0xE6), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_Vector_CVT_Int_To_Float, OpSize::i32Bit, true>},\n {OPD(1, 0b11, 0xE6), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_Vector_CVT_Float_To_Int, OpSize::i64Bit, true>},\n\n {OPD(1, 0b01, 0xE7), 1, &OpDispatchBuilder::AVX128_MOVVectorNT},\n\n {OPD(1, 0b01, 0xE8), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorALU, IR::OP_VSQSUB, OpSize::i8Bit>},\n {OPD(1, 0b01, 0xE9), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorALU, IR::OP_VSQSUB, OpSize::i16Bit>},\n {OPD(1, 0b01, 0xEA), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorALU, IR::OP_VSMIN, OpSize::i16Bit>},\n {OPD(1, 0b01, 0xEB), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorALU, IR::OP_VOR, OpSize::i128Bit>},\n {OPD(1, 0b01, 0xEC), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorALU, IR::OP_VSQADD, OpSize::i8Bit>},\n {OPD(1, 0b01, 0xED), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorALU, IR::OP_VSQADD, OpSize::i16Bit>},\n {OPD(1, 0b01, 0xEE), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorALU, IR::OP_VSMAX, OpSize::i16Bit>},\n {OPD(1, 0b01, 0xEF), 1, &OpDispatchBuilder::AVX128_VectorXOR},\n\n {OPD(1, 0b11, 0xF0), 1, &OpDispatchBuilder::AVX128_MOVVectorUnaligned},\n {OPD(1, 0b01, 0xF1), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorShiftWideImpl, OpSize::i16Bit, IROps::OP_VUSHLSWIDE>}, // VPSLL\n {OPD(1, 0b01, 0xF2), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorShiftWideImpl, OpSize::i32Bit, IROps::OP_VUSHLSWIDE>}, // VPSLL\n {OPD(1, 0b01, 0xF3), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorShiftWideImpl, OpSize::i64Bit, IROps::OP_VUSHLSWIDE>}, // VPSLL\n {OPD(1, 0b01, 0xF4), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VPMULL, OpSize::i32Bit, false>},\n {OPD(1, 0b01, 0xF5), 1, &OpDispatchBuilder::AVX128_VPMADDWD},\n {OPD(1, 0b01, 0xF6), 1, &OpDispatchBuilder::AVX128_VPSADBW},\n {OPD(1, 0b01, 0xF7), 1, &OpDispatchBuilder::AVX128_MASKMOV},\n\n {OPD(1, 0b01, 0xF8), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorALU, IR::OP_VSUB, OpSize::i8Bit>},\n {OPD(1, 0b01, 0xF9), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorALU, IR::OP_VSUB, OpSize::i16Bit>},\n {OPD(1, 0b01, 0xFA), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorALU, IR::OP_VSUB, OpSize::i32Bit>},\n {OPD(1, 0b01, 0xFB), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorALU, IR::OP_VSUB, OpSize::i64Bit>},\n {OPD(1, 0b01, 0xFC), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorALU, IR::OP_VADD, OpSize::i8Bit>},\n {OPD(1, 0b01, 0xFD), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorALU, IR::OP_VADD, OpSize::i16Bit>},\n {OPD(1, 0b01, 0xFE), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorALU, IR::OP_VADD, OpSize::i32Bit>},\n\n {OPD(2, 0b01, 0x00), 1, &OpDispatchBuilder::AVX128_VPSHUFB},\n {OPD(2, 0b01, 0x01), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VHADDP, IR::OP_VADDP, OpSize::i16Bit>},\n {OPD(2, 0b01, 0x02), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VHADDP, IR::OP_VADDP, OpSize::i32Bit>},\n {OPD(2, 0b01, 0x03), 1, &OpDispatchBuilder::AVX128_VPHADDSW},\n {OPD(2, 0b01, 0x04), 1, &OpDispatchBuilder::AVX128_VPMADDUBSW},\n\n {OPD(2, 0b01, 0x05), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VPHSUB, OpSize::i16Bit>},\n {OPD(2, 0b01, 0x06), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VPHSUB, OpSize::i32Bit>},\n {OPD(2, 0b01, 0x07), 1, &OpDispatchBuilder::AVX128_VPHSUBSW},\n\n {OPD(2, 0b01, 0x08), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VPSIGN, OpSize::i8Bit>},\n {OPD(2, 0b01, 0x09), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VPSIGN, OpSize::i16Bit>},\n {OPD(2, 0b01, 0x0A), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VPSIGN, OpSize::i32Bit>},\n {OPD(2, 0b01, 0x0B), 1, &OpDispatchBuilder::AVX128_VPMULHRSW},\n {OPD(2, 0b01, 0x0C), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VPERMILReg, OpSize::i32Bit>},\n {OPD(2, 0b01, 0x0D), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VPERMILReg, OpSize::i64Bit>},\n {OPD(2, 0b01, 0x0E), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VTESTP, OpSize::i32Bit>},\n {OPD(2, 0b01, 0x0F), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VTESTP, OpSize::i64Bit>},\n\n\n {OPD(2, 0b01, 0x13), 1, &OpDispatchBuilder::AVX128_VCVTPH2PS},\n {OPD(2, 0b01, 0x16), 1, &OpDispatchBuilder::AVX128_VPERMD},\n {OPD(2, 0b01, 0x17), 1, &OpDispatchBuilder::AVX128_PTest},\n {OPD(2, 0b01, 0x18), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VBROADCAST, OpSize::i32Bit>},\n {OPD(2, 0b01, 0x19), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VBROADCAST, OpSize::i64Bit>},\n {OPD(2, 0b01, 0x1A), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VBROADCAST, OpSize::i128Bit>},\n {OPD(2, 0b01, 0x1C), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorUnary, IR::OP_VABS, OpSize::i8Bit>},\n {OPD(2, 0b01, 0x1D), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorUnary, IR::OP_VABS, OpSize::i16Bit>},\n {OPD(2, 0b01, 0x1E), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorUnary, IR::OP_VABS, OpSize::i32Bit>},\n\n {OPD(2, 0b01, 0x20), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_ExtendVectorElements, OpSize::i8Bit, OpSize::i16Bit, true>},\n {OPD(2, 0b01, 0x21), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_ExtendVectorElements, OpSize::i8Bit, OpSize::i32Bit, true>},\n {OPD(2, 0b01, 0x22), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_ExtendVectorElements, OpSize::i8Bit, OpSize::i64Bit, true>},\n {OPD(2, 0b01, 0x23), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_ExtendVectorElements, OpSize::i16Bit, OpSize::i32Bit, true>},\n {OPD(2, 0b01, 0x24), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_ExtendVectorElements, OpSize::i16Bit, OpSize::i64Bit, true>},\n {OPD(2, 0b01, 0x25), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_ExtendVectorElements, OpSize::i32Bit, OpSize::i64Bit, true>},\n\n {OPD(2, 0b01, 0x28), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VPMULL, OpSize::i32Bit, true>},\n {OPD(2, 0b01, 0x29), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorALU, IR::OP_VCMPEQ, OpSize::i64Bit>},\n {OPD(2, 0b01, 0x2A), 1, &OpDispatchBuilder::AVX128_MOVVectorNT},\n {OPD(2, 0b01, 0x2B), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VPACKUS, OpSize::i32Bit>},\n {OPD(2, 0b01, 0x2C), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VMASKMOV, OpSize::i32Bit, false>},\n {OPD(2, 0b01, 0x2D), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VMASKMOV, OpSize::i64Bit, false>},\n {OPD(2, 0b01, 0x2E), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VMASKMOV, OpSize::i32Bit, true>},\n {OPD(2, 0b01, 0x2F), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VMASKMOV, OpSize::i64Bit, true>},\n\n {OPD(2, 0b01, 0x30), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_ExtendVectorElements, OpSize::i8Bit, OpSize::i16Bit, false>},\n {OPD(2, 0b01, 0x31), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_ExtendVectorElements, OpSize::i8Bit, OpSize::i32Bit, false>},\n {OPD(2, 0b01, 0x32), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_ExtendVectorElements, OpSize::i8Bit, OpSize::i64Bit, false>},\n {OPD(2, 0b01, 0x33), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_ExtendVectorElements, OpSize::i16Bit, OpSize::i32Bit, false>},\n {OPD(2, 0b01, 0x34), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_ExtendVectorElements, OpSize::i16Bit, OpSize::i64Bit, false>},\n {OPD(2, 0b01, 0x35), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_ExtendVectorElements, OpSize::i32Bit, OpSize::i64Bit, false>},\n {OPD(2, 0b01, 0x36), 1, &OpDispatchBuilder::AVX128_VPERMD},\n\n {OPD(2, 0b01, 0x37), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorALU, IR::OP_VCMPGT, OpSize::i64Bit>},\n {OPD(2, 0b01, 0x38), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorALU, IR::OP_VSMIN, OpSize::i8Bit>},\n {OPD(2, 0b01, 0x39), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorALU, IR::OP_VSMIN, OpSize::i32Bit>},\n {OPD(2, 0b01, 0x3A), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorALU, IR::OP_VUMIN, OpSize::i16Bit>},\n {OPD(2, 0b01, 0x3B), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorALU, IR::OP_VUMIN, OpSize::i32Bit>},\n {OPD(2, 0b01, 0x3C), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorALU, IR::OP_VSMAX, OpSize::i8Bit>},\n {OPD(2, 0b01, 0x3D), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorALU, IR::OP_VSMAX, OpSize::i32Bit>},\n {OPD(2, 0b01, 0x3E), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorALU, IR::OP_VUMAX, OpSize::i16Bit>},\n {OPD(2, 0b01, 0x3F), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorALU, IR::OP_VUMAX, OpSize::i32Bit>},\n\n {OPD(2, 0b01, 0x40), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorALU, IR::OP_VMUL, OpSize::i32Bit>},\n {OPD(2, 0b01, 0x41), 1, &OpDispatchBuilder::AVX128_PHMINPOSUW},\n {OPD(2, 0b01, 0x45), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VariableShiftImpl, IROps::OP_VUSHR>}, // VPSRLV\n {OPD(2, 0b01, 0x46), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VariableShiftImpl, IROps::OP_VSSHR>}, // VPSRAVD\n {OPD(2, 0b01, 0x47), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VariableShiftImpl, IROps::OP_VUSHL>}, // VPSLLV\n\n {OPD(2, 0b01, 0x58), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VBROADCAST, OpSize::i32Bit>},\n {OPD(2, 0b01, 0x59), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VBROADCAST, OpSize::i64Bit>},\n {OPD(2, 0b01, 0x5A), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VBROADCAST, OpSize::i128Bit>},\n\n {OPD(2, 0b01, 0x78), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VBROADCAST, OpSize::i8Bit>},\n {OPD(2, 0b01, 0x79), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VBROADCAST, OpSize::i16Bit>},\n\n {OPD(2, 0b01, 0x8C), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VPMASKMOV, false>},\n {OPD(2, 0b01, 0x8E), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VPMASKMOV, true>},\n\n {OPD(2, 0b01, 0x90), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VPGATHER, OpSize::i32Bit>},\n {OPD(2, 0b01, 0x91), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VPGATHER, OpSize::i64Bit>},\n {OPD(2, 0b01, 0x92), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VPGATHER, OpSize::i32Bit>},\n {OPD(2, 0b01, 0x93), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VPGATHER, OpSize::i64Bit>},\n\n {OPD(2, 0b01, 0x96), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VFMAddSubImpl, true, 1, 3, 2>}, // VFMADDSUB\n {OPD(2, 0b01, 0x97), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VFMAddSubImpl, false, 1, 3, 2>}, // VFMSUBADD\n\n {OPD(2, 0b01, 0x98), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VFMAImpl, IR::OP_VFMLA, 1, 3, 2>}, // VFMADD\n {OPD(2, 0b01, 0x99), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VFMAScalarImpl, IR::OP_VFMLASCALARINSERT, 1, 3, 2>}, // VFMADD\n {OPD(2, 0b01, 0x9A), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VFMAImpl, IR::OP_VFMLS, 1, 3, 2>}, // VFMSUB\n {OPD(2, 0b01, 0x9B), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VFMAScalarImpl, IR::OP_VFMLSSCALARINSERT, 1, 3, 2>}, // VFMSUB\n {OPD(2, 0b01, 0x9C), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VFMAImpl, IR::OP_VFNMLA, 1, 3, 2>}, // VFNMADD\n {OPD(2, 0b01, 0x9D), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VFMAScalarImpl, IR::OP_VFNMLASCALARINSERT, 1, 3, 2>}, // VFNMADD\n {OPD(2, 0b01, 0x9E), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VFMAImpl, IR::OP_VFNMLS, 1, 3, 2>}, // VFNMSUB\n {OPD(2, 0b01, 0x9F), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VFMAScalarImpl, IR::OP_VFNMLSSCALARINSERT, 1, 3, 2>}, // VFNMSUB\n\n {OPD(2, 0b01, 0xA8), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VFMAImpl, IR::OP_VFMLA, 2, 1, 3>}, // VFMADD\n {OPD(2, 0b01, 0xA9), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VFMAScalarImpl, IR::OP_VFMLASCALARINSERT, 2, 1, 3>}, // VFMADD\n {OPD(2, 0b01, 0xAA), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VFMAImpl, IR::OP_VFMLS, 2, 1, 3>}, // VFMSUB\n {OPD(2, 0b01, 0xAB), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VFMAScalarImpl, IR::OP_VFMLSSCALARINSERT, 2, 1, 3>}, // VFMSUB\n {OPD(2, 0b01, 0xAC), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VFMAImpl, IR::OP_VFNMLA, 2, 1, 3>}, // VFNMADD\n {OPD(2, 0b01, 0xAD), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VFMAScalarImpl, IR::OP_VFNMLASCALARINSERT, 2, 1, 3>}, // VFNMADD\n {OPD(2, 0b01, 0xAE), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VFMAImpl, IR::OP_VFNMLS, 2, 1, 3>}, // VFNMSUB\n {OPD(2, 0b01, 0xAF), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VFMAScalarImpl, IR::OP_VFNMLSSCALARINSERT, 2, 1, 3>}, // VFNMSUB\n\n {OPD(2, 0b01, 0xB8), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VFMAImpl, IR::OP_VFMLA, 2, 3, 1>}, // VFMADD\n {OPD(2, 0b01, 0xB9), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VFMAScalarImpl, IR::OP_VFMLASCALARINSERT, 2, 3, 1>}, // VFMADD\n {OPD(2, 0b01, 0xBA), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VFMAImpl, IR::OP_VFMLS, 2, 3, 1>}, // VFMSUB\n {OPD(2, 0b01, 0xBB), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VFMAScalarImpl, IR::OP_VFMLSSCALARINSERT, 2, 3, 1>}, // VFMSUB\n {OPD(2, 0b01, 0xBC), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VFMAImpl, IR::OP_VFNMLA, 2, 3, 1>}, // VFNMADD\n {OPD(2, 0b01, 0xBD), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VFMAScalarImpl, IR::OP_VFNMLASCALARINSERT, 2, 3, 1>}, // VFNMADD\n {OPD(2, 0b01, 0xBE), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VFMAImpl, IR::OP_VFNMLS, 2, 3, 1>}, // VFNMSUB\n {OPD(2, 0b01, 0xBF), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VFMAScalarImpl, IR::OP_VFNMLSSCALARINSERT, 2, 3, 1>}, // VFNMSUB\n\n {OPD(2, 0b01, 0xA6), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VFMAddSubImpl, true, 2, 1, 3>}, // VFMADDSUB\n {OPD(2, 0b01, 0xA7), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VFMAddSubImpl, false, 2, 1, 3>}, // VFMSUBADD\n\n {OPD(2, 0b01, 0xB6), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VFMAddSubImpl, true, 2, 3, 1>}, // VFMADDSUB\n {OPD(2, 0b01, 0xB7), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VFMAddSubImpl, false, 2, 3, 1>}, // VFMSUBADD\n\n {OPD(2, 0b01, 0xDB), 1, &OpDispatchBuilder::AVX128_VAESImc},\n {OPD(2, 0b01, 0xDC), 1, &OpDispatchBuilder::AVX128_VAESEnc},\n {OPD(2, 0b01, 0xDD), 1, &OpDispatchBuilder::AVX128_VAESEncLast},\n {OPD(2, 0b01, 0xDE), 1, &OpDispatchBuilder::AVX128_VAESDec},\n {OPD(2, 0b01, 0xDF), 1, &OpDispatchBuilder::AVX128_VAESDecLast},\n\n {OPD(3, 0b01, 0x00), 1, &OpDispatchBuilder::AVX128_VPERMQ},\n {OPD(3, 0b01, 0x01), 1, &OpDispatchBuilder::AVX128_VPERMQ},\n {OPD(3, 0b01, 0x02), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VBLEND, OpSize::i32Bit>},\n {OPD(3, 0b01, 0x04), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VPERMILImm, OpSize::i32Bit>},\n {OPD(3, 0b01, 0x05), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VPERMILImm, OpSize::i64Bit>},\n {OPD(3, 0b01, 0x06), 1, &OpDispatchBuilder::AVX128_VPERM2},\n {OPD(3, 0b01, 0x08), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorRound, OpSize::i32Bit>},\n {OPD(3, 0b01, 0x09), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorRound, OpSize::i64Bit>},\n {OPD(3, 0b01, 0x0A), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_InsertScalarRound, OpSize::i32Bit>},\n {OPD(3, 0b01, 0x0B), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_InsertScalarRound, OpSize::i64Bit>},\n {OPD(3, 0b01, 0x0C), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VBLEND, OpSize::i32Bit>},\n {OPD(3, 0b01, 0x0D), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VBLEND, OpSize::i64Bit>},\n {OPD(3, 0b01, 0x0E), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VBLEND, OpSize::i16Bit>},\n {OPD(3, 0b01, 0x0F), 1, &OpDispatchBuilder::AVX128_VPALIGNR},\n\n {OPD(3, 0b01, 0x14), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_PExtr, OpSize::i8Bit>},\n {OPD(3, 0b01, 0x15), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_PExtr, OpSize::i16Bit>},\n {OPD(3, 0b01, 0x16), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_PExtr, OpSize::i32Bit>},\n {OPD(3, 0b01, 0x17), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_PExtr, OpSize::i32Bit>},\n\n {OPD(3, 0b01, 0x18), 1, &OpDispatchBuilder::AVX128_VINSERT},\n {OPD(3, 0b01, 0x19), 1, &OpDispatchBuilder::AVX128_VEXTRACT128},\n {OPD(3, 0b01, 0x1D), 1, &OpDispatchBuilder::AVX128_VCVTPS2PH},\n {OPD(3, 0b01, 0x20), 1, &OpDispatchBuilder::AVX128_VPINSRB},\n {OPD(3, 0b01, 0x21), 1, &OpDispatchBuilder::AVX128_VINSERTPS},\n {OPD(3, 0b01, 0x22), 1, &OpDispatchBuilder::AVX128_VPINSRDQ},\n\n {OPD(3, 0b01, 0x38), 1, &OpDispatchBuilder::AVX128_VINSERT},\n {OPD(3, 0b01, 0x39), 1, &OpDispatchBuilder::AVX128_VEXTRACT128},\n\n {OPD(3, 0b01, 0x40), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VDPP, OpSize::i32Bit>},\n {OPD(3, 0b01, 0x41), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VDPP, OpSize::i64Bit>},\n {OPD(3, 0b01, 0x42), 1, &OpDispatchBuilder::AVX128_VMPSADBW},\n {OPD(3, 0b01, 0x44), 1, &OpDispatchBuilder::AVX128_VPCLMULQDQ},\n\n {OPD(3, 0b01, 0x46), 1, &OpDispatchBuilder::AVX128_VPERM2},\n\n {OPD(3, 0b01, 0x4A), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorVariableBlend, OpSize::i32Bit>},\n {OPD(3, 0b01, 0x4B), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorVariableBlend, OpSize::i64Bit>},\n {OPD(3, 0b01, 0x4C), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorVariableBlend, OpSize::i8Bit>},\n\n {OPD(3, 0b01, 0x60), 1, &OpDispatchBuilder::AVX128_VPCMPESTRM},\n {OPD(3, 0b01, 0x61), 1, &OpDispatchBuilder::AVX128_VPCMPESTRI},\n {OPD(3, 0b01, 0x62), 1, &OpDispatchBuilder::AVX128_VPCMPISTRM},\n {OPD(3, 0b01, 0x63), 1, &OpDispatchBuilder::AVX128_VPCMPISTRI},\n\n {OPD(3, 0b01, 0xDF), 1, &OpDispatchBuilder::AVX128_VAESKeyGenAssist},\n };\n#undef OPD\n\n#define OPD(group, pp, opcode) (((group - X86Tables::TYPE_VEX_GROUP_12) << 4) | (pp << 3) | (opcode))\n constexpr DispatchTableEntry TableGroupOps[] {\n // VPSRLI\n {OPD(X86Tables::TYPE_VEX_GROUP_12, 1, 0b010), 1,\n &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorShiftImmImpl, OpSize::i16Bit, IROps::OP_VUSHRI>},\n // VPSLLI\n {OPD(X86Tables::TYPE_VEX_GROUP_12, 1, 0b110), 1,\n &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorShiftImmImpl, OpSize::i16Bit, IROps::OP_VSHLI>},\n // VPSRAI\n {OPD(X86Tables::TYPE_VEX_GROUP_12, 1, 0b100), 1,\n &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorShiftImmImpl, OpSize::i16Bit, IROps::OP_VSSHRI>},\n\n // VPSRLI\n {OPD(X86Tables::TYPE_VEX_GROUP_13, 1, 0b010), 1,\n &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorShiftImmImpl, OpSize::i32Bit, IROps::OP_VUSHRI>},\n // VPSLLI\n {OPD(X86Tables::TYPE_VEX_GROUP_13, 1, 0b110), 1,\n &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorShiftImmImpl, OpSize::i32Bit, IROps::OP_VSHLI>},\n // VPSRAI\n {OPD(X86Tables::TYPE_VEX_GROUP_13, 1, 0b100), 1,\n &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorShiftImmImpl, OpSize::i32Bit, IROps::OP_VSSHRI>},\n\n // VPSRLI\n {OPD(X86Tables::TYPE_VEX_GROUP_14, 1, 0b010), 1,\n &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorShiftImmImpl, OpSize::i64Bit, IROps::OP_VUSHRI>},\n // VPSRLDQ\n {OPD(X86Tables::TYPE_VEX_GROUP_14, 1, 0b011), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_ShiftDoubleImm, OpDispatchBuilder::ShiftDirection::RIGHT>},\n // VPSLLI\n {OPD(X86Tables::TYPE_VEX_GROUP_14, 1, 0b110), 1,\n &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorShiftImmImpl, OpSize::i64Bit, IROps::OP_VSHLI>},\n // VPSLLDQ\n {OPD(X86Tables::TYPE_VEX_GROUP_14, 1, 0b111), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_ShiftDoubleImm, OpDispatchBuilder::ShiftDirection::LEFT>},\n\n ///< Use the regular implementation. It just happens to be in the VEX table.\n {OPD(X86Tables::TYPE_VEX_GROUP_15, 0, 0b010), 1, &OpDispatchBuilder::LDMXCSR},\n {OPD(X86Tables::TYPE_VEX_GROUP_15, 0, 0b011), 1, &OpDispatchBuilder::STMXCSR},\n };\n#undef OPD\n}\n\nnamespace AVX256 {\n using namespace IR;\n#define OPD(map_select, pp, opcode) (((map_select - 1) << 10) | (pp << 8) | (opcode))\n constexpr DispatchTableEntry BaseTable[] = {\n {OPD(1, 0b00, 0x10), 1, &OpDispatchBuilder::VMOVUPS_VMOVUPDOp},\n {OPD(1, 0b01, 0x10), 1, &OpDispatchBuilder::VMOVUPS_VMOVUPDOp},\n {OPD(1, 0b10, 0x10), 1, &OpDispatchBuilder::VMOVSSOp},\n {OPD(1, 0b11, 0x10), 1, &OpDispatchBuilder::VMOVSDOp},\n {OPD(1, 0b00, 0x11), 1, &OpDispatchBuilder::VMOVUPS_VMOVUPDOp},\n {OPD(1, 0b01, 0x11), 1, &OpDispatchBuilder::VMOVUPS_VMOVUPDOp},\n {OPD(1, 0b10, 0x11), 1, &OpDispatchBuilder::VMOVSSOp},\n {OPD(1, 0b11, 0x11), 1, &OpDispatchBuilder::VMOVSDOp},\n\n {OPD(1, 0b00, 0x12), 1, &OpDispatchBuilder::VMOVLPOp},\n {OPD(1, 0b01, 0x12), 1, &OpDispatchBuilder::VMOVLPOp},\n {OPD(1, 0b10, 0x12), 1, &OpDispatchBuilder::VMOVSLDUPOp},\n {OPD(1, 0b11, 0x12), 1, &OpDispatchBuilder::VMOVDDUPOp},\n {OPD(1, 0b00, 0x13), 1, &OpDispatchBuilder::VMOVLPOp},\n {OPD(1, 0b01, 0x13), 1, &OpDispatchBuilder::VMOVLPOp},\n\n {OPD(1, 0b00, 0x14), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPUNPCKLOp, OpSize::i32Bit>},\n {OPD(1, 0b01, 0x14), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPUNPCKLOp, OpSize::i64Bit>},\n\n {OPD(1, 0b00, 0x15), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPUNPCKHOp, OpSize::i32Bit>},\n {OPD(1, 0b01, 0x15), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPUNPCKHOp, OpSize::i64Bit>},\n\n {OPD(1, 0b00, 0x16), 1, &OpDispatchBuilder::VMOVHPOp},\n {OPD(1, 0b01, 0x16), 1, &OpDispatchBuilder::VMOVHPOp},\n {OPD(1, 0b10, 0x16), 1, &OpDispatchBuilder::VMOVSHDUPOp},\n {OPD(1, 0b00, 0x17), 1, &OpDispatchBuilder::VMOVHPOp},\n {OPD(1, 0b01, 0x17), 1, &OpDispatchBuilder::VMOVHPOp},\n\n {OPD(1, 0b00, 0x28), 1, &OpDispatchBuilder::VMOVAPS_VMOVAPDOp},\n {OPD(1, 0b01, 0x28), 1, &OpDispatchBuilder::VMOVAPS_VMOVAPDOp},\n {OPD(1, 0b00, 0x29), 1, &OpDispatchBuilder::VMOVAPS_VMOVAPDOp},\n {OPD(1, 0b01, 0x29), 1, &OpDispatchBuilder::VMOVAPS_VMOVAPDOp},\n\n {OPD(1, 0b10, 0x2A), 1, &OpDispatchBuilder::AVXInsertCVTGPR_To_FPR},\n {OPD(1, 0b11, 0x2A), 1, &OpDispatchBuilder::AVXInsertCVTGPR_To_FPR},\n\n {OPD(1, 0b00, 0x2B), 1, &OpDispatchBuilder::MOVVectorNTOp},\n {OPD(1, 0b01, 0x2B), 1, &OpDispatchBuilder::MOVVectorNTOp},\n\n {OPD(1, 0b10, 0x2C), 1, &OpDispatchBuilder::CVTFPR_To_GPR},\n {OPD(1, 0b11, 0x2C), 1, &OpDispatchBuilder::CVTFPR_To_GPR},\n\n {OPD(1, 0b10, 0x2D), 1, &OpDispatchBuilder::CVTFPR_To_GPR},\n {OPD(1, 0b11, 0x2D), 1, &OpDispatchBuilder::CVTFPR_To_GPR},\n\n {OPD(1, 0b00, 0x2E), 1, &OpDispatchBuilder::UCOMISxOp},\n {OPD(1, 0b01, 0x2E), 1, &OpDispatchBuilder::UCOMISxOp},\n {OPD(1, 0b00, 0x2F), 1, &OpDispatchBuilder::UCOMISxOp},\n {OPD(1, 0b01, 0x2F), 1, &OpDispatchBuilder::UCOMISxOp},\n\n {OPD(1, 0b00, 0x50), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::MOVMSKOp, OpSize::i32Bit>},\n {OPD(1, 0b01, 0x50), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::MOVMSKOp, OpSize::i64Bit>},\n\n {OPD(1, 0b00, 0x51), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorUnaryOp, IR::OP_VFSQRT, OpSize::i32Bit>},\n {OPD(1, 0b01, 0x51), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorUnaryOp, IR::OP_VFSQRT, OpSize::i64Bit>},\n {OPD(1, 0b10, 0x51), 1, &OpDispatchBuilder::AVXVectorScalarUnaryInsertALUOp},\n {OPD(1, 0b11, 0x51), 1, &OpDispatchBuilder::AVXVectorScalarUnaryInsertALUOp},\n\n {OPD(1, 0b00, 0x52), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorUnaryOp, IR::OP_VFRSQRT, OpSize::i32Bit>},\n {OPD(1, 0b10, 0x52), 1, &OpDispatchBuilder::AVXVectorScalarUnaryInsertALUOp},\n\n {OPD(1, 0b00, 0x53), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorUnaryOp, IR::OP_VFRECP, OpSize::i32Bit>},\n {OPD(1, 0b10, 0x53), 1, &OpDispatchBuilder::AVXVectorScalarUnaryInsertALUOp},\n\n {OPD(1, 0b00, 0x54), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VAND, OpSize::i128Bit>},\n {OPD(1, 0b01, 0x54), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VAND, OpSize::i128Bit>},\n\n {OPD(1, 0b00, 0x55), 1, &OpDispatchBuilder::VANDNOp},\n {OPD(1, 0b01, 0x55), 1, &OpDispatchBuilder::VANDNOp},\n\n {OPD(1, 0b00, 0x56), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VOR, OpSize::i128Bit>},\n {OPD(1, 0b01, 0x56), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VOR, OpSize::i128Bit>},\n\n {OPD(1, 0b00, 0x57), 1, &OpDispatchBuilder::AVXVectorXOROp},\n {OPD(1, 0b01, 0x57), 1, &OpDispatchBuilder::AVXVectorXOROp},\n\n {OPD(1, 0b00, 0x58), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VFADD, OpSize::i32Bit>},\n {OPD(1, 0b01, 0x58), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VFADD, OpSize::i64Bit>},\n {OPD(1, 0b10, 0x58), 1, &OpDispatchBuilder::AVXVectorScalarInsertALUOp},\n {OPD(1, 0b11, 0x58), 1, &OpDispatchBuilder::AVXVectorScalarInsertALUOp},\n\n {OPD(1, 0b00, 0x59), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VFMUL, OpSize::i32Bit>},\n {OPD(1, 0b01, 0x59), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VFMUL, OpSize::i64Bit>},\n {OPD(1, 0b10, 0x59), 1, &OpDispatchBuilder::AVXVectorScalarInsertALUOp},\n {OPD(1, 0b11, 0x59), 1, &OpDispatchBuilder::AVXVectorScalarInsertALUOp},\n\n {OPD(1, 0b00, 0x5A), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::Vector_CVT_Float_To_Float, OpSize::i64Bit, OpSize::i32Bit, true>},\n {OPD(1, 0b01, 0x5A), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::Vector_CVT_Float_To_Float, OpSize::i32Bit, OpSize::i64Bit, true>},\n {OPD(1, 0b10, 0x5A), 1, &OpDispatchBuilder::AVXInsertScalar_CVT_Float_To_Float},\n {OPD(1, 0b11, 0x5A), 1, &OpDispatchBuilder::AVXInsertScalar_CVT_Float_To_Float},\n\n {OPD(1, 0b00, 0x5B), 1, &OpDispatchBuilder::Vector_CVT_Int_To_Float},\n {OPD(1, 0b01, 0x5B), 1, &OpDispatchBuilder::Vector_CVT_Float_To_Int},\n {OPD(1, 0b10, 0x5B), 1, &OpDispatchBuilder::Vector_CVT_Float_To_Int},\n\n {OPD(1, 0b00, 0x5C), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VFSUB, OpSize::i32Bit>},\n {OPD(1, 0b01, 0x5C), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VFSUB, OpSize::i64Bit>},\n {OPD(1, 0b10, 0x5C), 1, &OpDispatchBuilder::AVXVectorScalarInsertALUOp},\n {OPD(1, 0b11, 0x5C), 1, &OpDispatchBuilder::AVXVectorScalarInsertALUOp},\n\n {OPD(1, 0b00, 0x5D), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VFMIN, OpSize::i32Bit>},\n {OPD(1, 0b01, 0x5D), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VFMIN, OpSize::i64Bit>},\n {OPD(1, 0b10, 0x5D), 1, &OpDispatchBuilder::AVXVectorScalarInsertALUOp},\n {OPD(1, 0b11, 0x5D), 1, &OpDispatchBuilder::AVXVectorScalarInsertALUOp},\n\n {OPD(1, 0b00, 0x5E), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VFDIV, OpSize::i32Bit>},\n {OPD(1, 0b01, 0x5E), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VFDIV, OpSize::i64Bit>},\n {OPD(1, 0b10, 0x5E), 1, &OpDispatchBuilder::AVXVectorScalarInsertALUOp},\n {OPD(1, 0b11, 0x5E), 1, &OpDispatchBuilder::AVXVectorScalarInsertALUOp},\n\n {OPD(1, 0b00, 0x5F), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VFMAX, OpSize::i32Bit>},\n {OPD(1, 0b01, 0x5F), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VFMAX, OpSize::i64Bit>},\n {OPD(1, 0b10, 0x5F), 1, &OpDispatchBuilder::AVXVectorScalarInsertALUOp},\n {OPD(1, 0b11, 0x5F), 1, &OpDispatchBuilder::AVXVectorScalarInsertALUOp},\n\n {OPD(1, 0b01, 0x60), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPUNPCKLOp, OpSize::i8Bit>},\n {OPD(1, 0b01, 0x61), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPUNPCKLOp, OpSize::i16Bit>},\n {OPD(1, 0b01, 0x62), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPUNPCKLOp, OpSize::i32Bit>},\n {OPD(1, 0b01, 0x63), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPACKSSOp, OpSize::i16Bit>},\n {OPD(1, 0b01, 0x64), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VCMPGT, OpSize::i8Bit>},\n {OPD(1, 0b01, 0x65), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VCMPGT, OpSize::i16Bit>},\n {OPD(1, 0b01, 0x66), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VCMPGT, OpSize::i32Bit>},\n {OPD(1, 0b01, 0x67), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPACKUSOp, OpSize::i16Bit>},\n {OPD(1, 0b01, 0x68), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPUNPCKHOp, OpSize::i8Bit>},\n {OPD(1, 0b01, 0x69), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPUNPCKHOp, OpSize::i16Bit>},\n {OPD(1, 0b01, 0x6A), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPUNPCKHOp, OpSize::i32Bit>},\n {OPD(1, 0b01, 0x6B), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPACKSSOp, OpSize::i32Bit>},\n {OPD(1, 0b01, 0x6C), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPUNPCKLOp, OpSize::i64Bit>},\n {OPD(1, 0b01, 0x6D), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPUNPCKHOp, OpSize::i64Bit>},\n {OPD(1, 0b01, 0x6E), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::MOVBetweenGPR_FPR, OpDispatchBuilder::VectorOpType::AVX>},\n\n {OPD(1, 0b01, 0x6F), 1, &OpDispatchBuilder::VMOVAPS_VMOVAPDOp},\n {OPD(1, 0b10, 0x6F), 1, &OpDispatchBuilder::VMOVUPS_VMOVUPDOp},\n\n {OPD(1, 0b01, 0x70), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPSHUFWOp, OpSize::i32Bit, true>},\n {OPD(1, 0b10, 0x70), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPSHUFWOp, OpSize::i16Bit, false>},\n {OPD(1, 0b11, 0x70), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPSHUFWOp, OpSize::i16Bit, true>},\n\n {OPD(1, 0b01, 0x74), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VCMPEQ, OpSize::i8Bit>},\n {OPD(1, 0b01, 0x75), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VCMPEQ, OpSize::i16Bit>},\n {OPD(1, 0b01, 0x76), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VCMPEQ, OpSize::i32Bit>},\n\n {OPD(1, 0b00, 0x77), 1, &OpDispatchBuilder::VZEROOp},\n\n {OPD(1, 0b01, 0x7C), 1, &OpDispatchBuilder::VHADDPOp},\n {OPD(1, 0b11, 0x7C), 1, &OpDispatchBuilder::VHADDPOp},\n {OPD(1, 0b01, 0x7D), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VHSUBPOp, OpSize::i64Bit>},\n {OPD(1, 0b11, 0x7D), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VHSUBPOp, OpSize::i32Bit>},\n\n {OPD(1, 0b01, 0x7E), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::MOVBetweenGPR_FPR, OpDispatchBuilder::VectorOpType::AVX>},\n {OPD(1, 0b10, 0x7E), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::MOVQOp, OpDispatchBuilder::VectorOpType::AVX>},\n\n {OPD(1, 0b01, 0x7F), 1, &OpDispatchBuilder::VMOVAPS_VMOVAPDOp},\n {OPD(1, 0b10, 0x7F), 1, &OpDispatchBuilder::VMOVUPS_VMOVUPDOp},\n\n {OPD(1, 0b00, 0xC2), 1, &OpDispatchBuilder::AVXVFCMPOp},\n {OPD(1, 0b01, 0xC2), 1, &OpDispatchBuilder::AVXVFCMPOp},\n {OPD(1, 0b10, 0xC2), 1, &OpDispatchBuilder::AVXInsertScalarFCMPOp},\n {OPD(1, 0b11, 0xC2), 1, &OpDispatchBuilder::AVXInsertScalarFCMPOp},\n\n {OPD(1, 0b01, 0xC4), 1, &OpDispatchBuilder::VPINSRWOp},\n {OPD(1, 0b01, 0xC5), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PExtrOp, OpSize::i16Bit>},\n\n {OPD(1, 0b00, 0xC6), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VSHUFOp, OpSize::i32Bit>},\n {OPD(1, 0b01, 0xC6), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VSHUFOp, OpSize::i64Bit>},\n\n {OPD(1, 0b01, 0xD0), 1, &OpDispatchBuilder::VADDSUBPOp},\n {OPD(1, 0b11, 0xD0), 1, &OpDispatchBuilder::VADDSUBPOp},\n\n {OPD(1, 0b01, 0xD1), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPSRLDOp, OpSize::i16Bit>},\n {OPD(1, 0b01, 0xD2), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPSRLDOp, OpSize::i32Bit>},\n {OPD(1, 0b01, 0xD3), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPSRLDOp, OpSize::i64Bit>},\n {OPD(1, 0b01, 0xD4), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VADD, OpSize::i64Bit>},\n {OPD(1, 0b01, 0xD5), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VMUL, OpSize::i16Bit>},\n {OPD(1, 0b01, 0xD6), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::MOVQOp, OpDispatchBuilder::VectorOpType::AVX>},\n {OPD(1, 0b01, 0xD7), 1, &OpDispatchBuilder::MOVMSKOpOne},\n\n {OPD(1, 0b01, 0xD8), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VUQSUB, OpSize::i8Bit>},\n {OPD(1, 0b01, 0xD9), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VUQSUB, OpSize::i16Bit>},\n {OPD(1, 0b01, 0xDA), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VUMIN, OpSize::i8Bit>},\n {OPD(1, 0b01, 0xDB), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VAND, OpSize::i128Bit>},\n {OPD(1, 0b01, 0xDC), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VUQADD, OpSize::i8Bit>},\n {OPD(1, 0b01, 0xDD), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VUQADD, OpSize::i16Bit>},\n {OPD(1, 0b01, 0xDE), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VUMAX, OpSize::i8Bit>},\n {OPD(1, 0b01, 0xDF), 1, &OpDispatchBuilder::VANDNOp},\n\n {OPD(1, 0b01, 0xE0), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VURAVG, OpSize::i8Bit>},\n {OPD(1, 0b01, 0xE1), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPSRAOp, OpSize::i16Bit>},\n {OPD(1, 0b01, 0xE2), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPSRAOp, OpSize::i32Bit>},\n {OPD(1, 0b01, 0xE3), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VURAVG, OpSize::i16Bit>},\n {OPD(1, 0b01, 0xE4), 1, &OpDispatchBuilder::VPMULHWOp},\n {OPD(1, 0b01, 0xE5), 1, &OpDispatchBuilder::VPMULHWOp},\n\n {OPD(1, 0b01, 0xE6), 1, &OpDispatchBuilder::Vector_CVT_Float_To_Int},\n {OPD(1, 0b10, 0xE6), 1, &OpDispatchBuilder::Vector_CVT_Int_To_Float},\n {OPD(1, 0b11, 0xE6), 1, &OpDispatchBuilder::Vector_CVT_Float_To_Int},\n\n {OPD(1, 0b01, 0xE7), 1, &OpDispatchBuilder::MOVVectorNTOp},\n\n {OPD(1, 0b01, 0xE8), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VSQSUB, OpSize::i8Bit>},\n {OPD(1, 0b01, 0xE9), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VSQSUB, OpSize::i16Bit>},\n {OPD(1, 0b01, 0xEA), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VSMIN, OpSize::i16Bit>},\n {OPD(1, 0b01, 0xEB), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VOR, OpSize::i128Bit>},\n {OPD(1, 0b01, 0xEC), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VSQADD, OpSize::i8Bit>},\n {OPD(1, 0b01, 0xED), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VSQADD, OpSize::i16Bit>},\n {OPD(1, 0b01, 0xEE), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VSMAX, OpSize::i16Bit>},\n {OPD(1, 0b01, 0xEF), 1, &OpDispatchBuilder::AVXVectorXOROp},\n\n {OPD(1, 0b11, 0xF0), 1, &OpDispatchBuilder::MOVVectorUnalignedOp},\n {OPD(1, 0b01, 0xF1), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPSLLOp, OpSize::i16Bit>},\n {OPD(1, 0b01, 0xF2), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPSLLOp, OpSize::i32Bit>},\n {OPD(1, 0b01, 0xF3), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPSLLOp, OpSize::i64Bit>},\n {OPD(1, 0b01, 0xF4), 1, &OpDispatchBuilder::VPMULLOp},\n {OPD(1, 0b01, 0xF5), 1, &OpDispatchBuilder::VPMADDWDOp},\n {OPD(1, 0b01, 0xF6), 1, &OpDispatchBuilder::VPSADBWOp},\n {OPD(1, 0b01, 0xF7), 1, &OpDispatchBuilder::MASKMOVOp},\n\n {OPD(1, 0b01, 0xF8), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VSUB, OpSize::i8Bit>},\n {OPD(1, 0b01, 0xF9), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VSUB, OpSize::i16Bit>},\n {OPD(1, 0b01, 0xFA), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VSUB, OpSize::i32Bit>},\n {OPD(1, 0b01, 0xFB), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VSUB, OpSize::i64Bit>},\n {OPD(1, 0b01, 0xFC), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VADD, OpSize::i8Bit>},\n {OPD(1, 0b01, 0xFD), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VADD, OpSize::i16Bit>},\n {OPD(1, 0b01, 0xFE), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VADD, OpSize::i32Bit>},\n\n {OPD(2, 0b01, 0x00), 1, &OpDispatchBuilder::VPSHUFBOp},\n {OPD(2, 0b01, 0x01), 1, &OpDispatchBuilder::VHADDPOp},\n {OPD(2, 0b01, 0x02), 1, &OpDispatchBuilder::VHADDPOp},\n {OPD(2, 0b01, 0x03), 1, &OpDispatchBuilder::VPHADDSWOp},\n {OPD(2, 0b01, 0x04), 1, &OpDispatchBuilder::VPMADDUBSWOp},\n\n {OPD(2, 0b01, 0x05), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPHSUBOp, OpSize::i16Bit>},\n {OPD(2, 0b01, 0x06), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPHSUBOp, OpSize::i32Bit>},\n {OPD(2, 0b01, 0x07), 1, &OpDispatchBuilder::VPHSUBSWOp},\n\n {OPD(2, 0b01, 0x08), 1, &OpDispatchBuilder::VPSIGN},\n {OPD(2, 0b01, 0x09), 1, &OpDispatchBuilder::VPSIGN},\n {OPD(2, 0b01, 0x0A), 1, &OpDispatchBuilder::VPSIGN},\n {OPD(2, 0b01, 0x0B), 1, &OpDispatchBuilder::VPMULHRSWOp},\n {OPD(2, 0b01, 0x0C), 1, &OpDispatchBuilder::VPERMILRegOp},\n {OPD(2, 0b01, 0x0D), 1, &OpDispatchBuilder::VPERMILRegOp},\n {OPD(2, 0b01, 0x0E), 1, &OpDispatchBuilder::VTESTPOp},\n {OPD(2, 0b01, 0x0F), 1, &OpDispatchBuilder::VTESTPOp},\n\n {OPD(2, 0b01, 0x13), 1, &OpDispatchBuilder::VCVTPH2PSOp},\n {OPD(2, 0b01, 0x16), 1, &OpDispatchBuilder::VPERMDOp},\n {OPD(2, 0b01, 0x17), 1, &OpDispatchBuilder::PTestOp},\n {OPD(2, 0b01, 0x18), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VBROADCASTOp, OpSize::i32Bit>},\n {OPD(2, 0b01, 0x19), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VBROADCASTOp, OpSize::i64Bit>},\n {OPD(2, 0b01, 0x1A), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VBROADCASTOp, OpSize::i128Bit>},\n {OPD(2, 0b01, 0x1C), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorUnaryOp, IR::OP_VABS, OpSize::i8Bit>},\n {OPD(2, 0b01, 0x1D), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorUnaryOp, IR::OP_VABS, OpSize::i16Bit>},\n {OPD(2, 0b01, 0x1E), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorUnaryOp, IR::OP_VABS, OpSize::i32Bit>},\n\n {OPD(2, 0b01, 0x20), 1, &OpDispatchBuilder::ExtendVectorElements},\n {OPD(2, 0b01, 0x21), 1, &OpDispatchBuilder::ExtendVectorElements},\n {OPD(2, 0b01, 0x22), 1, &OpDispatchBuilder::ExtendVectorElements},\n {OPD(2, 0b01, 0x23), 1, &OpDispatchBuilder::ExtendVectorElements},\n {OPD(2, 0b01, 0x24), 1, &OpDispatchBuilder::ExtendVectorElements},\n {OPD(2, 0b01, 0x25), 1, &OpDispatchBuilder::ExtendVectorElements},\n\n {OPD(2, 0b01, 0x28), 1, &OpDispatchBuilder::VPMULLOp},\n {OPD(2, 0b01, 0x29), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VCMPEQ, OpSize::i64Bit>},\n {OPD(2, 0b01, 0x2A), 1, &OpDispatchBuilder::MOVVectorNTOp},\n {OPD(2, 0b01, 0x2B), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPACKUSOp, OpSize::i32Bit>},\n {OPD(2, 0b01, 0x2C), 1, &OpDispatchBuilder::VMASKMOVOp},\n {OPD(2, 0b01, 0x2D), 1, &OpDispatchBuilder::VMASKMOVOp},\n {OPD(2, 0b01, 0x2E), 1, &OpDispatchBuilder::VMASKMOVOp},\n {OPD(2, 0b01, 0x2F), 1, &OpDispatchBuilder::VMASKMOVOp},\n\n {OPD(2, 0b01, 0x30), 1, &OpDispatchBuilder::ExtendVectorElements},\n {OPD(2, 0b01, 0x31), 1, &OpDispatchBuilder::ExtendVectorElements},\n {OPD(2, 0b01, 0x32), 1, &OpDispatchBuilder::ExtendVectorElements},\n {OPD(2, 0b01, 0x33), 1, &OpDispatchBuilder::ExtendVectorElements},\n {OPD(2, 0b01, 0x34), 1, &OpDispatchBuilder::ExtendVectorElements},\n {OPD(2, 0b01, 0x35), 1, &OpDispatchBuilder::ExtendVectorElements},\n {OPD(2, 0b01, 0x36), 1, &OpDispatchBuilder::VPERMDOp},\n\n {OPD(2, 0b01, 0x37), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VCMPGT, OpSize::i64Bit>},\n {OPD(2, 0b01, 0x38), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VSMIN, OpSize::i8Bit>},\n {OPD(2, 0b01, 0x39), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VSMIN, OpSize::i32Bit>},\n {OPD(2, 0b01, 0x3A), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VUMIN, OpSize::i16Bit>},\n {OPD(2, 0b01, 0x3B), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VUMIN, OpSize::i32Bit>},\n {OPD(2, 0b01, 0x3C), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VSMAX, OpSize::i8Bit>},\n {OPD(2, 0b01, 0x3D), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VSMAX, OpSize::i32Bit>},\n {OPD(2, 0b01, 0x3E), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VUMAX, OpSize::i16Bit>},\n {OPD(2, 0b01, 0x3F), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VUMAX, OpSize::i32Bit>},\n\n {OPD(2, 0b01, 0x40), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VMUL, OpSize::i32Bit>},\n {OPD(2, 0b01, 0x41), 1, &OpDispatchBuilder::PHMINPOSUWOp},\n {OPD(2, 0b01, 0x45), 1, &OpDispatchBuilder::VPSRLVOp},\n {OPD(2, 0b01, 0x46), 1, &OpDispatchBuilder::VPSRAVDOp},\n {OPD(2, 0b01, 0x47), 1, &OpDispatchBuilder::VPSLLVOp},\n\n {OPD(2, 0b01, 0x58), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VBROADCASTOp, OpSize::i32Bit>},\n {OPD(2, 0b01, 0x59), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VBROADCASTOp, OpSize::i64Bit>},\n {OPD(2, 0b01, 0x5A), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VBROADCASTOp, OpSize::i128Bit>},\n\n {OPD(2, 0b01, 0x78), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VBROADCASTOp, OpSize::i8Bit>},\n {OPD(2, 0b01, 0x79), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VBROADCASTOp, OpSize::i16Bit>},\n\n {OPD(2, 0b01, 0x8C), 1, &OpDispatchBuilder::VPMASKMOVOp},\n {OPD(2, 0b01, 0x8E), 1, &OpDispatchBuilder::VPMASKMOVOp},\n\n {OPD(2, 0b01, 0x90), 1, &OpDispatchBuilder::VPGATHER},\n {OPD(2, 0b01, 0x91), 1, &OpDispatchBuilder::VPGATHER},\n {OPD(2, 0b01, 0x92), 1, &OpDispatchBuilder::VPGATHER},\n {OPD(2, 0b01, 0x93), 1, &OpDispatchBuilder::VPGATHER},\n\n {OPD(2, 0b01, 0x96), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAddSubImpl, true, 1, 3, 2>}, // VFMADDSUB\n {OPD(2, 0b01, 0x97), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAddSubImpl, false, 1, 3, 2>}, // VFMSUBADD\n\n {OPD(2, 0b01, 0x98), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAImpl, IR::OP_VFMLA, false, 1, 3, 2>}, // VFMADD\n {OPD(2, 0b01, 0x99), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAImpl, IR::OP_VFMLA, true, 1, 3, 2>}, // VFMADD\n {OPD(2, 0b01, 0x9A), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAImpl, IR::OP_VFMLS, false, 1, 3, 2>}, // VFMSUB\n {OPD(2, 0b01, 0x9B), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAImpl, IR::OP_VFMLS, true, 1, 3, 2>}, // VFMSUB\n {OPD(2, 0b01, 0x9C), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAImpl, IR::OP_VFNMLA, false, 1, 3, 2>}, // VFNMADD\n {OPD(2, 0b01, 0x9D), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAImpl, IR::OP_VFNMLA, true, 1, 3, 2>}, // VFNMADD\n {OPD(2, 0b01, 0x9E), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAImpl, IR::OP_VFNMLS, false, 1, 3, 2>}, // VFNMSUB\n {OPD(2, 0b01, 0x9F), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAImpl, IR::OP_VFNMLS, true, 1, 3, 2>}, // VFNMSUB\n\n {OPD(2, 0b01, 0xA8), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAImpl, IR::OP_VFMLA, false, 2, 1, 3>}, // VFMADD\n {OPD(2, 0b01, 0xA9), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAImpl, IR::OP_VFMLA, true, 2, 1, 3>}, // VFMADD\n {OPD(2, 0b01, 0xAA), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAImpl, IR::OP_VFMLS, false, 2, 1, 3>}, // VFMSUB\n {OPD(2, 0b01, 0xAB), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAImpl, IR::OP_VFMLS, true, 2, 1, 3>}, // VFMSUB\n {OPD(2, 0b01, 0xAC), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAImpl, IR::OP_VFNMLA, false, 2, 1, 3>}, // VFNMADD\n {OPD(2, 0b01, 0xAD), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAImpl, IR::OP_VFNMLA, true, 2, 1, 3>}, // VFNMADD\n {OPD(2, 0b01, 0xAE), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAImpl, IR::OP_VFNMLS, false, 2, 1, 3>}, // VFNMSUB\n {OPD(2, 0b01, 0xAF), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAImpl, IR::OP_VFNMLS, true, 2, 1, 3>}, // VFNMSUB\n\n {OPD(2, 0b01, 0xB8), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAImpl, IR::OP_VFMLA, false, 2, 3, 1>}, // VFMADD\n {OPD(2, 0b01, 0xB9), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAImpl, IR::OP_VFMLA, true, 2, 3, 1>}, // VFMADD\n {OPD(2, 0b01, 0xBA), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAImpl, IR::OP_VFMLS, false, 2, 3, 1>}, // VFMSUB\n {OPD(2, 0b01, 0xBB), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAImpl, IR::OP_VFMLS, true, 2, 3, 1>}, // VFMSUB\n {OPD(2, 0b01, 0xBC), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAImpl, IR::OP_VFNMLA, false, 2, 3, 1>}, // VFNMADD\n {OPD(2, 0b01, 0xBD), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAImpl, IR::OP_VFNMLA, true, 2, 3, 1>}, // VFNMADD\n {OPD(2, 0b01, 0xBE), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAImpl, IR::OP_VFNMLS, false, 2, 3, 1>}, // VFNMSUB\n {OPD(2, 0b01, 0xBF), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAImpl, IR::OP_VFNMLS, true, 2, 3, 1>}, // VFNMSUB\n\n {OPD(2, 0b01, 0xA6), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAddSubImpl, true, 2, 1, 3>}, // VFMADDSUB\n {OPD(2, 0b01, 0xA7), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAddSubImpl, false, 2, 1, 3>}, // VFMSUBADD\n\n {OPD(2, 0b01, 0xB6), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAddSubImpl, true, 2, 3, 1>}, // VFMADDSUB\n {OPD(2, 0b01, 0xB7), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAddSubImpl, false, 2, 3, 1>}, // VFMSUBADD\n\n {OPD(2, 0b01, 0xDB), 1, &OpDispatchBuilder::AESImcOp},\n {OPD(2, 0b01, 0xDC), 1, &OpDispatchBuilder::VAESEncOp},\n {OPD(2, 0b01, 0xDD), 1, &OpDispatchBuilder::VAESEncLastOp},\n {OPD(2, 0b01, 0xDE), 1, &OpDispatchBuilder::VAESDecOp},\n {OPD(2, 0b01, 0xDF), 1, &OpDispatchBuilder::VAESDecLastOp},\n\n {OPD(3, 0b01, 0x00), 1, &OpDispatchBuilder::VPERMQOp},\n {OPD(3, 0b01, 0x01), 1, &OpDispatchBuilder::VPERMQOp},\n {OPD(3, 0b01, 0x02), 1, &OpDispatchBuilder::VPBLENDDOp},\n {OPD(3, 0b01, 0x04), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPERMILImmOp, OpSize::i32Bit>},\n {OPD(3, 0b01, 0x05), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPERMILImmOp, OpSize::i64Bit>},\n {OPD(3, 0b01, 0x06), 1, &OpDispatchBuilder::VPERM2Op},\n {OPD(3, 0b01, 0x08), 1, &OpDispatchBuilder::AVXVectorRound},\n {OPD(3, 0b01, 0x09), 1, &OpDispatchBuilder::AVXVectorRound},\n {OPD(3, 0b01, 0x0A), 1, &OpDispatchBuilder::AVXInsertScalarRound},\n {OPD(3, 0b01, 0x0B), 1, &OpDispatchBuilder::AVXInsertScalarRound},\n {OPD(3, 0b01, 0x0C), 1, &OpDispatchBuilder::VPBLENDDOp},\n {OPD(3, 0b01, 0x0D), 1, &OpDispatchBuilder::VBLENDPDOp},\n {OPD(3, 0b01, 0x0E), 1, &OpDispatchBuilder::VPBLENDWOp},\n {OPD(3, 0b01, 0x0F), 1, &OpDispatchBuilder::VPALIGNROp},\n\n {OPD(3, 0b01, 0x14), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PExtrOp, OpSize::i8Bit>},\n {OPD(3, 0b01, 0x15), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PExtrOp, OpSize::i16Bit>},\n {OPD(3, 0b01, 0x16), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PExtrOp, OpSize::i32Bit>},\n {OPD(3, 0b01, 0x17), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PExtrOp, OpSize::i32Bit>},\n\n {OPD(3, 0b01, 0x18), 1, &OpDispatchBuilder::VINSERTOp},\n {OPD(3, 0b01, 0x19), 1, &OpDispatchBuilder::VEXTRACT128Op},\n {OPD(3, 0b01, 0x1D), 1, &OpDispatchBuilder::VCVTPS2PHOp},\n {OPD(3, 0b01, 0x20), 1, &OpDispatchBuilder::VPINSRBOp},\n {OPD(3, 0b01, 0x21), 1, &OpDispatchBuilder::VINSERTPSOp},\n {OPD(3, 0b01, 0x22), 1, &OpDispatchBuilder::VPINSRDQOp},\n\n {OPD(3, 0b01, 0x38), 1, &OpDispatchBuilder::VINSERTOp},\n {OPD(3, 0b01, 0x39), 1, &OpDispatchBuilder::VEXTRACT128Op},\n\n {OPD(3, 0b01, 0x40), 1, &OpDispatchBuilder::VDPPOp},\n {OPD(3, 0b01, 0x41), 1, &OpDispatchBuilder::VDPPOp},\n {OPD(3, 0b01, 0x42), 1, &OpDispatchBuilder::VMPSADBWOp},\n {OPD(3, 0b01, 0x44), 1, &OpDispatchBuilder::VPCLMULQDQOp},\n\n {OPD(3, 0b01, 0x46), 1, &OpDispatchBuilder::VPERM2Op},\n\n {OPD(3, 0b01, 0x4A), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorVariableBlend, OpSize::i32Bit>},\n {OPD(3, 0b01, 0x4B), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorVariableBlend, OpSize::i64Bit>},\n {OPD(3, 0b01, 0x4C), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorVariableBlend, OpSize::i8Bit>},\n\n {OPD(3, 0b01, 0x60), 1, &OpDispatchBuilder::VPCMPESTRMOp},\n {OPD(3, 0b01, 0x61), 1, &OpDispatchBuilder::VPCMPESTRIOp},\n {OPD(3, 0b01, 0x62), 1, &OpDispatchBuilder::VPCMPISTRMOp},\n {OPD(3, 0b01, 0x63), 1, &OpDispatchBuilder::VPCMPISTRIOp},\n\n {OPD(3, 0b01, 0xDF), 1, &OpDispatchBuilder::AESKeyGenAssist},\n };\n#undef OPD\n\n#define OPD(group, pp, opcode) (((group - X86Tables::TYPE_VEX_GROUP_12) << 4) | (pp << 3) | (opcode))\n constexpr DispatchTableEntry TableGroupOps[] {\n {OPD(X86Tables::TYPE_VEX_GROUP_12, 1, 0b010), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPSRLIOp, OpSize::i16Bit>},\n {OPD(X86Tables::TYPE_VEX_GROUP_12, 1, 0b110), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPSLLIOp, OpSize::i16Bit>},\n {OPD(X86Tables::TYPE_VEX_GROUP_12, 1, 0b100), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPSRAIOp, OpSize::i16Bit>},\n\n {OPD(X86Tables::TYPE_VEX_GROUP_13, 1, 0b010), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPSRLIOp, OpSize::i32Bit>},\n {OPD(X86Tables::TYPE_VEX_GROUP_13, 1, 0b110), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPSLLIOp, OpSize::i32Bit>},\n {OPD(X86Tables::TYPE_VEX_GROUP_13, 1, 0b100), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPSRAIOp, OpSize::i32Bit>},\n\n {OPD(X86Tables::TYPE_VEX_GROUP_14, 1, 0b010), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPSRLIOp, OpSize::i64Bit>},\n {OPD(X86Tables::TYPE_VEX_GROUP_14, 1, 0b011), 1, &OpDispatchBuilder::VPSRLDQOp},\n {OPD(X86Tables::TYPE_VEX_GROUP_14, 1, 0b110), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPSLLIOp, OpSize::i64Bit>},\n {OPD(X86Tables::TYPE_VEX_GROUP_14, 1, 0b111), 1, &OpDispatchBuilder::VPSLLDQOp},\n\n {OPD(X86Tables::TYPE_VEX_GROUP_15, 0, 0b010), 1, &OpDispatchBuilder::LDMXCSR},\n {OPD(X86Tables::TYPE_VEX_GROUP_15, 0, 0b011), 1, &OpDispatchBuilder::STMXCSR},\n };\n#undef OPD\n}\n\nauto BaseTableLambda = [](const auto RuntimeTable) consteval {\n std::array Table{};\n#define OPD(map_select, pp, opcode) (((map_select - 1) << 10) | (pp << 8) | (opcode))\n constexpr U16U8InfoStruct VEXTable[] = {\n // Map 0 (Reserved)\n // VEX Map 1\n {OPD(1, 0b00, 0x10), 1, X86InstInfo{\"VMOVUPS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b01, 0x10), 1, X86InstInfo{\"VMOVUPD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b10, 0x10), 1, X86InstInfo{\"VMOVSS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_VEX_L_IGNORE, 0}},\n {OPD(1, 0b11, 0x10), 1, X86InstInfo{\"VMOVSD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_VEX_L_IGNORE, 0}},\n\n {OPD(1, 0b00, 0x11), 1, X86InstInfo{\"VMOVUPS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b01, 0x11), 1, X86InstInfo{\"VMOVUPD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b10, 0x11), 1, X86InstInfo{\"VMOVSS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_VEX_L_IGNORE, 0}},\n {OPD(1, 0b11, 0x11), 1, X86InstInfo{\"VMOVSD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_VEX_L_IGNORE, 0}},\n\n {OPD(1, 0b00, 0x12), 1, X86InstInfo{\"VMOVLPS\",TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_VEX_1ST_SRC | FLAGS_VEX_L_0, 0}},\n {OPD(1, 0b01, 0x12), 1, X86InstInfo{\"VMOVLPD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MOD_MEM_ONLY | FLAGS_VEX_1ST_SRC | FLAGS_VEX_L_0, 0}},\n {OPD(1, 0b10, 0x12), 1, X86InstInfo{\"VMOVSLDUP\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b11, 0x12), 1, X86InstInfo{\"VMOVDDUP\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n\n {OPD(1, 0b00, 0x13), 1, X86InstInfo{\"VMOVLPS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY | FLAGS_SF_MOD_DST | FLAGS_XMM_FLAGS | FLAGS_VEX_L_0, 0}},\n {OPD(1, 0b01, 0x13), 1, X86InstInfo{\"VMOVLPD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY | FLAGS_SF_MOD_DST | FLAGS_XMM_FLAGS | FLAGS_VEX_L_0, 0}},\n\n {OPD(1, 0b00, 0x14), 1, X86InstInfo{\"VUNPCKLPS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b01, 0x14), 1, X86InstInfo{\"VUNPCKLPD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n\n {OPD(1, 0b00, 0x15), 1, X86InstInfo{\"VUNPCKHPS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b01, 0x15), 1, X86InstInfo{\"VUNPCKHPD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n\n {OPD(1, 0b00, 0x16), 1, X86InstInfo{\"VMOV(L)HPS\",TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_VEX_1ST_SRC | FLAGS_VEX_L_0, 0}},\n {OPD(1, 0b01, 0x16), 1, X86InstInfo{\"VMOVHPD\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY | FLAGS_XMM_FLAGS | FLAGS_VEX_1ST_SRC | FLAGS_VEX_L_0, 0}},\n {OPD(1, 0b10, 0x16), 1, X86InstInfo{\"VMOVSHDUP\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n\n {OPD(1, 0b00, 0x17), 1, X86InstInfo{\"VMOVHPS\", TYPE_INST, GenFlagsSizes(SIZE_64BIT, SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY | FLAGS_SF_MOD_DST | FLAGS_XMM_FLAGS | FLAGS_VEX_L_0, 0}},\n {OPD(1, 0b01, 0x17), 1, X86InstInfo{\"VMOVHPD\", TYPE_INST, GenFlagsSizes(SIZE_64BIT, SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY | FLAGS_SF_MOD_DST | FLAGS_XMM_FLAGS | FLAGS_VEX_L_0, 0}},\n\n {OPD(1, 0b00, 0x50), 1, X86InstInfo{\"VMOVMSKPS\", TYPE_INST, GenFlagsSizes(SIZE_32BIT, SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_REG_ONLY | FLAGS_XMM_FLAGS | FLAGS_SF_DST_GPR, 0}},\n {OPD(1, 0b01, 0x50), 1, X86InstInfo{\"VMOVMSKPD\", TYPE_INST, GenFlagsSizes(SIZE_32BIT, SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_REG_ONLY | FLAGS_XMM_FLAGS | FLAGS_SF_DST_GPR, 0}},\n\n {OPD(1, 0b00, 0x51), 1, X86InstInfo{\"VSQRTPS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b01, 0x51), 1, X86InstInfo{\"VSQRTPD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b10, 0x51), 1, X86InstInfo{\"VSQRTSS\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_VEX_L_IGNORE, 0}},\n {OPD(1, 0b11, 0x51), 1, X86InstInfo{\"VSQRTSD\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_VEX_L_IGNORE, 0}},\n\n {OPD(1, 0b00, 0x52), 1, X86InstInfo{\"VRSQRTPS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b10, 0x52), 1, X86InstInfo{\"VRSQRTSS\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_VEX_L_IGNORE, 0}},\n\n {OPD(1, 0b00, 0x53), 1, X86InstInfo{\"VRCPPS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b10, 0x53), 1, X86InstInfo{\"VRCPSS\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_VEX_L_IGNORE, 0}},\n\n {OPD(1, 0b00, 0x54), 1, X86InstInfo{\"VANDPS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b01, 0x54), 1, X86InstInfo{\"VANDPD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n\n {OPD(1, 0b00, 0x55), 1, X86InstInfo{\"VANDNPS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b01, 0x55), 1, X86InstInfo{\"VANDNPD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n\n {OPD(1, 0b00, 0x56), 1, X86InstInfo{\"VORPS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b01, 0x56), 1, X86InstInfo{\"VORPD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n\n {OPD(1, 0b00, 0x57), 1, X86InstInfo{\"VXORPS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b01, 0x57), 1, X86InstInfo{\"VXORPD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n\n {OPD(1, 0b01, 0x60), 1, X86InstInfo{\"VPUNPCKLBW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b01, 0x61), 1, X86InstInfo{\"VPUNPCKLWD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b01, 0x62), 1, X86InstInfo{\"VPUNPCKLDQ\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b01, 0x63), 1, X86InstInfo{\"VPACKSSWB\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b01, 0x64), 1, X86InstInfo{\"VPCMPGTB\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b01, 0x65), 1, X86InstInfo{\"VPCMPGTW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b01, 0x66), 1, X86InstInfo{\"VPCMPGTD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b01, 0x67), 1, X86InstInfo{\"VPACKUSWB\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n\n {OPD(1, 0b01, 0x70), 1, X86InstInfo{\"VPSHUFD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 1}},\n {OPD(1, 0b10, 0x70), 1, X86InstInfo{\"VPSHUFHW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 1}},\n {OPD(1, 0b11, 0x70), 1, X86InstInfo{\"VPSHUFLW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 1}},\n\n {OPD(1, 0b01, 0x71), 1, X86InstInfo{\"\", TYPE_VEX_GROUP_12, FLAGS_NONE, 0}}, // VEX Group 12\n {OPD(1, 0b01, 0x72), 1, X86InstInfo{\"\", TYPE_VEX_GROUP_13, FLAGS_NONE, 0}}, // VEX Group 13\n {OPD(1, 0b01, 0x73), 1, X86InstInfo{\"\", TYPE_VEX_GROUP_14, FLAGS_NONE, 0}}, // VEX Group 14\n\n {OPD(1, 0b01, 0x74), 1, X86InstInfo{\"VPCMPEQB\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b01, 0x75), 1, X86InstInfo{\"VPCMPEQW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b01, 0x76), 1, X86InstInfo{\"VPCMPEQD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n\n {OPD(1, 0b00, 0x77), 1, X86InstInfo{\"VZERO*\", TYPE_INST, GenFlagsDstSize(SIZE_128BIT), 0}},\n\n {OPD(1, 0b00, 0xC2), 1, X86InstInfo{\"VCMPccPS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 1}},\n {OPD(1, 0b01, 0xC2), 1, X86InstInfo{\"VCMPccPD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 1}},\n {OPD(1, 0b10, 0xC2), 1, X86InstInfo{\"VCMPccSS\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_L_IGNORE | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 1}},\n {OPD(1, 0b11, 0xC2), 1, X86InstInfo{\"VCMPccSD\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_VEX_L_IGNORE | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 1}},\n\n {OPD(1, 0b01, 0xC4), 1, X86InstInfo{\"VPINSRW\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_16BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_SF_SRC_GPR | FLAGS_XMM_FLAGS | FLAGS_VEX_L_0, 1}},\n {OPD(1, 0b01, 0xC5), 1, X86InstInfo{\"VPEXTRW\", TYPE_INST, GenFlagsSizes(SIZE_32BIT, SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_REG_ONLY | FLAGS_SF_DST_GPR | FLAGS_XMM_FLAGS | FLAGS_VEX_L_0, 1}},\n\n {OPD(1, 0b00, 0xC6), 1, X86InstInfo{\"VSHUFPS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 1}},\n {OPD(1, 0b01, 0xC6), 1, X86InstInfo{\"VSHUFPD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 1}},\n\n // The above ops are defined from `Table A-17. VEX Opcode Map 1, Low Nibble = [0h:7h]` of AMD Architecture programmer's manual Volume 3\n // This table doesn't state which VEX.pp is for which instruction\n // XXX: Confirm all the above encoding opcodes\n\n {OPD(1, 0b00, 0x28), 1, X86InstInfo{\"VMOVAPS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b01, 0x28), 1, X86InstInfo{\"VMOVAPD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n\n {OPD(1, 0b00, 0x29), 1, X86InstInfo{\"VMOVAPS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b01, 0x29), 1, X86InstInfo{\"VMOVAPD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_XMM_FLAGS, 0}},\n\n {OPD(1, 0b10, 0x2A), 1, X86InstInfo{\"VCVTSI2SS\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_SF_SRC_GPR | FLAGS_VEX_L_IGNORE, 0}},\n {OPD(1, 0b11, 0x2A), 1, X86InstInfo{\"VCVTSI2SD\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_SF_SRC_GPR | FLAGS_VEX_L_IGNORE, 0}},\n\n {OPD(1, 0b00, 0x2B), 1, X86InstInfo{\"VMOVNTPS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY | FLAGS_SF_MOD_DST | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b01, 0x2B), 1, X86InstInfo{\"VMOVNTPD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY | FLAGS_SF_MOD_DST | FLAGS_XMM_FLAGS, 0}},\n\n {OPD(1, 0b10, 0x2C), 1, X86InstInfo{\"VCVTTSS2SI\", TYPE_INST, FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_DST_GPR | FLAGS_VEX_L_IGNORE, 0}},\n {OPD(1, 0b11, 0x2C), 1, X86InstInfo{\"VCVTTSD2SI\", TYPE_INST, FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_DST_GPR | FLAGS_VEX_L_IGNORE, 0}},\n\n {OPD(1, 0b10, 0x2D), 1, X86InstInfo{\"VCVTSS2SI\", TYPE_INST, FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_DST_GPR | FLAGS_VEX_L_IGNORE, 0}},\n {OPD(1, 0b11, 0x2D), 1, X86InstInfo{\"VCVTSD2SI\", TYPE_INST, FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_DST_GPR | FLAGS_VEX_L_IGNORE, 0}},\n\n {OPD(1, 0b00, 0x2E), 1, X86InstInfo{\"VUCOMISS\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_VEX_L_IGNORE, 0}},\n {OPD(1, 0b01, 0x2E), 1, X86InstInfo{\"VUCOMISD\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_VEX_L_IGNORE, 0}},\n\n {OPD(1, 0b00, 0x2F), 1, X86InstInfo{\"VCOMISS\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_VEX_L_IGNORE, 0}},\n {OPD(1, 0b01, 0x2F), 1, X86InstInfo{\"VCOMISD\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_VEX_L_IGNORE, 0}},\n\n {OPD(1, 0b00, 0x58), 1, X86InstInfo{\"VADDPS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b01, 0x58), 1, X86InstInfo{\"VADDPD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b10, 0x58), 1, X86InstInfo{\"VADDSS\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_VEX_L_IGNORE, 0}},\n {OPD(1, 0b11, 0x58), 1, X86InstInfo{\"VADDSD\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_VEX_L_IGNORE, 0}},\n\n {OPD(1, 0b00, 0x59), 1, X86InstInfo{\"VMULPS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b01, 0x59), 1, X86InstInfo{\"VMULPD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b10, 0x59), 1, X86InstInfo{\"VMULSS\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_VEX_L_IGNORE, 0}},\n {OPD(1, 0b11, 0x59), 1, X86InstInfo{\"VMULSD\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_VEX_L_IGNORE, 0}},\n\n {OPD(1, 0b00, 0x5A), 1, X86InstInfo{\"VCVTPS2PD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b01, 0x5A), 1, X86InstInfo{\"VCVTPD2PS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b10, 0x5A), 1, X86InstInfo{\"VCVTSS2SD\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_L_IGNORE | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b11, 0x5A), 1, X86InstInfo{\"VCVTSD2SS\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_VEX_L_IGNORE |FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n\n {OPD(1, 0b00, 0x5B), 1, X86InstInfo{\"VCVTDQ2PS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b01, 0x5B), 1, X86InstInfo{\"VCVTPS2DQ\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b10, 0x5B), 1, X86InstInfo{\"VCVTTPS2DQ\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n\n {OPD(1, 0b00, 0x5C), 1, X86InstInfo{\"VSUBPS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b01, 0x5C), 1, X86InstInfo{\"VSUBPD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b10, 0x5C), 1, X86InstInfo{\"VSUBSS\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_VEX_L_IGNORE, 0}},\n {OPD(1, 0b11, 0x5C), 1, X86InstInfo{\"VSUBSD\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_VEX_L_IGNORE, 0}},\n\n {OPD(1, 0b00, 0x5D), 1, X86InstInfo{\"VMINPS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b01, 0x5D), 1, X86InstInfo{\"VMINPD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b10, 0x5D), 1, X86InstInfo{\"VMINSS\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_VEX_L_IGNORE, 0}},\n {OPD(1, 0b11, 0x5D), 1, X86InstInfo{\"VMINSD\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_VEX_L_IGNORE, 0}},\n\n {OPD(1, 0b00, 0x5E), 1, X86InstInfo{\"VDIVPS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b01, 0x5E), 1, X86InstInfo{\"VDIVPD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b10, 0x5E), 1, X86InstInfo{\"VDIVSS\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_VEX_L_IGNORE, 0}},\n {OPD(1, 0b11, 0x5E), 1, X86InstInfo{\"VDIVSD\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_VEX_L_IGNORE, 0}},\n\n {OPD(1, 0b00, 0x5F), 1, X86InstInfo{\"VMAXPS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b01, 0x5F), 1, X86InstInfo{\"VMAXPD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b10, 0x5F), 1, X86InstInfo{\"VMAXSS\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_VEX_L_IGNORE, 0}},\n {OPD(1, 0b11, 0x5F), 1, X86InstInfo{\"VMAXSD\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_VEX_L_IGNORE, 0}},\n\n {OPD(1, 0b01, 0x68), 1, X86InstInfo{\"VPUNPCKHBW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b01, 0x69), 1, X86InstInfo{\"VPUNPCKHWD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b01, 0x6A), 1, X86InstInfo{\"VPUNPCKHDQ\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b01, 0x6B), 1, X86InstInfo{\"VPACKSSDW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b01, 0x6C), 1, X86InstInfo{\"VPUNPCKLQDQ\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b01, 0x6D), 1, X86InstInfo{\"VPUNPCKHQDQ\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b01, 0x6E), 1, X86InstInfo{\"VMOV*\", TYPE_INST, GenFlagsDstSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_VEX_L_0 | FLAGS_SF_SRC_GPR, 0}},\n\n {OPD(1, 0b01, 0x6F), 1, X86InstInfo{\"VMOVDQA\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b10, 0x6F), 1, X86InstInfo{\"VMOVDQU\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n\n {OPD(1, 0b01, 0x7C), 1, X86InstInfo{\"VHADDPD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b11, 0x7C), 1, X86InstInfo{\"VHADDPS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n\n {OPD(1, 0b01, 0x7D), 1, X86InstInfo{\"VHSUBPD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b11, 0x7D), 1, X86InstInfo{\"VHSUBPS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n\n {OPD(1, 0b01, 0x7E), 1, X86InstInfo{\"VMOV*\", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_DST_GPR | FLAGS_VEX_L_0 | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b10, 0x7E), 1, X86InstInfo{\"VMOVQ\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_VEX_L_0 | FLAGS_XMM_FLAGS, 0}},\n\n {OPD(1, 0b01, 0x7F), 1, X86InstInfo{\"VMOVDQA\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b10, 0x7F), 1, X86InstInfo{\"VMOVDQU\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_XMM_FLAGS, 0}},\n\n {OPD(1, 0b00, 0xAE), 1, X86InstInfo{\"\", TYPE_VEX_GROUP_15, FLAGS_NONE, 0}}, // VEX Group 15\n {OPD(1, 0b01, 0xAE), 1, X86InstInfo{\"\", TYPE_VEX_GROUP_15, FLAGS_NONE, 0}}, // VEX Group 15\n {OPD(1, 0b10, 0xAE), 1, X86InstInfo{\"\", TYPE_VEX_GROUP_15, FLAGS_NONE, 0}}, // VEX Group 15\n {OPD(1, 0b11, 0xAE), 1, X86InstInfo{\"\", TYPE_VEX_GROUP_15, FLAGS_NONE, 0}}, // VEX Group 15\n\n {OPD(1, 0b01, 0xD0), 1, X86InstInfo{\"VADDSUBPD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b11, 0xD0), 1, X86InstInfo{\"VADDSUBPS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n\n {OPD(1, 0b01, 0xD1), 1, X86InstInfo{\"VPSRLW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b01, 0xD2), 1, X86InstInfo{\"VPSRLD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b01, 0xD3), 1, X86InstInfo{\"VPSRLQ\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b01, 0xD4), 1, X86InstInfo{\"VPADDQ\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b01, 0xD5), 1, X86InstInfo{\"VPMULLW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b01, 0xD6), 1, X86InstInfo{\"VMOVQ\", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_VEX_L_0 | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b01, 0xD7), 1, X86InstInfo{\"VPMOVMSKB\", TYPE_INST, GenFlagsSizes(SIZE_32BIT, SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_DST_GPR | FLAGS_SF_MOD_REG_ONLY, 0}},\n\n {OPD(1, 0b01, 0xD8), 1, X86InstInfo{\"VPSUBUSB\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b01, 0xD9), 1, X86InstInfo{\"VPSUBUSW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b01, 0xDA), 1, X86InstInfo{\"VPMINUB\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b01, 0xDB), 1, X86InstInfo{\"VPAND\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b01, 0xDC), 1, X86InstInfo{\"VPADDUSB\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b01, 0xDD), 1, X86InstInfo{\"VPADDUSW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b01, 0xDE), 1, X86InstInfo{\"VPMAXUB\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b01, 0xDF), 1, X86InstInfo{\"VPANDN\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n\n {OPD(1, 0b01, 0xE0), 1, X86InstInfo{\"VPAVGB\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b01, 0xE1), 1, X86InstInfo{\"VPSRAW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b01, 0xE2), 1, X86InstInfo{\"VPSRAD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b01, 0xE3), 1, X86InstInfo{\"VPAVGW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b01, 0xE4), 1, X86InstInfo{\"VPMULHUW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b01, 0xE5), 1, X86InstInfo{\"VPMULHW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n\n {OPD(1, 0b01, 0xE6), 1, X86InstInfo{\"VCVTTPD2DQ\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b10, 0xE6), 1, X86InstInfo{\"VCVTDQ2PD\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b11, 0xE6), 1, X86InstInfo{\"VCVTPD2DQ\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n\n {OPD(1, 0b01, 0xE7), 1, X86InstInfo{\"VMOVNTDQ\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_MOD_MEM_ONLY | FLAGS_XMM_FLAGS, 0}},\n\n {OPD(1, 0b01, 0xE8), 1, X86InstInfo{\"VPSUBSB\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b01, 0xE9), 1, X86InstInfo{\"VPSUBSW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b01, 0xEA), 1, X86InstInfo{\"VPMINSW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b01, 0xEB), 1, X86InstInfo{\"VPOR\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b01, 0xEC), 1, X86InstInfo{\"VPADDSB\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b01, 0xED), 1, X86InstInfo{\"VPADDSW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b01, 0xEE), 1, X86InstInfo{\"VPMAXSW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b01, 0xEF), 1, X86InstInfo{\"VPXOR\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n\n {OPD(1, 0b11, 0xF0), 1, X86InstInfo{\"VLDDQU\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY | FLAGS_XMM_FLAGS, 0}},\n\n {OPD(1, 0b01, 0xF1), 1, X86InstInfo{\"VPSLLW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b01, 0xF2), 1, X86InstInfo{\"VPSLLD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b01, 0xF3), 1, X86InstInfo{\"VPSLLQ\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b01, 0xF4), 1, X86InstInfo{\"VPMULUDQ\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b01, 0xF5), 1, X86InstInfo{\"VPMADDWD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b01, 0xF6), 1, X86InstInfo{\"VPSADBW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b01, 0xF7), 1, X86InstInfo{\"VMASKMOVDQU\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_REG_ONLY | FLAGS_XMM_FLAGS | FLAGS_VEX_L_0, 0}},\n\n {OPD(1, 0b01, 0xF8), 1, X86InstInfo{\"VPSUBB\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b01, 0xF9), 1, X86InstInfo{\"VPSUBW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b01, 0xFA), 1, X86InstInfo{\"VPSUBD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b01, 0xFB), 1, X86InstInfo{\"VPSUBQ\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b01, 0xFC), 1, X86InstInfo{\"VPADDB\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b01, 0xFD), 1, X86InstInfo{\"VPADDW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(1, 0b01, 0xFE), 1, X86InstInfo{\"VPADDD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n\n // VEX Map 2\n {OPD(2, 0b01, 0x00), 1, X86InstInfo{\"VPSHUFB\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(2, 0b01, 0x01), 1, X86InstInfo{\"VPHADDW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(2, 0b01, 0x02), 1, X86InstInfo{\"VPHADDD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(2, 0b01, 0x03), 1, X86InstInfo{\"VPHADDSW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(2, 0b01, 0x04), 1, X86InstInfo{\"VPMADDUBSW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(2, 0b01, 0x05), 1, X86InstInfo{\"VPHSUBW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(2, 0b01, 0x06), 1, X86InstInfo{\"VPHSUBD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(2, 0b01, 0x07), 1, X86InstInfo{\"VPHSUBSW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n\n {OPD(2, 0b01, 0x08), 1, X86InstInfo{\"VPSIGNB\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(2, 0b01, 0x09), 1, X86InstInfo{\"VPSIGNW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(2, 0b01, 0x0A), 1, X86InstInfo{\"VPSIGND\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(2, 0b01, 0x0B), 1, X86InstInfo{\"VPMULHRSW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(2, 0b01, 0x0C), 1, X86InstInfo{\"VPERMILPS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_REX_W_0 | FLAGS_XMM_FLAGS, 0}},\n {OPD(2, 0b01, 0x0D), 1, X86InstInfo{\"VPERMILPD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_REX_W_0 | FLAGS_XMM_FLAGS, 0}},\n {OPD(2, 0b01, 0x0E), 1, X86InstInfo{\"VTESTPS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_REX_W_0 | FLAGS_XMM_FLAGS, 0}},\n {OPD(2, 0b01, 0x0F), 1, X86InstInfo{\"VTESTPD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_REX_W_0 | FLAGS_XMM_FLAGS, 0}},\n\n {OPD(2, 0b01, 0x13), 1, X86InstInfo{\"VCVTPH2PS\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_REX_W_0 | FLAGS_XMM_FLAGS, 0}},\n {OPD(2, 0b01, 0x16), 1, X86InstInfo{\"VPERMPS\", TYPE_INST, GenFlagsSameSize(SIZE_256BIT) | FLAGS_MODRM | FLAGS_VEX_L_1 | FLAGS_VEX_1ST_SRC | FLAGS_REX_W_0 | FLAGS_XMM_FLAGS, 0}},\n {OPD(2, 0b01, 0x17), 1, X86InstInfo{\"VPTEST\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n\n {OPD(2, 0b01, 0x18), 1, X86InstInfo{\"VBROADCASTSS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_REX_W_0 | FLAGS_XMM_FLAGS, 0}},\n {OPD(2, 0b01, 0x19), 1, X86InstInfo{\"VBROADCASTSD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_L_1 | FLAGS_REX_W_0 | FLAGS_XMM_FLAGS, 0}},\n {OPD(2, 0b01, 0x1A), 1, X86InstInfo{\"VBROADCASTF128\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_L_1 | FLAGS_SF_MOD_MEM_ONLY | FLAGS_REX_W_0 | FLAGS_XMM_FLAGS, 0}},\n {OPD(2, 0b01, 0x1C), 1, X86InstInfo{\"VPABSB\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {OPD(2, 0b01, 0x1D), 1, X86InstInfo{\"VPABSW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {OPD(2, 0b01, 0x1E), 1, X86InstInfo{\"VPABSD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n\n {OPD(2, 0b01, 0x20), 1, X86InstInfo{\"VPMOVSXBW\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {OPD(2, 0b01, 0x21), 1, X86InstInfo{\"VPMOVSXBD\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {OPD(2, 0b01, 0x22), 1, X86InstInfo{\"VPMOVSXBQ\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_16BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {OPD(2, 0b01, 0x23), 1, X86InstInfo{\"VPMOVSXWD\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {OPD(2, 0b01, 0x24), 1, X86InstInfo{\"VPMOVSXWQ\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {OPD(2, 0b01, 0x25), 1, X86InstInfo{\"VPMOVSXDQ\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n\n {OPD(2, 0b01, 0x28), 1, X86InstInfo{\"VPMULDQ\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(2, 0b01, 0x29), 1, X86InstInfo{\"VPCMPEQQ\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(2, 0b01, 0x2A), 1, X86InstInfo{\"VMOVNTDQA\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY | FLAGS_XMM_FLAGS, 0}},\n {OPD(2, 0b01, 0x2B), 1, X86InstInfo{\"VPACKUSDW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(2, 0b01, 0x2C), 1, X86InstInfo{\"VMASKMOVPS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY | FLAGS_VEX_1ST_SRC | FLAGS_REX_W_0 | FLAGS_XMM_FLAGS, 0}},\n {OPD(2, 0b01, 0x2D), 1, X86InstInfo{\"VMASKMOVPD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY | FLAGS_VEX_1ST_SRC | FLAGS_REX_W_0 | FLAGS_XMM_FLAGS, 0}},\n {OPD(2, 0b01, 0x2E), 1, X86InstInfo{\"VMASKMOVPS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY | FLAGS_SF_MOD_DST | FLAGS_VEX_1ST_SRC | FLAGS_REX_W_0 | FLAGS_XMM_FLAGS, 0}},\n {OPD(2, 0b01, 0x2F), 1, X86InstInfo{\"VMASKMOVPD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY | FLAGS_SF_MOD_DST | FLAGS_VEX_1ST_SRC | FLAGS_REX_W_0 | FLAGS_XMM_FLAGS, 0}},\n\n {OPD(2, 0b01, 0x30), 1, X86InstInfo{\"VPMOVZXBW\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {OPD(2, 0b01, 0x31), 1, X86InstInfo{\"VPMOVZXBD\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {OPD(2, 0b01, 0x32), 1, X86InstInfo{\"VPMOVZXBQ\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_16BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {OPD(2, 0b01, 0x33), 1, X86InstInfo{\"VPMOVZXWD\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {OPD(2, 0b01, 0x34), 1, X86InstInfo{\"VPMOVZXWQ\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {OPD(2, 0b01, 0x35), 1, X86InstInfo{\"VPMOVZXDQ\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {OPD(2, 0b01, 0x36), 1, X86InstInfo{\"VPERMD\", TYPE_INST, GenFlagsSameSize(SIZE_256BIT) | FLAGS_MODRM | FLAGS_VEX_L_1 | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(2, 0b01, 0x37), 1, X86InstInfo{\"VPCMPGTQ\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n\n {OPD(2, 0b01, 0x38), 1, X86InstInfo{\"VPMINSB\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(2, 0b01, 0x39), 1, X86InstInfo{\"VPMINSD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(2, 0b01, 0x3A), 1, X86InstInfo{\"VPMINUW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(2, 0b01, 0x3B), 1, X86InstInfo{\"VPMINUD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(2, 0b01, 0x3C), 1, X86InstInfo{\"VPMAXSB\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(2, 0b01, 0x3D), 1, X86InstInfo{\"VPMAXSD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(2, 0b01, 0x3E), 1, X86InstInfo{\"VPMAXUW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(2, 0b01, 0x3F), 1, X86InstInfo{\"VPMAXUD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n\n {OPD(2, 0b01, 0x40), 1, X86InstInfo{\"VPMULLD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(2, 0b01, 0x41), 1, X86InstInfo{\"VPHMINPOSUW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_VEX_L_0, 0}},\n {OPD(2, 0b01, 0x45), 1, X86InstInfo{\"VPSRLV\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(2, 0b01, 0x46), 1, X86InstInfo{\"VPSRAVD\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_REX_W_0 | FLAGS_XMM_FLAGS, 0}},\n {OPD(2, 0b01, 0x47), 1, X86InstInfo{\"VPSLLV\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n\n {OPD(2, 0b01, 0x58), 1, X86InstInfo{\"VPBROADCASTD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_REX_W_0 | FLAGS_XMM_FLAGS, 0}},\n {OPD(2, 0b01, 0x59), 1, X86InstInfo{\"VPBROADCASTQ\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_REX_W_0 | FLAGS_XMM_FLAGS, 0}},\n {OPD(2, 0b01, 0x5A), 1, X86InstInfo{\"VBROADCASTI128\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_L_1 | FLAGS_SF_MOD_MEM_ONLY | FLAGS_REX_W_0 | FLAGS_XMM_FLAGS, 0}},\n\n {OPD(2, 0b01, 0x78), 1, X86InstInfo{\"VPBROADCASTB\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_REX_W_0 | FLAGS_XMM_FLAGS, 0}},\n {OPD(2, 0b01, 0x79), 1, X86InstInfo{\"VPBROADCASTW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_REX_W_0 | FLAGS_XMM_FLAGS, 0}},\n\n {OPD(2, 0b01, 0x8C), 1, X86InstInfo{\"VPMASKMOV\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(2, 0b01, 0x8E), 1, X86InstInfo{\"VPMASKMOV\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY | FLAGS_SF_MOD_DST | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n\n {OPD(2, 0b01, 0x90), 1, X86InstInfo{\"VPGATHERDD/Q\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_2ND_SRC | FLAGS_VEX_VSIB | FLAGS_XMM_FLAGS, 0}},\n {OPD(2, 0b01, 0x91), 1, X86InstInfo{\"VPGATHERQD/Q\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_2ND_SRC | FLAGS_VEX_VSIB | FLAGS_XMM_FLAGS, 0}},\n {OPD(2, 0b01, 0x92), 1, X86InstInfo{\"VGATHERDPS/D\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_2ND_SRC | FLAGS_VEX_VSIB | FLAGS_XMM_FLAGS, 0}},\n {OPD(2, 0b01, 0x93), 1, X86InstInfo{\"VGATHERQPS/D\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_2ND_SRC | FLAGS_VEX_VSIB | FLAGS_XMM_FLAGS, 0}},\n\n {OPD(2, 0b01, 0x96), 1, X86InstInfo{\"VFMADDSUB132\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(2, 0b01, 0x97), 1, X86InstInfo{\"VFMSUBADD132\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n\n {OPD(2, 0b01, 0x98), 1, X86InstInfo{\"VFMADD132\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(2, 0b01, 0x99), 1, X86InstInfo{\"VFMADD132_S\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_VEX_L_IGNORE, 0}},\n {OPD(2, 0b01, 0x9A), 1, X86InstInfo{\"VFMSUB132\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(2, 0b01, 0x9B), 1, X86InstInfo{\"VFMSUB132_S\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_VEX_L_IGNORE, 0}},\n {OPD(2, 0b01, 0x9C), 1, X86InstInfo{\"VFNMADD132\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(2, 0b01, 0x9D), 1, X86InstInfo{\"VFNMADD132_S\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_VEX_L_IGNORE, 0}},\n {OPD(2, 0b01, 0x9E), 1, X86InstInfo{\"VFNMSUB132\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(2, 0b01, 0x9F), 1, X86InstInfo{\"VFNMSUB132_S\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_VEX_L_IGNORE, 0}},\n\n {OPD(2, 0b01, 0xA8), 1, X86InstInfo{\"VFMADD213\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(2, 0b01, 0xA9), 1, X86InstInfo{\"VFMADD213_S\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_VEX_L_IGNORE, 0}},\n {OPD(2, 0b01, 0xAA), 1, X86InstInfo{\"VFMSUB213\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(2, 0b01, 0xAB), 1, X86InstInfo{\"VFMSUB213_S\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_VEX_L_IGNORE, 0}},\n {OPD(2, 0b01, 0xAC), 1, X86InstInfo{\"VFNMADD213\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(2, 0b01, 0xAD), 1, X86InstInfo{\"VFNMADD213_S\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_VEX_L_IGNORE, 0}},\n {OPD(2, 0b01, 0xAE), 1, X86InstInfo{\"VFNMSUB213\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(2, 0b01, 0xAF), 1, X86InstInfo{\"VFNMSUB213_S\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_VEX_L_IGNORE, 0}},\n\n {OPD(2, 0b01, 0xB8), 1, X86InstInfo{\"VFMADD231\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(2, 0b01, 0xB9), 1, X86InstInfo{\"VFMADD231_S\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_VEX_L_IGNORE, 0}},\n {OPD(2, 0b01, 0xBA), 1, X86InstInfo{\"VFMSUB231\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(2, 0b01, 0xBB), 1, X86InstInfo{\"VFMSUB231_S\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_VEX_L_IGNORE, 0}},\n {OPD(2, 0b01, 0xBC), 1, X86InstInfo{\"VFNMADD231\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(2, 0b01, 0xBD), 1, X86InstInfo{\"VFNMADD231_S\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_VEX_L_IGNORE, 0}},\n {OPD(2, 0b01, 0xBE), 1, X86InstInfo{\"VFNMSUB231\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(2, 0b01, 0xBF), 1, X86InstInfo{\"VFNMSUB231_S\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_VEX_L_IGNORE, 0}},\n\n {OPD(2, 0b01, 0xA6), 1, X86InstInfo{\"VFMADDSUB213\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(2, 0b01, 0xA7), 1, X86InstInfo{\"VFMSUBADD213\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n\n {OPD(2, 0b01, 0xB6), 1, X86InstInfo{\"VFMADDSUB231\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(2, 0b01, 0xB7), 1, X86InstInfo{\"VFMSUBADD231\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n\n {OPD(2, 0b01, 0xDB), 1, X86InstInfo{\"VAESIMC\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},\n {OPD(2, 0b01, 0xDC), 1, X86InstInfo{\"VAESENC\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(2, 0b01, 0xDD), 1, X86InstInfo{\"VAESENCLAST\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(2, 0b01, 0xDE), 1, X86InstInfo{\"VAESDEC\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n {OPD(2, 0b01, 0xDF), 1, X86InstInfo{\"VAESDECLAST\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},\n\n {OPD(2, 0b00, 0xF2), 1, X86InstInfo{\"ANDN\", TYPE_INST, FLAGS_MODRM | FLAGS_VEX_1ST_SRC, 0}},\n\n {OPD(2, 0b00, 0xF3), 1, X86InstInfo{\"\", TYPE_VEX_GROUP_17, FLAGS_NONE, 0}}, // VEX Group 17\n {OPD(2, 0b01, 0xF3), 1, X86InstInfo{\"\", TYPE_VEX_GROUP_17, FLAGS_NONE, 0}}, // VEX Group 17\n {OPD(2, 0b10, 0xF3), 1, X86InstInfo{\"\", TYPE_VEX_GROUP_17, FLAGS_NONE, 0}}, // VEX Group 17\n {OPD(2, 0b11, 0xF3), 1, X86InstInfo{\"\", TYPE_VEX_GROUP_17, FLAGS_NONE, 0}}, // VEX Group 17\n\n {OPD(2, 0b00, 0xF5), 1, X86InstInfo{\"BZHI\", TYPE_INST, FLAGS_MODRM | FLAGS_VEX_L_0 | FLAGS_VEX_2ND_SRC, 0}},\n // AMD reference manual is incorrect. PEXT actually maps to 0b10, not 0b01.\n {OPD(2, 0b10, 0xF5), 1, X86InstInfo{\"PEXT\", TYPE_INST, FLAGS_MODRM | FLAGS_VEX_L_0 | FLAGS_VEX_1ST_SRC, 0}},\n {OPD(2, 0b11, 0xF5), 1, X86InstInfo{\"PDEP\", TYPE_INST, FLAGS_MODRM | FLAGS_VEX_L_0 | FLAGS_VEX_1ST_SRC, 0}},\n\n {OPD(2, 0b11, 0xF6), 1, X86InstInfo{\"MULX\", TYPE_INST, FLAGS_MODRM | FLAGS_VEX_L_0 | FLAGS_VEX_1ST_SRC, 0}},\n\n {OPD(2, 0b00, 0xF7), 1, X86InstInfo{\"BEXTR\", TYPE_INST, FLAGS_MODRM | FLAGS_VEX_L_0 | FLAGS_VEX_2ND_SRC, 0}},\n {OPD(2, 0b01, 0xF7), 1, X86InstInfo{\"SHLX\", TYPE_INST, FLAGS_MODRM | FLAGS_VEX_L_0 | FLAGS_VEX_2ND_SRC, 0}},\n {OPD(2, 0b10, 0xF7), 1, X86InstInfo{\"SARX\", TYPE_INST, FLAGS_MODRM | FLAGS_VEX_L_0 | FLAGS_VEX_2ND_SRC, 0}},\n {OPD(2, 0b11, 0xF7), 1, X86InstInfo{\"SHRX\", TYPE_INST, FLAGS_MODRM | FLAGS_VEX_L_0 | FLAGS_VEX_2ND_SRC, 0}},\n\n // VEX Map 3\n {OPD(3, 0b01, 0x00), 1, X86InstInfo{\"VPERMQ\", TYPE_INST, GenFlagsSameSize(SIZE_256BIT) | FLAGS_MODRM | FLAGS_VEX_L_1 | FLAGS_REX_W_1 | FLAGS_XMM_FLAGS, 1}},\n {OPD(3, 0b01, 0x01), 1, X86InstInfo{\"VPERMPD\", TYPE_INST, GenFlagsSameSize(SIZE_256BIT) | FLAGS_MODRM | FLAGS_VEX_L_1 | FLAGS_REX_W_1 | FLAGS_XMM_FLAGS, 1}},\n {OPD(3, 0b01, 0x02), 1, X86InstInfo{\"VPBLENDD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_REX_W_0 | FLAGS_XMM_FLAGS, 1}},\n {OPD(3, 0b01, 0x04), 1, X86InstInfo{\"VPERMILPS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_REX_W_0 | FLAGS_XMM_FLAGS, 1}},\n {OPD(3, 0b01, 0x05), 1, X86InstInfo{\"VPERMILPD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_REX_W_0 | FLAGS_XMM_FLAGS, 1}},\n {OPD(3, 0b01, 0x06), 1, X86InstInfo{\"VPERM2F128\", TYPE_INST, GenFlagsSameSize(SIZE_256BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_REX_W_0 | FLAGS_XMM_FLAGS | FLAGS_VEX_L_1, 1}},\n\n {OPD(3, 0b01, 0x08), 1, X86InstInfo{\"VROUNDPS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 1}},\n {OPD(3, 0b01, 0x09), 1, X86InstInfo{\"VROUNDPD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 1}},\n {OPD(3, 0b01, 0x0A), 1, X86InstInfo{\"VROUNDSS\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 1}},\n {OPD(3, 0b01, 0x0B), 1, X86InstInfo{\"VROUNDSD\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 1}},\n {OPD(3, 0b01, 0x0C), 1, X86InstInfo{\"VBLENDPS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 1}},\n {OPD(3, 0b01, 0x0D), 1, X86InstInfo{\"VBLENDPD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 1}},\n {OPD(3, 0b01, 0x0E), 1, X86InstInfo{\"VPBLENDW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 1}},\n {OPD(3, 0b01, 0x0F), 1, X86InstInfo{\"VPALIGNR\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 1}},\n\n {OPD(3, 0b01, 0x14), 1, X86InstInfo{\"VPEXTRB\", TYPE_INST, GenFlagsSizes(SIZE_32BIT, SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_L_0 | FLAGS_SF_MOD_DST | FLAGS_SF_DST_GPR | FLAGS_XMM_FLAGS, 1}},\n {OPD(3, 0b01, 0x15), 1, X86InstInfo{\"VPEXTRW\", TYPE_INST, GenFlagsSizes(SIZE_16BIT, SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_L_0 | FLAGS_SF_MOD_DST | FLAGS_SF_DST_GPR | FLAGS_XMM_FLAGS, 1}},\n {OPD(3, 0b01, 0x16), 1, X86InstInfo{\"VPEXTRD\", TYPE_INST, GenFlagsSizes(SIZE_32BIT, SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_L_0 | FLAGS_SF_MOD_DST | FLAGS_SF_DST_GPR | FLAGS_XMM_FLAGS, 1}},\n {OPD(3, 0b01, 0x17), 1, X86InstInfo{\"VEXTRACTPS\", TYPE_INST, GenFlagsSizes(SIZE_32BIT, SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_L_0 | FLAGS_SF_MOD_DST | FLAGS_SF_DST_GPR | FLAGS_XMM_FLAGS, 1}},\n\n {OPD(3, 0b01, 0x18), 1, X86InstInfo{\"VINSERTF128\", TYPE_INST, GenFlagsSameSize(SIZE_256BIT) | FLAGS_MODRM | FLAGS_VEX_L_1 | FLAGS_REX_W_0 | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 1}},\n {OPD(3, 0b01, 0x19), 1, X86InstInfo{\"VEXTRACTF128\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_256BIT) | FLAGS_MODRM | FLAGS_VEX_L_1 | FLAGS_REX_W_0 | FLAGS_SF_MOD_DST | FLAGS_XMM_FLAGS, 1}},\n {OPD(3, 0b01, 0x1D), 1, X86InstInfo{\"VCVTPS2PH\", TYPE_INST, GenFlagsSizes(SIZE_64BIT, SIZE_128BIT) | FLAGS_MODRM | FLAGS_REX_W_0 | FLAGS_SF_MOD_DST | FLAGS_XMM_FLAGS, 1}},\n\n {OPD(3, 0b01, 0x20), 1, X86InstInfo{\"VPINSRB\", TYPE_INST, GenFlagsDstSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_L_0 | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_SF_SRC_GPR, 1}},\n {OPD(3, 0b01, 0x21), 1, X86InstInfo{\"VINSERTPS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_L_0 | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 1}},\n {OPD(3, 0b01, 0x22), 1, X86InstInfo{\"VPINSR{D,Q}\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_L_0 | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_SF_SRC_GPR, 1}},\n\n {OPD(3, 0b01, 0x38), 1, X86InstInfo{\"VINSERTI128\", TYPE_INST, GenFlagsSameSize(SIZE_256BIT) | FLAGS_MODRM | FLAGS_VEX_L_1 | FLAGS_REX_W_0 | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 1}},\n {OPD(3, 0b01, 0x39), 1, X86InstInfo{\"VEXTRACTI128\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_256BIT) | FLAGS_MODRM | FLAGS_VEX_L_1 | FLAGS_REX_W_0 | FLAGS_SF_MOD_DST | FLAGS_XMM_FLAGS, 1}},\n\n {OPD(3, 0b01, 0x40), 1, X86InstInfo{\"VDPPS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 1}},\n {OPD(3, 0b01, 0x41), 1, X86InstInfo{\"VDPPD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_VEX_L_0, 1}},\n {OPD(3, 0b01, 0x42), 1, X86InstInfo{\"VMPSADBW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 1}},\n {OPD(3, 0b01, 0x44), 1, X86InstInfo{\"VPCLMULQDQ\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 1}},\n {OPD(3, 0b01, 0x46), 1, X86InstInfo{\"VPERM2I128\", TYPE_INST, GenFlagsSameSize(SIZE_256BIT) | FLAGS_MODRM | FLAGS_VEX_L_1 | FLAGS_REX_W_0 | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 1}},\n\n {OPD(3, 0b01, 0x4A), 1, X86InstInfo{\"VBLENDVPS\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_REX_W_0 | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 1}},\n {OPD(3, 0b01, 0x4B), 1, X86InstInfo{\"VBLENDVPD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_REX_W_0 | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 1}},\n {OPD(3, 0b01, 0x4C), 1, X86InstInfo{\"VPBLENDVB\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_REX_W_0 | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 1}},\n\n {OPD(3, 0b01, 0x5C), 1, X86InstInfo{\"VFMADDSUBPS\", TYPE_UNDEC, FLAGS_NONE, 0}}, ///< FMA4\n {OPD(3, 0b01, 0x5D), 1, X86InstInfo{\"VFMADDSUBPD\", TYPE_UNDEC, FLAGS_NONE, 0}}, ///< FMA4\n {OPD(3, 0b01, 0x5E), 1, X86InstInfo{\"VFMSUBADDPS\", TYPE_UNDEC, FLAGS_NONE, 0}}, ///< FMA4\n {OPD(3, 0b01, 0x5F), 1, X86InstInfo{\"VFMSUBADDPD\", TYPE_UNDEC, FLAGS_NONE, 0}}, ///< FMA4\n\n {OPD(3, 0b01, 0x60), 1, X86InstInfo{\"VPCMPESTRM\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_VEX_L_0, 1}},\n {OPD(3, 0b01, 0x61), 1, X86InstInfo{\"VPCMPESTRI\", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_VEX_L_0, 1}},\n {OPD(3, 0b01, 0x62), 1, X86InstInfo{\"VPCMPISTRM\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_VEX_L_0, 1}},\n {OPD(3, 0b01, 0x63), 1, X86InstInfo{\"VPCMPISTRI\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_VEX_L_0, 1}},\n\n {OPD(3, 0b01, 0x68), 1, X86InstInfo{\"VFMADDPS\", TYPE_UNDEC, FLAGS_NONE, 0}}, ///< FMA4\n {OPD(3, 0b01, 0x69), 1, X86InstInfo{\"VFMADDPD\", TYPE_UNDEC, FLAGS_NONE, 0}}, ///< FMA4\n {OPD(3, 0b01, 0x6A), 1, X86InstInfo{\"VFMADDSS\", TYPE_UNDEC, FLAGS_NONE, 0}}, ///< FMA4\n {OPD(3, 0b01, 0x6B), 1, X86InstInfo{\"VFMADDSD\", TYPE_UNDEC, FLAGS_NONE, 0}}, ///< FMA4\n {OPD(3, 0b01, 0x6C), 1, X86InstInfo{\"VFMSUBPS\", TYPE_UNDEC, FLAGS_NONE, 0}}, ///< FMA4\n {OPD(3, 0b01, 0x6D), 1, X86InstInfo{\"VFMSUBPD\", TYPE_UNDEC, FLAGS_NONE, 0}}, ///< FMA4\n {OPD(3, 0b01, 0x6E), 1, X86InstInfo{\"VFMSUBSS\", TYPE_UNDEC, FLAGS_NONE, 0}}, ///< FMA4\n {OPD(3, 0b01, 0x6F), 1, X86InstInfo{\"VFMSUBSD\", TYPE_UNDEC, FLAGS_NONE, 0}}, ///< FMA4\n\n {OPD(3, 0b01, 0x78), 1, X86InstInfo{\"VFNMADDPS\", TYPE_UNDEC, FLAGS_NONE, 0}}, ///< FMA4\n {OPD(3, 0b01, 0x79), 1, X86InstInfo{\"VFNMADDPD\", TYPE_UNDEC, FLAGS_NONE, 0}}, ///< FMA4\n {OPD(3, 0b01, 0x7A), 1, X86InstInfo{\"VFNMADDSS\", TYPE_UNDEC, FLAGS_NONE, 0}}, ///< FMA4\n {OPD(3, 0b01, 0x7B), 1, X86InstInfo{\"VFNMADDSD\", TYPE_UNDEC, FLAGS_NONE, 0}}, ///< FMA4\n {OPD(3, 0b01, 0x7C), 1, X86InstInfo{\"VFNMSUBPS\", TYPE_UNDEC, FLAGS_NONE, 0}}, ///< FMA4\n {OPD(3, 0b01, 0x7D), 1, X86InstInfo{\"VFNMSUBPD\", TYPE_UNDEC, FLAGS_NONE, 0}}, ///< FMA4\n {OPD(3, 0b01, 0x7E), 1, X86InstInfo{\"VFNMSUBSS\", TYPE_UNDEC, FLAGS_NONE, 0}}, ///< FMA4\n {OPD(3, 0b01, 0x7F), 1, X86InstInfo{\"VFNMSUBSD\", TYPE_UNDEC, FLAGS_NONE, 0}}, ///< FMA4\n\n {OPD(3, 0b01, 0xDF), 1, X86InstInfo{\"VAESKEYGENASSIST\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_L_0 | FLAGS_XMM_FLAGS, 1}},\n\n {OPD(3, 0b11, 0xF0), 1, X86InstInfo{\"RORX\", TYPE_INST, FLAGS_MODRM | FLAGS_VEX_L_0, 1}},\n\n // VEX Map 4 - 31 (Reserved)\n };\n#undef OPD\n\n GenerateTable(Table.data(), VEXTable, std::size(VEXTable));\n\n IR::InstallToTable(Table, IR::OpDispatch_VEXTable);\n IR::InstallToTable(Table, RuntimeTable);\n return Table;\n};\n\nauto GroupTableLambda = [](const auto RuntimeTable) consteval {\n std::array Table{};\n\n#define OPD(group, pp, opcode) (((group - TYPE_VEX_GROUP_12) << 4) | (pp << 3) | (opcode))\n constexpr U8U8InfoStruct VEXGroupTable[] = {\n {OPD(TYPE_VEX_GROUP_12, 1, 0b010), 1, X86InstInfo{\"VPSRLW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_REG_ONLY | FLAGS_VEX_DST | FLAGS_XMM_FLAGS, 1}},\n {OPD(TYPE_VEX_GROUP_12, 1, 0b100), 1, X86InstInfo{\"VPSRAW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_REG_ONLY | FLAGS_VEX_DST | FLAGS_XMM_FLAGS, 1}},\n {OPD(TYPE_VEX_GROUP_12, 1, 0b110), 1, X86InstInfo{\"VPSLLW\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_REG_ONLY | FLAGS_VEX_DST | FLAGS_XMM_FLAGS, 1}},\n\n {OPD(TYPE_VEX_GROUP_13, 1, 0b010), 1, X86InstInfo{\"VPSRLD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_REG_ONLY | FLAGS_VEX_DST | FLAGS_XMM_FLAGS, 1}},\n {OPD(TYPE_VEX_GROUP_13, 1, 0b100), 1, X86InstInfo{\"VPSRAD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_REG_ONLY | FLAGS_VEX_DST | FLAGS_XMM_FLAGS, 1}},\n {OPD(TYPE_VEX_GROUP_13, 1, 0b110), 1, X86InstInfo{\"VPSLLD\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_REG_ONLY | FLAGS_VEX_DST | FLAGS_XMM_FLAGS, 1}},\n\n {OPD(TYPE_VEX_GROUP_14, 1, 0b010), 1, X86InstInfo{\"VPSRLQ\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_REG_ONLY | FLAGS_VEX_DST | FLAGS_XMM_FLAGS, 1}},\n {OPD(TYPE_VEX_GROUP_14, 1, 0b011), 1, X86InstInfo{\"VPSRLDQ\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_REG_ONLY | FLAGS_VEX_DST | FLAGS_XMM_FLAGS, 1}},\n {OPD(TYPE_VEX_GROUP_14, 1, 0b110), 1, X86InstInfo{\"VPSLLQ\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_REG_ONLY | FLAGS_VEX_DST | FLAGS_XMM_FLAGS, 1}},\n {OPD(TYPE_VEX_GROUP_14, 1, 0b111), 1, X86InstInfo{\"VPSLLDQ\", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_REG_ONLY | FLAGS_VEX_DST | FLAGS_XMM_FLAGS, 1}},\n\n {OPD(TYPE_VEX_GROUP_15, 0, 0b010), 1, X86InstInfo{\"VLDMXCSR\", TYPE_INST, GenFlagsSameSize(SIZE_32BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_VEX_L_0 | FLAGS_SF_MOD_MEM_ONLY, 0}},\n {OPD(TYPE_VEX_GROUP_15, 0, 0b011), 1, X86InstInfo{\"VSTMXCSR\", TYPE_INST, GenFlagsSameSize(SIZE_32BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_VEX_L_0 | FLAGS_SF_MOD_MEM_ONLY, 0}},\n\n {OPD(TYPE_VEX_GROUP_17, 0, 0b001), 1, X86InstInfo{\"BLSR\", TYPE_INST, FLAGS_MODRM | FLAGS_VEX_DST, 0}},\n {OPD(TYPE_VEX_GROUP_17, 0, 0b010), 1, X86InstInfo{\"BLSMSK\", TYPE_INST, FLAGS_MODRM | FLAGS_VEX_DST, 0}},\n {OPD(TYPE_VEX_GROUP_17, 0, 0b011), 1, X86InstInfo{\"BLSI\", TYPE_INST, FLAGS_MODRM | FLAGS_VEX_DST, 0}},\n };\n#undef OPD\n\n GenerateTable(Table.data(), VEXGroupTable, std::size(VEXGroupTable));\n\n IR::InstallToTable(Table, IR::OpDispatch_VEXGroupTable);\n IR::InstallToTable(Table, RuntimeTable);\n return Table;\n};\n\nconst std::array VEXTableOps = BaseTableLambda(std::to_array(AVX256::BaseTable));\nconst std::array VEXTableGroupOps = GroupTableLambda(std::to_array(AVX256::TableGroupOps));\n\nconst std::array VEXTableOps_AVX128 = BaseTableLambda(std::to_array(AVX128::BaseTable));\nconst std::array VEXTableGroupOps_AVX128 = GroupTableLambda(std::to_array(AVX128::TableGroupOps));\n}\n" }, { "path": "FEXCore/Source/Interface/Core/X86Tables/X86Tables.h", "content": "// SPDX-License-Identifier: MIT\n/*\n$info$\ntags: frontend|x86-tables\n$end_info$\n*/\n\n#pragma once\n\n#include \n#include \n\n#include \n#include \n#include \n#include \n#include \n\nnamespace FEXCore::IR {\nclass OpDispatchBuilder;\n}\n\nnamespace FEXCore::X86Tables {\nstruct X86InstInfo;\n\nnamespace DecodeFlags {\n constexpr uint32_t FLAG_OPERAND_SIZE = (1 << 0);\n constexpr uint32_t FLAG_ADDRESS_SIZE = (1 << 1);\n constexpr uint32_t FLAG_LOCK = (1 << 2);\n constexpr uint32_t FLAG_LEGACY_PREFIX = (1 << 3);\n constexpr uint32_t FLAG_REX_PREFIX = (1 << 4);\n constexpr uint32_t FLAG_VSIB_BYTE = (1 << 5);\n constexpr uint32_t FLAG_OPTION_AVX_W = (1 << 6);\n constexpr uint32_t FLAG_REX_WIDENING = (1 << 7);\n constexpr uint32_t FLAG_REX_XGPR_B = (1 << 8);\n constexpr uint32_t FLAG_REX_XGPR_X = (1 << 9);\n constexpr uint32_t FLAG_REX_XGPR_R = (1 << 10);\n constexpr uint32_t FLAG_NO_PREFIX = (0b000 << 11);\n constexpr uint32_t FLAG_ES_PREFIX = (0b001 << 11);\n constexpr uint32_t FLAG_CS_PREFIX = (0b010 << 11);\n constexpr uint32_t FLAG_SS_PREFIX = (0b011 << 11);\n constexpr uint32_t FLAG_DS_PREFIX = (0b100 << 11);\n constexpr uint32_t FLAG_FS_PREFIX = (0b101 << 11);\n constexpr uint32_t FLAG_GS_PREFIX = (0b110 << 11);\n constexpr uint32_t FLAG_SEGMENTS = (0b111 << 11);\n constexpr uint32_t FLAG_FORCE_TSO = (1 << 14);\n constexpr uint32_t FLAG_DECODED_MODRM = (1 << 15);\n constexpr uint32_t FLAG_DECODED_SIB = (1 << 16);\n constexpr uint32_t FLAG_REP_PREFIX = (1 << 17);\n constexpr uint32_t FLAG_REPNE_PREFIX = (1 << 18);\n // Size flags\n constexpr uint32_t FLAG_SIZE_DST_OFF = 19;\n constexpr uint32_t FLAG_SIZE_SRC_OFF = FLAG_SIZE_DST_OFF + 3;\n constexpr uint32_t SIZE_MASK = 0b111;\n constexpr uint32_t SIZE_DEF = 0b000; // This should be invalid past decoding\n constexpr uint32_t SIZE_8BIT = 0b001;\n constexpr uint32_t SIZE_16BIT = 0b010;\n constexpr uint32_t SIZE_32BIT = 0b011;\n constexpr uint32_t SIZE_64BIT = 0b100;\n constexpr uint32_t SIZE_128BIT = 0b101;\n constexpr uint32_t SIZE_256BIT = 0b110;\n\n constexpr uint32_t FLAG_OPADDR_OFF = (FLAG_SIZE_SRC_OFF + 3);\n constexpr uint32_t FLAG_OPADDR_STACKSIZE = 4; // Two level deep stack\n constexpr uint32_t FLAG_OPADDR_FLAG_SIZE = 2;\n constexpr uint32_t FLAG_OPADDR_MASK = (((1 << FLAG_OPADDR_STACKSIZE) - 1) << FLAG_OPADDR_OFF);\n\n // 00 = NONE\n constexpr uint32_t FLAG_OPERAND_SIZE_LAST = 0b01;\n constexpr uint32_t FLAG_WIDENING_SIZE_LAST = 0b10;\n\n constexpr uint32_t GetSizeDstFlags(uint32_t Flags) {\n return (Flags >> FLAG_SIZE_DST_OFF) & SIZE_MASK;\n }\n constexpr uint32_t GetSizeSrcFlags(uint32_t Flags) {\n return (Flags >> FLAG_SIZE_SRC_OFF) & SIZE_MASK;\n }\n\n constexpr uint32_t GenSizeDstSize(uint32_t Size) {\n return Size << FLAG_SIZE_DST_OFF;\n }\n constexpr uint32_t GenSizeSrcSize(uint32_t Size) {\n return Size << FLAG_SIZE_SRC_OFF;\n }\n\n constexpr uint32_t GetOpAddr(uint32_t Flags, uint32_t Index) {\n return (((Flags & FLAG_OPADDR_MASK) >> FLAG_OPADDR_OFF) >> (Index * 2)) & ((1 << FLAG_OPADDR_FLAG_SIZE) - 1);\n }\n\n inline void PushOpAddr(uint32_t* Flags, uint32_t Flag) {\n uint32_t TmpFlags = *Flags;\n uint32_t BottomOfStack = ((TmpFlags & FLAG_OPADDR_MASK) >> FLAG_OPADDR_OFF) & ((1 << FLAG_OPADDR_FLAG_SIZE) - 1);\n\n TmpFlags &= ~(FLAG_OPADDR_MASK);\n TmpFlags |= (BottomOfStack << (FLAG_OPADDR_OFF + FLAG_OPADDR_FLAG_SIZE)) | (Flag << FLAG_OPADDR_OFF);\n\n *Flags = TmpFlags;\n }\n\n inline void PopOpAddrIf(uint32_t* Flags, uint32_t Flag) {\n uint32_t TmpFlags = *Flags;\n uint32_t BottomOfStack = ((TmpFlags & FLAG_OPADDR_MASK) >> FLAG_OPADDR_OFF) & ((1 << FLAG_OPADDR_FLAG_SIZE) - 1);\n\n // Only pop the stack if the bottom flag is the one we care about\n // Necessary for escape prefixes that overlap regular prefixes\n if (BottomOfStack != Flag) {\n return;\n }\n\n uint32_t TopOfStack = ((TmpFlags & FLAG_OPADDR_MASK) >> (FLAG_OPADDR_OFF + FLAG_OPADDR_FLAG_SIZE)) & ((1 << FLAG_OPADDR_FLAG_SIZE) - 1);\n\n TmpFlags &= ~(FLAG_OPADDR_MASK);\n TmpFlags |= (TopOfStack << FLAG_OPADDR_OFF);\n\n *Flags = TmpFlags;\n }\n\n} // namespace DecodeFlags\n\nstruct DecodedOperand {\n enum class OpType : uint8_t {\n Nothing,\n GPR,\n GPRDirect,\n GPRIndirect,\n GPRIndirectRelocation,\n RIPRelative,\n RIPRelativeRelocation,\n Literal,\n LiteralRelocation,\n SIB,\n SIBRelocation\n };\n\n bool IsNone() const {\n return Type == OpType::Nothing;\n }\n bool IsGPR() const {\n return Type == OpType::GPR;\n }\n bool IsGPRDirect() const {\n return Type == OpType::GPRDirect;\n }\n bool IsGPRIndirect() const {\n return Type == OpType::GPRIndirect;\n }\n bool IsGPRIndirectRelocation() const {\n return Type == OpType::GPRIndirectRelocation;\n }\n bool IsRIPRelative() const {\n return Type == OpType::RIPRelative;\n }\n bool IsRIPRelativeRelocation() const {\n return Type == OpType::RIPRelativeRelocation;\n }\n bool IsLiteral() const {\n return Type == OpType::Literal;\n }\n bool IsLiteralRelocation() const {\n return Type == OpType::LiteralRelocation;\n }\n bool IsSIB() const {\n return Type == OpType::SIB;\n }\n bool IsSIBRelocation() const {\n return Type == OpType::SIBRelocation;\n }\n\n uint64_t Literal() const {\n LOGMAN_THROW_A_FMT(IsLiteral(), \"Precondition: must be a literal\");\n return Data.Literal.Value;\n }\n\n union TypeUnion {\n struct GPRType {\n bool HighBits;\n uint8_t GPR;\n auto operator<=>(const GPRType&) const = default;\n } GPR;\n\n struct {\n int64_t Displacement;\n uint8_t GPR;\n } GPRIndirect; // Shared with GPRIndirectRelocation\n\n struct {\n int64_t Value;\n } RIPLiteral; // Shared with RIPLiteralRelocation\n\n struct LiteralType {\n uint64_t Value;\n uint8_t Size;\n } Literal;\n\n struct {\n int64_t EntrypointOffset;\n } LiteralRelocation;\n\n struct {\n int64_t Offset;\n uint8_t Scale;\n uint8_t Index; // ~0 invalid\n uint8_t Base; // ~0 invalid\n } SIB; // Shared with SIBRelocation\n };\n\n TypeUnion Data;\n OpType Type;\n};\n\nstruct DecodedInst {\n uint64_t PC;\n\n DecodedOperand Dest;\n DecodedOperand Src[3];\n\n // Constains the dispatcher handler pointer\n const X86InstInfo* TableInfo;\n\n uint32_t Flags;\n uint16_t OP;\n uint8_t OPRaw;\n\n uint8_t ModRM;\n uint8_t SIB;\n uint8_t InstSize;\n int8_t REXIndex;\n};\n\nunion ModRMDecoded {\n uint8_t Hex {};\n struct {\n uint8_t rm : 3;\n uint8_t reg : 3;\n uint8_t mod : 2;\n };\n};\n\nunion SIBDecoded {\n uint8_t Hex {};\n struct {\n uint8_t base : 3;\n uint8_t index : 3;\n uint8_t scale : 2;\n };\n};\n\nenum InstType {\n TYPE_UNKNOWN,\n TYPE_LEGACY_PREFIX,\n TYPE_PREFIX,\n TYPE_REX_PREFIX,\n TYPE_SECONDARY_TABLE_PREFIX,\n TYPE_X87_TABLE_PREFIX,\n TYPE_VEX_TABLE_PREFIX,\n TYPE_INST,\n TYPE_X87 = TYPE_INST,\n TYPE_INVALID,\n TYPE_COPY_OTHER,\n // Changes `X86InstInfo::OpcodeDispatcher` member to use the `Indirect` version.\n // Points to a 2 member array of X86InstInfo to choose instruction description based on executing bitness.\n TYPE_ARCH_DISPATCHER,\n\n // Must be in order\n // Groups 1, 1a, 2, 3, 4, 5, 11 are for the primary op table\n // Groups 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 17, p are for the secondary op table\n TYPE_GROUP_1,\n TYPE_GROUP_1A,\n TYPE_GROUP_2,\n TYPE_GROUP_3,\n TYPE_GROUP_4,\n TYPE_GROUP_5,\n TYPE_GROUP_11,\n\n // Must be in order\n // Groups 6-p Are for the secondary op table\n TYPE_GROUP_6,\n TYPE_GROUP_7,\n TYPE_GROUP_8,\n TYPE_GROUP_9,\n TYPE_GROUP_10,\n TYPE_GROUP_12,\n TYPE_GROUP_13,\n TYPE_GROUP_14,\n TYPE_GROUP_15,\n TYPE_GROUP_16,\n TYPE_GROUP_17,\n TYPE_GROUP_P,\n\n // The secondary op extension table allows further extensions\n // Group 7 allows additional extensions to this table\n TYPE_SECOND_GROUP_MODRM,\n\n TYPE_VEX_GROUP_12,\n TYPE_VEX_GROUP_13,\n TYPE_VEX_GROUP_14,\n TYPE_VEX_GROUP_15,\n TYPE_VEX_GROUP_17,\n\n TYPE_GROUP_EVEX,\n\n // Exists in the table but isn't decoded correctly\n TYPE_UNDEC = TYPE_INVALID,\n TYPE_MMX = TYPE_INVALID,\n TYPE_PRIV = TYPE_INVALID,\n TYPE_0F38_TABLE = TYPE_INVALID,\n TYPE_0F3A_TABLE = TYPE_INVALID,\n TYPE_3DNOW_TABLE = TYPE_INVALID,\n};\n\nnamespace InstFlags {\n\n using InstFlagType = uint64_t;\n\n constexpr InstFlagType FLAGS_NONE = 0;\n // The secondary Opcode Map uses prefix bytes to overlay more instruction\n // But some instructions need to ignore this overlay and consume these prefixes.\n constexpr InstFlagType FLAGS_NO_OVERLAY = (1ULL << 0);\n // Some instructions partially ignore overlay\n // Ignore OpSize (0x66) in this case\n constexpr InstFlagType FLAGS_NO_OVERLAY66 = (1ULL << 1);\n constexpr InstFlagType FLAGS_DEBUG_MEM_ACCESS = (1ULL << 2);\n // Only SEXT if the instruction is operating in 64bit operand size\n constexpr InstFlagType FLAGS_SRC_SEXT64BIT = (1ULL << 3);\n constexpr InstFlagType FLAGS_BLOCK_END = (1ULL << 4);\n constexpr InstFlagType FLAGS_SETS_RIP = (1ULL << 5);\n\n constexpr InstFlagType FLAGS_DISPLACE_SIZE_MUL_2 = (1ULL << 6);\n constexpr InstFlagType FLAGS_DISPLACE_SIZE_DIV_2 = (1ULL << 7);\n constexpr InstFlagType FLAGS_SRC_SEXT = (1ULL << 8);\n constexpr InstFlagType FLAGS_MEM_OFFSET = (1ULL << 9);\n\n // Enables XMM based subflags\n // Current reserved range for this SF is [10, 15]\n constexpr InstFlagType FLAGS_XMM_FLAGS = (1ULL << 10);\n\n // X87 flags aliased to XMM flags selection\n // Allows X87 instruction table that is abusing the flag for 64BIT selection to work\n constexpr InstFlagType FLAGS_X87_FLAGS = (1ULL << 10);\n\n // Non-XMM subflags\n constexpr InstFlagType FLAGS_SF_DST_RAX = (1ULL << 11);\n constexpr InstFlagType FLAGS_SF_DST_RDX = (1ULL << 12);\n constexpr InstFlagType FLAGS_SF_SRC_RAX = (1ULL << 13);\n constexpr InstFlagType FLAGS_SF_SRC_RCX = (1ULL << 14);\n constexpr InstFlagType FLAGS_SF_REX_IN_BYTE = (1ULL << 15);\n\n // XMM subflags\n constexpr InstFlagType FLAGS_SF_UNUSED = (1ULL << 11); // No assigned behavior yet\n constexpr InstFlagType FLAGS_SF_DST_GPR = (1ULL << 12);\n constexpr InstFlagType FLAGS_SF_SRC_GPR = (1ULL << 13);\n constexpr InstFlagType FLAGS_SF_MMX_DST = (1ULL << 14);\n constexpr InstFlagType FLAGS_SF_MMX_SRC = (1ULL << 15);\n constexpr InstFlagType FLAGS_SF_MMX = FLAGS_SF_MMX_DST | FLAGS_SF_MMX_SRC;\n\n // Enables MODRM specific subflags\n // Current reserved range for this SF is [14, 17]\n constexpr InstFlagType FLAGS_MODRM = (1ULL << 16);\n\n // With ModRM SF flag enabled\n // Direction of ModRM. Dst ^ Src\n // Set means destination is rm bits\n // Unset means src is rm bits\n constexpr InstFlagType FLAGS_SF_MOD_DST = (1ULL << 17);\n\n // If the instruction is restricted to mem or reg only\n // 0b00 = Regular ModRM support\n // 0b01 = Memory accesses only\n // 0b10 = Register accesses only\n // 0b11 = \n constexpr InstFlagType FLAGS_SF_MOD_MEM_ONLY = (1ULL << 18);\n constexpr InstFlagType FLAGS_SF_MOD_REG_ONLY = (1ULL << 19);\n\n constexpr InstFlagType FLAGS_SF_MOD_ZERO_REG = (1ULL << 20);\n\n // x87\n constexpr InstFlagType FLAGS_POP = (1ULL << 21);\n\n // Whether or not the instruction has a VEX prefix for the dest, first, or second source.\n constexpr InstFlagType FLAGS_VEX_SRC_MASK = (0b11ULL << 22);\n constexpr InstFlagType FLAGS_VEX_NO_OPERAND = (0b00ULL << 22);\n constexpr InstFlagType FLAGS_VEX_DST = (0b01ULL << 22);\n constexpr InstFlagType FLAGS_VEX_1ST_SRC = (0b10ULL << 22);\n constexpr InstFlagType FLAGS_VEX_2ND_SRC = (0b11ULL << 22);\n // Whether or not the instruction has a VSIB byte\n constexpr InstFlagType FLAGS_VEX_VSIB = (1ULL << 24);\n constexpr InstFlagType FLAGS_VEX_L_IGNORE = (1ULL << 25);\n constexpr InstFlagType FLAGS_VEX_L_0 = (1ULL << 26);\n constexpr InstFlagType FLAGS_VEX_L_1 = (1ULL << 27);\n\n constexpr InstFlagType FLAGS_REX_W_0 = (1ULL << 28);\n constexpr InstFlagType FLAGS_REX_W_1 = (1ULL << 29);\n\n constexpr InstFlagType FLAGS_CALL = (1ULL << 30);\n\n constexpr InstFlagType FLAGS_SIZE_DST_OFF = 58;\n constexpr InstFlagType FLAGS_SIZE_SRC_OFF = FLAGS_SIZE_DST_OFF + 3;\n\n constexpr InstFlagType SIZE_MASK = 0b111;\n constexpr InstFlagType SIZE_DEF = 0b000;\n constexpr InstFlagType SIZE_8BIT = 0b001;\n constexpr InstFlagType SIZE_16BIT = 0b010;\n constexpr InstFlagType SIZE_32BIT = 0b011;\n constexpr InstFlagType SIZE_64BIT = 0b100;\n constexpr InstFlagType SIZE_128BIT = 0b101;\n constexpr InstFlagType SIZE_256BIT = 0b110;\n constexpr InstFlagType SIZE_64BITDEF = 0b111; // Default mode is 64bit instead of typical 32bit\n\n#ifndef _WIN32\n constexpr uint32_t DEFAULT_SYSCALL_FLAGS = FLAGS_NO_OVERLAY;\n#else\n // Syscall ends a block on WIN32 because the instruction can update the CPU's RIP.\n constexpr uint32_t DEFAULT_SYSCALL_FLAGS = FLAGS_NO_OVERLAY | FLAGS_BLOCK_END;\n#endif\n\n constexpr InstFlagType GetSizeDstFlags(InstFlagType Flags) {\n return (Flags >> FLAGS_SIZE_DST_OFF) & SIZE_MASK;\n }\n constexpr InstFlagType GetSizeSrcFlags(InstFlagType Flags) {\n return (Flags >> FLAGS_SIZE_SRC_OFF) & SIZE_MASK;\n }\n\n constexpr InstFlagType GenFlagsDstSize(InstFlagType Size) {\n return Size << FLAGS_SIZE_DST_OFF;\n }\n constexpr InstFlagType GenFlagsSrcSize(InstFlagType Size) {\n return Size << FLAGS_SIZE_SRC_OFF;\n }\n constexpr InstFlagType GenFlagsSameSize(InstFlagType Size) {\n return (Size << FLAGS_SIZE_DST_OFF) | (Size << FLAGS_SIZE_SRC_OFF);\n }\n constexpr InstFlagType GenFlagsSizes(InstFlagType Dest, InstFlagType Src) {\n return (Dest << FLAGS_SIZE_DST_OFF) | (Src << FLAGS_SIZE_SRC_OFF);\n }\n\n// If it has an xmm subflag\n#define HAS_XMM_SUBFLAG(x, flag) \\\n (((x) & (FEXCore::X86Tables::InstFlags::FLAGS_XMM_FLAGS | (flag))) == (FEXCore::X86Tables::InstFlags::FLAGS_XMM_FLAGS | (flag)))\n\n// If it has non-xmm subflag\n#define HAS_NON_XMM_SUBFLAG(x, flag) (((x) & (FEXCore::X86Tables::InstFlags::FLAGS_XMM_FLAGS | (flag))) == (flag))\n} // namespace InstFlags\n\nconstexpr uint8_t OpToIndex(uint8_t Op) {\n switch (Op) {\n // Group 1\n case 0x80: return 0;\n case 0x81: return 1;\n case 0x82: return 2;\n case 0x83: return 3;\n // Group 2\n case 0xC0: return 0;\n case 0xC1: return 1;\n case 0xD0: return 2;\n case 0xD1: return 3;\n case 0xD2: return 4;\n case 0xD3: return 5;\n // Group 3\n case 0xF6: return 0;\n case 0xF7: return 1;\n // Group 4\n case 0xFE: return 0;\n // Group 5\n case 0xFF: return 0;\n // Group 11\n case 0xC6: return 0;\n case 0xC7: return 1;\n }\n return 0;\n}\n\nusing DecodedOp = const DecodedInst*;\nusing OpDispatchPtr = void (IR::OpDispatchBuilder::*)(DecodedOp);\n\nunion OpDispatchPtrWrapper {\n OpDispatchPtr OpDispatch;\n const struct X86InstInfo* Indirect;\n};\n\nstruct X86InstInfo {\n const char* Name;\n InstType Type;\n InstFlags::InstFlagType Flags; ///< Must be larger than InstFlags enum\n uint8_t MoreBytes;\n OpDispatchPtrWrapper OpcodeDispatcher;\n\n bool operator==(const X86InstInfo& b) const {\n if (strcmp(Name, b.Name) != 0 || Type != b.Type || Flags != b.Flags || MoreBytes != b.MoreBytes) {\n return false;\n }\n\n // We don't care if the opcode dispatcher differs\n return true;\n }\n bool operator!=(const X86InstInfo& b) const {\n return !operator==(b);\n }\n};\n\nstatic_assert(std::is_trivially_copyable_v);\n\nconstexpr size_t MAX_PRIMARY_TABLE_SIZE = 256;\nconstexpr size_t MAX_SECOND_TABLE_SIZE = 256;\nconstexpr size_t MAX_REP_MOD_TABLE_SIZE = 256;\nconstexpr size_t MAX_REPNE_MOD_TABLE_SIZE = 256;\nconstexpr size_t MAX_OPSIZE_MOD_TABLE_SIZE = 256;\n// 6 (groups) | 6 (max indexes) | 8 ops = 0b111'111'111 = 9 bits\nconstexpr size_t MAX_INST_GROUP_TABLE_SIZE = 512;\n// 12 (groups) | 3(max indexes) | 8 ops = 0b1111'11'111 = 9 bits\nconstexpr size_t MAX_INST_SECOND_GROUP_TABLE_SIZE = 512;\nconstexpr size_t MAX_X87_TABLE_SIZE = 1 << 11;\nconstexpr size_t MAX_SECOND_MODRM_TABLE_SIZE = 32;\n// (3 bit prefixes) | 8 bit opcode\nconstexpr size_t MAX_0F_38_TABLE_SIZE = (1 << 11);\n// 1 REX | 1 prefixes | 8 bit opcode\nconstexpr size_t MAX_0F_3A_TABLE_SIZE = (1 << 11);\nconstexpr size_t MAX_3DNOW_TABLE_SIZE = 256;\n// VEX\n// map_select(2 bits for now) | vex.pp (2 bits) | opcode (8bit)\nconstexpr size_t MAX_VEX_TABLE_SIZE = (1 << 13);\n// VEX group ops\n// group select (3 bits for now) | ModRM opcode (3 bits)\nconstexpr size_t MAX_VEX_GROUP_TABLE_SIZE = (1 << 7);\n\nextern const std::array BaseOps;\nextern const std::array SecondBaseOps;\nextern const std::array RepModOps;\nextern const std::array RepNEModOps;\nextern const std::array OpSizeModOps;\n\nextern const std::array PrimaryInstGroupOps;\nextern const std::array SecondInstGroupOps;\nextern const std::array SecondModRMTableOps;\nextern const std::array X87F80Ops;\nextern const std::array X87F64Ops;\nextern const std::array DDDNowOps;\nextern const std::array H0F38TableOps;\nextern const std::array H0F3ATableOps;\n\n// VEX\nextern const std::array VEXTableOps;\nextern const std::array VEXTableGroupOps;\n\nextern const std::array VEXTableOps_AVX128;\nextern const std::array VEXTableGroupOps_AVX128;\n\ntemplate\nstruct X86TablesInfoStruct {\n OpcodeType first;\n uint8_t second;\n X86InstInfo Info;\n};\nusing U8U8InfoStruct = X86TablesInfoStruct;\nusing U16U8InfoStruct = X86TablesInfoStruct;\n\ntemplate\nconstexpr static inline void GenerateTable(X86InstInfo* FinalTable, const X86TablesInfoStruct* LocalTable, size_t TableSize) {\n for (size_t j = 0; j < TableSize; ++j) {\n const X86TablesInfoStruct& Op = LocalTable[j];\n auto OpNum = Op.first;\n const X86InstInfo& Info = Op.Info;\n for (uint32_t i = 0; i < Op.second; ++i) {\n if (FinalTable[OpNum + i].Type != TYPE_UNKNOWN) {\n LOGMAN_MSG_A_FMT(\"Duplicate Entry {}->{}\", FinalTable[OpNum + i].Name, Info.Name);\n }\n if (FinalTable[OpNum + i].OpcodeDispatcher.OpDispatch) {\n LOGMAN_MSG_A_FMT(\"Already installed an OpcodeDispatcher for 0x{:x}\", OpNum + i);\n }\n FinalTable[OpNum + i] = Info;\n }\n }\n};\n\ntemplate\nconstexpr static inline void GenerateTableWithCopy(X86InstInfo* FinalTable, const X86TablesInfoStruct* LocalTable,\n size_t TableSize, const X86InstInfo* OtherLocal) {\n for (size_t j = 0; j < TableSize; ++j) {\n const X86TablesInfoStruct& Op = LocalTable[j];\n auto OpNum = Op.first;\n const X86InstInfo& Info = Op.Info;\n for (uint32_t i = 0; i < Op.second; ++i) {\n if (FinalTable[OpNum + i].Type != TYPE_UNKNOWN) {\n LOGMAN_MSG_A_FMT(\"Duplicate Entry {}->{}\", FinalTable[OpNum + i].Name, Info.Name);\n }\n if (Info.Type == TYPE_COPY_OTHER) {\n FinalTable[OpNum + i] = OtherLocal[OpNum + i];\n } else {\n FinalTable[OpNum + i] = Info;\n }\n }\n }\n};\n\ntemplate\nconstexpr static inline void GenerateX87Table(X86InstInfo* FinalTable, const X86TablesInfoStruct* LocalTable, size_t TableSize) {\n for (size_t j = 0; j < TableSize; ++j) {\n const X86TablesInfoStruct& Op = LocalTable[j];\n auto OpNum = Op.first;\n const X86InstInfo& Info = Op.Info;\n for (uint32_t i = 0; i < Op.second; ++i) {\n if (FinalTable[OpNum + i].Type != TYPE_UNKNOWN) {\n LOGMAN_MSG_A_FMT(\"Duplicate Entry {}->{}\", FinalTable[OpNum + i].Name, Info.Name);\n }\n if ((OpNum & 0b11'000'000) == 0b11'000'000) {\n // If the mod field is 0b11 then it is a regular op\n FinalTable[OpNum + i] = Info;\n } else {\n // If the mod field is !0b11 then this instruction is duplicated through the whole mod [0b00, 0b10] range\n // and the modrm.rm space because that is used part of the instruction encoding\n if ((OpNum & 0b11'000'000) != 0) {\n ERROR_AND_DIE_FMT(\"Only support mod field of zero in this path\");\n }\n for (uint16_t mod = 0b00'000'000; mod < 0b11'000'000; mod += 0b01'000'000) {\n for (uint16_t rm = 0b000; rm < 0b1'000; ++rm) {\n FinalTable[(OpNum | mod | rm) + i] = Info;\n }\n }\n }\n }\n }\n};\n\nFEX_DEFINE_ENUM_FMT_PASSTHROUGH(FEXCore::X86Tables::DecodedOperand::OpType);\n\n} // namespace FEXCore::X86Tables\n" }, { "path": "FEXCore/Source/Interface/Core/X86Tables/X87Tables.cpp", "content": "// SPDX-License-Identifier: MIT\n/*\n$info$\ntags: frontend|x86-tables\n$end_info$\n*/\n\n#include \"Interface/Core/X86Tables/X86Tables.h\"\n#include \"Interface/Core/OpcodeDispatcher.h\"\n\n#include \n\nnamespace FEXCore::X86Tables {\nusing namespace InstFlags;\nusing namespace IR;\n// Top bit indicating if it needs to be repeated with {0x40, 0x80} or'd in\n// All OPDReg versions need it\n#define OPDReg(op, reg) ((1 << 15) | ((op - 0xD8) << 8) | (reg << 3))\n#define OPD(op, modrmop) (((op - 0xD8) << 8) | modrmop)\nconstexpr std::array X87F64OpTable = {{\n {OPDReg(0xD8, 0) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FADDF64, OpSize::i32Bit, false, OpDispatchBuilder::OpResult::RES_ST0>},\n\n {OPDReg(0xD8, 1) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FMULF64, OpSize::i32Bit, false, OpDispatchBuilder::OpResult::RES_ST0>},\n\n {OPDReg(0xD8, 2) | 0x00, 8,\n &OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMIF64, OpSize::i32Bit, false, OpDispatchBuilder::FCOMIFlags::FLAGS_X87, false>},\n\n {OPDReg(0xD8, 3) | 0x00, 8,\n &OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMIF64, OpSize::i32Bit, false, OpDispatchBuilder::FCOMIFlags::FLAGS_X87, false>},\n\n {OPDReg(0xD8, 4) | 0x00, 8,\n &OpDispatchBuilder::Bind<&OpDispatchBuilder::FSUBF64, OpSize::i32Bit, false, false, OpDispatchBuilder::OpResult::RES_ST0>},\n\n {OPDReg(0xD8, 5) | 0x00, 8,\n &OpDispatchBuilder::Bind<&OpDispatchBuilder::FSUBF64, OpSize::i32Bit, false, true, OpDispatchBuilder::OpResult::RES_ST0>},\n\n {OPDReg(0xD8, 6) | 0x00, 8,\n &OpDispatchBuilder::Bind<&OpDispatchBuilder::FDIVF64, OpSize::i32Bit, false, false, OpDispatchBuilder::OpResult::RES_ST0>},\n\n {OPDReg(0xD8, 7) | 0x00, 8,\n &OpDispatchBuilder::Bind<&OpDispatchBuilder::FDIVF64, OpSize::i32Bit, false, true, OpDispatchBuilder::OpResult::RES_ST0>},\n\n {OPD(0xD8, 0xC0), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FADDF64, OpSize::f80Bit, false, OpDispatchBuilder::OpResult::RES_ST0>},\n {OPD(0xD8, 0xC8), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FMULF64, OpSize::f80Bit, false, OpDispatchBuilder::OpResult::RES_ST0>},\n {OPD(0xD8, 0xD0), 8,\n &OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMIF64, OpSize::f80Bit, false, OpDispatchBuilder::FCOMIFlags::FLAGS_X87, false>},\n {OPD(0xD8, 0xD8), 8,\n &OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMIF64, OpSize::f80Bit, false, OpDispatchBuilder::FCOMIFlags::FLAGS_X87, false>},\n {OPD(0xD8, 0xE0), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FSUBF64, OpSize::f80Bit, false, false, OpDispatchBuilder::OpResult::RES_ST0>},\n {OPD(0xD8, 0xE8), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FSUBF64, OpSize::f80Bit, false, true, OpDispatchBuilder::OpResult::RES_ST0>},\n {OPD(0xD8, 0xF0), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FDIVF64, OpSize::f80Bit, false, false, OpDispatchBuilder::OpResult::RES_ST0>},\n {OPD(0xD8, 0xF8), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FDIVF64, OpSize::f80Bit, false, true, OpDispatchBuilder::OpResult::RES_ST0>},\n\n {OPDReg(0xD9, 0) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FLDF64, OpSize::i32Bit>},\n\n // 1 = Invalid\n\n {OPDReg(0xD9, 2) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FST, OpSize::i32Bit>},\n\n {OPDReg(0xD9, 3) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FST, OpSize::i32Bit>},\n\n {OPDReg(0xD9, 4) | 0x00, 8, &OpDispatchBuilder::X87LDENVF64},\n\n {OPDReg(0xD9, 5) | 0x00, 8, &OpDispatchBuilder::X87FLDCWF64},\n\n {OPDReg(0xD9, 6) | 0x00, 8, &OpDispatchBuilder::X87FNSTENV},\n\n {OPDReg(0xD9, 7) | 0x00, 8, &OpDispatchBuilder::X87FSTCW},\n\n {OPD(0xD9, 0xC0), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FLDFromStack>},\n {OPD(0xD9, 0xC8), 8, &OpDispatchBuilder::FXCH},\n {OPD(0xD9, 0xD0), 1, &OpDispatchBuilder::NOPOp}, // FNOP\n // D1 = Invalid\n // D8 = Invalid\n {OPD(0xD9, 0xE0), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87OpHelper, OP_F80STACKCHANGESIGN, false>},\n {OPD(0xD9, 0xE1), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87OpHelper, OP_F80STACKABS, false>},\n // E2 = Invalid\n {OPD(0xD9, 0xE4), 1, &OpDispatchBuilder::FTSTF64},\n {OPD(0xD9, 0xE5), 1, &OpDispatchBuilder::X87FXAM},\n // E6 = Invalid\n {OPD(0xD9, 0xE8), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FLDF64_Const, 0x3FF0000000000000>}, // 1.0\n {OPD(0xD9, 0xE9), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FLDF64_Const, 0x400A934F0979A372>}, // log2l(10)\n {OPD(0xD9, 0xEA), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FLDF64_Const, 0x3FF71547652B82FE>}, // log2l(e)\n {OPD(0xD9, 0xEB), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FLDF64_Const, 0x400921FB54442D18>}, // pi\n {OPD(0xD9, 0xEC), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FLDF64_Const, 0x3FD34413509F79FF>}, // log10l(2)\n {OPD(0xD9, 0xED), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FLDF64_Const, 0x3FE62E42FEFA39EF>}, // log(2)\n {OPD(0xD9, 0xEE), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FLDF64_Const, 0>}, // 0.0\n\n // EF = Invalid\n {OPD(0xD9, 0xF0), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87OpHelper, OP_F80F2XM1STACK, false>},\n {OPD(0xD9, 0xF1), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87FYL2X, false>},\n {OPD(0xD9, 0xF2), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87OpHelper, OP_F80PTANSTACK, true>},\n {OPD(0xD9, 0xF3), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87OpHelper, OP_F80ATANSTACK, false>},\n {OPD(0xD9, 0xF4), 1, &OpDispatchBuilder::X87FXTRACTF64},\n {OPD(0xD9, 0xF5), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87OpHelper, OP_F80FPREM1STACK, true>},\n {OPD(0xD9, 0xF6), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87ModifySTP, false>},\n {OPD(0xD9, 0xF7), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87ModifySTP, true>},\n {OPD(0xD9, 0xF8), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87OpHelper, OP_F80FPREMSTACK, true>},\n {OPD(0xD9, 0xF9), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87FYL2X, true>},\n {OPD(0xD9, 0xFA), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87OpHelper, OP_F80SQRTSTACK, false>},\n {OPD(0xD9, 0xFB), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87OpHelper, OP_F80SINCOSSTACK, true>},\n {OPD(0xD9, 0xFC), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87OpHelper, OP_F80ROUNDSTACK, false>},\n {OPD(0xD9, 0xFD), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87OpHelper, OP_F80SCALESTACK, false>},\n {OPD(0xD9, 0xFE), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87OpHelper, OP_F80SINSTACK, true>},\n {OPD(0xD9, 0xFF), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87OpHelper, OP_F80COSSTACK, true>},\n\n {OPDReg(0xDA, 0) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FADDF64, OpSize::i32Bit, true, OpDispatchBuilder::OpResult::RES_ST0>},\n\n {OPDReg(0xDA, 1) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FMULF64, OpSize::i32Bit, true, OpDispatchBuilder::OpResult::RES_ST0>},\n\n {OPDReg(0xDA, 2) | 0x00, 8,\n &OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMIF64, OpSize::i32Bit, true, OpDispatchBuilder::FCOMIFlags::FLAGS_X87, false>},\n\n {OPDReg(0xDA, 3) | 0x00, 8,\n &OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMIF64, OpSize::i32Bit, true, OpDispatchBuilder::FCOMIFlags::FLAGS_X87, false>},\n\n {OPDReg(0xDA, 4) | 0x00, 8,\n &OpDispatchBuilder::Bind<&OpDispatchBuilder::FSUBF64, OpSize::i32Bit, true, false, OpDispatchBuilder::OpResult::RES_ST0>},\n\n {OPDReg(0xDA, 5) | 0x00, 8,\n &OpDispatchBuilder::Bind<&OpDispatchBuilder::FSUBF64, OpSize::i32Bit, true, true, OpDispatchBuilder::OpResult::RES_ST0>},\n\n {OPDReg(0xDA, 6) | 0x00, 8,\n &OpDispatchBuilder::Bind<&OpDispatchBuilder::FDIVF64, OpSize::i32Bit, true, false, OpDispatchBuilder::OpResult::RES_ST0>},\n\n {OPDReg(0xDA, 7) | 0x00, 8,\n &OpDispatchBuilder::Bind<&OpDispatchBuilder::FDIVF64, OpSize::i32Bit, true, true, OpDispatchBuilder::OpResult::RES_ST0>},\n\n {OPD(0xDA, 0xC0), 8, &OpDispatchBuilder::X87FCMOV},\n {OPD(0xDA, 0xC8), 8, &OpDispatchBuilder::X87FCMOV},\n {OPD(0xDA, 0xD0), 8, &OpDispatchBuilder::X87FCMOV},\n {OPD(0xDA, 0xD8), 8, &OpDispatchBuilder::X87FCMOV},\n // E0 = Invalid\n // E8 = Invalid\n {OPD(0xDA, 0xE9), 1,\n &OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMIF64, OpSize::f80Bit, false, OpDispatchBuilder::FCOMIFlags::FLAGS_X87, true>},\n // EA = Invalid\n // F0 = Invalid\n // F8 = Invalid\n\n {OPDReg(0xDB, 0) | 0x00, 8, &OpDispatchBuilder::FILDF64},\n\n {OPDReg(0xDB, 1) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FISTF64, true>},\n\n {OPDReg(0xDB, 2) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FISTF64, false>},\n\n {OPDReg(0xDB, 3) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FISTF64, false>},\n\n // 4 = Invalid\n\n {OPDReg(0xDB, 5) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FLDF64, OpSize::f80Bit>},\n\n // 6 = Invalid\n\n {OPDReg(0xDB, 7) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FST, OpSize::f80Bit>},\n\n\n {OPD(0xDB, 0xC0), 8, &OpDispatchBuilder::X87FCMOV},\n {OPD(0xDB, 0xC8), 8, &OpDispatchBuilder::X87FCMOV},\n {OPD(0xDB, 0xD0), 8, &OpDispatchBuilder::X87FCMOV},\n {OPD(0xDB, 0xD8), 8, &OpDispatchBuilder::X87FCMOV},\n // E0 = Invalid\n {OPD(0xDB, 0xE2), 1, &OpDispatchBuilder::FNCLEX},\n {OPD(0xDB, 0xE3), 1, &OpDispatchBuilder::FNINIT},\n // E4 = Invalid\n {OPD(0xDB, 0xE8), 8,\n &OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMIF64, OpSize::f80Bit, false, OpDispatchBuilder::FCOMIFlags::FLAGS_RFLAGS, false>},\n {OPD(0xDB, 0xF0), 8,\n &OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMIF64, OpSize::f80Bit, false, OpDispatchBuilder::FCOMIFlags::FLAGS_RFLAGS, false>},\n\n // F8 = Invalid\n\n {OPDReg(0xDC, 0) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FADDF64, OpSize::i64Bit, false, OpDispatchBuilder::OpResult::RES_ST0>},\n\n {OPDReg(0xDC, 1) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FMULF64, OpSize::i64Bit, false, OpDispatchBuilder::OpResult::RES_ST0>},\n\n {OPDReg(0xDC, 2) | 0x00, 8,\n &OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMIF64, OpSize::i64Bit, false, OpDispatchBuilder::FCOMIFlags::FLAGS_X87, false>},\n\n {OPDReg(0xDC, 3) | 0x00, 8,\n &OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMIF64, OpSize::i64Bit, false, OpDispatchBuilder::FCOMIFlags::FLAGS_X87, false>},\n\n {OPDReg(0xDC, 4) | 0x00, 8,\n &OpDispatchBuilder::Bind<&OpDispatchBuilder::FSUBF64, OpSize::i64Bit, false, false, OpDispatchBuilder::OpResult::RES_ST0>},\n\n {OPDReg(0xDC, 5) | 0x00, 8,\n &OpDispatchBuilder::Bind<&OpDispatchBuilder::FSUBF64, OpSize::i64Bit, false, true, OpDispatchBuilder::OpResult::RES_ST0>},\n\n {OPDReg(0xDC, 6) | 0x00, 8,\n &OpDispatchBuilder::Bind<&OpDispatchBuilder::FDIVF64, OpSize::i64Bit, false, false, OpDispatchBuilder::OpResult::RES_ST0>},\n\n {OPDReg(0xDC, 7) | 0x00, 8,\n &OpDispatchBuilder::Bind<&OpDispatchBuilder::FDIVF64, OpSize::i64Bit, false, true, OpDispatchBuilder::OpResult::RES_ST0>},\n\n {OPD(0xDC, 0xC0), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FADDF64, OpSize::f80Bit, false, OpDispatchBuilder::OpResult::RES_STI>},\n {OPD(0xDC, 0xC8), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FMULF64, OpSize::f80Bit, false, OpDispatchBuilder::OpResult::RES_STI>},\n {OPD(0xDC, 0xD0), 8,\n &OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMIF64, OpSize::f80Bit, false, OpDispatchBuilder::FCOMIFlags::FLAGS_X87, false>},\n {OPD(0xDC, 0xD8), 8,\n &OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMIF64, OpSize::f80Bit, false, OpDispatchBuilder::FCOMIFlags::FLAGS_X87, false>},\n {OPD(0xDC, 0xE0), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FSUBF64, OpSize::f80Bit, false, true, OpDispatchBuilder::OpResult::RES_STI>},\n {OPD(0xDC, 0xE8), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FSUBF64, OpSize::f80Bit, false, false, OpDispatchBuilder::OpResult::RES_STI>},\n {OPD(0xDC, 0xF0), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FDIVF64, OpSize::f80Bit, false, true, OpDispatchBuilder::OpResult::RES_STI>},\n {OPD(0xDC, 0xF8), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FDIVF64, OpSize::f80Bit, false, false, OpDispatchBuilder::OpResult::RES_STI>},\n\n {OPDReg(0xDD, 0) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FLDF64, OpSize::i64Bit>},\n\n {OPDReg(0xDD, 1) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FISTF64, true>},\n\n {OPDReg(0xDD, 2) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FST, OpSize::i64Bit>},\n\n {OPDReg(0xDD, 3) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FST, OpSize::i64Bit>},\n\n {OPDReg(0xDD, 4) | 0x00, 8, &OpDispatchBuilder::X87FRSTOR},\n\n // 5 = Invalid\n {OPDReg(0xDD, 6) | 0x00, 8, &OpDispatchBuilder::X87FNSAVE},\n\n {OPDReg(0xDD, 7) | 0x00, 8, &OpDispatchBuilder::X87FNSTSW},\n\n {OPD(0xDD, 0xC0), 8, &OpDispatchBuilder::X87FFREE},\n {OPD(0xDD, 0xC8), 8, &OpDispatchBuilder::FXCH},\n {OPD(0xDD, 0xD0), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FSTToStack>}, // register-register from regular X87\n {OPD(0xDD, 0xD8), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FSTToStack>}, //^\n\n {OPD(0xDD, 0xE0), 8,\n &OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMIF64, OpSize::f80Bit, false, OpDispatchBuilder::FCOMIFlags::FLAGS_X87, false>},\n {OPD(0xDD, 0xE8), 8,\n &OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMIF64, OpSize::f80Bit, false, OpDispatchBuilder::FCOMIFlags::FLAGS_X87, false>},\n\n {OPDReg(0xDE, 0) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FADDF64, OpSize::i16Bit, true, OpDispatchBuilder::OpResult::RES_ST0>},\n\n {OPDReg(0xDE, 1) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FMULF64, OpSize::i16Bit, true, OpDispatchBuilder::OpResult::RES_ST0>},\n\n {OPDReg(0xDE, 2) | 0x00, 8,\n &OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMIF64, OpSize::i16Bit, true, OpDispatchBuilder::FCOMIFlags::FLAGS_X87, false>},\n\n {OPDReg(0xDE, 3) | 0x00, 8,\n &OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMIF64, OpSize::i16Bit, true, OpDispatchBuilder::FCOMIFlags::FLAGS_X87, false>},\n\n {OPDReg(0xDE, 4) | 0x00, 8,\n &OpDispatchBuilder::Bind<&OpDispatchBuilder::FSUBF64, OpSize::i16Bit, true, false, OpDispatchBuilder::OpResult::RES_ST0>},\n\n {OPDReg(0xDE, 5) | 0x00, 8,\n &OpDispatchBuilder::Bind<&OpDispatchBuilder::FSUBF64, OpSize::i16Bit, true, true, OpDispatchBuilder::OpResult::RES_ST0>},\n\n {OPDReg(0xDE, 6) | 0x00, 8,\n &OpDispatchBuilder::Bind<&OpDispatchBuilder::FDIVF64, OpSize::i16Bit, true, false, OpDispatchBuilder::OpResult::RES_ST0>},\n\n {OPDReg(0xDE, 7) | 0x00, 8,\n &OpDispatchBuilder::Bind<&OpDispatchBuilder::FDIVF64, OpSize::i16Bit, true, true, OpDispatchBuilder::OpResult::RES_ST0>},\n\n {OPD(0xDE, 0xC0), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FADDF64, OpSize::f80Bit, false, OpDispatchBuilder::OpResult::RES_STI>},\n {OPD(0xDE, 0xC8), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FMULF64, OpSize::f80Bit, false, OpDispatchBuilder::OpResult::RES_STI>},\n {OPD(0xDE, 0xD0), 8,\n &OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMIF64, OpSize::f80Bit, false, OpDispatchBuilder::FCOMIFlags::FLAGS_X87, false>},\n {OPD(0xDE, 0xD9), 1,\n &OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMIF64, OpSize::f80Bit, false, OpDispatchBuilder::FCOMIFlags::FLAGS_X87, true>},\n {OPD(0xDE, 0xE0), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FSUBF64, OpSize::f80Bit, false, true, OpDispatchBuilder::OpResult::RES_STI>},\n {OPD(0xDE, 0xE8), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FSUBF64, OpSize::f80Bit, false, false, OpDispatchBuilder::OpResult::RES_STI>},\n {OPD(0xDE, 0xF0), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FDIVF64, OpSize::f80Bit, false, true, OpDispatchBuilder::OpResult::RES_STI>},\n {OPD(0xDE, 0xF8), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FDIVF64, OpSize::f80Bit, false, false, OpDispatchBuilder::OpResult::RES_STI>},\n\n {OPDReg(0xDF, 0) | 0x00, 8, &OpDispatchBuilder::FILDF64},\n\n {OPDReg(0xDF, 1) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FISTF64, true>},\n\n {OPDReg(0xDF, 2) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FISTF64, false>},\n\n {OPDReg(0xDF, 3) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FISTF64, false>},\n\n {OPDReg(0xDF, 4) | 0x00, 8, &OpDispatchBuilder::FBLDF64},\n\n {OPDReg(0xDF, 5) | 0x00, 8, &OpDispatchBuilder::FILDF64},\n\n {OPDReg(0xDF, 6) | 0x00, 8, &OpDispatchBuilder::FBSTPF64},\n\n {OPDReg(0xDF, 7) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FISTF64, false>},\n\n // XXX: This should also set the x87 tag bits to empty\n // We don't support this currently, so just pop the stack\n {OPD(0xDF, 0xC0), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87ModifySTP, true>},\n {OPD(0xDF, 0xC8), 8, &OpDispatchBuilder::FXCH},\n {OPD(0xDF, 0xD0), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FSTToStack>},\n {OPD(0xDF, 0xD8), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FSTToStack>},\n\n {OPD(0xDF, 0xE0), 8, &OpDispatchBuilder::X87FNSTSW},\n {OPD(0xDF, 0xE8), 8,\n &OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMIF64, OpSize::f80Bit, false, OpDispatchBuilder::FCOMIFlags::FLAGS_RFLAGS, false>},\n {OPD(0xDF, 0xF0), 8,\n &OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMIF64, OpSize::f80Bit, false, OpDispatchBuilder::FCOMIFlags::FLAGS_RFLAGS, false>},\n}};\n\nconstexpr std::array X87F80OpTable = {{\n {OPDReg(0xD8, 0) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FADD, OpSize::i32Bit, false, OpDispatchBuilder::OpResult::RES_ST0>},\n\n {OPDReg(0xD8, 1) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FMUL, OpSize::i32Bit, false, OpDispatchBuilder::OpResult::RES_ST0>},\n\n {OPDReg(0xD8, 2) | 0x00, 8,\n &OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMI, OpSize::i32Bit, false, OpDispatchBuilder::FCOMIFlags::FLAGS_X87, false>},\n\n {OPDReg(0xD8, 3) | 0x00, 8,\n &OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMI, OpSize::i32Bit, false, OpDispatchBuilder::FCOMIFlags::FLAGS_X87, false>},\n\n {OPDReg(0xD8, 4) | 0x00, 8,\n &OpDispatchBuilder::Bind<&OpDispatchBuilder::FSUB, OpSize::i32Bit, false, false, OpDispatchBuilder::OpResult::RES_ST0>},\n\n {OPDReg(0xD8, 5) | 0x00, 8,\n &OpDispatchBuilder::Bind<&OpDispatchBuilder::FSUB, OpSize::i32Bit, false, true, OpDispatchBuilder::OpResult::RES_ST0>},\n\n {OPDReg(0xD8, 6) | 0x00, 8,\n &OpDispatchBuilder::Bind<&OpDispatchBuilder::FDIV, OpSize::i32Bit, false, false, OpDispatchBuilder::OpResult::RES_ST0>},\n\n {OPDReg(0xD8, 7) | 0x00, 8,\n &OpDispatchBuilder::Bind<&OpDispatchBuilder::FDIV, OpSize::i32Bit, false, true, OpDispatchBuilder::OpResult::RES_ST0>},\n\n {OPD(0xD8, 0xC0), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FADD, OpSize::f80Bit, false, OpDispatchBuilder::OpResult::RES_ST0>},\n {OPD(0xD8, 0xC8), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FMUL, OpSize::f80Bit, false, OpDispatchBuilder::OpResult::RES_ST0>},\n {OPD(0xD8, 0xD0), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMI, OpSize::f80Bit, false, OpDispatchBuilder::FCOMIFlags::FLAGS_X87, false>},\n {OPD(0xD8, 0xD8), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMI, OpSize::f80Bit, false, OpDispatchBuilder::FCOMIFlags::FLAGS_X87, false>},\n {OPD(0xD8, 0xE0), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FSUB, OpSize::f80Bit, false, false, OpDispatchBuilder::OpResult::RES_ST0>},\n {OPD(0xD8, 0xE8), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FSUB, OpSize::f80Bit, false, true, OpDispatchBuilder::OpResult::RES_ST0>},\n {OPD(0xD8, 0xF0), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FDIV, OpSize::f80Bit, false, false, OpDispatchBuilder::OpResult::RES_ST0>},\n {OPD(0xD8, 0xF8), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FDIV, OpSize::f80Bit, false, true, OpDispatchBuilder::OpResult::RES_ST0>},\n\n {OPDReg(0xD9, 0) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FLD, OpSize::i32Bit>},\n\n // 1 = Invalid\n\n {OPDReg(0xD9, 2) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FST, OpSize::i32Bit>},\n\n {OPDReg(0xD9, 3) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FST, OpSize::i32Bit>},\n\n {OPDReg(0xD9, 4) | 0x00, 8, &OpDispatchBuilder::X87LDENV},\n\n {OPDReg(0xD9, 5) | 0x00, 8, &OpDispatchBuilder::X87FLDCW}, // XXX: stubbed FLDCW\n\n {OPDReg(0xD9, 6) | 0x00, 8, &OpDispatchBuilder::X87FNSTENV},\n\n {OPDReg(0xD9, 7) | 0x00, 8, &OpDispatchBuilder::X87FSTCW},\n\n {OPD(0xD9, 0xC0), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FLDFromStack>},\n {OPD(0xD9, 0xC8), 8, &OpDispatchBuilder::FXCH},\n {OPD(0xD9, 0xD0), 1, &OpDispatchBuilder::NOPOp}, // FNOP\n // D1 = Invalid\n // D8 = Invalid\n {OPD(0xD9, 0xE0), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87OpHelper, OP_F80STACKCHANGESIGN, false>},\n {OPD(0xD9, 0xE1), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87OpHelper, OP_F80STACKABS, false>},\n // E2 = Invalid\n {OPD(0xD9, 0xE4), 1, &OpDispatchBuilder::FTST},\n {OPD(0xD9, 0xE5), 1, &OpDispatchBuilder::X87FXAM},\n // E6 = Invalid\n {OPD(0xD9, 0xE8), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FLD_Const, NamedVectorConstant::NAMED_VECTOR_X87_ONE>}, // 1.0\n {OPD(0xD9, 0xE9), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FLD_Const, NamedVectorConstant::NAMED_VECTOR_X87_LOG2_10>}, // log2l(10)\n {OPD(0xD9, 0xEA), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FLD_Const, NamedVectorConstant::NAMED_VECTOR_X87_LOG2_E>}, // log2l(e)\n {OPD(0xD9, 0xEB), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FLD_Const, NamedVectorConstant::NAMED_VECTOR_X87_PI>}, // pi\n {OPD(0xD9, 0xEC), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FLD_Const, NamedVectorConstant::NAMED_VECTOR_X87_LOG10_2>}, // log10l(2)\n {OPD(0xD9, 0xED), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FLD_Const, NamedVectorConstant::NAMED_VECTOR_X87_LOG_2>}, // log(2)\n {OPD(0xD9, 0xEE), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FLD_Const, NamedVectorConstant::NAMED_VECTOR_ZERO>}, // 0.0\n\n // EF = Invalid\n {OPD(0xD9, 0xF0), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87OpHelper, OP_F80F2XM1STACK, false>},\n {OPD(0xD9, 0xF1), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87FYL2X, false>},\n {OPD(0xD9, 0xF2), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87OpHelper, OP_F80PTANSTACK, true>},\n {OPD(0xD9, 0xF3), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87OpHelper, OP_F80ATANSTACK, false>},\n {OPD(0xD9, 0xF4), 1, &OpDispatchBuilder::X87FXTRACT},\n {OPD(0xD9, 0xF5), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87OpHelper, OP_F80FPREM1STACK, true>},\n {OPD(0xD9, 0xF6), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87ModifySTP, false>},\n {OPD(0xD9, 0xF7), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87ModifySTP, true>},\n {OPD(0xD9, 0xF8), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87OpHelper, OP_F80FPREMSTACK, true>},\n {OPD(0xD9, 0xF9), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87FYL2X, true>},\n {OPD(0xD9, 0xFA), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87OpHelper, OP_F80SQRTSTACK, false>},\n {OPD(0xD9, 0xFB), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87OpHelper, OP_F80SINCOSSTACK, true>},\n {OPD(0xD9, 0xFC), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87OpHelper, OP_F80ROUNDSTACK, false>},\n {OPD(0xD9, 0xFD), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87OpHelper, OP_F80SCALESTACK, false>},\n {OPD(0xD9, 0xFE), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87OpHelper, OP_F80SINSTACK, true>},\n {OPD(0xD9, 0xFF), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87OpHelper, OP_F80COSSTACK, true>},\n\n {OPDReg(0xDA, 0) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FADD, OpSize::i32Bit, true, OpDispatchBuilder::OpResult::RES_ST0>},\n\n {OPDReg(0xDA, 1) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FMUL, OpSize::i32Bit, true, OpDispatchBuilder::OpResult::RES_ST0>},\n\n {OPDReg(0xDA, 2) | 0x00, 8,\n &OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMI, OpSize::i32Bit, true, OpDispatchBuilder::FCOMIFlags::FLAGS_X87, false>},\n\n {OPDReg(0xDA, 3) | 0x00, 8,\n &OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMI, OpSize::i32Bit, true, OpDispatchBuilder::FCOMIFlags::FLAGS_X87, false>},\n\n {OPDReg(0xDA, 4) | 0x00, 8,\n &OpDispatchBuilder::Bind<&OpDispatchBuilder::FSUB, OpSize::i32Bit, true, false, OpDispatchBuilder::OpResult::RES_ST0>},\n\n {OPDReg(0xDA, 5) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FSUB, OpSize::i32Bit, true, true, OpDispatchBuilder::OpResult::RES_ST0>},\n\n {OPDReg(0xDA, 6) | 0x00, 8,\n &OpDispatchBuilder::Bind<&OpDispatchBuilder::FDIV, OpSize::i32Bit, true, false, OpDispatchBuilder::OpResult::RES_ST0>},\n\n {OPDReg(0xDA, 7) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FDIV, OpSize::i32Bit, true, true, OpDispatchBuilder::OpResult::RES_ST0>},\n\n {OPD(0xDA, 0xC0), 8, &OpDispatchBuilder::X87FCMOV},\n {OPD(0xDA, 0xC8), 8, &OpDispatchBuilder::X87FCMOV},\n {OPD(0xDA, 0xD0), 8, &OpDispatchBuilder::X87FCMOV},\n {OPD(0xDA, 0xD8), 8, &OpDispatchBuilder::X87FCMOV},\n // E0 = Invalid\n // E8 = Invalid\n {OPD(0xDA, 0xE9), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMI, OpSize::f80Bit, false, OpDispatchBuilder::FCOMIFlags::FLAGS_X87, true>},\n // EA = Invalid\n // F0 = Invalid\n // F8 = Invalid\n\n {OPDReg(0xDB, 0) | 0x00, 8, &OpDispatchBuilder::FILD},\n\n {OPDReg(0xDB, 1) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FIST, true>},\n\n {OPDReg(0xDB, 2) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FIST, false>},\n\n {OPDReg(0xDB, 3) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FIST, false>},\n\n // 4 = Invalid\n\n {OPDReg(0xDB, 5) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FLD, OpSize::f80Bit>},\n\n // 6 = Invalid\n\n {OPDReg(0xDB, 7) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FST, OpSize::f80Bit>},\n\n\n {OPD(0xDB, 0xC0), 8, &OpDispatchBuilder::X87FCMOV},\n {OPD(0xDB, 0xC8), 8, &OpDispatchBuilder::X87FCMOV},\n {OPD(0xDB, 0xD0), 8, &OpDispatchBuilder::X87FCMOV},\n {OPD(0xDB, 0xD8), 8, &OpDispatchBuilder::X87FCMOV},\n // E0 = Invalid\n {OPD(0xDB, 0xE2), 1, &OpDispatchBuilder::FNCLEX},\n {OPD(0xDB, 0xE3), 1, &OpDispatchBuilder::FNINIT},\n // E4 = Invalid\n {OPD(0xDB, 0xE8), 8,\n &OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMI, OpSize::f80Bit, false, OpDispatchBuilder::FCOMIFlags::FLAGS_RFLAGS, false>},\n {OPD(0xDB, 0xF0), 8,\n &OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMI, OpSize::f80Bit, false, OpDispatchBuilder::FCOMIFlags::FLAGS_RFLAGS, false>},\n\n // F8 = Invalid\n\n {OPDReg(0xDC, 0) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FADD, OpSize::i64Bit, false, OpDispatchBuilder::OpResult::RES_ST0>},\n\n {OPDReg(0xDC, 1) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FMUL, OpSize::i64Bit, false, OpDispatchBuilder::OpResult::RES_ST0>},\n\n {OPDReg(0xDC, 2) | 0x00, 8,\n &OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMI, OpSize::i64Bit, false, OpDispatchBuilder::FCOMIFlags::FLAGS_X87, false>},\n\n {OPDReg(0xDC, 3) | 0x00, 8,\n &OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMI, OpSize::i64Bit, false, OpDispatchBuilder::FCOMIFlags::FLAGS_X87, false>},\n\n {OPDReg(0xDC, 4) | 0x00, 8,\n &OpDispatchBuilder::Bind<&OpDispatchBuilder::FSUB, OpSize::i64Bit, false, false, OpDispatchBuilder::OpResult::RES_ST0>},\n\n {OPDReg(0xDC, 5) | 0x00, 8,\n &OpDispatchBuilder::Bind<&OpDispatchBuilder::FSUB, OpSize::i64Bit, false, true, OpDispatchBuilder::OpResult::RES_ST0>},\n\n {OPDReg(0xDC, 6) | 0x00, 8,\n &OpDispatchBuilder::Bind<&OpDispatchBuilder::FDIV, OpSize::i64Bit, false, false, OpDispatchBuilder::OpResult::RES_ST0>},\n\n {OPDReg(0xDC, 7) | 0x00, 8,\n &OpDispatchBuilder::Bind<&OpDispatchBuilder::FDIV, OpSize::i64Bit, false, true, OpDispatchBuilder::OpResult::RES_ST0>},\n\n {OPD(0xDC, 0xC0), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FADD, OpSize::f80Bit, false, OpDispatchBuilder::OpResult::RES_STI>},\n {OPD(0xDC, 0xC8), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FMUL, OpSize::f80Bit, false, OpDispatchBuilder::OpResult::RES_STI>},\n {OPD(0xDC, 0xD0), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMI, OpSize::f80Bit, false, OpDispatchBuilder::FCOMIFlags::FLAGS_X87, false>},\n {OPD(0xDC, 0xD8), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMI, OpSize::f80Bit, false, OpDispatchBuilder::FCOMIFlags::FLAGS_X87, false>},\n {OPD(0xDC, 0xE0), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FSUB, OpSize::f80Bit, false, true, OpDispatchBuilder::OpResult::RES_STI>},\n {OPD(0xDC, 0xE8), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FSUB, OpSize::f80Bit, false, false, OpDispatchBuilder::OpResult::RES_STI>},\n {OPD(0xDC, 0xF0), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FDIV, OpSize::f80Bit, false, true, OpDispatchBuilder::OpResult::RES_STI>},\n {OPD(0xDC, 0xF8), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FDIV, OpSize::f80Bit, false, false, OpDispatchBuilder::OpResult::RES_STI>},\n\n {OPDReg(0xDD, 0) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FLD, OpSize::i64Bit>},\n\n {OPDReg(0xDD, 1) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FIST, true>},\n\n {OPDReg(0xDD, 2) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FST, OpSize::i64Bit>},\n\n {OPDReg(0xDD, 3) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FST, OpSize::i64Bit>},\n\n {OPDReg(0xDD, 4) | 0x00, 8, &OpDispatchBuilder::X87FRSTOR},\n\n // 5 = Invalid\n {OPDReg(0xDD, 6) | 0x00, 8, &OpDispatchBuilder::X87FNSAVE},\n\n {OPDReg(0xDD, 7) | 0x00, 8, &OpDispatchBuilder::X87FNSTSW},\n\n {OPD(0xDD, 0xC0), 8, &OpDispatchBuilder::X87FFREE},\n {OPD(0xDD, 0xC8), 8, &OpDispatchBuilder::FXCH},\n {OPD(0xDD, 0xD0), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FSTToStack>},\n {OPD(0xDD, 0xD8), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FSTToStack>},\n\n {OPD(0xDD, 0xE0), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMI, OpSize::f80Bit, false, OpDispatchBuilder::FCOMIFlags::FLAGS_X87, false>},\n {OPD(0xDD, 0xE8), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMI, OpSize::f80Bit, false, OpDispatchBuilder::FCOMIFlags::FLAGS_X87, false>},\n\n {OPDReg(0xDE, 0) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FADD, OpSize::i16Bit, true, OpDispatchBuilder::OpResult::RES_ST0>},\n\n {OPDReg(0xDE, 1) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FMUL, OpSize::i16Bit, true, OpDispatchBuilder::OpResult::RES_ST0>},\n\n {OPDReg(0xDE, 2) | 0x00, 8,\n &OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMI, OpSize::i16Bit, true, OpDispatchBuilder::FCOMIFlags::FLAGS_X87, false>},\n\n {OPDReg(0xDE, 3) | 0x00, 8,\n &OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMI, OpSize::i16Bit, true, OpDispatchBuilder::FCOMIFlags::FLAGS_X87, false>},\n\n {OPDReg(0xDE, 4) | 0x00, 8,\n &OpDispatchBuilder::Bind<&OpDispatchBuilder::FSUB, OpSize::i16Bit, true, false, OpDispatchBuilder::OpResult::RES_ST0>},\n\n {OPDReg(0xDE, 5) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FSUB, OpSize::i16Bit, true, true, OpDispatchBuilder::OpResult::RES_ST0>},\n\n {OPDReg(0xDE, 6) | 0x00, 8,\n &OpDispatchBuilder::Bind<&OpDispatchBuilder::FDIV, OpSize::i16Bit, true, false, OpDispatchBuilder::OpResult::RES_ST0>},\n\n {OPDReg(0xDE, 7) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FDIV, OpSize::i16Bit, true, true, OpDispatchBuilder::OpResult::RES_ST0>},\n\n {OPD(0xDE, 0xC0), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FADD, OpSize::f80Bit, false, OpDispatchBuilder::OpResult::RES_STI>},\n {OPD(0xDE, 0xC8), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FMUL, OpSize::f80Bit, false, OpDispatchBuilder::OpResult::RES_STI>},\n {OPD(0xDE, 0xD0), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMI, OpSize::f80Bit, false, OpDispatchBuilder::FCOMIFlags::FLAGS_X87, false>},\n {OPD(0xDE, 0xD9), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMI, OpSize::f80Bit, false, OpDispatchBuilder::FCOMIFlags::FLAGS_X87, true>},\n {OPD(0xDE, 0xE0), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FSUB, OpSize::f80Bit, false, true, OpDispatchBuilder::OpResult::RES_STI>},\n {OPD(0xDE, 0xE8), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FSUB, OpSize::f80Bit, false, false, OpDispatchBuilder::OpResult::RES_STI>},\n {OPD(0xDE, 0xF0), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FDIV, OpSize::f80Bit, false, true, OpDispatchBuilder::OpResult::RES_STI>},\n {OPD(0xDE, 0xF8), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FDIV, OpSize::f80Bit, false, false, OpDispatchBuilder::OpResult::RES_STI>},\n\n {OPDReg(0xDF, 0) | 0x00, 8, &OpDispatchBuilder::FILD},\n\n {OPDReg(0xDF, 1) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FIST, true>},\n\n {OPDReg(0xDF, 2) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FIST, false>},\n\n {OPDReg(0xDF, 3) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FIST, false>},\n\n {OPDReg(0xDF, 4) | 0x00, 8, &OpDispatchBuilder::FBLD},\n\n {OPDReg(0xDF, 5) | 0x00, 8, &OpDispatchBuilder::FILD},\n\n {OPDReg(0xDF, 6) | 0x00, 8, &OpDispatchBuilder::FBSTP},\n\n {OPDReg(0xDF, 7) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FIST, false>},\n\n // XXX: This should also set the x87 tag bits to empty\n // We don't support this currently, so just pop the stack\n {OPD(0xDF, 0xC0), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87ModifySTP, true>},\n {OPD(0xDF, 0xC8), 8, &OpDispatchBuilder::FXCH},\n {OPD(0xDF, 0xD0), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FSTToStack>},\n {OPD(0xDF, 0xD8), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FSTToStack>},\n\n {OPD(0xDF, 0xE0), 8, &OpDispatchBuilder::X87FNSTSW},\n {OPD(0xDF, 0xE8), 8,\n &OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMI, OpSize::f80Bit, false, OpDispatchBuilder::FCOMIFlags::FLAGS_RFLAGS, false>},\n {OPD(0xDF, 0xF0), 8,\n &OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMI, OpSize::f80Bit, false, OpDispatchBuilder::FCOMIFlags::FLAGS_RFLAGS, false>},\n}};\n#undef OPD\n#undef OPDReg\n\nauto GenerateX87TableLambda = [](const auto DispatchTable) consteval {\n#define OPD(op, modrmop) (((op - 0xD8) << 8) | modrmop)\n#define OPDReg(op, reg) (((op - 0xD8) << 8) | (reg << 3))\n std::array Table{};\n constexpr U16U8InfoStruct X87OpTable[] = {\n // 0xD8\n {OPDReg(0xD8, 0), 1, X86InstInfo{\"FADD\", TYPE_X87, FLAGS_MODRM, 0}},\n {OPDReg(0xD8, 1), 1, X86InstInfo{\"FMUL\", TYPE_X87, FLAGS_MODRM, 0}},\n {OPDReg(0xD8, 2), 1, X86InstInfo{\"FCOM\", TYPE_X87, FLAGS_MODRM, 0}},\n {OPDReg(0xD8, 3), 1, X86InstInfo{\"FCOMP\", TYPE_X87, FLAGS_MODRM | FLAGS_POP, 0}},\n {OPDReg(0xD8, 4), 1, X86InstInfo{\"FSUB\", TYPE_X87, FLAGS_MODRM, 0}},\n {OPDReg(0xD8, 5), 1, X86InstInfo{\"FSUBR\", TYPE_X87, FLAGS_MODRM, 0}},\n {OPDReg(0xD8, 6), 1, X86InstInfo{\"FDIV\", TYPE_X87, FLAGS_MODRM, 0}},\n {OPDReg(0xD8, 7), 1, X86InstInfo{\"FDIVR\", TYPE_X87, FLAGS_MODRM, 0}},\n // / 0\n {OPD(0xD8, 0xC0), 8, X86InstInfo{\"FADD\", TYPE_X87, FLAGS_NONE, 0}},\n // / 1\n {OPD(0xD8, 0xC8), 8, X86InstInfo{\"FMUL\", TYPE_X87, FLAGS_NONE, 0}},\n // / 2\n {OPD(0xD8, 0xD0), 8, X86InstInfo{\"FCOM\", TYPE_X87, FLAGS_NONE, 0}},\n // / 3\n {OPD(0xD8, 0xD8), 8, X86InstInfo{\"FCOMP\", TYPE_X87, FLAGS_POP, 0}},\n // / 4\n {OPD(0xD8, 0xE0), 8, X86InstInfo{\"FSUB\", TYPE_X87, FLAGS_NONE, 0}},\n // / 5\n {OPD(0xD8, 0xE8), 8, X86InstInfo{\"FSUBR\", TYPE_X87, FLAGS_NONE, 0}},\n // / 6\n {OPD(0xD8, 0xF0), 8, X86InstInfo{\"FDIV\", TYPE_X87, FLAGS_NONE, 0}},\n // / 7\n {OPD(0xD8, 0xF8), 8, X86InstInfo{\"FDIVR\", TYPE_X87, FLAGS_NONE, 0}},\n // 0xD9\n {OPDReg(0xD9, 0), 1, X86InstInfo{\"FLD\", TYPE_INST, FLAGS_MODRM, 0}},\n {OPDReg(0xD9, 1), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPDReg(0xD9, 2), 1, X86InstInfo{\"FST\", TYPE_X87, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {OPDReg(0xD9, 3), 1, X86InstInfo{\"FSTP\", TYPE_X87, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_POP, 0}},\n {OPDReg(0xD9, 4), 1, X86InstInfo{\"FLDENV\", TYPE_X87, FLAGS_MODRM, 0}},\n {OPDReg(0xD9, 5), 1, X86InstInfo{\"FLDCW\", TYPE_X87, GenFlagsSameSize(SIZE_16BIT) | FLAGS_MODRM, 0}},\n {OPDReg(0xD9, 6), 1, X86InstInfo{\"FNSTENV\", TYPE_X87, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {OPDReg(0xD9, 7), 1, X86InstInfo{\"FNSTCW\", TYPE_INST, GenFlagsSameSize(SIZE_16BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n // / 0\n {OPD(0xD9, 0xC0), 8, X86InstInfo{\"FLD\", TYPE_INST, FLAGS_NONE, 0}},\n // / 1\n {OPD(0xD9, 0xC8), 8, X86InstInfo{\"FXCH\", TYPE_X87, FLAGS_NONE, 0}},\n // / 2\n {OPD(0xD9, 0xD0), 1, X86InstInfo{\"FNOP\", TYPE_X87, FLAGS_NONE, 0}},\n {OPD(0xD9, 0xD1), 7, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n // / 3\n {OPD(0xD9, 0xD8), 8, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n // / 4\n {OPD(0xD9, 0xE0), 1, X86InstInfo{\"FCHS\", TYPE_X87, FLAGS_NONE, 0}},\n {OPD(0xD9, 0xE1), 1, X86InstInfo{\"FABS\", TYPE_X87, FLAGS_NONE, 0}},\n {OPD(0xD9, 0xE2), 2, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(0xD9, 0xE4), 1, X86InstInfo{\"FTST\", TYPE_X87, FLAGS_NONE, 0}},\n {OPD(0xD9, 0xE5), 1, X86InstInfo{\"FXAM\", TYPE_INST, FLAGS_NONE, 0}},\n {OPD(0xD9, 0xE6), 2, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n // / 5\n {OPD(0xD9, 0xE8), 1, X86InstInfo{\"FLD1\", TYPE_X87, FLAGS_NONE, 0}},\n {OPD(0xD9, 0xE9), 1, X86InstInfo{\"FLDL2T\", TYPE_X87, FLAGS_NONE, 0}},\n {OPD(0xD9, 0xEA), 1, X86InstInfo{\"FLDL2E\", TYPE_X87, FLAGS_NONE, 0}},\n {OPD(0xD9, 0xEB), 1, X86InstInfo{\"FLDPI\", TYPE_X87, FLAGS_NONE, 0}},\n {OPD(0xD9, 0xEC), 1, X86InstInfo{\"FLDLG2\", TYPE_X87, FLAGS_NONE, 0}},\n {OPD(0xD9, 0xED), 1, X86InstInfo{\"FLDLN2\", TYPE_X87, FLAGS_NONE, 0}},\n {OPD(0xD9, 0xEE), 1, X86InstInfo{\"FLDZ\", TYPE_X87, FLAGS_NONE, 0}},\n {OPD(0xD9, 0xEF), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n // / 6\n {OPD(0xD9, 0xF0), 1, X86InstInfo{\"F2XM1\", TYPE_X87, FLAGS_NONE, 0}},\n {OPD(0xD9, 0xF1), 1, X86InstInfo{\"FYL2X\", TYPE_X87, FLAGS_NONE, 0}},\n {OPD(0xD9, 0xF2), 1, X86InstInfo{\"FPTAN\", TYPE_X87, FLAGS_NONE, 0}},\n {OPD(0xD9, 0xF3), 1, X86InstInfo{\"FPATAN\", TYPE_X87, FLAGS_NONE, 0}},\n {OPD(0xD9, 0xF4), 1, X86InstInfo{\"FXTRACT\", TYPE_X87, FLAGS_NONE, 0}},\n {OPD(0xD9, 0xF5), 1, X86InstInfo{\"FPREM1\", TYPE_X87, FLAGS_NONE, 0}},\n {OPD(0xD9, 0xF6), 1, X86InstInfo{\"FDECSTP\", TYPE_X87, FLAGS_POP, 0}},\n {OPD(0xD9, 0xF7), 1, X86InstInfo{\"FINCSTP\", TYPE_X87, FLAGS_POP, 0}},\n // / 7\n {OPD(0xD9, 0xF8), 1, X86InstInfo{\"FPREM\", TYPE_X87, FLAGS_NONE, 0}},\n {OPD(0xD9, 0xF9), 1, X86InstInfo{\"FYL2XP1\", TYPE_X87, FLAGS_NONE, 0}},\n {OPD(0xD9, 0xFA), 1, X86InstInfo{\"FSQRT\", TYPE_X87, FLAGS_NONE, 0}},\n {OPD(0xD9, 0xFB), 1, X86InstInfo{\"FSINCOS\", TYPE_X87, FLAGS_NONE, 0}},\n {OPD(0xD9, 0xFC), 1, X86InstInfo{\"FRNDINT\", TYPE_X87, FLAGS_NONE, 0}},\n {OPD(0xD9, 0xFD), 1, X86InstInfo{\"FSCALE\", TYPE_X87, FLAGS_NONE, 0}},\n {OPD(0xD9, 0xFE), 1, X86InstInfo{\"FSIN\", TYPE_X87, FLAGS_NONE, 0}},\n {OPD(0xD9, 0xFF), 1, X86InstInfo{\"FCOS\", TYPE_X87, FLAGS_NONE, 0}},\n // 0xDA\n {OPDReg(0xDA, 0), 1, X86InstInfo{\"FIADD\", TYPE_X87, GenFlagsSrcSize(SIZE_32BIT) | FLAGS_MODRM, 0}},\n {OPDReg(0xDA, 1), 1, X86InstInfo{\"FIMUL\", TYPE_X87, GenFlagsSrcSize(SIZE_32BIT) | FLAGS_MODRM, 0}},\n {OPDReg(0xDA, 2), 1, X86InstInfo{\"FICOM\", TYPE_X87, GenFlagsSrcSize(SIZE_32BIT) | FLAGS_MODRM, 0}},\n {OPDReg(0xDA, 3), 1, X86InstInfo{\"FICOMP\", TYPE_X87, GenFlagsSrcSize(SIZE_32BIT) | FLAGS_MODRM | FLAGS_POP, 0}},\n {OPDReg(0xDA, 4), 1, X86InstInfo{\"FISUB\", TYPE_X87, GenFlagsSrcSize(SIZE_32BIT) | FLAGS_MODRM, 0}},\n {OPDReg(0xDA, 5), 1, X86InstInfo{\"FISUBR\", TYPE_X87, GenFlagsSrcSize(SIZE_32BIT) | FLAGS_MODRM, 0}},\n {OPDReg(0xDA, 6), 1, X86InstInfo{\"FIDIV\", TYPE_X87, GenFlagsSrcSize(SIZE_32BIT) | FLAGS_MODRM, 0}},\n {OPDReg(0xDA, 7), 1, X86InstInfo{\"FIDIVR\", TYPE_X87, GenFlagsSrcSize(SIZE_32BIT) | FLAGS_MODRM, 0}},\n // / 0\n {OPD(0xDA, 0xC0), 8, X86InstInfo{\"FCMOVB\", TYPE_X87, FLAGS_NONE, 0}},\n // / 1\n {OPD(0xDA, 0xC8), 8, X86InstInfo{\"FCMOVE\", TYPE_X87, FLAGS_NONE, 0}},\n // / 2\n {OPD(0xDA, 0xD0), 8, X86InstInfo{\"FCMOVBE\", TYPE_X87, FLAGS_NONE, 0}},\n // / 3\n {OPD(0xDA, 0xD8), 8, X86InstInfo{\"FCMOVU\", TYPE_X87, FLAGS_NONE, 0}},\n // / 4\n {OPD(0xDA, 0xE0), 8, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n // / 5\n {OPD(0xDA, 0xE8), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(0xDA, 0xE9), 1, X86InstInfo{\"FUCOMPP\", TYPE_X87, FLAGS_POP, 0}},\n {OPD(0xDA, 0xEA), 6, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n // / 6\n {OPD(0xDA, 0xF0), 8, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n // / 7\n {OPD(0xDA, 0xF8), 8, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n // 0xDB\n {OPDReg(0xDB, 0), 1, X86InstInfo{\"FILD\", TYPE_X87, GenFlagsSrcSize(SIZE_32BIT) | FLAGS_MODRM, 0}},\n {OPDReg(0xDB, 1), 1, X86InstInfo{\"FISTTP\", TYPE_X87, GenFlagsSrcSize(SIZE_32BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_POP, 0}},\n {OPDReg(0xDB, 2), 1, X86InstInfo{\"FIST\", TYPE_X87, GenFlagsSrcSize(SIZE_32BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {OPDReg(0xDB, 3), 1, X86InstInfo{\"FISTP\", TYPE_X87, GenFlagsSrcSize(SIZE_32BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_POP, 0}},\n {OPDReg(0xDB, 4), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPDReg(0xDB, 5), 1, X86InstInfo{\"FLD\", TYPE_X87, FLAGS_MODRM, 0}},\n {OPDReg(0xDB, 6), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPDReg(0xDB, 7), 1, X86InstInfo{\"FSTP\", TYPE_X87, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_POP, 0}},\n // / 0\n {OPD(0xDB, 0xC0), 8, X86InstInfo{\"FCMOVNB\", TYPE_X87, FLAGS_NONE, 0}},\n // / 1\n {OPD(0xDB, 0xC8), 8, X86InstInfo{\"FCMOVNE\", TYPE_X87, FLAGS_NONE, 0}},\n // / 2\n {OPD(0xDB, 0xD0), 8, X86InstInfo{\"FCMOVNBE\", TYPE_X87, FLAGS_NONE, 0}},\n // / 3\n {OPD(0xDB, 0xD8), 8, X86InstInfo{\"FCMOVNU\", TYPE_X87, FLAGS_NONE, 0}},\n // / 4\n {OPD(0xDB, 0xE0), 2, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(0xDB, 0xE2), 1, X86InstInfo{\"FNCLEX\", TYPE_X87, FLAGS_NONE, 0}},\n {OPD(0xDB, 0xE3), 1, X86InstInfo{\"FNINIT\", TYPE_X87, FLAGS_NONE, 0}},\n {OPD(0xDB, 0xE4), 4, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n // / 5\n {OPD(0xDB, 0xE8), 8, X86InstInfo{\"FUCOMI\", TYPE_INST, FLAGS_NONE, 0}},\n // / 6\n {OPD(0xDB, 0xF0), 8, X86InstInfo{\"FCOMI\", TYPE_X87, FLAGS_NONE, 0}},\n // / 7\n {OPD(0xDB, 0xF8), 8, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n // 0xDC\n {OPDReg(0xDC, 0), 1, X86InstInfo{\"FADD\", TYPE_X87, GenFlagsSrcSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_X87_FLAGS, 0}},\n {OPDReg(0xDC, 1), 1, X86InstInfo{\"FMUL\", TYPE_X87, GenFlagsSrcSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_X87_FLAGS, 0}},\n {OPDReg(0xDC, 2), 1, X86InstInfo{\"FCOM\", TYPE_X87, GenFlagsSrcSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_X87_FLAGS, 0}},\n {OPDReg(0xDC, 3), 1, X86InstInfo{\"FCOMP\", TYPE_X87, GenFlagsSrcSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_X87_FLAGS | FLAGS_POP, 0}},\n {OPDReg(0xDC, 4), 1, X86InstInfo{\"FSUB\", TYPE_X87, GenFlagsSrcSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_X87_FLAGS, 0}},\n {OPDReg(0xDC, 5), 1, X86InstInfo{\"FSUBR\", TYPE_X87, GenFlagsSrcSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_X87_FLAGS, 0}},\n {OPDReg(0xDC, 6), 1, X86InstInfo{\"FDIV\", TYPE_X87, GenFlagsSrcSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_X87_FLAGS, 0}},\n {OPDReg(0xDC, 7), 1, X86InstInfo{\"FDIVR\", TYPE_X87, GenFlagsSrcSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_X87_FLAGS, 0}},\n // / 0\n {OPD(0xDC, 0xC0), 8, X86InstInfo{\"FADD\", TYPE_X87, FLAGS_NONE, 0}},\n // / 1\n {OPD(0xDC, 0xC8), 8, X86InstInfo{\"FMUL\", TYPE_X87, FLAGS_NONE, 0}},\n // / 2\n {OPD(0xDC, 0xD0), 8, X86InstInfo{\"FCOM\", TYPE_X87, FLAGS_X87_FLAGS, 0}},\n // / 3\n {OPD(0xDC, 0xD8), 8, X86InstInfo{\"FCOMP\", TYPE_X87, FLAGS_X87_FLAGS | FLAGS_POP, 0}},\n // / 4\n {OPD(0xDC, 0xE0), 8, X86InstInfo{\"FSUBR\", TYPE_X87, FLAGS_NONE, 0}},\n // / 5\n {OPD(0xDC, 0xE8), 8, X86InstInfo{\"FSUB\", TYPE_X87, FLAGS_NONE, 0}},\n // / 6\n {OPD(0xDC, 0xF0), 8, X86InstInfo{\"FDIVR\", TYPE_X87, FLAGS_NONE, 0}},\n // / 7\n {OPD(0xDC, 0xF8), 8, X86InstInfo{\"FDIV\", TYPE_X87, FLAGS_NONE, 0}},\n // 0xDD\n {OPDReg(0xDD, 0), 1, X86InstInfo{\"FLD\", TYPE_X87, FLAGS_MODRM, 0}},\n {OPDReg(0xDD, 1), 1, X86InstInfo{\"FISTTP\", TYPE_X87, GenFlagsSrcSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_X87_FLAGS | FLAGS_SF_MOD_DST | FLAGS_POP, 0}},\n {OPDReg(0xDD, 2), 1, X86InstInfo{\"FST\", TYPE_X87, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {OPDReg(0xDD, 3), 1, X86InstInfo{\"FSTP\", TYPE_X87, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_POP, 0}},\n {OPDReg(0xDD, 4), 1, X86InstInfo{\"FRSTOR\", TYPE_X87, FLAGS_MODRM, 0}},\n {OPDReg(0xDD, 5), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPDReg(0xDD, 6), 1, X86InstInfo{\"FNSAVE\", TYPE_X87, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {OPDReg(0xDD, 7), 1, X86InstInfo{\"FNSTSW\", TYPE_X87, GenFlagsSameSize(SIZE_16BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n // / 0\n {OPD(0xDD, 0xC0), 8, X86InstInfo{\"FFREE\", TYPE_X87, FLAGS_NONE, 0}},\n // / 1\n {OPD(0xDD, 0xC8), 8, X86InstInfo{\"FXCH\", TYPE_X87, FLAGS_NONE, 0}},\n // / 2\n {OPD(0xDD, 0xD0), 8, X86InstInfo{\"FST\", TYPE_INST, FLAGS_SF_MOD_DST, 0}},\n // / 3\n {OPD(0xDD, 0xD8), 8, X86InstInfo{\"FSTP\", TYPE_X87, FLAGS_SF_MOD_DST | FLAGS_POP, 0}},\n // / 4\n {OPD(0xDD, 0xE0), 8, X86InstInfo{\"FUCOM\", TYPE_X87, FLAGS_NONE, 0}},\n // / 5\n {OPD(0xDD, 0xE8), 8, X86InstInfo{\"FUCOMP\", TYPE_X87, FLAGS_POP, 0}},\n // / 6\n {OPD(0xDD, 0xF0), 8, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n // / 7\n {OPD(0xDD, 0xF8), 8, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n // 0xDE\n {OPDReg(0xDE, 0), 1, X86InstInfo{\"FIADD\", TYPE_X87, GenFlagsSrcSize(SIZE_16BIT) | FLAGS_MODRM, 0}},\n {OPDReg(0xDE, 1), 1, X86InstInfo{\"FIMUL\", TYPE_X87, GenFlagsSrcSize(SIZE_16BIT) | FLAGS_MODRM, 0}},\n {OPDReg(0xDE, 2), 1, X86InstInfo{\"FICOM\", TYPE_X87, GenFlagsSrcSize(SIZE_16BIT) | FLAGS_MODRM, 0}},\n {OPDReg(0xDE, 3), 1, X86InstInfo{\"FICOMP\", TYPE_X87, GenFlagsSrcSize(SIZE_16BIT) | FLAGS_MODRM | FLAGS_POP, 0}},\n {OPDReg(0xDE, 4), 1, X86InstInfo{\"FISUB\", TYPE_X87, GenFlagsSrcSize(SIZE_16BIT) | FLAGS_MODRM, 0}},\n {OPDReg(0xDE, 5), 1, X86InstInfo{\"FISUBR\", TYPE_X87, GenFlagsSrcSize(SIZE_16BIT) | FLAGS_MODRM, 0}},\n {OPDReg(0xDE, 6), 1, X86InstInfo{\"FIDIV\", TYPE_X87, GenFlagsSrcSize(SIZE_16BIT) | FLAGS_MODRM, 0}},\n {OPDReg(0xDE, 7), 1, X86InstInfo{\"FIDIVR\", TYPE_X87, GenFlagsSrcSize(SIZE_16BIT) | FLAGS_MODRM, 0}},\n // / 0\n {OPD(0xDE, 0xC0), 8, X86InstInfo{\"FADDP\", TYPE_X87, FLAGS_POP, 0}},\n // / 1\n {OPD(0xDE, 0xC8), 8, X86InstInfo{\"FMULP\", TYPE_X87, FLAGS_POP, 0}},\n // / 2\n {OPD(0xDE, 0xD0), 8, X86InstInfo{\"FCOMP\", TYPE_X87, FLAGS_X87_FLAGS | FLAGS_POP, 0}},\n // / 3\n {OPD(0xDE, 0xD8), 1, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n {OPD(0xDE, 0xD9), 1, X86InstInfo{\"FCOMPP\", TYPE_X87, FLAGS_POP, 0}},\n {OPD(0xDE, 0xDA), 6, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n // / 4\n {OPD(0xDE, 0xE0), 8, X86InstInfo{\"FSUBRP\", TYPE_X87, FLAGS_POP, 0}},\n // / 5\n {OPD(0xDE, 0xE8), 8, X86InstInfo{\"FSUBP\", TYPE_X87, FLAGS_POP, 0}},\n // / 6\n {OPD(0xDE, 0xF0), 8, X86InstInfo{\"FDIVRP\", TYPE_X87, FLAGS_POP, 0}},\n // / 7\n {OPD(0xDE, 0xF8), 8, X86InstInfo{\"FDIVP\", TYPE_X87, FLAGS_POP, 0}},\n // 0xDF\n {OPDReg(0xDF, 0), 1, X86InstInfo{\"FILD\", TYPE_X87, GenFlagsSrcSize(SIZE_16BIT) | FLAGS_MODRM, 0}},\n {OPDReg(0xDF, 1), 1, X86InstInfo{\"FISTTP\", TYPE_X87, GenFlagsSrcSize(SIZE_16BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_POP, 0}},\n {OPDReg(0xDF, 2), 1, X86InstInfo{\"FIST\", TYPE_X87, GenFlagsSrcSize(SIZE_16BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},\n {OPDReg(0xDF, 3), 1, X86InstInfo{\"FISTP\", TYPE_X87, GenFlagsSrcSize(SIZE_16BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_POP, 0}},\n {OPDReg(0xDF, 4), 1, X86InstInfo{\"FBLD\", TYPE_X87, FLAGS_MODRM, 0}},\n {OPDReg(0xDF, 5), 1, X86InstInfo{\"FILD\", TYPE_X87, GenFlagsSrcSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_X87_FLAGS, 0}},\n {OPDReg(0xDF, 6), 1, X86InstInfo{\"FBSTP\", TYPE_X87, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_POP, 0}},\n {OPDReg(0xDF, 7), 1, X86InstInfo{\"FISTP\", TYPE_X87, GenFlagsSrcSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_X87_FLAGS | FLAGS_SF_MOD_DST | FLAGS_POP, 0}},\n // / 0\n // This instruction is a bit special. This is an undocumented(Almost) x87 instruction.\n // https://en.wikipedia.org/wiki/X86_instruction_listings#Undocumented_x87_instructions\n // https://www.pagetable.com/?p=16\n // AMD Athlon Processor x86 Code Optimization Guide - `Use FFREEP Macro to Pop One Register from the FPU Stack`\n // ISA architecture manuals don't talk about this instruction at all\n // At some point the Nvidia OpenGL binary driver uses this instruction.\n // GCC may also end up emitting this instruction in some rare edge case!\n // Almost all x86 CPUs implement this, and it is expected to be around\n {OPD(0xDF, 0xC0), 8, X86InstInfo{\"FFREEP\", TYPE_X87, FLAGS_POP, 0}},\n // / 1\n {OPD(0xDF, 0xC8), 8, X86InstInfo{\"FXCH\", TYPE_X87, FLAGS_NONE, 0}},\n // / 2\n {OPD(0xDF, 0xD0), 8, X86InstInfo{\"FSTP\", TYPE_X87, FLAGS_SF_MOD_DST | FLAGS_POP, 0}},\n // / 3\n {OPD(0xDF, 0xD8), 8, X86InstInfo{\"FSTP\", TYPE_X87, FLAGS_SF_MOD_DST | FLAGS_POP, 0}},\n // / 4\n {OPD(0xDF, 0xE0), 1, X86InstInfo{\"FNSTSW\", TYPE_INST, GenFlagsSameSize(SIZE_16BIT) | FLAGS_SF_DST_RAX, 0}},\n {OPD(0xDF, 0xE1), 7, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n // / 5\n {OPD(0xDF, 0xE8), 8, X86InstInfo{\"FUCOMIP\", TYPE_INST, FLAGS_POP, 0}},\n // / 6\n {OPD(0xDF, 0xF0), 8, X86InstInfo{\"FCOMIP\", TYPE_X87, FLAGS_POP, 0}},\n // / 7\n {OPD(0xDF, 0xF8), 8, X86InstInfo{\"\", TYPE_INVALID, FLAGS_NONE, 0}},\n };\n#undef OPD\n#undef OPDReg\n\n auto InstallToX87Table = [](auto& FinalTable, auto& LocalTable) {\n for (auto Op : LocalTable) {\n auto OpNum = Op.Op;\n bool Repeat = (OpNum & 0x8000) != 0;\n OpNum = OpNum & 0x7FF;\n auto Dispatcher = Op.Ptr;\n for (uint8_t i = 0; i < Op.Count; ++i) {\n LOGMAN_THROW_A_FMT(FinalTable[OpNum + i].OpcodeDispatcher.OpDispatch == nullptr, \"Duplicate Entry\");\n\n FinalTable[OpNum + i].OpcodeDispatcher.OpDispatch = Dispatcher;\n\n // Flag to indicate if we need to repeat this op in {0x40, 0x80} ranges\n if (Repeat) {\n FinalTable[(OpNum | 0x40) + i].OpcodeDispatcher.OpDispatch = Dispatcher;\n FinalTable[(OpNum | 0x80) + i].OpcodeDispatcher.OpDispatch = Dispatcher;\n }\n }\n }\n };\n\n GenerateX87Table(Table.data(), X87OpTable, std::size(X87OpTable));\n InstallToX87Table(Table, DispatchTable);\n return Table;\n};\n\nconstexpr std::array X87F80Ops = GenerateX87TableLambda(X87F80OpTable);\nconstexpr std::array X87F64Ops = GenerateX87TableLambda(X87F64OpTable);\n\n}\n" }, { "path": "FEXCore/Source/Interface/GDBJIT/GDBJIT.cpp", "content": "// SPDX-License-Identifier: MIT\n#include \"GDBJIT.h\"\n\n#include \n#include \n#include \n#include \n\n#if defined(GDB_SYMBOLS_ENABLED)\n\n#include \n\nextern \"C\" {\nenum jit_actions_t { JIT_NOACTION = 0, JIT_REGISTER_FN, JIT_UNREGISTER_FN };\n\nstruct jit_code_entry {\n jit_code_entry* next_entry;\n jit_code_entry* prev_entry;\n const char* symfile_addr;\n uint64_t symfile_size;\n};\n\nstruct jit_descriptor {\n uint32_t version;\n /* This type should be jit_actions_t, but we use uint32_t\n to be explicit about the bitwidth. */\n uint32_t action_flag;\n jit_code_entry* relevant_entry;\n jit_code_entry* first_entry;\n};\n\n/* Make sure to specify the version statically, because the\n debugger may check the version before we can set it. */\n\nconstinit jit_descriptor __jit_debug_descriptor = {.version = 1};\n\n/* GDB puts a breakpoint in this function. */\nvoid __attribute__((noinline)) __jit_debug_register_code() {\n asm volatile(\"\" ::\"r\"(&__jit_debug_descriptor));\n};\n}\n\nnamespace FEXCore {\n\nvoid GDBJITRegister(const FEXCore::ExecutableFileInfo& Entry, uintptr_t VAFileStart, uint64_t GuestRIP, uintptr_t HostEntry,\n FEXCore::Core::DebugData& DebugData) {\n auto map = Entry.SourcecodeMap.get();\n\n if (map) {\n auto FileOffset = GuestRIP - VAFileStart;\n\n auto Sym = map->FindSymbolMapping(FileOffset);\n\n auto SymName = HLE::SourcecodeSymbolMapping::SymName(Sym, Entry.Filename, HostEntry, FileOffset);\n\n fextl::vector Lines;\n for (const auto& GuestOpcode : DebugData.GuestOpcodes) {\n auto Line = map->FindLineMapping(GuestRIP + GuestOpcode.GuestEntryOffset - VAFileStart);\n if (Line) {\n Lines.push_back({Line->LineNumber, HostEntry + GuestOpcode.HostEntryOffset});\n }\n }\n\n size_t size = sizeof(info_t) + 1 * sizeof(blocks_t) + Lines.size() * sizeof(gdb_line_mapping);\n\n auto mem = (uint8_t*)malloc(size);\n auto base = mem;\n info_t* info = (info_t*)mem;\n mem += sizeof(info_t);\n\n strncpy(info->filename, map->SourceFile.c_str(), 511);\n\n info->nblocks = 1;\n\n auto blocks = (blocks_t*)mem;\n info->blocks_ofs = mem - base;\n\n mem += info->nblocks * sizeof(blocks_t);\n\n for (int i = 0; i < info->nblocks; i++) {\n strncpy(blocks[i].name, SymName.c_str(), 511);\n blocks[i].start = HostEntry;\n blocks[i].end = HostEntry + DebugData.HostCodeSize;\n }\n\n info->nlines = Lines.size();\n\n auto lines = (gdb_line_mapping*)mem;\n info->lines_ofs = mem - base;\n mem += info->nlines * sizeof(gdb_line_mapping);\n\n if (info->nlines) {\n memcpy(lines, Lines.data(), info->nlines * sizeof(gdb_line_mapping));\n }\n\n auto entry = new jit_code_entry {0, 0, 0, 0};\n\n entry->symfile_addr = (const char*)info;\n entry->symfile_size = size;\n\n if (__jit_debug_descriptor.first_entry) {\n __jit_debug_descriptor.relevant_entry->next_entry = entry;\n entry->prev_entry = __jit_debug_descriptor.relevant_entry;\n } else {\n __jit_debug_descriptor.first_entry = entry;\n }\n\n __jit_debug_descriptor.relevant_entry = entry;\n __jit_debug_descriptor.action_flag = JIT_REGISTER_FN;\n __jit_debug_register_code();\n }\n}\n} // namespace FEXCore\n#else\nnamespace FEXCore {\nvoid GDBJITRegister(const FEXCore::ExecutableFileInfo&, uintptr_t, uint64_t, uintptr_t, FEXCore::Core::DebugData&) {\n ERROR_AND_DIE_FMT(\"GDBSymbols support not compiled in\");\n}\n} // namespace FEXCore\n#endif\n" }, { "path": "FEXCore/Source/Interface/GDBJIT/GDBJIT.h", "content": "// SPDX-License-Identifier: MIT\n\n#include \n#include \n\nnamespace FEXCore {\nvoid GDBJITRegister(const FEXCore::ExecutableFileInfo&, uintptr_t VAFileStart, uint64_t GuestRIP, uintptr_t HostEntry, FEXCore::Core::DebugData&);\n}\n" }, { "path": "FEXCore/Source/Interface/IR/IR.h", "content": "// SPDX-License-Identifier: MIT\n#pragma once\n\n#include \n#include \n#include \n#include \n#include \n\n#include \n#include \n\n#include \n#include \n#include \n#include \n#include \n#include \n\nnamespace FEXCore::IR {\n\nclass OrderedNode;\n\n/**\n * @brief The IROp_Header is an dynamically sized array\n * At the end it contains a uint8_t for the number of arguments that Op has\n * Then there is an unsized array of NodeWrapper arguments for the number of arguments this op has\n * The op structures that are including the header must ensure that they pad themselves correctly to the number of arguments used\n */\nstruct IROp_Header;\n\n/**\n * @brief Represents the ID of a given IR node.\n *\n * Intended to provide strong typing from other integer values\n * to prevent passing incorrect values to certain API functions.\n */\nstruct NodeID final {\n using value_type = uint32_t;\n\n constexpr NodeID() noexcept = default;\n constexpr explicit NodeID(value_type Value_) noexcept\n : Value {Value_} {}\n\n constexpr NodeID(const NodeID&) noexcept = default;\n constexpr NodeID& operator=(const NodeID&) noexcept = default;\n\n constexpr NodeID(NodeID&&) noexcept = default;\n constexpr NodeID& operator=(NodeID&&) noexcept = default;\n\n [[nodiscard]]\n constexpr bool IsValid() const noexcept {\n return Value != 0;\n }\n [[nodiscard]]\n constexpr bool IsInvalid() const noexcept {\n return !IsValid();\n }\n constexpr void Invalidate() noexcept {\n Value = 0;\n }\n\n [[nodiscard]] constexpr auto operator<=>(const NodeID&) const noexcept = default;\n\n friend std::ostream& operator<<(std::ostream& out, NodeID ID) {\n out << ID.Value;\n return out;\n }\n friend std::istream& operator>>(std::istream& in, NodeID& ID) {\n in >> ID.Value;\n return in;\n }\n\n value_type Value {};\n};\n\n/**\n * @brief This is a very simple wrapper for our node pointers\n * You probably don't want to use this directly\n * Use OpNodeWrapper and OrderedNodeWrapper types below instead\n *\n * This is necessary to allow two things\n * - Reduce memory usage by having the pointer be an 32bit offset rather than the whole 64bit pointer\n * - Actually use an offset from a base so we aren't storing pointers for everything\n * - Makes IR list copying be as cheap as a memcpy\n * Downsides\n * - The IR nodes have to be allocated out of a linear array of memory\n * - We currently only allow a 32bit offset, so *only* 4 million nodes per list\n * - We have to have the base offset live somewhere else\n * - Has to be POD and trivially copyable\n * - Makes every real node access turn in to a [Base + Offset] access\n * - Can be confusing if you're mixing OpNodeWrapper and OrderedNodeWrapper usage\n */\ntemplate\nstruct FEX_PACKED NodeWrapperBase final {\n // 32bit or 64bit offset doesn't matter for addressing.\n // We use uint32_t to be more memory efficient (Cuts our node list size in half)\n using NodeOffsetType = uint32_t;\n NodeOffsetType NodeOffset;\n\n explicit NodeWrapperBase() = default;\n\n [[nodiscard]]\n static NodeWrapperBase WrapOffset(NodeOffsetType Offset) {\n NodeWrapperBase Wrapped;\n Wrapped.NodeOffset = Offset;\n return Wrapped;\n }\n\n [[nodiscard]]\n static NodeWrapperBase WrapPtr(uintptr_t Base, uintptr_t Value) {\n NodeWrapperBase Wrapped;\n Wrapped.SetOffset(Base, Value);\n return Wrapped;\n }\n\n [[nodiscard]]\n static void* UnwrapNode(uintptr_t Base, NodeWrapperBase Node) {\n return Node.GetNode(Base);\n }\n\n [[nodiscard]]\n NodeID ID() const;\n\n [[nodiscard]]\n bool IsInvalid() const {\n return NodeOffset == 0;\n }\n\n [[nodiscard]]\n bool IsImmediate() const {\n return NodeOffset & (1u << 31);\n }\n\n [[nodiscard]]\n bool HasKill() const {\n return NodeOffset & (1u << 30);\n }\n\n void ClearKill() {\n NodeOffset &= ~(1u << 30);\n }\n\n void SetKill() {\n NodeOffset |= (1u << 30);\n }\n\n [[nodiscard]]\n bool IsPointer() const {\n return !IsImmediate() && !HasKill();\n }\n\n [[nodiscard]]\n Type* GetNode(uintptr_t Base) {\n LOGMAN_THROW_A_FMT(IsPointer(), \"Precondition\");\n return reinterpret_cast(Base + NodeOffset);\n }\n [[nodiscard]]\n const Type* GetNode(uintptr_t Base) const {\n LOGMAN_THROW_A_FMT(IsPointer(), \"Precondition\");\n return reinterpret_cast(Base + NodeOffset);\n }\n\n void SetOffset(uintptr_t Base, uintptr_t Value) {\n NodeOffset = Value - Base;\n LOGMAN_THROW_A_FMT(IsPointer(), \"Offsets are within 2GiB range\");\n }\n\n void SetInvalid() {\n NodeOffset = 0;\n LOGMAN_THROW_A_FMT(IsInvalid(), \"Zero state\");\n }\n\n void SetImmediate(uint32_t Immediate) {\n LOGMAN_THROW_A_FMT(Immediate < (1u << 31), \"Bounded\");\n NodeOffset = Immediate | (1u << 31);\n LOGMAN_THROW_A_FMT(IsImmediate(), \"Encoded above\");\n }\n\n [[nodiscard]]\n uint32_t GetImmediate() const {\n LOGMAN_THROW_A_FMT(IsImmediate(), \"Precondition: must be an immediate\");\n return NodeOffset & ~(1u << 31);\n }\n\n [[nodiscard]]\n friend constexpr bool operator==(const NodeWrapperBase&, const NodeWrapperBase&) = default;\n\n [[nodiscard]]\n static NodeWrapperBase FromImmediate(uint32_t Immediate) {\n NodeWrapperBase A;\n A.SetImmediate(Immediate);\n return A;\n }\n};\n\nstatic_assert(std::is_trivially_copyable_v>);\n\nstatic_assert(sizeof(NodeWrapperBase) == sizeof(uint32_t));\n\nusing OpNodeWrapper = NodeWrapperBase;\nusing OrderedNodeWrapper = NodeWrapperBase;\n\nstruct OrderedNodeHeader {\n OpNodeWrapper Value;\n OrderedNodeWrapper Next;\n OrderedNodeWrapper Previous;\n};\n\nstatic_assert(sizeof(OrderedNodeHeader) == sizeof(uint32_t) * 3);\n\n/**\n * @brief This is a node in our IR representation\n * Is a doubly linked list node that lives in a representation of a linearly allocated node list\n * The links in the nodes can live in a list independent of the data IR data\n *\n * ex.\n * Region1 : ... <-> <-> <-> ...\n * | * |\n * v v\n * Region2 : ......\n *\n * In this example the OrderedNodes are allocated in one linear memory region (Not necessarily contiguous with one another linking)\n * The second region is contiguous but they don't have any relationship with one another directly\n */\nclass OrderedNode final {\npublic:\n // These three values are laid out very specifically to make it fast to access the NodeWrappers specifically\n OrderedNodeHeader Header;\n uint32_t NumUses;\n\n // After RA, the register allocated for the node. This is the register for the\n // node at the time it is written, even if it is shuffled into other registers\n // later. In other words, it is the register destination of the instruction\n // represented by this OrderedNode.\n //\n // This is the raw value of a PhysicalRegister data structure.\n uint8_t Reg;\n uint8_t Pad[3];\n\n using value_type = OrderedNodeWrapper;\n\n OrderedNode() = default;\n\n /**\n * @brief Appends a node to this current node\n *\n * Before. <-> <-> \n * After. <-> <-> <-> Next\n *\n * @return Pointer to the node being added\n */\n value_type append(uintptr_t Base, value_type Node) {\n // Set Next Node's Previous to incoming node\n SetPrevious(Base, Header.Next, Node);\n\n // Set Incoming node's links to this node's links\n SetPrevious(Base, Node, Wrapped(Base));\n SetNext(Base, Node, Header.Next);\n\n // Set this node's next to the incoming node\n SetNext(Base, Wrapped(Base), Node);\n\n // Return the node we are appending\n return Node;\n }\n\n OrderedNode* append(uintptr_t Base, OrderedNode* Node) {\n value_type WNode = Node->Wrapped(Base);\n // Set Next Node's Previous to incoming node\n SetPrevious(Base, Header.Next, WNode);\n\n // Set Incoming node's links to this node's links\n SetPrevious(Base, WNode, Wrapped(Base));\n SetNext(Base, WNode, Header.Next);\n\n // Set this node's next to the incoming node\n SetNext(Base, Wrapped(Base), WNode);\n\n // Return the node we are appending\n return Node;\n }\n\n /**\n * @brief Prepends a node to the current node\n * Before. <-> <-> \n * After. <-> <-> <-> Next\n *\n * @return Pointer to the node being added\n */\n value_type prepend(uintptr_t Base, value_type Node) {\n // Set the previous node's next to the incoming node\n SetNext(Base, Header.Previous, Node);\n\n // Set the incoming node's links\n SetPrevious(Base, Node, Header.Previous);\n SetNext(Base, Node, Wrapped(Base));\n\n // Set the current node's link\n SetPrevious(Base, Wrapped(Base), Node);\n\n // Return the node we are prepending\n return Node;\n }\n\n OrderedNode* prepend(uintptr_t Base, OrderedNode* Node) {\n value_type WNode = Node->Wrapped(Base);\n // Set the previous node's next to the incoming node\n SetNext(Base, Header.Previous, WNode);\n\n // Set the incoming node's links\n SetPrevious(Base, WNode, Header.Previous);\n SetNext(Base, WNode, Wrapped(Base));\n\n // Set the current node's link\n SetPrevious(Base, Wrapped(Base), WNode);\n\n // Return the node we are prepending\n return Node;\n }\n\n /**\n * @brief Gets the remaining size of the blocks from this point onward\n *\n * Doesn't find the head of the list\n *\n */\n [[nodiscard]]\n size_t size(uintptr_t Base) const {\n size_t Size = 1;\n // Walk the list forward until we hit a sentinel\n value_type Current = Header.Next;\n while (Current.NodeOffset != 0) {\n ++Size;\n OrderedNode* RealNode = Current.GetNode(Base);\n Current = RealNode->Header.Next;\n }\n return Size;\n }\n\n void Unlink(uintptr_t Base) {\n // This removes the node from the list. Orphaning it\n // Before: <-> <-> \n // After: \n SetNext(Base, Header.Previous, Header.Next);\n SetPrevious(Base, Header.Next, Header.Previous);\n }\n\n [[nodiscard]]\n const IROp_Header* Op(uintptr_t Base) const {\n return Header.Value.GetNode(Base);\n }\n [[nodiscard]]\n IROp_Header* Op(uintptr_t Base) {\n return Header.Value.GetNode(Base);\n }\n\n [[nodiscard]]\n uint32_t GetUses() const {\n return NumUses;\n }\n\n void AddUse() {\n ++NumUses;\n }\n void RemoveUse() {\n --NumUses;\n }\n\n [[nodiscard]]\n value_type Wrapped(uintptr_t Base) const {\n value_type Tmp;\n Tmp.SetOffset(Base, reinterpret_cast(this));\n return Tmp;\n }\n\nprivate:\n [[nodiscard]]\n value_type WrappedOffset(uint32_t Offset) const {\n value_type Tmp;\n Tmp.NodeOffset = Offset;\n return Tmp;\n }\n\n static void SetPrevious(uintptr_t Base, value_type Node, value_type New) {\n OrderedNode* RealNode = Node.GetNode(Base);\n RealNode->Header.Previous = New;\n }\n\n static void SetNext(uintptr_t Base, value_type Node, value_type New) {\n OrderedNode* RealNode = Node.GetNode(Base);\n RealNode->Header.Next = New;\n }\n\n void SetUses(uint32_t Uses) {\n NumUses = Uses;\n }\n};\n\nstatic_assert(std::is_trivially_constructible_v);\nstatic_assert(std::is_trivially_copyable_v);\nstatic_assert(offsetof(OrderedNode, Header) == 0);\nstatic_assert(sizeof(OrderedNode) == (sizeof(OrderedNodeHeader) + 2 * sizeof(uint32_t)));\n\n// This is temporary. We are transitioning away from OrderedNode's in favour of\n// flat Ref words. To ease porting, we have this typedef. Eventually OrderedNode\n// will be removed and this typedef will be replaced by something like:\n//\n// struct Ref {\n// uint Flags : 1;\n// uint ID : 23;\n// uint Reg : 8;\n// };\nusing Ref = OrderedNode*;\n\n/* This iterator can be used to step though nodes.\n * Due to how our IR is laid out, this can be used to either step\n * though the CodeBlocks or though the code within a single block.\n */\nclass NodeIterator {\npublic:\n struct value_type final {\n OrderedNode* Node;\n IROp_Header* Header;\n };\n using size_type = std::size_t;\n using difference_type = std::ptrdiff_t;\n using reference = value_type&;\n using const_reference = const value_type&;\n using pointer = value_type*;\n using const_pointer = const value_type*;\n using iterator = NodeIterator;\n using const_iterator = const NodeIterator;\n using reverse_iterator = iterator;\n using const_reverse_iterator = const_iterator;\n using iterator_category = std::bidirectional_iterator_tag;\n\n NodeIterator(uintptr_t Base, uintptr_t IRBase)\n : BaseList {Base}\n , IRList {IRBase} {}\n explicit NodeIterator(uintptr_t Base, uintptr_t IRBase, OrderedNodeWrapper Ptr)\n : BaseList {Base}\n , IRList {IRBase}\n , Node {Ptr} {}\n\n [[nodiscard]]\n bool operator==(const NodeIterator& rhs) const {\n return Node.NodeOffset == rhs.Node.NodeOffset;\n }\n\n [[nodiscard]]\n bool operator!=(const NodeIterator& rhs) const {\n return !operator==(rhs);\n }\n\n NodeIterator operator++() {\n OrderedNodeHeader* RealNode = reinterpret_cast(Node.GetNode(BaseList));\n Node = RealNode->Next;\n return *this;\n }\n\n NodeIterator operator--() {\n OrderedNodeHeader* RealNode = reinterpret_cast(Node.GetNode(BaseList));\n Node = RealNode->Previous;\n return *this;\n }\n\n [[nodiscard]]\n value_type operator*() {\n OrderedNode* RealNode = Node.GetNode(BaseList);\n return {RealNode, RealNode->Op(IRList)};\n }\n\n [[nodiscard]]\n value_type operator()() {\n OrderedNode* RealNode = Node.GetNode(BaseList);\n return {RealNode, RealNode->Op(IRList)};\n }\n\n [[nodiscard]]\n NodeID ID() const {\n return Node.ID();\n }\n\n [[nodiscard]]\n static NodeIterator Invalid() {\n return NodeIterator(0, 0);\n }\n\nprotected:\n uintptr_t BaseList {};\n uintptr_t IRList {};\n OrderedNodeWrapper Node {};\n};\n\n// This must directly match bytes to the named opsize.\n// Implicit sized IR operations does math to get between sizes.\nenum class OpSize : uint8_t {\n iUnsized = 0,\n i8Bit = 1,\n i16Bit = 2,\n i32Bit = 4,\n i64Bit = 8,\n f80Bit = 10,\n i128Bit = 16,\n i256Bit = 32,\n iInvalid = 0xFF,\n};\n\nenum class FloatCompareOp : uint8_t {\n EQ = 0,\n LT,\n LE,\n UNO,\n NEQ,\n ORD,\n};\n\nenum class ShiftType : uint8_t {\n LSL = 0,\n LSR,\n ASR,\n ROR,\n};\n\nenum class BranchHint : uint8_t { None = 0, Call, Return, CheckTF };\n\n\n// Converts a size stored as an integer in to an OpSize enum.\n// This is a nop operation and will be eliminated by the compiler.\nstatic inline OpSize SizeToOpSize(uint8_t Size) {\n switch (Size) {\n case 0: return OpSize::iUnsized;\n case 1: return OpSize::i8Bit;\n case 2: return OpSize::i16Bit;\n case 4: return OpSize::i32Bit;\n case 8: return OpSize::i64Bit;\n case 10: return OpSize::f80Bit;\n case 16: return OpSize::i128Bit;\n case 32: return OpSize::i256Bit;\n case 0xFF: return OpSize::iInvalid;\n default: FEX_UNREACHABLE;\n }\n}\n\n// This is a nop operation and will be eliminated by the compiler.\nstatic inline uint8_t OpSizeToSize(IR::OpSize Size) {\n switch (Size) {\n case OpSize::iUnsized: return 0;\n case OpSize::i8Bit: return 1;\n case OpSize::i16Bit: return 2;\n case OpSize::i32Bit: return 4;\n case OpSize::i64Bit: return 8;\n case OpSize::f80Bit: return 10;\n case OpSize::i128Bit: return 16;\n case OpSize::i256Bit: return 32;\n case OpSize::iInvalid: return 0xFF;\n default: FEX_UNREACHABLE;\n }\n}\n\nstatic inline uint16_t OpSizeAsBits(IR::OpSize Size) {\n LOGMAN_THROW_A_FMT(Size != IR::OpSize::iInvalid, \"Invalid Size\");\n return IR::OpSizeToSize(Size) * 8u;\n}\n\ntemplate\nrequires (std::is_integral_v)\nstatic inline OpSize operator<<(IR::OpSize Size, T Shift) {\n LOGMAN_THROW_A_FMT(Size != IR::OpSize::iInvalid, \"Invalid Size\");\n return IR::SizeToOpSize(IR::OpSizeToSize(Size) << Shift);\n}\n\ntemplate\nrequires (std::is_integral_v)\nstatic inline OpSize operator>>(IR::OpSize Size, T Shift) {\n LOGMAN_THROW_A_FMT(Size != IR::OpSize::iInvalid, \"Invalid Size\");\n return IR::SizeToOpSize(IR::OpSizeToSize(Size) >> Shift);\n}\n\nstatic inline OpSize operator/(IR::OpSize Size, IR::OpSize Divisor) {\n LOGMAN_THROW_A_FMT(Size != IR::OpSize::iInvalid, \"Invalid Size\");\n return IR::SizeToOpSize(IR::OpSizeToSize(Size) / IR::OpSizeToSize(Divisor));\n}\n\ntemplate\nrequires (std::is_integral_v)\nstatic inline OpSize operator/(IR::OpSize Size, T Divisor) {\n LOGMAN_THROW_A_FMT(Size != IR::OpSize::iInvalid, \"Invalid Size\");\n return IR::SizeToOpSize(IR::OpSizeToSize(Size) / Divisor);\n}\n\nstatic inline uint8_t NumElements(IR::OpSize RegisterSize, IR::OpSize ElementSize) {\n LOGMAN_THROW_A_FMT(RegisterSize != IR::OpSize::iInvalid && ElementSize != IR::OpSize::iInvalid && RegisterSize != IR::OpSize::iUnsized &&\n ElementSize != IR::OpSize::iUnsized,\n \"Invalid Size\");\n return IR::OpSizeToSize(RegisterSize) / IR::OpSizeToSize(ElementSize);\n}\n\n#define IROP_ENUM\n#define IROP_STRUCTS\n#define IROP_SIZES\n#define IROP_REG_CLASSES\n#include \n\n/* This iterator can be used to step though every single node in a multi-block in SSA order.\n *\n * Iterates in the order of:\n *\n * end <-- CodeBlockA <--> BlockAInst1 <--> BlockAInst2 <--> CodeBlockB <--> BlockBInst1 <--> BlockBInst2 --> end\n */\nclass AllNodesIterator : public NodeIterator {\npublic:\n AllNodesIterator(uintptr_t Base, uintptr_t IRBase)\n : NodeIterator(Base, IRBase) {}\n explicit AllNodesIterator(uintptr_t Base, uintptr_t IRBase, OrderedNodeWrapper Ptr)\n : NodeIterator(Base, IRBase, Ptr) {}\n AllNodesIterator(NodeIterator other)\n : NodeIterator(other) {} // Allow NodeIterator to be upgraded\n\n AllNodesIterator operator++() {\n OrderedNodeHeader* RealNode = reinterpret_cast(Node.GetNode(BaseList));\n auto IROp = Node.GetNode(BaseList)->Op(IRList);\n\n // If this is the last node of a codeblock, we need to continue to the next block\n if (IROp->Op == OP_ENDBLOCK) {\n auto EndBlock = IROp->C();\n\n auto CurrentBlock = EndBlock->BlockHeader.GetNode(BaseList);\n Node = CurrentBlock->Header.Next;\n } else if (IROp->Op == OP_CODEBLOCK) {\n auto CodeBlock = IROp->C();\n\n Node = CodeBlock->Begin;\n } else {\n Node = RealNode->Next;\n }\n\n return *this;\n }\n\n AllNodesIterator operator--() {\n auto IROp = Node.GetNode(BaseList)->Op(IRList);\n\n if (IROp->Op == OP_BEGINBLOCK) {\n auto BeginBlock = IROp->C();\n\n Node = BeginBlock->BlockHeader;\n } else if (IROp->Op == OP_CODEBLOCK) {\n auto PrevBlockWrapper = Node.GetNode(BaseList)->Header.Previous;\n auto PrevCodeBlock = PrevBlockWrapper.GetNode(BaseList)->Op(IRList)->C();\n\n Node = PrevCodeBlock->Last;\n } else {\n Node = Node.GetNode(BaseList)->Header.Previous;\n }\n\n return *this;\n }\n\n [[nodiscard]]\n static AllNodesIterator Invalid() {\n return AllNodesIterator(0, 0);\n }\n};\n\nclass IRListView;\nclass IREmitter;\n\ntemplate\ninline NodeID NodeWrapperBase::ID() const {\n return NodeID(NodeOffset / sizeof(IR::OrderedNode));\n}\n\n[[nodiscard]]\nbool IsBlockExit(FEXCore::IR::IROps Op);\n\nvoid Dump(fextl::stringstream* out, const IRListView* IR);\n\nconstexpr auto format_as(FEXCore::IR::NodeID ID) {\n return ID.Value;\n}\n\nFEX_DEFINE_ENUM_FMT_PASSTHROUGH(FEXCore::IR::FenceType)\nFEX_DEFINE_ENUM_FMT_PASSTHROUGH(FEXCore::IR::MemOffsetType)\nFEX_DEFINE_ENUM_FMT_PASSTHROUGH(FEXCore::IR::OpSize)\nFEX_DEFINE_ENUM_FMT_PASSTHROUGH(FEXCore::IR::RegClass)\n} // namespace FEXCore::IR\n\ntemplate<>\nstruct std::hash {\n size_t operator()(const FEXCore::IR::NodeID& ID) const noexcept {\n return std::hash {}(ID.Value);\n }\n};\n" }, { "path": "FEXCore/Source/Interface/IR/IR.json", "content": "{\n \"Docs\": [\n \"IRTypes define types that can be used directly in the IR.\",\n \"These will translate to the underlying C types when stored in the op data\",\n \"\",\n \"SSA types are special cased\",\n \" SSA = untyped\",\n \" GPR = GPR class type\",\n \" FPR = FPR class type\",\n \"Declaring the SSA types correctly will allow validation passes to ensure the op is getting passed correct arguments\",\n \"\",\n \"Arguments must always follow a particular order. :\",\n \"Type must always be an IRType\",\n \"Prefix currently can be one of the following: #, $\",\n \" #: This is a temporary argument that is in the IR Emitter arguments\",\n \" - This will not be stored in the resulting IR op data structure\",\n \" $: This is a value that will be stored inside of the IR op data structure\",\n \" - If it is type SSA, GPR, or FPR then it is an SSA type\",\n \" - These will get added to the SSA argument union to ensure RA happens\",\n \"\",\n \"IR op definition follows the structure of = \",\n \"\",\n \"Eg:\",\n \"IR op with no result and no arguments\",\n \" CallbackReturn\",\n \"\",\n \"IR op with result and no arguments\",\n \" GPR = ProcessorID\",\n \"\",\n \"IR op with no result and non-SSA argument\",\n \" Fence FenceType:$Type\",\n \"\",\n \"IR op with no result and SSA arguments\",\n \" SetRoundingMode GPR:$Mode\",\n \"\",\n \"IR op with result and SSA arguments\",\n \" GPR = Add GPR:$Src1, GPR:$Src2\",\n\n \"\",\n \"## Op members ##\",\n \"* Desc\",\n \" * List of text for documenting this IR op.\",\n \"* OpClass\",\n \" * Textual class to group IR ops by type\",\n \"* DestClass\",\n \" * SSA class of the return when the return type is `SSA`\",\n \" * Not used if the destination type is one of {GPR, FPR}\",\n \"* DestSize\",\n \" * The size of the destination type\",\n \"* EmitValidation\",\n \" * List of validations to emit for the IR emitter\",\n \" * These are validations that can't be automatically inferred and need to be hand-written\",\n \"\"\n ],\n \"Enums\": {\n \"class CondClass : uint8_t\": [\n \"EQ = 0,\",\n \"NEQ = 1,\",\n \"UGE = 2,\",\n \"ULT = 3,\",\n \"MI = 4,\",\n \"PL = 5,\",\n \"VS = 6,\",\n \"VC = 7,\",\n \"UGT = 8,\",\n \"ULE = 9,\",\n \"SGE = 10,\",\n \"SLT = 11,\",\n \"SGT = 12,\",\n \"SLE = 13,\",\n \"TSTZ = 14, /* bit test zero */\",\n \"TSTNZ = 15, /* bit test nonzero */\",\n \"\",\n \"FLU = 16, /* float less or unordered */\",\n \"FGE = 17, /* float greater or equal */\",\n \"FLEU = 18, /* float less or equal or unordered */\",\n \"FGT = 19, /* float greater */\",\n \"FU = 20, /* float unordered */\",\n \"FNU = 21, /* float not unordered */\",\n \"\",\n \"AL = 32, /* always */\"\n ],\n \"class FenceType : uint8_t\": [\n \"Load = 0,\",\n \"Store = 1,\",\n \"LoadStore = 2,\",\n \"Inst = 3,\"\n ],\n \"class MemOffsetType : uint8_t\": [\n \"SXTX = 0,\",\n \"UXTW = 1,\",\n \"SXTW = 2,\"\n ],\n \"class RegClass : uint32_t\": [\n \"Invalid = 0,\",\n \"GPR = 1,\",\n \"GPRFixed = 2,\",\n \"FPR = 3,\",\n \"FPRFixed = 4,\",\n \"Complex = 5,\"\n ],\n \"class RoundMode : uint8_t\": [\n \"Nearest = 0,\",\n \"NegInfinity = 1,\",\n \"PosInfinity = 2,\",\n \"TowardsZero = 3, /* Truncate */\",\n \"Host = 4,\"\n ],\n \"class ConstPad : uint8_t\": [\n \"NoPad = 0,\",\n \"DoPad = 1,\",\n \"AutoPad = 2,\"\n ]\n },\n \"Defines\": [\n \"constexpr uint8_t NumClasses {6}\",\n\n \"constexpr uint8_t FCMP_FLAG_EQ = 0\",\n \"constexpr uint8_t FCMP_FLAG_LT = 1\",\n \"constexpr uint8_t FCMP_FLAG_UNORDERED = 2\",\n\n \"struct BreakDefinition {\",\n \" uint16_t ErrorRegister;\",\n \" uint8_t Signal;\",\n \" uint8_t TrapNumber;\",\n \" uint8_t si_code;\",\n \"};\"\n ],\n \"IRTypes\" : {\n \"i1\": \"bool\",\n \"i8\": \"int8_t\",\n \"i16\": \"int16_t\",\n \"i32\": \"int32_t\",\n \"i64\": \"int64_t\",\n \"u8\": \"uint8_t\",\n \"u16\": \"uint16_t\",\n \"u32\": \"uint32_t\",\n \"u64\": \"uint64_t\",\n \"OpSize\": \"FEXCore::IR::OpSize\",\n \"SSA\": \"OrderedNode*\",\n \"GPR\": \"OrderedNode*\",\n \"FPR\": \"OrderedNode*\",\n \"FenceType\": \"FenceType\",\n \"RegisterClass\": \"RegClass\",\n \"CondClass\": \"CondClass\",\n \"SHA256Sum\": \"SHA256Sum\",\n \"MemOffsetType\": \"MemOffsetType\",\n \"BreakDefinition\": \"BreakDefinition\",\n \"RoundType\": \"RoundMode\",\n \"ConstPad\": \"ConstPad\",\n \"FloatCompareOp\": \"FloatCompareOp\",\n \"NamedVectorConstant\": \"FEXCore::IR::NamedVectorConstant\",\n \"IndexNamedVectorConstant\": \"FEXCore::IR::IndexNamedVectorConstant\",\n \"ShiftType\": \"FEXCore::IR::ShiftType\",\n \"BranchHint\": \"FEXCore::IR::BranchHint\",\n \"Array16\": \"std::array\"\n },\n \"Ops\": {\n \"Misc\": {\n \"Dummy\": {\n \"HasSideEffects\": true,\n \"SwitchGen\": false,\n \"JITDispatchOverride\": \"NoOp\"\n },\n \"IRHeader SSA:$Blocks, u64:$OriginalRIP, u32:$BlockCount, u32:$NumHostInstructions, u32:$SpillSlots, i1:$PostRA{false}, i1:$HasX87{false}, i1:$ReadsParity{false}\": {\n \"SwitchGen\": false,\n \"JITDispatchOverride\": \"NoOp\"\n },\n \"CodeBlock SSA:$Begin, SSA:$Last, u32:$ID, i1:$EntryPoint{false}, u32:$GuestEntryOffset{0}\": {\n \"SwitchGen\": false,\n \"RAOverride\": \"0\",\n \"JITDispatchOverride\": \"NoOp\"\n },\n \"BeginBlock SSA:$BlockHeader\": {\n \"HasSideEffects\": true,\n \"SwitchGen\": false,\n \"RAOverride\": \"0\",\n \"JITDispatchOverride\": \"NoOp\"\n },\n \"InvalidateFlags u64:$Flags\": {\n \"HasSideEffects\": true,\n \"JITDispatchOverride\": \"NoOp\"\n },\n\n \"EndBlock SSA:$BlockHeader\": {\n \"HasSideEffects\": true,\n \"SwitchGen\": false,\n \"RAOverride\": \"0\",\n \"JITDispatchOverride\": \"NoOp\"\n },\n\n \"GuestOpcode u32:$GuestEntryOffset\": {\n \"Desc\": [\"Marks the beginning of a guest opcode\"],\n \"HasSideEffects\": true\n },\n\n \"GPR = ValidateCode Array16:$CodeOriginal, GPR:$Address, u8:$CodeLength\": {\n \"HasSideEffects\": true,\n \"HasDest\": true,\n \"DestSize\": \"OpSize::i64Bit\"\n },\n\n \"ThreadRemoveCodeEntry\": {\n \"HasSideEffects\": true\n },\n\n \"GPR = ProcessorID\": {\n \"Desc\": [\"Returns the processor ID correlating to the current running CPU\",\n \"This may be out of date by time this instruction is executed so care must be taken\",\n \"This same information can be gotten from syscall getcpu(&cpu, &node)\",\n \"uint32_t Res = (node << 12) | cpu;\",\n \"This means it has a limitation of 4096 CPU cores. Which is fine and matches x86 behaviour\"\n ],\n \"DestSize\": \"OpSize::i64Bit\"\n },\n \"GPR = GetRoundingMode\": {\n \"Desc\": [\"Gets the current rounding mode options\"\n ],\n \"DestSize\": \"OpSize::i32Bit\"\n },\n\n \"SetRoundingMode GPR:$RoundMode, i1:$SetDAZ, GPR:$MXCSR\": {\n \"Desc\": [\"Sets the current rounding mode options for the thread\"\n ],\n \"HasSideEffects\": true\n },\n \"GPR = PushRoundingMode u8:$RoundMode\": {\n \"Desc\": [\"Override the current rounding mode options for the thread, returning old FPCR\"\n ],\n \"DestSize\": \"OpSize::i64Bit\",\n \"HasSideEffects\": true\n },\n \"PopRoundingMode GPR:$FPCR\": {\n \"Desc\": [\"Resets rounding mode after PushRoundingMode operation\"\n ],\n \"HasSideEffects\": true\n },\n \"Print SSA:$Value\": {\n \"HasSideEffects\": true,\n \"Desc\": [\"Debug operation that prints an SSA value to the console\",\n \"May only print 64bits of the value\"]\n },\n \"GPR = AllocateGPR i1:$ForPair\": {\n \"Desc\": [\"Silly pseudo-instruction to allocate a register for a future destination\",\n \"Note: if an instruction uses allocated destinations-as-sources,\",\n \"it cannot use a regular destination too. This ensures RA correctness.\",\n \"This is a kludge to deal with the IR's lack of multiple destinations\",\n \"If ForPair is set, RA will try to allocate the base of a register pair\"],\n \"DestSize\": \"OpSize::i64Bit\",\n \"JITDispatch\": false\n },\n \"FPR = AllocateFPR OpSize:#RegisterSize, OpSize:#ElementSize\": {\n \"Desc\": [\"Like AllocateGPR, but for FPR\"],\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\",\n \"JITDispatch\": false\n },\n \"GPR = AllocateGPRAfter GPR:$After\": {\n \"Desc\": [\"Silly pseudo-instruction to allocate a register for a future destination\",\n \"This is a kludge to deal with the IR's lack of multiple destinations\",\n \"RA will attempt to allocate to the register after $After.\",\n \"It may not succeed.\"],\n \"DestSize\": \"OpSize::i64Bit\",\n \"JITDispatch\": false\n },\n \"GPR = RDRAND i1:$GetReseeded\": {\n \"Desc\": [\"Uses the hardware random number generator to generate a 64bit number\",\n \"The boolean argument asks if we should be reading the reseeded number or not\",\n \"Reseeded RNG calculation is more expensive and will be heavier to use\",\n \"Returns the 64-bit number\",\n \"Sets the Z flag if the number is valid.\",\n \"RNG hardware is allowed to fail early and return. Software must always check this\"\n ],\n \"HasSideEffects\": true,\n \"DestSize\": \"OpSize::i64Bit\"\n },\n \"Yield\": {\n \"HasSideEffects\": true,\n \"Desc\": [\"This is a hint instruction that the CPU is likely to do a spin so it might want to pause to help out SMP\",\n \"Can be implemented as a NOP if necessary\"]\n },\n \"WFET GPR:$Upper, GPR:$Lower\": {\n \"HasSideEffects\": true,\n \"Desc\": [\n \"Implement a low power wait attempting to sleep until RDTSC >= Upper:Lower.\",\n \"Will spuriously wake up.\"\n\t]\n },\n \"MonoBackpatcherWrite OpSize:$Size, GPR:$Value, GPR:$Addr\": {\n \"HasSideEffects\": true,\n \"Desc\": [ \"Writes and invalidates the target address with the invalidation mutex locked. This is a fault-avoiding\",\n \"replacement for the atomic SMC writes used in the mono callsite backpatcher.\" ],\n \"EmitValidation\": [\n \"Size == FEXCore::IR::OpSize::i32Bit || Size == FEXCore::IR::OpSize::i64Bit\"\n ]\n }\n },\n \"Branch\": {\n \"Jump SSA:$TargetBlock\": {\n \"HasSideEffects\": true,\n \"RAOverride\": \"0\"\n },\n \"CondJump SSA:$Cmp1, SSA:$Cmp2, SSA:$TrueBlock, SSA:$FalseBlock, CondClass:$Cond{CondClass::NEQ}, OpSize:$CompareSize{OpSize::iInvalid}, i1:$FromNZCV{false}\": {\n \"Inline\": [\"\", \"AddSub\"],\n \"HasSideEffects\": true,\n \"RAOverride\": \"2\"\n },\n \"ExitFunction OpSize:#Size, GPR:$NewRIP, BranchHint:$Hint, GPR:$CallReturnAddress, SSA:$CallReturnBlock\": {\n \"Desc\": [\"Exits the current JIT function with a target RIP\"\n ],\n \"Inline\": [\"Any\"],\n \"HasSideEffects\": true,\n \"DestSize\": \"Size\",\n \"RAOverride\": \"2\"\n },\n \"Break BreakDefinition:$Reason\": {\n \"HasSideEffects\": true\n },\n \"CallbackReturn\": {\n \"HasSideEffects\": true\n },\n \"GPR = Syscall GPR:$SyscallID, GPR:$Arg0, GPR:$Arg1, GPR:$Arg2, GPR:$Arg3, GPR:$Arg4, GPR:$Arg5\": {\n \"HasSideEffects\": true,\n \"Desc\": [\"Dispatches a guest syscall through to the SyscallHandler class\"\n ],\n \"DestSize\": \"OpSize::i64Bit\"\n },\n\n \"Thunk GPR:$ArgPtr, SHA256Sum:$ThunkNameHash\": {\n \"HasSideEffects\": true\n },\n\n \"GPR:$EAX, GPR:$EBX, GPR:$ECX, GPR:$EDX = CPUID GPR:$Function, GPR:$Leaf\": {\n \"Desc\": [\"Calls in to the CPUID handler function to return emulated CPUID\"],\n \"DestSize\": \"OpSize::i32Bit\",\n \"HasSideEffects\": true\n },\n \"GPR:$EAX, GPR:$EDX = XGetBV GPR:$Function\": {\n \"Desc\": [\"Calls in to the XCR handler function to return emulated XCR\"],\n \"DestSize\": \"OpSize::i32Bit\",\n \"HasSideEffects\": true\n }\n },\n \"Moves\": {\n \"GPR = Copy GPR:$Source\": {\n \"Desc\": [\"GPR copy, generated by RA to split live ranges\"],\n \"DestSize\": \"OpSize::i64Bit\"\n }\n },\n \"StaticRA\": {\n \"SSA = LoadRegister u32:$Reg, RegisterClass:$Class, OpSize:#Size\": {\n \"Desc\": [\"Loads a value from the given register\",\n \"Size must match the execution mode.\"],\n \"DestSize\": \"Size\"\n },\n\n \"GPR = LoadPF OpSize:#Size\": {\n \"Desc\": [\"Loads raw PF\"],\n \"DestSize\": \"Size\"\n },\n\n \"GPR = LoadAF OpSize:#Size\": {\n \"Desc\": [\"Loads raw PF\"],\n \"DestSize\": \"Size\"\n },\n\n \"SSA = StoreRegister SSA:$Value, OpSize:#Size\": {\n \"HasSideEffects\": true,\n \"Desc\": [\"Stores a value to a given register.\",\n \"Size must match the execution mode.\"],\n \"DestSize\": \"Size\"\n },\n\n \"StorePF GPR:$Value, OpSize:#Size\": {\n \"HasSideEffects\": true,\n \"Desc\": [\"Stores raw PF\"],\n \"DestSize\": \"Size\"\n },\n\n \"StoreAF GPR:$Value, OpSize:#Size\": {\n \"HasSideEffects\": true,\n \"Desc\": [\"Stores raw AF\"],\n \"DestSize\": \"Size\"\n }\n },\n \"Memory\": {\n \"SSA = LoadContext OpSize:#ByteSize, RegisterClass:$Class, u32:$Offset\": {\n \"Desc\": [\"Loads a value from the context with offset\",\n \"Dest = Ctx[Offset]\"\n ],\n \"DestSize\": \"ByteSize\",\n \"EmitValidation\": [\n \"($Class == RegClass::GPR && (#ByteSize == IR::OpSize::i8Bit || #ByteSize == IR::OpSize::i16Bit || #ByteSize == IR::OpSize::i32Bit || #ByteSize == IR::OpSize::i64Bit)) || $Class == RegClass::FPR\",\n \"($Class == RegClass::FPR && (#ByteSize == IR::OpSize::i8Bit || #ByteSize == IR::OpSize::i16Bit || #ByteSize == IR::OpSize::i32Bit || #ByteSize == IR::OpSize::i64Bit || #ByteSize == IR::OpSize::i128Bit || #ByteSize == IR::OpSize::i256Bit)) || $Class == RegClass::GPR\",\n \"!($Offset >= offsetof(Core::CPUState, gregs[0]) && $Offset < offsetof(Core::CPUState, gregs[16])) && \\\"Can't LoadContext to GPR\\\"\",\n \"!($Offset >= offsetof(Core::CPUState, xmm.avx.data[0]) && $Offset < offsetof(Core::CPUState, xmm.avx.data[16])) && \\\"Can't LoadContext to XMM\\\"\"\n ]\n },\n\n \"SSA:$Value1, SSA:$Value2 = LoadContextPair OpSize:#ByteSize, RegisterClass:$Class, u32:$Offset\": {\n \"Desc\": [\"Loads a pair of values from the context with offset\",\n \"Value0 = Ctx[Offset], Value1 = Ctx[Offset + ByteSize]\"\n ],\n \"HasSideEffects\": true,\n \"DestSize\": \"ByteSize\",\n \"EmitValidation\": [\n \"($Class == RegClass::GPR && (#ByteSize == IR::OpSize::i8Bit || #ByteSize == IR::OpSize::i16Bit || #ByteSize == IR::OpSize::i32Bit || #ByteSize == IR::OpSize::i64Bit)) || $Class == RegClass::FPR\",\n \"($Class == RegClass::FPR && (#ByteSize == IR::OpSize::i8Bit || #ByteSize == IR::OpSize::i16Bit || #ByteSize == IR::OpSize::i32Bit || #ByteSize == IR::OpSize::i64Bit || #ByteSize == IR::OpSize::i128Bit || #ByteSize == IR::OpSize::i256Bit)) || $Class == RegClass::GPR\",\n \"!($Offset >= offsetof(Core::CPUState, gregs[0]) && $Offset < offsetof(Core::CPUState, gregs[16])) && \\\"Can't LoadContext to GPR\\\"\",\n \"!($Offset >= offsetof(Core::CPUState, xmm.avx.data[0]) && $Offset < offsetof(Core::CPUState, xmm.avx.data[16])) && \\\"Can't LoadContext to XMM\\\"\"\n ]\n },\n\n \"StoreContext OpSize:#ByteSize, RegisterClass:$Class, SSA:$Value, u32:$Offset\": {\n \"Desc\": [\"Stores a value to the context with offset\",\n \"Ctx[Offset] = Value\",\n \"Zero Extends if value's type is too small\",\n \"Truncates if value's type is too large\"\n ],\n \"Inline\": [\"Zero\", \"\"],\n \"HasSideEffects\": true,\n \"DestSize\": \"ByteSize\",\n \"EmitValidation\": [\n \"($Class == RegClass::GPR && (#ByteSize == IR::OpSize::i8Bit || #ByteSize == IR::OpSize::i16Bit || #ByteSize == IR::OpSize::i32Bit || #ByteSize == IR::OpSize::i64Bit)) || $Class == RegClass::FPR\",\n \"($Class == RegClass::FPR && (#ByteSize == IR::OpSize::i8Bit || #ByteSize == IR::OpSize::i16Bit || #ByteSize == IR::OpSize::i32Bit || #ByteSize == IR::OpSize::i64Bit || #ByteSize == IR::OpSize::i128Bit || #ByteSize == IR::OpSize::i256Bit)) || $Class == RegClass::GPR\",\n \"!($Offset >= offsetof(Core::CPUState, gregs[0]) && $Offset < offsetof(Core::CPUState, gregs[16])) && \\\"Can't StoreContext to GPR\\\"\",\n \"!($Offset >= offsetof(Core::CPUState, xmm.avx.data[0]) && $Offset < offsetof(Core::CPUState, xmm.avx.data[16])) && \\\"Can't StoreContext to XMM\\\"\"\n ]\n },\n\n \"StoreContextPair OpSize:#ByteSize, RegisterClass:$Class, SSA:$Value1, SSA:$Value2, u32:$Offset\": {\n \"Desc\": [\"Stores a pair of values to the context with offset\",\n \"Ctx[Offset] = Value1, Ctx[Offset + ByteSize] = Value2\",\n \"Zero Extends if value's type is too small\",\n \"Truncates if value's type is too large\"\n ],\n \"HasSideEffects\": true,\n \"DestSize\": \"ByteSize\",\n \"EmitValidation\": [\n \"WalkFindRegClass($Value1) == $Class\",\n \"WalkFindRegClass($Value2) == $Class\",\n \"($Class == RegClass::GPR && (#ByteSize == IR::OpSize::i8Bit || #ByteSize == IR::OpSize::i16Bit || #ByteSize == IR::OpSize::i32Bit || #ByteSize == IR::OpSize::i64Bit)) || $Class == RegClass::FPR\",\n \"($Class == RegClass::FPR && (#ByteSize == IR::OpSize::i8Bit || #ByteSize == IR::OpSize::i16Bit || #ByteSize == IR::OpSize::i32Bit || #ByteSize == IR::OpSize::i64Bit || #ByteSize == IR::OpSize::i128Bit || #ByteSize == IR::OpSize::i256Bit)) || $Class == RegClass::GPR\",\n \"!($Offset >= offsetof(Core::CPUState, gregs[0]) && $Offset < offsetof(Core::CPUState, gregs[16])) && \\\"Can't StoreContext to GPR\\\"\",\n \"!($Offset >= offsetof(Core::CPUState, xmm.avx.data[0]) && $Offset < offsetof(Core::CPUState, xmm.avx.data[16])) && \\\"Can't StoreContext to XMM\\\"\"\n ]\n },\n\n \"SSA = LoadContextIndexed GPR:$Index, OpSize:#ByteSize, u32:$BaseOffset, u32:$Stride, RegisterClass:$Class\": {\n \"Desc\": [\"Loads a value from the context with offset and indexed by SSA value\",\n \"Dest = Ctx[BaseOffset + Index * Stride]\"\n ],\n \"DestSize\": \"ByteSize\",\n \"EmitValidation\": [\n \"($Class == RegClass::GPR && (#ByteSize == IR::OpSize::i8Bit || #ByteSize == IR::OpSize::i16Bit || #ByteSize == IR::OpSize::i32Bit || #ByteSize == IR::OpSize::i64Bit)) || $Class == RegClass::FPR\",\n \"($Class == RegClass::FPR && (#ByteSize == IR::OpSize::i8Bit || #ByteSize == IR::OpSize::i16Bit || #ByteSize == IR::OpSize::i32Bit || #ByteSize == IR::OpSize::i64Bit || #ByteSize == IR::OpSize::i128Bit || #ByteSize == IR::OpSize::i256Bit)) || $Class == RegClass::GPR\",\n \"!($BaseOffset >= offsetof(Core::CPUState, gregs[0]) && $BaseOffset < offsetof(Core::CPUState, gregs[16])) && \\\"Can't LoadContextIndexed to GPR\\\"\",\n \"!($BaseOffset >= offsetof(Core::CPUState, xmm.avx.data[0]) && $BaseOffset < offsetof(Core::CPUState, xmm.avx.data[16])) && \\\"Can't LoadContextIndexed to XMM\\\"\"\n ]\n },\n \"StoreContextIndexed SSA:$Value, GPR:$Index, OpSize:#ByteSize, u32:$BaseOffset, u32:$Stride, RegisterClass:$Class\": {\n \"HasSideEffects\": true,\n \"Desc\": [\"Stores a value to the context with offset and indexed by SSA value\",\n \"Ctx[BaseOffset + Index * Stride] = Value\"\n ],\n \"DestSize\": \"ByteSize\",\n \"EmitValidation\": [\n \"WalkFindRegClass($Value) == $Class\",\n \"($Class == RegClass::GPR && (#ByteSize == IR::OpSize::i8Bit || #ByteSize == IR::OpSize::i16Bit || #ByteSize == IR::OpSize::i32Bit || #ByteSize == IR::OpSize::i64Bit)) || $Class == RegClass::FPR\",\n \"($Class == RegClass::FPR && (#ByteSize == IR::OpSize::i8Bit || #ByteSize == IR::OpSize::i16Bit || #ByteSize == IR::OpSize::i32Bit || #ByteSize == IR::OpSize::i64Bit || #ByteSize == IR::OpSize::i128Bit || #ByteSize == IR::OpSize::i256Bit)) || $Class == RegClass::GPR\",\n \"!($BaseOffset >= offsetof(Core::CPUState, gregs[0]) && $BaseOffset < offsetof(Core::CPUState, gregs[16])) && \\\"Can't StoreContextIndexed to GPR\\\"\",\n \"!($BaseOffset >= offsetof(Core::CPUState, xmm.avx.data[0]) && $BaseOffset < offsetof(Core::CPUState, xmm.avx.data[16])) && \\\"Can't StoreContextIndexed to XMM\\\"\"\n ]\n },\n \"GPR = FormContextAddress OpSize:#Size, GPR:$Index, u32:$Stride\": {\n \"Desc\": [\"Forms an address into the context structure indexed by SSA value\",\n \"Dest = Ctx + Index * Stride\",\n \"This allows backends to compute the address once and reuse it for multiple memory operations\",\n \"Stride must be a power of 2\"\n ],\n \"DestSize\": \"Size\",\n \"EmitValidation\": [\n \"#Size == IR::OpSize::i64Bit\"\n ]\n },\n\n \"SpillRegister SSA:$Value, u32:$Slot, RegisterClass:$Class\": {\n \"HasSideEffects\": true,\n \"Desc\": [\"Spills an SSA value to memory\",\n \"Spill slots are register allocated and has live ranges calculated to handle slot calculation\",\n \"!Don't use this op. It is for RA to handle spilling and filling!\"\n ],\n \"EmitValidation\": [\n \"WalkFindRegClass($Value) == $Class\"\n ]\n },\n\n \"SSA = FillRegister OpSize:#Size, OpSize:#ElementSize, u32:$Slot, RegisterClass:$Class\": {\n \"Desc\": [\"Fills a register from a spill slot\",\n \"Spill slots are register allocated and has live ranges calculated to handle slot calculation\",\n \"!Don't use this op. It is for RA to handle spilling and filling!\"\n ],\n \"DestSize\": \"Size\",\n \"ElementSize\": \"ElementSize\"\n },\n\n \"GPR = LoadNZCV\": {\n \"Desc\": [\"Loads value of NZCV register\"],\n \"DestSize\": \"OpSize::i32Bit\"\n },\n\n \"StoreNZCV GPR:$Value\": {\n \"HasSideEffects\": true,\n \"Desc\": [\"Stores value to NZCV register\"],\n \"DestSize\": \"OpSize::i32Bit\"\n },\n\n \"GPR = LoadDF\": {\n \"Desc\": [\"Loads the decimal flag from the context object in -1/1\",\n \"representation for easy consumption\"\n ],\n \"DestSize\": \"OpSize::i64Bit\"\n },\n\n \"SSA = LoadMem RegisterClass:$Class, OpSize:#Size, GPR:$Addr, GPR:$Offset, OpSize:$Align, MemOffsetType:$OffsetType, u8:$OffsetScale\": {\n \"Inline\": [\"\", \"Mem\"],\n \"DestSize\": \"Size\"\n },\n\n \"SSA:$Value1, SSA:$Value2 = LoadMemPair RegisterClass:$Class, OpSize:#Size, GPR:$Addr, u32:$Offset\": {\n \"Desc\": [\"Load a pair of values from memory.\"],\n \"DestSize\": \"Size\",\n \"HasSideEffects\": true\n },\n\n \"StoreMem RegisterClass:$Class, OpSize:#Size, SSA:$Value, GPR:$Addr, GPR:$Offset, OpSize:$Align, MemOffsetType:$OffsetType, u8:$OffsetScale\": {\n \"Desc\": [ \"Stores a value to memory.\",\n \"Zero Extends if value's type is too small\",\n \"Truncates if value's type is too large\"\n ],\n \"Inline\": [\"Zero\", \"\", \"Mem\"],\n \"HasSideEffects\": true,\n \"DestSize\": \"Size\"\n },\n\n \"StoreMemPair RegisterClass:$Class, OpSize:#Size, SSA:$Value1, SSA:$Value2, GPR:$Addr, u32:$Offset\": {\n \"Desc\": [ \"Stores a pair of values to memory.\",\n \"Zero Extends if value's type is too small\",\n \"Truncates if value's type is too large\"\n ],\n \"Inline\": [\"Zero\", \"Zero\"],\n \"HasSideEffects\": true,\n \"DestSize\": \"Size\"\n },\n\n \"StoreMemX87SVEOptPredicate OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Value, GPR:$Addr\": {\n \"Desc\": [ \"Stores a value to memory using SVE predicate mask that's designed\",\n \"specifically for use in the X87 SVE Ldst optimization.\" ],\n \"DestSize\": \"RegisterSize\",\n \"HasSideEffects\": true,\n \"ElementSize\": \"ElementSize\"\n },\n \"FPR = LoadMemX87SVEOptPredicate OpSize:#RegisterSize, OpSize:#ElementSize, GPR:$Addr\": {\n \"Desc\": [ \"Loads a value to memory using SVE predicate mask that's designed\",\n \"specifically for use in the X87 SVE Ldst optimization.\" ],\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n\n \"SSA = LoadMemTSO RegisterClass:$Class, OpSize:#Size, GPR:$Addr, GPR:$Offset, OpSize:$Align, MemOffsetType:$OffsetType, u8:$OffsetScale\": {\n \"Desc\": [\"Does a x86 TSO compatible load from memory. Offset must be Invalid().\"\n ],\n \"Inline\": [\"\", \"Memtso\"],\n \"DestSize\": \"Size\"\n },\n\n \"StoreMemTSO RegisterClass:$Class, OpSize:#Size, SSA:$Value, GPR:$Addr, GPR:$Offset, OpSize:$Align, MemOffsetType:$OffsetType, u8:$OffsetScale\": {\n \"Desc\": [\"Does a x86 TSO compatible store to memory. Offset must be Invalid().\"\n ],\n \"Inline\": [\"Zero\", \"\", \"Memtso\"],\n \"HasSideEffects\": true,\n \"DestSize\": \"Size\"\n },\n\n \"FPR = VLoadVectorMasked OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Mask, GPR:$Addr, GPR:$Offset, MemOffsetType:$OffsetType, u8:$OffsetScale\": {\n \"Desc\": [\"Does a masked load similar to VPMASKMOV/VMASKMOV where the upper bit of each element\",\n \"determines whether or not that element will be loaded from memory\"],\n \"ImplicitFlagClobber\": true,\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n \"VStoreVectorMasked OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Mask, FPR:$Data, GPR:$Addr, GPR:$Offset, MemOffsetType:$OffsetType, u8:$OffsetScale\": {\n \"Desc\": [\"Does a masked store similar to VPMASKMOV/VMASKMOV where the upper bit of each element\",\n \"determines whether or not that element will be stored to memory\"],\n \"HasSideEffects\": true,\n \"ImplicitFlagClobber\": true,\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n \"FPR = VLoadVectorGatherMasked OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Incoming, FPR:$Mask, GPR:$AddrBase, FPR:$VectorIndexLow, FPR:$VectorIndexHigh, OpSize:$VectorIndexElementSize, u8:$OffsetScale, u8:$DataElementOffsetStart, u8:$IndexElementOffsetStart, OpSize:$AddrSize\": {\n \"Desc\": [\n \"Does a masked load similar to VPGATHERD* where the upper bit of each element\",\n \"determines whether or not that element will be loaded from memory.\",\n \"Most of VSIB encoding is passed directly through to the IR operation.\"\n ],\n \"TiedSource\": 0,\n \"ImplicitFlagClobber\": true,\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\",\n \"EmitValidation\": [\n \"$VectorIndexElementSize == OpSize::i32Bit || $VectorIndexElementSize == OpSize::i64Bit\"\n ]\n },\n \"FPR = VLoadVectorGatherMaskedQPS OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Incoming, FPR:$MaskReg, GPR:$AddrBase, FPR:$VectorIndexLow, FPR:$VectorIndexHigh, u8:$OffsetScale, OpSize:$AddrSize\": {\n \"Desc\": [\n \"Does a masked load similar to VPGATHERQPS where the upper bit of each element\",\n \"determines whether or not that element will be loaded from memory.\",\n \"Most of VSIB encoding is passed directly through to the IR operation.\",\n \"Only supports the case of 32-bit data element sizes from 64-bit addresses\"\n ],\n \"TiedSource\": 0,\n \"ImplicitFlagClobber\": true,\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\",\n \"EmitValidation\": [\n \"ElementSize == OpSize::i32Bit\",\n \"RegisterSize != FEXCore::IR::OpSize::i256Bit && \\\"What does 256-bit mean in this context?\\\"\"\n ]\n },\n \"FPR = VLoadVectorElement OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$DstSrc, u8:$Index, GPR:$Addr\": {\n \"Desc\": [\"Does a memory load to a single element of a vector.\",\n \"Leaves the rest of the vector's data intact.\",\n \"Matches arm64 ld1 semantics\"],\n \"TiedSource\": 0,\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n \"VStoreVectorElement OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Value, u8:$Index, GPR:$Addr\": {\n \"Desc\": [\"Does a memory store of a single element of a vector.\",\n \"Matches arm64 st1 semantics\"],\n \"HasSideEffects\": true,\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n \"FPR = VBroadcastFromMem OpSize:#RegisterSize, OpSize:#ElementSize, GPR:$Address\": {\n \"Desc\": [\"Broadcasts an ElementSize value from memory into each element of a vector.\"],\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n \"GPR = Push OpSize:#Size, OpSize:$ValueSize, GPR:$Value, GPR:$Addr\": {\n \"Desc\": [\n \"Pushes a value to the address, returning the new pointer after incrementing.\",\n \"The address is decremented by the value size while.\",\n \"The return value size is the size of the current operating mode\"\n ],\n \"TiedSource\": 1,\n \"HasSideEffects\": true,\n \"DestSize\": \"Size\"\n },\n \"PushTwo OpSize:#Size, OpSize:$ValueSize, GPR:$Value1, GPR:$Value2, GPR:$Addr\": {\n \"Desc\": [\n \"Push two values to the address, incrementing the pointer in the place.\",\n \"Fused post-RA so doesn't have a destination.\"\n ],\n \"HasSideEffects\": true\n },\n \"GPR = RMWHandle GPR:$Value\": {\n \"Desc\": [\n \"This is a special move that indicates the result will be poisoned by a non-SSA instruction writing to its result.\",\n \"In effect, it serves to prevent invalid optimizations with non-SSA instructions.\"\n ],\n \"DestSize\": \"OpSize::i64Bit\",\n \"HasSideEffects\": true,\n \"TiedSource\": 0\n },\n \"GPR:$Addr, GPR:$Value = Pop OpSize:$Size, GPR:$Addr\": {\n \"Desc\": [\n \"Pops a value from the address, updating the new pointer after incrementing.\",\n \"The address is incremented by the size via an RMW source/destintaion.\"\n ],\n \"HasSideEffects\": true,\n \"DestSize\": \"Size\"\n },\n \"GPR:$Addr, GPR:$Value1, GPR:$Value2 = PopTwo OpSize:$Size, GPR:$Addr\": {\n \"Desc\": [\"Pop two values from the address. Fused post-RA.\"],\n \"HasSideEffects\": true,\n \"DestSize\": \"Size\"\n },\n \"GPR = MemSet i1:$IsAtomic, OpSize:$Size, GPR:$Prefix, GPR:$Addr, GPR:$Value, GPR:$Length, GPR:$Direction\": {\n \"Desc\": [\"Duplicates behaviour of x86 STOS repeat\",\n \"Returns the final address that gets generated without the prefix appended.\"\n ],\n \"Inline\": [\"\", \"\", \"Zero\", \"\", \"Any\"],\n \"HasSideEffects\": true,\n \"DestSize\": \"OpSize::i64Bit\"\n },\n \"GPR:$DstAddress, GPR:$SrcAddress = MemCpy i1:$IsAtomic, OpSize:$Size, GPR:$Dest, GPR:$Src, GPR:$Length, GPR:$Direction\": {\n \"Desc\": [\"Duplicates behaviour of x86 MOVS repeat\",\n \"Returns the final addresses after they have been incremented or decremented\"\n ],\n \"Inline\": [\"\", \"\", \"\", \"Any\"],\n \"HasSideEffects\": true,\n \"DestSize\": \"OpSize::i64Bit\"\n },\n \"CacheLineClear GPR:$Addr, i1:$Serialize\": {\n \"Desc\": [\"Does a 64 byte cacheline clear at the address specified\",\n \"Only clears the data cachelines. Doesn't do any zeroing\",\n \"Can skip serialization if requested.\"\n ],\n \"HasSideEffects\": true\n },\n \"CacheLineClean GPR:$Addr\": {\n \"Desc\": [\"Does a 64 byte cacheline cleanat the address specified\",\n \"Only cleans the data cachelines. Doesn't do any zeroing\",\n \"Skips the invalidation step of the CacheLineClear operation\"\n ],\n \"HasSideEffects\": true\n },\n \"CacheLineZero GPR:$Addr\": {\n \"Desc\": [\"Does a 64 byte zero at the address specified\",\n \"Writing zeroes to memory\",\n \"It is specifically non-temporal and weakly ordered\",\n \"This matches CLZero behaviour\"\n ],\n \"HasSideEffects\": true\n },\n \"Fence FenceType:$Fence\": {\n \"Desc\": [\"Does a memory fence operation of the desired type\",\n \"FenceType::Load: Ensures load memory operations are serialized\",\n \"FenceType::Store: Ensures store memory operations are serialized\",\n \"FenceType::LoadStore: Ensures loads and store memory operations are serialized\",\n \"FenceType::Inst: Instruction barrier. Ensures all instructions after this point will be explicitly fetched\",\n \"Ensures the memory operations are globally visible\"\n ],\n \"HasSideEffects\": true\n },\n \"Prefetch i1:$ForStore, i1:$Stream, i8:$CacheLevel, GPR:$Addr, GPR:$Offset, MemOffsetType:$OffsetType, u8:$OffsetScale\": {\n \"Desc\": [\"Does a cacheline prefetch operation\"\n ],\n \"Inline\": [\"\", \"Mem\"],\n \"EmitValidation\": [\"CacheLevel > 0 && CacheLevel < 4\"],\n \"HasSideEffects\": true,\n \"DestSize\": \"OpSize::i64Bit\"\n },\n \"VStoreNonTemporal OpSize:#RegisterSize, FPR:$Value, GPR:$Addr, i8:$Offset\": {\n \"Desc\": [\"Does a non-temporal memory store of a vector.\",\n \"Matches arm64 SVE stnt1b semantics.\",\n \"Specifically weak-memory model ordered to match x86 non-temporal stores.\"\n ],\n \"HasSideEffects\": true,\n \"DestSize\": \"RegisterSize\",\n \"EmitValidation\": [\n \"RegisterSize == FEXCore::IR::OpSize::i128Bit || RegisterSize == FEXCore::IR::OpSize::i256Bit\",\n \"Offset % IR::OpSizeToSize(RegisterSize) == 0\"\n ]\n },\n \"VStoreNonTemporalPair OpSize:#RegisterSize, FPR:$ValueLow, FPR:$ValueHigh, GPR:$Addr, i8:$Offset\": {\n \"Desc\": [\"Does a non-temporal memory store of two vector registers.\",\n \"Matches arm64 stnp semantics.\",\n \"Specifically weak-memory model ordered to match x86 non-temporal stores.\"\n ],\n \"HasSideEffects\": true,\n \"DestSize\": \"RegisterSize\",\n \"EmitValidation\": [\n \"RegisterSize == FEXCore::IR::OpSize::i128Bit\",\n \"Offset % IR::OpSizeToSize(RegisterSize) == 0\"\n ]\n },\n \"FPR = VLoadNonTemporal OpSize:#RegisterSize, GPR:$Addr, i8:$Offset\": {\n \"Desc\": [\"Does a non-temporal memory load of a vector.\",\n \"Matches arm64 SVE ldnt1b semantics.\",\n \"Specifically weak-memory model ordered to match x86 non-temporal stores.\"\n ],\n \"HasSideEffects\": true,\n \"DestSize\": \"RegisterSize\",\n \"EmitValidation\": [\n \"RegisterSize == FEXCore::IR::OpSize::i128Bit || RegisterSize == FEXCore::IR::OpSize::i256Bit\",\n \"Offset % IR::OpSizeToSize(RegisterSize) == 0\"\n ]\n },\n \"ContextClear u32:$Offset, u32:$Size\": {\n \"Desc\": [\n \"Clears a region of the context by CLZero size\",\n \"Both the offset and size alignment need to be by CLZero size\"\n ],\n \"HasSideEffects\": true,\n \"EmitValidation\": [\n \"Offset % 64 == 0\",\n \"Size % 64 == 0\"\n ]\n }\n },\n \"Atomic\": {\n \"GPR = CAS OpSize:#Size, GPR:$Expected, GPR:$Desired, GPR:$Addr\": {\n \"HasSideEffects\": true,\n \"Desc\": [\"Does a compare and swap of values to a memory location\",\n \"This mostly matches the C++ atomic_compare_exchange_strong function\",\n \"Dest = atomic_compare_exchange_strong(%Addr, %Expected, %Desired)\",\n \"Depending on if the value in %Addr is Expected the results destination will be different\",\n \"Behaves like the following but atomically\",\n \"Dest = %Expected\",\n \"if (deref(%Addr) != %Expected) Dest = deref(%Addr)\"\n ],\n \"TiedSource\": 0,\n \"DestSize\": \"Size\",\n \"ImplicitFlagClobber\": true,\n \"EmitValidation\": [\n \"Size == FEXCore::IR::OpSize::i8Bit || Size == FEXCore::IR::OpSize::i16Bit || Size == FEXCore::IR::OpSize::i32Bit || Size == FEXCore::IR::OpSize::i64Bit\"\n ]\n },\n \"GPR:$Lo, GPR:$Hi = CASPair OpSize:#Size, GPR:$ExpectedLo, GPR:$ExpectedHi, GPR:$DesiredLo, GPR:$DesiredHi, GPR:$Addr\": {\n \"Desc\": [\"Does a compare and exchange with two pairs of values\",\n \"ssa0 is the comparison value\",\n \"ssa1 is the new value\",\n \"ssa2 is the memory location\",\n \"Returns the lower & upper halves of the value in memory.\"\n ],\n \"HasSideEffects\": true,\n \"DestSize\": \"Size\",\n \"EmitValidation\": [\n \"Size == FEXCore::IR::OpSize::i32Bit || Size == FEXCore::IR::OpSize::i64Bit\"\n ]\n },\n \"GPR = AtomicSwap OpSize:#Size, GPR:$Value, GPR:$Addr\": {\n \"HasSideEffects\": true,\n \"Desc\": [\"Atomic integer swap\"\n ],\n \"DestSize\": \"Size\",\n \"EmitValidation\": [\n \"Size == FEXCore::IR::OpSize::i8Bit || Size == FEXCore::IR::OpSize::i16Bit || Size == FEXCore::IR::OpSize::i32Bit || Size == FEXCore::IR::OpSize::i64Bit\"\n ]\n },\n \"GPR = AtomicFetchAdd OpSize:#Size, GPR:$Value, GPR:$Addr\": {\n \"HasSideEffects\": true,\n \"Desc\": [\"Atomic integer fetch and add\",\n \"Atomically fetches %Addr and adds %value to the memory location\",\n \"Dest is the value prior to operating on the value in memory\",\n \"IR layout must match NonFetch-variant, otherwise DCE IR optimization breaks!\"\n ],\n \"DestSize\": \"Size\",\n \"EmitValidation\": [\n \"Size == FEXCore::IR::OpSize::i8Bit || Size == FEXCore::IR::OpSize::i16Bit || Size == FEXCore::IR::OpSize::i32Bit || Size == FEXCore::IR::OpSize::i64Bit\"\n ]\n },\n \"GPR = AtomicFetchSub OpSize:#Size, GPR:$Value, GPR:$Addr\": {\n \"HasSideEffects\": true,\n \"Desc\": [\"Atomic integer fetch and sub\",\n \"Atomically fetches %Addr and subtracts %value to the memory location\",\n \"Dest is the value prior to operating on the value in memory\",\n \"IR layout must match NonFetch-variant, otherwise DCE IR optimization breaks!\"\n ],\n \"DestSize\": \"Size\",\n \"EmitValidation\": [\n \"Size == FEXCore::IR::OpSize::i8Bit || Size == FEXCore::IR::OpSize::i16Bit || Size == FEXCore::IR::OpSize::i32Bit || Size == FEXCore::IR::OpSize::i64Bit\"\n ]\n },\n \"GPR = AtomicFetchAnd OpSize:#Size, GPR:$Value, GPR:$Addr\": {\n \"HasSideEffects\": true,\n \"Desc\": [\"Atomic integer fetch and binary and\",\n \"Atomically fetches %Addr and binary ands %value to the memory location\",\n \"Dest is the value prior to operating on the value in memory\",\n \"IR layout must match NonFetch-variant, otherwise DCE IR optimization breaks!\"\n ],\n \"DestSize\": \"Size\",\n \"EmitValidation\": [\n \"Size == FEXCore::IR::OpSize::i8Bit || Size == FEXCore::IR::OpSize::i16Bit || Size == FEXCore::IR::OpSize::i32Bit || Size == FEXCore::IR::OpSize::i64Bit\"\n ]\n },\n \"GPR = AtomicFetchCLR OpSize:#Size, GPR:$Value, GPR:$Addr\": {\n \"HasSideEffects\": true,\n \"Desc\": [\"Atomic integer fetch and binary clear\",\n \"Atomically fetches %Addr and binary clears %value to the memory location\",\n \"Dest is the value prior to operating on the value in memory\",\n \"Matches ARM ldclral semantics\",\n \"eg: Dest[Addr] &= ~Value\",\n \"IR layout must match NonFetch-variant, otherwise DCE IR optimization breaks!\"\n ],\n \"DestSize\": \"Size\",\n \"EmitValidation\": [\n \"Size == FEXCore::IR::OpSize::i8Bit || Size == FEXCore::IR::OpSize::i16Bit || Size == FEXCore::IR::OpSize::i32Bit || Size == FEXCore::IR::OpSize::i64Bit\"\n ]\n },\n \"GPR = AtomicFetchOr OpSize:#Size, GPR:$Value, GPR:$Addr\": {\n \"HasSideEffects\": true,\n \"Desc\": [\"Atomic integer fetch and binary or\",\n \"Atomically fetches %Addr and binary ors %value to the memory location\",\n \"Dest is the value prior to operating on the value in memory\",\n \"IR layout must match NonFetch-variant, otherwise DCE IR optimization breaks!\"\n ],\n \"DestSize\": \"Size\",\n \"EmitValidation\": [\n \"Size == FEXCore::IR::OpSize::i8Bit || Size == FEXCore::IR::OpSize::i16Bit || Size == FEXCore::IR::OpSize::i32Bit || Size == FEXCore::IR::OpSize::i64Bit\"\n ]\n },\n \"GPR = AtomicFetchXor OpSize:#Size, GPR:$Value, GPR:$Addr\": {\n \"HasSideEffects\": true,\n \"Desc\": [\"Atomic integer fetch and binary exclusive or\",\n \"Atomically fetches %Addr and binary exclusive ors %value to the memory location\",\n \"Dest is the value prior to operating on the value in memory\",\n \"IR layout must match NonFetch-variant, otherwise DCE IR optimization breaks!\"\n ],\n \"DestSize\": \"Size\",\n \"EmitValidation\": [\n \"Size == FEXCore::IR::OpSize::i8Bit || Size == FEXCore::IR::OpSize::i16Bit || Size == FEXCore::IR::OpSize::i32Bit || Size == FEXCore::IR::OpSize::i64Bit\"\n ]\n },\n \"GPR = AtomicFetchNeg OpSize:#Size, GPR:$Addr\": {\n \"HasSideEffects\": true,\n \"Desc\": [\"Atomic integer fetch and two's complement negate\",\n \"Dest is the value prior to operating on the value in memory\",\n \"IR layout must match NonFetch-variant, otherwise DCE IR optimization breaks!\"\n ],\n \"DestSize\": \"Size\",\n \"EmitValidation\": [\n \"Size == FEXCore::IR::OpSize::i8Bit || Size == FEXCore::IR::OpSize::i16Bit || Size == FEXCore::IR::OpSize::i32Bit || Size == FEXCore::IR::OpSize::i64Bit\"\n ]\n },\n \"TelemetrySetValue GPR:$Value, u8:$TelemetryValueIndex\": {\n \"HasSideEffects\": true,\n \"Desc\": [\"Set Telemetry value if the passed in 32-bit value isn't zero.\",\n \"Only useful for 32-bit applications.\"\n ],\n \"ImplicitFlagClobber\": true,\n \"DestSize\": \"OpSize::i64Bit\"\n }\n },\n \"ALU\": {\n \"GPR = EntrypointOffset OpSize:#Size, i64:$Offset\": {\n \"Desc\": [\"Returns the + Offset address\",\n \"When the size is 4 bytes then 32-bit overflow and underflow needs to work\"\n ],\n \"DestSize\": \"Size\",\n \"EmitValidation\": [\n \"Size == FEXCore::IR::OpSize::i32Bit || Size == FEXCore::IR::OpSize::i64Bit\"\n ]\n },\n\n \"InlineEntrypointOffset OpSize:#Size, i64:$Offset\": {\n \"Desc\": [\"Returns the + Offset address\",\n \"When the size is 4 bytes then 32-bit overflow and underflow needs to work\"\n ],\n \"HasSideEffects\": true,\n \"RAOverride\": \"0\",\n \"DestSize\": \"Size\",\n \"EmitValidation\": [\n \"Size == FEXCore::IR::OpSize::i32Bit || Size == FEXCore::IR::OpSize::i64Bit\"\n ]\n },\n\n \"GPR = Constant i64:$Constant, ConstPad:$Pad{IR::ConstPad::NoPad}, i32:$MaxBytes{0}\": {\n \"Desc\": [\"Generates a 64bit constant inside of a GPR\",\n \"Unsupported to create a constant in FPR\"\n ],\n \"DestSize\": \"OpSize::i64Bit\",\n \"EmitValidation\": [\n \"MaxBytes >= 0 && MaxBytes <= 8 && (MaxBytes & 1) == 0\",\n \"MaxBytes == 0 || (Constant >> (MaxBytes * 8)) == 0\"\n ]\n },\n\n \"InlineConstant i64:$Constant\": {\n \"Desc\": [\"Generates a 64bit constant to be used directly, non-FPR\"],\n \"HasSideEffects\": true,\n \"RAOverride\": \"0\",\n \"DestSize\": \"OpSize::i64Bit\"\n },\n\n \"GPR = CycleCounter i1:$SelfSynchronizingLoads\": {\n \"Desc\": [\"Returns the host 64bit cycle counter\",\n \"Useful when emulating rdtsc\",\n \"Be careful, the frequency of this counter changes based on host\",\n \"On AArch64 make sure to query the CNTFRQ_EL0 system register to get the frequency\",\n \"On x86-64 make sure to query CPUID fn8000_0008[EDX_8] for constant TSC\",\n \"x86-64 constant frequency lives in MSR_PLATFORM_INFO. Which is only available to kernel\",\n \"Part of the ART frequency equation can be pulled from CPUID fn0000_0015[EBX & EAX]\",\n \"But it's missing the ART multiplier still?\",\n \"If the self-synchronizing flag is toggled then all instructions and loads must be completed before the cycle counter read\"\n ],\n \"DestSize\": \"OpSize::i64Bit\"\n },\n\n \"GPR = Neg OpSize:#Size, GPR:$Src, CondClass:$Cond{CondClass::AL}\": {\n \"Desc\": [\"Integer negation, with optional predication\",\n \"Dest = Cond ? -Src : Src\",\n \"Will truncate to 64 or 32bits\"\n ],\n \"DestSize\": \"Size\",\n \"EmitValidation\": [\n \"Size == FEXCore::IR::OpSize::i32Bit || Size == FEXCore::IR::OpSize::i64Bit\"\n ]\n },\n \"GPR = Not OpSize:#Size, GPR:$Src\": {\n \"Desc\": [\"Integer binary not\",\n \"op:\",\n \"Dest = ~Src\"\n ],\n \"DestSize\": \"Size\",\n \"EmitValidation\": [\n \"Size == FEXCore::IR::OpSize::i32Bit || Size == FEXCore::IR::OpSize::i64Bit\"\n ]\n },\n \"GPR = Popcount OpSize:#Size, GPR:$Src\": {\n \"Desc\": [\"Population count of source register\",\n \"Returns the number of bits set\"\n ],\n \"DestSize\": \"Size\",\n \"EmitValidation\": [\n \"Size == FEXCore::IR::OpSize::i8Bit || Size == FEXCore::IR::OpSize::i16Bit || Size == FEXCore::IR::OpSize::i32Bit || Size == FEXCore::IR::OpSize::i64Bit\"\n ]\n },\n \"GPR = FindLSB OpSize:#Size, GPR:$Src\": {\n \"Desc\": [\"Find least-significant-bit set\",\n \"Returns the index of the least significant bit set\",\n \"Undefined result if Src is zero.\"\n ],\n \"DestSize\": \"Size\",\n \"EmitValidation\": [\n \"Size == FEXCore::IR::OpSize::i8Bit || Size == FEXCore::IR::OpSize::i16Bit || Size == FEXCore::IR::OpSize::i32Bit || Size == FEXCore::IR::OpSize::i64Bit\"\n ]\n },\n \"GPR = FindMSB OpSize:#Size, GPR:$Src\": {\n \"Desc\": [\"Find most-significant-bit set\",\n \"Returns the index of the most significant bit set\",\n \"Undefined result if Src is zero.\"\n ],\n \"DestSize\": \"Size\",\n \"EmitValidation\": [\n \"Size == FEXCore::IR::OpSize::i16Bit || Size == FEXCore::IR::OpSize::i32Bit || Size == FEXCore::IR::OpSize::i64Bit\"\n ]\n },\n \"GPR = FindTrailingZeroes OpSize:#Size, GPR:$Src\": {\n \"Desc\": [\"Counts the number of trailing zero bits in a GPR\",\n \"Returns the number of bits that are zero trailing\",\n \"In the case of zero returns the size in bits of the input\"\n ],\n \"DestSize\": \"Size\",\n \"EmitValidation\": [\n \"Size == FEXCore::IR::OpSize::i16Bit || Size == FEXCore::IR::OpSize::i32Bit || Size == FEXCore::IR::OpSize::i64Bit\"\n ]\n },\n \"GPR = CountLeadingZeroes OpSize:#Size, GPR:$Src\": {\n \"Desc\": [\"Counts the number of leading zero bits in a GPR\",\n \"Returns the number of bits that are zero leading\",\n \"In the case of zero returns the size in bits of the input\"\n ],\n \"DestSize\": \"Size\",\n \"EmitValidation\": [\n \"Size == FEXCore::IR::OpSize::i16Bit || Size == FEXCore::IR::OpSize::i32Bit || Size == FEXCore::IR::OpSize::i64Bit\"\n ]\n },\n \"GPR = Rev OpSize:#Size, GPR:$Src\": {\n \"Desc\": [\"Reverses the byte order of the register\",\n \"Specifically 8bit byte swap size. (Not 16bit or 32bit word swapping)\"\n ],\n \"DestSize\": \"Size\",\n \"EmitValidation\": [\n \"Size == FEXCore::IR::OpSize::i16Bit || Size == FEXCore::IR::OpSize::i32Bit || Size == FEXCore::IR::OpSize::i64Bit\"\n ]\n },\n \"GPR = Rbit OpSize:#Size, GPR:$Src\": {\n \"Desc\": [\"Reverses the bit order of the register\"],\n \"DestSize\": \"Size\",\n \"EmitValidation\": [\n \"Size == FEXCore::IR::OpSize::i32Bit || Size == FEXCore::IR::OpSize::i64Bit\"\n ]\n },\n \"GPR = Add OpSize:#Size, GPR:$Src1, GPR:$Src2\": {\n \"Desc\": [ \"Integer Add\",\n \"Will truncate to 64 or 32bits\"\n ],\n \"Inline\": [\"\", \"LargeAddSub\"],\n \"DestSize\": \"Size\",\n \"EmitValidation\": [\n \"Size == FEXCore::IR::OpSize::i32Bit || Size == FEXCore::IR::OpSize::i64Bit\"\n ]\n },\n \"GPR = Adc OpSize:#Size, GPR:$Src1, GPR:$Src2\": {\n \"Desc\": [ \"Integer Add with carry\",\n \"Will truncate to 64 or 32bits\"\n ],\n \"Inline\": [\"Zero\", \"\"],\n \"DestSize\": \"Size\",\n \"EmitValidation\": [\n \"Size == FEXCore::IR::OpSize::i32Bit || Size == FEXCore::IR::OpSize::i64Bit\"\n ]\n },\n \"GPR = Sbb OpSize:#Size, GPR:$Src1, GPR:$Src2\": {\n \"Desc\": [ \"Integer Subtract with carry/borrow\",\n \"Will truncate to 64 or 32bits\"\n ],\n \"DestSize\": \"Size\",\n \"EmitValidation\": [\n \"Size == FEXCore::IR::OpSize::i32Bit || Size == FEXCore::IR::OpSize::i64Bit\"\n ]\n },\n \"GPR = AddShift OpSize:#Size, GPR:$Src1, GPR:$Src2, ShiftType:$Shift{ShiftType::LSL}, u8:$ShiftAmount{0}\": {\n \"Desc\": [ \"Integer Add with shifted register\",\n \"Will truncate to 64 or 32bits\",\n \"Dest = Src1 + (Src2 << ShiftAmount)\"\n ],\n \"DestSize\": \"Size\",\n \"EmitValidation\": [\n \"Size == FEXCore::IR::OpSize::i32Bit || Size == FEXCore::IR::OpSize::i64Bit\",\n \"Shift != ShiftType::ROR\"\n ]\n },\n \"GPR = AddWithFlags OpSize:#Size, GPR:$Src1, GPR:$Src2\": {\n \"Desc\": [ \"Integer add. Truncates and sets NZCV per AddNZCV\"],\n \"Inline\": [\"\", \"LargeAddSub\"],\n \"DestSize\": \"Size\",\n \"HasSideEffects\": true,\n \"EmitValidation\": [\n \"Size == FEXCore::IR::OpSize::i32Bit || Size == FEXCore::IR::OpSize::i64Bit\"\n ]\n },\n \"AddNZCV OpSize:#Size, GPR:$Src1, GPR:$Src2\": {\n \"Desc\": [\"Set NZCV for the sum of two GPRs\"],\n \"Inline\": [\"\", \"LargeAddSub\"],\n \"HasSideEffects\": true,\n \"DestSize\": \"Size\"\n },\n \"SetSmallNZV OpSize:#Size, GPR:$Src\": {\n \"Desc\": [\"Set NZV with a SETF instruction. Preserves CF.\"],\n \"HasSideEffects\": true,\n \"DestSize\": \"Size\",\n \"EmitValidation\": [\n \"Size == FEXCore::IR::OpSize::i8Bit || Size == FEXCore::IR::OpSize::i16Bit\"\n ]\n },\n \"CarryInvert\": {\n \"Desc\": [\"Invert carry flag in NZCV\"],\n \"HasSideEffects\": true\n },\n \"AXFlag GPR:$V_inv\": {\n \"Desc\": [\"After an FCmp, converts NZCV flags from the Arm format to a mysterious eXternal format\",\n \"On FlagM2-less platforms, takes the inverted 1/0 overflow flag\"],\n \"HasSideEffects\": true\n },\n \"GPR = Parity GPR:$Raw, i1:$Mask, i1:$Invert\": {\n \"Desc\": [\"Calculates PF\"],\n \"DestSize\": \"OpSize::i32Bit\"\n },\n \"RmifNZCV GPR:$Src, u8:$Rotate, u8:$Mask\": {\n \"Desc\": [\"Rotate, mask, and insert into NZCV on FlagM platforms\"],\n \"Inline\": [\"Zero\", \"\"],\n \"HasSideEffects\": true\n },\n \"CondAddNZCV OpSize:#Size, GPR:$Src1, GPR:$Src2, CondClass:$Cond, u8:$FalseNZCV\": {\n \"Desc\": [\"If condition is true, set NZCV per sum of GPRs, else force NZCV to a constant.\"],\n \"Inline\": [\"Zero\", \"AddSub\"],\n \"HasSideEffects\": true,\n \"DestSize\": \"Size\",\n \"EmitValidation\": [\n \"Size == FEXCore::IR::OpSize::i32Bit || Size == FEXCore::IR::OpSize::i64Bit\"\n ]\n },\n \"CondSubNZCV OpSize:#Size, GPR:$Src1, GPR:$Src2, CondClass:$Cond, u8:$FalseNZCV\": {\n \"Desc\": [\"If condition is true, set NZCV per difference of GPRs, else force NZCV to a constant.\"],\n \"Inline\": [\"Zero\", \"AddSub\"],\n \"HasSideEffects\": true,\n \"DestSize\": \"Size\",\n \"EmitValidation\": [\n \"Size == FEXCore::IR::OpSize::i32Bit || Size == FEXCore::IR::OpSize::i64Bit\"\n ]\n },\n \"GPR = AdcWithFlags OpSize:#Size, GPR:$Src1, GPR:$Src2\": {\n \"Desc\": [\"Adds and set NZCV for the sum of two GPRs and carry-in given as NZCV\"],\n \"Inline\": [\"Zero\", \"\"],\n \"HasSideEffects\": true,\n \"DestSize\": \"Size\",\n \"EmitValidation\": [\n \"Size == FEXCore::IR::OpSize::i32Bit || Size == FEXCore::IR::OpSize::i64Bit\"\n ]\n },\n \"GPR = AdcZero OpSize:#Size, GPR:$Src1\": {\n \"Desc\": [\"Adds GPR with inverted carry-in\"],\n \"DestSize\": \"Size\",\n \"EmitValidation\": [\n \"Size == FEXCore::IR::OpSize::i32Bit || Size == FEXCore::IR::OpSize::i64Bit\"\n ]\n },\n \"GPR = AdcZeroWithFlags OpSize:#Size, GPR:$Src1\": {\n \"Desc\": [\"Adds and set NZCV for the sum of GPR and inverted carry-in given as NZCV\"],\n \"HasSideEffects\": true,\n \"DestSize\": \"Size\",\n \"EmitValidation\": [\n \"Size == FEXCore::IR::OpSize::i32Bit || Size == FEXCore::IR::OpSize::i64Bit\"\n ]\n },\n \"GPR = SbbWithFlags OpSize:#Size, GPR:$Src1, GPR:$Src2\": {\n \"Desc\": [\"Subtracts and set NZCV for the difference of two GPRs and carry-in given as NZCV\"],\n \"HasSideEffects\": true,\n \"DestSize\": \"Size\",\n \"EmitValidation\": [\n \"Size == FEXCore::IR::OpSize::i32Bit || Size == FEXCore::IR::OpSize::i64Bit\"\n ]\n },\n \"AdcNZCV OpSize:#Size, GPR:$Src1, GPR:$Src2\": {\n \"Desc\": [\"Set NZCV for the sum of two GPRs and carry-in given as NZCV\"],\n \"HasSideEffects\": true,\n \"DestSize\": \"Size\",\n \"EmitValidation\": [\n \"Size == FEXCore::IR::OpSize::i32Bit || Size == FEXCore::IR::OpSize::i64Bit\"\n ]\n },\n \"SbbNZCV OpSize:#Size, GPR:$Src1, GPR:$Src2\": {\n \"Desc\": [\"Set NZCV for the difference of two GPRs and carry-in given as NZCV\"],\n \"HasSideEffects\": true,\n \"DestSize\": \"Size\",\n \"EmitValidation\": [\n \"Size == FEXCore::IR::OpSize::i32Bit || Size == FEXCore::IR::OpSize::i64Bit\"\n ]\n },\n \"GPR = Sub OpSize:#Size, GPR:$Src1, GPR:$Src2\": {\n \"Desc\": [ \"Integer Sub\",\n \"Will truncate to 64 or 32bits\"\n ],\n \"Inline\": [\"SubtractZero\", \"LargeAddSub\"],\n \"DestSize\": \"Size\",\n \"EmitValidation\": [\n \"Size == FEXCore::IR::OpSize::i32Bit || Size == FEXCore::IR::OpSize::i64Bit\"\n ]\n },\n \"GPR = SubShift OpSize:#Size, GPR:$Src1, GPR:$Src2, ShiftType:$Shift{ShiftType::LSL}, u8:$ShiftAmount{0}\": {\n \"Desc\": [ \"Integer Sub with shifted register\",\n \"Will truncate to 64 or 32bits\"\n ],\n \"DestSize\": \"Size\",\n \"EmitValidation\": [\n \"Size == FEXCore::IR::OpSize::i32Bit || Size == FEXCore::IR::OpSize::i64Bit\",\n \"Shift != ShiftType::ROR\"\n ]\n },\n \"GPR = SubWithFlags OpSize:#Size, GPR:$Src1, GPR:$Src2\": {\n \"Desc\": [ \"Integer Sub. Truncates and sets NZCV per SubNZCV\"],\n \"Inline\": [\"SubtractZero\", \"LargeAddSub\"],\n \"DestSize\": \"Size\",\n \"HasSideEffects\": true,\n \"EmitValidation\": [\n \"Size == FEXCore::IR::OpSize::i32Bit || Size == FEXCore::IR::OpSize::i64Bit\"\n ]\n },\n \"CmpPairZ OpSize:#Size, GPR:$Src1Lo, GPR:$Src1Hi, GPR:$Src2Lo, GPR:$Src2Hi\": {\n \"Desc\": [\"Compares register pairs and sets Z accordingly, preserving N/Z/V.\",\n \"This accelerates cmpxchg.\"],\n \"HasSideEffects\": true,\n \"DestSize\": \"Size\"\n },\n \"SubNZCV OpSize:#Size, GPR:$Src1, GPR:$Src2\": {\n \"Desc\": [\"Set NZCV for the difference of two GPRs. \",\n \"Carry flag uses arm64 definition, inverted x86.\",\n \"\"],\n \"Inline\": [\"Zero\", \"LargeAddSub\"],\n \"DestSize\": \"Size\",\n \"HasSideEffects\": true\n },\n \"GPR = Or OpSize:#Size, GPR:$Src1, GPR:$Src2\": {\n \"Desc\": [\"Integer binary or\"\n ],\n \"DestSize\": \"Size\",\n \"Inline\": [\"\", \"Logical\"],\n \"EmitValidation\": [\n \"Size == FEXCore::IR::OpSize::i32Bit || Size == FEXCore::IR::OpSize::i64Bit\"\n ]\n },\n \"GPR = Orlshl OpSize:#Size, GPR:$Src1, GPR:$Src2, u8:$BitShift\": {\n \"Desc\": [\"Integer binary or with logical shift left\"\n ],\n \"DestSize\": \"Size\",\n \"EmitValidation\": [\n \"Size == FEXCore::IR::OpSize::i32Bit || Size == FEXCore::IR::OpSize::i64Bit\"\n ]\n },\n \"GPR = Orlshr OpSize:#Size, GPR:$Src1, GPR:$Src2, u8:$BitShift\": {\n \"Desc\": [\"Integer binary or with logical shift right\"\n ],\n \"DestSize\": \"Size\",\n \"EmitValidation\": [\n \"Size == FEXCore::IR::OpSize::i32Bit || Size == FEXCore::IR::OpSize::i64Bit\"\n ]\n },\n \"GPR = Ornror OpSize:#Size, GPR:$Src1, GPR:$Src2, u8:$BitShift\": {\n \"Desc\": [\"Integer binary or with NOT on second source and rotation right\"\n ],\n \"DestSize\": \"Size\",\n \"EmitValidation\": [\n \"Size == FEXCore::IR::OpSize::i32Bit || Size == FEXCore::IR::OpSize::i64Bit\"\n ]\n },\n \"GPR = Xor OpSize:#Size, GPR:$Src1, GPR:$Src2\": {\n \"Desc\": [\"Integer binary exclusive or\"],\n \"Inline\": [\"\", \"Logical\"],\n \"DestSize\": \"Size\",\n \"EmitValidation\": [\n \"Size == FEXCore::IR::OpSize::i32Bit || Size == FEXCore::IR::OpSize::i64Bit\"\n ]\n },\n \"GPR = XorShift OpSize:#Size, GPR:$Src1, GPR:$Src2, ShiftType:$Shift{ShiftType::LSL}, u8:$ShiftAmount{0}\": {\n \"Desc\": [ \"Integer binary exclusive or with shifted register\"],\n \"DestSize\": \"Size\",\n \"EmitValidation\": [\n \"Size == FEXCore::IR::OpSize::i32Bit || Size == FEXCore::IR::OpSize::i64Bit\"\n ]\n },\n \"GPR = XornShift OpSize:#Size, GPR:$Src1, GPR:$Src2, ShiftType:$Shift{ShiftType::LSL}, u8:$ShiftAmount{0}\": {\n \"Desc\": [ \"Integer binary exclusive or not with shifted register\"],\n \"DestSize\": \"Size\",\n \"EmitValidation\": [\n \"Size == FEXCore::IR::OpSize::i32Bit || Size == FEXCore::IR::OpSize::i64Bit\"\n ]\n },\n \"GPR = And OpSize:#Size, GPR:$Src1, GPR:$Src2\": {\n \"Desc\": [\"Integer binary and\"],\n \"Inline\": [\"\", \"Logical\"],\n \"DestSize\": \"Size\",\n \"EmitValidation\": [\n \"Size == FEXCore::IR::OpSize::i32Bit || Size == FEXCore::IR::OpSize::i64Bit\"\n ]\n },\n \"GPR = AndShift OpSize:#Size, GPR:$Src1, GPR:$Src2, ShiftType:$Shift{ShiftType::LSL}, u8:$ShiftAmount{0}\": {\n \"Desc\": [ \"Integer binary and with shifted register\"],\n \"DestSize\": \"Size\",\n \"EmitValidation\": [\n \"Size == FEXCore::IR::OpSize::i32Bit || Size == FEXCore::IR::OpSize::i64Bit\"\n ]\n },\n \"GPR = AndWithFlags OpSize:#Size, GPR:$Src1, GPR:$Src2\": {\n \"Desc\": [\"Integer binary and\"\n ],\n \"Inline\": [\"\", \"Logical\"],\n \"DestSize\": \"Size\",\n \"TiedSource\": 0,\n \"HasSideEffects\": true\n },\n \"GPR = Andn OpSize:#Size, GPR:$Src1, GPR:$Src2\": {\n \"Desc\": [\"Integer binary AND NOT. Performs the equivalent of Src1 & ~Src2\"],\n \"DestSize\": \"Size\",\n \"Inline\": [\"\", \"Logical\"],\n \"EmitValidation\": [\n \"Size == FEXCore::IR::OpSize::i32Bit || Size == FEXCore::IR::OpSize::i64Bit\"\n ]\n },\n \"TestNZ OpSize:#Size, GPR:$Src1, GPR:$Src2\": {\n \"Desc\": [\"Set NZCV for the binary AND of two GPRs, setting N and Z accordingly and zeroing C and V\"],\n \"Inline\": [\"\", \"Logical\"],\n \"DestSize\": \"Size\",\n \"HasSideEffects\": true\n },\n \"TestZ OpSize:#Size, GPR:$Src1, GPR:$Src2\": {\n \"Desc\": [\"Set NZCV for the binary AND of two GPRs, setting Z accordingly and zeroing C and V. N is undefined.\"],\n \"DestSize\": \"Size\",\n \"HasSideEffects\": true\n },\n \"GPR = Lshl OpSize:#Size, GPR:$Src1, GPR:$Src2\": {\n \"Desc\": [\"Integer logical shift left\"],\n \"Inline\": [\"\", \"Any\"],\n \"DestSize\": \"Size\",\n \"EmitValidation\": [\n \"Size == FEXCore::IR::OpSize::i32Bit || Size == FEXCore::IR::OpSize::i64Bit\"\n ]\n },\n \"GPR = Lshr OpSize:#Size, GPR:$Src1, GPR:$Src2\": {\n \"Desc\": [\"Integer logical shift right\"],\n \"Inline\": [\"\", \"Any\"],\n \"DestSize\": \"Size\",\n \"EmitValidation\": [\n \"Size == FEXCore::IR::OpSize::i32Bit || Size == FEXCore::IR::OpSize::i64Bit\"\n ]\n },\n \"GPR = Ashr OpSize:#Size, GPR:$Src1, GPR:$Src2\": {\n \"Desc\": [\"Integer arithmetic shift right\"],\n \"Inline\": [\"\", \"Any\"],\n \"DestSize\": \"Size\",\n \"EmitValidation\": [\n \"Size == FEXCore::IR::OpSize::i32Bit || Size == FEXCore::IR::OpSize::i64Bit\"\n ]\n },\n \"GPR = ShiftFlags OpSize:$Size, GPR:$Result, GPR:$Src1, ShiftType:$Shift, GPR:$Src2, GPR:$PFInput, i1:$InvertCF\": {\n \"Desc\": [\"Set NZCV flags for specified variable integer shift with given result.\",\n \"Returns updated raw PF.\"],\n \"HasSideEffects\": true,\n \"DestSize\": \"OpSize::i64Bit\"\n },\n \"RotateFlags OpSize:$Size, GPR:$Result, GPR:$Shift, i1:$Left\": {\n \"Desc\": [\"Set NZCV flags for specified variable integer rotate with given result.\"],\n \"HasSideEffects\": true\n },\n \"GPR = Ror OpSize:#Size, GPR:$Src1, GPR:$Src2\": {\n \"Desc\": [\"Integer rotate right\"],\n \"Inline\": [\"\", \"Any\"],\n \"DestSize\": \"Size\",\n \"EmitValidation\": [\n \"Size == FEXCore::IR::OpSize::i32Bit || Size == FEXCore::IR::OpSize::i64Bit\"\n ]\n },\n \"GPR = Mul OpSize:#Size, GPR:$Src1, GPR:$Src2\": {\n \"Desc\": [\"Integer signed multiplication\"\n ],\n \"DestSize\": \"Size\",\n \"EmitValidation\": [\n \"Size == FEXCore::IR::OpSize::i32Bit || Size == FEXCore::IR::OpSize::i64Bit\"\n ]\n },\n \"GPR = UMul OpSize:#Size, GPR:$Src1, GPR:$Src2\": {\n \"Desc\": [\"Integer unsigned multiplication\"\n ],\n \"DestSize\": \"Size\",\n \"EmitValidation\": [\n \"Size == FEXCore::IR::OpSize::i32Bit || Size == FEXCore::IR::OpSize::i64Bit\"\n ]\n },\n \"GPR = UMull GPR:$Src1, GPR:$Src2\": {\n \"Desc\": [\"Integer unsigned multiplication long\",\n \"Multiplies two 32-bit numbers, returning a 64-bit destination register.\"\n ],\n \"DestSize\": \"FEXCore::IR::OpSize::i64Bit\"\n },\n \"GPR = SMull GPR:$Src1, GPR:$Src2\": {\n \"Desc\": [\"Integer signed multiplication long\",\n \"Multiplies two 32-bit numbers, returning a 64-bit destination register.\"\n ],\n \"DestSize\": \"FEXCore::IR::OpSize::i64Bit\"\n },\n \"GPR = MulH OpSize:#Size, GPR:$Src1, GPR:$Src2\": {\n \"Desc\": [\"Integer signed multiply returning high results\",\n \"op:\",\n \"Tmp = Src1 * Src2;\",\n \"Dest = Tmp >> (size * 8);\"\n ],\n \"DestSize\": \"Size\",\n \"EmitValidation\": [\n \"Size == FEXCore::IR::OpSize::i32Bit || Size == FEXCore::IR::OpSize::i64Bit\"\n ]\n },\n \"GPR = UMulH OpSize:#Size, GPR:$Src1, GPR:$Src2\": {\n \"Desc\": [\"Integer unsigned multiply returning high results\",\n \"op:\",\n \"Tmp = Src1 * Src2;\",\n \"Dest = Tmp >> (size * 8);\"\n ],\n \"DestSize\": \"Size\",\n \"EmitValidation\": [\n \"Size == FEXCore::IR::OpSize::i32Bit || Size == FEXCore::IR::OpSize::i64Bit\"\n ]\n },\n \"GPR = Bfi OpSize:#Size, u8:$Width, u8:$lsb, GPR:$Dest, GPR:$Src\": {\n \"Desc\": [\"Copies a bitfield from one GPR to another\",\n \"The source bitfield is from Src[Width:0]\",\n \"The bitfield is copied in to Dest[(Width + lsb):lsb]\"\n ],\n \"DestSize\": \"Size\",\n \"TiedSource\": 0,\n \"EmitValidation\": [\n \"Size == FEXCore::IR::OpSize::i32Bit || Size == FEXCore::IR::OpSize::i64Bit\",\n \"(Width + lsb) <= IR::OpSizeAsBits(Size)\"\n ]\n },\n \"GPR = Bfxil OpSize:#Size, u8:$Width, u8:$lsb, GPR:$Dest, GPR:$Src\": {\n \"Desc\": [\"Copies a bitfield from one GPR to another\",\n \"Inserting in to the low bits of the destination\",\n \"The source bitfield is from Src[(Width + lsb):lsb]\",\n \"The bitfield is copied in to Dest[Width:0]\"\n ],\n \"DestSize\": \"Size\",\n \"TiedSource\": 0,\n \"EmitValidation\": [\n \"Size == FEXCore::IR::OpSize::i32Bit || Size == FEXCore::IR::OpSize::i64Bit\",\n \"(Width + lsb) <= IR::OpSizeAsBits(Size)\"\n ]\n },\n \"GPR = Bfe OpSize:#Size, u8:$Width, u8:$lsb, GPR:$Src\": {\n \"Desc\": [\"Extracts a bitfield from one GPR with zext\",\n \"The source bitfield is from Src[Width:0]\",\n \"The bitfield is then zero extended\"\n ],\n \"DestSize\": \"Size\",\n \"EmitValidation\": [\n \"Size == FEXCore::IR::OpSize::i32Bit || Size == FEXCore::IR::OpSize::i64Bit\",\n \"(Width + lsb) <= IR::OpSizeAsBits(Size)\"\n ]\n },\n \"GPR = Sbfe OpSize:#Size, u8:$Width, u8:$lsb, GPR:$Src\": {\n \"Desc\": [\"Extracts a bitfield from one GPR with sext\",\n \"The source bitfield is from Src[Width:0]\",\n \"The bitfield is then sign extended\"\n ],\n \"DestSize\": \"Size\",\n \"EmitValidation\": [\n \"Size == FEXCore::IR::OpSize::i32Bit || Size == FEXCore::IR::OpSize::i64Bit\",\n \"(Width + lsb) <= IR::OpSizeAsBits(Size)\"\n ]\n },\n \"GPR = NZCVSelect OpSize:#ResultSize, CondClass:$Cond, GPR:$TrueVal, GPR:$FalseVal\": {\n \"Desc\": [\"Select based on value in NZCV flags\",\n \"op:\",\n \"Dest = Cond ? TrueVal : FalseVal\"\n ],\n \"DestSize\": \"ResultSize\",\n \"EmitValidation\": [\n \"ResultSize == FEXCore::IR::OpSize::i32Bit || ResultSize == FEXCore::IR::OpSize::i64Bit\"\n ]\n },\n \"FPR = NZCVSelectV OpSize:#ResultSize, CondClass:$Cond, FPR:$TrueVal, FPR:$FalseVal\": {\n \"Desc\": [\n \"Select based on value in NZCV flags, where TrueVal and FalseVal are both FPRs.\",\n \"op:\",\n \"Dest = Cond ? TrueVal : FalseVal\"\n ],\n \"DestSize\": \"ResultSize\"\n },\n \"GPR = NZCVSelectIncrement OpSize:#ResultSize, CondClass:$Cond, GPR:$TrueVal, GPR:$FalseVal\": {\n \"Desc\": [\"Select and increment based on value in NZCV flags\",\n \"op:\",\n \"Dest = Cond ? TrueVal : (FalseVal + 1)\"\n ],\n \"DestSize\": \"ResultSize\",\n \"EmitValidation\": [\n \"ResultSize == FEXCore::IR::OpSize::i32Bit || ResultSize == FEXCore::IR::OpSize::i64Bit\"\n ]\n },\n \"GPR = Select OpSize:#ResultSize, OpSize:$CompareSize, CondClass:$Cond, SSA:$Cmp1, SSA:$Cmp2, GPR:$TrueVal, GPR:$FalseVal\": {\n \"Desc\": [\"Ternary selection of GPRs\",\n \"op:\",\n \"Dest = Cmp1 Cmp2 ? TrueVal : FalseVal\"\n ],\n \"Inline\": [\"\", \"AddSub\", \"\", \"\"],\n \"DestSize\": \"ResultSize\",\n \"ImplicitFlagClobber\": true,\n \"EmitValidation\": [\n \"CompareSize == FEXCore::IR::OpSize::i32Bit || CompareSize == FEXCore::IR::OpSize::i64Bit || CompareSize == FEXCore::IR::OpSize::i128Bit\",\n \"ResultSize == FEXCore::IR::OpSize::i32Bit || ResultSize == FEXCore::IR::OpSize::i64Bit\"\n ]\n },\n \"GPR = MaskGenerateFromBitWidth GPR:$BitWidth\": {\n \"Desc\": [\"Generates a bit mask from with a value from [0, 63]\",\n \"0 is special cased to full-mask\",\n \"Special operation for SSE4a bitmask generation.\"\n ],\n \"DestSize\": \"FEXCore::IR::OpSize::i64Bit\",\n \"ImplicitFlagClobber\": true\n },\n\n \"GPR = Extr OpSize:#Size, GPR:$Upper, GPR:$Lower, u8:$LSB\": {\n \"Desc\": [\"Concats the two GPRs to create a value that is the size of the full two GPRs\",\n \"It then extracts a bitfield width that size of a GPR from the LSB\",\n \"Valid LSB range is 0-31 for 32bit and 0-63 for 64bit\",\n \" ConcatValue = $Upper:$Lower\",\n \"Result = ConcatValue\"\n ],\n \"DestSize\": \"Size\",\n \"EmitValidation\": [\n \"Size == FEXCore::IR::OpSize::i32Bit || Size == FEXCore::IR::OpSize::i64Bit\"\n ]\n },\n \"GPR = PDep OpSize:#Size, GPR:$Input, GPR:$Mask\": {\n \"Desc\": [\"Performs a parallel bit deposit.\",\n \"Takes the contiguous low-order bits and deposits them into\",\n \"the destination at the locations specified by the Mask.\"\n ],\n \"DestSize\": \"Size\",\n \"EmitValidation\": [\n \"Size == FEXCore::IR::OpSize::i32Bit || Size == FEXCore::IR::OpSize::i64Bit\"\n ]\n },\n\n \"GPR = PExt OpSize:#Size, GPR:$Input, GPR:$Mask\": {\n \"Desc\": [\"Performs a parallel bit extract.\",\n \"Each bit set in the mask will select the corresponding bit in the Input\",\n \"and transfers them to the lower contiguous bits in the destination.\"\n ],\n \"DestSize\": \"Size\",\n \"EmitValidation\": [\n \"Size == FEXCore::IR::OpSize::i32Bit || Size == FEXCore::IR::OpSize::i64Bit\"\n ]\n },\n\n \"GPR:$Quotient, GPR:$Remainder = Div OpSize:#Size, GPR:$Lower, GPR:$Upper, GPR:$Divisor\": {\n \"Desc\": [\"Integer long signed division returning lower bits\",\n \"The Lower and Upper registers will be concated together to generate a dividend twice the size\",\n \"Then the divisor divides the temporary dividend and returns the results in the original sized register\",\n \"If Upper is invalid, this is a non-long division.\"\n ],\n \"DestSize\": \"Size\",\n \"HasSideEffects\": true\n },\n \"GPR:$Quotient, GPR:$Remainder = UDiv OpSize:#Size, GPR:$Lower, GPR:$Upper, GPR:$Divisor\": {\n \"Desc\": [\"Integer long unsigned division returning lower bits\",\n \"The Lower and Upper registers will be concated together to generate a dividend twice the size\",\n \"Then the divisor divides the temporary dividend and returns the results in the original sized register\",\n \"If Upper is invalid, this is a non-long division.\"\n ],\n \"DestSize\": \"Size\",\n \"HasSideEffects\": true\n },\n\n \"Float to GPR\": {\"Ignore\": 1},\n \"GPR = VExtractToGPR OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector, u8:$Index\": {\n \"Desc\": [\"Extracts an element from a vector and places it in a GPR\",\n \"The element that is extracted from the vector is zero extended to the GPR size\"\n ],\n \"DestSize\": \"ElementSize\"\n },\n\n \"GPR = Float_ToGPR_S OpSize:#DestElementSize, OpSize:$SrcElementSize, FPR:$Scalar\": {\n \"Desc\": [\"Moves the scalar element to a GPR with conversion\",\n \"Converts the 32bit or 64bit float to an signed integer\",\n \"Rounding mode determined by host flag's rounding mode\"\n ],\n \"DestSize\": \"DestElementSize\"\n },\n\n \"GPR = Float_ToGPR_ZS OpSize:#DestElementSize, OpSize:$SrcElementSize, FPR:$Scalar\": {\n \"Desc\": [\"Moves the scalar element to a GPR with conversion\",\n \"Converts the 32bit or 64bit float to an signed integer rounding towards zero (Truncating)\"\n ],\n \"DestSize\": \"DestElementSize\"\n },\n\n \"FCmp OpSize:$ElementSize, FPR:$Scalar1, FPR:$Scalar2\": {\n \"Desc\": [\"Does a scalar unordered compare and sets NZCV accordingly.\",\n \"NZCV follows Arm conventions, a separate AXFLAG instruction is required for x86\",\n \"Ordering flag result is true if either float input is NaN\"\n ],\n \"HasSideEffects\": true\n }\n },\n \"VectorScalar\": {\n \"FPR = VFAddScalarInsert OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector1, FPR:$Vector2, i1:$ZeroUpperBits\": {\n \"Desc\": [\"Does a scalar 'add' between Vector1 and Vector2.\",\n \"Inserting the result in to the lower element of Vector1 and returning the results.\",\n \"If ZeroUpperBits is set then in a 256-bit wide operation it will zero the upper 128-bits of the destination.\",\n \"For 128-bit operation this matches SSE insert semantics.\",\n \"For 256-bit operation with ZeroUpperBits, this matches AVX insert semantics.\"\n ],\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n \"FPR = VFSubScalarInsert OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector1, FPR:$Vector2, i1:$ZeroUpperBits\": {\n \"Desc\": [\"Does a scalar 'sub' between Vector1 and Vector2.\",\n \"Inserting the result in to the lower element of Vector1 and returning the results.\",\n \"If ZeroUpperBits is set then in a 256-bit wide operation it will zero the upper 128-bits of the destination.\",\n \"For 128-bit operation this matches SSE insert semantics.\",\n \"For 256-bit operation with ZeroUpperBits, this matches AVX insert semantics.\"\n ],\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n \"FPR = VFMulScalarInsert OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector1, FPR:$Vector2, i1:$ZeroUpperBits\": {\n \"Desc\": [\"Does a scalar 'mul' between Vector1 and Vector2.\",\n \"Inserting the result in to the lower element of Vector1 and returning the results.\",\n \"If ZeroUpperBits is set then in a 256-bit wide operation it will zero the upper 128-bits of the destination.\",\n \"For 128-bit operation this matches SSE insert semantics.\",\n \"For 256-bit operation with ZeroUpperBits, this matches AVX insert semantics.\"\n ],\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n \"FPR = VFDivScalarInsert OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector1, FPR:$Vector2, i1:$ZeroUpperBits\": {\n \"Desc\": [\"Does a scalar 'div' between Vector1 and Vector2.\",\n \"Inserting the result in to the lower element of Vector1 and returning the results.\",\n \"If ZeroUpperBits is set then in a 256-bit wide operation it will zero the upper 128-bits of the destination.\",\n \"For 128-bit operation this matches SSE insert semantics.\",\n \"For 256-bit operation with ZeroUpperBits, this matches AVX insert semantics.\"\n ],\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n \"FPR = VFMinScalarInsert OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector1, FPR:$Vector2, i1:$ZeroUpperBits\": {\n \"Desc\": [\"Does a scalar 'min' between Vector1 and Vector2.\",\n \"Inserting the result in to the lower element of Vector1 and returning the results.\",\n \"If ZeroUpperBits is set then in a 256-bit wide operation it will zero the upper 128-bits of the destination.\",\n \"For 128-bit operation this matches SSE insert semantics.\",\n \"For 256-bit operation with ZeroUpperBits, this matches AVX insert semantics.\",\n \"Additionally matches x86 zero and NaN semantics\",\n \"If both source operands are zero, return the second operand (in the case of negative and positive zero)\",\n \"If either source operand is NaN then return the second operand.\"\n ],\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\",\n \"ImplicitFlagClobber\": true\n },\n \"FPR = VFMaxScalarInsert OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector1, FPR:$Vector2, i1:$ZeroUpperBits\": {\n \"Desc\": [\"Does a scalar 'max' between Vector1 and Vector2.\",\n \"Inserting the result in to the lower element of Vector1 and returning the results.\",\n \"If ZeroUpperBits is set then in a 256-bit wide operation it will zero the upper 128-bits of the destination.\",\n \"For 128-bit operation this matches SSE insert semantics.\",\n \"For 256-bit operation with ZeroUpperBits, this matches AVX insert semantics.\",\n \"Additionally matches x86 zero and NaN semantics\",\n \"If both source operands are zero, return the second operand (in the case of negative and positive zero)\",\n \"If either source operand is NaN then return the second operand.\"\n ],\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\",\n \"ImplicitFlagClobber\": true\n },\n \"FPR = VFSqrtScalarInsert OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector1, FPR:$Vector2, i1:$ZeroUpperBits\": {\n \"Desc\": [\"Does a scalar 'sqrt' on Vector2, inserting in to Vector1 and storing in to the destination.\",\n \"Inserting the result in to the lower element of Vector1 and returning the results.\",\n \"If ZeroUpperBits is set then in a 256-bit wide operation it will zero the upper 128-bits of the destination.\",\n \"For 128-bit operation this matches SSE insert semantics.\",\n \"For 256-bit operation with ZeroUpperBits, this matches AVX insert semantics.\"\n ],\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n \"FPR = VFRSqrtScalarInsert OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector1, FPR:$Vector2, i1:$ZeroUpperBits\": {\n \"Desc\": [\"Does a scalar 'rsqrt' on Vector2, inserting in to Vector1 and storing in to the destination.\",\n \"Inserting the result in to the lower element of Vector1 and returning the results.\",\n \"If ZeroUpperBits is set then in a 256-bit wide operation it will zero the upper 128-bits of the destination.\",\n \"For 128-bit operation this matches SSE insert semantics.\",\n \"For 256-bit operation with ZeroUpperBits, this matches AVX insert semantics.\"\n ],\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n \"FPR = VFRecpScalarInsert OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector1, FPR:$Vector2, i1:$ZeroUpperBits\": {\n \"Desc\": [\"Does a scalar 'recip' on Vector2, inserting in to Vector1 and storing in to the destination.\",\n \"Inserting the result in to the lower element of Vector1 and returning the results.\",\n \"If ZeroUpperBits is set then in a 256-bit wide operation it will zero the upper 128-bits of the destination.\",\n \"For 128-bit operation this matches SSE insert semantics.\",\n \"For 256-bit operation with ZeroUpperBits, this matches AVX insert semantics.\"\n ],\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n \"FPR = VFToFScalarInsert OpSize:#RegisterSize, OpSize:#DstElementSize, OpSize:$SrcElementSize, FPR:$Vector1, FPR:$Vector2, i1:$ZeroUpperBits\": {\n \"Desc\": [\"Does a scalar 'cvt' between Vector1 and Vector2.\",\n \"Inserting the result in to the lower element of Vector1 and returning the results.\",\n \"If ZeroUpperBits is set then in a 256-bit wide operation it will zero the upper 128-bits of the destination.\",\n \"For 128-bit operation this matches SSE insert semantics.\",\n \"For 256-bit operation with ZeroUpperBits, this matches AVX insert semantics.\"\n ],\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"DstElementSize\"\n },\n \"FPR = VSToFVectorInsert OpSize:#RegisterSize, OpSize:#DstElementSize, OpSize:$SrcElementSize, FPR:$Vector1, FPR:$Vector2, i8:$HasTwoElements, i1:$ZeroUpperBits\": {\n \"Desc\": [\"Does a Vector 'scvt' between Vector1 and Vector2.\",\n \"Inserting the result in to the lower element of Vector1 and returning the results.\",\n \"If ZeroUpperBits is set then in a 256-bit wide operation it will zero the upper 128-bits of the destination.\",\n \"For 128-bit operation this matches SSE insert semantics.\",\n \"For 256-bit operation with ZeroUpperBits, this matches AVX insert semantics.\",\n \"HasTwoElements is slightly different than most of these scalar operations.\",\n \"Handles the edge case of cvtpi2ps xmm0, mm0 which is two elements in the lower 64-bits\"\n ],\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"DstElementSize\"\n },\n \"FPR = VSToFGPRInsert OpSize:#RegisterSize, OpSize:#DstElementSize, OpSize:$SrcElementSize, FPR:$Vector, GPR:$Src, i1:$ZeroUpperBits\": {\n \"Desc\": [\"Does a scalar 'cvt' between Vector1 and GPR.\",\n \"Inserting the result in to the lower element of Vector1 and returning the results.\",\n \"If ZeroUpperBits is set then in a 256-bit wide operation it will zero the upper 128-bits of the destination.\",\n \"For 128-bit operation this matches SSE insert semantics.\",\n \"For 256-bit operation with ZeroUpperBits, this matches AVX insert semantics.\"\n ],\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"DstElementSize\"\n },\n \"FPR = VFToIScalarInsert OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector1, FPR:$Vector2, RoundType:$Round, i1:$ZeroUpperBits\": {\n \"Desc\": [\"Does a scalar round float to integral on Vector2, inserting in to Vector1 and storing in to the destination.\",\n \"Rounding mode determined by argument\",\n \"Inserting the result in to the lower element of Vector1 and returning the results.\",\n \"If ZeroUpperBits is set then in a 256-bit wide operation it will zero the upper 128-bits of the destination.\",\n \"For 128-bit operation this matches SSE insert semantics.\",\n \"For 256-bit operation with ZeroUpperBits, this matches AVX insert semantics.\"\n ],\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n \"FPR = VFCMPScalarInsert OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector1, FPR:$Vector2, FloatCompareOp:$Op, i1:$ZeroUpperBits\": {\n \"Desc\": [\"Does a scalar 'cmp' between Vector1 and Vecto2, inserting in to Vector1 and storing in to the destination.\",\n \"Compare op determined by argument\",\n \"Inserting the result in to the lower element of Vector1 and returning the results.\",\n \"If ZeroUpperBits is set then in a 256-bit wide operation it will zero the upper 128-bits of the destination.\",\n \"For 128-bit operation this matches SSE insert semantics.\",\n \"For 256-bit operation with ZeroUpperBits, this matches AVX insert semantics.\"\n ],\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n \"FPR = VFMLAScalarInsert OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Upper, FPR:$Vector1, FPR:$Vector2, FPR:$Addend\": {\n \"Desc\": [\n \"Dest = (Vector1 * Vector2) + Addend\",\n \"This explicitly matches x86 FMA semantics because ARM semantics are mind-bending.\",\n \"Upper elements copied from Upper\"\n ],\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\",\n \"TiedSource\": 0\n },\n \"FPR = VFMLSScalarInsert OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Upper, FPR:$Vector1, FPR:$Vector2, FPR:$Addend\": {\n \"Desc\": [\n \"Dest = (Vector1 * Vector2) - Addend\",\n \"This explicitly matches x86 FMA semantics because ARM semantics are mind-bending.\",\n \"Upper elements copied from Upper\"\n ],\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\",\n \"TiedSource\": 0\n },\n \"FPR = VFNMLAScalarInsert OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Upper, FPR:$Vector1, FPR:$Vector2, FPR:$Addend\": {\n \"Desc\": [\n \"Dest = (-Vector1 * Vector2) + Addend\",\n \"This explicitly matches x86 FMA semantics because ARM semantics are mind-bending.\",\n \"Upper elements copied from Upper\"\n ],\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\",\n \"TiedSource\": 0\n },\n \"FPR = VFNMLSScalarInsert OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Upper, FPR:$Vector1, FPR:$Vector2, FPR:$Addend\": {\n \"Desc\": [\n \"Dest = (-Vector1 * Vector2) - Addend\",\n \"This explicitly matches x86 FMA semantics because ARM semantics are mind-bending.\",\n \"Upper elements copied from Upper\"\n ],\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\",\n \"TiedSource\": 0\n },\n \"FPR = VFCopySign OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector1, FPR:$Vector2\": {\n \"Desc\": [\"Returns a vector where each element has has the magniture of each corresponding element in vector1 and the sign of vector 2.\"],\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\",\n \"TiedSource\": 0\n }\n },\n \"Vector\": {\n \"FPR = VMov OpSize:#RegisterSize, FPR:$Source\": {\n \"Desc\" : [\"Copy vector register\",\n \"When Register size is smaller than Source register size,\",\n \"this op is defined to truncate and zero extend\"\n ],\n \"DestSize\": \"RegisterSize\"\n },\n\n \"FPR = VectorImm OpSize:#RegisterSize, OpSize:#ElementSize, u8:$Immediate, u8:$ShiftAmount{0}\": {\n \"Desc\": [\"Generates a vector with each element containg the immediate zexted\"\n ],\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n\n \"FPR = LoadNamedVectorConstant OpSize:#RegisterSize, NamedVectorConstant:$Constant\": {\n \"Desc\": [\"Load a named vector constant.\",\n \"The list of vector constants can be found in \"\n ],\n \"DestSize\": \"RegisterSize\"\n },\n \"FPR = LoadNamedVectorIndexedConstant OpSize:#RegisterSize, IndexNamedVectorConstant:$Constant, u32:$Index\": {\n \"Desc\": [\"Load a named vector constant from Indexable table.\",\n \"Index needs to be aligned register size.\",\n \"The list of indexable vector constants can be found in \"\n ],\n \"DestSize\": \"RegisterSize\"\n },\n \"FPR = VNeg OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector\": {\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n \"FPR = VNot OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector\": {\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n\n \"FPR = VAbs OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector\": {\n \"Desc\": [\"Does an signed integer absolute\"\n ],\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n\n \"FPR = VPopcount OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector\": {\n \"Desc\": [\"Does a popcount for each element of the register\"\n ],\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n\n \"FPR = VAddV OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector\": {\n \"Desc\": [\"Does a horizontal vector add of elements across the source vector\",\n \"Result is a zero extended scalar\"\n ],\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n \"FPR = VUMinV OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector\": {\n \"Desc\": [\"Does a horizontal vector unsigned minimum of elements across the source vector\",\n \"Result is a zero extended scalar\"\n ],\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n \"FPR = VUMaxV OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector\": {\n \"Desc\": [\"Does a horizontal vector unsigned maximum of elements across the source vector\",\n \"Result is a zero extended scalar\"\n ],\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n \"FPR = VFAbs OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector\": {\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n\n \"FPR = VFNeg OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector\": {\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n\n \"FPR = VFRecp OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector\": {\n \"Desc\": [\n \"Reciprocal value - matches the precision required by the x86 spec.\",\n \"It has a relative error of at most 1.5 * 2^-12\"\n ],\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n \"FPR = VFRecpPrecision OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector\": {\n \"Desc\": [\n \"Similar to VFRecp but carrying more precision for 3DNow!\",\n \"It provides at least 14 bits precision, with a relative error of at most 2^-14\"\n ],\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\",\n \"EmitValidation\": [\n \"RegisterSize == FEXCore::IR::OpSize::i64Bit || RegisterSize == FEXCore::IR::OpSize::i32Bit\",\n \"ElementSize == FEXCore::IR::OpSize::i32Bit\"\n ]\n },\n\n \"FPR = VFSqrt OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector\": {\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n\n \"FPR = VFRSqrt OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector\": {\n \"Desc\": [\n \"Reciprocal Square Root - matches the precision required by the x86 spec.\",\n \"It has a relative error of at most 1.5 * 2^-12\"\n ],\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n \"FPR = VFRSqrtPrecision OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector\": {\n \"Desc\": [\n \"Similar to VFRSqrt but carrying more precision for 3DNow!\",\n \"It provides at least 15 bits precision, with a relative error of at most 2^-15\"\n ],\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\",\n \"EmitValidation\": [\n \"RegisterSize == FEXCore::IR::OpSize::i64Bit || RegisterSize == FEXCore::IR::OpSize::i32Bit\",\n \"ElementSize == FEXCore::IR::OpSize::i32Bit\"\n ]\n },\n\n \"FPR = VCMPEQZ OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector\": {\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n \"FPR = VCMPGTZ OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector\": {\n \"Desc\": [\"Vector compare signed greater than\",\n \"Each element is compared, if the result is true then the resulting element is ~0, else zero\",\n \"Compares the vector against zero\"\n ],\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n \"FPR = VCMPLTZ OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector\": {\n \"Desc\": [\"Vector compare signed less than\",\n \"Each element is compared, if the result is true then the resulting element is ~0, else zero\",\n \"Compares the vector against zero\"\n ],\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n \"FPR = VDupElement OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector, u8:$Index\": {\n \"Desc\": [\"Duplicates one element from the source register across the whole register\"],\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n \"FPR = VShlI OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector, u8:$BitShift\": {\n \"TiedSource\": 0,\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\",\n \"EmitValidation\": [\n \"ElementSize >= FEXCore::IR::OpSize::i8Bit && ElementSize <= FEXCore::IR::OpSize::i64Bit\",\n \"BitShift > 0\"\n ]\n },\n \"FPR = VUShrI OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector, u8:$BitShift\": {\n \"TiedSource\": 0,\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\",\n \"EmitValidation\": [\n \"ElementSize >= FEXCore::IR::OpSize::i8Bit && ElementSize <= FEXCore::IR::OpSize::i64Bit\",\n \"BitShift > 0\"\n ]\n },\n \"FPR = VUShraI OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$DestVector, FPR:$Vector, u8:$BitShift\": {\n \"TiedSource\": 0,\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\",\n \"EmitValidation\": [\n \"ElementSize >= FEXCore::IR::OpSize::i8Bit && ElementSize <= FEXCore::IR::OpSize::i64Bit\",\n \"BitShift > 0 && BitShift <= IR::OpSizeAsBits(ElementSize)\"\n ]\n },\n \"FPR = VSShrI OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector, u8:$BitShift\": {\n \"TiedSource\": 0,\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\",\n \"EmitValidation\": [\n \"ElementSize >= FEXCore::IR::OpSize::i8Bit && ElementSize <= FEXCore::IR::OpSize::i64Bit\",\n \"BitShift > 0\"\n ]\n },\n\n \"FPR = VUShrNI OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector, u8:$BitShift\": {\n \"TiedSource\": 0,\n \"Desc\": \"Unsigned shifts right each element and then narrows to the next lower element size\",\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize >> 1\",\n \"EmitValidation\": [\n \"ElementSize >= FEXCore::IR::OpSize::i16Bit && ElementSize <= FEXCore::IR::OpSize::i64Bit\",\n \"BitShift <= IR::OpSizeAsBits(ElementSize)\"\n ]\n },\n\n \"FPR = VUShrNI2 OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$VectorLower, FPR:$VectorUpper, u8:$BitShift\": {\n \"TiedSource\": 0,\n \"Desc\": [\"Unsigned shifts right each element and then narrows to the next lower element size\",\n \"Inserts results in to the high elements of the first argument\"\n ],\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize >> 1\",\n \"EmitValidation\": [\n \"ElementSize >= FEXCore::IR::OpSize::i16Bit && ElementSize <= FEXCore::IR::OpSize::i64Bit\",\n \"BitShift > 0 && BitShift <= IR::OpSizeAsBits(ElementSize)\"\n ]\n },\n \"FPR = VSXTL OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector\": {\n \"Desc\": \"Sign extends elements from the source element size to the next size up\",\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize << 1\"\n },\n \"FPR = VSXTL2 OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector\": {\n \"Desc\": [\"Sign extends elements from the source element size to the next size up\",\n \"Source elements come from the upper half of the register\"\n ],\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize << 1\"\n },\n \"FPR = VSSHLL OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector, u8:$BitShift{0}\": {\n \"Desc\": \"Sign extends elements from the source element size to the next size up\",\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize << 1\"\n },\n \"FPR = VSSHLL2 OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector, u8:$BitShift{0}\": {\n \"Desc\": [\"Sign extends elements from the source element size to the next size up\",\n \"Source elements come from the upper half of the register\"\n ],\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize << 1\"\n },\n \"FPR = VUXTL OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector\": {\n \"Desc\": \"Zero extends elements from the source element size to the next size up\",\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize << 1\"\n },\n \"FPR = VUXTL2 OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector\": {\n \"Desc\": [\"Zero extends elements from the source element size to the next size up\",\n \"Source elements come from the upper half of the register\"\n ],\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize << 1\"\n },\n \"FPR = VSQXTN OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector\": {\n \"TiedSource\": 0,\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize >> 1\"\n },\n \"FPR = VSQXTN2 OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$VectorLower, FPR:$VectorUpper\": {\n \"TiedSource\": 0,\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize >> 1\"\n },\n \"FPR = VSQXTNPair OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$VectorLower, FPR:$VectorUpper\": {\n \"Desc\": [\"Does both VSQXTN and VSQXTN2 in a combined operation.\"\n ],\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize >> 1\"\n },\n \"FPR = VSQXTUN OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector\": {\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize >> 1\"\n },\n \"FPR = VSQXTUN2 OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$VectorLower, FPR:$VectorUpper\": {\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize >> 1\"\n },\n \"FPR = VSQXTUNPair OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$VectorLower, FPR:$VectorUpper\": {\n \"Desc\": [\"Does both VSQXTUN and VSQXTUN2 in a combined operation.\"\n ],\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize >> 1\"\n },\n \"FPR = VSRSHR OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector, u8:$BitShift\": {\n \"Desc\": [\"Signed rounding shift right by immediate\",\n \"Exactly matching Arm64 srshr semantics\"\n ],\n \"TiedSource\": 0,\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n \"FPR = VSQSHL OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector, u8:$BitShift\": {\n \"Desc\": [\"Signed satuating shift left by immediate\",\n \"Exactly matching Arm64 sqshl semantics\"\n ],\n \"TiedSource\": 0,\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n \"FPR = VRev32 OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector\": {\n \"Desc\" : [\"Reverses elements in 32-bit halfwords\",\n \"Available element size: 1byte, 2 byte\"\n ],\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n \"FPR = VRev64 OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector\": {\n \"Desc\" : [\"Reverses elements in 64-bit halfwords\",\n \"Available element size: 1byte, 2 byte, 4 byte\"\n ],\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n\n \"FPR = VAdd OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector1, FPR:$Vector2\": {\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n\n \"FPR = VSub OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector1, FPR:$Vector2\": {\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n\n \"FPR = VAnd OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector1, FPR:$Vector2\": {\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\",\n \"EmitValidation\": [\n \"RegisterSize == FEXCore::IR::OpSize::i256Bit || RegisterSize == FEXCore::IR::OpSize::i128Bit || RegisterSize == FEXCore::IR::OpSize::i64Bit\"\n ]\n },\n\n \"FPR = VAndn OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector1, FPR:$Vector2\": {\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\",\n \"EmitValidation\": [\n \"RegisterSize == FEXCore::IR::OpSize::i256Bit || RegisterSize == FEXCore::IR::OpSize::i128Bit || RegisterSize == FEXCore::IR::OpSize::i64Bit\"\n ]\n },\n\n \"FPR = VOrn OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector1, FPR:$Vector2\": {\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\",\n \"EmitValidation\": [\n \"RegisterSize == FEXCore::IR::OpSize::i256Bit || RegisterSize == FEXCore::IR::OpSize::i128Bit || RegisterSize == FEXCore::IR::OpSize::i64Bit\"\n ]\n },\n\n \"FPR = VOr OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector1, FPR:$Vector2\": {\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\",\n \"EmitValidation\": [\n \"RegisterSize == FEXCore::IR::OpSize::i256Bit || RegisterSize == FEXCore::IR::OpSize::i128Bit || RegisterSize == FEXCore::IR::OpSize::i64Bit\"\n ]\n },\n\n \"FPR = VXor OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector1, FPR:$Vector2\": {\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\",\n \"EmitValidation\": [\n \"RegisterSize == FEXCore::IR::OpSize::i256Bit || RegisterSize == FEXCore::IR::OpSize::i128Bit || RegisterSize == FEXCore::IR::OpSize::i64Bit\"\n ]\n },\n\n \"FPR = VUQAdd OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector1, FPR:$Vector2\": {\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n\n \"FPR = VUQSub OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector1, FPR:$Vector2\": {\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n\n \"FPR = VSQAdd OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector1, FPR:$Vector2\": {\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n\n \"FPR = VSQSub OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector1, FPR:$Vector2\": {\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n\n \"FPR = VAddP OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$VectorLower, FPR:$VectorUpper\": {\n \"Desc\": \"Does a horizontal pairwise add of elements across the two source vectors\",\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n\n \"FPR = VURAvg OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector1, FPR:$Vector2\": {\n \"Desc\": [\"Does an unsigned rounded average\", \"dst_elem = (src1_elem + src2_elem + 1) >> 1\"],\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n \"FPR = VUMin OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector1, FPR:$Vector2\": {\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n \"FPR = VUMax OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector1, FPR:$Vector2\": {\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n \"FPR = VSMin OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector1, FPR:$Vector2\": {\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n \"FPR = VSMax OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector1, FPR:$Vector2\": {\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n\n \"FPR = VZip OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$VectorLower, FPR:$VectorUpper\": {\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n \"FPR = VZip2 OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$VectorLower, FPR:$VectorUpper\": {\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n \"FPR = VUnZip OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$VectorLower, FPR:$VectorUpper\": {\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n \"FPR = VUnZip2 OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$VectorLower, FPR:$VectorUpper\": {\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n \"FPR = VTrn OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$VectorLower, FPR:$VectorUpper\": {\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n \"FPR = VTrn2 OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$VectorLower, FPR:$VectorUpper\": {\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n\n \"FPR = VFAdd OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector1, FPR:$Vector2\": {\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n \"FPR = VFAddP OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$VectorLower, FPR:$VectorUpper\": {\n \"Desc\": \"Does a horizontal pairwise add of elements across the two source vectors with float element types\",\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n \"FPR = VFAddV OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector\": {\n \"Desc\": [\"Does a horizontal float vector add of elements across the source vector\",\n \"Result is a zero extended scalar\"\n ],\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n \"FPR = VFSub OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector1, FPR:$Vector2\": {\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n \"FPR = VFMul OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector1, FPR:$Vector2\": {\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n \"FPR = VFDiv OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector1, FPR:$Vector2\": {\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n\n \"FPR = VFMin OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector1, FPR:$Vector2\": {\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\",\n \"TiedSource\": 0\n },\n \"FPR = VFMax OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector1, FPR:$Vector2\": {\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\",\n \"TiedSource\": 0\n },\n \"FPR = VMul OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector1, FPR:$Vector2\": {\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n \"FPR = VUMull OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector1, FPR:$Vector2\": {\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize << 1\"\n },\n \"FPR = VSMull OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector1, FPR:$Vector2\": {\n \"Desc\": [ \"Does a signed integer multiply with extend.\",\n \"ElementSize is the source size\"\n ],\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize << 1\"\n },\n \"FPR = VUMull2 OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector1, FPR:$Vector2\": {\n \"Desc\": \"Multiplies the high elements with size extension\",\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize << 1\"\n },\n \"FPR = VSMull2 OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector1, FPR:$Vector2\": {\n \"Desc\": \"Multiplies the high elements with size extension\",\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize << 1\"\n },\n \"FPR = VUMulH OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector1, FPR:$Vector2\": {\n \"Desc\": \"Wide unsigned multiply returning the high results\",\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n \"FPR = VSMulH OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector1, FPR:$Vector2\": {\n \"Desc\": \"Wide signed multiply returning the high results\",\n\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n \"FPR = VUABDL OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector1, FPR:$Vector2\": {\n \"Desc\": [\"Unsigned Absolute Difference Long\"\n ],\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize << 1\"\n },\n \"FPR = VUABDL2 OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector1, FPR:$Vector2\": {\n \"Desc\": [\"Unsigned Absolute Difference Long\",\n \"Using the high elements of the source vectors\"\n ],\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize << 1\"\n },\n \"FPR = VUShl OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector, FPR:$ShiftVector, i1:$RangeCheck\": {\n \"TiedSource\": 0,\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n \"FPR = VUShr OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector, FPR:$ShiftVector, i1:$RangeCheck\": {\n \"TiedSource\": 0,\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n \"FPR = VSShr OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector, FPR:$ShiftVector, i1:$RangeCheck\": {\n \"TiedSource\": 0,\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n \"FPR = VUShlS OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector, FPR:$ShiftScalar\": {\n \"TiedSource\": 0,\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n \"FPR = VUShrS OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector, FPR:$ShiftScalar\": {\n \"TiedSource\": 0,\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n \"FPR = VSShrS OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector, FPR:$ShiftScalar\": {\n \"TiedSource\": 0,\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n \"FPR = VUShrSWide OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector, FPR:$ShiftScalar\": {\n \"TiedSource\": 0,\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n \"FPR = VSShrSWide OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector, FPR:$ShiftScalar\": {\n \"TiedSource\": 0,\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n \"FPR = VUShlSWide OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector, FPR:$ShiftScalar\": {\n \"TiedSource\": 0,\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n \"FPR = VInsElement OpSize:#RegisterSize, OpSize:#ElementSize, u8:$DestIdx, u8:$SrcIdx, FPR:$DestVector, FPR:$SrcVector\": {\n \"TiedSource\": 0,\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n \"FPR = VInsGPR OpSize:#RegisterSize, OpSize:#ElementSize, u8:$DestIdx, FPR:$DestVector, GPR:$Src\": {\n \"TiedSource\": 0,\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n\n \"FPR = VExtr OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$VectorLower, FPR:$VectorUpper, u8:$Index\": {\n \"Desc\": [\"Concats two vector registers together and extracts a full width register from the element index\",\n \"Index is an element index. So it is offset by ElementSize argument\",\n \"op:\",\n \"TmpVector = concat(Upper:Lower)\",\n \"Dest = TmpVector >> (ElementSize * Index * 8); // Or can be thought of `concat(&TmpVector[Index], i128)`\"\n ],\n \"TiedSource\": 1,\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n\n \"FPR = VCMPEQ OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector1, FPR:$Vector2\": {\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n\n \"FPR = VCMPGT OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector1, FPR:$Vector2\": {\n \"Desc\": [\"Vector compare signed greater than\",\n \"Each element is compared, if the result is true then the resulting element is ~0, else zero\"\n ],\n\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n \"FPR = VFCMPEQ OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector1, FPR:$Vector2\": {\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n \"FPR = VFCMPNEQ OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector1, FPR:$Vector2\": {\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n \"FPR = VFCMPLT OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector1, FPR:$Vector2\": {\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n \"FPR = VFCMPGT OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector1, FPR:$Vector2\": {\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n \"FPR = VFCMPLE OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector1, FPR:$Vector2\": {\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n \"FPR = VFCMPORD OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector1, FPR:$Vector2\": {\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n \"FPR = VFCMPUNO OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector1, FPR:$Vector2\": {\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n \"FPR = VTBL1 OpSize:#RegisterSize, FPR:$VectorTable, FPR:$VectorIndices\": {\n \"Desc\": [\"Does a vector table lookup from one register in to the destination\",\n \"Lookup is byte sized per byte element.\",\n \"Any index larger than what the registers provide will result in zero for that element\",\n \"Table is always treated as a 128bit register\",\n \"Indices matches destination size. Either 64bit or 128bit\"\n ],\n \"DestSize\": \"RegisterSize\"\n },\n \"FPR = VTBL2 OpSize:#RegisterSize, FPR:$VectorTable1, FPR:$VectorTable2, FPR:$VectorIndices\": {\n \"Desc\": [\"Does a vector table lookup from two registers in to the destination\",\n \"Lookup is byte sized per byte element.\",\n \"Any index larger than what the registers provide will result in zero for that element\",\n \"Table is always treated as a two 128bit registers\",\n \"Indices matches destination size. Either 64bit or 128bit\",\n \"Careful about not using sequential table registers, will result in some moves if they aren't sequential.\"\n ],\n \"DestSize\": \"RegisterSize\"\n },\n \"FPR = VTBX1 OpSize:#RegisterSize, FPR:$VectorSrcDst, FPR:$VectorTable, FPR:$VectorIndices\": {\n \"Desc\": [\"Does a vector table lookup from one register in to the destination\",\n \"Lookup is byte sized per byte element.\",\n \"Any index larger than what the registers provide will result in not modifying that element\",\n \"Table is always treated as a 128bit register\",\n \"Indices matches destination size. Either 64bit or 128bit\"\n ],\n \"TiedSource\": 0,\n \"DestSize\": \"RegisterSize\"\n },\n \"FPR = VBSL OpSize:#RegisterSize, FPR:$VectorMask, FPR:$VectorTrue, FPR:$VectorFalse\": {\n \"Desc\": [\"Does a vector bitwise select.\",\n \"If the bit in the field is 1 then the corresponding bit is pulled from VectorTrue\",\n \"If the bit in the field is 0 then the corresponding bit is pulled from VectorFalse\"\n ],\n \"TiedSource\": 0,\n \"DestSize\": \"RegisterSize\"\n },\n\n \"GPR = VPCMPESTRX FPR:$LHS, FPR:$RHS, GPR:$RAX, GPR:$RDX, u16:$Control\": {\n \"Desc\": [\"Performs intermediate behavior analogous to the x86 PCMPESTRI/PCMPESTRM instruction\",\n \"This will return the intermediate result of a PCMPESTR-type operation, but NOT the final\",\n \"result. This must be derived from the intermediate result\",\n\n \"NOTE: On top of returning the intermediate result, the returned value also combines the status\",\n \"flags into the upper 16-bits of the 32-bit result, as these can also be derived over the\",\n \"course of creating the intermediate result\"\n ],\n \"DestSize\": \"OpSize::i32Bit\",\n \"JITDispatch\": false\n },\n \"GPR = VPCMPISTRX FPR:$LHS, FPR:$RHS, u8:$Control\": {\n \"Desc\": [\"Performs intermediate behavior analogous to the x86 PCMPISTRI/PCMPISTRM instruction\",\n \"This will return the intermediate result of a PCMPISTR-type operation, but NOT the final\",\n \"result. This must be derived from the intermediate result\",\n\n \"NOTE: On top of returning the intermediate result, the returned value also combines the status\",\n \"flags into the upper 16-bits of the 32-bit result, as these can also be derived over the\",\n \"course of creating the intermediate result\"\n ],\n \"DestSize\": \"OpSize::i32Bit\",\n \"JITDispatch\": false\n },\n \"FPR = VFCADD OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector1, FPR:$Vector2, u16:$Rotate\": {\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n \"FPR = VFMLA OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector1, FPR:$Vector2, FPR:$Addend\": {\n \"Desc\": [\n \"Dest = (Vector1 * Vector2) + Addend\",\n \"This explicitly matches x86 FMA semantics because ARM semantics are mind-bending.\"\n ],\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\",\n \"TiedSource\": 2\n },\n \"FPR = VFMLS OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector1, FPR:$Vector2, FPR:$Addend\": {\n \"Desc\": [\n \"Dest = (Vector1 * Vector2) - Addend\",\n \"This explicitly matches x86 FMA semantics because ARM semantics are mind-bending.\"\n ],\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\",\n \"TiedSource\": 2\n },\n \"FPR = VFNMLA OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector1, FPR:$Vector2, FPR:$Addend\": {\n \"Desc\": [\n \"Dest = (-Vector1 * Vector2) + Addend\",\n \"This explicitly matches x86 FMA semantics because ARM semantics are mind-bending.\"\n ],\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\",\n \"TiedSource\": 2\n },\n \"FPR = VFNMLS OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector1, FPR:$Vector2, FPR:$Addend\": {\n \"Desc\": [\n \"Dest = (-Vector1 * Vector2) - Addend\",\n \"This explicitly matches x86 FMA semantics because ARM semantics are mind-bending.\"\n ],\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\",\n \"TiedSource\": 2\n }\n },\n \"Conv\": {\n \"FPR = VCastFromGPR OpSize:#RegisterSize, OpSize:#ElementSize, GPR:$Src\": {\n \"Desc\": [\"Moves a GPR to a Vector register with zero extension to full length of the register.\",\n \"No conversion is done on the data as it moves register files\"\n ],\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n\n \"FPR = VDupFromGPR OpSize:#RegisterSize, OpSize:#ElementSize, GPR:$Src\": {\n \"Desc\": [\"Broadcasts a value in a GPR into each ElementSize-sized element in a vector\"],\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n\n \"FPR = VLoadTwoGPRs GPR:$Lower, GPR:$Upper\": {\n \"Desc\": [\"Moves two 64-bit registers to a vector register optimally\"],\n \"DestSize\": \"OpSize::i128Bit\",\n \"ElementSize\": \"OpSize::i64Bit\"\n },\n\n \"FPR = Float_FromGPR_S OpSize:#DstElementSize, OpSize:$SrcElementSize, GPR:$Src\": {\n \"Desc\": [\"Scalar op: Converts signed GPR to Scalar float\",\n \"Zeroes the upper bits of the vector register\"\n ],\n \"DestSize\": \"DstElementSize\"\n },\n \"FPR = Float_FToF OpSize:#DstElementSize, OpSize:$SrcElementSize, FPR:$Scalar\": {\n \"Desc\": [\"Scalar op: Converts float from one size to another\",\n \"Zeroes the upper bits of the vector register\"\n ],\n \"DestSize\": \"DstElementSize\"\n },\n\n \"FPR = Vector_SToF OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector\": {\n \"Desc\": \"Vector op: Converts signed integer to same size float\",\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n \"FPR = Vector_FToS OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector\": {\n \"Desc\": [\"Vector op: Converts float to signed integer, rounding towards zero\",\n \"Rounding mode determined by host rounding mode\"\n ],\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n \"FPR = Vector_FToZS OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector\": {\n \"Desc\": \"Vector op: Converts float to signed integer, rounding towards zero\",\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n \"FPR = Vector_FToF OpSize:#RegisterSize, OpSize:#DestElementSize, FPR:$Vector, OpSize:$SrcElementSize\": {\n \"Desc\": \"Vector op: Converts float from source element size to destination size (fp32<->fp64)\",\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"DestElementSize\"\n },\n\n \"FPR = VFCVTL2 OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector\": {\n \"Desc\": [\n \"Vector op: Converts float from source element size to destination size (fp32->fp64)\",\n \"Selecting from the high half of the register.\"\n ],\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize << 1\",\n \"EmitValidation\": [\n \"RegisterSize != FEXCore::IR::OpSize::i256Bit && \\\"What does 256-bit mean in this context?\\\"\"\n ]\n },\n \"FPR = VFCVTN2 OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$VectorLower, FPR:$VectorUpper\": {\n \"TiedSource\": 0,\n \"Desc\": [\n \"Vector op: Converts float from source element size and inserting in to the high bits.\",\n \"Bottom half is untouched\",\n \"Narrowing to the element size below what is passed in.\",\n \"F64->F32, F32->F16\"\n ],\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize >> 1\",\n \"EmitValidation\": [\n \"RegisterSize != FEXCore::IR::OpSize::i256Bit && \\\"What does 256-bit mean in this context?\\\"\"\n ]\n },\n \"FPR = Vector_FToI OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector, RoundType:$Round\": {\n \"Desc\": [\"Vector op: Rounds float to integral\",\n \"Rounding mode determined by argument\"\n ],\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n \"FPR = Vector_FToISized OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector, i1:$HostRound, OpSize:$IntSize\": {\n \"Desc\": [\"Vector op: Rounds float to sized integral\",\n \"Either host rounding or round-to-zero\",\n \"Rounding mode determined by argument\"\n ],\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"ElementSize\"\n },\n \"FPR = Vector_F64ToI32 OpSize:#RegisterSize, FPR:$Vector, RoundType:$Round, i1:$EnsureZeroUpperHalf\": {\n \"Desc\": [\"Vector op: Rounds 64-bit float to 32-bit integral with round mode\",\n \"Matches CVTPD2DQ/CVTTPD2DQ behaviour\"\n ],\n \"DestSize\": \"RegisterSize\",\n \"ElementSize\": \"FEXCore::IR::OpSize::i32Bit\"\n }\n },\n \"Crypto\": {\n \"FPR = VAESImc FPR:$Vector\": {\n \"Desc\": \"Does a stage of the inverse mix column transformation\",\n \"DestSize\": \"OpSize::i128Bit\"\n },\n \"FPR = VAESEnc OpSize:#RegisterSize, FPR:$State, FPR:$Key, FPR:$ZeroReg\": {\n \"Desc\": \"Does a step of AES encryption\",\n \"DestSize\": \"RegisterSize\"\n },\n \"FPR = VAESEncLast OpSize:#RegisterSize, FPR:$State, FPR:$Key, FPR:$ZeroReg\": {\n \"Desc\": \"Does the last step of AES encryption\",\n \"DestSize\": \"RegisterSize\"\n },\n \"FPR = VAESDec OpSize:#RegisterSize, FPR:$State, FPR:$Key, FPR:$ZeroReg\": {\n \"Desc\": \"Does a step of AES decryption\",\n \"DestSize\": \"RegisterSize\"\n },\n \"FPR = VAESDecLast OpSize:#RegisterSize, FPR:$State, FPR:$Key, FPR:$ZeroReg\": {\n \"Desc\": \"Does the last step of AES decryption\",\n \"DestSize\": \"RegisterSize\"\n },\n \"FPR = VAESKeyGenAssist FPR:$Src, FPR:$KeyGenTBLSwizzle, FPR:$ZeroReg, u8:$RCON\": {\n \"Desc\": \"Assists in key generation\",\n \"DestSize\": \"OpSize::i128Bit\"\n },\n \"FPR = VSha1H FPR:$Src\": {\n \"Desc\": \"Does vector scalar SHA1H instruction\",\n \"DestSize\": \"FEXCore::IR::OpSize::i32Bit\"\n },\n \"FPR = VSha1C FPR:$Src1, FPR:$Src2, FPR:$Src3\": {\n \"Desc\": \"Does vector SHA1C instruction\",\n \"DestSize\": \"FEXCore::IR::OpSize::i128Bit\",\n \"TiedSource\": 0\n },\n \"FPR = VSha1M FPR:$Src1, FPR:$Src2, FPR:$Src3\": {\n \"Desc\": \"Does vector SHA1M instruction\",\n \"DestSize\": \"FEXCore::IR::OpSize::i128Bit\",\n \"TiedSource\": 0\n },\n \"FPR = VSha1P FPR:$Src1, FPR:$Src2, FPR:$Src3\": {\n \"Desc\": \"Does vector SHA1P instruction\",\n \"DestSize\": \"FEXCore::IR::OpSize::i128Bit\",\n \"TiedSource\": 0\n },\n \"FPR = VSha1SU1 FPR:$Src1, FPR:$Src2\": {\n \"Desc\": \"Does vector scalar SHA1H instruction\",\n \"DestSize\": \"FEXCore::IR::OpSize::i128Bit\",\n \"TiedSource\": 0\n },\n \"FPR = VSha256U0 FPR:$Src1, FPR:$Src2\": {\n \"Desc\": \"Does vector scalar VSha256U0 instruction\",\n \"DestSize\": \"FEXCore::IR::OpSize::i128Bit\",\n \"TiedSource\": 0\n },\n \"FPR = VSha256U1 FPR:$Src1, FPR:$Src2\": {\n \"Desc\": \"Does vector scalar VSha256U1 instruction\",\n \"DestSize\": \"FEXCore::IR::OpSize::i128Bit\"\n },\n \"FPR = VSha256H FPR:$Src1, FPR:$Src2, FPR:$Src3\": {\n \"Desc\": \"Does vector scalar VSha256H instruction\",\n \"DestSize\": \"FEXCore::IR::OpSize::i128Bit\",\n \"TiedSource\": 0\n },\n \"FPR = VSha256H2 FPR:$Src1, FPR:$Src2, FPR:$Src3\": {\n \"Desc\": \"Does vector scalar VSha256H2 instruction\",\n \"DestSize\": \"FEXCore::IR::OpSize::i128Bit\",\n \"TiedSource\": 0\n },\n \"GPR = CRC32 GPR:$Src1, GPR:$Src2, OpSize:$SrcSize\": {\n \"Desc\": [\"CRC32 using polynomial 0x1EDC6F41\"\n ],\n \"DestSize\": \"OpSize::i32Bit\"\n },\n \"FPR = PCLMUL OpSize:#RegisterSize, FPR:$Src1, FPR:$Src2, u8:$Selector\": {\n \"Desc\": [\n \"Performs carryless multiplication of 64-bit elements depending on the selector.\",\n \"Selector = 0b00000000: Uses low 64-bit elements from both input vectors\",\n \"Selector = 0b00000001: Uses high 64-bit element from Src1 and low 64-bit element from Src2\",\n \"Selector = 0b00010000: Uses low 64-bit element from Src1 and high 64-bit element from Src2\",\n \"Selector = 0b00010001: Uses high 64-bit elements from both input vectors\"\n ],\n \"DestSize\": \"RegisterSize\"\n }\n },\n \"F64\": {\n \"FPR = F64ATAN FPR:$Src1, FPR:$Src2\": {\n \"DestSize\": \"OpSize::i64Bit\",\n \"JITDispatch\": false\n },\n \"FPR = F64FPREM FPR:$Src1, FPR:$Src2\": {\n \"DestSize\": \"OpSize::i64Bit\",\n \"JITDispatch\": false\n },\n \"FPR = F64FPREM1 FPR:$Src1, FPR:$Src2\": {\n \"DestSize\": \"OpSize::i64Bit\",\n \"JITDispatch\": false\n },\n \"FPR = F64SCALE FPR:$Src1, FPR:$Src2\": {\n \"DestSize\": \"OpSize::i64Bit\",\n \"JITDispatch\": false\n },\n \"FPR = F64F2XM1 FPR:$Src\": {\n \"DestSize\": \"OpSize::i64Bit\",\n \"JITDispatch\": false\n },\n \"FPR = F64FYL2X FPR:$Src, FPR:$Src2\": {\n \"DestSize\": \"OpSize::i64Bit\",\n \"JITDispatch\": false\n },\n \"FPR = F64TAN FPR:$Src\": {\n \"DestSize\": \"OpSize::i64Bit\",\n \"JITDispatch\": true\n },\n \"FPR = F64SIN FPR:$Src\": {\n \"DestSize\": \"OpSize::i64Bit\",\n \"JITDispatch\": true\n },\n \"FPR = F64COS FPR:$Src\": {\n \"DestSize\": \"OpSize::i64Bit\",\n \"JITDispatch\": true\n },\n \"FPR:$Sin, FPR:$Cos = F64SINCOS FPR:$Src\": {\n \"DestSize\": \"OpSize::i64Bit\",\n \"HasSideEffects\": true,\n \"JITDispatch\": false\n }\n },\n \"F80\": {\n \"GPR = SyncStackToSlow\": {\n \"Desc\": [\n \"Synchronizes the virtual stack environment to the physical registers.\",\n \"Returns the current stack top.\"\n ],\n \"X87\": true,\n \"HasSideEffects\": true,\n \"DestSize\": \"OpSize::i64Bit\"\n },\n \"StackForceSlow\": {\n \"Desc\": [\n \"Forces the slow path.\"\n ],\n \"X87\": true,\n \"HasSideEffects\": true\n },\n \"InitStack\": {\n \"Desc\": [\n \"Initializes the stack by marking all tags as invalid and setting top to zero.\"\n ],\n \"X87\": true,\n \"HasSideEffects\": true\n },\n \"IncStackTop\": {\n \"Desc\": [\n \"Increase stack top-pointer.\"\n ],\n \"X87\": true,\n \"HasSideEffects\": true\n },\n \"DecStackTop\": {\n \"Desc\": [\n \"Decrease stack top-pointer.\"\n ],\n \"X87\": true,\n \"HasSideEffects\": true\n },\n \"InvalidateStack u8:$StackLocation\": {\n \"Desc\": [\n \"Marks the value in TOP+$StackLocation as empty / invalid 0b11.\",\n \"If the StackLocation is 0xff, we invalidate all locations.\"\n ],\n \"X87\": true,\n \"HasSideEffects\": true\n },\n \"PushStack FPR:$X80Src, FPR:$OriginalValue, OpSize:$LoadSize\": {\n \"Desc\": [\n \"Pushes the provided X80Src source on to the x87 stack.\",\n \"Tracks OriginalValue as the original value of X80Src. OriginalValue can be Invalid() in which case no tracking is done.\",\n \"Opsize is 128bit for F80 values, 64-bit for low precision.\",\n \"LoadSize the original load size, i.e. of size of OriginalValue.\",\n \"Float: 80-bit, 64-bit, 32-bit\"\n ],\n \"HasSideEffects\": true,\n \"X87\": true\n },\n \"CopyPushStack u8:$StackLocation\": {\n \"Desc\": [\n \"Pushes an element already on the stack onto the top.\"\n ],\n \"HasSideEffects\": true,\n \"X87\": true\n },\n \"StoreStackMem OpSize:$SourceSize, OpSize:$StoreSize, GPR:$Addr, GPR:$Offset, OpSize:$Align, MemOffsetType:$OffsetType, u8:$OffsetScale\": {\n \"Desc\": [\n \"Takes the top value off the x87 stack and stores it to memory.\",\n \"SourceSize is 128bit for F80 values, 64-bit for low precision.\",\n \"StoreSize is the store size for conversion:\",\n \"Float: 80-bit, 64-bit, or 32-bit\"\n ],\n \"HasSideEffects\": true,\n \"X87\": true\n },\n \"StoreStackToStack u8:$StackLocation\": {\n \"Desc\": [\n \"Takes the top value off the x87 stack and stores it to stack location TOP+StackLocation\",\n \"Float: 80-bit, 64-bit, or 32-bit\",\n \"Int: 64-bit, 32-bit, 16-bit\"\n ],\n \"HasSideEffects\": true,\n \"X87\": true\n },\n \"PopStackDestroy\": {\n \"Desc\": [\n \"Pops the top value off the stack but doesn't save it anywhere.\"\n ],\n \"HasSideEffects\": true,\n \"X87\": true\n },\n \"FPR = ReadStackValue u8:$StackLocation\": {\n \"Desc\": [\n \"Reads a value off the stack at the offset\"\n ],\n \"DestSize\": \"OpSize::i128Bit\",\n \"X87\": true\n },\n \"GPR = StackValidTag u8:$StackLocation\": {\n \"Desc\": [\n \"Returns 1 if the value in location TOP+$StackLocation is valid, 0 otherwise.\"\n ],\n \"DestSize\": \"OpSize::i32Bit\",\n \"X87\": true\n },\n \"F80AddStack u8:$SrcStack1, u8:$SrcStack2\": {\n \"Desc\": [\n \"Adds two stack locations together, storing the result in to the first stack location\"\n ],\n \"HasSideEffects\": true,\n \"X87\": true\n },\n \"F80AddValue u8:$SrcStack, FPR:$X80Src\": {\n \"Desc\": [\n \"Adds a operand value to a stack location. The result stored in to the stack location provided.\"\n ],\n \"HasSideEffects\": true,\n \"X87\": true\n },\n \"FPR = F80Add FPR:$X80Src1, FPR:$X80Src2\": {\n \"DestSize\": \"OpSize::i128Bit\",\n \"JITDispatch\": false\n },\n \"F80SubStack u8:$DstStack, u8:$SrcStack1, u8:$SrcStack2\": {\n \"Desc\": [\n \"Subtracts the value in stack location TOP+$SrcStack2 from the value in stack location TOP+$SrcStack1.\",\n \"The result is stored in stack location TOP+$DstStack.\"\n ],\n \"HasSideEffects\": true,\n \"X87\": true\n },\n \"F80SubValue u8:$SrcStack, FPR:$X80Src\": {\n \"Desc\": [\n \"Subtracts the value $X80Src from the value in stack location TOP+$SrcStack.\",\n \"The result is stored in stack location TOP.\"\n ],\n \"HasSideEffects\": true,\n \"X87\": true\n },\n \"F80SubRValue FPR:$X80Src, u8:$SrcStack\": {\n \"Desc\": [\n \"Subtracts the value in stack location TOP+$SrcStack from the value $X80Src.\",\n \"The result is stored in stack location TOP.\"\n ],\n \"HasSideEffects\": true,\n \"X87\": true\n },\n \"FPR = F80Sub FPR:$X80Src1, FPR:$X80Src2\": {\n \"Desc\": [\n \"Subtracts the value in $X80Src1 from the value in $X80Src2.\",\n \"The result is returned.\",\n \"`FPR = X80Src2 - X80Src1`\"\n ],\n \"DestSize\": \"OpSize::i128Bit\",\n \"JITDispatch\": false\n },\n \"F80MulStack u8:$SrcStack1, u8:$SrcStack2\": {\n \"Desc\": [\n \"Multiplies two stack locations together, storing the result in to the first stack location\"\n ],\n \"HasSideEffects\": true,\n \"X87\": true\n },\n \"F80MulValue u8:$SrcStack, FPR:$X80Src\": {\n \"Desc\": [\n \"Multiplies a operand value to a stack location. The result stored in to the stack location provided.\"\n ],\n \"HasSideEffects\": true,\n \"X87\": true\n },\n \"FPR = F80Mul FPR:$X80Src1, FPR:$X80Src2\": {\n \"DestSize\": \"OpSize::i128Bit\",\n \"JITDispatch\": false\n },\n \"F80DivStack u8:$DstStack, u8:$SrcStack1, u8:$SrcStack2\": {\n \"Desc\": [\n \"Divides the value in stack location TOP+$SrcStack1 by the value in stack location TOP+$SrcStack2.\",\n \"The result is stored in stack location TOP+$DstStack.\",\n \"`FPR|Stack[TOP+DstStack] = Stack[TOP+SrcStack1] / Stack[TOP+SrcStack2]`\"\n ],\n \"HasSideEffects\": true,\n \"X87\": true\n },\n \"F80DivValue u8:$SrcStack, FPR:$X80Src\": {\n \"Desc\": [\n \"Divides the value in stack location TOP+$SrcStack by the value $X80Src.\",\n \"The result is stored in stack location TOP and returned.\",\n \"`FPR|Stack[TOP] = Stack[TOP+SrcStack] / X80Src`\"\n ],\n \"HasSideEffects\": true,\n \"X87\": true\n },\n \"F80DivRValue FPR:$X80Src, u8:$SrcStack\": {\n \"Desc\": [\n \"Divides the value X80Src by the value in stack location TOP+$SrcStack.\",\n \"The result is stored in stack location TOP.\",\n \"`FPR|Stack[TOP] = X80Src / Stack[TOP+SrcStack]`\"\n ],\n \"HasSideEffects\": true,\n \"X87\": true\n },\n \"FPR = F80Div FPR:$X80Src1, FPR:$X80Src2\": {\n \"Desc\": [\n \"Divides the value in $X80Src1 by the value in $X80Src2.\",\n \"The result is returned.\",\n \"`FPR = X80Src1 / X80Src2`\"\n ],\n \"DestSize\": \"OpSize::i128Bit\",\n \"JITDispatch\": false\n },\n \"F80StackXchange u8:$SrcStack\": {\n \"Desc\": [\n \"Exchanges the value at the top of the stack with the value at TOP+$SrcStack.\"\n ],\n \"X87\": true,\n \"HasSideEffects\": true\n },\n \"FPR = F80StackChangeSign\": {\n \"Desc\": [\n \"Complements the sign bit of the value at the top of the stack.\",\n \"Returns the new value at the top of the stack.\"\n ],\n \"HasSideEffects\": true,\n \"DestSize\": \"OpSize::i128Bit\",\n \"X87\": true\n },\n \"FPR = F80StackAbs\": {\n \"Desc\": [\n \"Clears the sign bit of the value at the top of the stack.\",\n \"Returns the new value at the top of the stack.\"\n ],\n \"HasSideEffects\": true,\n \"DestSize\": \"OpSize::i128Bit\",\n \"X87\": true\n },\n \"F80PTANStack\": {\n \"Desc\": [\n \"Computes the approximate tangent of the source operand in register ST(0), stores the result in ST(0), and pushes a 1.0 onto the FPU register stack.\"\n ],\n \"X87\": true,\n \"HasSideEffects\": true\n },\n \"FPR = F80ATANStack\": {\n \"Desc\": [\n \"Computes arctan(st1/st0) and stores it in st0. Then pops the stack.\"\n ],\n \"DestSize\": \"OpSize::i128Bit\",\n \"X87\": true,\n \"HasSideEffects\": true\n },\n \"FPR = F80ATAN FPR:$X80Src1, FPR:$X80Src2\": {\n \"DestSize\": \"OpSize::i128Bit\",\n \"JITDispatch\": false\n },\n \"F80FPREMStack\": {\n \"X87\": true,\n \"HasSideEffects\": true\n },\n \"FPR = F80FPREM FPR:$X80Src1, FPR:$X80Src2\": {\n \"DestSize\": \"OpSize::i128Bit\",\n \"JITDispatch\": false\n },\n \"F80FPREM1Stack\": {\n \"X87\": true,\n \"HasSideEffects\": true\n },\n \"FPR = F80FPREM1 FPR:$X80Src1, FPR:$X80Src2\": {\n \"DestSize\": \"OpSize::i128Bit\",\n \"JITDispatch\": false\n },\n \"F80SCALEStack\": {\n \"X87\": true,\n \"HasSideEffects\": true\n },\n \"FPR = F80SCALE FPR:$X80Src1, FPR:$X80Src2\": {\n \"DestSize\": \"OpSize::i128Bit\",\n \"JITDispatch\": false\n },\n \"FPR = F80CVT OpSize:#Size, FPR:$X80Src\": {\n \"DestSize\": \"Size\",\n \"JITDispatch\": false\n },\n \"GPR = F80CVTInt OpSize:#Size, FPR:$X80Src, i1:$Truncate\": {\n \"DestSize\": \"Size\",\n \"JITDispatch\": false\n },\n \"FPR = F80CVTTo FPR:$X80Src, OpSize:$SrcSize\": {\n \"DestSize\": \"OpSize::i128Bit\",\n \"JITDispatch\": false\n },\n \"FPR = F80CVTToInt GPR:$Src, OpSize:$SrcSize\": {\n \"DestSize\": \"OpSize::i128Bit\",\n \"JITDispatch\": false\n },\n \"F80RoundStack\": {\n \"Desc\": [\n \"Replaces the value at the top of the stack with its nearest integral value.\"\n ],\n \"X87\": true,\n \"HasSideEffects\": true\n },\n \"FPR = F80Round FPR:$X80Src\": {\n \"DestSize\": \"OpSize::i128Bit\",\n \"JITDispatch\": false\n },\n \"F80F2XM1Stack\": {\n \"X87\": true,\n \"HasSideEffects\": true\n },\n \"FPR = F80F2XM1 FPR:$X80Src\": {\n \"DestSize\": \"OpSize::i128Bit\",\n \"JITDispatch\": false\n },\n \"FPR = F80TAN FPR:$X80Src\": {\n \"DestSize\": \"OpSize::i128Bit\",\n \"JITDispatch\": false\n },\n \"F80SINStack\": {\n \"X87\": true,\n \"HasSideEffects\": true\n },\n \"FPR = F80SIN FPR:$X80Src\": {\n \"DestSize\": \"OpSize::i128Bit\",\n \"JITDispatch\": false\n },\n \"F80COSStack\": {\n \"X87\": true,\n \"HasSideEffects\": true\n },\n \"FPR = F80COS FPR:$X80Src\": {\n \"DestSize\": \"OpSize::i128Bit\",\n \"JITDispatch\": false\n },\n \"FPR:$Sin, FPR:$Cos = F80SINCOS FPR:$X80Src\": {\n \"DestSize\": \"OpSize::i128Bit\",\n \"HasSideEffects\": true,\n \"JITDispatch\": false\n },\n \"F80SINCOSStack\": {\n \"X87\": true,\n \"HasSideEffects\": true\n },\n \"F80SQRTStack\": {\n \"X87\": true,\n \"HasSideEffects\": true\n },\n \"FPR = F80SQRT FPR:$X80Src\": {\n \"DestSize\": \"OpSize::i128Bit\",\n \"JITDispatch\": false\n },\n \"FPR = F80XTRACT_EXP FPR:$X80Src\": {\n \"DestSize\": \"OpSize::i128Bit\",\n \"JITDispatch\": false\n },\n \"FPR = F80XTRACT_SIG FPR:$X80Src\": {\n \"DestSize\": \"OpSize::i128Bit\",\n \"JITDispatch\": false\n },\n \"GPR = F80StackTest u8:$SrcStack\": {\n \"Desc\": [\n \"Does comparison between value in stack at TOP + SrcStack\"\n ],\n \"DestSize\": \"OpSize::i32Bit\",\n \"X87\": true\n },\n \"GPR = F80CmpStack u8:$SrcStack\": {\n \"Desc\": [\n \"Does a scalar unordered compare between the value at the top of the stack and the value in stack position TOP+$SrcStack and stores the flags in to a GPR\",\n \"Ordering flag result is true if either float input is NaN\"\n ],\n \"DestSize\": \"OpSize::i32Bit\",\n \"X87\": true\n },\n \"GPR = F80CmpValue FPR:$X80Src\": {\n \"Desc\": [\n \"Does a scalar unordered compare between the value at the top of the stack and $X80Src and stores the asked for flags in to a GPR\",\n \"Ordering flag result is true if either float input is NaN\"\n ],\n \"DestSize\": \"OpSize::i32Bit\",\n \"HasSideEffects\": true,\n \"X87\": true\n },\n \"GPR = F80Cmp FPR:$X80Src1, FPR:$X80Src2\": {\n \"Desc\": [\"Does a scalar unordered compare and stores the flags in to a GPR\",\n \"Ordering flag result is true if either float input is NaN\"\n ],\n \"DestSize\": \"OpSize::i32Bit\",\n \"JITDispatch\": false\n },\n \"FPR = F80BCDLoad FPR:$X80Src\": {\n \"DestSize\": \"OpSize::i128Bit\",\n \"JITDispatch\": false\n },\n \"FPR = F80BCDStore FPR:$X80Src\": {\n \"DestSize\": \"OpSize::i128Bit\",\n \"JITDispatch\": false\n },\n \"FPR = F80FYL2XStack\": {\n \"Desc\": [\n \"Computes ST1 * log2(ST0)\",\n \"Stores the result in ST1, and pops the top of the stack.\",\n \"Returns the new value at the top of the stack, i.e. the result of the operation.\"\n ],\n \"HasSideEffects\": true,\n \"DestSize\": \"OpSize::i128Bit\",\n \"X87\": true\n },\n \"FPR = F80FYL2X FPR:$X80Src1, FPR:$X80Src2\": {\n \"DestSize\": \"OpSize::i128Bit\",\n \"JITDispatch\": false\n },\n \"F80VBSLStack OpSize:#RegisterSize, FPR:$VectorMask, u8:$SrcStack1, u8:$SrcStack2\": {\n \"Desc\": [\n \"Does a vector bitwise select.\",\n \"If the bit in the field is 1 then the corresponding bit is pulled from VectorTrue\",\n \"If the bit in the field is 0 then the corresponding bit is pulled from VectorFalse\",\n \"Writes the result to the top of the stack.\"\n ],\n \"X87\": true,\n \"HasSideEffects\": true\n }\n },\n \"Backend\": {\n \"Last\": {\n \"HasSideEffects\": true\n }\n }\n }\n}\n" }, { "path": "FEXCore/Source/Interface/IR/IRDumper.cpp", "content": "// SPDX-License-Identifier: MIT\n/*\n$info$\nmeta: ir|dumper ~ IR -> Text\ntags: ir|dumper\n$end_info$\n*/\n\n#include \"Interface/IR/IntrusiveIRList.h\"\n#include \"Interface/IR/RegisterAllocationData.h\"\n\n#include \n#include \n#include \n\n#include \n#include \n#include \n#include \n#include \n#include \n\nnamespace FEXCore::IR {\n#define IROP_GETNAME_IMPL\n#define IROP_GETRAARGS_IMPL\n#define IROP_REG_CLASSES_IMPL\n#define IROP_HASSIDEEFFECTS_IMPL\n#define IROP_SIZES_IMPL\n#define IROP_GETHASDEST_IMPL\n\n#include \n\nstatic void PrintArg(fextl::stringstream* out, const IRListView*, const SHA256Sum& Arg) {\n *out << fextl::fmt::format(\"sha256:{:02x}\", fmt::join(Arg.data, \"\"));\n}\n\nstatic void PrintArg(fextl::stringstream* out, const IRListView*, uint64_t Arg) {\n *out << fextl::fmt::format(\"#{:#x}\", Arg);\n}\n\nstatic void PrintArg(fextl::stringstream* out, const IRListView*, CondClass Arg) {\n if (Arg == CondClass::AL) {\n *out << \"ALWAYS\";\n return;\n }\n\n static constexpr std::array CondNames = {\"EQ\", \"NEQ\", \"UGE\", \"ULT\", \"MI\", \"PL\", \"VS\", \"VC\",\n \"UGT\", \"ULE\", \"SGE\", \"SLT\", \"SGT\", \"SLE\", \"TSTZ\", \"TSTNZ\",\n \"FLU\", \"FGE\", \"FLEU\", \"FGT\", \"FU\", \"FNU\"};\n\n *out << CondNames[FEXCore::ToUnderlying(Arg)];\n}\n\nstatic void PrintArg(fextl::stringstream* out, const IRListView*, MemOffsetType Arg) {\n static constexpr std::array Names = {\n \"SXTX\",\n \"UXTW\",\n \"SXTW\",\n };\n\n *out << Names[FEXCore::ToUnderlying(Arg)];\n}\n\nstatic void PrintArg(fextl::stringstream* out, const IRListView*, RegClass Arg) {\n *out << [Arg] {\n switch (Arg) {\n case RegClass::Invalid: return \"Invalid\";\n case RegClass::GPR: return \"GPR\";\n case RegClass::GPRFixed: return \"GPRFixed\";\n case RegClass::FPR: return \"FPR\";\n case RegClass::FPRFixed: return \"FPRFixed\";\n case RegClass::Complex: return \"Complex\";\n }\n return \"\";\n }();\n}\n\nstatic void PrintArg(fextl::stringstream* out, const IRListView* IR, OrderedNodeWrapper Arg) {\n if (Arg.IsImmediate()) {\n auto PhyReg = PhysicalRegister(Arg);\n\n switch (PhyReg.AsRegClass()) {\n case RegClass::GPR: *out << \"r\"; break;\n case RegClass::GPRFixed: *out << \"R\"; break;\n case RegClass::FPR: *out << \"v\"; break;\n case RegClass::FPRFixed: *out << \"V\"; break;\n case RegClass::Complex: *out << \"c\"; break;\n case RegClass::Invalid: *out << \"invalid\"; break;\n default: *out << \"unknown\"; break;\n }\n\n if (PhyReg.AsRegClass() != RegClass::Invalid) {\n *out << std::dec << uint32_t(PhyReg.Reg);\n }\n\n return;\n }\n\n auto [CodeNode, IROp] = IR->at(Arg)();\n const auto ArgID = Arg.ID();\n\n if (ArgID.IsInvalid()) {\n *out << \"%Invalid\";\n } else {\n *out << \"%\" << std::dec << ArgID;\n }\n\n if (GetHasDest(IROp->Op)) {\n auto ElementSize = IROp->ElementSize;\n uint32_t NumElements = 0;\n if (IROp->ElementSize == OpSize::iUnsized) {\n ElementSize = IROp->Size;\n }\n\n if (ElementSize != OpSize::iUnsized) {\n NumElements = IR::NumElements(IROp->Size, ElementSize);\n }\n\n *out << \" i\" << std::dec << IR::OpSizeAsBits(ElementSize);\n\n if (NumElements > 1) {\n *out << \"v\" << std::dec << NumElements;\n }\n }\n}\n\nstatic void PrintArg(fextl::stringstream* out, const IRListView*, FenceType Arg) {\n *out << [Arg] {\n switch (Arg) {\n case FenceType::Load: return \"Loads\";\n case FenceType::Store: return \"Stores\";\n case FenceType::LoadStore: return \"LoadStores\";\n case FenceType::Inst: return \"Instruction\";\n }\n return \"\";\n }();\n}\n\nstatic void PrintArg(fextl::stringstream* out, const IRListView*, RoundMode Arg) {\n *out << [Arg] {\n switch (Arg) {\n case RoundMode::Nearest: return \"Nearest\";\n case RoundMode::NegInfinity: return \"-Inf\";\n case RoundMode::PosInfinity: return \"+Inf\";\n case RoundMode::TowardsZero: return \"Towards Zero\";\n case RoundMode::Host: return \"Host\";\n }\n return \"\";\n }();\n}\n\nstatic void PrintArg(fextl::stringstream* out, const IRListView*, ConstPad Arg) {\n *out << [Arg] {\n switch (Arg) {\n case ConstPad::NoPad: return \"NoPad\";\n case ConstPad::DoPad: return \"DoPad\";\n case ConstPad::AutoPad: return \"AutoPad\";\n }\n return \"\";\n }();\n}\n\nstatic void PrintArg(fextl::stringstream* out, const IRListView*, NamedVectorConstant Arg) {\n *out << [Arg] {\n // clang-format off\n switch (Arg) {\n case NamedVectorConstant::NAMED_VECTOR_INCREMENTAL_U16_INDEX:\n return \"u16_incremental_index\";\n case NamedVectorConstant::NAMED_VECTOR_INCREMENTAL_U16_INDEX_UPPER:\n return \"u16_incremental_index_upper\";\n case NamedVectorConstant::NAMED_VECTOR_PADDSUBPS_INVERT:\n return \"addsubps_invert\";\n case NamedVectorConstant::NAMED_VECTOR_PADDSUBPS_INVERT_UPPER:\n return \"addsubps_invert_upper\";\n case NamedVectorConstant::NAMED_VECTOR_PADDSUBPD_INVERT:\n return \"addsubpd_invert\";\n case NamedVectorConstant::NAMED_VECTOR_PADDSUBPD_INVERT_UPPER:\n return \"addsubpd_invert_upper\";\n case NamedVectorConstant::NAMED_VECTOR_PSUBADDPS_INVERT:\n return \"subaddps_invert\";\n case NamedVectorConstant::NAMED_VECTOR_PSUBADDPS_INVERT_UPPER:\n return \"subaddps_invert_upper\";\n case NamedVectorConstant::NAMED_VECTOR_PSUBADDPD_INVERT:\n return \"subaddpd_invert\";\n case NamedVectorConstant::NAMED_VECTOR_PSUBADDPD_INVERT_UPPER:\n return \"subaddpd_invert_upper\";\n case NamedVectorConstant::NAMED_VECTOR_MOVMSKPS_SHIFT:\n return \"movmskps_shift\";\n case NamedVectorConstant::NAMED_VECTOR_AESKEYGENASSIST_SWIZZLE:\n return \"aeskeygenassist_swizzle\";\n case NamedVectorConstant::NAMED_VECTOR_BLENDPS_0110B:\n return \"blendps_0110b\";\n case NamedVectorConstant::NAMED_VECTOR_BLENDPS_0111B:\n return \"blendps_0111b\";\n case NamedVectorConstant::NAMED_VECTOR_BLENDPS_1001B:\n return \"blendps_1001b\";\n case NamedVectorConstant::NAMED_VECTOR_BLENDPS_1011B:\n return \"blendps_1011b\";\n case NamedVectorConstant::NAMED_VECTOR_BLENDPS_1101B:\n return \"blendps_1101b\";\n case NamedVectorConstant::NAMED_VECTOR_BLENDPS_1110B:\n return \"blendps_1110b\";\n case NamedVectorConstant::NAMED_VECTOR_MOVMASKB:\n return \"movmaskb\";\n case NamedVectorConstant::NAMED_VECTOR_MOVMASKB_UPPER:\n return \"movmaskb_upper\";\n case NamedVectorConstant::NAMED_VECTOR_ZERO:\n return \"vectorzero\";\n case NamedVectorConstant::NAMED_VECTOR_X87_ONE:\n return \"x87_1_0\";\n case NamedVectorConstant::NAMED_VECTOR_X87_LOG2_10:\n return \"x87_log2_10\";\n case NamedVectorConstant::NAMED_VECTOR_X87_LOG2_E:\n return \"x87_log2_e\";\n case NamedVectorConstant::NAMED_VECTOR_X87_PI:\n return \"x87_pi\";\n case NamedVectorConstant::NAMED_VECTOR_X87_LOG10_2:\n return \"x87_log10_2\";\n case NamedVectorConstant::NAMED_VECTOR_X87_LOG_2:\n return \"x87_log2\";\n case NamedVectorConstant::NAMED_VECTOR_CVTMAX_F32_I32:\n return \"cvtmax_f32_i32\";\n case NamedVectorConstant::NAMED_VECTOR_CVTMAX_F32_I32_UPPER:\n return \"cvtmax_f32_i32_upper\";\n case NamedVectorConstant::NAMED_VECTOR_CVTMAX_F32_I64:\n return \"cvtmax_f32_i64\";\n case NamedVectorConstant::NAMED_VECTOR_CVTMAX_F64_I32:\n return \"cvtmax_f64_i32\";\n case NamedVectorConstant::NAMED_VECTOR_CVTMAX_F64_I32_UPPER:\n return \"cvtmax_f64_i32_upper\";\n case NamedVectorConstant::NAMED_VECTOR_CVTMAX_F64_I64:\n return \"cvtmax_f64_i64\";\n case NamedVectorConstant::NAMED_VECTOR_CVTMAX_I32:\n return \"cvtmax_i32\";\n case NamedVectorConstant::NAMED_VECTOR_CVTMAX_I64:\n return \"cvtmax_i64\";\n case NamedVectorConstant::NAMED_VECTOR_F80_SIGN_MASK:\n return \"f80_sign_mask\";\n case NamedVectorConstant::NAMED_VECTOR_SHA1RNDS_K0:\n return \"sha1rnds_k0\";\n case NamedVectorConstant::NAMED_VECTOR_SHA1RNDS_K1:\n return \"sha1rnds_k1\";\n case NamedVectorConstant::NAMED_VECTOR_SHA1RNDS_K2:\n return \"sha1rnds_k2\";\n case NamedVectorConstant::NAMED_VECTOR_SHA1RNDS_K3:\n return \"sha1rnds_k3\";\n case NamedVectorConstant::NAMED_VECTOR_MAX:\n return \"\";\n }\n return \"\";\n // clang-format on\n }();\n}\n\nstatic void PrintArg(fextl::stringstream* out, const IRListView*, IndexNamedVectorConstant Arg) {\n *out << [Arg] {\n // clang-format off\n switch (Arg) {\n case IndexNamedVectorConstant::INDEXED_NAMED_VECTOR_PSHUFLW:\n return \"pshuflw\";\n case IndexNamedVectorConstant::INDEXED_NAMED_VECTOR_PSHUFHW:\n return \"pshufhw\";\n case IndexNamedVectorConstant::INDEXED_NAMED_VECTOR_PSHUFD:\n return \"pshufd\";\n case IndexNamedVectorConstant::INDEXED_NAMED_VECTOR_SHUFPS:\n return \"shufps\";\n case IndexNamedVectorConstant::INDEXED_NAMED_VECTOR_DPPS_MASK:\n return \"dpps_mask\";\n case IndexNamedVectorConstant::INDEXED_NAMED_VECTOR_DPPD_MASK:\n return \"dppd_mask\";\n case IndexNamedVectorConstant::INDEXED_NAMED_VECTOR_PBLENDW:\n return \"pblendw\";\n case INDEXED_NAMED_VECTOR_MAX:\n return \"\";\n }\n return \"\";\n // clang-format on\n }();\n}\n\nstatic void PrintArg(fextl::stringstream* out, const IRListView*, OpSize Arg) {\n *out << [Arg] {\n switch (Arg) {\n case OpSize::iUnsized: return \"Unsized\";\n case OpSize::i8Bit: return \"i8\";\n case OpSize::i16Bit: return \"i16\";\n case OpSize::i32Bit: return \"i32\";\n case OpSize::i64Bit: return \"i64\";\n case OpSize::f80Bit: return \"f80\";\n case OpSize::i128Bit: return \"i128\";\n case OpSize::i256Bit: return \"i256\";\n case OpSize::iInvalid: return \"Invalid\";\n }\n return \"\";\n }();\n}\n\nstatic void PrintArg(fextl::stringstream* out, const IRListView*, FloatCompareOp Arg) {\n *out << [Arg] {\n switch (Arg) {\n case FloatCompareOp::EQ: return \"FEQ\";\n case FloatCompareOp::LT: return \"FLT\";\n case FloatCompareOp::LE: return \"FLE\";\n case FloatCompareOp::UNO: return \"UNO\";\n case FloatCompareOp::NEQ: return \"NEQ\";\n case FloatCompareOp::ORD: return \"ORD\";\n }\n return \"\";\n }();\n}\n\nstatic void PrintArg(fextl::stringstream* out, const IRListView*, FEXCore::IR::BreakDefinition Arg) {\n *out << \"{\" << Arg.ErrorRegister << \".\";\n *out << static_cast(Arg.Signal) << \".\";\n *out << static_cast(Arg.TrapNumber) << \".\";\n *out << static_cast(Arg.si_code) << \"}\";\n}\n\nstatic void PrintArg(fextl::stringstream* out, const IRListView*, ShiftType Arg) {\n *out << [Arg] {\n switch (Arg) {\n case ShiftType::LSL: return \"LSL\";\n case ShiftType::LSR: return \"LSR\";\n case ShiftType::ASR: return \"ASR\";\n case ShiftType::ROR: return \"ROR\";\n }\n return \"\";\n }();\n}\n\nstatic void PrintArg(fextl::stringstream* out, const IRListView*, BranchHint Arg) {\n *out << [Arg] {\n switch (Arg) {\n case BranchHint::None: return \"None\";\n case BranchHint::Call: return \"Call\";\n case BranchHint::Return: return \"Return\";\n case BranchHint::CheckTF: return \"CheckTF\";\n }\n return \"\";\n }();\n}\n\nstatic void PrintArg(fextl::stringstream* out, const IRListView*, const std::array& Arg) {\n *out << fextl::fmt::format(\"{:02x}\", fmt::join(Arg, \"\"));\n}\n\nvoid Dump(fextl::stringstream* out, const IRListView* IR) {\n auto HeaderOp = IR->GetHeader();\n\n int8_t CurrentIndent = 0;\n auto AddIndent = [&out, &CurrentIndent]() {\n for (uint8_t i = 0; i < CurrentIndent; ++i) {\n *out << \"\\t\";\n }\n };\n\n ++CurrentIndent;\n AddIndent();\n *out << fextl::fmt::format(\"(%0) IRHeader %{}, #{:#x}, #{}, #{}\\n\", HeaderOp->Blocks.ID(), +HeaderOp->OriginalRIP, +HeaderOp->BlockCount,\n +HeaderOp->NumHostInstructions);\n\n for (auto [BlockNode, BlockHeader] : IR->GetBlocks()) {\n {\n auto BlockIROp = BlockHeader->C();\n\n AddIndent();\n *out << \"(%\" << IR->GetID(BlockNode) << \") \" << \"CodeBlock \";\n\n *out << \"%\" << BlockIROp->Begin.ID() << \", \";\n *out << \"%\" << BlockIROp->Last.ID() << std::endl;\n }\n\n ++CurrentIndent;\n for (auto [CodeNode, IROp] : IR->GetCode(BlockNode)) {\n const auto ID = IR->GetID(CodeNode);\n const auto Name = FEXCore::IR::GetName(IROp->Op);\n\n {\n AddIndent();\n if (GetHasDest(IROp->Op)) {\n\n auto ElementSize = IROp->ElementSize;\n uint8_t NumElements = 0;\n if (IROp->ElementSize != OpSize::iUnsized) {\n ElementSize = IROp->Size;\n }\n\n if (ElementSize != OpSize::iUnsized) {\n NumElements = IR::NumElements(IROp->Size, ElementSize);\n }\n\n *out << \"%\" << std::dec << ID;\n\n auto PhyReg = PhysicalRegister(CodeNode);\n if (!PhyReg.IsInvalid()) {\n switch (PhyReg.AsRegClass()) {\n case RegClass::GPR: *out << \"(r\"; break;\n case RegClass::GPRFixed: *out << \"(R\"; break;\n case RegClass::FPR: *out << \"(v\"; break;\n case RegClass::FPRFixed: *out << \"(V\"; break;\n case RegClass::Complex: *out << \"(complex\"; break;\n case RegClass::Invalid: *out << \"(invalid\"; break;\n default: *out << \"(unknown\"; break;\n }\n if (PhyReg.AsRegClass() != RegClass::Invalid) {\n *out << std::dec << uint32_t(PhyReg.Reg) << \")\";\n } else {\n *out << \")\";\n }\n }\n\n *out << \" i\" << std::dec << IR::OpSizeAsBits(ElementSize);\n\n if (NumElements > 1) {\n *out << \"v\" << std::dec << NumElements;\n }\n\n *out << \" = \";\n } else {\n\n auto ElementSize = IROp->ElementSize;\n if (IROp->ElementSize == OpSize::iUnsized) {\n ElementSize = IROp->Size;\n }\n uint32_t NumElements = 0;\n if (ElementSize != OpSize::iUnsized) {\n NumElements = IR::NumElements(IROp->Size, ElementSize);\n }\n\n *out << \"(%\" << std::dec << ID << ' ';\n *out << 'i' << std::dec << IR::OpSizeAsBits(ElementSize);\n if (NumElements > 1) {\n *out << 'v' << std::dec << NumElements;\n }\n *out << \") \";\n }\n *out << Name;\n\n#define IROP_ARGPRINTER_HELPER\n#include \n default: *out << \"\"; break;\n }\n\n //*out << \" (\" << std::dec << CodeNode->GetUses() << \")\";\n\n *out << \"\\n\";\n }\n }\n\n CurrentIndent = std::max(0, CurrentIndent - 1);\n}\n}\n}\n" }, { "path": "FEXCore/Source/Interface/IR/IREmitter.cpp", "content": "// SPDX-License-Identifier: MIT\n/*\n$info$\nmeta: ir|emitter ~ C++ Functions to generate IR. See IR.json for spec.\ntags: ir|emitter\n$end_info$\n*/\n\n#include \"Interface/IR/IREmitter.h\"\n\n#include \n#include \n#include \n\n#include \n#include \n\nnamespace FEXCore::IR {\n\nstatic bool IsFragmentExit(FEXCore::IR::IROps Op) {\n switch (Op) {\n case OP_EXITFUNCTION:\n case OP_BREAK: return true;\n default: return false;\n }\n}\n\nbool IsBlockExit(FEXCore::IR::IROps Op) {\n switch (Op) {\n case OP_JUMP:\n case OP_CONDJUMP: return true;\n default: return IsFragmentExit(Op);\n }\n}\n\nRegClass IREmitter::WalkFindRegClass(Ref Node) {\n auto Class = GetOpRegClass(Node);\n switch (Class) {\n case RegClass::GPR:\n case RegClass::FPR:\n case RegClass::GPRFixed:\n case RegClass::FPRFixed:\n case RegClass::Invalid: return Class;\n default: break;\n }\n\n // Complex case, needs to be handled on an op by op basis\n uintptr_t DataBegin = DualListData.DataBegin();\n\n FEXCore::IR::IROp_Header* IROp = Node->Op(DataBegin);\n\n switch (IROp->Op) {\n case IROps::OP_LOADREGISTER: {\n auto Op = IROp->C();\n return Op->Class;\n break;\n }\n case IROps::OP_LOADCONTEXT: {\n auto Op = IROp->C();\n return Op->Class;\n break;\n }\n case IROps::OP_LOADCONTEXTINDEXED: {\n auto Op = IROp->C();\n return Op->Class;\n break;\n }\n case IROps::OP_FILLREGISTER: {\n auto Op = IROp->C();\n return Op->Class;\n break;\n }\n case IROps::OP_LOADMEM: {\n auto Op = IROp->C();\n return Op->Class;\n break;\n }\n case IROps::OP_LOADMEMTSO: {\n auto Op = IROp->C();\n return Op->Class;\n break;\n }\n default: LOGMAN_MSG_A_FMT(\"Unhandled op type: {} {} in argument class validation\", ToUnderlying(IROp->Op), GetOpName(Node)); break;\n }\n return RegClass::Invalid;\n}\n\nvoid IREmitter::ResetWorkingList() {\n DualListData.Reset();\n CodeBlocks.clear();\n CurrentWriteCursor = nullptr;\n // This is necessary since we do \"null\" pointer checks\n InvalidNode = reinterpret_cast(DualListData.ListAllocate(sizeof(OrderedNode)));\n memset(InvalidNode, 0, sizeof(OrderedNode));\n CurrentCodeBlock = nullptr;\n}\n\nvoid IREmitter::ReplaceAllUsesWithRange(Ref Node, Ref NewNode, AllNodesIterator Begin, AllNodesIterator End) {\n uintptr_t ListBegin = DualListData.ListBegin();\n auto NodeId = Node->Wrapped(ListBegin).ID();\n\n while (Begin != End) {\n auto [RealNode, IROp] = Begin();\n\n const uint8_t NumArgs = IR::GetArgs(IROp->Op);\n for (uint8_t i = 0; i < NumArgs; ++i) {\n if (IROp->Args[i].ID() == NodeId) {\n Node->RemoveUse();\n NewNode->AddUse();\n IROp->Args[i].NodeOffset = NewNode->Wrapped(ListBegin).NodeOffset;\n\n // We can stop searching once all uses of the node are gone.\n if (Node->NumUses == 0) {\n return;\n }\n }\n }\n\n ++Begin;\n }\n}\n\nvoid IREmitter::ReplaceNodeArgument(Ref Node, uint8_t Arg, Ref NewArg) {\n uintptr_t ListBegin = DualListData.ListBegin();\n uintptr_t DataBegin = DualListData.DataBegin();\n\n FEXCore::IR::IROp_Header* IROp = Node->Op(DataBegin);\n OrderedNodeWrapper OldArgWrapper = IROp->Args[Arg];\n Ref OldArg = OldArgWrapper.GetNode(ListBegin);\n OldArg->RemoveUse();\n NewArg->AddUse();\n IROp->Args[Arg].NodeOffset = NewArg->Wrapped(ListBegin).NodeOffset;\n}\n\nvoid IREmitter::RemoveArgUses(Ref Node) {\n uintptr_t ListBegin = DualListData.ListBegin();\n uintptr_t DataBegin = DualListData.DataBegin();\n\n FEXCore::IR::IROp_Header* IROp = Node->Op(DataBegin);\n\n const uint8_t NumArgs = IR::GetArgs(IROp->Op);\n for (uint8_t i = 0; i < NumArgs; ++i) {\n auto ArgNode = IROp->Args[i].GetNode(ListBegin);\n ArgNode->RemoveUse();\n }\n}\n\nvoid IREmitter::RemovePostRA(Ref Node) {\n Node->Unlink(DualListData.ListBegin());\n}\n\nvoid IREmitter::Remove(Ref Node) {\n RemoveArgUses(Node);\n\n Node->Unlink(DualListData.ListBegin());\n}\n\nIREmitter::IRPair IREmitter::CreateNewCodeBlockAfter(Ref insertAfter) {\n auto OldCursor = GetWriteCursor();\n\n auto CodeNode = CreateCodeNode();\n\n if (insertAfter) {\n LinkCodeBlocks(insertAfter, CodeNode);\n } else {\n LOGMAN_THROW_A_FMT(CurrentCodeBlock != nullptr, \"CurrentCodeBlock must not be null here\");\n\n // Find last block\n auto LastBlock = CurrentCodeBlock;\n\n while (LastBlock->Header.Next.GetNode(DualListData.ListBegin()) != InvalidNode) {\n LastBlock = LastBlock->Header.Next.GetNode(DualListData.ListBegin());\n }\n\n // Append it after the last block\n LinkCodeBlocks(LastBlock, CodeNode);\n }\n\n SetWriteCursor(OldCursor);\n\n return CodeNode;\n}\n\nvoid IREmitter::SetCurrentCodeBlock(Ref Node) {\n CurrentCodeBlock = Node;\n LOGMAN_THROW_A_FMT(Node->Op(DualListData.DataBegin())->Op == OP_CODEBLOCK, \"Node wasn't codeblock. It was '{}'\",\n IR::GetName(Node->Op(DualListData.DataBegin())->Op));\n SetWriteCursor(Node->Op(DualListData.DataBegin())->CW()->Begin.GetNode(DualListData.ListBegin()));\n\n // Constants are pooled only within a single block.\n NrConstants = 0;\n}\n\n} // namespace FEXCore::IR\n" }, { "path": "FEXCore/Source/Interface/IR/IREmitter.h", "content": "// SPDX-License-Identifier: MIT\n#pragma once\n\n#include \"CodeEmitter/Emitter.h\"\n#include \"Interface/IR/IR.h\"\n#include \"Interface/IR/IntrusiveIRList.h\"\n\n#include \n#include \n\n#include \n#include \n\n#include \n#include \n#include \n\nnamespace FEXCore::IR {\n\nclass IREmitter {\npublic:\n IREmitter(FEXCore::Utils::IntrusivePooledAllocator& ThreadAllocator, bool SupportsTSOImm9)\n : DualListData {ThreadAllocator, 8 * 1024 * 1024}\n , SupportsTSOImm9(SupportsTSOImm9) {}\n\n virtual ~IREmitter() = default;\n\n void ReownOrClaimBuffer() {\n DualListData.ReownOrClaimBuffer();\n\n // Reset the working list on new buffer.\n ResetWorkingList();\n }\n\n void DelayedDisownBuffer() {\n DualListData.DelayedDisownBuffer();\n }\n\n IRListView ViewIR() {\n return IRListView(&DualListData);\n }\n\n /**\n * @name IR allocation routines\n *\n * @{ */\n\n RegClass WalkFindRegClass(Ref Node);\n\n // These inlining helpers are used by IRDefines.inc so define first.\n Ref InlineMem(OpSize Size, Ref Offset, MemOffsetType OffsetType, uint8_t& OffsetScale, bool TSO = false) {\n uint64_t Imm {};\n if (OffsetType != MemOffsetType::SXTX || !IsValueConstant(WrapNode(Offset), &Imm)) {\n return Offset;\n }\n\n // The immediate may be scaled in the IR, we need to correct for that.\n Imm *= OffsetScale;\n\n // Signed immediate unscaled 9-bit range for both regular and LRCPC2 ops.\n bool IsSIMM9 = ((int64_t)Imm >= -256) && ((int64_t)Imm <= 255);\n IsSIMM9 &= (SupportsTSOImm9 || !TSO);\n\n // Extended offsets for regular loadstore only.\n LOGMAN_THROW_A_FMT(Size >= IR::OpSize::i8Bit && Size <= IR::OpSize::i256Bit, \"Must be sized\");\n\n bool IsExtended = (Imm & (IR::OpSizeToSize(Size) - 1)) == 0 && Imm / IR::OpSizeToSize(Size) <= 4095;\n IsExtended &= !TSO;\n\n if (IsSIMM9 || IsExtended) {\n OffsetScale = 1;\n return _InlineConstant(Imm);\n } else {\n return Offset;\n }\n }\n\n#define DEF_INLINE(Type, Variable, Filter) \\\n Ref Inline##Type(OpSize Size, Ref Source) { \\\n uint64_t Variable; \\\n if (IsValueConstant(WrapNode(Source), &Variable) && (Filter)) { \\\n return _InlineConstant(Variable); \\\n } else { \\\n return Source; \\\n } \\\n }\n\n DEF_INLINE(Any, _, true)\n DEF_INLINE(Zero, X, X == 0)\n DEF_INLINE(AddSub, X, ARMEmitter::IsImmAddSub(X))\n DEF_INLINE(LargeAddSub, X, ARMEmitter::IsImmAddSub(X) && Size >= OpSize::i32Bit);\n DEF_INLINE(Logical, X, ARMEmitter::Emitter::IsImmLogical(X, std::max((int)IR::OpSizeAsBits(Size), 32)));\n\n Ref InlineSubtractZero(OpSize Size, Ref Src1, Ref Src2) {\n // Only inline a zero if we won't inline the other source.\n return IsValueConstant(WrapNode(Src2)) ? Src1 : InlineZero(Size, Src1);\n }\n#undef DEF_INLINE\n\n// These handlers add cost to the constructor and destructor\n// If it becomes an issue then blow them away\n// GCC also generates some pretty atrocious code around these\n// Use Clang!\n#define IROP_ALLOCATE_HELPERS\n#define IROP_DISPATCH_HELPERS\n#include \n IRPair _Jump() {\n return _Jump(InvalidNode);\n }\n IRPair _CondJump(Ref ssa0, CondClass cond = CondClass::NEQ) {\n return _CondJump(ssa0, _Constant(0), InvalidNode, InvalidNode, cond, GetOpSize(ssa0));\n }\n IRPair _CondJump(Ref ssa0, Ref ssa1, Ref ssa2, CondClass cond = CondClass::NEQ) {\n return _CondJump(ssa0, _Constant(0), ssa1, ssa2, cond, GetOpSize(ssa0));\n }\n\n IRPair _LoadContextGPR(OpSize ByteSize, uint32_t Offset) {\n return _LoadContext(ByteSize, RegClass::GPR, Offset);\n }\n IRPair _LoadContextFPR(OpSize ByteSize, uint32_t Offset) {\n return _LoadContext(ByteSize, RegClass::FPR, Offset);\n }\n IRPair _StoreContextGPR(OpSize ByteSize, Ref Value, uint32_t Offset) {\n return _StoreContext(ByteSize, RegClass::GPR, Value, Offset);\n }\n IRPair _StoreContextFPR(OpSize ByteSize, Ref Value, uint32_t Offset) {\n return _StoreContext(ByteSize, RegClass::FPR, Value, Offset);\n }\n\n IRPair _LoadContextGPRIndexed(Ref Index, OpSize ByteSize, uint32_t BaseOffset, uint32_t Stride) {\n return _LoadContextIndexed(Index, ByteSize, BaseOffset, Stride, RegClass::GPR);\n }\n IRPair _LoadContextFPRIndexed(Ref Index, OpSize ByteSize, uint32_t BaseOffset, uint32_t Stride) {\n return _LoadContextIndexed(Index, ByteSize, BaseOffset, Stride, RegClass::FPR);\n }\n IRPair _StoreContextGPRIndexed(Ref Value, Ref Index, OpSize ByteSize, uint32_t BaseOffset, uint32_t Stride) {\n return _StoreContextIndexed(Value, Index, ByteSize, BaseOffset, Stride, RegClass::GPR);\n }\n IRPair _StoreContextFPRIndexed(Ref Value, Ref Index, OpSize ByteSize, uint32_t BaseOffset, uint32_t Stride) {\n return _StoreContextIndexed(Value, Index, ByteSize, BaseOffset, Stride, RegClass::FPR);\n }\n\n IRPair _LoadMem(RegClass Class, OpSize Size, Ref ssa0, OpSize Align = OpSize::i8Bit) {\n return _LoadMem(Class, Size, ssa0, Invalid(), Align, MemOffsetType::SXTX, 1);\n }\n IRPair _LoadMemGPR(OpSize Size, Ref ssa0, OpSize Align = OpSize::i8Bit) {\n return _LoadMem(RegClass::GPR, Size, ssa0, Invalid(), Align, MemOffsetType::SXTX, 1);\n }\n IRPair _LoadMemGPR(OpSize Size, Ref Addr, Ref Offset, OpSize Align, MemOffsetType OffsetType, uint8_t OffsetScale) {\n return _LoadMem(RegClass::GPR, Size, Addr, Offset, Align, OffsetType, OffsetScale);\n }\n IRPair _LoadMemFPR(OpSize Size, Ref ssa0, OpSize Align = OpSize::i8Bit) {\n return _LoadMem(RegClass::FPR, Size, ssa0, Invalid(), Align, MemOffsetType::SXTX, 1);\n }\n IRPair _LoadMemFPR(OpSize Size, Ref Addr, Ref Offset, OpSize Align, MemOffsetType OffsetType, uint8_t OffsetScale) {\n return _LoadMem(RegClass::FPR, Size, Addr, Offset, Align, OffsetType, OffsetScale);\n }\n IRPair _StoreMem(RegClass Class, OpSize Size, Ref Addr, Ref Value, OpSize Align = OpSize::i8Bit) {\n return _StoreMem(Class, Size, Value, Addr, Invalid(), Align, MemOffsetType::SXTX, 1);\n }\n IRPair _StoreMemGPR(OpSize Size, Ref Addr, Ref Value, OpSize Align = OpSize::i8Bit) {\n return _StoreMem(RegClass::GPR, Size, Value, Addr, Invalid(), Align, MemOffsetType::SXTX, 1);\n }\n IRPair _StoreMemGPR(OpSize Size, Ref Value, Ref Addr, Ref Offset, OpSize Align, MemOffsetType OffsetType, uint8_t OffsetScale) {\n return _StoreMem(RegClass::GPR, Size, Value, Addr, Offset, Align, OffsetType, OffsetScale);\n }\n IRPair _StoreMemFPR(OpSize Size, Ref Addr, Ref Value, OpSize Align = OpSize::i8Bit) {\n return _StoreMem(RegClass::FPR, Size, Value, Addr, Invalid(), Align, MemOffsetType::SXTX, 1);\n }\n IRPair _StoreMemFPR(OpSize Size, Ref Value, Ref Addr, Ref Offset, OpSize Align, MemOffsetType OffsetType, uint8_t OffsetScale) {\n return _StoreMem(RegClass::FPR, Size, Value, Addr, Offset, Align, OffsetType, OffsetScale);\n }\n\n IRPair _StoreMemPairGPR(OpSize Size, Ref Value1, Ref Value2, Ref Addr, uint32_t Offset) {\n return _StoreMemPair(RegClass::GPR, Size, Value1, Value2, Addr, Offset);\n }\n IRPair _StoreMemPairFPR(OpSize Size, Ref Value1, Ref Value2, Ref Addr, uint32_t Offset) {\n return _StoreMemPair(RegClass::FPR, Size, Value1, Value2, Addr, Offset);\n }\n\n IRPair Select01(FEXCore::IR::OpSize CompareSize, CondClass Cond, OrderedNode* Cmp1, OrderedNode* Cmp2) {\n return _Select(OpSize::i64Bit, CompareSize, Cond, Cmp1, Cmp2, _InlineConstant(1), _InlineConstant(0));\n }\n\n IRPair To01(FEXCore::IR::OpSize CompareSize, OrderedNode* Cmp1) {\n return Select01(CompareSize, CondClass::NEQ, Cmp1, Constant(0));\n }\n\n IRPair _NZCVSelect01(CondClass Cond) {\n return _NZCVSelect(OpSize::i64Bit, Cond, _InlineConstant(1), _InlineConstant(0));\n }\n\n Ref Addsub(IR::OpSize Size, IROps Op, IROps NegatedOp, Ref Src1, uint64_t Src2) {\n // Sign-extend the constant\n if (Size == OpSize::i32Bit) {\n Src2 = (int64_t)(int32_t)Src2;\n }\n\n // Negative constants need to be negated to inline.\n if (Src2 & (1ull << 63) && ARMEmitter::IsImmAddSub(-Src2)) {\n Op = NegatedOp;\n Src2 = -Src2;\n }\n\n auto Dest = _Add(Size, Src1, Constant(Src2));\n Dest.first->Header.Op = Op;\n return Dest;\n }\n\n Ref Add(IR::OpSize Size, Ref Src1, uint64_t Src2) {\n return Addsub(Size, OP_ADD, OP_SUB, Src1, Src2);\n }\n\n Ref Sub(IR::OpSize Size, Ref Src1, uint64_t Src2) {\n return Addsub(Size, OP_SUB, OP_ADD, Src1, Src2);\n }\n\n Ref AddWithFlags(IR::OpSize Size, Ref Src1, uint64_t Src2) {\n return Addsub(Size, OP_ADDWITHFLAGS, OP_SUBWITHFLAGS, Src1, Src2);\n }\n\n Ref SubWithFlags(IR::OpSize Size, Ref Src1, uint64_t Src2) {\n return Addsub(Size, OP_SUBWITHFLAGS, OP_ADDWITHFLAGS, Src1, Src2);\n }\n\n#define DEF_ADDSUB(Op) \\\n Ref Op(IR::OpSize Size, Ref Src1, Ref Src2) { \\\n uint64_t Constant; \\\n if (IsValueConstant(WrapNode(Src2), &Constant)) { \\\n return Op(Size, Src1, Constant); \\\n } else { \\\n return _##Op(Size, Src1, Src2); \\\n } \\\n }\n\n DEF_ADDSUB(Add)\n DEF_ADDSUB(Sub)\n DEF_ADDSUB(AddWithFlags)\n DEF_ADDSUB(SubWithFlags)\n\n struct ConstantData {\n int64_t Value;\n ConstPad Pad;\n int32_t MaxBytes;\n [[nodiscard]] auto operator<=>(const ConstantData&) const noexcept = default;\n };\n ConstantData Constants[32];\n Ref ConstantRefs[32];\n uint32_t NrConstants;\n\n Ref Constant(int64_t Value, ConstPad Pad = IR::ConstPad::NoPad, int32_t MaxBytes = 0) {\n const ConstantData Data {\n .Value = Value,\n .Pad = Pad,\n .MaxBytes = MaxBytes,\n };\n // Search for the constant in the pool.\n for (unsigned i = 0; i < std::min(NrConstants, 32u); ++i) {\n if (Constants[i] == Data) {\n return ConstantRefs[i];\n }\n }\n\n // Otherwise, materialize a fresh constant and pool it.\n Ref R = _Constant(Value, Pad, MaxBytes);\n unsigned i = (NrConstants++) & 31;\n Constants[i] = Data;\n ConstantRefs[i] = R;\n return R;\n }\n\n Ref Invalid() {\n return InvalidNode;\n }\n\n void SetJumpTarget(IR::IROp_Jump* Op, Ref Target) {\n LOGMAN_THROW_A_FMT(Target->Op(DualListData.DataBegin())->Op == OP_CODEBLOCK, \"Tried setting Jump target to %{} {}\",\n Target->Wrapped(DualListData.ListBegin()).ID(), IR::GetName(Target->Op(DualListData.DataBegin())->Op));\n\n Op->Header.Args[0].NodeOffset = Target->Wrapped(DualListData.ListBegin()).NodeOffset;\n }\n void SetTrueJumpTarget(IR::IROp_CondJump* Op, Ref Target) {\n LOGMAN_THROW_A_FMT(Target->Op(DualListData.DataBegin())->Op == OP_CODEBLOCK, \"Tried setting CondJump target to %{} {}\",\n Target->Wrapped(DualListData.ListBegin()).ID(), IR::GetName(Target->Op(DualListData.DataBegin())->Op));\n\n Op->TrueBlock.NodeOffset = Target->Wrapped(DualListData.ListBegin()).NodeOffset;\n }\n void SetFalseJumpTarget(IR::IROp_CondJump* Op, Ref Target) {\n LOGMAN_THROW_A_FMT(Target->Op(DualListData.DataBegin())->Op == OP_CODEBLOCK, \"Tried setting CondJump target to %{} {}\",\n Target->Wrapped(DualListData.ListBegin()).ID(), IR::GetName(Target->Op(DualListData.DataBegin())->Op));\n\n Op->FalseBlock.NodeOffset = Target->Wrapped(DualListData.ListBegin()).NodeOffset;\n }\n\n void SetJumpTarget(IRPair Op, Ref Target) {\n LOGMAN_THROW_A_FMT(Target->Op(DualListData.DataBegin())->Op == OP_CODEBLOCK, \"Tried setting Jump target to %{} {}\",\n Target->Wrapped(DualListData.ListBegin()).ID(), IR::GetName(Target->Op(DualListData.DataBegin())->Op));\n\n Op.first->Header.Args[0].NodeOffset = Target->Wrapped(DualListData.ListBegin()).NodeOffset;\n }\n void SetTrueJumpTarget(IRPair Op, Ref Target) {\n LOGMAN_THROW_A_FMT(Target->Op(DualListData.DataBegin())->Op == OP_CODEBLOCK, \"Tried setting CondJump target to %{} {}\",\n Target->Wrapped(DualListData.ListBegin()).ID(), IR::GetName(Target->Op(DualListData.DataBegin())->Op));\n Op.first->TrueBlock.NodeOffset = Target->Wrapped(DualListData.ListBegin()).NodeOffset;\n }\n void SetFalseJumpTarget(IRPair Op, Ref Target) {\n LOGMAN_THROW_A_FMT(Target->Op(DualListData.DataBegin())->Op == OP_CODEBLOCK, \"Tried setting CondJump target to %{} {}\",\n Target->Wrapped(DualListData.ListBegin()).ID(), IR::GetName(Target->Op(DualListData.DataBegin())->Op));\n Op.first->FalseBlock.NodeOffset = Target->Wrapped(DualListData.ListBegin()).NodeOffset;\n }\n\n /** @} */\n RegClass WalkFindRegClass(OrderedNodeWrapper ssa) {\n Ref RealNode = ssa.GetNode(DualListData.ListBegin());\n return WalkFindRegClass(RealNode);\n }\n\n bool IsValueConstant(OrderedNodeWrapper ssa, uint64_t* Constant = nullptr) {\n Ref RealNode = ssa.GetNode(DualListData.ListBegin());\n FEXCore::IR::IROp_Header* IROp = RealNode->Op(DualListData.DataBegin());\n if (IROp->Op == OP_CONSTANT) {\n auto Op = IROp->C();\n if (Constant) {\n *Constant = Op->Constant;\n }\n return true;\n }\n return false;\n }\n\n bool IsValueInlineConstant(OrderedNodeWrapper ssa) {\n Ref RealNode = ssa.GetNode(DualListData.ListBegin());\n FEXCore::IR::IROp_Header* IROp = RealNode->Op(DualListData.DataBegin());\n if (IROp->Op == OP_INLINECONSTANT) {\n return true;\n }\n return false;\n }\n\n FEXCore::IR::IROp_Header* GetOpHeader(OrderedNodeWrapper ssa) {\n Ref RealNode = ssa.GetNode(DualListData.ListBegin());\n return RealNode->Op(DualListData.DataBegin());\n }\n\n Ref UnwrapNode(OrderedNodeWrapper ssa) {\n return ssa.GetNode(DualListData.ListBegin());\n }\n\n OrderedNodeWrapper WrapNode(Ref node) {\n return node->Wrapped(DualListData.ListBegin());\n }\n\n NodeIterator GetIterator(OrderedNodeWrapper wrapper) {\n return NodeIterator(DualListData.ListBegin(), DualListData.DataBegin(), wrapper);\n }\n\n void ReplaceAllUsesWithRange(Ref Node, Ref NewNode, AllNodesIterator Begin, AllNodesIterator End);\n\n void ReplaceUsesWithAfter(Ref Node, Ref NewNode, AllNodesIterator After) {\n ++After;\n ReplaceAllUsesWithRange(Node, NewNode, After, AllNodesIterator(DualListData.ListBegin(), DualListData.DataBegin()));\n }\n\n void ReplaceUsesWithAfter(Ref Node, Ref NewNode, Ref After) {\n auto Wrapped = After->Wrapped(DualListData.ListBegin());\n AllNodesIterator It = AllNodesIterator(DualListData.ListBegin(), DualListData.DataBegin(), Wrapped);\n\n ReplaceUsesWithAfter(Node, NewNode, It);\n }\n\n void ReplaceNodeArgument(Ref Node, uint8_t Arg, Ref NewArg);\n\n void Remove(Ref Node);\n void RemovePostRA(Ref Node);\n\n void CopyData(const IREmitter& rhs) {\n LOGMAN_THROW_A_FMT(rhs.DualListData.DataBackingSize() <= DualListData.DataBackingSize(), \"Trying to take ownership of data that is too \"\n \"large\");\n LOGMAN_THROW_A_FMT(rhs.DualListData.ListBackingSize() <= DualListData.ListBackingSize(), \"Trying to take ownership of data that is too \"\n \"large\");\n DualListData.CopyData(rhs.DualListData);\n InvalidNode = rhs.InvalidNode->Wrapped(rhs.DualListData.ListBegin()).GetNode(DualListData.ListBegin());\n CurrentWriteCursor = rhs.CurrentWriteCursor;\n CodeBlocks = rhs.CodeBlocks;\n for (auto& CodeBlock : CodeBlocks) {\n CodeBlock = CodeBlock->Wrapped(rhs.DualListData.ListBegin()).GetNode(DualListData.ListBegin());\n }\n }\n\n void SetWriteCursor(Ref Node) {\n CurrentWriteCursor = Node;\n }\n\n // Set cursor to write before Node\n void SetWriteCursorBefore(Ref Node) {\n auto IR = ViewIR();\n auto Before = IR.at(Node);\n --Before;\n\n SetWriteCursor((*Before).Node);\n }\n\n Ref GetWriteCursor() {\n return CurrentWriteCursor;\n }\n\n Ref GetCurrentBlock() {\n return CurrentCodeBlock;\n }\n\n /**\n * @brief This creates an orphaned code node\n * The IROp backing is in the correct list but the OrderedNode lives outside of the list\n *\n * XXX: This is because we don't want code blocks to interleave with current instruction IR ops currently\n * We can change this behaviour once we remove the old BeginBlock/EndBlock types\n *\n * @return OrderedNode\n */\n IRPair CreateCodeNode(bool EntryPoint = false, uint32_t GuestEntryOffset = 0) {\n SetWriteCursor(nullptr); // Orphan from any previous nodes\n\n auto ID = ViewIR().GetHeader()->BlockCount++;\n auto CodeNode = _CodeBlock(InvalidNode, InvalidNode, ID, EntryPoint, GuestEntryOffset);\n\n CodeBlocks.emplace_back(CodeNode);\n\n SetWriteCursor(nullptr); // Orphan from any future nodes\n\n auto Begin = _BeginBlock(CodeNode);\n CodeNode.first->Begin = Begin.Node->Wrapped(DualListData.ListBegin());\n\n auto EndBlock = _EndBlock(CodeNode);\n CodeNode.first->Last = EndBlock.Node->Wrapped(DualListData.ListBegin());\n\n return CodeNode;\n }\n\n /**\n * @name Links codeblocks together\n * Codeblocks are singly linked so we need to walk the list forward if the linked block isn't isn't the last\n *\n * eq.\n * CodeNode->Next -> Next\n * to\n * CodeNode->Next -> New -> Next\n *\n * @{ */\n /** @} */\n void LinkCodeBlocks(Ref CodeNode, Ref Next) {\n [[maybe_unused]] auto CurrentIROp = CodeNode->Op(DualListData.DataBegin())->CW();\n#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED\n LOGMAN_THROW_A_FMT(CurrentIROp->Header.Op == IROps::OP_CODEBLOCK, \"Invalid\");\n#endif\n\n CodeNode->append(DualListData.ListBegin(), Next);\n }\n\n IRPair CreateNewCodeBlockAtEnd() {\n return CreateNewCodeBlockAfter(nullptr);\n }\n IRPair CreateNewCodeBlockAfter(Ref insertAfter);\n void SetCurrentCodeBlock(Ref Node);\n\nprotected:\n void RemoveArgUses(Ref Node);\n\n Ref CreateNode(IROp_Header* Op) {\n uintptr_t ListBegin = DualListData.ListBegin();\n size_t Size = sizeof(OrderedNode);\n void* Ptr = DualListData.ListAllocate(Size);\n Ref Node = new (Ptr) OrderedNode();\n Node->Header.Value.SetOffset(DualListData.DataBegin(), reinterpret_cast(Op));\n\n if (CurrentWriteCursor) {\n CurrentWriteCursor->append(ListBegin, Node);\n }\n CurrentWriteCursor = Node;\n return Node;\n }\n\n Ref GetNode(uint32_t SSANode) {\n uintptr_t ListBegin = DualListData.ListBegin();\n Ref Node = reinterpret_cast(ListBegin + SSANode * sizeof(OrderedNode));\n return Node;\n }\n\n Ref EmplaceOrphanedNode(Ref OldNode) {\n size_t Size = sizeof(OrderedNode);\n Ref Ptr = reinterpret_cast(DualListData.ListAllocate(Size));\n memcpy(Ptr, OldNode, Size);\n return Ptr;\n }\n\n // MMX State can be either MMX (for 64bit) or x87 FPU (for 80bit)\n enum { MMXState_MMX, MMXState_X87 } MMXState = MMXState_MMX;\n\n // Overriden by dispatcher, stubbed for IR tests\n virtual void RecordX87Use() {}\n virtual void ChgStateX87_MMX() {}\n virtual void ChgStateMMX_X87() {}\n virtual void SaveNZCV(IROps Op) {}\n\n Ref CurrentWriteCursor = nullptr;\n\n // These could be combined with a little bit of work to be more efficient with memory usage. Isn't a big deal\n DualIntrusiveAllocatorThreadPool DualListData;\n\n Ref InvalidNode {};\n Ref CurrentCodeBlock {};\n fextl::vector CodeBlocks;\n uint64_t Entry {};\n bool SupportsTSOImm9 {};\n\nprivate:\n void ResetWorkingList();\n};\n\n} // namespace FEXCore::IR\n" }, { "path": "FEXCore/Source/Interface/IR/IntrusiveIRList.h", "content": "// SPDX-License-Identifier: MIT\n#pragma once\n\n#include \"Interface/IR/IR.h\"\n\n#include \n#include \n#include \n\n#include \n#include \n#include \n#include \n\nnamespace FEXCore::IR {\n/**\n * @brief This is purely an intrusive allocator\n * This doesn't support any form of ordering at all\n * Just provides a chunk of memory for allocating IR nodes from\n *\n * Can potentially support reallocation if we are smart and make sure to invalidate anything holding a true pointer\n */\nclass DualIntrusiveAllocator {\npublic:\n [[nodiscard]]\n bool DataCheckSize(size_t Size) const {\n size_t NewOffset = DataCurrentOffset + Size;\n return NewOffset <= MemorySize;\n }\n\n [[nodiscard]]\n bool ListCheckSize(size_t Size) const {\n size_t NewOffset = ListCurrentOffset + Size;\n return NewOffset <= MemorySize;\n }\n\n [[nodiscard]]\n void* DataAllocate(size_t Size) {\n LOGMAN_THROW_A_FMT(DataCheckSize(Size), \"Ran out of space in DualIntrusiveAllocator during allocation\");\n size_t NewOffset = DataCurrentOffset + Size;\n uintptr_t NewPointer = Data + DataCurrentOffset;\n DataCurrentOffset = NewOffset;\n return reinterpret_cast(NewPointer);\n }\n\n [[nodiscard]]\n void* ListAllocate(size_t Size) {\n LOGMAN_THROW_A_FMT(ListCheckSize(Size), \"Ran out of space in DualIntrusiveAllocator during allocation\");\n size_t NewOffset = ListCurrentOffset + Size;\n uintptr_t NewPointer = List + ListCurrentOffset;\n ListCurrentOffset = NewOffset;\n return reinterpret_cast(NewPointer);\n }\n\n [[nodiscard]]\n size_t DataSize() const {\n return DataCurrentOffset;\n }\n [[nodiscard]]\n size_t DataBackingSize() const {\n return MemorySize;\n }\n\n [[nodiscard]]\n size_t ListSize() const {\n return ListCurrentOffset;\n }\n [[nodiscard]]\n size_t ListBackingSize() const {\n return MemorySize;\n }\n\n [[nodiscard]]\n uintptr_t DataBegin() const {\n return Data;\n }\n [[nodiscard]]\n uintptr_t ListBegin() const {\n return List;\n }\n\n void Reset() {\n DataCurrentOffset = 0;\n ListCurrentOffset = 0;\n }\n\n void CopyData(const DualIntrusiveAllocator& rhs) {\n DataCurrentOffset = rhs.DataCurrentOffset;\n ListCurrentOffset = rhs.ListCurrentOffset;\n memcpy(reinterpret_cast(Data), reinterpret_cast(rhs.Data), DataCurrentOffset);\n memcpy(reinterpret_cast(List), reinterpret_cast(rhs.List), ListCurrentOffset);\n }\n\nprotected:\n DualIntrusiveAllocator(size_t Size)\n : MemorySize {Size} {}\n\n uintptr_t Data {};\n uintptr_t List {};\n size_t DataCurrentOffset {0};\n size_t ListCurrentOffset {0};\n size_t MemorySize {};\n};\n\nclass DualIntrusiveAllocatorMalloc final : public DualIntrusiveAllocator {\npublic:\n DualIntrusiveAllocatorMalloc(size_t Size)\n : DualIntrusiveAllocator {Size} {\n Data = reinterpret_cast(FEXCore::Allocator::malloc(Size * 2));\n List = reinterpret_cast(Data + Size);\n }\n\n ~DualIntrusiveAllocatorMalloc() {\n FEXCore::Allocator::free(reinterpret_cast(Data));\n }\n};\n\nclass DualIntrusiveAllocatorThreadPool final : public DualIntrusiveAllocator {\npublic:\n DualIntrusiveAllocatorThreadPool(FEXCore::Utils::IntrusivePooledAllocator& ThreadAllocator, size_t Size)\n : DualIntrusiveAllocator {Size}\n , PoolObject {ThreadAllocator, Size * 2} {}\n void ReownOrClaimBuffer() {\n Data = PoolObject.ReownOrClaimBuffer();\n List = Data + MemorySize;\n }\n\n void DelayedDisownBuffer() {\n PoolObject.DelayedDisownBuffer();\n }\n\nprivate:\n Utils::PoolBufferWithTimedRetirement PoolObject;\n};\n\nclass IRListView final {\npublic:\n IRListView() = delete;\n\n IRListView(DualIntrusiveAllocator* Data)\n : IRListView(reinterpret_cast(Data->DataBegin()), reinterpret_cast(Data->ListBegin()), Data->DataSize(), Data->ListSize()) {}\n\n IRListView(IRListView* Old)\n : IRListView(Old->IRDataInternal, Old->ListDataInternal, Old->DataSize, Old->ListSize) {}\n\n IRListView(void* IRData_, void* ListData_, size_t DataSize_, size_t ListSize_)\n : IRDataInternal(IRData_)\n , ListDataInternal(ListData_)\n , DataSize(DataSize_)\n , ListSize(ListSize_) {}\n\n [[nodiscard]]\n size_t GetInlineSize() const {\n static_assert(sizeof(*this) == 32);\n return sizeof(*this) + DataSize + ListSize;\n }\n\n [[nodiscard]]\n size_t GetDataSize() const {\n return DataSize;\n }\n [[nodiscard]]\n size_t GetListSize() const {\n return ListSize;\n }\n [[nodiscard]]\n size_t GetSSACount() const {\n return ListSize / sizeof(OrderedNode);\n }\n\n [[nodiscard]]\n NodeID GetID(const Ref Node) const {\n return Node->Wrapped(GetListData()).ID();\n }\n\n [[nodiscard]]\n Ref GetHeaderNode() const {\n OrderedNodeWrapper Wrapped;\n Wrapped.NodeOffset = sizeof(OrderedNode);\n return Wrapped.GetNode(GetListData());\n }\n\n [[nodiscard]]\n IROp_IRHeader* GetHeader() const {\n return GetOp(GetHeaderNode());\n }\n\n [[nodiscard]]\n unsigned PostRA() const {\n return GetHeader()->PostRA;\n }\n\n [[nodiscard]]\n unsigned SpillSlots() const {\n return GetHeader()->SpillSlots;\n }\n\n template\n [[nodiscard]]\n T* GetOp(Ref Node) const {\n auto OpHeader = Node->Op(GetData());\n auto Op = OpHeader->template CW();\n\n // If we are casting to something narrower than just the header, check the opcode.\n if constexpr (!std::is_same::value) {\n LOGMAN_THROW_A_FMT(Op->OPCODE == Op->Header.Op, \"Expected Node to be '{}'. Found '{}' instead\", GetName(Op->OPCODE),\n GetName(Op->Header.Op));\n }\n\n return Op;\n }\n\n template\n [[nodiscard]]\n T* GetOp(OrderedNodeWrapper Wrapper) const {\n auto Node = Wrapper.GetNode(GetListData());\n return GetOp(Node);\n }\n\n [[nodiscard]]\n Ref GetNode(OrderedNodeWrapper Wrapper) const {\n return Wrapper.GetNode(GetListData());\n }\n\n ///< Gets an OrderedNode from the IRListView as an OrderedNodeWrapper.\n [[nodiscard]]\n OrderedNodeWrapper WrapNode(Ref Node) const {\n return Node->Wrapped(GetListData());\n }\n\nprivate:\n struct BlockRange {\n using iterator = NodeIterator;\n const IRListView* View;\n\n BlockRange(const IRListView* parent)\n : View(parent) {};\n\n [[nodiscard]]\n iterator begin() const noexcept {\n auto Header = View->GetHeader();\n return iterator(View->GetListData(), View->GetData(), Header->Blocks);\n }\n\n [[nodiscard]]\n iterator end() const noexcept {\n return iterator(View->GetListData(), View->GetData());\n }\n };\n\n struct CodeRange {\n using iterator = NodeIterator;\n const IRListView* View;\n const OrderedNodeWrapper BlockWrapper;\n\n CodeRange(const IRListView* parent, OrderedNodeWrapper block)\n : View(parent)\n , BlockWrapper(block) {};\n\n [[nodiscard]]\n iterator begin() const noexcept {\n auto Block = View->GetOp(BlockWrapper);\n return iterator(View->GetListData(), View->GetData(), Block->Begin);\n }\n\n [[nodiscard]]\n iterator end() const noexcept {\n return iterator(View->GetListData(), View->GetData());\n }\n };\n\n struct AllCodeRange {\n using iterator = AllNodesIterator; // Diffrent Iterator\n const IRListView* View;\n\n AllCodeRange(const IRListView* parent)\n : View(parent) {};\n\n [[nodiscard]]\n iterator begin() const noexcept {\n auto Header = View->GetHeader();\n return iterator(View->GetListData(), View->GetData(), Header->Blocks);\n }\n\n [[nodiscard]]\n iterator end() const noexcept {\n return iterator(View->GetListData(), View->GetData());\n }\n };\n\npublic:\n using iterator = NodeIterator;\n\n [[nodiscard]]\n BlockRange GetBlocks() const {\n return BlockRange(this);\n }\n\n [[nodiscard]]\n CodeRange GetCode(const Ref block) const {\n return CodeRange(this, block->Wrapped(GetListData()));\n }\n\n [[nodiscard]]\n AllCodeRange GetAllCode() const {\n return AllCodeRange(this);\n }\n\n [[nodiscard]]\n iterator begin() const noexcept {\n OrderedNodeWrapper Wrapped;\n Wrapped.NodeOffset = sizeof(OrderedNode);\n return iterator(GetListData(), GetData(), Wrapped);\n }\n\n /**\n * @brief This is not an iterator that you can reverse iterator through!\n *\n * @return Our iterator sentinel to ensure ending correctly\n */\n [[nodiscard]]\n iterator end() const noexcept {\n OrderedNodeWrapper Wrapped;\n Wrapped.NodeOffset = 0;\n return iterator(GetListData(), GetData(), Wrapped);\n }\n\n /**\n * @brief Convert a OrderedNodeWrapper to an interator that we can iterate over\n * @return Iterator for this op\n */\n [[nodiscard]]\n iterator at(OrderedNodeWrapper Wrapped) const noexcept {\n return iterator(GetListData(), GetData(), Wrapped);\n }\n\n [[nodiscard]]\n iterator at(NodeID ID) const noexcept {\n OrderedNodeWrapper Wrapped;\n Wrapped.NodeOffset = ID.Value * sizeof(OrderedNode);\n return iterator(GetListData(), GetData(), Wrapped);\n }\n\n [[nodiscard]]\n iterator at(const Ref Node) const noexcept {\n const auto ListData = GetListData();\n auto Wrapped = Node->Wrapped(ListData);\n return iterator(ListData, GetData(), Wrapped);\n }\n\n [[nodiscard]]\n uintptr_t GetData() const {\n return reinterpret_cast(IRDataInternal ? IRDataInternal : InlineData);\n }\n\n [[nodiscard]]\n uintptr_t GetListData() const {\n return reinterpret_cast(ListDataInternal ? ListDataInternal : &InlineData[DataSize]);\n }\n\nprivate:\n void* IRDataInternal;\n void* ListDataInternal;\n size_t DataSize;\n size_t ListSize;\n uint8_t InlineData[0];\n};\n\n} // namespace FEXCore::IR\n" }, { "path": "FEXCore/Source/Interface/IR/PassManager.cpp", "content": "// SPDX-License-Identifier: MIT\n/*\n$info$\nmeta: ir|opts ~ IR to IR Optimization\ntags: ir|opts\ndesc: Defines which passes are run, and runs them\n$end_info$\n*/\n\n#include \"Interface/Context/Context.h\"\n#include \"Interface/IR/PassManager.h\"\n#include \"Interface/IR/Passes.h\"\n#include \"Interface/IR/Passes/RegisterAllocationPass.h\"\n\n#include \n#include \n\nnamespace FEXCore::IR {\nclass IREmitter;\n\nvoid PassManager::Finalize() {\n if (!PassManagerDumpIR()) {\n // Not configured to dump any IR, just return.\n return;\n }\n\n auto it = Passes.begin();\n // Walk the passes and add them where asked.\n if (PassManagerDumpIR() & FEXCore::Config::PassManagerDumpIR::BEFOREOPT) {\n // Insert at the start.\n it = InsertAt(it, Debug::CreateIRDumper());\n ++it; // Skip what we inserted.\n }\n\n if ((PassManagerDumpIR() & FEXCore::Config::PassManagerDumpIR::BEFOREPASS) ||\n (PassManagerDumpIR() & FEXCore::Config::PassManagerDumpIR::AFTERPASS)) {\n\n bool SkipFirstBefore = PassManagerDumpIR() & FEXCore::Config::PassManagerDumpIR::BEFOREOPT;\n for (; it != Passes.end();) {\n if (PassManagerDumpIR() & FEXCore::Config::PassManagerDumpIR::BEFOREPASS) {\n if (SkipFirstBefore) {\n // If we need to skip the first one, then continue.\n SkipFirstBefore = false;\n ++it;\n continue;\n }\n\n // Insert before\n it = InsertAt(it, Debug::CreateIRDumper());\n ++it; // Skip what we inserted.\n }\n\n ++it; // Skip current pass.\n if (PassManagerDumpIR() & FEXCore::Config::PassManagerDumpIR::AFTERPASS) {\n // Insert after\n it = InsertAt(it, Debug::CreateIRDumper());\n ++it; // Skip what we inserted.\n }\n }\n }\n if (PassManagerDumpIR() & FEXCore::Config::PassManagerDumpIR::AFTEROPT) {\n if (!(PassManagerDumpIR() & FEXCore::Config::PassManagerDumpIR::AFTERPASS)) {\n // Insert final IRDumper.\n InsertAt(Passes.end(), Debug::CreateIRDumper());\n }\n }\n}\n\nvoid PassManager::AddDefaultPasses(FEXCore::Context::ContextImpl* ctx) {\n FEX_CONFIG_OPT(DisablePasses, O0);\n\n if (!DisablePasses()) {\n InsertPass(CreateX87StackOptimizationPass(ctx->HostFeatures, ctx->Config.Is64BitMode ? IR::OpSize::i64Bit : IR::OpSize::i32Bit));\n InsertPass(CreateDeadFlagCalculationEliminination());\n }\n}\n\nvoid PassManager::AddDefaultValidationPasses() {\n#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED\n InsertValidationPass(Validation::CreateIRValidation(), \"IRValidation\");\n#endif\n}\n\nvoid PassManager::InsertRegisterAllocationPass(FEXCore::Context::ContextImpl* ctx) {\n InsertPass(IR::CreateRegisterAllocationPass(&ctx->CPUID), \"RA\");\n}\n\nvoid PassManager::Run(IREmitter* IREmit) {\n FEXCORE_PROFILE_SCOPED(\"PassManager::Run\");\n\n for (const auto& Pass : Passes) {\n Pass->Run(IREmit);\n }\n\n#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED\n for (const auto& Pass : ValidationPasses) {\n Pass->Run(IREmit);\n }\n#endif\n}\n} // namespace FEXCore::IR\n" }, { "path": "FEXCore/Source/Interface/IR/PassManager.h", "content": "// SPDX-License-Identifier: MIT\n/*\n$info$\ntags: ir|opts\n$end_info$\n*/\n\n#pragma once\n\n#include \n#include \n#include \n#include \n#include \n#include \n\n#include \n#include \n\nnamespace FEXCore::Context {\nclass ContextImpl;\n}\n\nnamespace FEXCore::HLE {\nclass SyscallHandler;\n}\n\nnamespace FEXCore::IR {\nclass PassManager;\nclass IREmitter;\n\nclass Pass {\npublic:\n virtual ~Pass() = default;\n virtual void Run(IREmitter* IREmit) = 0;\n\n void RegisterPassManager(PassManager* _Manager) {\n Manager = _Manager;\n }\n\nprotected:\n PassManager* Manager {};\n};\n\nclass PassManager final {\npublic:\n void AddDefaultPasses(FEXCore::Context::ContextImpl* ctx);\n void AddDefaultValidationPasses();\n Pass* InsertPass(fextl::unique_ptr Pass, fextl::string Name = \"\") {\n auto PassPtr = InsertAt(Passes.end(), std::move(Pass))->get();\n\n if (!Name.empty()) {\n NameToPassMaping[Name] = PassPtr;\n }\n return PassPtr;\n }\n\n void InsertRegisterAllocationPass(FEXCore::Context::ContextImpl* ctx);\n\n void Run(IREmitter* IREmit);\n\n bool HasPass(fextl::string Name) const {\n return NameToPassMaping.contains(Name);\n }\n\n template\n T* GetPass(fextl::string Name) {\n return dynamic_cast(NameToPassMaping[Name]);\n }\n\n Pass* GetPass(fextl::string Name) {\n return NameToPassMaping[Name];\n }\n\n void RegisterSyscallHandler(FEXCore::HLE::SyscallHandler* Handler) {\n SyscallHandler = Handler;\n }\n\n void Finalize();\n\nprotected:\n FEXCore::HLE::SyscallHandler* SyscallHandler {};\n\nprivate:\n using PassArrayType = fextl::vector>;\n PassArrayType::iterator InsertAt(PassArrayType::iterator pos, fextl::unique_ptr Pass) {\n Pass->RegisterPassManager(this);\n return Passes.insert(pos, std::move(Pass));\n }\n PassArrayType Passes;\n fextl::unordered_map NameToPassMaping;\n\n#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED\n fextl::vector> ValidationPasses;\n void InsertValidationPass(fextl::unique_ptr Pass, fextl::string Name = \"\") {\n Pass->RegisterPassManager(this);\n auto PassPtr = ValidationPasses.emplace_back(std::move(Pass)).get();\n\n if (!Name.empty()) {\n NameToPassMaping[Name] = PassPtr;\n }\n }\n#endif\n\n FEX_CONFIG_OPT(Is64BitMode, IS64BIT_MODE);\n FEX_CONFIG_OPT(PassManagerDumpIR, PASSMANAGERDUMPIR);\n};\n} // namespace FEXCore::IR\n" }, { "path": "FEXCore/Source/Interface/IR/Passes/IRDumperPass.cpp", "content": "// SPDX-License-Identifier: MIT\n/*\n$info$\ntags: ir|debug\ndesc: Prints IR\n$end_info$\n*/\n\n#include \"Interface/IR/IR.h\"\n#include \"Interface/IR/IREmitter.h\"\n#include \"Interface/IR/PassManager.h\"\n#include \"Interface/IR/Passes/RegisterAllocationPass.h\"\n#include \"Interface/Core/OpcodeDispatcher.h\"\n\n#include \n\nnamespace FEXCore::IR::Debug {\nclass IRDumper final : public FEXCore::IR::Pass {\npublic:\n IRDumper();\n void Run(IREmitter* IREmit) override;\n\nprivate:\n FEX_CONFIG_OPT(DumpIR, DUMPIR);\n bool DumpToFile {};\n bool DumpToLog {};\n};\n\nIRDumper::IRDumper() {\n const auto& DumpIRStr = DumpIR();\n if (DumpIRStr == \"stderr\" || DumpIRStr == \"stdout\" || DumpIRStr == \"no\") {\n // Intentionally do nothing\n } else if (DumpIRStr == \"server\") {\n DumpToLog = true;\n } else {\n DumpToFile = true;\n }\n}\n\nvoid IRDumper::Run(IREmitter* IREmit) {\n FEXCore::File::File FD {};\n if (DumpIR() == \"stderr\") {\n FD = FEXCore::File::File::GetStdERR();\n } else if (DumpIR() == \"stdout\") {\n FD = FEXCore::File::File::GetStdOUT();\n }\n\n auto IR = IREmit->ViewIR();\n auto HeaderOp = IR.GetHeader();\n LOGMAN_THROW_A_FMT(HeaderOp->Header.Op == OP_IRHEADER, \"First op wasn't IRHeader\");\n\n // DumpIRStr might be no if not dumping but ShouldDump is set in OpDisp\n if (DumpToFile) {\n const auto fileName = fextl::fmt::format(\"{}/{:x}{}\", DumpIR(), +HeaderOp->OriginalRIP, IR.PostRA() ? \"-post.ir\" : \"-pre.ir\");\n FD = FEXCore::File::File(fileName.c_str(),\n FEXCore::File::FileModes::WRITE | FEXCore::File::FileModes::CREATE | FEXCore::File::FileModes::TRUNCATE);\n }\n\n if (FD.IsValid() || DumpToLog) {\n fextl::stringstream out;\n FEXCore::IR::Dump(&out, &IR);\n if (FD.IsValid()) {\n fextl::fmt::print(FD, \"IR-{} 0x{:x}:\\n{}\\n@@@@@\\n\", IR.PostRA() ? \"post\" : \"pre\", +HeaderOp->OriginalRIP, out.str());\n } else {\n LogMan::Msg::IFmt(\"IR-{} 0x{:x}:\\n{}\\n@@@@@\\n\", IR.PostRA() ? \"post\" : \"pre\", +HeaderOp->OriginalRIP, out.str());\n }\n }\n}\n\nfextl::unique_ptr CreateIRDumper() {\n return fextl::make_unique();\n}\n} // namespace FEXCore::IR::Debug\n" }, { "path": "FEXCore/Source/Interface/IR/Passes/IRValidation.cpp", "content": "// SPDX-License-Identifier: MIT\n/*\n$info$\ntags: ir|opts\ndesc: Sanity checking pass\n$end_info$\n*/\n\n#include \"Interface/IR/IR.h\"\n#include \"Interface/IR/IREmitter.h\"\n#include \"Interface/IR/PassManager.h\"\n#include \"Interface/IR/RegisterAllocationData.h\"\n#include \"Interface/IR/Passes/IRValidation.h\"\n#include \"Interface/IR/Passes/RegisterAllocationPass.h\"\n\n#include \n#include \n#include \n#include \n#include \n\n#include \n#include \n#include \n#include \n#include \n\nnamespace FEXCore::IR::Validation {\n\n\nIRValidation::~IRValidation() {\n NodeIsLive.Free();\n}\n\nvoid IRValidation::Run(IREmitter* IREmit) {\n FEXCORE_PROFILE_SCOPED(\"PassManager::IRValidation\");\n\n bool HadError = false;\n bool HadWarning = false;\n\n fextl::ostringstream Errors;\n fextl::ostringstream Warnings;\n\n auto CurrentIR = IREmit->ViewIR();\n\n OffsetToBlockMap.clear();\n EntryBlock = nullptr;\n\n uint32_t Count = CurrentIR.GetSSACount();\n if (Count > MaxNodes) {\n NodeIsLive.Realloc(Count);\n }\n\n fextl::vector Uses(Count, 0);\n\n#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED\n auto HeaderOp = CurrentIR.GetHeader();\n LOGMAN_THROW_A_FMT(HeaderOp->Header.Op == OP_IRHEADER, \"First op wasn't IRHeader\");\n#endif\n\n for (auto [BlockNode, BlockHeader] : CurrentIR.GetBlocks()) {\n auto BlockIROp = BlockHeader->CW();\n LOGMAN_THROW_A_FMT(BlockIROp->Header.Op == OP_CODEBLOCK, \"IR type failed to be a code block\");\n\n if (!EntryBlock) {\n EntryBlock = BlockNode;\n }\n\n const auto BlockID = CurrentIR.GetID(BlockNode);\n BlockInfo* CurrentBlock = &OffsetToBlockMap.try_emplace(BlockID).first->second;\n\n // We only allow defs local to a single block, so clear live set per block\n NodeIsLive.MemClear(Count);\n\n for (auto [CodeNode, IROp] : CurrentIR.GetCode(BlockNode)) {\n const auto ID = CurrentIR.GetID(CodeNode);\n const auto OpSize = IROp->Size;\n\n if (GetHasDest(IROp->Op)) {\n HadError |= OpSize == IR::OpSize::iInvalid;\n // Does the op have a destination of size 0?\n if (OpSize == IR::OpSize::iInvalid) {\n Errors << \"%\" << ID << \": Had destination but with no size\" << std::endl;\n }\n\n // Does the node have zero uses? Should have been DCE'd\n if (CodeNode->GetUses() == 0) {\n HadWarning |= true;\n Warnings << \"%\" << ID << \": Destination created but had no uses\" << std::endl;\n }\n\n if (CurrentIR.PostRA()) {\n // After RA, the destination needs to be assigned a register and class\n auto PhyReg = PhysicalRegister(CodeNode);\n\n const auto ExpectedClass = IR::GetRegClass(IROp->Op);\n const auto AssignedClass = PhyReg.AsRegClass();\n\n // If no register class was assigned\n if (AssignedClass == IR::RegClass::Invalid) {\n HadError |= true;\n Errors << \"%\" << ID << \": Had destination but with no register class assigned\" << std::endl;\n }\n\n // If no physical register was assigned\n if (PhyReg.IsInvalid()) {\n HadError |= true;\n Errors << \"%\" << ID << \": Had destination but with no register assigned\" << std::endl;\n }\n\n // Assigned class wasn't the expected class and it is a non-complex op\n if (AssignedClass != ExpectedClass && ExpectedClass != IR::RegClass::Complex) {\n HadWarning |= true;\n Warnings << \"%\" << ID << \": Destination had register class \" << uint32_t(AssignedClass) << \" When register class \"\n << uint32_t(ExpectedClass) << \" Was expected\" << std::endl;\n }\n }\n }\n\n uint8_t NumArgs = IR::GetRAArgs(IROp->Op);\n\n for (uint32_t i = 0; i < NumArgs; ++i) {\n OrderedNodeWrapper Arg = IROp->Args[i];\n const auto ArgID = Arg.ID();\n if (Arg.IsImmediate()) {\n continue;\n }\n\n IROps Op = CurrentIR.GetOp(Arg)->Op;\n\n if (ArgID.IsValid()) {\n Uses[ArgID.Value]++;\n }\n\n // We do not validate the location of inline constants because it's\n // irrelevant, they're ignored by RA and always inlined to where they\n // need to be. This lets us pool inline constants globally.\n bool Ignore = (Op == OP_IRHEADER || Op == OP_INLINECONSTANT);\n\n if (!Ignore && ArgID.IsValid() && !NodeIsLive.Get(ArgID.Value)) {\n HadError |= true;\n Errors << \"%\" << ID << \": Arg[\" << i << \"] references invalid %\" << ArgID << std::endl;\n }\n }\n\n NodeIsLive.Set(ID.Value);\n\n switch (IROp->Op) {\n case IR::OP_EXITFUNCTION: {\n CurrentBlock->HasExit = true;\n break;\n }\n case IR::OP_CONDJUMP: {\n auto Op = IROp->C();\n\n OrderedNode* TrueTargetNode = CurrentIR.GetNode(Op->TrueBlock);\n OrderedNode* FalseTargetNode = CurrentIR.GetNode(Op->FalseBlock);\n\n CurrentBlock->Successors.emplace_back(TrueTargetNode);\n CurrentBlock->Successors.emplace_back(FalseTargetNode);\n\n const FEXCore::IR::IROp_Header* TrueTargetOp = CurrentIR.GetOp(TrueTargetNode);\n const FEXCore::IR::IROp_Header* FalseTargetOp = CurrentIR.GetOp(FalseTargetNode);\n\n if (TrueTargetOp->Op != OP_CODEBLOCK) {\n HadError |= true;\n Errors << \"CondJump %\" << ID << \": True Target Jumps to Op that isn't the begining of a block\" << std::endl;\n } else {\n auto Block = OffsetToBlockMap.try_emplace(Op->TrueBlock.ID()).first;\n Block->second.Predecessors.emplace_back(BlockNode);\n }\n\n if (FalseTargetOp->Op != OP_CODEBLOCK) {\n HadError |= true;\n Errors << \"CondJump %\" << ID << \": False Target Jumps to Op that isn't the begining of a block\" << std::endl;\n } else {\n auto Block = OffsetToBlockMap.try_emplace(Op->FalseBlock.ID()).first;\n Block->second.Predecessors.emplace_back(BlockNode);\n }\n\n break;\n }\n case IR::OP_JUMP: {\n auto Op = IROp->C();\n OrderedNode* TargetNode = CurrentIR.GetNode(Op->Header.Args[0]);\n CurrentBlock->Successors.emplace_back(TargetNode);\n\n const FEXCore::IR::IROp_Header* TargetOp = CurrentIR.GetOp(TargetNode);\n if (TargetOp->Op != OP_CODEBLOCK) {\n HadError |= true;\n Errors << \"Jump %\" << ID << \": Jump to Op that isn't the begining of a block\" << std::endl;\n } else {\n auto Block = OffsetToBlockMap.try_emplace(Op->Header.Args[0].ID()).first;\n Block->second.Predecessors.emplace_back(BlockNode);\n }\n break;\n }\n default:\n // LOGMAN_MSG_A_FMT(\"Unknown IR Op: {}({})\", IROp->Op, FEXCore::IR::GetName(IROp->Op));\n break;\n }\n }\n\n // Blocks can only have zero (Exit), 1 (Unconditional branch) or 2 (Conditional) successors\n size_t NumSuccessors = CurrentBlock->Successors.size();\n if (NumSuccessors > 2) {\n HadError |= true;\n Errors << \"%\" << BlockID << \" Has \" << NumSuccessors << \" successors which is too many\" << std::endl;\n }\n\n {\n auto GetOp = [](auto Code) {\n auto [CodeNode, IROp] = Code();\n return IROp->Op;\n };\n\n auto CodeCurrent = CurrentIR.at(BlockIROp->Last);\n\n // Last instruction in the block must be EndBlock\n {\n auto Op = GetOp(CodeCurrent);\n if (Op != IR::OP_ENDBLOCK) {\n HadError |= true;\n Errors << \"%\" << BlockID << \" Failed to end block with EndBlock\" << std::endl;\n }\n }\n\n --CodeCurrent;\n\n // Blocks need to have an instruction that leaves the block in some way before the EndBlock instruction\n {\n auto Op = GetOp(CodeCurrent);\n if (!IsBlockExit(Op)) {\n HadError |= true;\n Errors << \"%\" << BlockID << \" Didn't have a block exit IR op as its last instruction\" << std::endl;\n }\n }\n }\n }\n\n // Use counts are only relevant pre-RA.\n if (!CurrentIR.PostRA()) {\n for (uint32_t i = 0; i < CurrentIR.GetSSACount(); i++) {\n auto [Node, IROp] = CurrentIR.at(IR::NodeID {i})();\n if (Node->NumUses != Uses[i] && IROp->Op != OP_CODEBLOCK && IROp->Op != OP_IRHEADER) {\n HadError |= true;\n Errors << \"%\" << i << \" Has \" << Uses[i] << \" Uses, but reports \" << Node->NumUses << std::endl;\n }\n }\n }\n\n HadWarning = false;\n if (HadError || HadWarning) {\n fextl::stringstream Out;\n FEXCore::IR::Dump(&Out, &CurrentIR);\n\n if (HadError) {\n Out << \"Errors:\" << std::endl << Errors.str() << std::endl;\n }\n\n if (HadWarning) {\n Out << \"Warnings:\" << std::endl << Warnings.str() << std::endl;\n }\n\n LogMan::Msg::EFmt(\"{}\", Out.str());\n\n LOGMAN_MSG_A_FMT(\"Encountered IR validation Error\");\n\n Errors.clear();\n Warnings.clear();\n }\n}\n\nfextl::unique_ptr CreateIRValidation() {\n return fextl::make_unique();\n}\n} // namespace FEXCore::IR::Validation\n" }, { "path": "FEXCore/Source/Interface/IR/Passes/IRValidation.h", "content": "// SPDX-License-Identifier: MIT\n#pragma once\n\n#include \"Common/BitSet.h\"\n#include \n#include \n#include \n\nnamespace FEXCore::IR::Validation {\n\nstruct BlockInfo {\n bool HasExit;\n const OrderedNode* BlockNode;\n\n fextl::vector Predecessors;\n fextl::vector Successors;\n};\n\nclass IRValidation final : public FEXCore::IR::Pass {\npublic:\n ~IRValidation();\n void Run(IREmitter* IREmit) override;\n\nprivate:\n\n BitSet NodeIsLive {};\n OrderedNode* EntryBlock {};\n fextl::unordered_map OffsetToBlockMap;\n size_t MaxNodes {};\n};\n} // namespace FEXCore::IR::Validation\n" }, { "path": "FEXCore/Source/Interface/IR/Passes/RedundantFlagCalculationElimination.cpp", "content": "// SPDX-License-Identifier: MIT\n/*\n$info$\ntags: ir|opts\n$end_info$\n*/\n\n#include \"Interface/IR/IR.h\"\n#include \"Interface/IR/IREmitter.h\"\n#include \"Interface/IR/PassManager.h\"\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n// Flag bit flags\n#define FLAG_V (1U << 0)\n#define FLAG_C (1U << 1)\n#define FLAG_Z (1U << 2)\n#define FLAG_N (1U << 3)\n#define FLAG_P (1U << 4)\n#define FLAG_A (1U << 5)\n\n#define FLAG_ZCV (FLAG_Z | FLAG_C | FLAG_V)\n#define FLAG_NZCV (FLAG_N | FLAG_ZCV)\n#define FLAG_ALL (FLAG_NZCV | FLAG_A | FLAG_P)\n\nnamespace FEXCore::IR {\n\nstruct FlagInfoUnpacked {\n // Set of flags read by the instruction.\n unsigned Read;\n\n // Set of flags written by the instruction. Happens AFTER the reads.\n unsigned Write;\n\n // If true, the instruction can be be eliminated if its flag writes can all be\n // eliminated.\n bool CanEliminate;\n\n // If set, the opcode can be replaced with Replacement if its flag writes can\n // all be eliminated, or ReplacementNoWrite if its register write can be\n // eliminated.\n IROps Replacement;\n IROps ReplacementNoWrite;\n\n // Needs speical handling\n bool Special;\n};\n\nstruct FlagInfo {\n uint64_t Raw;\n\n static constexpr struct FlagInfo Pack(struct FlagInfoUnpacked F) {\n uint64_t R = F.Read | (F.Write << 8) | (F.CanEliminate << 16) | (((uint64_t)F.Replacement) << 32) |\n ((uint64_t)F.ReplacementNoWrite << 48) | (F.Special ? (1ull << 63) : 0);\n return {.Raw = R};\n }\n\n bool Trivial() const {\n return Raw == 0;\n }\n\n unsigned Read() const {\n return Bits(0, 8);\n }\n\n unsigned Write() const {\n return Bits(8, 8);\n }\n\n bool CanEliminate() const {\n return Bits(16, 1);\n }\n\n bool Special() const {\n return Bits(63, 1);\n }\n\n IROps Replacement() const {\n return (IROps)Bits(32, 16);\n }\n\n IROps ReplacementNoWrite() const {\n return (IROps)Bits(48, 16);\n }\n\nprivate:\n unsigned Bits(unsigned Start, unsigned Count) const {\n return (Raw >> Start) & ((1u << Count) - 1);\n }\n};\n\nstruct BlockInfo {\n fextl::vector Predecessors;\n Ref Node;\n uint8_t Flags;\n bool InWorklist;\n};\n\nstruct ControlFlowGraph {\n fextl::vector BlockMap;\n IRListView& IR;\n\n void Init(fextl::deque& Worklist, uint32_t BlockCount) {\n BlockMap.resize(BlockCount);\n\n for (unsigned ID = 0; ID < BlockCount; ++ID) {\n // Add the block with conservative flags and already in the worklist.\n auto Info = BlockInfo {{}, nullptr, FLAG_ALL, true};\n\n // Add some initial capacity\n Info.Predecessors.reserve(2);\n\n BlockMap[ID] = std::move(Info);\n Worklist.push_back(ID);\n }\n }\n\n BlockInfo* Get(uint32_t Block) {\n return &BlockMap[Block];\n }\n\n BlockInfo* Get(IROp_CodeBlock* Block) {\n return &BlockMap[Block->ID];\n }\n\n BlockInfo* Get(OrderedNodeWrapper Block) {\n return Get(IR.GetOp(Block));\n }\n\n void RecordEdge(uint32_t From, OrderedNodeWrapper To) {\n auto Info = Get(To);\n Info->Predecessors.push_back(From);\n }\n\n void AddWorklist(fextl::deque& Worklist, uint32_t Block) {\n auto Info = Get(Block);\n if (!Info->InWorklist) {\n Info->InWorklist = true;\n Worklist.push_front(Block);\n }\n }\n};\n\nclass DeadFlagCalculationEliminination final : public FEXCore::IR::Pass {\npublic:\n void Run(IREmitter* IREmit) override;\n\nprivate:\n FlagInfo Classify(IROp_Header* Node);\n unsigned FlagsForCondClassType(CondClass Cond);\n bool EliminateDeadCode(IREmitter* IREmit, Ref CodeNode, IROp_Header* IROp);\n void FoldBranch(IREmitter* IREmit, IRListView& CurrentIR, IROp_CondJump* Op, Ref CodeNode);\n CondClass X86ToArmFloatCond(CondClass X86);\n bool ProcessBlock(IREmitter* IREmit, IRListView& CurrentIR, Ref Block, ControlFlowGraph& CFG);\n void OptimizeParity(IREmitter* IREmit, IRListView& CurrentIR, ControlFlowGraph& CFG);\n};\n\nunsigned DeadFlagCalculationEliminination::FlagsForCondClassType(CondClass Cond) {\n switch (Cond) {\n case CondClass::AL: return 0;\n\n case CondClass::MI:\n case CondClass::PL: return FLAG_N;\n\n case CondClass::EQ:\n case CondClass::NEQ: return FLAG_Z;\n\n case CondClass::UGE:\n case CondClass::ULT: return FLAG_C;\n\n case CondClass::VS:\n case CondClass::VC:\n case CondClass::FU:\n case CondClass::FNU: return FLAG_V;\n\n case CondClass::UGT:\n case CondClass::ULE: return FLAG_Z | FLAG_C;\n\n case CondClass::SGE:\n case CondClass::SLT:\n case CondClass::FLU:\n case CondClass::FGE: return FLAG_N | FLAG_V;\n\n case CondClass::SGT:\n case CondClass::SLE:\n case CondClass::FLEU:\n case CondClass::FGT: return FLAG_N | FLAG_Z | FLAG_V;\n\n default: LOGMAN_THROW_A_FMT(false, \"unknown cond class type\"); return FLAG_NZCV;\n }\n}\n\nconstexpr FlagInfo ClassifyConst(IROps Op) {\n switch (Op) {\n case OP_ANDWITHFLAGS:\n return FlagInfo::Pack({\n .Write = FLAG_NZCV,\n .Replacement = OP_AND,\n .ReplacementNoWrite = OP_TESTNZ,\n });\n\n case OP_ADDWITHFLAGS:\n return FlagInfo::Pack({\n .Write = FLAG_NZCV,\n .Replacement = OP_ADD,\n .ReplacementNoWrite = OP_ADDNZCV,\n });\n\n case OP_SUBWITHFLAGS:\n return FlagInfo::Pack({\n .Write = FLAG_NZCV,\n .Replacement = OP_SUB,\n .ReplacementNoWrite = OP_SUBNZCV,\n });\n\n case OP_ADCWITHFLAGS:\n return FlagInfo::Pack({\n .Read = FLAG_C,\n .Write = FLAG_NZCV,\n .Replacement = OP_ADC,\n .ReplacementNoWrite = OP_ADCNZCV,\n });\n\n case OP_ADCZEROWITHFLAGS:\n return FlagInfo::Pack({\n .Read = FLAG_C,\n .Write = FLAG_NZCV,\n .Replacement = OP_ADCZERO,\n });\n\n case OP_SBBWITHFLAGS:\n return FlagInfo::Pack({\n .Read = FLAG_C,\n .Write = FLAG_NZCV,\n .Replacement = OP_SBB,\n .ReplacementNoWrite = OP_SBBNZCV,\n });\n\n case OP_SHIFTFLAGS:\n // _ShiftFlags conditionally sets NZCV+PF, which we model here as a\n // read-modify-write. Logically, it also conditionally makes AF undefined,\n // which we model by omitting AF from both Read and Write sets (since\n // \"cond ? AF : undef\" may be optimized to \"AF\").\n return FlagInfo::Pack({\n .Read = FLAG_NZCV | FLAG_P,\n .Write = FLAG_NZCV | FLAG_P,\n .CanEliminate = true,\n });\n\n case OP_ROTATEFLAGS:\n // _RotateFlags conditionally sets CV, again modeled as RMW.\n return FlagInfo::Pack({\n .Read = FLAG_C | FLAG_V,\n .Write = FLAG_C | FLAG_V,\n .CanEliminate = true,\n });\n\n case OP_RDRAND: return FlagInfo::Pack({.Write = FLAG_NZCV});\n\n case OP_ADDNZCV:\n case OP_SUBNZCV:\n case OP_TESTNZ:\n case OP_FCMP:\n case OP_STORENZCV:\n return FlagInfo::Pack({\n .Write = FLAG_NZCV,\n .CanEliminate = true,\n });\n\n case OP_AXFLAG:\n // Per the Arm spec, axflag reads Z/V/C but not N. It writes all flags.\n return FlagInfo::Pack({\n .Read = FLAG_ZCV,\n .Write = FLAG_NZCV,\n .CanEliminate = true,\n });\n\n case OP_CMPPAIRZ:\n return FlagInfo::Pack({\n .Write = FLAG_Z,\n .CanEliminate = true,\n });\n\n case OP_CARRYINVERT:\n return FlagInfo::Pack({\n .Read = FLAG_C,\n .Write = FLAG_C,\n .CanEliminate = true,\n });\n\n case OP_SETSMALLNZV:\n return FlagInfo::Pack({\n .Write = FLAG_N | FLAG_Z | FLAG_V,\n .CanEliminate = true,\n });\n\n case OP_LOADNZCV: return FlagInfo::Pack({.Read = FLAG_NZCV});\n\n case OP_ADC:\n case OP_ADCZERO:\n case OP_SBB: return FlagInfo::Pack({.Read = FLAG_C});\n\n case OP_ADCNZCV:\n case OP_SBBNZCV:\n return FlagInfo::Pack({\n .Read = FLAG_C,\n .Write = FLAG_NZCV,\n .CanEliminate = true,\n });\n\n case OP_LOADPF: return FlagInfo::Pack({.Read = FLAG_P});\n case OP_LOADAF: return FlagInfo::Pack({.Read = FLAG_A});\n case OP_STOREPF: return FlagInfo::Pack({.Write = FLAG_P, .CanEliminate = true});\n case OP_STOREAF: return FlagInfo::Pack({.Write = FLAG_A, .CanEliminate = true});\n\n case OP_NZCVSELECT:\n case OP_NZCVSELECTV:\n case OP_NZCVSELECTINCREMENT:\n case OP_NEG:\n case OP_CONDJUMP:\n case OP_CONDSUBNZCV:\n case OP_CONDADDNZCV:\n case OP_RMIFNZCV:\n case OP_INVALIDATEFLAGS: return FlagInfo::Pack({.Special = true});\n default: return FlagInfo::Pack({});\n }\n}\n\nconstexpr auto FlagInfos = std::invoke([] {\n std::array ret = {};\n\n for (unsigned i = 0; i < OP_LAST; ++i) {\n ret[i] = ClassifyConst((IROps)i);\n }\n\n return ret;\n});\n\nFlagInfo DeadFlagCalculationEliminination::Classify(IROp_Header* IROp) {\n FlagInfo Info = FlagInfos[IROp->Op];\n if (!Info.Special()) {\n return Info;\n }\n\n switch (IROp->Op) {\n case OP_NZCVSELECT:\n case OP_NZCVSELECTINCREMENT: {\n auto Op = IROp->CW();\n return FlagInfo::Pack({.Read = FlagsForCondClassType(Op->Cond)});\n }\n\n case OP_NZCVSELECTV: {\n auto Op = IROp->CW();\n return FlagInfo::Pack({.Read = FlagsForCondClassType(Op->Cond)});\n }\n\n case OP_NEG: {\n auto Op = IROp->CW();\n return FlagInfo::Pack({.Read = FlagsForCondClassType(Op->Cond)});\n }\n\n case OP_CONDJUMP: {\n auto Op = IROp->CW();\n if (!Op->FromNZCV) {\n return FlagInfo::Pack({});\n }\n\n return FlagInfo::Pack({.Read = FlagsForCondClassType(Op->Cond)});\n }\n\n case OP_CONDSUBNZCV:\n case OP_CONDADDNZCV: {\n auto Op = IROp->CW();\n return FlagInfo::Pack({\n .Read = FlagsForCondClassType(Op->Cond),\n .Write = FLAG_NZCV,\n .CanEliminate = true,\n });\n }\n\n case OP_RMIFNZCV: {\n auto Op = IROp->CW();\n\n static_assert(FLAG_N == (1 << 3), \"rmif mask lines up with our bits\");\n static_assert(FLAG_Z == (1 << 2), \"rmif mask lines up with our bits\");\n static_assert(FLAG_C == (1 << 1), \"rmif mask lines up with our bits\");\n static_assert(FLAG_V == (1 << 0), \"rmif mask lines up with our bits\");\n\n return FlagInfo::Pack({\n .Write = Op->Mask,\n .CanEliminate = true,\n });\n }\n\n case OP_INVALIDATEFLAGS: {\n auto Op = IROp->CW();\n unsigned Flags = 0;\n\n // TODO: Make this translation less silly\n if (Op->Flags & (1u << X86State::RFLAG_SF_RAW_LOC)) {\n Flags |= FLAG_N;\n }\n\n if (Op->Flags & (1u << X86State::RFLAG_ZF_RAW_LOC)) {\n Flags |= FLAG_Z;\n }\n\n if (Op->Flags & (1u << X86State::RFLAG_CF_RAW_LOC)) {\n Flags |= FLAG_C;\n }\n\n if (Op->Flags & (1u << X86State::RFLAG_OF_RAW_LOC)) {\n Flags |= FLAG_V;\n }\n\n if (Op->Flags & (1u << X86State::RFLAG_PF_RAW_LOC)) {\n Flags |= FLAG_P;\n }\n\n if (Op->Flags & (1u << X86State::RFLAG_AF_RAW_LOC)) {\n Flags |= FLAG_A;\n }\n\n // The mental model of InvalidateFlags is writing undefined values to all\n // of the selected flags, allowing the write-after-write optimizations to\n // optimize invalidate-after-write for free.\n return FlagInfo::Pack({\n .Write = Flags,\n .CanEliminate = true,\n });\n }\n\n default: LOGMAN_THROW_A_FMT(false, \"invalid special op\"); FEX_UNREACHABLE;\n }\n\n FEX_UNREACHABLE;\n}\n\n// General purpose dead code elimination. Returns whether flag handling should\n// be skipped (because it was removed or could not possibly affect flags).\nbool DeadFlagCalculationEliminination::EliminateDeadCode(IREmitter* IREmit, Ref CodeNode, IROp_Header* IROp) {\n // Can't remove anything used or with side effects.\n if (CodeNode->GetUses() > 0 || IR::HasSideEffects(IROp->Op)) {\n return false;\n }\n\n IREmit->Remove(CodeNode);\n return true;\n}\n\nCondClass DeadFlagCalculationEliminination::X86ToArmFloatCond(CondClass X86) {\n // Table of x86 condition codes that map to arm64 condition codes, in the\n // sense that fcmp+axflag+branch(x86) is equivalent to fcmp+branch(arm).\n //\n // E would be \"equal or unordered\", no condition code.\n // G would be \"greater than or less than\", no condition code.\n //\n // SF/OF conditions are trivial and therefore shouldn't actually be generated\n switch (X86) {\n case CondClass::UGE /* A */: return CondClass::FGE /* GE */;\n case CondClass::UGT /* AE */: return CondClass::FGT /* GT */;\n case CondClass::ULT /* B */: return CondClass::SLT /* LT */;\n case CondClass::ULE /* BE */: return CondClass::SLE /* LE */;\n case CondClass::SLE /* LE */: return CondClass::SLE /* LE */;\n default: return CondClass::AL;\n }\n}\n\nvoid DeadFlagCalculationEliminination::FoldBranch(IREmitter* IREmit, IRListView& CurrentIR, IROp_CondJump* Op, Ref CodeNode) {\n // Skip past StoreRegisters at the end -- they don't touch flags.\n auto PrevWrap = CodeNode->Header.Previous;\n while (CurrentIR.GetOp(PrevWrap)->Op == OP_STOREREGISTER ||\n CurrentIR.GetOp(PrevWrap)->Op == OP_STOREPF || CurrentIR.GetOp(PrevWrap)->Op == OP_STOREAF) {\n PrevWrap = CurrentIR.GetNode(PrevWrap)->Header.Previous;\n }\n\n auto Prev = CurrentIR.GetOp(PrevWrap);\n if (Prev->Op == OP_AXFLAG) {\n // Pattern match a branch fed by AXFLAG.\n CondClass ArmCond = X86ToArmFloatCond(Op->Cond);\n if (ArmCond == CondClass::AL) {\n return;\n }\n\n Op->Cond = ArmCond;\n } else if (Prev->Op == OP_SUBNZCV) {\n // Pattern match a branch fed by a compare. We could also handle bit tests\n // here, but tbz/tbnz has a limited offset range which we don't have a way to\n // deal with yet. Let's hope that's not a big deal.\n if (!(Op->Cond == CondClass::NEQ || Op->Cond == CondClass::EQ) || (Prev->Size < OpSize::i32Bit)) {\n return;\n }\n\n auto SecondArg = CurrentIR.GetOp(Prev->Args[1]);\n if (SecondArg->Op != OP_INLINECONSTANT || SecondArg->C()->Constant != 0) {\n return;\n }\n\n // We've matched. Fold the compare into branch.\n IREmit->ReplaceNodeArgument(CodeNode, 0, CurrentIR.GetNode(Prev->Args[0]));\n IREmit->ReplaceNodeArgument(CodeNode, 1, CurrentIR.GetNode(Prev->Args[1]));\n Op->FromNZCV = false;\n Op->CompareSize = Prev->Size;\n } else {\n return;\n }\n\n // The compare/test/axflag sets flags but does not write registers. Flags are\n // dead after the jump. The jump does not read flags anymore. There is no\n // intervening instruction. Therefore the compare is dead.\n IREmit->Remove(CurrentIR.GetNode(PrevWrap));\n}\n\n/**\n * @brief This pass removes dead code locally.\n */\nbool DeadFlagCalculationEliminination::ProcessBlock(IREmitter* IREmit, IRListView& CurrentIR, Ref Block, ControlFlowGraph& CFG) {\n uint32_t FlagsRead = FLAG_ALL;\n\n // Reverse iteration is not yet working with the iterators\n auto BlockIROp = CurrentIR.GetOp(Block);\n\n // We grab these nodes this way so we can iterate easily\n auto CodeBegin = CurrentIR.at(BlockIROp->Begin);\n auto CodeLast = CurrentIR.at(BlockIROp->Last);\n\n // Advance past EndBlock to get at the exit.\n --CodeLast;\n\n // Initialize the FlagsRead mask according to the exit instruction.\n auto [ExitNode, ExitOp] = CodeLast();\n if (ExitOp->Op == IR::OP_CONDJUMP) {\n auto Op = ExitOp->CW();\n FlagsRead = CFG.Get(Op->TrueBlock)->Flags | CFG.Get(Op->FalseBlock)->Flags;\n } else if (ExitOp->Op == IR::OP_JUMP) {\n FlagsRead = CFG.Get(ExitOp->Args[0])->Flags;\n }\n\n // Iterate the block in reverse\n while (true) {\n auto [CodeNode, IROp] = CodeLast();\n\n // Optimizing flags can cause earlier flag reads to become dead but dead\n // flag reads should not impede optimiation of earlier dead flag writes.\n // We must DCE as we go to ensure we converge in a single iteration.\n if (!EliminateDeadCode(IREmit, CodeNode, IROp)) {\n // Optimiation algorithm: For each flag written...\n //\n // If the flag has a later read (per FlagsRead), remove the flag from\n // FlagsRead, since the reader is covered by this write.\n //\n // Else, there is no later read, so remove the flag write (if we can).\n // This is the active part of the optimization.\n //\n // Then, add each flag read to FlagsRead.\n //\n // This order is important: instructions that read-modify-write flags\n // (like adcs) first read flags, then write flags. Since we're iterating\n // the block backwards, that means we handle the write first.\n struct FlagInfo Info = Classify(IROp);\n\n if (!Info.Trivial()) {\n bool Eliminated = false;\n\n if ((FlagsRead & Info.Write()) == 0) {\n if ((Info.CanEliminate() || Info.Replacement()) && CodeNode->GetUses() == 0) {\n IREmit->Remove(CodeNode);\n Eliminated = true;\n } else if (Info.Replacement()) {\n IROp->Op = Info.Replacement();\n }\n } else if (Info.ReplacementNoWrite() && CodeNode->GetUses() == 0) {\n IROp->Op = Info.ReplacementNoWrite();\n }\n\n // If we don't care about the sign or carry, we can optimize testnz.\n // Carry is inverted between testz and testnz so we check that too. Note\n // this flag is outside of the if, since the TestNZ might result from\n // optimizing AndWithFlags, and we need to converge locally in a single\n // iteration.\n if (IROp->Op == OP_TESTNZ && IROp->Size < OpSize::i32Bit && !(FlagsRead & (FLAG_N | FLAG_C))) {\n IROp->Op = OP_TESTZ;\n }\n\n FlagsRead &= ~Info.Write();\n\n // If we eliminated the instruction, we eliminate its read too. This\n // check is required to ensure the pass converges locally in a single\n // iteration.\n if (!Eliminated) {\n FlagsRead |= Info.Read();\n }\n }\n }\n\n // Iterate in reverse\n if (CodeLast == CodeBegin) {\n break;\n }\n --CodeLast;\n }\n\n // For the purposes of global propagation, the content of our progress doesn't\n // matter -- only the difference in our final FlagsRead contributes to changes\n // in the predecessors.\n uint32_t OldFlagsRead = CFG.Get(BlockIROp->ID)->Flags;\n CFG.Get(BlockIROp->ID)->Flags = FlagsRead;\n return (OldFlagsRead != FlagsRead);\n}\n\nvoid DeadFlagCalculationEliminination::OptimizeParity(IREmitter* IREmit, IRListView& CurrentIR, ControlFlowGraph& CFG) {\n // Mapping for flags inside this pass.\n const uint8_t PARTIAL = 0;\n const uint8_t FULL = 1;\n\n // Initialize conservatively: all blocks need full parity. This initialization\n // matters for proper handling of backedges.\n for (auto [Block, BlockHeader] : CurrentIR.GetBlocks()) {\n auto ID = BlockHeader->C()->ID;\n CFG.Get(ID)->Flags = FULL;\n }\n\n for (auto [Block, BlockHeader] : CurrentIR.GetBlocks()) {\n const auto ID = BlockHeader->C()->ID;\n const auto& Predecessors = CFG.Get(ID)->Predecessors;\n bool Full = false;\n\n if (Predecessors.empty()) {\n // Conservatively assume there was full parity before the start block\n Full = true;\n } else {\n // If any predecessor needs full parity at the end, we need full parity.\n for (auto Pred : Predecessors) {\n Full |= (CFG.Get(Pred)->Flags == FULL);\n }\n }\n\n for (auto [CodeNode, IROp] : CurrentIR.GetCode(Block)) {\n if (IROp->Op == OP_STOREPF) {\n auto Op = IROp->CW();\n auto Generator = CurrentIR.GetOp(Op->Value);\n\n // Determine if we only write 0/1 to the parity flag.\n Full = true;\n if (Generator->Op == OP_NZCVSELECT) {\n auto C0 = CurrentIR.GetOp(Generator->Args[0]);\n auto C1 = CurrentIR.GetOp(Generator->Args[1]);\n if (C0->Op == C1->Op && C0->Op == OP_INLINECONSTANT) {\n auto IC0 = CurrentIR.GetOp(Generator->Args[0]);\n auto IC1 = CurrentIR.GetOp(Generator->Args[1]);\n\n // We need the full 8 if the constant has upper bits set.\n Full = (IC0->Constant | IC1->Constant) & ~1;\n }\n }\n } else if (IROp->Op == OP_PARITY && !Full) {\n // Eliminate parity calculations if it's only 1-bit.\n auto Parity = IROp->C();\n Ref Value = CurrentIR.GetNode(Parity->Raw);\n\n if (Parity->Invert) {\n IREmit->SetWriteCursor(CodeNode);\n Value = IREmit->_Xor(OpSize::i32Bit, Value, IREmit->_InlineConstant(1));\n }\n\n IREmit->ReplaceUsesWithAfter(CodeNode, Value, CurrentIR.at(CodeNode));\n IREmit->Remove(CodeNode);\n }\n }\n\n // Record our final state for our successors to read.\n CFG.Get(ID)->Flags = Full ? FULL : PARTIAL;\n }\n}\n\nvoid DeadFlagCalculationEliminination::Run(IREmitter* IREmit) {\n FEXCORE_PROFILE_SCOPED(\"PassManager::DFE\");\n\n auto CurrentIR = IREmit->ViewIR();\n fextl::deque Worklist;\n\n // Initialize CFG\n ControlFlowGraph CFG {.IR = CurrentIR};\n CFG.Init(Worklist, CurrentIR.GetHeader()->BlockCount);\n\n // Gather CFG\n for (auto [BlockNode, BlockHeader] : CurrentIR.GetBlocks()) {\n auto Block = BlockHeader->C();\n auto CodeLast = CurrentIR.at(Block->Last);\n --CodeLast;\n auto [ExitNode, ExitOp] = CodeLast();\n if (ExitOp->Op == IR::OP_CONDJUMP) {\n auto Op = ExitOp->CW();\n\n CFG.RecordEdge(Block->ID, Op->TrueBlock);\n CFG.RecordEdge(Block->ID, Op->FalseBlock);\n } else if (ExitOp->Op == IR::OP_JUMP) {\n CFG.RecordEdge(Block->ID, ExitOp->Args[0]);\n }\n\n CFG.Get(Block->ID)->Node = BlockNode;\n }\n\n // After processing a block, if we made progress, we must process its\n // predecessors to propagate globally. A block will be reprocessed only if\n // there is a loop backedge.\n for (; !Worklist.empty(); Worklist.pop_back()) {\n auto Block = Worklist.back();\n auto Info = CFG.Get(Block);\n Info->InWorklist = false;\n\n if (ProcessBlock(IREmit, CurrentIR, Info->Node, CFG)) {\n for (auto Pred : Info->Predecessors) {\n CFG.AddWorklist(Worklist, Pred);\n }\n }\n }\n\n // Fold compares into branches now that we're otherwise optimized. This needs\n // to run after eliminating carries etc and it needs the global flag metadata.\n // But it only needs to run once, we don't do it in the loop.\n for (auto [Block, _] : CurrentIR.GetBlocks()) {\n // Grab the jump\n auto BlockIROp = CurrentIR.GetOp(Block);\n auto CodeLast = CurrentIR.at(BlockIROp->Last);\n --CodeLast;\n\n auto [ExitNode, ExitOp] = CodeLast();\n if (ExitOp->Op == IR::OP_CONDJUMP) {\n auto Op = ExitOp->CW();\n uint32_t FlagsOut = CFG.Get(Op->TrueBlock)->Flags | CFG.Get(Op->FalseBlock)->Flags;\n\n if ((FlagsOut & FLAG_NZCV) == 0 && Op->FromNZCV) {\n FoldBranch(IREmit, CurrentIR, Op, ExitNode);\n }\n }\n }\n\n if (CurrentIR.GetHeader()->ReadsParity) {\n OptimizeParity(IREmit, CurrentIR, CFG);\n }\n}\n\nfextl::unique_ptr CreateDeadFlagCalculationEliminination() {\n return fextl::make_unique();\n}\n\n} // namespace FEXCore::IR\n" }, { "path": "FEXCore/Source/Interface/IR/Passes/RegisterAllocationPass.cpp", "content": "// SPDX-License-Identifier: MIT\n/*\n$info$\ntags: ir|opts\n$end_info$\n*/\n\n#include \"Interface/IR/Passes/RegisterAllocationPass.h\"\n#include \"Interface/IR/IR.h\"\n#include \"Interface/IR/IREmitter.h\"\n#include \"Interface/IR/RegisterAllocationData.h\"\n#include \"Interface/IR/Passes.h\"\n#include \"Interface/Core/CPUID.h\"\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\nusing namespace FEXCore;\n\nnamespace FEXCore::IR {\nnamespace {\n struct RegisterClassData {\n uint32_t Available;\n uint32_t Count;\n\n // If bit R of Available is 0, then RegToSSA[R] is the node currently\n // allocated to R. Else, RegToSSA[R] is UNDEFINED, no need to clear this\n // when freeing registers.\n Ref RegToSSA[32];\n };\n\n IR::RegClass GetRegClassFromNode(IR::IRListView* IR, IR::IROp_Header* IROp) {\n const auto Class = IR::GetRegClass(IROp->Op);\n if (Class != IR::RegClass::Complex) {\n return Class;\n }\n\n // Complex register class handling\n switch (IROp->Op) {\n case IR::OP_LOADCONTEXT: return IROp->C()->Class;\n case IR::OP_LOADREGISTER: return IROp->C()->Class;\n case IR::OP_LOADCONTEXTINDEXED: return IROp->C()->Class;\n case IR::OP_LOADMEM:\n case IR::OP_LOADMEMTSO: return IROp->C()->Class;\n case IR::OP_FILLREGISTER: return IROp->C()->Class;\n default: return IR::RegClass::Invalid;\n }\n };\n} // Anonymous namespace\n\nclass ConstrainedRAPass final : public RegisterAllocationPass {\npublic:\n explicit ConstrainedRAPass(const FEXCore::CPUIDEmu* CPUID)\n : CPUID {CPUID} {}\n void Run(IREmitter* IREmit) override;\n void AddRegisters(IR::RegClass Class, uint32_t RegisterCount) override;\n bool TryPostRAMerge(Ref LastNode, Ref CodeNode, IROp_Header* IROp);\n\nprivate:\n RegisterClassData Classes[IR::NumClasses];\n\n IREmitter* IREmit {};\n IRListView* IR {};\n const FEXCore::CPUIDEmu* CPUID {};\n\n // Map of nodes to their preferred register, to coalesce load/store reg.\n fextl::vector PreferredReg;\n\n // Map of assigned registers. Does not grow beyond the initial set.\n fextl::vector SSAToReg;\n\n // Maps defs to their assigned spill slot + 1, or 0 if not spilled.\n fextl::vector SpillSlots;\n\n // Next-use distance relative to the block end of each source, last first.\n fextl::vector SourcesNextUses;\n\n // Sources that have been seen\n fextl::vector Seen;\n\n // SourcesNextUses is read backwards, this tracks the index\n int64_t SourceIndex {};\n\n bool Rematerializable(IROp_Header* IROp) {\n return IROp->Op == OP_CONSTANT;\n }\n\n Ref InsertFill(Ref Node) {\n IROp_Header* IROp = IR->GetOp(Node);\n\n // Remat if we can\n if (Rematerializable(IROp)) {\n const auto Op = IROp->C();\n uint64_t Const = Op->Constant;\n return IREmit->_Constant(Const, Op->Pad, Op->MaxBytes);\n }\n\n // Otherwise fill from stack\n uint32_t SlotPlusOne = SpillSlots[IR->GetID(Node).Value];\n LOGMAN_THROW_A_FMT(SlotPlusOne >= 1, \"Node must have been spilled\");\n\n const auto RegClass = GetRegClassFromNode(IR, IROp);\n return IREmit->_FillRegister(IROp->Size, IROp->ElementSize, SlotPlusOne - 1, RegClass);\n };\n\n // IP of next-use of each source. IPs are measured from the end of the\n // block, so we don't need to size the block up-front.\n fextl::vector NextUses;\n\n bool AnySpilled {};\n\n bool IsValidArg(OrderedNodeWrapper Arg) {\n if (Arg.IsInvalid()) {\n return false;\n }\n\n auto Op = IR->GetOp(Arg)->Op;\n return Op != OP_INLINECONSTANT && Op != OP_INLINEENTRYPOINTOFFSET;\n };\n\n RegisterClassData* GetClass(PhysicalRegister Reg) {\n return &Classes[Reg.Class];\n };\n\n uint32_t GetRegBits(PhysicalRegister Reg) {\n return 1 << Reg.Reg;\n };\n\n bool IsInRegisterFile(Ref Node) {\n auto ID = IR->GetID(Node).Value;\n LOGMAN_THROW_A_FMT(ID < SSAToReg.size(), \"Only old nodes looked up\");\n\n PhysicalRegister Reg = SSAToReg[ID];\n RegisterClassData* Class = GetClass(Reg);\n\n return (Class->Available & GetRegBits(Reg)) == 0 && Class->RegToSSA[Reg.Reg] == Node;\n };\n\n void FreeReg(PhysicalRegister Reg) {\n RegisterClassData* Class = GetClass(Reg);\n uint32_t RegBits = GetRegBits(Reg);\n\n LOGMAN_THROW_A_FMT(!(Class->Available & RegBits), \"Register double-free\");\n\n Class->Available |= RegBits;\n };\n\n bool HasSource(IROp_Header* I, PhysicalRegister Reg) {\n int NumArgs = IR::GetRAArgs(I->Op);\n for (int s = 0; s < NumArgs; ++s) {\n if (I->Args[s].IsImmediate()) {\n // When spilling for a destination, we'll see register sources\n if (PhysicalRegister(I->Args[s]) == Reg) {\n return true;\n }\n } else {\n // When spilling for SRA correctness, we'll see SSA sources. This is\n // pretty obscure.\n auto V = I->Args[s];\n V.ClearKill();\n\n if (IsValidArg(V) && SSAToReg[V.ID().Value] == Reg) {\n return true;\n }\n }\n }\n\n return false;\n };\n\n Ref DecodeSRANode(const IROp_Header* IROp, Ref Node) {\n if (IROp->Op == OP_LOADREGISTER || IROp->Op == OP_LOADPF || IROp->Op == OP_LOADAF) {\n return Node;\n } else if (IROp->Op == OP_STOREREGISTER) {\n auto V = IROp->C()->Value;\n V.ClearKill();\n return IR->GetNode(V);\n } else if (IROp->Op == OP_STOREPF || IROp->Op == OP_STOREAF) {\n auto V = IROp->C()->Value;\n V.ClearKill();\n return IR->GetNode(V);\n }\n\n return nullptr;\n };\n\n PhysicalRegister DecodeSRAReg(const IROp_Header* IROp, Ref Node) {\n uint8_t FlagOffset = Classes[FEXCore::ToUnderlying(RegClass::GPRFixed)].Count - 2;\n\n if (IROp->Op == OP_STOREREGISTER) {\n return PhysicalRegister(Node);\n } else if (IROp->Op == OP_LOADPF || IROp->Op == OP_STOREPF) {\n return PhysicalRegister {RegClass::GPRFixed, FlagOffset};\n } else if (IROp->Op == OP_LOADAF || IROp->Op == OP_STOREAF) {\n return PhysicalRegister {RegClass::GPRFixed, uint8_t(FlagOffset + 1)};\n } else {\n const IROp_LoadRegister* Op = IROp->C();\n\n LOGMAN_THROW_A_FMT(Op->Class == RegClass::GPR || Op->Class == RegClass::FPR, \"SRA classes\");\n if (Op->Class == RegClass::FPR) {\n return PhysicalRegister {RegClass::FPRFixed, uint8_t(Op->Reg)};\n } else {\n return PhysicalRegister {RegClass::GPRFixed, uint8_t(Op->Reg)};\n }\n }\n };\n\n bool IsTrivial(Ref Node, const IROp_Header* Header) {\n switch (Header->Op) {\n case OP_ALLOCATEGPR: return true;\n case OP_ALLOCATEGPRAFTER: return true;\n case OP_ALLOCATEFPR: return true;\n case OP_RMWHANDLE: return PhysicalRegister(Node) == PhysicalRegister(Header->Args[0]);\n case OP_LOADREGISTER: return PhysicalRegister(Node) == DecodeSRAReg(Header, Node);\n case OP_STOREREGISTER: return PhysicalRegister(Header->Args[0]) == DecodeSRAReg(Header, Node);\n default: return false;\n }\n }\n\n // Helper macro to walk the set bits b in a 32-bit word x, using ffs to get\n // the next set bit and then clearing on each iteration.\n#define foreach_bit(b, x) for (uint32_t __x = (x), b; ((b) = __builtin_ffs(__x) - 1, __x); __x &= ~(1 << (b)))\n\n void CalculateNextUses(IROp_CodeBlock* BlockIROp, IROp_Header* Until) {\n SourcesNextUses.clear();\n NextUses.resize(IR->GetSSACount(), 0);\n\n // IP relative to the end of the block.\n uint32_t IP = 1;\n\n // We grab these nodes this way so we can iterate easily\n auto CodeBegin = IR->at(BlockIROp->Begin);\n auto CodeLast = IR->at(BlockIROp->Last);\n\n while (1) {\n auto [CodeNode, IROp] = CodeLast();\n if (IROp == Until) {\n break;\n }\n // End of iteration gunk\n\n const int NumArgs = IR::GetRAArgs(IROp->Op);\n for (int i = NumArgs - 1; i >= 0; --i) {\n auto V = IROp->Args[i];\n V.ClearKill();\n\n if (IsValidArg(V)) {\n const uint32_t Index = V.ID().Value;\n\n SourcesNextUses.push_back(NextUses[Index]);\n NextUses[Index] = IP;\n }\n }\n\n // IP is relative to block end and we iterate backwards, so increment.\n ++IP;\n\n // Rest is iteration gunk\n if (CodeLast == CodeBegin) {\n break;\n }\n --CodeLast;\n }\n\n SourceIndex = SourcesNextUses.size();\n }\n\n void SpillReg(RegisterClassData* Class, IROp_CodeBlock* Block, IROp_Header* Exclude) {\n // We're about to use next-use information, so calculate it.\n if (!AnySpilled) {\n CalculateNextUses(Block, Exclude);\n }\n\n // Find the best node to spill according to the \"furthest-first\" heuristic.\n // Since we defined IPs relative to the end of the block, the furthest\n // next-use has the /smallest/ unsigned IP.\n Ref Candidate = nullptr;\n uint32_t BestDistance = UINT32_MAX;\n uint8_t BestReg = ~0;\n uint32_t Allocated = ((1u << Class->Count) - 1) & ~Class->Available;\n\n foreach_bit(i, Allocated) {\n Ref Node = Class->RegToSSA[i];\n auto Reg = SSAToReg[IR->GetID(Node).Value];\n\n LOGMAN_THROW_A_FMT(Node != nullptr, \"Invariant3\");\n LOGMAN_THROW_A_FMT(Reg.Reg == i, \"Invariant4\");\n\n // Skip any source used by the current instruction, it is unspillable.\n if (!HasSource(Exclude, Reg)) {\n uint32_t NextUse = NextUses[IR->GetID(Node).Value];\n\n // Prioritize remat over spilling. It is typically cheaper to remat a\n // constant multiple times than to spill a single value.\n if (!Rematerializable(IR->GetOp(Node))) {\n NextUse += 100000;\n }\n\n if (NextUse < BestDistance) {\n BestDistance = NextUse;\n BestReg = i;\n Candidate = Node;\n }\n }\n }\n\n LOGMAN_THROW_A_FMT(Candidate != nullptr, \"must've found something..\");\n\n PhysicalRegister Reg = SSAToReg[IR->GetID(Candidate).Value];\n LOGMAN_THROW_A_FMT(Reg.Reg == BestReg, \"Invariant6\");\n\n IROp_Header* Header = IR->GetOp(Candidate);\n uint32_t Value = IR->GetID(Candidate).Value;\n bool Spilled = !SpillSlots.empty() && SpillSlots[Value] != 0;\n\n // If we already spilled the Candidate, we don't need to spill again.\n // Similarly, if we can rematerialize the instruction, we don't spill it.\n if (!Spilled && Header->Op != OP_CONSTANT) {\n LOGMAN_THROW_A_FMT(Reg.AsRegClass() == GetRegClassFromNode(IR, Header), \"Consistent\");\n\n // SpillSlots allocation is deferred.\n if (SpillSlots.empty()) {\n SpillSlots.resize(IR->GetSSACount(), 0);\n }\n\n // TODO: we should colour spill slots\n uint32_t Slot = IR->GetHeader()->SpillSlots++;\n\n // We must map here in case we're spilling something we shuffled.\n auto SpillOp = IREmit->_SpillRegister(OrderedNodeWrapper::FromImmediate(Reg.Raw), Slot, Reg.AsRegClass());\n SpillOp.first->Header.Size = Header->Size;\n SpillOp.first->Header.ElementSize = Header->ElementSize;\n SpillSlots[Value] = Slot + 1;\n }\n\n // Now that we've spilled the value, take it out of the register file\n FreeReg(Reg);\n AnySpilled = true;\n };\n\n void RemapReg(Ref Node, PhysicalRegister Reg) {\n RegisterClassData* Class = GetClass(Reg);\n Class->RegToSSA[Reg.Reg] = Node;\n\n uint32_t Index = IR->GetID(Node).Value;\n if (Index < SSAToReg.size()) {\n SSAToReg[Index] = Reg;\n }\n };\n\n // Record a given assignment of register Reg to Node.\n void SetReg(Ref Node, PhysicalRegister Reg) {\n RegisterClassData* Class = GetClass(Reg);\n uint32_t RegBits = GetRegBits(Reg);\n\n LOGMAN_THROW_A_FMT((Class->Available & RegBits) == RegBits, \"Precondition\");\n\n Class->Available &= ~RegBits;\n\n RemapReg(Node, Reg);\n Node->Reg = Reg.Raw;\n };\n\n // Assign a register for a given Node, spilling if necessary.\n void AssignReg(IROp_Header* IROp, IROp_CodeBlock* Block, Ref CodeNode, IROp_Header* Pivot) {\n const uint32_t Node = IR->GetID(CodeNode).Value;\n\n // Prioritize preferred registers.\n if (Node < PreferredReg.size()) {\n if (PhysicalRegister Reg = PreferredReg[Node]; !Reg.IsInvalid()) {\n RegisterClassData* Class = GetClass(Reg);\n uint32_t RegBits = GetRegBits(Reg);\n\n if ((Class->Available & RegBits) == RegBits) {\n SetReg(CodeNode, Reg);\n return;\n }\n }\n }\n\n // Try to handle tied registers. This can fail, the JIT will insert moves.\n if (int TiedIdx = IR::TiedSource(IROp->Op); TiedIdx >= 0) {\n auto Reg = PhysicalRegister(IROp->Args[TiedIdx]);\n RegisterClassData* Class = GetClass(Reg);\n uint32_t RegBits = GetRegBits(Reg);\n\n if (Reg.AsRegClass() != RegClass::GPRFixed && Reg.AsRegClass() != RegClass::FPRFixed && (Class->Available & RegBits) == RegBits) {\n SetReg(CodeNode, Reg);\n return;\n }\n }\n\n // Try to coalesce reserved pairs. Just a heuristic to remove some moves.\n if (IROp->Op == OP_ALLOCATEGPR && IROp->C()->ForPair) {\n uint32_t Available = Classes[FEXCore::ToUnderlying(RegClass::GPR)].Available;\n\n // Only choose base register R if R and R + 1 are both free\n Available &= (Available >> 1);\n\n // Only consider aligned registers in the pair region\n constexpr uint32_t EVEN_BITS = 0x55555555;\n Available &= (EVEN_BITS & ((1u << PairRegs) - 1));\n\n if (Available) {\n unsigned Reg = std::countr_zero(Available);\n SetReg(CodeNode, PhysicalRegister(RegClass::GPR, Reg));\n return;\n }\n } else if (IROp->Op == OP_ALLOCATEGPRAFTER) {\n uint32_t Available = Classes[FEXCore::ToUnderlying(RegClass::GPR)].Available;\n auto After = PhysicalRegister(IROp->Args[0]);\n if ((After.Reg & 1) == 0 && Available & (1ull << (After.Reg + 1))) {\n SetReg(CodeNode, PhysicalRegister(RegClass::GPR, After.Reg + 1));\n return;\n }\n }\n\n RegClass ClassType = GetRegClassFromNode(IR, IROp);\n RegisterClassData* Class = &Classes[FEXCore::ToUnderlying(ClassType)];\n\n // Spill to make room in the register file.\n if (!Class->Available) {\n IREmit->SetWriteCursorBefore(CodeNode);\n SpillReg(Class, Block, Pivot);\n }\n\n // Assign a free register in the appropriate class.\n LOGMAN_THROW_A_FMT(Class->Available != 0, \"Post-condition of spilling\");\n unsigned Reg = std::countr_zero(Class->Available);\n SetReg(CodeNode, PhysicalRegister(ClassType, Reg));\n };\n};\n\nvoid ConstrainedRAPass::AddRegisters(IR::RegClass Class, uint32_t RegisterCount) {\n LOGMAN_THROW_A_FMT(RegisterCount <= 31, \"Up to 31 regs supported\");\n\n Classes[FEXCore::ToUnderlying(Class)].Count = RegisterCount;\n}\n\ninline bool KillMove(IROp_Header* LastOp, IROp_Header* IROp, Ref LastNode, Ref CodeNode) {\n // 32-bit moves in x86_64 are represented as a Bfe, detect them.\n if (LastOp->Op == OP_BFE && LastOp->C()->lsb == 0 && LastOp->C()->Width == 32) {\n auto Op = IROp->Op;\n\n if (Op == OP_AND) {\n // Rewrite \"mov wA, wB; and xA, xA, xC\" into \"and wA, wB, wC\", since\n // ((b & 0xffffffff) & c) == (b & c) & 0xffffffff.\n IROp->Size = OpSize::i32Bit;\n return true;\n } else if (IROp->Size == OpSize::i32Bit) {\n return Op == OP_OR || Op == OP_XOR || Op == OP_AND || Op == OP_SUB || Op == OP_LSHL || Op == OP_LSHR || Op == OP_ASHR;\n }\n }\n\n return LastOp->Op == OP_STOREREGISTER;\n}\n\ninline bool IsSignext(const IROp_Header* IROp, OrderedNodeWrapper Src, OpSize Size) {\n if (IROp->Op == OP_SBFE) {\n auto Sbfe = IROp->C();\n return Sbfe->Width == 1 && Sbfe->lsb == (IR::OpSizeAsBits(Size) - 1) && Sbfe->Src == Src;\n } else {\n return false;\n }\n}\n\ninline bool IsZero(const IROp_Header* IROp) {\n return IROp->Op == OP_CONSTANT && IROp->C()->Constant == 0;\n}\n\nbool ConstrainedRAPass::TryPostRAMerge(Ref LastNode, Ref CodeNode, IROp_Header* IROp) {\n auto LastOp = IR->GetOp(LastNode);\n\n if (IROp->Op == OP_PUSH && LastOp->Op == OP_PUSH) {\n auto SP = PhysicalRegister(CodeNode);\n auto Push = IR->GetOp(CodeNode);\n auto LastPush = IR->GetOp(LastNode);\n\n if (LastOp->Size == IROp->Size && LastPush->ValueSize == Push->ValueSize && SP == PhysicalRegister(LastNode) &&\n SP == PhysicalRegister(IROp->Args[1]) && SP == PhysicalRegister(LastOp->Args[1]) && SP != PhysicalRegister(IROp->Args[0]) &&\n SP != PhysicalRegister(LastOp->Args[0]) && Push->ValueSize >= OpSize::i32Bit) {\n\n IREmit->SetWriteCursorBefore(LastNode);\n IREmit->_PushTwo(IROp->Size, Push->ValueSize, IROp->Args[0], LastOp->Args[0], IROp->Args[1]);\n IREmit->RemovePostRA(CodeNode);\n return true;\n }\n } else if (IROp->Op == OP_POP) {\n auto SP = PhysicalRegister(IROp->Args[0]);\n\n if (LastOp->Op == OP_POP && LastOp->Size == IROp->Size && IROp->Size >= OpSize::i32Bit && SP == PhysicalRegister(LastOp->Args[0])) {\n IREmit->SetWriteCursorBefore(LastNode);\n IREmit->_PopTwo(IROp->Size, IROp->Args[0], LastOp->Args[1], IROp->Args[1]);\n IREmit->RemovePostRA(CodeNode);\n return true;\n }\n } else if ((IROp->Op == OP_DIV || IROp->Op == OP_UDIV) && IROp->Size >= OpSize::i32Bit) {\n // If Upper came from a sign/zero extension, we only need a 64-bit division.\n auto Op = IROp->CW();\n if (!Op->Upper.IsInvalid() && PhysicalRegister(Op->Upper) == PhysicalRegister(LastNode)) {\n if (IROp->Op == OP_DIV ? IsSignext(LastOp, Op->Lower, IROp->Size) : IsZero(LastOp)) {\n Op->Upper.SetInvalid();\n return PhysicalRegister(LastNode) == PhysicalRegister(Op->OutRemainder);\n }\n }\n } else if (IROp->Op == OP_XGETBV && PhysicalRegister(IROp->Args[0]) == PhysicalRegister(LastNode) && LastOp->Op == OP_CONSTANT) {\n // Try to constant fold\n uint64_t ConstantFunction = LastOp->C()->Constant;\n auto Op = IROp->CW();\n if (CPUID->DoesXCRFunctionReportConstantData(ConstantFunction)) {\n const auto Result = CPUID->RunXCRFunction(ConstantFunction);\n IREmit->SetWriteCursorBefore(CodeNode);\n IREmit->_Constant(Result.eax).Node->Reg = PhysicalRegister(Op->OutEAX).Raw;\n IREmit->_Constant(Result.edx).Node->Reg = PhysicalRegister(Op->OutEDX).Raw;\n IREmit->RemovePostRA(CodeNode);\n return false;\n }\n } else if (IROp->Op == OP_CPUID && PhysicalRegister(IROp->Args[0]) == PhysicalRegister(LastNode) && LastOp->Op == OP_CONSTANT) {\n // Try to constant fold. As a limitation of merging only 2 instructions, we\n // can only handle constant functions, not constant leafs. This could be\n // lifted if we generalized at a (significant) complexity cost.\n uint64_t ConstantFunction = LastOp->C()->Constant;\n auto Op = IROp->CW();\n\n const auto SupportsConstant = CPUID->DoesFunctionReportConstantData(ConstantFunction);\n if (SupportsConstant.SupportsConstantFunction == CPUIDEmu::SupportsConstant::CONSTANT &&\n SupportsConstant.NeedsLeaf != CPUIDEmu::NeedsLeafConstant::NEEDSLEAFCONSTANT) {\n const auto Result = CPUID->RunFunction(ConstantFunction, 0 /* leaf */);\n\n IREmit->SetWriteCursorBefore(CodeNode);\n IREmit->_Fence(IR::FenceType::Inst);\n IREmit->_Constant(Result.eax).Node->Reg = PhysicalRegister(Op->OutEAX).Raw;\n IREmit->_Constant(Result.ebx).Node->Reg = PhysicalRegister(Op->OutEBX).Raw;\n IREmit->_Constant(Result.ecx).Node->Reg = PhysicalRegister(Op->OutECX).Raw;\n IREmit->_Constant(Result.edx).Node->Reg = PhysicalRegister(Op->OutEDX).Raw;\n IREmit->RemovePostRA(CodeNode);\n return false;\n }\n }\n\n // Merge moves that are immediately consumed.\n //\n // x86 code inserts such moves to workaround x86's 2-address code. Because\n // arm64 is 3-address code, we can optimize these out.\n //\n // Note we rely on the short-circuiting here.\n if (PhysicalRegister(LastNode) == PhysicalRegister(CodeNode) && KillMove(LastOp, IROp, LastNode, CodeNode)) {\n LOGMAN_THROW_A_FMT(!PhysicalRegister(CodeNode).IsInvalid(), \"invariant\");\n\n int NumArgs = IR::GetRAArgs(IROp->Op);\n for (int s = 0; s < NumArgs; ++s) {\n if (IROp->Args[s].IsImmediate() && PhysicalRegister(IROp->Args[s]) == PhysicalRegister(LastNode)) {\n IROp->Args[s].SetImmediate(PhysicalRegister(LastOp->Args[0]).Raw);\n }\n }\n\n return true;\n }\n\n return false;\n}\n\nvoid ConstrainedRAPass::Run(IREmitter* IREmit_) {\n FEXCORE_PROFILE_SCOPED(\"PassManager::RA\");\n\n IREmit = IREmit_;\n auto IR_ = IREmit->ViewIR();\n IR = &IR_;\n\n PreferredReg.resize(IR->GetSSACount(), PhysicalRegister::Invalid());\n SSAToReg.resize(IR->GetSSACount(), PhysicalRegister::Invalid());\n Seen.resize(IR->GetSSACount(), false);\n\n for (auto [BlockNode, BlockHeader] : IR->GetBlocks()) {\n // Spilling is local, so reset this per-block\n AnySpilled = false;\n\n // At the start of each block, all registers are available.\n for (auto& Class : Classes) {\n Class.Available = (1u << Class.Count) - 1;\n }\n\n auto BlockIROp = BlockHeader->CW();\n\n // Backwards pass: analyze kill bits and SRA affinities\n {\n // Reverse iteration is not yet working with the iterators\n // We grab these nodes this way so we can iterate easily\n auto CodeBegin = IR->at(BlockIROp->Begin);\n auto CodeLast = IR->at(BlockIROp->Last);\n\n while (1) {\n auto [CodeNode, IROp] = CodeLast();\n // End of iteration gunk\n\n // Record preferred registers for SRA. We also record the Node accessing\n // each register, used below. Since we initialized Class->Available,\n // RegToSSA is otherwise undefined so we can stash our temps there.\n if (auto Node = DecodeSRANode(IROp, CodeNode); Node != nullptr) {\n auto Reg = DecodeSRAReg(IROp, CodeNode);\n\n PreferredReg[IR->GetID(Node).Value] = Reg;\n GetClass(Reg)->RegToSSA[Reg.Reg] = CodeNode;\n }\n\n // Coalescing an SRA store is equivalent to hoisting the store,\n // implying write-after-write and read-after-write hazards. We can only\n // coalesce if there is no intervening load/store.\n //\n // Since we're walking backwards, RegToSSA tracks\n // the first load/store after CodeNode. That first instruction is the\n // store in question iff there is no intervening load/store.\n //\n // Reset PreferredReg if that is not the case, ensuring SRA correctness.\n if (auto Reg = PreferredReg[IR->GetID(CodeNode).Value]; !Reg.IsInvalid()) {\n auto Node = GetClass(Reg)->RegToSSA[Reg.Reg];\n IROp_Header* Header = IR->GetOp(Node);\n\n if (CodeNode != DecodeSRANode(Header, Node)) {\n PreferredReg[IR->GetID(CodeNode).Value] = PhysicalRegister::Invalid();\n }\n }\n\n const int NumArgs = IR::GetRAArgs(IROp->Op);\n for (int i = NumArgs - 1; i >= 0; --i) {\n const auto& Arg = IROp->Args[i];\n if (!Arg.IsInvalid()) {\n const uint32_t Index = Arg.ID().Value;\n if (!Seen[Index]) {\n Seen[Index] = true;\n IROp->Args[i].SetKill();\n }\n }\n }\n\n // Rest is iteration gunk\n if (CodeLast == CodeBegin) {\n break;\n }\n --CodeLast;\n }\n }\n\n // NextUses currently contains first use distances, the exact initialization\n // assumed by the forward pass. Do not reset it.\n\n // Last nontrivial instruction, for merging as we go.\n Ref LastNode = nullptr;\n\n // Forward pass: Assign registers, spilling & optimizing as we go.\n for (auto [CodeNode, IROp] : IR->GetCode(BlockNode)) {\n bool AnySpilledBeforeThisInstruction = AnySpilled;\n\n // These do not read or write registers, and must be skipped for merging.\n // Since we'd be doing this check anyway for merging, do the check now so\n // we can skip the rest of the logic too.\n if (IROp->Op == OP_GUESTOPCODE || IROp->Op == OP_INLINECONSTANT) {\n continue;\n }\n\n // Static registers must be consistent at SRA load/store. Evict to ensure.\n if (auto Node = DecodeSRANode(IROp, CodeNode); Node != nullptr) {\n auto Reg = DecodeSRAReg(IROp, CodeNode);\n RegisterClassData* Class = &Classes[Reg.Class];\n\n if (!(Class->Available & (1u << Reg.Reg))) {\n Ref Old = Class->RegToSSA[Reg.Reg];\n\n if (Old != Node) {\n // Before inserting instructions, we need to set the cursor and\n // reset LastNode so we don't merge across an inserted copy.\n // Otherwise, we would erroneously miss the copy when determining if\n // we can merge, and end up unsoundly merging a mov+xchg sequence.\n IREmit->SetWriteCursorBefore(CodeNode);\n LastNode = nullptr;\n\n Ref Copy;\n\n if (Reg.AsRegClass() == RegClass::FPRFixed) {\n IROp_Header* Header = IR->GetOp(Old);\n Copy = IREmit->_VMov(Header->Size, OrderedNodeWrapper::FromImmediate(Reg.Raw));\n } else {\n Copy = IREmit->_Copy(OrderedNodeWrapper::FromImmediate(Reg.Raw));\n }\n\n FreeReg(Reg);\n AssignReg(IR->GetOp(Copy), BlockIROp, Copy, IROp);\n RemapReg(Old, PhysicalRegister(Copy));\n }\n }\n }\n\n // Fill all sources that are not already in the register file.\n //\n // This happens before freeing killed sources, since we need all sources in\n // the register file simultaneously.\n //\n // Also update next-use info, again only relevant if we've spilled.\n int NumArgs = IR::GetRAArgs(IROp->Op);\n\n if (AnySpilledBeforeThisInstruction) {\n for (int s = 0; s < NumArgs; ++s) {\n auto V = IROp->Args[s];\n V.ClearKill();\n\n if (!IsValidArg(V)) {\n continue;\n }\n\n Ref Old = IR->GetNode(V);\n\n SourceIndex--;\n LOGMAN_THROW_A_FMT(SourceIndex >= 0, \"Consistent source count\");\n NextUses[V.ID().Value] = SourcesNextUses[SourceIndex];\n\n if (!IsInRegisterFile(Old)) {\n IREmit->SetWriteCursorBefore(CodeNode);\n LastNode = nullptr;\n\n Ref Fill = InsertFill(Old);\n\n AssignReg(IR->GetOp(Fill), BlockIROp, Fill, IROp);\n RemapReg(Old, PhysicalRegister(Fill));\n }\n }\n }\n\n for (int s = 0; s < NumArgs; ++s) {\n if (IROp->Args[s].IsInvalid()) {\n continue;\n }\n\n bool Kill = IROp->Args[s].HasKill();\n IROp->Args[s].ClearKill();\n Ref Node = IR->GetNode(IROp->Args[s]);\n auto ID = IR->GetID(Node).Value;\n auto Reg = SSAToReg[ID];\n\n if (!Reg.IsInvalid()) {\n if (Kill) {\n LOGMAN_THROW_A_FMT(IsInRegisterFile(Node), \"sources in file\");\n FreeReg(Reg);\n }\n\n IROp->Args[s].SetImmediate(Reg.Raw);\n }\n }\n\n // Assign destinations.\n if (GetHasDest(IROp->Op) && PhysicalRegister(CodeNode).IsInvalid()) {\n AssignReg(IROp, BlockIROp, CodeNode, IROp);\n }\n\n if (IsTrivial(CodeNode, IROp)) {\n // Delete instructions that only exist for RA\n IREmit->RemovePostRA(CodeNode);\n } else if (LastNode && TryPostRAMerge(LastNode, CodeNode, IROp)) {\n // Merge adjacent instructions\n IREmit->RemovePostRA(LastNode);\n LastNode = nullptr;\n } else {\n LastNode = CodeNode;\n }\n }\n\n if (AnySpilled) {\n LOGMAN_THROW_A_FMT(SourceIndex == 0, \"Consistent source count in block\");\n }\n }\n\n PreferredReg.clear();\n SSAToReg.clear();\n SpillSlots.clear();\n NextUses.clear();\n Seen.clear();\n\n IR->GetHeader()->PostRA = true;\n}\n\nfextl::unique_ptr CreateRegisterAllocationPass(const FEXCore::CPUIDEmu* CPUID) {\n return fextl::make_unique(CPUID);\n}\n} // namespace FEXCore::IR\n" }, { "path": "FEXCore/Source/Interface/IR/Passes/RegisterAllocationPass.h", "content": "// SPDX-License-Identifier: MIT\n/*\n$info$\ntags: ir|opts\n$end_info$\n*/\n\n#pragma once\n#include \"Interface/IR/PassManager.h\"\n\n#include \n#include \n\nnamespace FEXCore::IR {\nenum class RegClass : uint32_t;\n\nclass RegisterAllocationPass : public FEXCore::IR::Pass {\npublic:\n virtual void AddRegisters(RegClass Class, uint32_t RegisterCount) = 0;\n\n // Number of GPRs usable for pairs at start of GPR set. Must be even.\n uint32_t PairRegs {};\n};\n\n} // namespace FEXCore::IR\n" }, { "path": "FEXCore/Source/Interface/IR/Passes/x87StackOptimizationPass.cpp", "content": "// SPDX-License-Identifier: MIT\n#include \"FEXCore/Utils/LogManager.h\"\n#include \"Interface/Core/Interpreter/Fallbacks/FallbackOpHandler.h\"\n#include \"Interface/IR/IR.h\"\n#include \"Interface/IR/IREmitter.h\"\n#include \"Interface/IR/PassManager.h\"\n#include \"FEXCore/IR/IR.h\"\n#include \"FEXCore/Utils/Profiler.h\"\n#include \"FEXCore/Core/HostFeatures.h\"\n#include \"Interface/Core/Addressing.h\"\n\n#include \n#include \n#include \n#include \n\n// This file adds a pass to process X87 stack instructions.\n// These instructions are marked in IR.json with `X87: true` and are generated\n// by X87 guest instructions.\n// The way is works is that there's a virtual stack `StackData`, where we load and store\n// and apply the operations in a block of code. Once the block finishes, we emit the necessary operations\n// that we recorded onto the virtual stack. This allows us to save a lot of code movement\n// to and from stack registers, top management and valid flags. It also allows us to\n// perform memcpy optimizations like the one performed in STORESTACKMEM.\n//\n// By default we run on the fast path - i.e. we assume all values are in the stack and we have a complete\n// stack overview. However, if we encounter a value that's not in the virtual stack - maybe it was added\n// to the stack in a previous block, we move onto the slow path which loads and stores values to the stack\n// registers.\n// Once in a slow path, we won't return to the fast pass until the beginning of the following block.\n\nnamespace FEXCore::IR {\n\n// FIXME(pmatos): copy from OpcodeDispatcher.h\ninline uint32_t MMBaseOffset() {\n return static_cast(offsetof(Core::CPUState, mm[0][0]));\n}\n\n// Similar helper to the one in OpcodeDispatcher.h except we do not\n// need to handle flags, etc.\ntemplate\nvoid DeriveOp(Ref& RefV, IROps NewOp, IREmitter::IRPair Expr) {\n Expr.first->Header.Op = NewOp;\n RefV = Expr;\n}\n\nenum class StackSlot { UNUSED, INVALID, VALID };\n// FixedSizeStack is a model of the x87 Stack where each element in this\n// fixed size stack lives at an offset from top. The top of the stack is at\n// index 0.\ntemplate\nclass FixedSizeStack {\npublic:\n struct StackSlotEntry final {\n StackSlot Type;\n T Value;\n };\n\n static constexpr uint8_t size = 8;\n\n // Real top as an offset from stored top value (or the one at the beginning of the block)\n // For example, if we start and push a value to our simulated stack, because we don't\n // update top straight away the TopOffset is 1.\n // If SlowPath is true, then TopOffset is always zero.\n int8_t TopOffset = 0;\n\n FixedSizeStack()\n : buffer(FixedSizeStack::size, {StackSlot::UNUSED, T::Invalid}) {}\n\n void push(const T& Value) {\n rotate();\n buffer.front() = {StackSlot::VALID, Value};\n }\n\n // Rotate the elements with the direction controlled by Right\n void rotate(bool Right = true) {\n if (Right) {\n std::rotate(buffer.begin(), buffer.end() - 1, buffer.end());\n TopOffset++;\n } else {\n std::rotate(buffer.begin(), buffer.begin() + 1, buffer.end());\n TopOffset--;\n }\n }\n\n void pop() {\n buffer.front() = {StackSlot::INVALID, T::Invalid};\n rotate(false);\n }\n\n const StackSlotEntry& top(size_t Offset = 0) const {\n return buffer[Offset];\n }\n\n void setTop(T Value, size_t Offset = 0) {\n buffer[Offset] = {StackSlot::VALID, Value};\n }\n\n bool isValid(size_t Offset) const {\n return buffer[Offset].first;\n }\n\n void clear() {\n for (auto& Elem : buffer) {\n Elem = {StackSlot::UNUSED, T::Invalid};\n }\n TopOffset = 0;\n }\n\n void dump() const {\n LogMan::Msg::DFmt(\"-- Stack\");\n\n for (size_t i = 0; i < 8; i++) {\n const auto& [Valid, Element] = buffer[i];\n if (Valid == StackSlot::VALID) {\n LogMan::Msg::DFmt(\"| ST{}: 0x{:x}\", i, (uintptr_t)(Element.StackDataNode));\n } else if (Valid == StackSlot::INVALID) {\n LogMan::Msg::DFmt(\"| ST{}: INVALID\", i);\n }\n }\n LogMan::Msg::DFmt(\"--\");\n }\n\n void setTagInvalid(size_t Index) {\n buffer[Index].Type = StackSlot::INVALID;\n }\n\n // Returns a mask to set in AbridgedTagWord\n uint8_t getValidMask() {\n uint8_t Mask = 0;\n for (size_t i = 0; i < buffer.size(); i++) {\n if (buffer[i].Type == StackSlot::VALID) {\n Mask |= 1U << i;\n }\n }\n return Mask;\n }\n\n // Returns a mask to set in AbridgedTagWord\n uint8_t getInvalidMask() {\n uint8_t Mask = 0;\n for (size_t i = 0; i < buffer.size(); i++) {\n if (buffer[i].Type == StackSlot::INVALID) {\n Mask |= 1U << i;\n }\n }\n return Mask;\n }\n\nprivate:\n fextl::vector buffer;\n};\n\nclass X87StackOptimization final : public Pass {\npublic:\n X87StackOptimization(const FEXCore::HostFeatures& Features, OpSize GPROpSize)\n : Features(Features)\n , GPROpSize(GPROpSize) {\n FEX_CONFIG_OPT(ReducedPrecision, X87REDUCEDPRECISION);\n ReducedPrecisionMode = ReducedPrecision;\n }\n void Run(IREmitter* Emit) override;\n\nprivate:\n const FEXCore::HostFeatures& Features;\n const OpSize GPROpSize;\n bool ReducedPrecisionMode;\n FEX_CONFIG_OPT(DisableVixlIndirectCalls, DISABLE_VIXL_INDIRECT_RUNTIME_CALLS);\n\n // Helpers\n Ref RotateRight8(uint32_t V, Ref Amount);\n\n void F80SplitStore_Helper(const IROp_StoreStackMem* Op, Ref StackNode, Ref AddrNode, Ref Offset, OpSize Align, MemOffsetType OffsetType,\n uint8_t OffsetScale) {\n IREmit->_StoreMemFPR(OpSize::i64Bit, StackNode, AddrNode, Offset, Align, OffsetType, OffsetScale);\n auto Upper = IREmit->_VExtractToGPR(OpSize::i128Bit, OpSize::i64Bit, StackNode, 1);\n\n // Store the Upper part of the register (the remaining 2 bytes) into memory.\n AddressMode A {.Base = AddrNode,\n .Index = Op->Offset.IsInvalid() ? nullptr : Offset,\n .Offset = 8,\n .IndexType = MemOffsetType::SXTX,\n .IndexScale = OffsetScale,\n .AddrSize = OpSize::i64Bit};\n A = SelectAddressMode(IREmit, A, GPROpSize, Features.SupportsTSOImm9, false, false, OpSize::i16Bit);\n IREmit->_StoreMemGPR(OpSize::i16Bit, Upper, A.Base, A.Index, OpSize::i64Bit, MemOffsetType::SXTX, A.IndexScale);\n }\n\n void Store80BitToMem(const IROp_StoreStackMem* Op, Ref StackNode, Ref AddrNode, Ref Offset, OpSize Align, MemOffsetType OffsetType,\n uint8_t OffsetScale) {\n if (Features.SupportsSVE128 || Features.SupportsSVE256) {\n AddressMode A {.Base = AddrNode,\n .Index = Op->Offset.IsInvalid() ? nullptr : Offset,\n .IndexType = MemOffsetType::SXTX,\n .IndexScale = OffsetScale,\n .AddrSize = OpSize::i64Bit};\n AddrNode = LoadEffectiveAddress(IREmit, A, GPROpSize, false);\n IREmit->_StoreMemX87SVEOptPredicate(OpSize::i128Bit, OpSize::i16Bit, StackNode, AddrNode);\n } else {\n F80SplitStore_Helper(Op, StackNode, AddrNode, Offset, Align, OffsetType, OffsetScale);\n }\n }\n\n void StoreStackMem_Helper(const IROp_StoreStackMem* Op, Ref StackNode) {\n LOGMAN_THROW_A_FMT(!ReducedPrecisionMode, \"Full precision mode expected.\");\n\n Ref AddrNode = IR->GetNode(Op->Addr);\n Ref Offset = IR->GetNode(Op->Offset);\n OpSize Align = Op->Align;\n MemOffsetType OffsetType = Op->OffsetType;\n uint8_t OffsetScale = Op->OffsetScale;\n\n // Normal Precision Mode\n switch (Op->StoreSize) {\n case OpSize::i32Bit:\n case OpSize::i64Bit: {\n StackNode = IREmit->_F80CVT(Op->StoreSize, StackNode);\n IREmit->_StoreMemFPR(Op->StoreSize, StackNode, AddrNode, Offset, Align, OffsetType, OffsetScale);\n break;\n }\n\n case OpSize::f80Bit: {\n Store80BitToMem(Op, StackNode, AddrNode, Offset, Align, OffsetType, OffsetScale);\n break;\n }\n default: ERROR_AND_DIE_FMT(\"Unsupported x87 size\");\n }\n }\n\n // Performs a store to memory from a value the stack passed in as StackNode.\n // This is the version dealing with the reduced precision case.\n void StoreStackMem_Reduced_Helper(const IROp_StoreStackMem* Op, Ref StackNode) {\n LOGMAN_THROW_A_FMT(ReducedPrecisionMode, \"Reduced precision mode expected.\");\n\n Ref AddrNode = IR->GetNode(Op->Addr);\n Ref Offset = IR->GetNode(Op->Offset);\n OpSize Align = Op->Align;\n MemOffsetType OffsetType = Op->OffsetType;\n uint8_t OffsetScale = Op->OffsetScale;\n\n switch (Op->StoreSize) {\n case OpSize::i32Bit: {\n StackNode = IREmit->_Float_FToF(OpSize::i32Bit, OpSize::i64Bit, StackNode);\n [[fallthrough]];\n }\n case OpSize::i64Bit: {\n IREmit->_StoreMemFPR(Op->StoreSize, StackNode, AddrNode, Offset, Align, OffsetType, OffsetScale);\n break;\n }\n\n case OpSize::f80Bit: {\n StackNode = IREmit->_F80CVTTo(StackNode, OpSize::i64Bit);\n Store80BitToMem(Op, StackNode, AddrNode, Offset, Align, OffsetType, OffsetScale);\n break;\n }\n default: ERROR_AND_DIE_FMT(\"Unsupported x87 size\");\n }\n }\n\n // Handles a Unary operation.\n // Takes the op we are handling, the Node for the reduced precision case and the node for the normal case.\n // Depending on the type of Op64, we might need to pass a couple of extra constant arguments, this happens\n // when VFOp64 is true.\n void HandleUnop(IROps Op64, bool VFOp64, IROps Op80);\n void HandleBinopValue(IROps Op64, bool VFOp64, IROps Op80, uint8_t DestStackOffset, bool MarkDestValid, uint8_t StackOffset,\n Ref ValueNode, bool Reverse = false);\n void HandleBinopStack(IROps Op64, bool VFOp64, IROps Op80, uint8_t DestStackOffset, uint8_t StackOffset1, uint8_t StackOffset2,\n bool Reverse = false);\n\n // Top Management Helpers\n /// Set the valid tag for Value as valid (if Valid is true), or invalid (if Valid is false).\n void SetX87ValidTag(uint8_t Offset, bool Valid);\n // Generates slow code to load/store a value from an offset from the top of the stack\n Ref LoadStackValueAtOffset_Slow(uint8_t Offset = 0);\n void StoreStackValueAtOffset_Slow(Ref Value, uint8_t Offset = 0, bool SetValid = true);\n // Update Top value in slow path for a pop\n void UpdateTopForPop_Slow();\n void UpdateTopForPush_Slow();\n // Synchronizes the current simulated stack with the actual values.\n // Returns a new value for Top, that's synchronized between the simulated stack\n // and the actual FPU stack.\n Ref SynchronizeStackValues();\n // Moves us from the fast to the slow path if ShouldMigrate is true.\n void MigrateToSlowPathIf(bool ShouldMigrate);\n // Top Cache Management\n Ref GetTopWithCache_Slow();\n Ref GetOffsetTopWithCache_Slow(uint8_t Offset, bool Reverse = false);\n Ref GetOffsetTopAddressWithCache_Slow(uint8_t Offset);\n void SetTopWithCache_Slow(Ref Value);\n Ref GetX87ValidTag_Slow(uint8_t Offset);\n // Resets fields to initial values\n void Reset();\n\n struct StackMemberInfo {\n StackMemberInfo() = delete;\n StackMemberInfo(Ref Data)\n : StackDataNode(Data) {}\n StackMemberInfo(Ref Data, Ref Source, OpSize Size)\n : StackDataNode(Data)\n , Source({Size, Source}) {}\n Ref StackDataNode {}; // Reference to the data in the Stack.\n // This is the source data node in the stack format, possibly converted to 64/80 bits.\n struct StackMemberData final {\n OpSize Size;\n Ref Node;\n };\n\n static const StackMemberInfo Invalid;\n\n // Tuple is only valid if we have information about the Source of the Stack Data Node.\n // In it's valid then OpSize is the original source size and Ref is the original source node.\n std::optional Source {};\n };\n\n // StackData, TopCache need to be always properly set to ensure\n // they reflect the current state of the FPU. This sync only makes sense while\n // taking the fast path. Once in the slow path, these don't make sense anymore\n // and we are syncing everything.\n\n // Index on vector is offset to top value at start of block\n // If slow path is true, then StackData is always empty.\n FixedSizeStack StackData;\n\n void InvalidateCaches();\n void InvalidateCachedRegs();\n\n // Path Migration helper management\n std::optional MigrateToSlowPath_IfInvalid(uint8_t Offset = 0);\n Ref LoadStackValue(uint8_t Offset = 0);\n void StoreStackValue(Ref Value, uint8_t Offset = 0, bool SetValid = false);\n void StackPop();\n\n // Cache for Constants\n // ConstantPoll[i] has IREmit->_Constant(i);\n std::array ConstantPool {};\n Ref GetConstant(ssize_t Offset);\n\n // Cached value for Top\n // If slowpath is false, then TopCache is nullptr.\n bool FlushTopPending = false;\n std::array FlushValuesPending {};\n bool FlushValidPending = false;\n void FlushCachedRegs();\n\n Ref GetFTW();\n\n Ref FTWCached {};\n std::array TopOffsetCache {};\n std::array TopOffsetAddressCache {};\n std::array TopValueCache {};\n std::array TopValidCache {};\n\n // Are we on the slow path?\n // Once we enter the slow path, we never come out.\n // This just simplifies the code atm. If there's a need to return to the fast path in the future\n // we can implement that but I would expect that there would be very few cases where that's necessary.\n // On the slow path TopCache is always the last obtained version of top.\n // TopOffset is ignored\n bool SlowPath = false;\n // Keeping IREmitter not to pass arguments around\n IREmitter* IREmit = nullptr;\n IRListView* IR = nullptr;\n};\n\ninline const X87StackOptimization::StackMemberInfo X87StackOptimization::StackMemberInfo::Invalid {nullptr};\n\ninline void X87StackOptimization::InvalidateCaches() {\n InvalidateCachedRegs();\n ConstantPool.fill(nullptr);\n}\n\ninline void X87StackOptimization::InvalidateCachedRegs() {\n FlushCachedRegs();\n FTWCached = {};\n TopOffsetCache.fill(nullptr);\n TopOffsetAddressCache.fill(nullptr);\n TopValueCache.fill(nullptr);\n TopValidCache.fill(StackSlot::UNUSED);\n}\n\ninline void X87StackOptimization::Reset() {\n SlowPath = false;\n StackData.clear();\n InvalidateCaches();\n}\n\ninline Ref X87StackOptimization::GetConstant(ssize_t Offset) {\n if (Offset < 0 || Offset >= X87StackOptimization::ConstantPool.size()) {\n // not dealt by pool\n return IREmit->_Constant(Offset);\n }\n if (ConstantPool[Offset] == nullptr) {\n\n ConstantPool[Offset] = IREmit->_Constant(Offset);\n }\n return ConstantPool[Offset];\n}\n\ninline void X87StackOptimization::MigrateToSlowPathIf(bool ShouldMigrate) {\n if (ShouldMigrate && !SlowPath) {\n SynchronizeStackValues();\n StackData.clear();\n SlowPath = true;\n }\n}\n\ninline Ref X87StackOptimization::GetTopWithCache_Slow() {\n if (!TopOffsetCache[0]) {\n TopOffsetCache[0] = IREmit->_LoadContextGPR(OpSize::i8Bit, offsetof(FEXCore::Core::CPUState, flags) + FEXCore::X86State::X87FLAG_TOP_LOC);\n }\n return TopOffsetCache[0];\n}\n\ninline Ref X87StackOptimization::GetOffsetTopWithCache_Slow(uint8_t Offset, bool Reverse) {\n if (Reverse) {\n Offset = 8 - Offset;\n }\n\n Offset &= 7;\n\n if (TopOffsetCache[Offset]) {\n return TopOffsetCache[Offset];\n }\n\n auto* OffsetTop = GetTopWithCache_Slow();\n if (Offset != 0) {\n OffsetTop = IREmit->_And(OpSize::i32Bit, IREmit->Add(OpSize::i32Bit, OffsetTop, Offset), GetConstant(7));\n // GetTopWithCache_Slow already sets the cache so we don't need to set it here for offset == 0\n TopOffsetCache[Offset] = OffsetTop;\n }\n\n return OffsetTop;\n}\n\ninline Ref X87StackOptimization::GetOffsetTopAddressWithCache_Slow(uint8_t Offset) {\n if (TopOffsetAddressCache[Offset]) {\n return TopOffsetAddressCache[Offset];\n }\n\n Ref OffsetRef = GetOffsetTopWithCache_Slow(Offset);\n TopOffsetAddressCache[Offset] = IREmit->_FormContextAddress(OpSize::i64Bit, OffsetRef, 16);\n\n return TopOffsetAddressCache[Offset];\n}\n\ninline void X87StackOptimization::SetTopWithCache_Slow(Ref Value) {\n InvalidateCachedRegs();\n TopOffsetCache[0] = Value;\n FlushTopPending = true;\n}\n\ninline Ref X87StackOptimization::GetFTW() {\n if (!FTWCached) {\n FTWCached = IREmit->_LoadContextGPR(OpSize::i8Bit, offsetof(FEXCore::Core::CPUState, AbridgedFTW));\n }\n return FTWCached;\n}\n\ninline void X87StackOptimization::SetX87ValidTag(uint8_t Offset, bool Valid) {\n TopValidCache[Offset] = Valid ? StackSlot::VALID : StackSlot::INVALID;\n FlushValidPending = true;\n}\n\ninline Ref X87StackOptimization::GetX87ValidTag_Slow(uint8_t Offset) {\n switch (TopValidCache[Offset]) {\n case StackSlot::UNUSED:\n return IREmit->_And(OpSize::i32Bit, IREmit->_Lshr(OpSize::i32Bit, GetFTW(), GetOffsetTopWithCache_Slow(Offset)), GetConstant(1));\n case StackSlot::INVALID: return GetConstant(0);\n case StackSlot::VALID: return GetConstant(1);\n }\n}\n\ninline Ref X87StackOptimization::LoadStackValueAtOffset_Slow(uint8_t Offset) {\n OrderedNode* TopOffsetAddress = GetOffsetTopAddressWithCache_Slow(Offset);\n auto Size = ReducedPrecisionMode ? OpSize::i64Bit : OpSize::i128Bit;\n if (!TopValueCache[Offset]) {\n TopValueCache[Offset] = IREmit->_LoadMemFPR(Size, TopOffsetAddress, IREmit->_InlineConstant(MMBaseOffset()), Size, MemOffsetType::SXTX, 1);\n }\n return TopValueCache[Offset];\n}\n\ninline void X87StackOptimization::StoreStackValueAtOffset_Slow(Ref Value, uint8_t Offset, bool SetValid) {\n TopValueCache[Offset] = Value;\n FlushValuesPending[Offset] = true;\n // mark it valid\n // In some cases we might already know it has been previously set as valid so we don't need to do it again\n if (SetValid) {\n SetX87ValidTag(Offset, true);\n }\n}\n\ninline Ref X87StackOptimization::RotateRight8(uint32_t V, Ref Amount) {\n return IREmit->_Lshr(OpSize::i32Bit, GetConstant(V | (V << 8)), Amount);\n}\n\ninline std::optional X87StackOptimization::MigrateToSlowPath_IfInvalid(uint8_t Offset) {\n const auto& [Valid, StackMember] = StackData.top(Offset);\n MigrateToSlowPathIf(Valid != StackSlot::VALID);\n if (Valid == StackSlot::VALID) {\n return StackMember;\n }\n return {};\n}\n\ninline Ref X87StackOptimization::LoadStackValue(uint8_t Offset) {\n const auto& StackValue = MigrateToSlowPath_IfInvalid(Offset);\n return SlowPath ? LoadStackValueAtOffset_Slow(Offset) : StackValue->StackDataNode;\n}\n\ninline void X87StackOptimization::StoreStackValue(Ref Value, uint8_t Offset, bool SetValid) {\n if (SlowPath) {\n StoreStackValueAtOffset_Slow(Value, Offset, SetValid);\n } else {\n StackData.setTop(StackMemberInfo {Value}, Offset);\n }\n}\n\ninline void X87StackOptimization::StackPop() {\n if (SlowPath) {\n UpdateTopForPop_Slow();\n } else {\n StackData.pop();\n }\n}\n\n\nvoid X87StackOptimization::HandleUnop(IROps Op64, bool VFOp64, IROps Op80) {\n Ref St0 = LoadStackValue();\n Ref Value {};\n\n if (ReducedPrecisionMode) {\n if (VFOp64) {\n DeriveOp(Value, Op64, IREmit->_VFSqrt(OpSize::i64Bit, OpSize::i64Bit, St0));\n } else {\n DeriveOp(Value, Op64, IREmit->_F64SIN(St0));\n }\n } else {\n DeriveOp(Value, Op80, IREmit->_F80SQRT(St0));\n }\n\n StoreStackValue(Value);\n}\n\n\nvoid X87StackOptimization::HandleBinopValue(IROps Op64, bool VFOp64, IROps Op80, uint8_t DestStackOffset, bool MarkDestValid,\n uint8_t StackOffset, Ref ValueNode, bool Reverse) {\n LOGMAN_THROW_A_FMT(!Reverse || VFOp64, \"There are no reverse operations using non VFOp64 ops\");\n auto StackNode = LoadStackValue(StackOffset);\n\n Ref Node = {};\n if (ReducedPrecisionMode) {\n if (Reverse) {\n DeriveOp(Node, Op64, IREmit->_VFAdd(OpSize::i64Bit, OpSize::i64Bit, ValueNode, StackNode));\n } else {\n if (VFOp64) {\n DeriveOp(Node, Op64, IREmit->_VFAdd(OpSize::i64Bit, OpSize::i64Bit, StackNode, ValueNode));\n } else {\n DeriveOp(Node, Op64, IREmit->_F64FPREM(StackNode, ValueNode));\n }\n }\n } else {\n if (Reverse) {\n DeriveOp(Node, Op80, IREmit->_F80Add(ValueNode, StackNode));\n } else {\n DeriveOp(Node, Op80, IREmit->_F80Add(StackNode, ValueNode));\n }\n }\n\n StoreStackValue(Node, DestStackOffset, MarkDestValid && StackOffset != DestStackOffset);\n}\n\nvoid X87StackOptimization::HandleBinopStack(IROps Op64, bool VFOp64, IROps Op80, uint8_t DestStackOffset, uint8_t StackOffset1,\n uint8_t StackOffset2, bool Reverse) {\n auto StackNode = LoadStackValue(StackOffset2);\n HandleBinopValue(Op64, VFOp64, Op80, DestStackOffset, StackOffset2 != DestStackOffset, StackOffset1, StackNode, Reverse);\n}\n\ninline void X87StackOptimization::UpdateTopForPop_Slow() {\n // Pop the top of the x87 stack\n GetOffsetTopWithCache_Slow(1);\n std::rotate(TopOffsetCache.begin(), std::next(TopOffsetCache.begin()), TopOffsetCache.end());\n std::rotate(TopOffsetAddressCache.begin(), std::next(TopOffsetAddressCache.begin()), TopOffsetAddressCache.end());\n std::rotate(TopValueCache.begin(), std::next(TopValueCache.begin()), TopValueCache.end());\n std::rotate(FlushValuesPending.begin(), std::next(FlushValuesPending.begin()), FlushValuesPending.end());\n std::rotate(TopValidCache.begin(), std::next(TopValidCache.begin()), TopValidCache.end());\n FlushTopPending = true;\n}\n\ninline void X87StackOptimization::UpdateTopForPush_Slow() {\n // Pop the top of the x87 stack\n GetOffsetTopWithCache_Slow(1, true);\n std::rotate(TopOffsetCache.begin(), std::prev(TopOffsetCache.end()), TopOffsetCache.end());\n std::rotate(TopOffsetAddressCache.begin(), std::prev(TopOffsetAddressCache.end()), TopOffsetAddressCache.end());\n std::rotate(TopValueCache.begin(), std::prev(TopValueCache.end()), TopValueCache.end());\n std::rotate(FlushValuesPending.begin(), std::prev(FlushValuesPending.end()), FlushValuesPending.end());\n std::rotate(TopValidCache.begin(), std::prev(TopValidCache.end()), TopValidCache.end());\n FlushTopPending = true;\n}\n\nvoid X87StackOptimization::FlushCachedRegs() {\n if (FlushTopPending) {\n IREmit->_StoreContextGPR(OpSize::i8Bit, TopOffsetCache[0], offsetof(FEXCore::Core::CPUState, flags) + FEXCore::X86State::X87FLAG_TOP_LOC);\n FlushTopPending = false;\n }\n\n auto Size = ReducedPrecisionMode ? OpSize::i64Bit : OpSize::i128Bit;\n for (size_t i = 0; i < FlushValuesPending.size(); i++) {\n if (FlushValuesPending[i]) {\n OrderedNode* TopOffsetAddress = GetOffsetTopAddressWithCache_Slow(i);\n IREmit->_StoreMemFPR(Size, TopValueCache[i], TopOffsetAddress, IREmit->_InlineConstant(MMBaseOffset()), Size, MemOffsetType::SXTX, 1);\n // store\n FlushValuesPending[i] = false;\n }\n }\n\n if (FlushValidPending) {\n uint8_t ValidMask = 0;\n uint8_t InvalidMask = 0;\n for (auto It = TopValidCache.rbegin(); It != TopValidCache.rend(); It++) {\n ValidMask <<= 1;\n InvalidMask <<= 1;\n if (*It == StackSlot::VALID) {\n ValidMask |= 1;\n } else if (*It == StackSlot::INVALID) {\n InvalidMask |= 1;\n }\n }\n\n if (ValidMask || InvalidMask) {\n Ref NewFTW = [&]() {\n if (ValidMask == 0xff || InvalidMask == 0xff) {\n // If InvalidMask == 0xff then ValidMask = 0\n return GetConstant(ValidMask);\n } else {\n Ref NewFTW = GetFTW();\n Ref RotAmount {};\n if (std::popcount(ValidMask) == 1) {\n uint8_t BitIdx = std::countr_zero(ValidMask);\n Ref RegMask = IREmit->_Lshl(OpSize::i32Bit, GetConstant(1), GetOffsetTopWithCache_Slow(BitIdx));\n NewFTW = IREmit->_Or(OpSize::i32Bit, NewFTW, RegMask);\n } else if (ValidMask) {\n RotAmount = IREmit->_Sub(OpSize::i32Bit, GetConstant(8), GetTopWithCache_Slow());\n // perform a rotate right on mask by top\n NewFTW = IREmit->_Or(OpSize::i32Bit, NewFTW, RotateRight8(ValidMask, RotAmount));\n }\n\n if (std::popcount(InvalidMask) == 1) {\n uint8_t BitIdx = std::countr_zero(InvalidMask);\n Ref RegMask = IREmit->_Lshl(OpSize::i32Bit, GetConstant(1), GetOffsetTopWithCache_Slow(BitIdx));\n NewFTW = IREmit->_Andn(OpSize::i32Bit, NewFTW, RegMask);\n } else if (InvalidMask) {\n if (!RotAmount) {\n RotAmount = IREmit->_Sub(OpSize::i32Bit, GetConstant(8), GetTopWithCache_Slow());\n }\n NewFTW = IREmit->_Andn(OpSize::i32Bit, NewFTW, RotateRight8(InvalidMask, RotAmount));\n }\n return NewFTW;\n }\n }();\n\n IREmit->_StoreContextGPR(OpSize::i8Bit, NewFTW, offsetof(FEXCore::Core::CPUState, AbridgedFTW));\n FTWCached = NewFTW;\n }\n\n FlushValidPending = false;\n }\n}\n\n// We synchronize stack values in a few occasions but one of the most important of those,\n// is when we move from fast to a slow path and need to make sure that the context is properly\n// written.\nRef X87StackOptimization::SynchronizeStackValues() {\n if (SlowPath) {\n return GetTopWithCache_Slow();\n }\n\n // Store new top which is now the original top minus recorded top offset\n // Careful with underflow wraparound.\n const auto TopOffset = StackData.TopOffset;\n\n if (TopOffset != 0) {\n Ref NewTop = GetOffsetTopWithCache_Slow(TopOffset, true);\n SetTopWithCache_Slow(NewTop);\n }\n StackData.TopOffset = 0;\n\n // Before leaving we need to write the current values in the stack to\n // context so that the values are correct. Copy SourceDataNode in the\n // stack to the respective mmX register.\n Ref TopValue = GetTopWithCache_Slow();\n for (size_t i = 0; i < StackData.size; ++i) {\n const auto& [Valid, StackMember] = StackData.top(i);\n\n if (Valid == StackSlot::VALID) {\n StoreStackValueAtOffset_Slow(StackMember.StackDataNode, i, false);\n }\n }\n { // Set valid tags\n uint8_t ValidMask = StackData.getValidMask();\n uint8_t InvalidMask = StackData.getInvalidMask();\n for (auto& Elem : TopValidCache) {\n Elem = (ValidMask & 1) ? StackSlot::VALID : ((InvalidMask & 1) ? StackSlot::INVALID : StackSlot::UNUSED);\n\n ValidMask >>= 1;\n InvalidMask >>= 1;\n }\n FlushValidPending = true;\n }\n\n return TopValue;\n}\n\nvoid X87StackOptimization::Run(IREmitter* Emit) {\n FEXCORE_PROFILE_SCOPED(\"PassManager::x87StackOpt\");\n\n auto CurrentIR = Emit->ViewIR();\n auto* HeaderOp = CurrentIR.GetHeader();\n LOGMAN_THROW_A_FMT(HeaderOp->Header.Op == OP_IRHEADER, \"First op wasn't IRHeader\");\n\n if (!HeaderOp->HasX87) {\n // If there is no x87 in this, just early exit.\n return;\n }\n\n // Initialize IREmit member\n IREmit = Emit;\n IR = &CurrentIR;\n\n // Run optimization proper\n for (auto [BlockNode, BlockHeader] : CurrentIR.GetBlocks()) {\n auto BlockIROp = BlockHeader->CW();\n // Each time we deal with a new block we need to start over.\n // The optimization should run per-block\n Reset();\n\n IREmit->SetCurrentCodeBlock(BlockNode);\n for (auto [CodeNode, IROp] : CurrentIR.GetCode(BlockNode)) {\n if (!LoweredX87(IROp->Op)) {\n continue;\n }\n IREmit->SetWriteCursor(CodeNode);\n switch (IROp->Op) {\n case OP_F80ADDSTACK: {\n const auto* Op = IROp->C();\n HandleBinopStack(OP_VFADD, true, OP_F80ADD, Op->SrcStack1, Op->SrcStack1, Op->SrcStack2);\n break;\n }\n\n case OP_F80SUBSTACK: {\n const auto* Op = IROp->C();\n HandleBinopStack(OP_VFSUB, true, OP_F80SUB, Op->DstStack, Op->SrcStack1, Op->SrcStack2);\n break;\n }\n\n case OP_F80MULSTACK: {\n const auto* Op = IROp->C();\n HandleBinopStack(OP_VFMUL, true, OP_F80MUL, Op->SrcStack1, Op->SrcStack1, Op->SrcStack2);\n break;\n }\n\n case OP_F80DIVSTACK: {\n const auto* Op = IROp->C();\n HandleBinopStack(OP_VFDIV, true, OP_F80DIV, Op->DstStack, Op->SrcStack1, Op->SrcStack2);\n break;\n }\n\n case OP_F80FPREMSTACK: {\n HandleBinopStack(OP_F64FPREM, false, OP_F80FPREM, 0, 0, 1);\n break;\n }\n\n case OP_F80FPREM1STACK: {\n HandleBinopStack(OP_F64FPREM1, false, OP_F80FPREM1, 0, 0, 1);\n break;\n }\n\n case OP_F80SCALESTACK: {\n HandleBinopStack(OP_F64SCALE, false, OP_F80SCALE, 0, 0, 1);\n break;\n }\n\n case OP_F80FYL2XSTACK: {\n HandleBinopStack(OP_F64FYL2X, false, OP_F80FYL2X, 1, 0, 1);\n StackPop();\n break;\n }\n\n case OP_F80ATANSTACK: {\n HandleBinopStack(OP_F64ATAN, false, OP_F80ATAN, 1, 1, 0);\n StackPop();\n break;\n }\n\n case OP_F80ADDVALUE: {\n const auto* Op = IROp->C();\n HandleBinopValue(OP_VFADD, true, OP_F80ADD, 0, true, Op->SrcStack, CurrentIR.GetNode(Op->X80Src));\n break;\n }\n\n case OP_F80SUBRVALUE:\n case OP_F80SUBVALUE: {\n const auto* Op = IROp->C();\n HandleBinopValue(OP_VFSUB, true, OP_F80SUB, 0, true, Op->SrcStack, CurrentIR.GetNode(Op->X80Src), IROp->Op == OP_F80SUBRVALUE);\n break;\n }\n\n case OP_F80DIVRVALUE:\n case OP_F80DIVVALUE: {\n const auto* Op = IROp->C();\n HandleBinopValue(OP_VFDIV, true, OP_F80DIV, 0, true, Op->SrcStack, CurrentIR.GetNode(Op->X80Src), IROp->Op == OP_F80DIVRVALUE);\n break;\n }\n\n case OP_F80MULVALUE: {\n const auto* Op = IROp->C();\n HandleBinopValue(OP_VFMUL, true, OP_F80MUL, 0, true, Op->SrcStack, CurrentIR.GetNode(Op->X80Src));\n break;\n }\n\n case OP_F80SQRTSTACK: {\n HandleUnop(OP_VFSQRT, true, OP_F80SQRT);\n break;\n }\n\n case OP_F80SINSTACK: {\n HandleUnop(OP_F64SIN, false, OP_F80SIN);\n break;\n }\n\n case OP_F80COSSTACK: {\n HandleUnop(OP_F64COS, false, OP_F80COS);\n break;\n }\n\n case OP_F80F2XM1STACK: {\n HandleUnop(OP_F64F2XM1, false, OP_F80F2XM1);\n break;\n }\n\n\n case OP_F80PTANSTACK: {\n HandleUnop(OP_F64TAN, false, OP_F80TAN);\n Ref OneConst {};\n if (ReducedPrecisionMode) {\n OneConst = IREmit->_VCastFromGPR(OpSize::i64Bit, OpSize::i64Bit, GetConstant(0x3FF0000000000000));\n } else {\n OneConst = IREmit->_LoadNamedVectorConstant(OpSize::i128Bit, NamedVectorConstant::NAMED_VECTOR_X87_ONE);\n }\n\n if (SlowPath) {\n UpdateTopForPush_Slow();\n StoreStackValueAtOffset_Slow(OneConst);\n } else {\n StackData.push(StackMemberInfo {OneConst});\n }\n break;\n }\n\n case OP_F80SINCOSSTACK: {\n Ref St0 = LoadStackValue();\n\n Ref SinValue {};\n Ref CosValue {};\n\n if (ReducedPrecisionMode) {\n SinValue = IREmit->_F64SIN(St0);\n CosValue = IREmit->_F64COS(St0);\n }\n#ifdef VIXL_SIMULATOR\n else if (DisableVixlIndirectCalls() == 0) {\n SinValue = IREmit->_F80SIN(St0);\n CosValue = IREmit->_F80COS(St0);\n }\n#endif\n else {\n SinValue = IREmit->_AllocateFPR(OpSize::i128Bit, OpSize::i128Bit);\n CosValue = IREmit->_AllocateFPR(OpSize::i128Bit, OpSize::i128Bit);\n IREmit->_F80SINCOS(St0, SinValue, CosValue);\n }\n\n // Push values\n if (SlowPath) {\n StoreStackValueAtOffset_Slow(SinValue, 0, false);\n UpdateTopForPush_Slow();\n StoreStackValueAtOffset_Slow(CosValue, 0, true);\n } else {\n StackData.setTop(StackMemberInfo {SinValue});\n StackData.push(StackMemberInfo {CosValue});\n }\n break;\n }\n\n case OP_INITSTACK: {\n StackData.clear();\n InvalidateCachedRegs();\n break;\n }\n\n case OP_INVALIDATESTACK: {\n const auto* Op = IROp->C();\n auto Offset = Op->StackLocation;\n\n if (Offset != 0xff) { // invalidate single offset\n if (SlowPath) {\n SetX87ValidTag(Offset, false);\n } else {\n StackData.setTagInvalid(Offset);\n }\n } else { // invalidate all\n if (SlowPath) {\n TopValidCache.fill(StackSlot::INVALID);\n FlushValidPending = true;\n } else {\n for (size_t i = 0; i < StackData.size; i++) {\n StackData.setTagInvalid(i);\n }\n }\n }\n break;\n }\n\n case OP_PUSHSTACK: {\n const auto* Op = IROp->C();\n auto* SourceNode = CurrentIR.GetNode(Op->X80Src);\n\n if (SlowPath) {\n UpdateTopForPush_Slow();\n StoreStackValueAtOffset_Slow(SourceNode);\n } else {\n auto* SourceNode = CurrentIR.GetNode(Op->X80Src);\n if (Op->OriginalValue.IsInvalid()) {\n // No original value to track - just push the converted data\n StackData.push(StackMemberInfo {SourceNode});\n } else {\n auto* OriginalNode = CurrentIR.GetNode(Op->OriginalValue);\n StackData.push(StackMemberInfo {SourceNode, OriginalNode, Op->LoadSize});\n }\n }\n break;\n }\n\n case OP_COPYPUSHSTACK: {\n const auto* Op = IROp->C();\n auto Offset = Op->StackLocation;\n auto Value = MigrateToSlowPath_IfInvalid(Offset);\n\n if (SlowPath) {\n Ref St0 = LoadStackValueAtOffset_Slow(Offset);\n UpdateTopForPush_Slow();\n StoreStackValueAtOffset_Slow(St0);\n } else {\n StackData.push(*Value);\n }\n break;\n }\n\n case OP_READSTACKVALUE: {\n const auto* Op = IROp->C();\n auto Offset = Op->StackLocation;\n Ref NewValue = LoadStackValue(Offset);\n\n IREmit->ReplaceUsesWithAfter(CodeNode, NewValue, CodeNode);\n break;\n }\n\n case OP_STACKVALIDTAG: {\n // Returns 0 if value is valid and 1 otherwise.\n const auto* Op = IROp->C();\n auto Offset = Op->StackLocation;\n auto Value = MigrateToSlowPath_IfInvalid(Offset);\n\n Ref Tag {};\n if (SlowPath) {\n Tag = GetX87ValidTag_Slow(Offset);\n } else {\n Tag = Value ? GetConstant(1) : GetConstant(0);\n }\n\n IREmit->ReplaceUsesWithAfter(CodeNode, Tag, CodeNode);\n break;\n }\n\n case OP_STORESTACKMEM: {\n const auto* Op = IROp->C();\n const auto& Value = MigrateToSlowPath_IfInvalid();\n Ref StackNode = SlowPath ? LoadStackValueAtOffset_Slow() : Value->StackDataNode;\n Ref AddrNode = CurrentIR.GetNode(Op->Addr);\n Ref Offset = CurrentIR.GetNode(Op->Offset);\n OpSize Align = Op->Align;\n MemOffsetType OffsetType = Op->OffsetType;\n uint8_t OffsetScale = Op->OffsetScale;\n\n // On the fast path we can optimize memory copies.\n // If we are doing:\n // fld dword [rax]\n // fst dword [rbx]\n // We can optimize this to:\n // ldr w2, [x0]\n // str w2, [x1]\n // or similar. As long as the source size and dest size are one and the same.\n // This will avoid any conversions between source and stack element size and conversion back.\n OpSize StoreSize = Op->StoreSize;\n LOGMAN_THROW_A_FMT(Op->StoreSize == OpSize::i32Bit || Op->StoreSize == OpSize::i64Bit || Op->StoreSize == OpSize::f80Bit,\n \"Invalid store size in x87 store stack mem\");\n if (!SlowPath && Value->Source && Value->Source->Size == StoreSize) {\n Ref SourceValue = Value->Source->Node;\n if (Op->StoreSize == OpSize::f80Bit) {\n Store80BitToMem(Op, SourceValue, AddrNode, Offset, Align, OffsetType, OffsetScale);\n } else {\n IREmit->_StoreMemFPR(StoreSize, SourceValue, AddrNode, Offset, Align, OffsetType, OffsetScale);\n }\n break;\n }\n\n if (ReducedPrecisionMode) {\n StoreStackMem_Reduced_Helper(Op, StackNode);\n break;\n }\n\n StoreStackMem_Helper(Op, StackNode);\n break;\n }\n\n case OP_STORESTACKTOSTACK: { // stores top of stack in another place in stack.\n const auto* Op = IROp->C();\n auto Offset = Op->StackLocation;\n\n if (Offset != 0) {\n auto Value = MigrateToSlowPath_IfInvalid();\n\n // Need to store st0 to stack location - basically a copy.\n if (SlowPath) {\n StoreStackValueAtOffset_Slow(LoadStackValueAtOffset_Slow(), Offset);\n } else {\n StackData.setTop(*Value, Offset);\n }\n }\n break;\n }\n case OP_POPSTACKDESTROY: {\n if (SlowPath) {\n SetX87ValidTag(0, false);\n }\n StackPop();\n break;\n }\n\n case OP_F80STACKXCHANGE: {\n const auto* Op = IROp->C();\n auto Offset = Op->SrcStack;\n\n if (Offset == 0) {\n // No-op\n break;\n }\n\n const auto [ValidTop, StackMemberTop] = StackData.top(0);\n const auto [ValidOffset, StackMemberOffset] = StackData.top(Offset);\n\n if (ValidTop != StackSlot::VALID || ValidOffset != StackSlot::VALID) {\n // Slow path: do actual memory operations\n Ref ValueTop = LoadStackValue();\n Ref ValueOffset = LoadStackValue(Offset);\n StoreStackValue(ValueOffset);\n StoreStackValue(ValueTop, Offset);\n } else {\n // Fast path: swap complete StackMemberInfo preserving Source metadata\n StackData.setTop(StackMemberOffset, 0);\n StackData.setTop(StackMemberTop, Offset);\n }\n break;\n }\n\n case OP_F80STACKCHANGESIGN: {\n Ref Value = LoadStackValue();\n\n // We need a couple of intermediate instructions to change the sign\n // of a value\n Ref ResultNode {};\n if (ReducedPrecisionMode) {\n ResultNode = IREmit->_VFNeg(OpSize::i64Bit, OpSize::i64Bit, Value);\n } else {\n Ref HelperNode = IREmit->_LoadNamedVectorConstant(OpSize::i128Bit, IR::NamedVectorConstant::NAMED_VECTOR_F80_SIGN_MASK);\n ResultNode = IREmit->_VXor(OpSize::i128Bit, OpSize::i8Bit, Value, HelperNode);\n }\n StoreStackValue(ResultNode);\n break;\n }\n\n case OP_F80STACKABS: {\n Ref Value = LoadStackValue();\n\n Ref ResultNode {};\n if (ReducedPrecisionMode) {\n ResultNode = IREmit->_VFAbs(OpSize::i64Bit, OpSize::i64Bit, Value);\n } else {\n // Intermediate insts\n Ref HelperNode = IREmit->_LoadNamedVectorConstant(OpSize::i128Bit, IR::NamedVectorConstant::NAMED_VECTOR_F80_SIGN_MASK);\n ResultNode = IREmit->_VAndn(OpSize::i128Bit, OpSize::i8Bit, Value, HelperNode);\n }\n StoreStackValue(ResultNode);\n break;\n }\n\n case OP_F80CMPSTACK: {\n const auto* Op = IROp->C();\n auto Offset = Op->SrcStack;\n Ref StackValue1 = LoadStackValue();\n Ref StackValue2 = LoadStackValue(Offset);\n\n Ref CmpNode {};\n if (ReducedPrecisionMode) {\n CmpNode = IREmit->_FCmp(OpSize::i64Bit, StackValue1, StackValue2);\n } else {\n CmpNode = IREmit->_F80Cmp(StackValue1, StackValue2);\n }\n\n IREmit->ReplaceUsesWithAfter(CodeNode, CmpNode, CodeNode);\n break;\n }\n case OP_F80STACKTEST: {\n const auto* Op = IROp->C();\n auto Offset = Op->SrcStack;\n auto StackNode = LoadStackValue(Offset);\n Ref ZeroConst = IREmit->_VCastFromGPR(ReducedPrecisionMode ? OpSize::i64Bit : OpSize::i128Bit, OpSize::i64Bit, GetConstant(0));\n\n Ref CmpNode {};\n if (ReducedPrecisionMode) {\n CmpNode = IREmit->_FCmp(OpSize::i64Bit, StackNode, ZeroConst);\n } else {\n CmpNode = IREmit->_F80Cmp(StackNode, ZeroConst);\n }\n IREmit->ReplaceUsesWithAfter(CodeNode, CmpNode, CodeNode);\n break;\n }\n\n\n case OP_F80CMPVALUE: {\n const auto* Op = IROp->C();\n const auto& Value = CurrentIR.GetNode(Op->X80Src);\n auto StackNode = LoadStackValue();\n\n Ref CmpNode {};\n if (ReducedPrecisionMode) {\n CmpNode = IREmit->_FCmp(OpSize::i64Bit, StackNode, Value);\n } else {\n CmpNode = IREmit->_F80Cmp(StackNode, Value);\n }\n IREmit->ReplaceUsesWithAfter(CodeNode, CmpNode, CodeNode);\n break;\n }\n\n case OP_SYNCSTACKTOSLOW: {\n // This synchronizes stack values but doesn't necessarily moves us off the FastPath!\n Ref NewTop = SynchronizeStackValues();\n FlushCachedRegs();\n IREmit->ReplaceUsesWithAfter(CodeNode, NewTop, CodeNode);\n break;\n }\n\n case OP_STACKFORCESLOW: {\n MigrateToSlowPathIf(true);\n InvalidateCachedRegs();\n break;\n }\n\n case OP_INCSTACKTOP: {\n if (SlowPath) {\n UpdateTopForPop_Slow();\n } else {\n StackData.rotate(false);\n }\n break;\n }\n\n case OP_DECSTACKTOP: {\n if (SlowPath) {\n UpdateTopForPush_Slow();\n } else {\n StackData.rotate(true);\n }\n break;\n }\n\n case OP_F80ROUNDSTACK: {\n Ref St0 = LoadStackValue();\n\n Ref Value {};\n if (ReducedPrecisionMode) {\n Value = IREmit->_Vector_FToI(OpSize::i64Bit, OpSize::i64Bit, St0, RoundMode::Host);\n } else {\n Value = IREmit->_F80Round(St0);\n }\n StoreStackValue(Value);\n break;\n }\n\n case OP_F80VBSLSTACK: {\n const auto* Op = IROp->C();\n\n auto StackOffset1 = Op->SrcStack1;\n auto StackOffset2 = Op->SrcStack2;\n Ref Value1 = LoadStackValue(StackOffset1);\n Ref Value2 = LoadStackValue(StackOffset2);\n\n Ref StackNode = IREmit->_VBSL(OpSize::i128Bit, CurrentIR.GetNode(Op->VectorMask), Value1, Value2);\n StoreStackValue(StackNode, 0, StackOffset1 && StackOffset2);\n break;\n }\n\n default: LOGMAN_THROW_A_FMT(false, \"IROp was expected to be lowered\");\n }\n IREmit->Remove(CodeNode);\n }\n\n auto Last = CurrentIR.at(BlockIROp->Last);\n --Last;\n auto [LastCodeNode, LastIROp] = Last();\n LOGMAN_THROW_A_FMT(IsBlockExit(LastIROp->Op), \"must be exit\");\n IREmit->SetWriteCursorBefore(LastCodeNode);\n SynchronizeStackValues();\n FlushCachedRegs();\n }\n\n return;\n}\n\nfextl::unique_ptr CreateX87StackOptimizationPass(const FEXCore::HostFeatures& Features, OpSize GPROpSize) {\n return fextl::make_unique(Features, GPROpSize);\n}\n} // namespace FEXCore::IR\n" }, { "path": "FEXCore/Source/Interface/IR/Passes.h", "content": "// SPDX-License-Identifier: MIT\n#pragma once\n\n#include \n\nnamespace FEXCore {\nclass CPUIDEmu;\nstruct HostFeatures;\n} // namespace FEXCore\n\nnamespace FEXCore::Utils {\nclass IntrusivePooledAllocator;\n}\n\nnamespace FEXCore::IR {\nclass Pass;\nclass RegisterAllocationPass;\n\nfextl::unique_ptr CreateDeadFlagCalculationEliminination();\nfextl::unique_ptr CreateRegisterAllocationPass(const FEXCore::CPUIDEmu* CPUID);\nfextl::unique_ptr CreateX87StackOptimizationPass(const FEXCore::HostFeatures&, OpSize GPROpSize);\n\nnamespace Validation {\n fextl::unique_ptr CreateIRValidation();\n} // namespace Validation\n\nnamespace Debug {\n fextl::unique_ptr CreateIRDumper();\n}\n} // namespace FEXCore::IR\n" }, { "path": "FEXCore/Source/Interface/IR/RegisterAllocationData.h", "content": "// SPDX-License-Identifier: MIT\n#pragma once\n\n#include \"IR.h\"\n#include \n\nnamespace FEXCore::IR {\n\nunion PhysicalRegister {\n uint8_t Raw;\n struct {\n // 32 maximum physical registers\n uint8_t Reg : 5;\n // 8 Maximum classes\n uint8_t Class : 3;\n };\n\n bool operator==(const PhysicalRegister& Other) const {\n return Raw == Other.Raw;\n }\n\n PhysicalRegister(RegClass Class, uint8_t Reg)\n : Reg(Reg)\n , Class(uint8_t(Class)) {}\n\n PhysicalRegister(OrderedNodeWrapper Arg)\n : Raw(Arg.GetImmediate()) {}\n\n PhysicalRegister(Ref Node)\n : Raw(Node->Reg) {}\n\n RegClass AsRegClass() const {\n return RegClass {Class};\n }\n\n static const PhysicalRegister Invalid() {\n return PhysicalRegister(RegClass::Invalid, 0);\n }\n\n bool IsInvalid() const {\n static_assert(uint8_t(RegClass::Invalid) == 0);\n return Raw == 0;\n }\n};\n\nstatic_assert(sizeof(PhysicalRegister) == 1);\n\n} // namespace FEXCore::IR\n" }, { "path": "FEXCore/Source/Utils/Allocator/64BitAllocator.cpp", "content": "// SPDX-License-Identifier: MIT\n#include \"Utils/Allocator/FlexBitSet.h\"\n#include \"Utils/Allocator/HostAllocator.h\"\n#include \"Utils/Allocator/IntrusiveArenaAllocator.h\"\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\nnamespace Alloc::OSAllocator {\n\nthread_local FEXCore::Core::InternalThreadState* TLSThread {};\n\nclass OSAllocator_64Bit final : public Alloc::HostAllocator {\npublic:\n OSAllocator_64Bit();\n OSAllocator_64Bit(fextl::vector& Regions);\n\n virtual ~OSAllocator_64Bit();\n void* AllocateSlab(size_t Size) override {\n return nullptr;\n }\n void DeallocateSlab(void* Ptr, size_t Size) override {}\n\n void* Mmap(void* addr, size_t length, int prot, int flags, int fd, off_t offset) override;\n int Munmap(void* addr, size_t length) override;\n\n void LockBeforeFork(FEXCore::Core::InternalThreadState* Thread) override {\n AllocationMutex.lock();\n }\n\n void UnlockAfterFork(FEXCore::Core::InternalThreadState* Thread, bool Child) override {\n if (Child) {\n AllocationMutex.StealAndDropActiveLocks();\n } else {\n AllocationMutex.unlock();\n }\n }\n\nprivate:\n // Upper bound is the maximum virtual address space of the host processor\n uintptr_t UPPER_BOUND = (1ULL << 57);\n\n // Lower bound is the starting of the range just past the lower 32bits\n constexpr static uintptr_t LOWER_BOUND = 0x1'0000'0000ULL;\n\n uintptr_t UPPER_BOUND_PAGE = UPPER_BOUND / FEXCore::Utils::FEX_PAGE_SIZE;\n constexpr static uintptr_t LOWER_BOUND_PAGE = LOWER_BOUND / FEXCore::Utils::FEX_PAGE_SIZE;\n\n struct ReservedVMARegion {\n uintptr_t Base;\n // Could be number of pages if we want to pack this in to 12 bytes\n uint64_t RegionSize;\n };\n\n bool MergeReservedRegionIfPossible(ReservedVMARegion* Region, uintptr_t NextPtr, uint64_t NextSize) {\n constexpr uint64_t MaxReservedRegionSize = 64ULL * 1024 * 1024 * 1024; // 64GB\n uintptr_t RegionEnd = Region->Base + Region->RegionSize;\n uint64_t NewRegionSize = Region->RegionSize + NextSize;\n if (RegionEnd == NextPtr && NewRegionSize <= MaxReservedRegionSize) {\n // Append the contiguous region\n Region->RegionSize = NewRegionSize;\n return true;\n }\n return false;\n }\n\n struct LiveVMARegion {\n ReservedVMARegion* SlabInfo;\n uint64_t FreeSpace {};\n uint64_t NumManagedPages {};\n uint32_t LastPageAllocation {};\n bool HadMunmap {};\n\n // Align UsedPages so it pads to the next page.\n // Necessary to take advantage of madvise zero page pooling.\n using FlexBitElementType = uint64_t;\n alignas(FEXCore::Utils::FEX_PAGE_SIZE) FEXCore::FlexBitSet UsedPages;\n\n // This returns the size of the LiveVMARegion in addition to the flex set that tracks the used data\n // The LiveVMARegion lives at the start of the VMA region which means on initialization we need to set that\n // tracked ranged as used immediately\n static size_t GetFEXManagedVMARegionSize(size_t Size) {\n // One element per page\n\n // 0x10'0000'0000 bytes\n // 0x100'0000 Pages\n // 1 bit per page for tracking means 0x20'0000 (Pages / 8) bytes of flex space\n // Which is 2MB of tracking\n const uint64_t NumElements = Size >> FEXCore::Utils::FEX_PAGE_SHIFT;\n return sizeof(LiveVMARegion) + FEXCore::FlexBitSet::SizeInBytes(NumElements);\n }\n\n static void InitializeVMARegionUsed(LiveVMARegion* Region, size_t AdditionalSize) {\n size_t SizeOfLiveRegion =\n FEXCore::AlignUp(LiveVMARegion::GetFEXManagedVMARegionSize(Region->SlabInfo->RegionSize), FEXCore::Utils::FEX_PAGE_SIZE);\n size_t SizePlusManagedData = SizeOfLiveRegion + AdditionalSize;\n\n Region->FreeSpace = Region->SlabInfo->RegionSize - SizePlusManagedData;\n\n size_t NumManagedPages = SizePlusManagedData >> FEXCore::Utils::FEX_PAGE_SHIFT;\n size_t ManagedSize = NumManagedPages << FEXCore::Utils::FEX_PAGE_SHIFT;\n\n // Use madvise to set the full tracking region to zero.\n // This ensures unused pages are zero, while not having the backing pages consuming memory.\n ::madvise(Region->UsedPages.Memory + ManagedSize, (Region->SlabInfo->RegionSize >> FEXCore::Utils::FEX_PAGE_SHIFT) - ManagedSize,\n MADV_DONTNEED);\n\n // Use madvise to claim WILLNEED on the beginning pages for initial state tracking.\n // Improves performance of the following MemClear by not doing a page level fault dance for data necessary to track >170TB of used pages.\n ::madvise(Region->UsedPages.Memory, ManagedSize, MADV_WILLNEED);\n\n // Set our reserved pages\n Region->UsedPages.MemSet(NumManagedPages);\n Region->LastPageAllocation = NumManagedPages;\n Region->NumManagedPages = NumManagedPages;\n }\n };\n\n static_assert(sizeof(LiveVMARegion) == FEXCore::Utils::FEX_PAGE_SIZE, \"Needs to be the size of a page\");\n\n static_assert(std::is_trivially_copyable::value, \"Needs to be trivially copyable\");\n static_assert(offsetof(LiveVMARegion, UsedPages) == sizeof(LiveVMARegion), \"FlexBitSet needs to be at the end\");\n\n using ReservedRegionListType = fex_pmr::list;\n using LiveRegionListType = fex_pmr::list;\n ReservedRegionListType* ReservedRegions {};\n LiveRegionListType* LiveRegions {};\n\n Alloc::ForwardOnlyIntrusiveArenaAllocator* ObjectAlloc {};\n FEXCore::ForkableUniqueMutex AllocationMutex;\n void DetermineVASize();\n\n LiveVMARegion* MakeRegionActive(ReservedRegionListType::iterator ReservedIterator, uint64_t UsedSize) {\n ReservedVMARegion* ReservedRegion = *ReservedIterator;\n\n ReservedRegions->erase(ReservedIterator);\n\n // mprotect the new region we've allocated\n size_t SizeOfLiveRegion =\n FEXCore::AlignUp(LiveVMARegion::GetFEXManagedVMARegionSize(ReservedRegion->RegionSize), FEXCore::Utils::FEX_PAGE_SIZE);\n size_t SizePlusManagedData = UsedSize + SizeOfLiveRegion;\n\n auto Res = mprotect(reinterpret_cast(ReservedRegion->Base), SizePlusManagedData, PROT_READ | PROT_WRITE);\n LOGMAN_THROW_A_FMT(Res != -1, \"Couldn't mprotect region: {} '{}' Likely occurs when running out of memory or Maximum VMAs\", errno,\n strerror(errno));\n\n FEXCore::Allocator::VirtualName(\"FEXMem_Misc\", reinterpret_cast(ReservedRegion->Base), SizePlusManagedData);\n LiveVMARegion* LiveRange = new (reinterpret_cast(ReservedRegion->Base)) LiveVMARegion();\n\n // Copy over the reserved data\n LiveRange->SlabInfo = ReservedRegion;\n\n // Initialize VMA\n LiveVMARegion::InitializeVMARegionUsed(LiveRange, UsedSize);\n\n // Add to our active tracked ranges\n auto LiveIter = LiveRegions->emplace_back(LiveRange);\n return LiveIter;\n }\n\n void AllocateMemoryRegions(fextl::vector& Ranges);\n LiveVMARegion* FindLiveRegionForAddress(uintptr_t Addr, uintptr_t AddrEnd);\n};\n\nvoid OSAllocator_64Bit::DetermineVASize() {\n size_t Bits = FEXCore::Allocator::DetermineVASize();\n uintptr_t Size = 1ULL << Bits;\n\n UPPER_BOUND = Size;\n\n#if ARCHITECTURE_x86_64 // Last page cannot be allocated on x86\n UPPER_BOUND -= FEXCore::Utils::FEX_PAGE_SIZE;\n#endif\n\n UPPER_BOUND_PAGE = UPPER_BOUND / FEXCore::Utils::FEX_PAGE_SIZE;\n}\n\nOSAllocator_64Bit::LiveVMARegion* OSAllocator_64Bit::FindLiveRegionForAddress(uintptr_t Addr, uintptr_t AddrEnd) {\n LiveVMARegion* LiveRegion {};\n\n // Check active slabs to see if we can fit this\n for (auto it = LiveRegions->begin(); it != LiveRegions->end(); ++it) {\n uintptr_t RegionBegin = (*it)->SlabInfo->Base;\n uintptr_t RegionEnd = RegionBegin + (*it)->SlabInfo->RegionSize;\n\n if (Addr >= RegionBegin && AddrEnd < RegionEnd) {\n LiveRegion = *it;\n // Leave our loop\n break;\n }\n }\n\n // Couldn't find an active region that fit\n // Check reserved regions\n if (!LiveRegion) {\n // Didn't have a slab that fit this range\n // Check our reserved regions to see if we have one that fits\n for (auto it = ReservedRegions->begin(); it != ReservedRegions->end(); ++it) {\n ReservedVMARegion* ReservedRegion = *it;\n uintptr_t RegionEnd = ReservedRegion->Base + ReservedRegion->RegionSize;\n if (Addr >= ReservedRegion->Base && AddrEnd < RegionEnd) {\n // Found one, let's make it active\n LiveRegion = MakeRegionActive(it, 0);\n break;\n }\n }\n }\n\n return LiveRegion;\n}\n\nvoid* OSAllocator_64Bit::Mmap(void* addr, size_t length, int prot, int flags, int fd, off_t offset) {\n if (addr != 0 && addr < reinterpret_cast(LOWER_BOUND)) {\n // If we are asked to allocate something outside of the 64-bit space\n // Then we need to just hand this to the OS\n return ::mmap(addr, length, prot, flags, fd, offset);\n }\n\n uint64_t Addr = reinterpret_cast(addr);\n // Addr must be page aligned\n if (Addr & ~FEXCore::Utils::FEX_PAGE_MASK) {\n return reinterpret_cast(-EINVAL);\n }\n\n // If FD is provided then offset must also be page aligned\n if (fd != -1 && offset & ~FEXCore::Utils::FEX_PAGE_MASK) {\n return reinterpret_cast(-EINVAL);\n }\n\n // 64bit address overflow\n if (Addr + length < Addr) {\n return reinterpret_cast(-EOVERFLOW);\n }\n\n bool Fixed = (flags & MAP_FIXED) || (flags & MAP_FIXED_NOREPLACE);\n length = FEXCore::AlignUp(length, FEXCore::Utils::FEX_PAGE_SIZE);\n\n uint64_t AddrEnd = Addr + length;\n size_t NumberOfPages = length / FEXCore::Utils::FEX_PAGE_SIZE;\n\n // This needs a mutex to be thread safe\n auto lk = FEXCore::GuardSignalDeferringSectionWithFallback(AllocationMutex, TLSThread);\n\n uint64_t AllocatedOffset {};\n LiveVMARegion* LiveRegion {};\n\n if (Fixed || Addr != 0) {\n LiveRegion = FindLiveRegionForAddress(Addr, AddrEnd);\n }\n\nagain:\n\n struct RangeResult final {\n LiveVMARegion* RegionInsertedInto;\n void* Ptr;\n };\n\n auto CheckIfRangeFits = [&AllocatedOffset](LiveVMARegion* Region, uint64_t length, int prot, int flags, int fd, off_t offset,\n uint64_t StartingPosition = 0) -> RangeResult {\n uint64_t AllocatedPage {~0ULL};\n uint64_t NumberOfPages = length >> FEXCore::Utils::FEX_PAGE_SHIFT;\n\n if (Region->FreeSpace >= length) {\n uint64_t LastAllocation =\n StartingPosition ? (StartingPosition - Region->SlabInfo->Base) >> FEXCore::Utils::FEX_PAGE_SHIFT : Region->LastPageAllocation;\n size_t RegionNumberOfPages = Region->SlabInfo->RegionSize >> FEXCore::Utils::FEX_PAGE_SHIFT;\n\n\n if (Region->HadMunmap) {\n // Backward scan\n // We need to do a backward scan first to fill any holes\n // Otherwise we will very quickly run out of VMA regions (65k maximum)\n auto SearchResult = Region->UsedPages.BackwardScanForRange(LastAllocation, NumberOfPages, Region->NumManagedPages);\n\n AllocatedPage = SearchResult.FoundElement;\n\n // If we didn't even have a one page free in the backward search, then unclaim HadMunmap.\n // Switching over to default forward search.\n if (SearchResult.FoundElement == ~0ULL && !SearchResult.FoundHole) {\n Region->HadMunmap = false;\n }\n }\n\n // Foward Scan\n if (AllocatedPage == ~0ULL) {\n auto SearchResult = Region->UsedPages.ForwardScanForRange(LastAllocation, NumberOfPages, RegionNumberOfPages);\n AllocatedPage = SearchResult.FoundElement;\n }\n\n if (AllocatedPage != ~0ULL) {\n AllocatedOffset = Region->SlabInfo->Base + AllocatedPage * FEXCore::Utils::FEX_PAGE_SIZE;\n\n // We need to setup protections for this\n void* MMapResult = ::mmap(reinterpret_cast(AllocatedOffset), length, prot, (flags & ~MAP_FIXED_NOREPLACE) | MAP_FIXED, fd, offset);\n\n if (MMapResult == MAP_FAILED) {\n return RangeResult {Region, reinterpret_cast(-errno)};\n }\n return RangeResult {Region, MMapResult};\n }\n }\n\n return {};\n };\n\n if (Fixed) {\n // Found a region let's allocate to it\n if (LiveRegion) {\n // Found a slab that fits this\n if (flags & MAP_FIXED_NOREPLACE) {\n auto Fits = CheckIfRangeFits(LiveRegion, length, prot, flags, fd, offset, Addr);\n if (Fits.RegionInsertedInto && Fits.Ptr == reinterpret_cast(Addr)) {\n // We fit correctly\n AllocatedOffset = Addr;\n } else {\n // Intersected with something that already existed\n return reinterpret_cast(-EEXIST);\n }\n } else {\n // We need to mmap the file to this location\n void* MMapResult = ::mmap(reinterpret_cast(Addr), length, prot, (flags & ~MAP_FIXED_NOREPLACE) | MAP_FIXED, fd, offset);\n\n if (MMapResult == MAP_FAILED) {\n return reinterpret_cast(-errno);\n }\n\n AllocatedOffset = Addr;\n }\n // Fall through to live region tracking\n }\n } else {\n // Check our active slabs to see if we can fit the allocation\n // Slightly different than fixed since it doesn't need exact placement\n if (LiveRegion && Addr != 0) {\n // We found a LiveRegion that could hold this address. Let's try to place it\n // Check if this area is free\n auto Fits = CheckIfRangeFits(LiveRegion, length, prot, flags, fd, offset, Addr);\n if (Fits.RegionInsertedInto && Fits.Ptr == reinterpret_cast(Addr)) {\n // We fit correctly\n AllocatedOffset = Addr;\n } else {\n // Couldn't fit\n // We can continue past this point still\n LiveRegion = nullptr;\n }\n }\n\n if (!LiveRegion) {\n for (auto it = LiveRegions->begin(); it != LiveRegions->end(); ++it) {\n auto Fits = CheckIfRangeFits(*it, length, prot, flags, fd, offset);\n if (Fits.RegionInsertedInto && Fits.Ptr == reinterpret_cast(AllocatedOffset)) {\n // We fit correctly\n LiveRegion = Fits.RegionInsertedInto;\n break;\n }\n\n // Couldn't fit but mmap gave us an error\n if (!Fits.RegionInsertedInto && Fits.Ptr) {\n return Fits.Ptr;\n }\n\n // nullptr on both means no error and couldn't fit\n }\n }\n\n if (!LiveRegion) {\n // Couldn't find a fit in the live regions\n // Allocate a new reserved region\n size_t lengthOfLiveRegion = FEXCore::AlignUp(LiveVMARegion::GetFEXManagedVMARegionSize(length), FEXCore::Utils::FEX_PAGE_SIZE);\n size_t lengthPlusManagedData = length + lengthOfLiveRegion;\n for (auto it = ReservedRegions->begin(); it != ReservedRegions->end(); ++it) {\n if ((*it)->RegionSize >= lengthPlusManagedData) {\n MakeRegionActive(it, 0);\n goto again;\n }\n }\n }\n }\n\n if (LiveRegion) {\n // Mark the pages as used\n uintptr_t RegionBegin = LiveRegion->SlabInfo->Base;\n uintptr_t MappedBegin = (AllocatedOffset - RegionBegin) >> FEXCore::Utils::FEX_PAGE_SHIFT;\n size_t PagesSet {};\n\n for (size_t i = 0; i < NumberOfPages; ++i) {\n PagesSet += LiveRegion->UsedPages.TestAndSet(MappedBegin + i) == false;\n }\n\n // Change our last allocation region\n LiveRegion->LastPageAllocation = MappedBegin + NumberOfPages;\n LiveRegion->FreeSpace -= PagesSet * FEXCore::Utils::FEX_PAGE_SIZE;\n LOGMAN_THROW_A_FMT(LiveRegion->FreeSpace <= LiveRegion->SlabInfo->RegionSize,\n \"Corrupt LiveRegion free space! 0x{:x} > 0x{:x}. After allocating 0x{:x} (0x{:x} overlapped)\", LiveRegion->FreeSpace,\n LiveRegion->SlabInfo->RegionSize, length, PagesSet);\n }\n\n if (!AllocatedOffset) {\n AllocatedOffset = -ENOMEM;\n }\n return reinterpret_cast(AllocatedOffset);\n}\n\nint OSAllocator_64Bit::Munmap(void* addr, size_t length) {\n if (addr < reinterpret_cast(LOWER_BOUND)) {\n // If we are asked to allocate something outside of the 64-bit space\n // Then we need to just hand this to the OS\n return ::munmap(addr, length);\n }\n\n uint64_t Addr = reinterpret_cast(addr);\n\n if (Addr & ~FEXCore::Utils::FEX_PAGE_MASK) {\n return -EINVAL;\n }\n\n if (length & ~FEXCore::Utils::FEX_PAGE_MASK) {\n return -EINVAL;\n }\n\n if (Addr + length < Addr) {\n return -EOVERFLOW;\n }\n\n // This needs a mutex to be thread safe\n auto lk = FEXCore::GuardSignalDeferringSectionWithFallback(AllocationMutex, TLSThread);\n\n length = FEXCore::AlignUp(length, FEXCore::Utils::FEX_PAGE_SIZE);\n\n uintptr_t PtrBegin = reinterpret_cast(addr);\n uintptr_t PtrEnd = PtrBegin + length;\n // Walk all of the live ranges and find this slab then delete it\n for (auto it = LiveRegions->begin(); it != LiveRegions->end(); ++it) {\n uintptr_t RegionBegin = (*it)->SlabInfo->Base;\n uintptr_t RegionEnd = RegionBegin + (*it)->SlabInfo->RegionSize;\n\n if (RegionBegin <= PtrBegin && RegionEnd > PtrEnd) {\n // Live region fully encompasses slab range\n\n uint64_t FreedPages {};\n uint32_t SlabPageBegin = (PtrBegin - RegionBegin) >> FEXCore::Utils::FEX_PAGE_SHIFT;\n uint64_t PagesToFree = length >> FEXCore::Utils::FEX_PAGE_SHIFT;\n\n for (size_t i = 0; i < PagesToFree; ++i) {\n FreedPages += (*it)->UsedPages.TestAndClear(SlabPageBegin + i) ? 1 : 0;\n }\n\n if (FreedPages != 0) {\n // If we were contiuous freeing then make sure to give back the physical address space\n // If the region was locked then madvise won't remove the physical backing\n // This woul be a bug in the frontend application\n // So be careful with mlock/munlock\n ::madvise(addr, length, MADV_DONTNEED);\n ::mmap(addr, length, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0);\n }\n\n (*it)->FreeSpace += FreedPages * FEXCore::Utils::FEX_PAGE_SIZE;\n\n // Set the last allocated page to the minimum of last page allocation or this slab\n // This will let us more quickly fill holes\n (*it)->LastPageAllocation = std::min((*it)->LastPageAllocation, SlabPageBegin);\n\n (*it)->HadMunmap = true;\n\n // XXX: Move region back to reserved list\n return 0;\n }\n }\n\n // If it didn't match at all then no error\n return 0;\n}\n\nvoid OSAllocator_64Bit::AllocateMemoryRegions(fextl::vector& Ranges) {\n // Need to allocate the ObjectAlloc up front. Find a region that is larger than our minimum size first.\n const size_t ObjectAllocSize = 64 * 1024 * 1024;\n\n for (auto& it : Ranges) {\n if (ObjectAllocSize > it.Size) {\n continue;\n }\n\n // Allocate up to 64 MiB the first allocation for an intrusive allocator\n mprotect(it.Ptr, ObjectAllocSize, PROT_READ | PROT_WRITE);\n\n // This enables the kernel to use transparent large pages in the allocator which can reduce memory pressure\n ::madvise(it.Ptr, ObjectAllocSize, MADV_HUGEPAGE);\n\n FEXCore::Allocator::VirtualName(\"FEXMem_Misc\", reinterpret_cast(it.Ptr), ObjectAllocSize);\n\n ObjectAlloc = new (it.Ptr) Alloc::ForwardOnlyIntrusiveArenaAllocator(it.Ptr, ObjectAllocSize);\n ReservedRegions = ObjectAlloc->new_construct(ReservedRegions, ObjectAlloc);\n LiveRegions = ObjectAlloc->new_construct(LiveRegions, ObjectAlloc);\n\n if (it.Size >= ObjectAllocSize) {\n // Modify region size\n it.Size -= ObjectAllocSize;\n (uint8_t*&)it.Ptr += ObjectAllocSize;\n }\n\n break;\n }\n\n if (!ObjectAlloc) {\n ERROR_AND_DIE_FMT(\"Couldn't allocate object allocator!\");\n }\n\n for (auto [Ptr, AllocationSize] : Ranges) {\n // Skip using any regions that are <= two pages. FEX's VMA allocator requires two pages\n // for tracking data. So three pages are minimum for a single page VMA allocation.\n if (AllocationSize <= (FEXCore::Utils::FEX_PAGE_SIZE * 2)) {\n continue;\n }\n\n ReservedVMARegion* Region = ObjectAlloc->new_construct();\n Region->Base = reinterpret_cast(Ptr);\n Region->RegionSize = AllocationSize;\n ReservedRegions->emplace_back(Region);\n }\n}\n\n\nOSAllocator_64Bit::OSAllocator_64Bit() {\n DetermineVASize();\n\n auto Ranges = FEXCore::Allocator::StealMemoryRegion(LOWER_BOUND, UPPER_BOUND);\n\n AllocateMemoryRegions(Ranges);\n}\n\nOSAllocator_64Bit::OSAllocator_64Bit(fextl::vector& Regions) {\n AllocateMemoryRegions(Regions);\n}\n\nOSAllocator_64Bit::~OSAllocator_64Bit() {\n // This needs a mutex to be thread safe\n auto lk = FEXCore::GuardSignalDeferringSectionWithFallback(AllocationMutex, TLSThread);\n\n // Walk the pages and deallocate\n // First walk the live regions\n for (auto it = LiveRegions->begin(); it != LiveRegions->end(); ++it) {\n ::munmap(reinterpret_cast((*it)->SlabInfo->Base), (*it)->SlabInfo->RegionSize);\n }\n\n // Now walk the reserved regions\n for (auto it = ReservedRegions->begin(); it != ReservedRegions->end(); ++it) {\n ::munmap(reinterpret_cast((*it)->Base), (*it)->RegionSize);\n }\n}\n\nfextl::unique_ptr Create64BitAllocator() {\n return fextl::make_unique();\n}\n\ntemplate\nstruct alloc_delete : public std::default_delete {\n void operator()(T* ptr) const {\n if (ptr) {\n const auto size = sizeof(T);\n const auto MinPage = FEXCore::AlignUp(size, FEXCore::Utils::FEX_PAGE_SIZE);\n\n std::destroy_at(ptr);\n ::munmap(ptr, MinPage);\n }\n }\n\n template\n requires (std::is_base_of_v)\n operator fextl::default_delete() {\n return fextl::default_delete();\n }\n};\n\ntemplate\nrequires (!std::is_array_v)\nfextl::unique_ptr make_alloc_unique(FEXCore::Allocator::MemoryRegion& Base, Args&&... args) {\n const auto size = sizeof(T);\n const auto MinPage = FEXCore::AlignUp(size, FEXCore::Utils::FEX_PAGE_SIZE);\n if (Base.Size < size || MinPage != FEXCore::Utils::FEX_PAGE_SIZE) {\n ERROR_AND_DIE_FMT(\"Couldn't fit allocator in to page!\");\n }\n\n auto ptr = ::mmap(Base.Ptr, MinPage, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0);\n if (ptr == MAP_FAILED) {\n ERROR_AND_DIE_FMT(\"Couldn't allocate memory region\");\n }\n\n FEXCore::Allocator::VirtualName(\"FEXMem_Misc\", reinterpret_cast(ptr), MinPage);\n\n // Remove the page from the base region.\n // Could be zero after this.\n Base.Size -= MinPage;\n Base.Ptr = reinterpret_cast(reinterpret_cast(Base.Ptr) + MinPage);\n\n auto Result = ::new (ptr) T(std::forward(args)...);\n return fextl::unique_ptr>(Result);\n}\n\nfextl::unique_ptr Create64BitAllocatorWithRegions(fextl::vector& Regions) {\n // This is a bit tricky as we can't allocate memory safely except from the Regions provided. Otherwise we might overwrite memory pages we\n // don't own. Scan the memory regions and find the smallest one.\n FEXCore::Allocator::MemoryRegion& Smallest = Regions[0];\n for (auto& it : Regions) {\n if (it.Size <= Smallest.Size) {\n Smallest = it;\n }\n }\n\n return make_alloc_unique(Smallest, Regions);\n}\n\n} // namespace Alloc::OSAllocator\n\nnamespace FEXCore::Allocator {\nvoid RegisterTLSData(FEXCore::Core::InternalThreadState* Thread) {\n Alloc::OSAllocator::TLSThread = Thread;\n}\n\nvoid UninstallTLSData(FEXCore::Core::InternalThreadState* Thread) {\n Alloc::OSAllocator::TLSThread = nullptr;\n}\n} // namespace FEXCore::Allocator\n" }, { "path": "FEXCore/Source/Utils/Allocator/FlexBitSet.h", "content": "// SPDX-License-Identifier: MIT\n#pragma once\n\n#include \n#include \n\n#include \n#include \n#include \n#include \n\nnamespace FEXCore {\n\ntemplate\nstruct FlexBitSet final {\n using ElementType = T;\n constexpr static size_t MinimumSize = sizeof(ElementType);\n constexpr static size_t MinimumSizeBits = sizeof(ElementType) * 8;\n\n T Memory[];\n\n bool Get(size_t Element) const {\n return (Memory[Element / MinimumSizeBits] & (1ULL << (Element % MinimumSizeBits))) != 0;\n }\n bool TestAndClear(size_t Element) {\n bool Value = Get(Element);\n Memory[Element / MinimumSizeBits] &= ~(1ULL << (Element % MinimumSizeBits));\n return Value;\n }\n bool TestAndSet(size_t Element) {\n bool Value = Get(Element);\n Memory[Element / MinimumSizeBits] |= (1ULL << (Element % MinimumSizeBits));\n return Value;\n }\n void Set(size_t Element) {\n Memory[Element / MinimumSizeBits] |= (1ULL << (Element % MinimumSizeBits));\n }\n void Clear(size_t Element) {\n Memory[Element / MinimumSizeBits] &= ~(1ULL << (Element % MinimumSizeBits));\n }\n void MemClear(size_t Elements) {\n memset(Memory, 0, FEXCore::AlignUp(Elements / MinimumSizeBits, MinimumSizeBits));\n }\n void MemSet(size_t Elements) {\n memset(Memory, 0xFF, FEXCore::AlignUp(Elements / MinimumSizeBits, MinimumSizeBits));\n }\n\n // Range scanning results\n struct BitsetScanResults {\n // Which element was found. ~0ULL if not found.\n size_t FoundElement;\n // During the scan, found a hole in the allocations that didn't fit.\n bool FoundHole;\n };\n\n // TODO: Make {Forward,Backward}ScanForRange faster\n // Currently these functions test a single bit at a time, which is fairly costly.\n // The compiler emits a full element load per iteration, wasting a bunch of time on loads.\n // If we change these functions to have a pre-amble and post-amble to align the primary loop to the element size then this can go\n // significantly faster.\n //\n // Once the element scanning is aligned to the element size, we can then use native count leading zero(CLZ) and count trailing zero(CTZ)\n // instructions on a full element to scan uint64_t elements per loop iteration.\n\n // Implementation details:\n // Template argument WantUnset\n // Used to determine if the desired range is for set or unset ranges.\n // Typically `WantUnset` should be true. Used for finding a unset range inside of a range will set elements.\n //\n // @param BeginningElement - The first element in the set to start scanning from.\n // @param ElementCount - How many elements to find a range for fitting.\n // @param MinimumElement - Minimum element in the set to search to\n //\n // @return The scan results\n template\n BitsetScanResults BackwardScanForRange(size_t BeginningElement, size_t ElementCount, size_t MinimumElement) {\n bool FoundHole {};\n\n // Final element to iterate to.\n const size_t FinalElement = MinimumElement + ElementCount - 1;\n\n for (size_t CurrentPage = BeginningElement; CurrentPage >= FinalElement;) {\n size_t Remaining = ElementCount;\n LOGMAN_THROW_A_FMT(CurrentPage <= BeginningElement && CurrentPage >= FinalElement, \"BackwardScanForRange: Scanning less than \"\n \"available range\");\n\n while (Remaining) {\n if (this->Get(CurrentPage - Remaining + 1) == WantUnset) {\n // Has an intersecting range\n break;\n }\n --Remaining;\n }\n\n if (Remaining) {\n // If we found at least one Element hole then track that\n if (Remaining != ElementCount) {\n FoundHole = true;\n }\n\n // Didn't find a slab range\n CurrentPage -= Remaining;\n } else {\n // We have a slab range\n return BitsetScanResults {CurrentPage - ElementCount + 1, FoundHole};\n }\n }\n\n return BitsetScanResults {~0ULL, FoundHole};\n }\n\n // @param BeginningElement - The first element in the set to start scanning from.\n // @param ElementCount - How many elements to find a range for fitting.\n // @param ElementsInSet - How many elements are in the full set.\n //\n // @return The scan results\n template\n BitsetScanResults ForwardScanForRange(size_t BeginningElement, size_t ElementCount, size_t ElementsInSet) {\n bool FoundHole {};\n\n // Final element to iterate to.\n const size_t FinalElement = ElementsInSet - ElementCount + 1;\n\n for (size_t CurrentElement = BeginningElement; CurrentElement <= FinalElement;) {\n // If we have enough free space, check if we have enough free pages that are contiguous\n size_t Remaining = ElementCount;\n\n LOGMAN_THROW_A_FMT(CurrentElement >= BeginningElement && CurrentElement <= FinalElement, \"ForwardScanForRange: Scanning less than \"\n \"available range\");\n\n while (Remaining) {\n if (this->Get(CurrentElement + Remaining - 1) == WantUnset) {\n // Has an intersecting range\n break;\n }\n --Remaining;\n }\n\n if (Remaining) {\n // If we found at least one Element hole then track that\n if (Remaining != ElementCount) {\n FoundHole = true;\n }\n\n // Didn't find a slab range\n CurrentElement += Remaining;\n } else {\n // We have a slab range\n return BitsetScanResults {CurrentElement, FoundHole};\n }\n }\n\n return BitsetScanResults {~0ULL, FoundHole};\n }\n\n // This very explicitly doesn't let you take an address\n // Is only a getter\n bool operator[](size_t Element) const {\n return Get(Element);\n }\n\n // Returns the number of bits required to hold the number of elements.\n // Just rounds up to the MinimumSizeInBits.\n constexpr static size_t SizeInBits(uint64_t Elements) {\n return FEXCore::AlignUp(Elements, MinimumSizeBits);\n }\n // Returns the number of bytes required to hold the number of elements.\n constexpr static size_t SizeInBytes(uint64_t Elements) {\n return SizeInBits(Elements) / 8;\n }\n};\n\nstatic_assert(sizeof(FlexBitSet) == 0, \"This needs to be a flex member\");\nstatic_assert(std::is_trivially_copyable_v>, \"Needs to be trivially copyable\");\n\n} // namespace FEXCore\n" }, { "path": "FEXCore/Source/Utils/Allocator/HostAllocator.h", "content": "// SPDX-License-Identifier: MIT\n#pragma once\n\n#include \n#include \n\n#include \n#include \n\nnamespace FEXCore::Allocator {\nstruct MemoryRegion;\n}\nnamespace FEXCore::Core {\nstruct InternalThreadState;\n}\n\nnamespace Alloc {\n// HostAllocator is just a page pased slab allocator\n// Similar to mmap and munmap only mapping at the page level\nclass HostAllocator {\npublic:\n HostAllocator() = default;\n virtual ~HostAllocator() = default;\n virtual void* AllocateSlab(size_t Size) = 0;\n virtual void DeallocateSlab(void* Ptr, size_t Size) = 0;\n\n virtual void* Mmap(void* addr, size_t length, int prot, int flags, int fd, off_t offset) {\n return nullptr;\n }\n virtual int Munmap(void* addr, size_t length) {\n return -1;\n }\n\n virtual void LockBeforeFork(FEXCore::Core::InternalThreadState* Thread) {}\n virtual void UnlockAfterFork(FEXCore::Core::InternalThreadState* Thread, bool Child) {}\n};\n\nclass GlobalAllocator {\npublic:\n HostAllocator* Alloc {};\n GlobalAllocator(HostAllocator* _Alloc)\n : Alloc {_Alloc} {}\n\n virtual ~GlobalAllocator() = default;\n virtual void* malloc(size_t Size) = 0;\n virtual void* calloc(size_t num, size_t size) = 0;\n virtual void* realloc(void* ptr, size_t size) = 0;\n virtual void* memalign(size_t alignment, size_t size) = 0;\n virtual void free(void* ptr) = 0;\n};\n} // namespace Alloc\n\nnamespace Alloc::OSAllocator {\nfextl::unique_ptr Create64BitAllocator();\nfextl::unique_ptr Create64BitAllocatorWithRegions(fextl::vector& Regions);\nstatic inline void ReleaseAllocatorWorkaround(fextl::unique_ptr Allocator) {\n // XXX: This is currently a leak.\n // We can't work around this yet until static initializers that allocate memory are completely removed from our codebase\n // The allocator is also intrusively allocated, so the unique_ptr tries to double free the HostAllocator object.\n // Luckily we only remove this on process shutdown, so the kernel will do the cleanup for us\n Allocator.release();\n}\n\n} // namespace Alloc::OSAllocator\n" }, { "path": "FEXCore/Source/Utils/Allocator/IntrusiveArenaAllocator.h", "content": "// SPDX-License-Identifier: MIT\n#pragma once\n\n#include \"FlexBitSet.h\"\n#include \"HostAllocator.h\"\n\n#include \n#include \n\n#include \n#include \n#include \nnamespace fex_pmr = std::pmr;\n#include \n\n#include \n\nnamespace Alloc {\nclass ForwardOnlyIntrusiveArenaAllocator final : public fex_pmr::memory_resource {\npublic:\n ForwardOnlyIntrusiveArenaAllocator(void* Ptr, size_t _Size)\n : Begin {reinterpret_cast(Ptr)}\n , Size {_Size} {\n LastAllocation = sizeof(ForwardOnlyIntrusiveArenaAllocator);\n }\n\n ~ForwardOnlyIntrusiveArenaAllocator() = default;\n\n template\n U* new_construct(Args&&... args) {\n void* Ptr = do_allocate(sizeof(U), alignof(U));\n return new (Ptr) U(args...);\n }\n\n template\n U* new_construct(U* Class, Args&&... args) {\n void* Ptr = do_allocate(sizeof(U), alignof(U));\n return new (Ptr) U(args...);\n }\n\n size_t AmountAllocated() const {\n return LastAllocation;\n }\n\nprivate:\n void* do_allocate(std::size_t bytes, std::size_t alignment) override {\n size_t PreviousAligned = FEXCore::AlignUp(LastAllocation, alignment);\n size_t NewOffset = PreviousAligned + bytes;\n\n if (NewOffset > Size) {\n return nullptr;\n }\n\n LastAllocation = NewOffset;\n\n return reinterpret_cast(Begin + PreviousAligned);\n }\n\n void do_deallocate(void*, std::size_t, std::size_t) override {\n // Do nothing\n }\n\n bool do_is_equal(const fex_pmr::memory_resource& other) const noexcept override {\n // Only if the allocator pointers are the same are they equal\n if (this == &other) {\n return true;\n }\n // We don't share state with another allocator so we can't share anything\n return false;\n }\n\n uintptr_t Begin;\n size_t Size;\n size_t LastAllocation {};\n};\n\nclass IntrusiveArenaAllocator final : public fex_pmr::memory_resource {\npublic:\n IntrusiveArenaAllocator(void* Ptr, size_t _Size)\n : Begin {reinterpret_cast(Ptr)}\n , Size {_Size} {\n uint64_t NumberOfPages = _Size / FEXCore::Utils::FEX_PAGE_SIZE;\n uint64_t UsedBits =\n FEXCore::AlignUp(sizeof(IntrusiveArenaAllocator) + Size / FEXCore::Utils::FEX_PAGE_SIZE / 8, FEXCore::Utils::FEX_PAGE_SIZE);\n for (size_t i = 0; i < UsedBits; ++i) {\n UsedPages.Set(i);\n }\n\n FreePages = NumberOfPages - UsedBits;\n }\n\n template\n U* new_construct(Args&&... args) {\n void* Ptr = do_allocate(sizeof(U), alignof(U));\n return new (Ptr) U(args...);\n }\n\n template\n U* new_construct(U* Class, Args&&... args) {\n void* Ptr = do_allocate(sizeof(U), alignof(U));\n return new (Ptr) U(args...);\n }\n\n uintptr_t GetSlabBase() const {\n return Begin;\n }\n uint64_t GetSlabSize() const {\n return Size;\n }\n uint64_t GetFreePages() const {\n return FreePages;\n }\n\nprivate:\n void* do_allocate(std::size_t bytes, std::size_t alignment) override {\n std::scoped_lock lk {AllocationMutex};\n\n size_t NumberPages = FEXCore::AlignUp(bytes, FEXCore::Utils::FEX_PAGE_SIZE) / FEXCore::Utils::FEX_PAGE_SIZE;\n\n uintptr_t AllocatedOffset {};\n\ntry_again:\n for (uintptr_t CurrentPage = LastAllocatedPageOffset; CurrentPage <= (Size - NumberPages);) {\n size_t Remaining = NumberPages;\n\n while (Remaining) {\n if (UsedPages[CurrentPage + Remaining - 1]) {\n // Has an intersecting range\n break;\n }\n --Remaining;\n }\n\n if (Remaining) {\n // Didn't find an allocation range\n CurrentPage += Remaining;\n } else {\n // We have a range to allocate\n AllocatedOffset = CurrentPage;\n break;\n }\n }\n\n if (!AllocatedOffset && LastAllocatedPageOffset != 0) {\n // Try again but starting from the beginning\n LastAllocatedPageOffset = 0;\n // Using goto so we don't have recursive mutex shenanigans\n goto try_again;\n }\n\n // Allocated offset must be valid or zero at this point\n if (AllocatedOffset) {\n // Map the range as no longer available\n for (size_t i = 0; i < NumberPages; ++i) {\n UsedPages.Set(AllocatedOffset + i);\n }\n\n LastAllocatedPageOffset = AllocatedOffset + NumberPages;\n\n // Now convert this base page to a pointer and return it\n return reinterpret_cast(Begin + AllocatedOffset * FEXCore::Utils::FEX_PAGE_SIZE);\n }\n\n return nullptr;\n }\n\n void do_deallocate(void* p, std::size_t bytes, std::size_t alignment) override {\n std::scoped_lock lk {AllocationMutex};\n\n uintptr_t PageOffset = (reinterpret_cast(p) - Begin) / FEXCore::Utils::FEX_PAGE_SIZE;\n size_t NumPages = FEXCore::AlignUp(bytes, FEXCore::Utils::FEX_PAGE_SIZE) / FEXCore::Utils::FEX_PAGE_SIZE;\n\n // Walk the allocation list and deallocate\n uint64_t FreedPages {};\n for (size_t i = 0; i < NumPages; ++i) {\n FreedPages += UsedPages.TestAndClear(PageOffset + i) ? 1 : 0;\n }\n FreePages += FreedPages;\n }\n\n bool do_is_equal(const fex_pmr::memory_resource& other) const noexcept override {\n // Only if the allocator pointers are the same are they equal\n if (this == &other) {\n return true;\n }\n // We don't share state with another allocator so we can't share anything\n return false;\n }\n\n uintptr_t Begin;\n size_t Size;\n uint64_t FreePages {};\n size_t LastAllocatedPageOffset {};\n std::mutex AllocationMutex {};\n // For up to 64GB regions this will require up to 2MB tracking\n // Needs to be the last element\n FEXCore::FlexBitSet UsedPages;\n};\n} // namespace Alloc\n" }, { "path": "FEXCore/Source/Utils/Allocator.cpp", "content": "// SPDX-License-Identifier: MIT\n#include \"Utils/Allocator/HostAllocator.h\"\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#ifndef _WIN32\n#include \n#include \n#endif\n\nnamespace fextl::pmr {\nstatic fextl::pmr::default_resource FEXDefaultResource;\nstd::pmr::memory_resource* get_default_resource() {\n return &FEXDefaultResource;\n}\n} // namespace fextl::pmr\n\n#ifndef _WIN32\nnamespace FEXCore::Allocator {\nMMAP_Hook mmap {::mmap};\nMUNMAP_Hook munmap {::munmap};\n\nuint64_t HostVASize {};\n\nusing GLIBC_MALLOC_Hook = void* (*)(size_t, const void* caller);\nusing GLIBC_REALLOC_Hook = void* (*)(void*, size_t, const void* caller);\nusing GLIBC_FREE_Hook = void (*)(void*, const void* caller);\n\nfextl::unique_ptr Alloc64 {};\n\nvoid* FEX_mmap(void* addr, size_t length, int prot, int flags, int fd, off_t offset) {\n void* Result = Alloc64->Mmap(addr, length, prot, flags, fd, offset);\n if (Result >= (void*)-4096) {\n errno = -(uint64_t)Result;\n return (void*)-1;\n }\n\n if (flags & MAP_ANONYMOUS) {\n VirtualName(\"FEXMem\", Result, length);\n }\n return Result;\n}\n\nvoid VirtualName(const char* Name, void* Ptr, size_t Size) {\n static bool Supports {true};\n if (Supports) {\n auto Result = prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, Ptr, Size, Name);\n if (Result == -1) {\n // Disable any additional attempts.\n Supports = false;\n }\n }\n}\n\nint FEX_munmap(void* addr, size_t length) {\n int Result = Alloc64->Munmap(addr, length);\n\n if (Result != 0) {\n errno = -Result;\n return -1;\n }\n return Result;\n}\n\n#pragma GCC diagnostic push\n#pragma GCC diagnostic ignored \"-Wdeprecated-declarations\"\n\nstatic void AssignHookOverrides(size_t PageSize) {\n SetupAllocatorHooks(FEX_mmap, FEX_munmap);\n FEXCore::Allocator::mmap = FEX_mmap;\n FEXCore::Allocator::munmap = FEX_munmap;\n InitializeAllocator(PageSize);\n}\n\nvoid SetupHooks(size_t PageSize) {\n Alloc64 = Alloc::OSAllocator::Create64BitAllocator();\n AssignHookOverrides(PageSize);\n}\n\nvoid ClearHooks() {\n SetupAllocatorHooks(::mmap, ::munmap);\n FEXCore::Allocator::mmap = ::mmap;\n FEXCore::Allocator::munmap = ::munmap;\n\n Alloc::OSAllocator::ReleaseAllocatorWorkaround(std::move(Alloc64));\n}\n#pragma GCC diagnostic pop\n\nFEX_DEFAULT_VISIBILITY size_t DetermineVASize() {\n if (HostVASize) {\n return HostVASize;\n }\n\n static constexpr std::array TLBSizes = {\n 57, 52, 48, 47, 42, 39, 36,\n };\n\n for (auto Bits : TLBSizes) {\n uintptr_t Size = 1ULL << Bits;\n // Just try allocating\n // We can't actually determine VA size on ARM safely\n auto Find = [](uintptr_t Size) -> bool {\n for (int i = 0; i < 64; ++i) {\n // Try grabbing a some of the top pages of the range\n // x86 allocates some high pages in the top end\n void* Ptr = ::mmap(reinterpret_cast(Size - FEXCore::Utils::FEX_PAGE_SIZE * i), FEXCore::Utils::FEX_PAGE_SIZE, PROT_NONE,\n MAP_FIXED_NOREPLACE | MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);\n if (Ptr != (void*)~0ULL) {\n ::munmap(Ptr, FEXCore::Utils::FEX_PAGE_SIZE);\n if (Ptr == (void*)(Size - FEXCore::Utils::FEX_PAGE_SIZE * i)) {\n return true;\n }\n }\n }\n return false;\n };\n\n if (Find(Size)) {\n HostVASize = Bits;\n return Bits;\n }\n }\n\n LOGMAN_MSG_A_FMT(\"Couldn't determine host VA size\");\n FEX_UNREACHABLE;\n}\n\n#define STEAL_LOG(...) // fprintf(stderr, __VA_ARGS__)\n\nfextl::vector CollectMemoryGaps(uintptr_t Begin, uintptr_t End, int MapsFD) {\n fextl::vector Regions;\n\n uintptr_t RegionEnd = 0;\n\n char Buffer[2048];\n const char* Cursor = Buffer;\n ssize_t Remaining = 0;\n\n bool EndOfFileReached = false;\n\n while (true) {\n const auto line_begin = Cursor;\n auto line_end = std::find(line_begin, Cursor + Remaining, '\\n');\n\n // Check if the buffered data covers the entire line.\n // If not, try buffering more data.\n if (line_end == Cursor + Remaining) {\n if (EndOfFileReached) {\n // No more data to buffer. Add remaining memory and return.\n const auto MapBegin = std::max(RegionEnd, Begin);\n STEAL_LOG(\"[%d] EndOfFile; MapBegin: %016lX MapEnd: %016lX\\n\", __LINE__, MapBegin, End);\n if (End > MapBegin) {\n Regions.push_back({(void*)MapBegin, End - MapBegin});\n }\n\n return Regions;\n }\n\n // Move pending content back to the beginning, then buffer more data.\n std::copy(Cursor, Cursor + Remaining, std::begin(Buffer));\n auto PendingBytes = Remaining;\n do {\n Remaining = read(MapsFD, Buffer + PendingBytes, sizeof(Buffer) - PendingBytes);\n } while (Remaining == -1 && errno == EAGAIN);\n\n if (Remaining < sizeof(Buffer) - PendingBytes) {\n EndOfFileReached = true;\n }\n\n Remaining += PendingBytes;\n\n Cursor = Buffer;\n continue;\n }\n\n // Parse mapped region in the format \"fffff7cc3000-fffff7cc4000 r--p ...\"\n {\n uintptr_t RegionBegin {};\n auto result = std::from_chars(Cursor, line_end, RegionBegin, 16);\n LogMan::Throw::AFmt(result.ec == std::errc {} && *result.ptr == '-', \"Unexpected line format\");\n Cursor = result.ptr + 1;\n\n // Add gap between the previous region and the current one\n const auto MapBegin = std::max(RegionEnd, Begin);\n const auto MapEnd = std::min(RegionBegin, End);\n if (MapEnd > MapBegin) {\n Regions.push_back({(void*)MapBegin, MapEnd - MapBegin});\n }\n\n result = std::from_chars(Cursor, line_end, RegionEnd, 16);\n LogMan::Throw::AFmt(result.ec == std::errc {} && *result.ptr == ' ', \"Unexpected line format\");\n Cursor = result.ptr + 1;\n\n STEAL_LOG(\"[%d] parsed line: RegionBegin=%016lX RegionEnd=%016lX\\n\", __LINE__, RegionBegin, RegionEnd);\n\n if (RegionEnd >= End) {\n // Early return if we are completely beyond the allocation space.\n return Regions;\n }\n }\n\n Remaining -= line_end + 1 - line_begin;\n Cursor = line_end + 1;\n }\n FEX_UNREACHABLE;\n}\n\nfextl::vector StealMemoryRegion(uintptr_t Begin, uintptr_t End) {\n const uintptr_t StackLocation_u64 = reinterpret_cast(alloca(0));\n\n const int MapsFD = open(\"/proc/self/maps\", O_RDONLY);\n LogMan::Throw::AFmt(MapsFD != -1, \"Failed to open /proc/self/maps\");\n\n auto Regions = CollectMemoryGaps(Begin, End, MapsFD);\n close(MapsFD);\n\n // If the memory bounds include the stack, blocking all memory regions will\n // limit the stack size to the current value. To allow some stack growth,\n // we don't block the memory gap directly below the stack memory but\n // instead map it as readable+writable.\n {\n auto StackRegionIt = std::find_if(Regions.begin(), Regions.end(), [StackLocation_u64](auto& Region) {\n return reinterpret_cast(Region.Ptr) + Region.Size > StackLocation_u64;\n });\n\n // If no gap crossing the stack pointer was found but the SP is within\n // the given bounds, the stack mapping is right after the last gap.\n bool IsStackMapping = StackRegionIt != Regions.end() || StackLocation_u64 <= End;\n\n if (IsStackMapping && StackRegionIt != Regions.begin() &&\n reinterpret_cast(std::prev(StackRegionIt)->Ptr) + std::prev(StackRegionIt)->Size <= End) {\n // Allocate the region under the stack as READ | WRITE so the stack can still grow\n --StackRegionIt;\n\n auto Alloc =\n ::mmap(StackRegionIt->Ptr, StackRegionIt->Size, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE | MAP_FIXED, -1, 0);\n\n LogMan::Throw::AFmt(Alloc != MAP_FAILED, \"StealMemoryRegion:Stack: mmap({}, {:x}) failed: {}\", fmt::ptr(StackRegionIt->Ptr),\n StackRegionIt->Size, errno);\n LogMan::Throw::AFmt(Alloc == StackRegionIt->Ptr, \"mmap returned {} instead of {}\", Alloc, fmt::ptr(StackRegionIt->Ptr));\n\n Regions.erase(StackRegionIt);\n }\n }\n\n // Block remaining memory gaps\n for (auto RegionIt = Regions.begin(); RegionIt != Regions.end(); ++RegionIt) {\n auto Alloc = ::mmap(RegionIt->Ptr, RegionIt->Size, PROT_NONE, MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE | MAP_FIXED_NOREPLACE, -1, 0);\n\n LogMan::Throw::AFmt(Alloc != MAP_FAILED, \"StealMemoryRegion: mmap({}, {:x}) failed: {}\", fmt::ptr(RegionIt->Ptr), RegionIt->Size, errno);\n LogMan::Throw::AFmt(Alloc == RegionIt->Ptr, \"mmap returned {} instead of {}\", Alloc, fmt::ptr(RegionIt->Ptr));\n }\n\n return Regions;\n}\n\nfextl::vector Setup48BitAllocatorIfExists(size_t PageSize) {\n size_t Bits = FEXCore::Allocator::DetermineVASize();\n if (Bits < 48) {\n return {};\n }\n\n uintptr_t Begin48BitVA = 0x0'8000'0000'0000ULL;\n uintptr_t End48BitVA = 0x1'0000'0000'0000ULL;\n auto Regions = StealMemoryRegion(Begin48BitVA, End48BitVA);\n\n Alloc64 = Alloc::OSAllocator::Create64BitAllocatorWithRegions(Regions);\n AssignHookOverrides(PageSize);\n\n return Regions;\n}\n\nvoid ReclaimMemoryRegion(const fextl::vector& Regions) {\n for (const auto& Region : Regions) {\n ::munmap(Region.Ptr, Region.Size);\n }\n}\n\nvoid LockBeforeFork(FEXCore::Core::InternalThreadState* Thread) {\n if (Alloc64) {\n Alloc64->LockBeforeFork(Thread);\n }\n}\n\nvoid UnlockAfterFork(FEXCore::Core::InternalThreadState* Thread, bool Child) {\n if (Alloc64) {\n Alloc64->UnlockAfterFork(Thread, Child);\n }\n}\n} // namespace FEXCore::Allocator\n#endif\n" }, { "path": "FEXCore/Source/Utils/Allocator.h", "content": "// SPDX-License-Identifier: MIT\n#pragma once\n\nnamespace FEXCore::Core {\nstruct InternalThreadState;\n}\n\nnamespace FEXCore::Allocator {\nvoid LockBeforeFork(FEXCore::Core::InternalThreadState* Thread);\nvoid UnlockAfterFork(FEXCore::Core::InternalThreadState* Thread, bool Child);\n} // namespace FEXCore::Allocator\n" }, { "path": "FEXCore/Source/Utils/AllocatorHooks.cpp", "content": "// SPDX-License-Identifier: MIT\n#ifdef ENABLE_FEX_ALLOCATOR\n#include \n#ifndef _WIN32\n#include \n#include \n#include \n#else\n#define NTDDI_VERSION 0x0A000005\n#include \n#endif\n#endif\n\n#include \n#include \n#include \n#include \n#include \n#include \n\nnamespace FEXCore::Allocator {\nusing mmap_hook_type = void* (*)(void* addr, size_t length, int prot, int flags, int fd, off_t offset);\nusing munmap_hook_type = int (*)(void* addr, size_t length);\n\n#ifdef ENABLE_FEX_ALLOCATOR\ntypedef void* (*rp_mmap_hook_type)(size_t size, size_t alignment, size_t* offset, size_t* mapped_size);\ntypedef void (*rp_munmap_hook_type)(void* address, size_t offset, size_t mapped_size);\nextern \"C\" rp_mmap_hook_type rp_mmap_hook;\nextern \"C\" rp_munmap_hook_type rp_munmap_hook;\n\n#ifndef _WIN32\nmmap_hook_type fex_mmap_hook = ::mmap;\nmunmap_hook_type fex_munmap_hook = ::munmap;\n#endif\n\n// Assume a 64KB page size until told otherwise.\nstatic rpmalloc_config_t global_config {\n .page_size = 64 * 1024,\n // THP causes crashes for some reason.\n .enable_huge_pages = 0,\n .disable_decommit = 0,\n .page_name = \"FEXAllocator\",\n .huge_page_name = \"FEXAllocator\",\n .unmap_on_finalize = 0,\n};\n\nvoid* malloc(size_t size) {\n return ::rpmalloc(size);\n}\nvoid* calloc(size_t n, size_t size) {\n return ::rpcalloc(n, size);\n}\nvoid* memalign(size_t align, size_t s) {\n return ::rpmemalign(align, s);\n}\nvoid* valloc(size_t size) {\n return ::rpaligned_alloc(global_config.page_size, size);\n}\nint posix_memalign(void** r, size_t a, size_t s) {\n void* ptr;\n auto res = ::rpposix_memalign(&ptr, a, s);\n *r = ptr;\n return res;\n}\nvoid* realloc(void* ptr, size_t size) {\n return ::rprealloc(ptr, size);\n}\nvoid free(void* ptr) {\n return ::rpfree(ptr);\n}\nsize_t malloc_usable_size(void* ptr) {\n return ::rpmalloc_usable_size(ptr);\n}\nvoid* aligned_alloc(size_t a, size_t s) {\n return ::rpaligned_alloc(a, s);\n}\nvoid aligned_free(void* ptr) {\n return ::rpfree(ptr);\n}\n\nvoid InitializeThread() {\n rpmalloc_thread_initialize();\n}\n\n#ifndef _WIN32\n[[nodiscard]]\nconstexpr uint64_t AlignUp(uint64_t value, uint64_t size) {\n return value + (size - value % size) % size;\n}\n\nstatic void* FEX_rp_mmap(size_t size, size_t alignment, size_t* offset, size_t* mapped_size) {\n#define pointer_offset(ptr, ofs) (void*)((char*)(ptr) + (ptrdiff_t)(ofs))\n // If the alignment is less than the operating page size then alignment is guaranteed. Just remove it.\n if (alignment < global_config.page_size) {\n alignment = 0;\n }\n\n size_t map_size = AlignUp(size + alignment, global_config.page_size);\n auto ptr = fex_mmap_hook(0, map_size, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);\n\n if (ptr == MAP_FAILED) {\n ptr = nullptr;\n } else {\n#ifndef PR_SET_VMA\n#define PR_SET_VMA 0x53564d41\n#endif\n\n#ifndef PR_SET_VMA_ANON_NAME\n#define PR_SET_VMA_ANON_NAME 0\n#endif\n prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, ptr, map_size, global_config.page_name);\n\n // Disable HUGEPAGE on allocation from rpmalloc.\n madvise(ptr, map_size, MADV_NOHUGEPAGE);\n }\n\n if (ptr == nullptr) {\n fprintf(stderr, \"Failed to map VMA region.\");\n return nullptr;\n }\n\n if (alignment) {\n size_t padding = ((uintptr_t)ptr & (uintptr_t)(alignment - 1));\n if (padding) {\n padding = alignment - padding;\n }\n ptr = pointer_offset(ptr, padding);\n *offset = padding;\n }\n *mapped_size = map_size;\n return ptr;\n}\n\nstatic void FEX_rp_memory_commit(void* address, size_t size) {\n // NOP-implementation.\n}\n\nstatic void FEX_rp_memory_decommit(void* address, size_t size) {\n if (global_config.disable_decommit) {\n return;\n }\n\n if (madvise(address, size, MADV_DONTNEED)) {\n fprintf(stderr, \"Failed to decommit VMA region.\");\n }\n}\n\nstatic void FEX_rp_memory_unmap(void* address, size_t offset, size_t mapped_size) {\n address = pointer_offset(address, -(int32_t)offset);\n int Result = fex_munmap_hook(address, mapped_size);\n if (Result == -1) {\n fprintf(stderr, \"Failed to unmap VMA region.\");\n }\n#undef pointer_offset\n}\n\nvoid SetupAllocatorHooks(mmap_hook_type MMapHook, munmap_hook_type MunmapHook) {\n fex_mmap_hook = MMapHook;\n fex_munmap_hook = MunmapHook;\n}\n\nstatic rpmalloc_interface_t global_interface {\n .memory_map = FEX_rp_mmap,\n .memory_commit = FEX_rp_memory_commit,\n .memory_decommit = FEX_rp_memory_decommit,\n .memory_unmap = FEX_rp_memory_unmap,\n .map_fail_callback = nullptr,\n .error_callback = nullptr,\n};\n\nvoid InitializeAllocator(size_t PageSize) {\n global_config.page_size = PageSize;\n rpmalloc_initialize_config(&global_interface, &global_config);\n rp_mmap_hook = FEX_rp_mmap;\n rp_munmap_hook = FEX_rp_memory_unmap;\n}\n#endif\n\n#elif defined(_WIN32)\n#error \"Tried building _WIN32 without jemalloc\"\n\n#else\nvoid InitializeThread() {}\n\nvoid* malloc(size_t size) {\n return ::malloc(size);\n}\nvoid* calloc(size_t n, size_t size) {\n return ::calloc(n, size);\n}\nvoid* memalign(size_t align, size_t s) {\n return ::memalign(align, s);\n}\nvoid* valloc(size_t size) {\n return ::valloc(size);\n}\nint posix_memalign(void** r, size_t a, size_t s) {\n return ::posix_memalign(r, a, s);\n}\nvoid* realloc(void* ptr, size_t size) {\n return ::realloc(ptr, size);\n}\nvoid free(void* ptr) {\n return ::free(ptr);\n}\nsize_t malloc_usable_size(void* ptr) {\n return ::malloc_usable_size(ptr);\n}\nvoid* aligned_alloc(size_t a, size_t s) {\n return ::aligned_alloc(a, s);\n}\nvoid aligned_free(void* ptr) {\n return ::free(ptr);\n}\n\nvoid SetupAllocatorHooks(mmap_hook_type MMapHook, munmap_hook_type MunmapHook) {}\n\nvoid InitializeAllocator(size_t PageSize) {}\n\n#endif\n} // namespace FEXCore::Allocator\n" }, { "path": "FEXCore/Source/Utils/AllocatorOverride.cpp", "content": "// SPDX-License-Identifier: MIT\n#include \n#include \n#include \n\n#include \n\n#include \n#include \n#include \n#include \n\nextern \"C\" {\n// The majority of FEX internal code should avoid using the glibc allocator. To ensure glibc allocations don't accidentally slip\n// in, FEX overrides these glibc functions with faulting variants.\n//\n// A notable exception is thunks, which should still use glibc allocations and avoid using `fextl::` namespace.\n//\n// Other minor exceptions throughout FEX use the `YesIKnowImNotSupposedToUseTheGlibcAllocator` helper to temporarily disable faulting.\n#define GLIBC_ALIAS_FUNCTION(func) __attribute__((alias(#func), visibility(\"default\")))\nextern void* __libc_calloc(size_t, size_t);\nvoid* calloc(size_t, size_t) GLIBC_ALIAS_FUNCTION(fault_calloc);\n\nextern void __libc_free(void*);\nvoid free(void*) GLIBC_ALIAS_FUNCTION(fault_free);\n\nextern void* __libc_malloc(size_t);\nvoid* malloc(size_t) GLIBC_ALIAS_FUNCTION(fault_malloc);\n\nextern void* __libc_memalign(size_t, size_t);\nvoid* memalign(size_t, size_t) GLIBC_ALIAS_FUNCTION(fault_memalign);\n\nextern void* __libc_realloc(void*, size_t);\nvoid* realloc(void*, size_t) GLIBC_ALIAS_FUNCTION(fault_realloc);\n\nextern void* __libc_valloc(size_t);\nvoid* valloc(size_t) GLIBC_ALIAS_FUNCTION(fault_valloc);\n\nextern int __posix_memalign(void**, size_t, size_t);\nint posix_memalign(void**, size_t, size_t) GLIBC_ALIAS_FUNCTION(fault_posix_memalign);\n\nextern size_t __malloc_usable_size(void*);\nsize_t malloc_usable_size(void*) GLIBC_ALIAS_FUNCTION(fault_malloc_usable_size);\n\n// Reuse __libc_memalign\nvoid* aligned_alloc(size_t, size_t) GLIBC_ALIAS_FUNCTION(fault_aligned_alloc);\n}\n\nnamespace FEXCore::Allocator {\n// Enable or disable allocation faulting globally.\nstatic bool GlobalEvaluate {};\n\n// Enable or disable allocation faulting per-thread.\nstatic thread_local uint64_t SkipEvalForThread {};\n\n// Internal memory allocation hooks to allow non-faulting allocations through.\nauto calloc_ptr = __libc_calloc;\nauto free_ptr = __libc_free;\nauto malloc_ptr = __libc_malloc;\nauto memalign_ptr = __libc_memalign;\nauto realloc_ptr = __libc_realloc;\nauto valloc_ptr = __libc_valloc;\nauto posix_memalign_ptr = ::posix_memalign;\nauto malloc_usable_size_ptr = ::malloc_usable_size;\nauto aligned_alloc_ptr = __libc_memalign;\n\n// Constructor for per-thread allocation faulting check.\nYesIKnowImNotSupposedToUseTheGlibcAllocator::YesIKnowImNotSupposedToUseTheGlibcAllocator() {\n ++SkipEvalForThread;\n}\n\n// Destructor for per-thread allocation faulting check.\nYesIKnowImNotSupposedToUseTheGlibcAllocator::~YesIKnowImNotSupposedToUseTheGlibcAllocator() {\n --SkipEvalForThread;\n}\n\n// Hard disabling of per-thread allocation fault checking.\n// No coming back from this, used on thread destruction.\nFEX_DEFAULT_VISIBILITY void YesIKnowImNotSupposedToUseTheGlibcAllocator::HardDisable() {\n // Just set it to half of its maximum value so it never wraps back around.\n SkipEvalForThread = std::numeric_limits::max() / 2;\n}\n\n// Enable global fault checking.\nvoid SetupFaultEvaluate() {\n GlobalEvaluate = true;\n}\n\n// Disable global fault checking.\nvoid ClearFaultEvaluate() {\n GlobalEvaluate = false;\n}\n\n// Evaluate if a glibc hooked allocation should fault.\nvoid EvaluateReturnAddress(void* Return) {\n if (!GlobalEvaluate) {\n // Fault evaluation disabled globally.\n return;\n }\n\n if (SkipEvalForThread) {\n // Fault evaluation currently disabled for this thread.\n return;\n }\n\n // We don't know where we are when allocating. Make sure to be safe and generate the string on the stack.\n // Print an error message to let a developer know that an allocation faulted.\n char Tmp[512];\n auto Res = fmt::format_to_n(Tmp, 512, \"ERROR: Requested memory using non-FEX allocator at 0x{:x}\\n\", reinterpret_cast(Return));\n Tmp[Res.size] = 0;\n write(STDERR_FILENO, Tmp, Res.size);\n\n // Trap the execution to stop FEX in its tracks.\n FEX_TRAP_EXECUTION;\n}\n} // namespace FEXCore::Allocator\n\nextern \"C\" {\n// These are the glibc allocator override symbols.\n// These will override the glibc allocators and then check if the allocation should fault.\nvoid* fault_calloc(size_t n, size_t size) {\n FEXCore::Allocator::EvaluateReturnAddress(__builtin_extract_return_addr(__builtin_return_address(0)));\n return FEXCore::Allocator::calloc_ptr(n, size);\n}\nvoid fault_free(void* ptr) {\n FEXCore::Allocator::EvaluateReturnAddress(__builtin_extract_return_addr(__builtin_return_address(0)));\n FEXCore::Allocator::free_ptr(ptr);\n}\nvoid* fault_malloc(size_t size) {\n FEXCore::Allocator::EvaluateReturnAddress(__builtin_extract_return_addr(__builtin_return_address(0)));\n return FEXCore::Allocator::malloc_ptr(size);\n}\nvoid* fault_memalign(size_t align, size_t s) {\n FEXCore::Allocator::EvaluateReturnAddress(__builtin_extract_return_addr(__builtin_return_address(0)));\n return FEXCore::Allocator::memalign_ptr(align, s);\n}\nvoid* fault_realloc(void* ptr, size_t size) {\n FEXCore::Allocator::EvaluateReturnAddress(__builtin_extract_return_addr(__builtin_return_address(0)));\n return FEXCore::Allocator::realloc_ptr(ptr, size);\n}\nvoid* fault_valloc(size_t size) {\n FEXCore::Allocator::EvaluateReturnAddress(__builtin_extract_return_addr(__builtin_return_address(0)));\n return FEXCore::Allocator::valloc_ptr(size);\n}\nint fault_posix_memalign(void** r, size_t a, size_t s) {\n FEXCore::Allocator::EvaluateReturnAddress(__builtin_extract_return_addr(__builtin_return_address(0)));\n return FEXCore::Allocator::posix_memalign_ptr(r, a, s);\n}\nsize_t fault_malloc_usable_size(void* ptr) {\n FEXCore::Allocator::EvaluateReturnAddress(__builtin_extract_return_addr(__builtin_return_address(0)));\n return FEXCore::Allocator::malloc_usable_size_ptr(ptr);\n}\nvoid* fault_aligned_alloc(size_t a, size_t s) {\n FEXCore::Allocator::EvaluateReturnAddress(__builtin_extract_return_addr(__builtin_return_address(0)));\n return FEXCore::Allocator::aligned_alloc_ptr(a, s);\n}\n}\n" }, { "path": "FEXCore/Source/Utils/ArchHelpers/Arm64.cpp", "content": "// SPDX-License-Identifier: MIT\n\n#include \"Interface/Core/CPUBackend.h\"\n#include \"Interface/Context/Context.h\"\n#include \"Utils/SpinWaitLock.h\"\n\n#include \n#include \n#include \n#include \n#include \n\n#include \n#include \n\nnamespace FEXCore::ArchHelpers::Arm64 {\nconstexpr uint32_t CASPAL_MASK = 0xBF'E0'FC'00;\nconstexpr uint32_t CASPAL_INST = 0x08'60'FC'00;\n\nconstexpr uint32_t CASAL_MASK = 0x3F'E0'FC'00;\nconstexpr uint32_t CASAL_INST = 0x08'E0'FC'00;\n\nconstexpr uint32_t ATOMIC_MEM_MASK = 0x3B200C00;\nconstexpr uint32_t ATOMIC_MEM_INST = 0x38200000;\n\nconstexpr uint32_t RCPC2_MASK = 0x3F'E0'0C'00;\nconstexpr uint32_t LDAPUR_INST = 0x19'40'00'00;\nconstexpr uint32_t STLUR_INST = 0x19'00'00'00;\n\nconstexpr uint32_t LDAXP_MASK = 0xBF'FF'80'00;\nconstexpr uint32_t LDAXP_INST = 0x88'7F'80'00;\n\nconstexpr uint32_t STLXP_MASK = 0xBF'E0'80'00;\nconstexpr uint32_t STLXP_INST = 0x88'20'80'00;\n\nconstexpr uint32_t LDAXR_MASK = 0x3F'FF'FC'00;\nconstexpr uint32_t LDAXR_INST = 0x08'5F'FC'00;\nconstexpr uint32_t LDAR_INST = 0x08'DF'FC'00;\nconstexpr uint32_t LDAPR_INST = 0x38'BF'C0'00;\nconstexpr uint32_t STLR_INST = 0x08'9F'FC'00;\n\nconstexpr uint32_t STLXR_MASK = 0x3F'E0'FC'00;\nconstexpr uint32_t STLXR_INST = 0x08'00'FC'00;\n\n// Load/store register (register offset) (Rm encoded as xzr)\nconstexpr uint32_t LDSTREGISTER_MASK = 0b0011'1111'1111'1111'1111'1100'0000'0000;\nconstexpr uint32_t LDR_INST = 0b0011'1000'0111'1111'0110'1000'0000'0000;\nconstexpr uint32_t STR_INST = 0b0011'1000'0011'1111'0110'1000'0000'0000;\n\nconstexpr uint32_t LDSTUNSCALED_MASK = 0b0011'1011'1110'0000'0000'1100'0000'0000;\nconstexpr uint32_t LDUR_INST = 0b0011'1000'0100'0000'0000'0000'0000'0000;\nconstexpr uint32_t STUR_INST = 0b0011'1000'0000'0000'0000'0000'0000'0000;\n\nconstexpr uint32_t LDSTP_MASK = 0b0011'1011'1000'0000'0000'0000'0000'0000;\nconstexpr uint32_t STP_INST = 0b0010'1001'0000'0000'0000'0000'0000'0000;\n\nconstexpr uint32_t CBNZ_MASK = 0x7F'00'00'00;\nconstexpr uint32_t CBNZ_INST = 0x35'00'00'00;\n\nconstexpr uint32_t ALU_OP_MASK = 0x7F'20'00'00;\nconstexpr uint32_t ADD_INST = 0x0B'00'00'00;\nconstexpr uint32_t SUB_INST = 0x4B'00'00'00;\nconstexpr uint32_t ADD_SHIFT_INST = 0x0B'20'00'00;\nconstexpr uint32_t SUB_SHIFT_INST = 0x4B'20'00'00;\nconstexpr uint32_t CMP_INST = 0x6B'00'00'00;\nconstexpr uint32_t CMP_SHIFT_INST = 0x6B'20'00'00;\nconstexpr uint32_t AND_INST = 0x0A'00'00'00;\nconstexpr uint32_t BIC_INST = 0x0A'20'00'00;\nconstexpr uint32_t OR_INST = 0x2A'00'00'00;\nconstexpr uint32_t ORN_INST = 0x2A'20'00'00;\nconstexpr uint32_t EOR_INST = 0x4A'00'00'00;\nconstexpr uint32_t EON_INST = 0x4A'20'00'00;\n\nconstexpr uint32_t CCMP_MASK = 0x7F'E0'0C'10;\nconstexpr uint32_t CCMP_INST = 0x7A'40'00'00;\n\nconstexpr uint32_t CLREX_MASK = 0xFF'FF'F0'FF;\nconstexpr uint32_t CLREX_INST = 0xD5'03'30'5F;\n\nenum ExclusiveAtomicPairType {\n TYPE_SWAP,\n TYPE_ADD,\n TYPE_SUB,\n TYPE_AND,\n TYPE_BIC,\n TYPE_OR,\n TYPE_ORN,\n TYPE_EOR,\n TYPE_EON,\n TYPE_NEG, // This is just a sub with zero. Need to know the differences\n};\n\n// Load ops are 4 bits\n// Acquire and release bits are independent on the instruction\nconstexpr uint32_t ATOMIC_ADD_OP = 0b0000;\nconstexpr uint32_t ATOMIC_CLR_OP = 0b0001;\nconstexpr uint32_t ATOMIC_EOR_OP = 0b0010;\nconstexpr uint32_t ATOMIC_SET_OP = 0b0011;\nconstexpr uint32_t ATOMIC_SWAP_OP = 0b1000;\n\nconstexpr uint32_t REGISTER_MASK = 0b11111;\nconstexpr uint32_t RD_OFFSET = 0;\nconstexpr uint32_t RN_OFFSET = 5;\nconstexpr uint32_t RM_OFFSET = 16;\n\nconstexpr uint32_t DMB = 0b1101'0101'0000'0011'0011'0000'1011'1111 | 0b1011'0000'0000; // Inner shareable all\n\nconstexpr uint32_t DMB_LD = 0b1101'0101'0000'0011'0011'0000'1011'1111 | 0b1101'0000'0000; // Inner shareable load\n\nstatic constexpr uint32_t GetRdReg(uint32_t Instr) {\n return (Instr >> RD_OFFSET) & REGISTER_MASK;\n}\n\nstatic constexpr uint32_t GetRnReg(uint32_t Instr) {\n return (Instr >> RN_OFFSET) & REGISTER_MASK;\n}\n\nstatic constexpr uint32_t GetRmReg(uint32_t Instr) {\n return (Instr >> RM_OFFSET) & REGISTER_MASK;\n}\n\nstatic void ClearICache(void* Begin, std::size_t Length) {\n __builtin___clear_cache(static_cast(Begin), static_cast(Begin) + Length);\n}\n\nstatic __uint128_t LoadAcquire128(uint64_t Addr) {\n __uint128_t Result {};\n uint64_t Lower;\n uint64_t Upper;\n // This specifically avoids using std::atomic<__uint128_t>\n // std::atomic helper does a ldaxp + stxp pair that crashes when the page is only mapped readable\n __asm volatile(\n R\"(\n ldaxp %[ResultLower], %[ResultUpper], [%[Addr]];\n clrex;\n)\"\n : [ResultLower] \"=r\"(Lower), [ResultUpper] \"=r\"(Upper)\n : [Addr] \"r\"(Addr)\n : \"memory\");\n Result = Upper;\n Result <<= 64;\n Result |= Lower;\n return Result;\n}\n\nstatic uint64_t LoadAcquire64(uint64_t Addr) {\n auto Atom = std::atomic_ref(*reinterpret_cast(Addr));\n return Atom.load(std::memory_order_acquire);\n}\n\nstatic bool StoreCAS64(uint64_t& Expected, uint64_t Val, uint64_t Addr) {\n auto Atom = std::atomic_ref(*reinterpret_cast(Addr));\n return Atom.compare_exchange_strong(Expected, Val);\n}\n\nstatic uint32_t LoadAcquire32(uint64_t Addr) {\n auto Atom = std::atomic_ref(*reinterpret_cast(Addr));\n return Atom.load(std::memory_order_acquire);\n}\n\nstatic bool StoreCAS32(uint32_t& Expected, uint32_t Val, uint64_t Addr) {\n auto Atom = std::atomic_ref(*reinterpret_cast(Addr));\n return Atom.compare_exchange_strong(Expected, Val);\n}\n\nstatic uint8_t LoadAcquire8(uint64_t Addr) {\n auto Atom = std::atomic_ref(*reinterpret_cast(Addr));\n return Atom.load(std::memory_order_acquire);\n}\n\nstatic bool StoreCAS8(uint8_t& Expected, uint8_t Val, uint64_t Addr) {\n auto Atom = std::atomic_ref(*reinterpret_cast(Addr));\n return Atom.compare_exchange_strong(Expected, Val);\n}\n\nstatic uint16_t DoLoad16(uint64_t Addr) {\n uint64_t AlignmentMask = 0b1111;\n if ((Addr & AlignmentMask) == 15) {\n // Address crosses over 16byte or 64byte threshold\n // Needs two loads\n uint64_t AddrUpper = Addr + 1;\n uint8_t ActualUpper {};\n uint8_t ActualLower {};\n // Careful ordering here\n ActualUpper = LoadAcquire8(AddrUpper);\n ActualLower = LoadAcquire8(Addr);\n\n uint16_t Result = ActualUpper;\n Result <<= 8;\n Result |= ActualLower;\n return Result;\n } else {\n AlignmentMask = 0b111;\n if ((Addr & AlignmentMask) == 7) {\n // Crosses 8byte boundary\n // Needs 128bit load\n // Fits within a 16byte region\n uint64_t Alignment = Addr & 0b1111;\n Addr &= ~0b1111ULL;\n\n __uint128_t TmpResult = LoadAcquire128(Addr);\n\n // Zexts the result\n uint16_t Result = TmpResult >> (Alignment * 8);\n return Result;\n } else {\n AlignmentMask = 0b11;\n if ((Addr & AlignmentMask) == 3) {\n // Crosses 4byte boundary\n // Needs 64bit Load\n uint64_t Alignment = Addr & AlignmentMask;\n Addr &= ~AlignmentMask;\n\n auto Atomic = std::atomic_ref(*reinterpret_cast(Addr));\n uint64_t TmpResult = Atomic.load();\n\n // Zexts the result\n uint16_t Result = TmpResult >> (Alignment * 8);\n return Result;\n } else {\n // Fits within 4byte boundary\n // Only needs 32bit Load\n // Only alignment offset will be 1 here\n uint64_t Alignment = Addr & AlignmentMask;\n Addr &= ~AlignmentMask;\n\n auto Atomic = std::atomic_ref(*reinterpret_cast(Addr));\n uint32_t TmpResult = Atomic.load();\n\n // Zexts the result\n uint16_t Result = TmpResult >> (Alignment * 8);\n return Result;\n }\n }\n }\n}\n\nstatic uint32_t DoLoad32(uint64_t Addr) {\n uint64_t AlignmentMask = 0b1111;\n if ((Addr & AlignmentMask) > 12) {\n // Address crosses over 16byte threshold\n // Needs dual 32bit load\n uint64_t Alignment = Addr & 0b11;\n Addr &= ~0b11ULL;\n\n uint64_t AddrUpper = Addr + 4;\n\n // Careful ordering here\n uint32_t ActualUpper = LoadAcquire32(AddrUpper);\n uint32_t ActualLower = LoadAcquire32(Addr);\n\n uint64_t Result = ActualUpper;\n Result <<= 32;\n Result |= ActualLower;\n return Result >> (Alignment * 8);\n } else {\n AlignmentMask = 0b111;\n if ((Addr & AlignmentMask) >= 5) {\n // Crosses 8byte boundary\n // Needs 128bit load\n // Fits within a 16byte region\n uint64_t Alignment = Addr & 0b1111;\n Addr &= ~0b1111ULL;\n\n __uint128_t TmpResult = LoadAcquire128(Addr);\n\n return TmpResult >> (Alignment * 8);\n } else {\n // Fits within 8byte boundary\n // Only needs 64bit CAS\n // Alignments can be [1,5)\n uint64_t Alignment = Addr & AlignmentMask;\n Addr &= ~AlignmentMask;\n\n auto Atomic = std::atomic_ref(*reinterpret_cast(Addr));\n uint64_t TmpResult = Atomic.load();\n\n return TmpResult >> (Alignment * 8);\n }\n }\n}\n\nstatic uint64_t DoLoad64(uint64_t Addr) {\n uint64_t AlignmentMask = 0b1111;\n if ((Addr & AlignmentMask) > 8) {\n uint64_t Alignment = Addr & 0b111;\n Addr &= ~0b111ULL;\n uint64_t AddrUpper = Addr + 8;\n\n // Crosses a 16byte boundary\n // Needs two 8 byte loads\n uint64_t ActualUpper {};\n uint64_t ActualLower {};\n // Careful ordering here\n ActualUpper = LoadAcquire64(AddrUpper);\n ActualLower = LoadAcquire64(Addr);\n\n __uint128_t Result = ActualUpper;\n Result <<= 64;\n Result |= ActualLower;\n return Result >> (Alignment * 8);\n } else {\n // Fits within a 16byte region\n uint64_t Alignment = Addr & AlignmentMask;\n Addr &= ~AlignmentMask;\n __uint128_t TmpResult = LoadAcquire128(Addr);\n uint64_t Result = TmpResult >> (Alignment * 8);\n return Result;\n }\n}\n\nstatic __uint128_t DoLoad128(uint64_t Addr) {\n // Any misalignment here means we cross a 16byte boundary\n // So we need two 128bit loads\n uint64_t Alignment = Addr & 0b1111;\n Addr &= ~0b1111ULL;\n uint64_t AddrUpper = Addr + 16;\n\n union AlignedData {\n struct {\n __uint128_t Lower;\n __uint128_t Upper;\n } Large;\n struct {\n uint8_t Data[32];\n } Bytes;\n };\n\n AlignedData* Data = reinterpret_cast(alloca(sizeof(AlignedData)));\n Data->Large.Upper = LoadAcquire128(AddrUpper);\n Data->Large.Lower = LoadAcquire128(Addr);\n\n __uint128_t Result {};\n memcpy(&Result, &Data->Bytes.Data[Alignment], sizeof(Result));\n return Result;\n}\n\nstatic bool RunCASPAL(uint64_t* GPRs, uint32_t Size, uint32_t DesiredReg1, uint32_t DesiredReg2, uint32_t ExpectedReg1,\n uint32_t ExpectedReg2, uint32_t AddressReg, uint32_t* StrictSplitLockMutex) {\n\n std::optional> Lock {};\n if (Size == 0) {\n // 32bit\n uint64_t Addr = GPRs[AddressReg];\n\n uint32_t DesiredLower = GPRs[DesiredReg1];\n uint32_t DesiredUpper = GPRs[DesiredReg2];\n\n uint32_t ExpectedLower = GPRs[ExpectedReg1];\n uint32_t ExpectedUpper = GPRs[ExpectedReg2];\n\n // Cross-cacheline CAS doesn't work on ARM\n // It isn't even guaranteed to work on x86\n // Intel will do a \"split lock\" which locks the full bus\n // AMD will tear instead\n // Both cross-cacheline and cross 16byte both need dual CAS loops that can tear\n // ARMv8.4 LSE2 solves all atomic issues except cross-cacheline\n\n // Check for Split lock across a cacheline\n if ((Addr & 63) > 56) {\n FEXCORE_TELEMETRY_SET(TYPE_HAS_SPLIT_LOCKS, 1);\n if (StrictSplitLockMutex && !Lock.has_value()) {\n Lock.emplace(StrictSplitLockMutex);\n }\n }\n\n uint64_t AlignmentMask = 0b1111;\n if ((Addr & AlignmentMask) > 8) {\n FEXCORE_TELEMETRY_SET(TYPE_16BYTE_SPLIT, 1);\n if (StrictSplitLockMutex && !Lock.has_value()) {\n Lock.emplace(StrictSplitLockMutex);\n }\n\n uint64_t Alignment = Addr & 0b111;\n Addr &= ~0b111ULL;\n uint64_t AddrUpper = Addr + 8;\n\n // Crosses a 16byte boundary\n // Need to do 256bit atomic, but since that doesn't exist we need to do a dual CAS loop\n __uint128_t Mask = ~0ULL;\n Mask <<= Alignment * 8;\n __uint128_t NegMask = ~Mask;\n __uint128_t TmpExpected {};\n __uint128_t TmpDesired {};\n\n __uint128_t Desired = DesiredUpper;\n Desired <<= 32;\n Desired |= DesiredLower;\n Desired <<= Alignment * 8;\n\n __uint128_t Expected = ExpectedUpper;\n Expected <<= 32;\n Expected |= ExpectedLower;\n Expected <<= Alignment * 8;\n\n while (1) {\n __uint128_t LoadOrderUpper = LoadAcquire64(AddrUpper);\n LoadOrderUpper <<= 64;\n __uint128_t TmpActual = LoadOrderUpper | LoadAcquire64(Addr);\n\n // Set up expected\n TmpExpected = TmpActual;\n TmpExpected &= NegMask;\n TmpExpected |= Expected;\n\n // Set up desired\n TmpDesired = TmpExpected;\n TmpDesired &= NegMask;\n TmpDesired |= Desired;\n\n uint64_t TmpExpectedLower = TmpExpected;\n uint64_t TmpExpectedUpper = TmpExpected >> 64;\n\n uint64_t TmpDesiredLower = TmpDesired;\n uint64_t TmpDesiredUpper = TmpDesired >> 64;\n\n if (TmpExpected == TmpActual) {\n if (StoreCAS64(TmpExpectedUpper, TmpDesiredUpper, AddrUpper)) {\n if (StoreCAS64(TmpExpectedLower, TmpDesiredLower, Addr)) {\n // Stored successfully\n return true;\n } else {\n // CAS managed to tear, we can't really solve this\n // Continue down the path to let the guest know values weren't expected\n FEXCORE_TELEMETRY_SET(TYPE_CAS_128BIT_TEAR, 1);\n }\n }\n\n TmpExpected = TmpExpectedUpper;\n TmpExpected <<= 64;\n TmpExpected |= TmpExpectedLower;\n } else {\n // Mismatch up front\n TmpExpected = TmpActual;\n }\n\n // Not successful\n // Now we need to check the results to see if we need to try again\n __uint128_t FailedResultOurBits = TmpExpected & Mask;\n __uint128_t FailedResultNotOurBits = TmpExpected & NegMask;\n\n __uint128_t FailedDesiredOurBits = TmpDesired & Mask;\n __uint128_t FailedDesiredNotOurBits = TmpDesired & NegMask;\n if ((FailedResultNotOurBits ^ FailedDesiredNotOurBits) != 0) {\n // If the bits changed that weren't part of our regular CAS then we need to try again\n continue;\n }\n if ((FailedResultOurBits ^ FailedDesiredOurBits) != 0) {\n // If the bits changed that we were wanting to change then we have failed and can return\n // We need to extract the bits and return them in EXPECTED\n uint64_t FailedResult = FailedResultOurBits >> (Alignment * 8);\n GPRs[ExpectedReg1] = FailedResult & ~0U;\n GPRs[ExpectedReg2] = FailedResult >> 32;\n return true;\n }\n\n // This happens in the case that between Load and CAS that something has store our desired in to the memory location\n // This means our CAS fails because what we wanted to store was already stored\n uint64_t FailedResult = FailedResultOurBits >> (Alignment * 8);\n GPRs[ExpectedReg1] = FailedResult & ~0U;\n GPRs[ExpectedReg2] = FailedResult >> 32;\n return true;\n }\n } else {\n // Fits within a 16byte region\n uint64_t Alignment = Addr & 0b1111;\n Addr &= ~0b1111ULL;\n auto Atomic128 = std::atomic_ref<__uint128_t>(*reinterpret_cast<__uint128_t*>(Addr));\n\n __uint128_t Mask = ~0ULL;\n Mask <<= Alignment * 8;\n __uint128_t NegMask = ~Mask;\n __uint128_t TmpExpected {};\n __uint128_t TmpDesired {};\n\n __uint128_t Desired = (uint64_t)DesiredUpper << 32 | DesiredLower;\n Desired <<= Alignment * 8;\n\n __uint128_t Expected = (uint64_t)ExpectedUpper << 32 | ExpectedLower;\n Expected <<= Alignment * 8;\n\n while (1) {\n TmpExpected = Atomic128.load();\n\n // Set up expected\n TmpExpected &= NegMask;\n TmpExpected |= Expected;\n\n // Set up desired\n TmpDesired = TmpExpected;\n TmpDesired &= NegMask;\n TmpDesired |= Desired;\n\n bool CASResult = Atomic128.compare_exchange_strong(TmpExpected, TmpDesired);\n if (CASResult) {\n // Successful, so we are done\n return true;\n } else {\n // Not successful\n // Now we need to check the results to see if we need to try again\n __uint128_t FailedResultOurBits = TmpExpected & Mask;\n __uint128_t FailedResultNotOurBits = TmpExpected & NegMask;\n\n __uint128_t FailedDesiredNotOurBits = TmpDesired & NegMask;\n if ((FailedResultNotOurBits ^ FailedDesiredNotOurBits) != 0) {\n // If the bits changed that weren't part of our regular CAS then we need to try again\n continue;\n }\n\n // This happens in the case that between Load and CAS that something has store our desired in to the memory location\n // This means our CAS fails because what we wanted to store was already stored\n uint64_t FailedResult = FailedResultOurBits >> (Alignment * 8);\n GPRs[ExpectedReg1] = FailedResult & ~0U;\n GPRs[ExpectedReg2] = FailedResult >> 32;\n return true;\n }\n }\n }\n }\n return false;\n}\n\nstatic bool HandleCASPAL(uint32_t Instr, uint64_t* GPRs, uint32_t* StrictSplitLockMutex) {\n uint32_t Size = (Instr >> 30) & 1;\n\n uint32_t DesiredReg1 = Instr & 0b11111;\n uint32_t DesiredReg2 = DesiredReg1 + 1;\n uint32_t ExpectedReg1 = (Instr >> 16) & 0b11111;\n uint32_t ExpectedReg2 = ExpectedReg1 + 1;\n uint32_t AddressReg = (Instr >> 5) & 0b11111;\n\n return RunCASPAL(GPRs, Size, DesiredReg1, DesiredReg2, ExpectedReg1, ExpectedReg2, AddressReg, StrictSplitLockMutex);\n}\n\nstatic uint64_t HandleCASPAL_ARMv8(uint32_t Instr, uintptr_t ProgramCounter, uint64_t* GPRs, uint32_t* StrictSplitLockMutex) {\n // caspair\n // [1] ldaxp(TMP2.W(), TMP3.W(), MemOperand(MemSrc)); <-- DataReg & AddrReg\n // [2] cmp(TMP2.W(), Expected.first.W()); <-- ExpectedReg1\n // [3] ccmp(TMP3.W(), Expected.second.W(), NoFlag, Condition::eq); <-- ExpectedREg2\n // [4] b(&LoopNotExpected, Condition::ne);\n // [5] stlxp(TMP2.W(), Desired.first.W(), Desired.second.W(), MemOperand(MemSrc)); <-- DesiredReg\n // [6] cbnz(TMP2.W(), &LoopTop);\n // [7] mov(Dst.first.W(), Expected.first.W());\n // [8] mov(Dst.second.W(), Expected.second.W());\n // [9] b(&LoopExpected);\n // [10] mov(Dst.first.W(), TMP2.W());\n // [11] mov(Dst.second.W(), TMP3.W());\n // [12] clrex();\n\n uint32_t* PC = (uint32_t*)ProgramCounter;\n\n uint32_t Size = (Instr >> 30) & 1;\n uint32_t AddrReg = (Instr >> 5) & 0x1F;\n uint32_t DataReg = Instr & 0x1F;\n uint32_t DataReg2 = (Instr >> 10) & 0x1F;\n\n uint32_t ExpectedReg1 {};\n uint32_t ExpectedReg2 {};\n\n uint32_t DesiredReg1 {};\n uint32_t DesiredReg2 {};\n\n if (Size == 1) {\n // 64-bit pair happens on paranoid vector loads\n // [1] ldaxp(TMP1, TMP2, MemSrc);\n // [2] clrex();\n //\n // 64-bit pair happens on paranoid vector stores\n // [1] ldaxp(xzr, TMP3, MemSrc); // <- Can hit SIGBUS\n // [2] stlxp(TMP3, TMP1, TMP2, MemSrc); // <- Can also hit SIGBUS\n // [3] cbnz(TMP3, &B); // < Overwritten with DMB\n\n if (DataReg == 31) {\n } else {\n uint32_t NextInstr = PC[1];\n if ((NextInstr & ArchHelpers::Arm64::CLREX_MASK) == ArchHelpers::Arm64::CLREX_INST) {\n uint64_t Addr = GPRs[AddrReg];\n\n auto Res = DoLoad128(Addr);\n // We set the result register if it isn't a zero register\n if (DataReg != 31) {\n GPRs[DataReg] = Res;\n }\n if (DataReg2 != 31) {\n GPRs[DataReg2] = Res >> 64;\n }\n\n // Skip ldaxp and clrex\n return 2 * sizeof(uint32_t);\n }\n }\n return 0;\n }\n\n // Only 32-bit pairs\n for (int i = 1; i < 10; i++) {\n uint32_t NextInstr = PC[i];\n if ((NextInstr & ArchHelpers::Arm64::ALU_OP_MASK) == ArchHelpers::Arm64::CMP_INST ||\n (NextInstr & ArchHelpers::Arm64::ALU_OP_MASK) == ArchHelpers::Arm64::CMP_SHIFT_INST) {\n ExpectedReg1 = GetRmReg(NextInstr);\n } else if ((NextInstr & ArchHelpers::Arm64::CCMP_MASK) == ArchHelpers::Arm64::CCMP_INST) {\n ExpectedReg2 = GetRmReg(NextInstr);\n } else if ((NextInstr & ArchHelpers::Arm64::STLXP_MASK) == ArchHelpers::Arm64::STLXP_INST) {\n DesiredReg1 = (NextInstr & 0x1F);\n DesiredReg2 = (NextInstr >> 10) & 0x1F;\n }\n }\n\n // mov expected into the temp registers used by JIT\n GPRs[DataReg] = GPRs[ExpectedReg1];\n GPRs[DataReg2] = GPRs[ExpectedReg2];\n\n if (RunCASPAL(GPRs, Size, DesiredReg1, DesiredReg2, DataReg, DataReg2, AddrReg, StrictSplitLockMutex)) {\n return 9 * sizeof(uint32_t); // skip to mov + clrex\n } else {\n return 0;\n }\n}\n\ntemplate\nusing CASExpectedFn = T (*)(T Src, T Expected);\ntemplate\nusing CASDesiredFn = T (*)(T Src, T Desired);\n\ntemplate\nstatic uint16_t DoCAS16(uint16_t DesiredSrc, uint16_t ExpectedSrc, uint64_t Addr, CASExpectedFn ExpectedFunction,\n CASDesiredFn DesiredFunction, uint32_t* StrictSplitLockMutex) {\n std::optional> Lock {};\n\n if ((Addr & 63) == 63) {\n FEXCORE_TELEMETRY_SET(TYPE_HAS_SPLIT_LOCKS, 1);\n if (StrictSplitLockMutex && !Lock.has_value()) {\n Lock.emplace(StrictSplitLockMutex);\n }\n }\n\n // 16 bit\n uint64_t AlignmentMask = 0b1111;\n if ((Addr & AlignmentMask) == 15) {\n FEXCORE_TELEMETRY_SET(TYPE_16BYTE_SPLIT, 1);\n if (StrictSplitLockMutex && !Lock.has_value()) {\n Lock.emplace(StrictSplitLockMutex);\n }\n\n // Address crosses over 16byte or 64byte threshold\n // Need a dual 8bit CAS loop\n uint64_t AddrUpper = Addr + 1;\n\n while (1) {\n uint8_t ActualUpper {};\n uint8_t ActualLower {};\n // Careful ordering here\n ActualUpper = LoadAcquire8(AddrUpper);\n ActualLower = LoadAcquire8(Addr);\n\n uint16_t Actual = ActualUpper;\n Actual <<= 8;\n Actual |= ActualLower;\n\n uint16_t Desired = DesiredFunction(Actual, DesiredSrc);\n uint8_t DesiredLower = Desired;\n uint8_t DesiredUpper = Desired >> 8;\n\n uint16_t Expected = ExpectedFunction(Actual, ExpectedSrc);\n uint8_t ExpectedLower = Expected;\n uint8_t ExpectedUpper = Expected >> 8;\n\n bool Tear = false;\n if (ActualUpper == ExpectedUpper && ActualLower == ExpectedLower) {\n if (StoreCAS8(ExpectedUpper, DesiredUpper, AddrUpper)) {\n if (StoreCAS8(ExpectedLower, DesiredLower, Addr)) {\n // Stored successfully\n return Expected;\n } else {\n // CAS managed to tear, we can't really solve this\n // Continue down the path to let the guest know values weren't expected\n Tear = true;\n FEXCORE_TELEMETRY_SET(TYPE_CAS_16BIT_TEAR, 1);\n }\n }\n\n ActualLower = ExpectedLower;\n }\n\n // If the bits changed that we were wanting to change then we have failed and can return\n // We need to extract the bits and return them in EXPECTED\n uint16_t FailedResult = ActualUpper;\n FailedResult <<= 8;\n FailedResult |= ActualLower;\n\n if constexpr (Retry) {\n if (Tear) {\n // If we are retrying and tearing then we can't do anything here\n // XXX: Resolve with TME\n return FailedResult;\n } else {\n // We can retry safely\n }\n } else {\n // Without Retry (CAS) then we have failed regardless of tear\n // CAS failed but handled successfully\n return FailedResult;\n }\n }\n } else {\n AlignmentMask = 0b111;\n if ((Addr & AlignmentMask) == 7) {\n // Crosses 8byte boundary\n // Needs 128bit CAS\n // Fits within a 16byte region\n uint64_t Alignment = Addr & 0b1111;\n Addr &= ~0b1111ULL;\n auto Atomic128 = std::atomic_ref<__uint128_t>(*reinterpret_cast<__uint128_t*>(Addr));\n\n __uint128_t Mask = 0xFFFF;\n Mask <<= Alignment * 8;\n __uint128_t NegMask = ~Mask;\n __uint128_t TmpExpected {};\n __uint128_t TmpDesired {};\n\n while (1) {\n TmpExpected = Atomic128.load();\n\n __uint128_t Desired = DesiredFunction(TmpExpected >> (Alignment * 8), DesiredSrc);\n Desired <<= Alignment * 8;\n\n __uint128_t Expected = ExpectedFunction(TmpExpected >> (Alignment * 8), ExpectedSrc);\n Expected <<= Alignment * 8;\n\n // Set up expected\n TmpExpected &= NegMask;\n TmpExpected |= Expected;\n\n // Set up desired\n TmpDesired = TmpExpected;\n TmpDesired &= NegMask;\n TmpDesired |= Desired;\n\n bool CASResult = Atomic128.compare_exchange_strong(TmpExpected, TmpDesired);\n if (CASResult) {\n // Successful, so we are done\n return Expected >> (Alignment * 8);\n } else {\n if constexpr (Retry) {\n // If we failed but we have enabled retry then just retry without checking results\n // CAS can't retry but atomic memory ops need to retry until passing\n continue;\n }\n // Not successful\n // Now we need to check the results to see if we need to try again\n __uint128_t FailedResultOurBits = TmpExpected & Mask;\n __uint128_t FailedResultNotOurBits = TmpExpected & NegMask;\n\n __uint128_t FailedDesiredNotOurBits = TmpDesired & NegMask;\n if ((FailedResultNotOurBits ^ FailedDesiredNotOurBits) != 0) {\n // If the bits changed that weren't part of our regular CAS then we need to try again\n continue;\n }\n\n // This happens in the case that between Load and CAS that something has store our desired in to the memory location\n // This means our CAS fails because what we wanted to store was already stored\n uint16_t FailedResult = FailedResultOurBits >> (Alignment * 8);\n // CAS failed but handled successfully\n return FailedResult;\n }\n }\n } else {\n AlignmentMask = 0b11;\n if ((Addr & AlignmentMask) == 3) {\n // Crosses 4byte boundary\n // Needs 64bit CAS\n uint64_t Alignment = Addr & AlignmentMask;\n Addr &= ~AlignmentMask;\n\n uint64_t Mask = 0xFFFF;\n Mask <<= Alignment * 8;\n\n uint64_t NegMask = ~Mask;\n\n uint64_t TmpExpected {};\n uint64_t TmpDesired {};\n\n auto Atomic = std::atomic_ref(*reinterpret_cast(Addr));\n while (1) {\n TmpExpected = Atomic.load();\n\n uint64_t Desired = DesiredFunction(TmpExpected >> (Alignment * 8), DesiredSrc);\n Desired <<= Alignment * 8;\n\n uint64_t Expected = ExpectedFunction(TmpExpected >> (Alignment * 8), ExpectedSrc);\n Expected <<= Alignment * 8;\n\n // Set up expected\n TmpExpected &= NegMask;\n TmpExpected |= Expected;\n\n // Set up desired\n TmpDesired = TmpExpected;\n TmpDesired &= NegMask;\n TmpDesired |= Desired;\n\n bool CASResult = Atomic.compare_exchange_strong(TmpExpected, TmpDesired);\n if (CASResult) {\n // Successful, so we are done\n return Expected >> (Alignment * 8);\n } else {\n if constexpr (Retry) {\n // If we failed but we have enabled retry then just retry without checking results\n // CAS can't retry but atomic memory ops need to retry until passing\n continue;\n }\n // Not successful\n // Now we need to check the results to see if we can try again\n uint64_t FailedResultOurBits = TmpExpected & Mask;\n uint64_t FailedResultNotOurBits = TmpExpected & NegMask;\n\n uint64_t FailedDesiredNotOurBits = TmpDesired & NegMask;\n\n if ((FailedResultNotOurBits ^ FailedDesiredNotOurBits) != 0) {\n // If the bits changed that weren't part of our regular CAS then we need to try again\n continue;\n }\n\n // This happens in the case that between Load and CAS that something has store our desired in to the memory location\n // This means our CAS fails because what we wanted to store was already stored\n uint16_t FailedResult = FailedResultOurBits >> (Alignment * 8);\n // CAS failed but handled successfully\n return FailedResult;\n }\n }\n } else {\n // Fits within 4byte boundary\n // Only needs 32bit CAS\n // Only alignment offset will be 1 here\n uint64_t Alignment = Addr & AlignmentMask;\n Addr &= ~AlignmentMask;\n\n uint32_t Mask = 0xFFFF;\n Mask <<= Alignment * 8;\n\n uint32_t NegMask = ~Mask;\n\n uint32_t TmpExpected {};\n uint32_t TmpDesired {};\n\n auto Atomic = std::atomic_ref(*reinterpret_cast(Addr));\n while (1) {\n TmpExpected = Atomic.load();\n\n\n uint32_t Desired = DesiredFunction(TmpExpected >> (Alignment * 8), DesiredSrc);\n Desired <<= Alignment * 8;\n\n uint32_t Expected = ExpectedFunction(TmpExpected >> (Alignment * 8), ExpectedSrc);\n Expected <<= Alignment * 8;\n\n // Set up expected\n TmpExpected &= NegMask;\n TmpExpected |= Expected;\n\n // Set up desired\n TmpDesired = TmpExpected;\n TmpDesired &= NegMask;\n TmpDesired |= Desired;\n\n bool CASResult = Atomic.compare_exchange_strong(TmpExpected, TmpDesired);\n if (CASResult) {\n // Successful, so we are done\n return Expected >> (Alignment * 8);\n } else {\n if constexpr (Retry) {\n // If we failed but we have enabled retry then just retry without checking results\n // CAS can't retry but atomic memory ops need to retry until passing\n continue;\n }\n // Not successful\n // Now we need to check the results to see if we can try again\n uint32_t FailedResultOurBits = TmpExpected & Mask;\n uint32_t FailedResultNotOurBits = TmpExpected & NegMask;\n\n uint32_t FailedDesiredNotOurBits = TmpDesired & NegMask;\n\n if ((FailedResultNotOurBits ^ FailedDesiredNotOurBits) != 0) {\n // If the bits changed that weren't part of our regular CAS then we need to try again\n continue;\n }\n\n // This happens in the case that between Load and CAS that something has store our desired in to the memory location\n // This means our CAS fails because what we wanted to store was already stored\n uint16_t FailedResult = FailedResultOurBits >> (Alignment * 8);\n // CAS failed but handled successfully\n return FailedResult;\n }\n }\n }\n }\n }\n}\n\ntemplate\nstatic uint32_t DoCAS32(uint32_t DesiredSrc, uint32_t ExpectedSrc, uint64_t Addr, CASExpectedFn ExpectedFunction,\n CASDesiredFn DesiredFunction, uint32_t* StrictSplitLockMutex) {\n std::optional> Lock {};\n\n if ((Addr & 63) > 60) {\n FEXCORE_TELEMETRY_SET(TYPE_HAS_SPLIT_LOCKS, 1);\n if (StrictSplitLockMutex && !Lock.has_value()) {\n Lock.emplace(StrictSplitLockMutex);\n }\n }\n\n // 32 bit\n uint64_t AlignmentMask = 0b1111;\n if ((Addr & AlignmentMask) > 12) {\n FEXCORE_TELEMETRY_SET(TYPE_16BYTE_SPLIT, 1);\n if (StrictSplitLockMutex && !Lock.has_value()) {\n Lock.emplace(StrictSplitLockMutex);\n }\n\n // Address crosses over 16byte threshold\n // Needs dual 4 byte CAS loop\n uint64_t Alignment = Addr & 0b11;\n Addr &= ~0b11;\n\n uint64_t AddrUpper = Addr + 4;\n\n uint64_t Mask = ~0U;\n Mask <<= Alignment * 8;\n uint64_t NegMask = ~Mask;\n\n // Careful ordering here\n while (1) {\n uint64_t LoadOrderUpper = LoadAcquire32(AddrUpper);\n LoadOrderUpper <<= 32;\n uint64_t TmpActual = LoadOrderUpper | LoadAcquire32(Addr);\n\n uint64_t Desired = DesiredFunction(TmpActual >> (Alignment * 8), DesiredSrc);\n uint64_t Expected = ExpectedFunction(TmpActual >> (Alignment * 8), ExpectedSrc);\n\n uint64_t TmpExpected = TmpActual;\n TmpExpected &= NegMask;\n TmpExpected |= Expected << (Alignment * 8);\n\n uint64_t TmpDesired = TmpExpected;\n TmpDesired &= NegMask;\n TmpDesired |= Desired << (Alignment * 8);\n\n bool Tear = false;\n if (TmpExpected == TmpActual) {\n uint32_t TmpExpectedLower = TmpExpected;\n uint32_t TmpExpectedUpper = TmpExpected >> 32;\n\n uint32_t TmpDesiredLower = TmpDesired;\n uint32_t TmpDesiredUpper = TmpDesired >> 32;\n\n if (StoreCAS32(TmpExpectedUpper, TmpDesiredUpper, AddrUpper)) {\n if (StoreCAS32(TmpExpectedLower, TmpDesiredLower, Addr)) {\n // Stored successfully\n return Expected;\n } else {\n // CAS managed to tear, we can't really solve this\n // Continue down the path to let the guest know values weren't expected\n Tear = true;\n FEXCORE_TELEMETRY_SET(TYPE_CAS_32BIT_TEAR, 1);\n }\n }\n\n TmpExpected = TmpExpectedUpper;\n TmpExpected <<= 32;\n TmpExpected |= TmpExpectedLower;\n } else {\n // Mismatch up front\n TmpExpected = TmpActual;\n }\n\n // Not successful\n // Now we need to check the results to see if we need to try again\n uint64_t FailedResultOurBits = TmpExpected & Mask;\n uint64_t FailedResultNotOurBits = TmpExpected & NegMask;\n\n uint64_t FailedDesiredNotOurBits = TmpDesired & NegMask;\n if ((FailedResultNotOurBits ^ FailedDesiredNotOurBits) != 0) {\n // If the bits changed that weren't part of our regular CAS then we need to try again\n continue;\n }\n\n // This happens in the case that between Load and CAS that something has store our desired in to the memory location\n // This means our CAS fails because what we wanted to store was already stored\n uint32_t FailedResult = FailedResultOurBits >> (Alignment * 8);\n\n if constexpr (Retry) {\n if (Tear) {\n // If we are retrying and tearing then we can't do anything here\n // XXX: Resolve with TME\n return FailedResult;\n } else {\n // We can retry safely\n }\n } else {\n // Without Retry (CAS) then we have failed regardless of tear\n // CAS failed but handled successfully\n return FailedResult;\n }\n }\n } else {\n AlignmentMask = 0b111;\n if ((Addr & AlignmentMask) >= 5) {\n // Crosses 8byte boundary\n // Needs 128bit CAS\n // Fits within a 16byte region\n uint64_t Alignment = Addr & 0b1111;\n Addr &= ~0b1111ULL;\n auto Atomic128 = std::atomic_ref<__uint128_t>(*reinterpret_cast<__uint128_t*>(Addr));\n\n __uint128_t Mask = ~0U;\n Mask <<= Alignment * 8;\n __uint128_t NegMask = ~Mask;\n __uint128_t TmpExpected {};\n __uint128_t TmpDesired {};\n\n while (1) {\n __uint128_t TmpActual = Atomic128.load();\n\n __uint128_t Desired = DesiredFunction(TmpActual >> (Alignment * 8), DesiredSrc);\n __uint128_t Expected = ExpectedFunction(TmpActual >> (Alignment * 8), ExpectedSrc);\n\n // Set up expected\n TmpExpected = TmpActual;\n TmpExpected &= NegMask;\n TmpExpected |= Expected << (Alignment * 8);\n\n // Set up desired\n TmpDesired = TmpExpected;\n TmpDesired &= NegMask;\n TmpDesired |= Desired << (Alignment * 8);\n\n bool CASResult = Atomic128.compare_exchange_strong(TmpExpected, TmpDesired);\n if (CASResult) {\n // Stored successfully\n return Expected;\n } else {\n if constexpr (Retry) {\n // If we failed but we have enabled retry then just retry without checking results\n // CAS can't retry but atomic memory ops need to retry until passing\n continue;\n }\n\n // Not successful\n // Now we need to check the results to see if we need to try again\n __uint128_t FailedResultOurBits = TmpExpected & Mask;\n __uint128_t FailedResultNotOurBits = TmpExpected & NegMask;\n\n __uint128_t FailedDesiredNotOurBits = TmpDesired & NegMask;\n if ((FailedResultNotOurBits ^ FailedDesiredNotOurBits) != 0) {\n // If the bits changed that weren't part of our regular CAS then we need to try again\n continue;\n }\n\n // This happens in the case that between Load and CAS that something has store our desired in to the memory location\n // This means our CAS fails because what we wanted to store was already stored\n uint32_t FailedResult = FailedResultOurBits >> (Alignment * 8);\n // CAS failed but handled successfully\n return FailedResult;\n }\n }\n } else {\n // Fits within 8byte boundary\n // Only needs 64bit CAS\n // Alignments can be [1,5)\n uint64_t Alignment = Addr & AlignmentMask;\n Addr &= ~AlignmentMask;\n\n uint64_t Mask = ~0U;\n Mask <<= Alignment * 8;\n\n uint64_t NegMask = ~Mask;\n\n uint64_t TmpExpected {};\n uint64_t TmpDesired {};\n\n auto Atomic = std::atomic_ref(*reinterpret_cast(Addr));\n while (1) {\n uint64_t TmpActual = Atomic.load();\n\n uint64_t Desired = DesiredFunction(TmpActual >> (Alignment * 8), DesiredSrc);\n uint64_t Expected = ExpectedFunction(TmpActual >> (Alignment * 8), ExpectedSrc);\n\n // Set up expected\n TmpExpected = TmpActual;\n TmpExpected &= NegMask;\n TmpExpected |= Expected << (Alignment * 8);\n\n // Set up desired\n TmpDesired = TmpExpected;\n TmpDesired &= NegMask;\n TmpDesired |= Desired << (Alignment * 8);\n\n bool CASResult = Atomic.compare_exchange_strong(TmpExpected, TmpDesired);\n if (CASResult) {\n // Stored successfully\n return Expected;\n } else {\n if constexpr (Retry) {\n // If we failed but we have enabled retry then just retry without checking results\n // CAS can't retry but atomic memory ops need to retry until passing\n continue;\n }\n\n // Not successful\n // Now we need to check the results to see if we can try again\n uint64_t FailedResultOurBits = TmpExpected & Mask;\n uint64_t FailedResultNotOurBits = TmpExpected & NegMask;\n\n uint64_t FailedDesiredNotOurBits = TmpDesired & NegMask;\n\n if ((FailedResultNotOurBits ^ FailedDesiredNotOurBits) != 0) {\n // If the bits changed that weren't part of our regular CAS then we need to try again\n continue;\n }\n\n // This happens in the case that between Load and CAS that something has store our desired in to the memory location\n // This means our CAS fails because what we wanted to store was already stored\n uint32_t FailedResult = FailedResultOurBits >> (Alignment * 8);\n // CAS failed but handled successfully\n return FailedResult;\n }\n }\n }\n }\n}\n\ntemplate\nstatic uint64_t DoCAS64(uint64_t DesiredSrc, uint64_t ExpectedSrc, uint64_t Addr, CASExpectedFn ExpectedFunction,\n CASDesiredFn DesiredFunction, uint32_t* StrictSplitLockMutex) {\n std::optional> Lock {};\n\n if ((Addr & 63) > 56) {\n FEXCORE_TELEMETRY_SET(TYPE_HAS_SPLIT_LOCKS, 1);\n if (StrictSplitLockMutex && !Lock.has_value()) {\n Lock.emplace(StrictSplitLockMutex);\n }\n }\n\n // 64bit\n uint64_t AlignmentMask = 0b1111;\n if ((Addr & AlignmentMask) > 8) {\n FEXCORE_TELEMETRY_SET(TYPE_16BYTE_SPLIT, 1);\n if (StrictSplitLockMutex && !Lock.has_value()) {\n Lock.emplace(StrictSplitLockMutex);\n }\n\n uint64_t Alignment = Addr & 0b111;\n Addr &= ~0b111ULL;\n uint64_t AddrUpper = Addr + 8;\n\n // Crosses a 16byte boundary\n // Need to do 256bit atomic, but since that doesn't exist we need to do a dual CAS loop\n __uint128_t Mask = ~0ULL;\n Mask <<= Alignment * 8;\n __uint128_t NegMask = ~Mask;\n __uint128_t TmpExpected {};\n __uint128_t TmpDesired {};\n\n while (1) {\n __uint128_t LoadOrderUpper = LoadAcquire64(AddrUpper);\n LoadOrderUpper <<= 64;\n __uint128_t TmpActual = LoadOrderUpper | LoadAcquire64(Addr);\n\n __uint128_t Desired = DesiredFunction(TmpActual >> (Alignment * 8), DesiredSrc);\n __uint128_t Expected = ExpectedFunction(TmpActual >> (Alignment * 8), ExpectedSrc);\n\n // Set up expected\n TmpExpected = TmpActual;\n TmpExpected &= NegMask;\n TmpExpected |= Expected << (Alignment * 8);\n\n // Set up desired\n TmpDesired = TmpExpected;\n TmpDesired &= NegMask;\n TmpDesired |= Desired << (Alignment * 8);\n\n uint64_t TmpExpectedLower = TmpExpected;\n uint64_t TmpExpectedUpper = TmpExpected >> 64;\n\n uint64_t TmpDesiredLower = TmpDesired;\n uint64_t TmpDesiredUpper = TmpDesired >> 64;\n\n bool Tear = false;\n if (TmpExpected == TmpActual) {\n if (StoreCAS64(TmpExpectedUpper, TmpDesiredUpper, AddrUpper)) {\n if (StoreCAS64(TmpExpectedLower, TmpDesiredLower, Addr)) {\n // Stored successfully\n return Expected;\n } else {\n // CAS managed to tear, we can't really solve this\n // Continue down the path to let the guest know values weren't expected\n Tear = true;\n FEXCORE_TELEMETRY_SET(TYPE_CAS_64BIT_TEAR, 1);\n }\n }\n\n TmpExpected = TmpExpectedUpper;\n TmpExpected <<= 64;\n TmpExpected |= TmpExpectedLower;\n } else {\n // Mismatch up front\n TmpExpected = TmpActual;\n }\n\n // Not successful\n // Now we need to check the results to see if we need to try again\n __uint128_t FailedResultOurBits = TmpExpected & Mask;\n __uint128_t FailedResultNotOurBits = TmpExpected & NegMask;\n\n __uint128_t FailedDesiredNotOurBits = TmpDesired & NegMask;\n if ((FailedResultNotOurBits ^ FailedDesiredNotOurBits) != 0) {\n // If the bits changed that weren't part of our regular CAS then we need to try again\n continue;\n }\n\n // This happens in the case that between Load and CAS that something has store our desired in to the memory location\n // This means our CAS fails because what we wanted to store was already stored\n uint64_t FailedResult = FailedResultOurBits >> (Alignment * 8);\n\n if constexpr (Retry) {\n if (Tear) {\n // If we are retrying and tearing then we can't do anything here\n // XXX: Resolve with TME\n return FailedResult;\n } else {\n // We can retry safely\n }\n } else {\n // Without Retry (CAS) then we have failed regardless of tear\n // CAS failed but handled successfully\n return FailedResult;\n }\n }\n } else {\n // Fits within a 16byte region\n uint64_t Alignment = Addr & AlignmentMask;\n Addr &= ~AlignmentMask;\n auto Atomic128 = std::atomic_ref<__uint128_t>(*reinterpret_cast<__uint128_t*>(Addr));\n\n __uint128_t Mask = ~0ULL;\n Mask <<= Alignment * 8;\n __uint128_t NegMask = ~Mask;\n __uint128_t TmpExpected {};\n __uint128_t TmpDesired {};\n\n while (1) {\n __uint128_t TmpActual = Atomic128.load();\n\n __uint128_t Desired = DesiredFunction(TmpActual >> (Alignment * 8), DesiredSrc);\n __uint128_t Expected = ExpectedFunction(TmpActual >> (Alignment * 8), ExpectedSrc);\n\n // Set up expected\n TmpExpected = TmpActual;\n TmpExpected &= NegMask;\n TmpExpected |= Expected << (Alignment * 8);\n\n // Set up desired\n TmpDesired = TmpExpected;\n TmpDesired &= NegMask;\n TmpDesired |= Desired << (Alignment * 8);\n\n bool CASResult = Atomic128.compare_exchange_strong(TmpExpected, TmpDesired);\n if (CASResult) {\n // Stored successfully\n return Expected;\n } else {\n if constexpr (Retry) {\n // If we failed but we have enabled retry then just retry without checking results\n // CAS can't retry but atomic memory ops need to retry until passing\n continue;\n }\n\n // Not successful\n // Now we need to check the results to see if we need to try again\n __uint128_t FailedResultOurBits = TmpExpected & Mask;\n __uint128_t FailedResultNotOurBits = TmpExpected & NegMask;\n\n __uint128_t FailedDesiredNotOurBits = TmpDesired & NegMask;\n if ((FailedResultNotOurBits ^ FailedDesiredNotOurBits) != 0) {\n // If the bits changed that weren't part of our regular CAS then we need to try again\n continue;\n }\n\n // This happens in the case that between Load and CAS that something has store our desired in to the memory location\n // This means our CAS fails because what we wanted to store was already stored\n uint64_t FailedResult = FailedResultOurBits >> (Alignment * 8);\n // CAS failed but handled successfully\n return FailedResult;\n }\n }\n }\n}\n\nstatic std::optional DoCAS(uint32_t Size, uint64_t Desired, uint64_t Expected, uint64_t Addr, uint32_t* StrictSplitLockMutex) {\n // Cross-cacheline CAS doesn't work on ARM\n // It isn't even guaranteed to work on x86\n // Intel will do a \"split lock\" which locks the full bus\n // AMD will tear instead\n // Both cross-cacheline and cross 16byte both need dual CAS loops that can tear\n // ARMv8.4 LSE2 solves all atomic issues except cross-cacheline\n // ARM's TME extension solves the cross-cacheline problem\n\n // 8bit can't be unaligned\n // Only need to handle 16, 32, 64\n if (Size == 2) {\n auto Res = DoCAS16(\n Desired, Expected, Addr,\n [](uint16_t, uint16_t Expected) -> uint16_t {\n // Expected is just Expected\n return Expected;\n },\n [](uint16_t, uint16_t Desired) -> uint16_t {\n // Desired is just Desired\n return Desired;\n },\n StrictSplitLockMutex);\n return Res;\n } else if (Size == 4) {\n auto Res = DoCAS32(\n Desired, Expected, Addr,\n [](uint32_t, uint32_t Expected) -> uint32_t {\n // Expected is just Expected\n return Expected;\n },\n [](uint32_t, uint32_t Desired) -> uint32_t {\n // Desired is just Desired\n return Desired;\n },\n StrictSplitLockMutex);\n return Res;\n } else if (Size == 8) {\n auto Res = DoCAS64(\n Desired, Expected, Addr,\n [](uint64_t, uint64_t Expected) -> uint64_t {\n // Expected is just Expected\n return Expected;\n },\n [](uint64_t, uint64_t Desired) -> uint64_t {\n // Desired is just Desired\n return Desired;\n },\n StrictSplitLockMutex);\n return Res;\n }\n\n return std::nullopt;\n}\n\nstatic bool RunCASAL(uint64_t* GPRs, uint32_t Size, uint32_t DesiredReg, uint32_t ExpectedReg, uint32_t AddressReg, uint32_t* StrictSplitLockMutex) {\n std::optional Res = DoCAS(Size, GPRs[DesiredReg], GPRs[ExpectedReg], GPRs[AddressReg], StrictSplitLockMutex);\n if (!Res.has_value()) {\n return false;\n }\n\n // Regardless of pass or fail\n // We set the result register if it isn't a zero register\n if (ExpectedReg != 31) {\n GPRs[ExpectedReg] = *Res;\n }\n return true;\n}\n\nstatic bool HandleCASAL(uint64_t* GPRs, uint32_t Instr, uint32_t* StrictSplitLockMutex) {\n uint32_t Size = 1 << (Instr >> 30);\n\n uint32_t DesiredReg = Instr & 0b11111;\n uint32_t ExpectedReg = (Instr >> 16) & 0b11111;\n uint32_t AddressReg = (Instr >> 5) & 0b11111;\n return RunCASAL(GPRs, Size, DesiredReg, ExpectedReg, AddressReg, StrictSplitLockMutex);\n}\n\nstatic bool HandleAtomicMemOp(uint32_t Instr, uint64_t* GPRs, uint32_t* StrictSplitLockMutex) {\n uint32_t Size = 1 << (Instr >> 30);\n uint32_t ResultReg = Instr & 0b11111;\n uint32_t SourceReg = (Instr >> 16) & 0b11111;\n uint32_t AddressReg = (Instr >> 5) & 0b11111;\n\n uint64_t Addr = GPRs[AddressReg];\n\n uint8_t Op = (Instr >> 12) & 0xF;\n\n if (Size == 2) {\n auto NOPExpected = [](uint16_t SrcVal, uint16_t) -> uint16_t {\n return SrcVal;\n };\n\n auto ADDDesired = [](uint16_t SrcVal, uint16_t Desired) -> uint16_t {\n return SrcVal + Desired;\n };\n\n auto CLRDesired = [](uint16_t SrcVal, uint16_t Desired) -> uint16_t {\n return SrcVal & ~Desired;\n };\n\n auto EORDesired = [](uint16_t SrcVal, uint16_t Desired) -> uint16_t {\n return SrcVal ^ Desired;\n };\n\n auto SETDesired = [](uint16_t SrcVal, uint16_t Desired) -> uint16_t {\n return SrcVal | Desired;\n };\n\n auto SWAPDesired = [](uint16_t SrcVal, uint16_t Desired) -> uint16_t {\n return Desired;\n };\n\n CASDesiredFn DesiredFunction {};\n\n switch (Op) {\n case ATOMIC_ADD_OP: DesiredFunction = ADDDesired; break;\n case ATOMIC_CLR_OP: DesiredFunction = CLRDesired; break;\n case ATOMIC_EOR_OP: DesiredFunction = EORDesired; break;\n case ATOMIC_SET_OP: DesiredFunction = SETDesired; break;\n case ATOMIC_SWAP_OP: DesiredFunction = SWAPDesired; break;\n default: LogMan::Msg::EFmt(\"Unhandled JIT SIGBUS Atomic mem op 0x{:02x}\", Op); return false;\n }\n\n auto Res = DoCAS16(GPRs[SourceReg],\n 0, // Unused\n Addr, NOPExpected, DesiredFunction, StrictSplitLockMutex);\n // If we passed and our destination register is not zero\n // Then we need to update the result register with what was in memory\n if (ResultReg != 31) {\n GPRs[ResultReg] = Res;\n }\n return true;\n } else if (Size == 4) {\n auto NOPExpected = [](uint32_t SrcVal, uint32_t) -> uint32_t {\n return SrcVal;\n };\n\n auto ADDDesired = [](uint32_t SrcVal, uint32_t Desired) -> uint32_t {\n return SrcVal + Desired;\n };\n\n auto CLRDesired = [](uint32_t SrcVal, uint32_t Desired) -> uint32_t {\n return SrcVal & ~Desired;\n };\n\n auto EORDesired = [](uint32_t SrcVal, uint32_t Desired) -> uint32_t {\n return SrcVal ^ Desired;\n };\n\n auto SETDesired = [](uint32_t SrcVal, uint32_t Desired) -> uint32_t {\n return SrcVal | Desired;\n };\n\n auto SWAPDesired = [](uint32_t SrcVal, uint32_t Desired) -> uint32_t {\n return Desired;\n };\n\n CASDesiredFn DesiredFunction {};\n\n switch (Op) {\n case ATOMIC_ADD_OP: DesiredFunction = ADDDesired; break;\n case ATOMIC_CLR_OP: DesiredFunction = CLRDesired; break;\n case ATOMIC_EOR_OP: DesiredFunction = EORDesired; break;\n case ATOMIC_SET_OP: DesiredFunction = SETDesired; break;\n case ATOMIC_SWAP_OP: DesiredFunction = SWAPDesired; break;\n default: LogMan::Msg::EFmt(\"Unhandled JIT SIGBUS Atomic mem op 0x{:02x}\", Op); return false;\n }\n\n auto Res = DoCAS32(GPRs[SourceReg],\n 0, // Unused\n Addr, NOPExpected, DesiredFunction, StrictSplitLockMutex);\n // If we passed and our destination register is not zero\n // Then we need to update the result register with what was in memory\n if (ResultReg != 31) {\n GPRs[ResultReg] = Res;\n }\n return true;\n } else if (Size == 8) {\n auto NOPExpected = [](uint64_t SrcVal, uint64_t) -> uint64_t {\n return SrcVal;\n };\n\n auto ADDDesired = [](uint64_t SrcVal, uint64_t Desired) -> uint64_t {\n return SrcVal + Desired;\n };\n\n auto CLRDesired = [](uint64_t SrcVal, uint64_t Desired) -> uint64_t {\n return SrcVal & ~Desired;\n };\n\n auto EORDesired = [](uint64_t SrcVal, uint64_t Desired) -> uint64_t {\n return SrcVal ^ Desired;\n };\n\n auto SETDesired = [](uint64_t SrcVal, uint64_t Desired) -> uint64_t {\n return SrcVal | Desired;\n };\n\n auto SWAPDesired = [](uint64_t SrcVal, uint64_t Desired) -> uint64_t {\n return Desired;\n };\n\n CASDesiredFn DesiredFunction {};\n\n switch (Op) {\n case ATOMIC_ADD_OP: DesiredFunction = ADDDesired; break;\n case ATOMIC_CLR_OP: DesiredFunction = CLRDesired; break;\n case ATOMIC_EOR_OP: DesiredFunction = EORDesired; break;\n case ATOMIC_SET_OP: DesiredFunction = SETDesired; break;\n case ATOMIC_SWAP_OP: DesiredFunction = SWAPDesired; break;\n default: LogMan::Msg::EFmt(\"Unhandled JIT SIGBUS Atomic mem op 0x{:02x}\", Op); return false;\n }\n\n auto Res = DoCAS64(GPRs[SourceReg],\n 0, // Unused\n Addr, NOPExpected, DesiredFunction, StrictSplitLockMutex);\n // If we passed and our destination register is not zero\n // Then we need to update the result register with what was in memory\n if (ResultReg != 31) {\n GPRs[ResultReg] = Res;\n }\n return true;\n }\n\n return false;\n}\n\nstatic bool HandleAtomicLoad(uint32_t Instr, uint64_t* GPRs, int64_t Offset, Core::UnalignedExclusiveStore* Store = nullptr) {\n uint32_t Size = 1 << (Instr >> 30);\n\n uint32_t ResultReg = Instr & 0b11111;\n uint32_t AddressReg = (Instr >> 5) & 0b11111;\n\n uint64_t Addr = GPRs[AddressReg] + Offset;\n uint64_t Res;\n\n if (Size == 2) {\n Res = DoLoad16(Addr);\n // We set the result register if it isn't a zero register\n if (ResultReg != 31) {\n GPRs[ResultReg] = Res;\n }\n } else if (Size == 4) {\n Res = DoLoad32(Addr);\n // We set the result register if it isn't a zero register\n if (ResultReg != 31) {\n GPRs[ResultReg] = Res;\n }\n } else if (Size == 8) {\n Res = DoLoad64(Addr);\n // We set the result register if it isn't a zero register\n if (ResultReg != 31) {\n GPRs[ResultReg] = Res;\n }\n } else {\n return false;\n }\n\n if (Store) {\n Store->Addr = Addr;\n Store->Store = Res;\n Store->Size = Size;\n }\n return true;\n}\n\nstatic bool HandleAtomicStore(uint32_t Instr, uint64_t* GPRs, int64_t Offset, uint32_t* StrictSplitLockMutex) {\n uint32_t Size = 1 << (Instr >> 30);\n\n uint32_t DataReg = Instr & 0x1F;\n uint32_t AddressReg = (Instr >> 5) & 0b11111;\n\n uint64_t Addr = GPRs[AddressReg] + Offset;\n\n constexpr bool DoRetry = false;\n if (Size == 2) {\n DoCAS16(\n GPRs[DataReg],\n 0, // Unused\n Addr,\n [](uint16_t SrcVal, uint16_t) -> uint16_t {\n // Expected is just src\n return SrcVal;\n },\n [](uint16_t, uint16_t Desired) -> uint16_t {\n // Desired is just Desired\n return Desired;\n },\n StrictSplitLockMutex);\n return true;\n } else if (Size == 4) {\n DoCAS32(\n GPRs[DataReg],\n 0, // Unused\n Addr,\n [](uint32_t SrcVal, uint32_t) -> uint32_t {\n // Expected is just src\n return SrcVal;\n },\n [](uint32_t, uint32_t Desired) -> uint32_t {\n // Desired is just Desired\n return Desired;\n },\n StrictSplitLockMutex);\n return true;\n } else if (Size == 8) {\n DoCAS64(\n GPRs[DataReg],\n 0, // Unused\n Addr,\n [](uint64_t SrcVal, uint64_t) -> uint64_t {\n // Expected is just src\n return SrcVal;\n },\n [](uint64_t, uint64_t Desired) -> uint64_t {\n // Desired is just Desired\n return Desired;\n },\n StrictSplitLockMutex);\n return true;\n }\n\n return false;\n}\n\nstatic uint64_t HandleCAS_NoAtomics(uintptr_t ProgramCounter, uint64_t* GPRs, uint32_t* StrictSplitLockMutex) {\n // ARMv8.0 CAS\n // [1] ldaxrb(TMP2.W(), MemOperand(MemSrc))\n // [2] cmp (TMP2.W(), Expected.W())\n // [3] b\n // [4] stlxrb(TMP3.W(), Desired.W(), MemOperand(MemSrc)\n // [5] cbnz\n // [6] mov\n // [7] b\n // [8] mov (.., TMP2.W());\n // [9] clrex\n\n uint32_t* PC = (uint32_t*)ProgramCounter;\n uint32_t Instr = PC[0];\n uint32_t Size = 1 << (Instr >> 30);\n uint32_t AddressReg = GetRnReg(Instr);\n uint32_t ResultReg = GetRdReg(Instr); // TMP2\n uint32_t DesiredReg = 0;\n uint32_t ExpectedReg = 0;\n for (size_t i = 1; i < 6; ++i) {\n uint32_t NextInstr = PC[i];\n if ((NextInstr & ArchHelpers::Arm64::STLXR_MASK) == ArchHelpers::Arm64::STLXR_INST) {\n#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED\n // Just double check that the memory destination matches\n const uint32_t StoreAddressReg = GetRnReg(NextInstr);\n LOGMAN_THROW_A_FMT(StoreAddressReg == AddressReg, \"StoreExclusive memory register didn't match the store exclusive register\");\n#endif\n DesiredReg = GetRdReg(NextInstr);\n } else if ((NextInstr & ArchHelpers::Arm64::ALU_OP_MASK) == ArchHelpers::Arm64::CMP_INST ||\n (NextInstr & ArchHelpers::Arm64::ALU_OP_MASK) == ArchHelpers::Arm64::CMP_SHIFT_INST) {\n ExpectedReg = GetRmReg(NextInstr);\n }\n }\n // set up CASAL by doing mov(TMP2, Expected)\n GPRs[ResultReg] = GPRs[ExpectedReg];\n\n if (RunCASAL(GPRs, Size, DesiredReg, ResultReg, AddressReg, StrictSplitLockMutex)) {\n return 7 * sizeof(uint32_t); // jump to mov to allocated register\n } else {\n return 0;\n }\n}\n\nstatic uint64_t HandleAtomicLoadstoreExclusive(uintptr_t ProgramCounter, uint64_t* GPRs, uint32_t* StrictSplitLockMutex) {\n uint32_t* PC = (uint32_t*)ProgramCounter;\n uint32_t Instr = PC[0];\n\n // Atomic Add\n // [1] ldaxrb(TMP2.W(), MemOperand(MemSrc));\n // [2] add(TMP2.W(), TMP2.W(), GetReg(Op->Header.Args[1].ID()));\n // [3] stlxrb(TMP2.W(), TMP2.W(), MemOperand(MemSrc));\n // [4] cbnz(TMP2.W(), &LoopTop);\n //\n // Atomic Fetch Add\n // [1] ldaxrb(TMP2.W(), MemOperand(MemSrc));\n // [2] add(TMP3.W(), TMP2.W(), GetReg(Op->Header.Args[1].ID()));\n // [3] stlxrb(TMP4.W(), TMP3.W(), MemOperand(MemSrc));\n // [4] cbnz(TMP4.W(), &LoopTop);\n // [5] mov(GetReg(Node), TMP2.W());\n //\n // Atomic Swap\n //\n // [1] ldaxrb(TMP2.W(), MemOperand(MemSrc));\n // [2] stlxrb(TMP4.W(), GetReg(Op->Header.Args[1].ID()), MemOperand(MemSrc));\n // [3] cbnz(TMP4.W(), &LoopTop);\n // [4] uxtb(GetReg(Node), TMP2.W());\n //\n // ASSUMPTIONS:\n // - Both cases:\n // - The [2]ALU op: (Non NEG case)\n // - First source is from [1]ldaxr\n // - Second source is incoming value\n // - The [2]ALU op: (NEG case)\n // - First source is zero register\n // - The second source is the from [1]ldaxr\n // - No ALU op: (SWAP case)\n // - No DataSourceRegister\n //\n // - In Atomic case (non-fetch)\n // - The [3]stlxr instruction status + memory register are the SAME register\n //\n // - In Atomic FETCH case\n // - The [3]stlxr instruction's status + memory register are never the same register\n // - The [5]mov instruction source is always the destination register from [1] ldaxr*\n uint32_t ResultReg = GetRdReg(Instr);\n uint32_t AddressReg = GetRnReg(Instr);\n uint64_t Addr = GPRs[AddressReg];\n\n size_t NumInstructionsToSkip = 0;\n\n // Are we an Atomic op or AtomicFetch?\n bool AtomicFetch = false;\n\n // This is the register that is the incoming source to the ALU operation\n // = \n // NEG case is special\n // = Zero \n // DataSourceRegister must always be the Rm register\n uint32_t DataSourceReg {};\n ExclusiveAtomicPairType AtomicOp {ExclusiveAtomicPairType::TYPE_SWAP};\n\n // Scan forward at most five instructions to find our instructions\n for (size_t i = 1; i < 6; ++i) {\n uint32_t NextInstr = PC[i];\n if ((NextInstr & ArchHelpers::Arm64::ALU_OP_MASK) == ArchHelpers::Arm64::ADD_INST ||\n (NextInstr & ArchHelpers::Arm64::ALU_OP_MASK) == ArchHelpers::Arm64::ADD_SHIFT_INST) {\n AtomicOp = ExclusiveAtomicPairType::TYPE_ADD;\n DataSourceReg = GetRmReg(NextInstr);\n } else if ((NextInstr & ArchHelpers::Arm64::ALU_OP_MASK) == ArchHelpers::Arm64::SUB_INST ||\n (NextInstr & ArchHelpers::Arm64::ALU_OP_MASK) == ArchHelpers::Arm64::SUB_SHIFT_INST) {\n uint32_t RnReg = GetRnReg(NextInstr);\n if (RnReg == REGISTER_MASK) {\n // Zero reg means neg\n AtomicOp = ExclusiveAtomicPairType::TYPE_NEG;\n } else {\n AtomicOp = ExclusiveAtomicPairType::TYPE_SUB;\n }\n DataSourceReg = GetRmReg(NextInstr);\n } else if ((NextInstr & ArchHelpers::Arm64::ALU_OP_MASK) == ArchHelpers::Arm64::CMP_INST ||\n (NextInstr & ArchHelpers::Arm64::ALU_OP_MASK) == ArchHelpers::Arm64::CMP_SHIFT_INST) {\n return HandleCAS_NoAtomics(ProgramCounter, GPRs, StrictSplitLockMutex); // ARMv8.0 CAS\n } else if ((NextInstr & ArchHelpers::Arm64::ALU_OP_MASK) == ArchHelpers::Arm64::AND_INST) {\n AtomicOp = ExclusiveAtomicPairType::TYPE_AND;\n DataSourceReg = GetRmReg(NextInstr);\n } else if ((NextInstr & ArchHelpers::Arm64::ALU_OP_MASK) == ArchHelpers::Arm64::BIC_INST) {\n AtomicOp = ExclusiveAtomicPairType::TYPE_BIC;\n DataSourceReg = GetRmReg(NextInstr);\n } else if ((NextInstr & ArchHelpers::Arm64::ALU_OP_MASK) == ArchHelpers::Arm64::OR_INST) {\n AtomicOp = ExclusiveAtomicPairType::TYPE_OR;\n DataSourceReg = GetRmReg(NextInstr);\n } else if ((NextInstr & ArchHelpers::Arm64::ALU_OP_MASK) == ArchHelpers::Arm64::ORN_INST) {\n AtomicOp = ExclusiveAtomicPairType::TYPE_ORN;\n DataSourceReg = GetRmReg(NextInstr);\n } else if ((NextInstr & ArchHelpers::Arm64::ALU_OP_MASK) == ArchHelpers::Arm64::EOR_INST) {\n AtomicOp = ExclusiveAtomicPairType::TYPE_EOR;\n DataSourceReg = GetRmReg(NextInstr);\n } else if ((NextInstr & ArchHelpers::Arm64::ALU_OP_MASK) == ArchHelpers::Arm64::EON_INST) {\n AtomicOp = ExclusiveAtomicPairType::TYPE_EON;\n DataSourceReg = GetRmReg(NextInstr);\n } else if ((NextInstr & ArchHelpers::Arm64::STLXR_MASK) == ArchHelpers::Arm64::STLXR_INST) {\n#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED\n // Just double check that the memory destination matches\n const uint32_t StoreAddressReg = GetRnReg(NextInstr);\n LOGMAN_THROW_A_FMT(StoreAddressReg == AddressReg, \"StoreExclusive memory register didn't match the store exclusive register\");\n#endif\n uint32_t StatusReg = GetRmReg(NextInstr);\n uint32_t StoreResultReg = GetRdReg(NextInstr);\n // We are an atomic fetch instruction if the data register isn't the status register\n AtomicFetch = !(StatusReg == StoreResultReg);\n if (AtomicOp == ExclusiveAtomicPairType::TYPE_SWAP) {\n // In the case of swap we don't have an ALU op inbetween\n // Source is directly in STLXR\n DataSourceReg = StoreResultReg;\n }\n } else if ((NextInstr & ArchHelpers::Arm64::CBNZ_MASK) == ArchHelpers::Arm64::CBNZ_INST) {\n // Found the CBNZ, we want to skip to just after this instruction when done\n NumInstructionsToSkip = i + 1;\n // This is the last instruction we care about. Leave now\n break;\n } else {\n LogMan::Msg::AFmt(\"Unknown instruction 0x{:08x}\", NextInstr);\n }\n }\n\n uint32_t Size = 1 << (Instr >> 30);\n\n constexpr bool DoRetry = true;\n\n auto NOPExpected = [](AtomicType SrcVal, AtomicType) -> AtomicType {\n return SrcVal;\n };\n\n auto ADDDesired = [](AtomicType SrcVal, AtomicType Desired) -> AtomicType {\n return SrcVal + Desired;\n };\n\n auto SUBDesired = [](AtomicType SrcVal, AtomicType Desired) -> AtomicType {\n return SrcVal - Desired;\n };\n\n auto ANDDesired = [](AtomicType SrcVal, AtomicType Desired) -> AtomicType {\n return SrcVal & Desired;\n };\n\n auto BICDesired = [](AtomicType SrcVal, AtomicType Desired) -> AtomicType {\n return SrcVal & ~Desired;\n };\n\n auto ORDesired = [](AtomicType SrcVal, AtomicType Desired) -> AtomicType {\n return SrcVal | Desired;\n };\n\n auto ORNDesired = [](AtomicType SrcVal, AtomicType Desired) -> AtomicType {\n return SrcVal | ~Desired;\n };\n\n auto EORDesired = [](AtomicType SrcVal, AtomicType Desired) -> AtomicType {\n return SrcVal ^ Desired;\n };\n\n auto EONDesired = [](AtomicType SrcVal, AtomicType Desired) -> AtomicType {\n return SrcVal ^ ~Desired;\n };\n\n auto NEGDesired = [](AtomicType SrcVal, AtomicType Desired) -> AtomicType {\n return -SrcVal;\n };\n\n auto SWAPDesired = [](AtomicType SrcVal, AtomicType Desired) -> AtomicType {\n return Desired;\n };\n\n if (Size == 2) {\n using AtomicType = uint16_t;\n CASDesiredFn DesiredFunction {};\n\n switch (AtomicOp) {\n case ExclusiveAtomicPairType::TYPE_SWAP: DesiredFunction = SWAPDesired; break;\n case ExclusiveAtomicPairType::TYPE_ADD: DesiredFunction = ADDDesired; break;\n case ExclusiveAtomicPairType::TYPE_SUB: DesiredFunction = SUBDesired; break;\n case ExclusiveAtomicPairType::TYPE_AND: DesiredFunction = ANDDesired; break;\n case ExclusiveAtomicPairType::TYPE_BIC: DesiredFunction = BICDesired; break;\n case ExclusiveAtomicPairType::TYPE_OR: DesiredFunction = ORDesired; break;\n case ExclusiveAtomicPairType::TYPE_ORN: DesiredFunction = ORNDesired; break;\n case ExclusiveAtomicPairType::TYPE_EOR: DesiredFunction = EORDesired; break;\n case ExclusiveAtomicPairType::TYPE_EON: DesiredFunction = EONDesired; break;\n case ExclusiveAtomicPairType::TYPE_NEG: DesiredFunction = NEGDesired; break;\n default: LogMan::Msg::EFmt(\"Unhandled JIT SIGBUS Atomic mem op 0x{:02x}\", FEXCore::ToUnderlying(AtomicOp)); return false;\n }\n\n auto Res = DoCAS16(GPRs[DataSourceReg],\n 0, // Unused\n Addr, NOPExpected, DesiredFunction, StrictSplitLockMutex);\n\n if (AtomicFetch && ResultReg != 31) {\n // On atomic fetch then we store the resulting value back in to the loadacquire destination register\n // We want the memory value BEFORE the ALU op\n GPRs[ResultReg] = Res;\n }\n } else if (Size == 4) {\n using AtomicType = uint32_t;\n CASDesiredFn DesiredFunction {};\n\n switch (AtomicOp) {\n case ExclusiveAtomicPairType::TYPE_SWAP: DesiredFunction = SWAPDesired; break;\n case ExclusiveAtomicPairType::TYPE_ADD: DesiredFunction = ADDDesired; break;\n case ExclusiveAtomicPairType::TYPE_SUB: DesiredFunction = SUBDesired; break;\n case ExclusiveAtomicPairType::TYPE_AND: DesiredFunction = ANDDesired; break;\n case ExclusiveAtomicPairType::TYPE_BIC: DesiredFunction = BICDesired; break;\n case ExclusiveAtomicPairType::TYPE_OR: DesiredFunction = ORDesired; break;\n case ExclusiveAtomicPairType::TYPE_ORN: DesiredFunction = ORNDesired; break;\n case ExclusiveAtomicPairType::TYPE_EOR: DesiredFunction = EORDesired; break;\n case ExclusiveAtomicPairType::TYPE_EON: DesiredFunction = EONDesired; break;\n case ExclusiveAtomicPairType::TYPE_NEG: DesiredFunction = NEGDesired; break;\n default: LogMan::Msg::EFmt(\"Unhandled JIT SIGBUS Atomic mem op 0x{:02x}\", FEXCore::ToUnderlying(AtomicOp)); return false;\n }\n\n auto Res = DoCAS32(GPRs[DataSourceReg],\n 0, // Unused\n Addr, NOPExpected, DesiredFunction, StrictSplitLockMutex);\n\n if (AtomicFetch && ResultReg != 31) {\n // On atomic fetch then we store the resulting value back in to the loadacquire destination register\n // We want the memory value BEFORE the ALU op\n GPRs[ResultReg] = Res;\n }\n } else if (Size == 8) {\n using AtomicType = uint64_t;\n CASDesiredFn DesiredFunction {};\n\n switch (AtomicOp) {\n case ExclusiveAtomicPairType::TYPE_SWAP: DesiredFunction = SWAPDesired; break;\n case ExclusiveAtomicPairType::TYPE_ADD: DesiredFunction = ADDDesired; break;\n case ExclusiveAtomicPairType::TYPE_SUB: DesiredFunction = SUBDesired; break;\n case ExclusiveAtomicPairType::TYPE_AND: DesiredFunction = ANDDesired; break;\n case ExclusiveAtomicPairType::TYPE_BIC: DesiredFunction = BICDesired; break;\n case ExclusiveAtomicPairType::TYPE_OR: DesiredFunction = ORDesired; break;\n case ExclusiveAtomicPairType::TYPE_ORN: DesiredFunction = ORNDesired; break;\n case ExclusiveAtomicPairType::TYPE_EOR: DesiredFunction = EORDesired; break;\n case ExclusiveAtomicPairType::TYPE_EON: DesiredFunction = EONDesired; break;\n case ExclusiveAtomicPairType::TYPE_NEG: DesiredFunction = NEGDesired; break;\n default: LogMan::Msg::EFmt(\"Unhandled JIT SIGBUS Atomic mem op 0x{:02x}\", FEXCore::ToUnderlying(AtomicOp)); return false;\n }\n\n auto Res = DoCAS64(GPRs[DataSourceReg],\n 0, // Unused\n Addr, NOPExpected, DesiredFunction, StrictSplitLockMutex);\n if (AtomicFetch && ResultReg != 31) {\n // On atomic fetch then we store the resulting value back in to the loadacquire destination register\n // We want the memory value BEFORE the ALU op\n GPRs[ResultReg] = Res;\n }\n }\n\n // Multiply by 4 for number of bytes to skip\n return NumInstructionsToSkip * 4;\n}\n\n[[nodiscard]]\nstd::optional HandleUnalignedAccess(FEXCore::Core::InternalThreadState* Thread, UnalignedHandlerType HandleType,\n uintptr_t ProgramCounter, uint64_t* GPRs, bool IsJIT) {\n#ifdef ARCHITECTURE_arm64\n constexpr bool is_arm64 = true;\n#else\n constexpr bool is_arm64 = false;\n#endif\n\n if constexpr (!is_arm64) {\n return std::nullopt;\n }\n\n uint32_t* PC = (uint32_t*)ProgramCounter;\n uint32_t Instr = PC[0];\n\n // 1 = 16bit\n // 2 = 32bit\n // 3 = 64bit\n uint32_t Size = (Instr & 0xC000'0000) >> 30;\n uint32_t AddrReg = (Instr >> 5) & 0x1F;\n uint32_t DataReg = Instr & 0x1F;\n\n auto CTX = static_cast(Thread->CTX);\n uint32_t* StrictSplitLockMutex {CTX->Config.StrictInProcessSplitLocks ? &CTX->StrictSplitLockMutex : nullptr};\n\n if (!IsJIT) [[unlikely]] {\n if ((Instr & LDAXR_MASK) == LDAR_INST || // LDAR*\n (Instr & LDAXR_MASK) == LDAPR_INST) { // LDAPR*\n if (ArchHelpers::Arm64::HandleAtomicLoad(Instr, GPRs, 0)) {\n // Skip this instruction now\n return 4;\n } else {\n LogMan::Msg::EFmt(\"Unhandled JIT SIGBUS LDAR*: PC: 0x{:x} Instruction: 0x{:08x}\\n\", ProgramCounter, PC[0]);\n return std::nullopt;\n }\n } else if ((Instr & LDAXR_MASK) == STLR_INST) { // STLR*\n if (ArchHelpers::Arm64::HandleAtomicStore(Instr, GPRs, 0, StrictSplitLockMutex)) {\n // Skip this instruction now\n return 4;\n } else {\n LogMan::Msg::EFmt(\"Unhandled JIT SIGBUS STLR*: PC: 0x{:x} Instruction: 0x{:08x}\\n\", ProgramCounter, PC[0]);\n return std::nullopt;\n }\n } else if ((Instr & RCPC2_MASK) == LDAPUR_INST) { // LDAPUR*\n // Extract the 9-bit offset from the instruction\n int32_t Offset = static_cast(Instr) << 11 >> 23;\n if (ArchHelpers::Arm64::HandleAtomicLoad(Instr, GPRs, Offset)) {\n // Skip this instruction now\n return 4;\n } else {\n LogMan::Msg::EFmt(\"Unhandled JIT SIGBUS LDAPUR*: PC: 0x{:x} Instruction: 0x{:08x}\\n\", ProgramCounter, PC[0]);\n return std::nullopt;\n }\n } else if ((Instr & RCPC2_MASK) == STLUR_INST) { // STLUR*\n // Extract the 9-bit offset from the instruction\n int32_t Offset = static_cast(Instr) << 11 >> 23;\n if (ArchHelpers::Arm64::HandleAtomicStore(Instr, GPRs, Offset, StrictSplitLockMutex)) {\n // Skip this instruction now\n return 4;\n } else {\n LogMan::Msg::EFmt(\"Unhandled JIT SIGBUS LDLUR*: PC: 0x{:x} Instruction: 0x{:08x}\\n\", ProgramCounter, PC[0]);\n return std::nullopt;\n }\n } else if ((Instr & ArchHelpers::Arm64::LDAXR_MASK) == ArchHelpers::Arm64::LDAXR_INST) { // LDAXR*\n if (ArchHelpers::Arm64::HandleAtomicLoad(Instr, GPRs, 0, &Thread->ExclusiveStore)) {\n return 4;\n }\n } else if ((Instr & ArchHelpers::Arm64::STLXR_MASK) == ArchHelpers::Arm64::STLXR_INST) { // STLXR*\n uint32_t StatusReg = Instr << 11 >> 27;\n // // Emulate exclusive store by validating the address and value against the last unaligned LDAXR*.\n if (GPRs[AddrReg] != Thread->ExclusiveStore.Addr || Size > Thread->ExclusiveStore.Size) {\n if (StatusReg != 31) {\n GPRs[StatusReg] = 1;\n }\n return 4;\n }\n if (std::optional Prev =\n DoCAS(Size, DataReg == 31 ? 0 : GPRs[DataReg], Thread->ExclusiveStore.Store, GPRs[AddrReg], StrictSplitLockMutex)) {\n if (StatusReg != 31) {\n GPRs[StatusReg] = !!memcmp(&Thread->ExclusiveStore.Store, &*Prev, Size);\n }\n Thread->ExclusiveStore.Size = 0;\n return 4;\n }\n }\n return 0;\n }\n\n const auto Frame = Thread->CurrentFrame;\n const uint64_t BlockBegin = Frame->State.InlineJITBlockHeader;\n auto InlineHeader = reinterpret_cast(BlockBegin);\n auto InlineTail = reinterpret_cast(Frame->State.InlineJITBlockHeader + InlineHeader->OffsetToBlockTail);\n\n // Check some instructions first that don't do any backpatching.\n if ((Instr & ArchHelpers::Arm64::CASPAL_MASK) == ArchHelpers::Arm64::CASPAL_INST) { // CASPAL\n if (ArchHelpers::Arm64::HandleCASPAL(Instr, GPRs, StrictSplitLockMutex)) {\n // Skip this instruction now\n return 4;\n } else {\n LogMan::Msg::EFmt(\"Unhandled JIT SIGBUS CASPAL: PC: 0x{:x} Instruction: 0x{:08x}\\n\", ProgramCounter, PC[0]);\n return std::nullopt;\n }\n } else if ((Instr & ArchHelpers::Arm64::CASAL_MASK) == ArchHelpers::Arm64::CASAL_INST) { // CASAL\n if (ArchHelpers::Arm64::HandleCASAL(GPRs, Instr, StrictSplitLockMutex)) {\n // Skip this instruction now\n return 4;\n } else {\n LogMan::Msg::EFmt(\"Unhandled JIT SIGBUS CASAL: PC: 0x{:x} Instruction: 0x{:08x}\\n\", ProgramCounter, PC[0]);\n return std::nullopt;\n }\n } else if ((Instr & LDAXR_MASK) == LDAR_INST || // LDAR*\n (Instr & LDAXR_MASK) == LDAPR_INST || // LDAPR*\n (Instr & LDAXR_MASK) == STLR_INST) { // STLR*\n // This must fall through to the spin-lock implementation below.\n // This mask has a partial overlap with ATOMIC_MEM_INST so we need to check this here.\n } else if ((Instr & ArchHelpers::Arm64::ATOMIC_MEM_MASK) == ArchHelpers::Arm64::ATOMIC_MEM_INST) { // Atomic memory op\n if (ArchHelpers::Arm64::HandleAtomicMemOp(Instr, GPRs, StrictSplitLockMutex)) {\n // Skip this instruction now\n return 4;\n } else {\n uint8_t Op = (PC[0] >> 12) & 0xF;\n LogMan::Msg::EFmt(\"Unhandled JIT SIGBUS Atomic mem op 0x{:02x}: PC: 0x{:x} Instruction: 0x{:08x}\\n\", Op, ProgramCounter, PC[0]);\n return std::nullopt;\n }\n } else if ((Instr & ArchHelpers::Arm64::LDAXR_MASK) == ArchHelpers::Arm64::LDAXR_INST) { // LDAXR*\n uint64_t BytesToSkip = ArchHelpers::Arm64::HandleAtomicLoadstoreExclusive(ProgramCounter, GPRs, StrictSplitLockMutex);\n if (BytesToSkip) {\n // Skip this instruction now\n return BytesToSkip;\n }\n // Explicit fallthrough to the backpatch handler below!\n } else if ((Instr & ArchHelpers::Arm64::LDAXP_MASK) == ArchHelpers::Arm64::LDAXP_INST) { // LDAXP\n // Should be compare and swap pair only. LDAXP not used elsewhere\n uint64_t BytesToSkip = ArchHelpers::Arm64::HandleCASPAL_ARMv8(Instr, ProgramCounter, GPRs, StrictSplitLockMutex);\n if (BytesToSkip) {\n // Skip this instruction now\n return BytesToSkip;\n } else {\n LogMan::Msg::EFmt(\"Unhandled JIT SIGBUS CASPAL: PC: 0x{:x} Instruction: 0x{:08x}\\n\", ProgramCounter, PC[0]);\n return std::nullopt;\n }\n }\n\n // Lock code mutex during any SIGBUS handling that potentially changes code.\n // Due to code buffer sharing between threads, code must be carefully backpatched from last to first.\n // Multiple threads can be attempting to handle the SIGBUS or even be executing the code being backpatched.\n FEXCore::Utils::SpinWaitLock::UniqueSpinMutex lk(&InlineTail->SpinLockFutex);\n\n if ((Instr & LDAXR_MASK) == LDAR_INST || // LDAR*\n (Instr & LDAXR_MASK) == LDAPR_INST) { // LDAPR*\n uint32_t LDR = LDR_INST;\n LDR |= Size << 30;\n LDR |= AddrReg << 5;\n LDR |= DataReg;\n if (HandleType != UnalignedHandlerType::NonAtomic) {\n // Ordering matters with cross-thread visibility!\n std::atomic_ref(PC[1]).store(DMB_LD, std::memory_order_release); // Back-patch the half-barrier.\n }\n std::atomic_ref(PC[0]).store(LDR, std::memory_order_release);\n ClearICache(&PC[0], 8);\n // With the instruction modified, now execute again.\n return 0;\n } else if ((Instr & LDAXR_MASK) == STLR_INST) { // STLR*\n uint32_t STR = STR_INST;\n STR |= Size << 30;\n STR |= AddrReg << 5;\n STR |= DataReg;\n if (HandleType != UnalignedHandlerType::NonAtomic) {\n std::atomic_ref(PC[-1]).store(DMB, std::memory_order_release); // Back-patch the half-barrier.\n }\n std::atomic_ref(PC[0]).store(STR, std::memory_order_release);\n ClearICache(&PC[-1], 8);\n // Back up one instruction and have another go\n return -4;\n } else if ((Instr & RCPC2_MASK) == LDAPUR_INST) { // LDAPUR*\n // Extract the 9-bit offset from the instruction\n uint32_t LDUR = LDUR_INST;\n LDUR |= Size << 30;\n LDUR |= AddrReg << 5;\n LDUR |= DataReg;\n LDUR |= Instr & (0b1'1111'1111 << 12);\n if (HandleType != UnalignedHandlerType::NonAtomic) {\n // Ordering matters with cross-thread visibility!\n std::atomic_ref(PC[1]).store(DMB_LD, std::memory_order_release); // Back-patch the half-barrier.\n }\n std::atomic_ref(PC[0]).store(LDUR, std::memory_order_release);\n ClearICache(&PC[0], 8);\n // With the instruction modified, now execute again.\n return 0;\n } else if ((Instr & RCPC2_MASK) == STLUR_INST) { // STLUR*\n uint32_t STUR = STUR_INST;\n STUR |= Size << 30;\n STUR |= AddrReg << 5;\n STUR |= DataReg;\n STUR |= Instr & (0b1'1111'1111 << 12);\n if (HandleType != UnalignedHandlerType::NonAtomic) {\n std::atomic_ref(PC[-1]).store(DMB, std::memory_order_release); // Back-patch the half-barrier.\n }\n std::atomic_ref(PC[0]).store(STUR, std::memory_order_release);\n\n ClearICache(&PC[-1], 8);\n // Back up one instruction and have another go\n return -4;\n }\n\n // Check if another thread backpatched this instruction before this thread got here\n // Since we got here, this can happen in a couple situations:\n // - Unhandled instruction (Shouldn't occur, FEX programmer error added a new unhandled atomic)\n // - Another thread backpatched an atomic access to be a non-atomic access\n auto AtomicInst = std::atomic_ref(PC[0]).load(std::memory_order_acquire);\n if ((AtomicInst & LDSTREGISTER_MASK) == LDR_INST || (AtomicInst & LDSTUNSCALED_MASK) == LDUR_INST) {\n // This atomic instruction was backpatched to a load.\n if (HandleType != UnalignedHandlerType::NonAtomic) {\n // Check if the next instruction is a DMB.\n auto DMBInst = std::atomic_ref(PC[1]).load(std::memory_order_acquire);\n if (DMBInst == DMB_LD) {\n return 0;\n }\n } else {\n // No DMB instruction with this HandleType.\n return 0;\n }\n } else if ((AtomicInst & LDSTREGISTER_MASK) == STR_INST || (AtomicInst & LDSTUNSCALED_MASK) == STUR_INST) {\n if (HandleType != UnalignedHandlerType::NonAtomic) {\n // Check if the previous instruction is a DMB.\n auto DMBInst = std::atomic_ref(PC[-1]).load(std::memory_order_acquire);\n if (DMBInst == DMB) {\n // Return handled, make sure to adjust PC so we run the DMB.\n return -4;\n }\n } else {\n // No DMB instruction with this HandleType.\n return 0;\n }\n } else if (AtomicInst == DMB) {\n // ARMv8.0-a LDAXP backpatch handling. Will have turned in to the following:\n // - PC[0] = DMB\n // - PC[1] = STP\n // - PC[2] = DMB\n auto STPInst = std::atomic_ref(PC[1]).load(std::memory_order_acquire);\n auto DMBInst = std::atomic_ref(PC[2]).load(std::memory_order_acquire);\n if ((STPInst & LDSTP_MASK) == STP_INST && DMBInst == DMB) {\n // Code that was backpatched is what was expected for ARMv8.0-a LDAXP.\n return 0;\n }\n }\n\n LogMan::Msg::EFmt(\"Unhandled JIT SIGBUS: PC: 0x{:x} Instruction: 0x{:08x}\\n\", ProgramCounter, PC[0]);\n return std::nullopt;\n}\n\n\n} // namespace FEXCore::ArchHelpers::Arm64\n" }, { "path": "FEXCore/Source/Utils/ArchHelpers/Arm64_stubs.cpp", "content": "// SPDX-License-Identifier: MIT\n#include \n#include \n#include \n\nnamespace FEXCore::ArchHelpers::Arm64 {\n\n#ifndef ARCHITECTURE_arm64\n// These are stub implementations that exist only to allow instantiating the arm64 jit\n// on non arm platforms.\n\n// Obvously such a configuration can't do the actual arm64-specific stuff\n\nstd::optional\nHandleUnalignedAccess(FEXCore::Core::InternalThreadState* Thread, UnalignedHandlerType HandleType, uintptr_t ProgramCounter, uint64_t* GPRs) {\n ERROR_AND_DIE_FMT(\"HandleAtomicMemOp Not Implemented\");\n}\n\n#endif\n\n} // namespace FEXCore::ArchHelpers::Arm64\n" }, { "path": "FEXCore/Source/Utils/BucketList.h", "content": "// SPDX-License-Identifier: MIT\n#pragma once\n\n#include \n#include \n#include \n\nnamespace FEXCore {\n\n// BucketList is an optimized container, it includes an inline array of Size\n// and can overflow to a linked list of further buckets\n//\n// To optimize for best performance, Size should be big enough to allocate one or two\n// buckets for the typical case\n// Picking a Size so sizeof(Bucket<...>) is a power of two is also a small win\ntemplate\nstruct BucketList {\n static constexpr size_t Size = _Size;\n\n T Items[Size];\n fextl::unique_ptr> Next;\n\n void Clear() {\n Items[0] = T {};\n#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED\n for (size_t i = 1; i < Size; i++) {\n Items[i] = T {0xDEADBEEF};\n }\n#endif\n Next.reset();\n }\n\n BucketList() {\n Clear();\n }\n\n template\n void Iterate(EnumeratorFn Enumerator) const {\n size_t i = 0;\n auto Bucket = this;\n\n while (true) {\n auto Item = Bucket->Items[i];\n if (Item == T {}) {\n break;\n }\n\n Enumerator(Item);\n\n if (++i == Size) {\n LOGMAN_THROW_A_FMT(Bucket->Next != nullptr, \"Interference bug\");\n Bucket = Bucket->Next.get();\n i = 0;\n }\n }\n }\n\n template\n bool Find(EnumeratorFn Enumerator) const {\n size_t i = 0;\n auto Bucket = this;\n\n while (true) {\n auto Item = Bucket->Items[i];\n if (Item == T {}) {\n break;\n }\n\n if (Enumerator(Item)) {\n return true;\n }\n\n if (++i == Size) {\n LOGMAN_THROW_A_FMT(Bucket->Next != nullptr, \"Bucket in bad state\");\n Bucket = Bucket->Next.get();\n i = 0;\n }\n }\n\n return false;\n }\n\n void Append(T Val) {\n auto that = this;\n\n while (that->Next) {\n that = that->Next.get();\n }\n\n size_t i;\n for (i = 0; i < Size; i++) {\n if (that->Items[i] == T {}) {\n that->Items[i] = Val;\n break;\n }\n }\n\n if (i < (Size - 1)) {\n that->Items[i + 1] = T {};\n } else {\n that->Next = fextl::make_unique>();\n }\n }\n void Erase(T Val) {\n size_t i = 0;\n auto that = this;\n auto foundThat = this;\n size_t foundI = 0;\n\n while (true) {\n if (that->Items[i] == Val) {\n foundThat = that;\n foundI = i;\n break;\n } else if (++i == Size) {\n i = 0;\n LOGMAN_THROW_A_FMT(that->Next != nullptr, \"Bucket::Erase but element not contained\");\n that = that->Next.get();\n }\n }\n\n while (true) {\n if (that->Items[i] == T {}) {\n foundThat->Items[foundI] = that->Items[i - 1];\n that->Items[i - 1] = T {};\n break;\n } else if (++i == Size) {\n if (that->Next->Items[0] == T {}) {\n that->Next.reset();\n foundThat->Items[foundI] = that->Items[Size - 1];\n that->Items[Size - 1] = T {};\n break;\n }\n i = 0;\n that = that->Next.get();\n }\n }\n }\n};\n\n} // namespace FEXCore\n" }, { "path": "FEXCore/Source/Utils/Config.h", "content": "// SPDX-License-Identifier: MIT\n#include \n\nnamespace FEXCore::Config {\nconst fextl::string& GetTelemetryDirectory();\n}\n" }, { "path": "FEXCore/Source/Utils/FileLoading.cpp", "content": "// SPDX-License-Identifier: MIT\n#include \n\n#include \n#include \n\n#include \n#include \n#include \n#include \n#ifdef _WIN32\n#include \n#endif\n\nnamespace FEXCore::FileLoading {\n\n#ifndef _WIN32\ntemplate\nstatic bool LoadFileImpl(T& Data, const fextl::string& Filepath, size_t FixedSize) {\n int FD = open(Filepath.c_str(), O_RDONLY);\n\n if (FD == -1) {\n return false;\n }\n\n size_t FileSize {};\n if (FixedSize == 0) {\n struct stat buf;\n if (fstat(FD, &buf) == 0) {\n FileSize = buf.st_size;\n }\n } else {\n FileSize = FixedSize;\n }\n\n ssize_t CurrentOffset = 0;\n ssize_t Read = -1;\n bool LoadedFile {};\n if (FileSize) {\n // File size is known upfront\n Data.resize(FileSize);\n size_t Remaining = FileSize;\n while (CurrentOffset != FileSize && (Read = pread(FD, &Data.at(CurrentOffset), Remaining, CurrentOffset)) > 0) {\n CurrentOffset += Read;\n Remaining -= Read;\n }\n\n LoadedFile = CurrentOffset == FileSize && Read != -1;\n } else {\n // The file is either empty or its size is unknown (e.g. procfs data).\n // Try reading in chunks instead\n constexpr size_t READ_SIZE = 4096;\n Data.resize(READ_SIZE);\n\n while ((Read = pread(FD, &Data.at(CurrentOffset), READ_SIZE, CurrentOffset)) > 0) {\n CurrentOffset += Read;\n if ((CurrentOffset + READ_SIZE) > Data.size()) {\n Data.resize(CurrentOffset + READ_SIZE);\n }\n }\n\n if (Read == -1) {\n Data.clear();\n close(FD);\n return false;\n }\n\n // Final resize to ensure there is no garbage data past the end.\n Data.resize(CurrentOffset + Read);\n\n LoadedFile = true;\n }\n close(FD);\n return LoadedFile;\n}\n\nssize_t LoadFileToBuffer(const fextl::string& Filepath, std::span Buffer) {\n int FD = open(Filepath.c_str(), O_RDONLY);\n\n if (FD == -1) {\n return -1;\n }\n\n ssize_t Read = pread(FD, Buffer.data(), Buffer.size(), 0);\n close(FD);\n return Read;\n}\n\n#else\ntemplate\nstatic bool LoadFileImpl(T& Data, const fextl::string& Filepath, size_t FixedSize) {\n std::ifstream f(Filepath.c_str(), std::ios::binary | std::ios::ate);\n if (f.fail()) {\n return false;\n }\n auto Size = f.tellg();\n f.seekg(0, std::ios::beg);\n Data.resize(Size);\n f.read(Data.data(), Size);\n return !f.fail();\n}\n\nssize_t LoadFileToBuffer(const fextl::string& Filepath, std::span Buffer) {\n std::ifstream f(Filepath.c_str(), std::ios::binary | std::ios::ate);\n return f.readsome(Buffer.data(), Buffer.size());\n}\n\n#endif\n\nbool LoadFile(fextl::vector& Data, const fextl::string& Filepath, size_t FixedSize) {\n return LoadFileImpl(Data, Filepath, FixedSize);\n}\n\nbool LoadFile(fextl::string& Data, const fextl::string& Filepath, size_t FixedSize) {\n return LoadFileImpl(Data, Filepath, FixedSize);\n}\n\n} // namespace FEXCore::FileLoading\n" }, { "path": "FEXCore/Source/Utils/ForcedAssert.cpp", "content": "// SPDX-License-Identifier: MIT\nnamespace FEXCore::Assert {\n// This function can not be inlined\n[[noreturn]]\n__attribute__((noinline, naked)) void ForcedAssert() {\n#ifdef ARCHITECTURE_x86_64\n asm volatile(\"ud2\");\n#else\n asm volatile(\"hlt #1\");\n#endif\n}\n} // namespace FEXCore::Assert\n" }, { "path": "FEXCore/Source/Utils/LogManager.cpp", "content": "// SPDX-License-Identifier: MIT\n/*\n$info$\ntags: glue|log-manager\n$end_info$\n*/\n\n#include \n#include \n#include \n\nnamespace LogMan {\n\nnamespace Throw {\n ThrowHandler Handler {};\n void InstallHandler(ThrowHandler _Handler) {\n Handler = _Handler;\n }\n void UnInstallHandler() {\n Handler = nullptr;\n }\n\n void MFmt(const char* fmt, const fmt::format_args& args) {\n if (Handler) {\n auto msg = fextl::fmt::vformat(fmt, args);\n Handler(msg.c_str());\n }\n\n FEX_TRAP_EXECUTION;\n }\n} // namespace Throw\n\nnamespace Msg {\n MsgHandler Handler {};\n void InstallHandler(MsgHandler _Handler) {\n Handler = _Handler;\n }\n void UnInstallHandler() {\n Handler = nullptr;\n }\n\n void MFmtImpl(DebugLevels level, const char* fmt, const fmt::format_args& args) {\n if (Handler) {\n const auto msg = fextl::fmt::vformat(fmt, args);\n Handler(level, msg.c_str());\n }\n }\n\n} // namespace Msg\n} // namespace LogMan\n" }, { "path": "FEXCore/Source/Utils/LongJump.cpp", "content": "// SPDX-License-Identifier: MIT\n#include \n#include \n\n#include \n\nnamespace FEXCore::UncheckedLongJump {\n#if defined(ARCHITECTURE_arm64)\n[[nodiscard]]\nFEX_DEFAULT_VISIBILITY FEX_NAKED uint64_t SetJump(JumpBuf& Buffer) {\n __asm volatile(R\"(\n // x0 contains the jumpbuffer\n stp x19, x20, [x0, #( 0 * 8)];\n stp x21, x22, [x0, #( 2 * 8)];\n stp x23, x24, [x0, #( 4 * 8)];\n stp x25, x26, [x0, #( 6 * 8)];\n stp x27, x28, [x0, #( 8 * 8)];\n stp x29, x30, [x0, #(10 * 8)];\n\n // FPRs\n stp d8, d9, [x0, #(12 * 8)];\n stp d10, d11, [x0, #(14 * 8)];\n stp d12, d13, [x0, #(16 * 8)];\n stp d14, d15, [x0, #(18 * 8)];\n\n // Move SP in to a temporary to store.\n mov x1, sp;\n str x1, [x0, #(20 * 8)];\n\n // Return zero to signify this is the SetJump.\n mov x0, #0;\n ret;\n )\" ::\n : \"memory\");\n}\n\n[[noreturn]]\nFEX_DEFAULT_VISIBILITY FEX_NAKED void LongJump(const JumpBuf& Buffer, uint64_t Value) {\n __asm volatile(R\"(\n // x0 contains the jumpbuffer\n ldp x19, x20, [x0, #( 0 * 8)];\n ldp x21, x22, [x0, #( 2 * 8)];\n ldp x23, x24, [x0, #( 4 * 8)];\n ldp x25, x26, [x0, #( 6 * 8)];\n ldp x27, x28, [x0, #( 8 * 8)];\n ldp x29, x30, [x0, #(10 * 8)];\n\n // FPRs\n ldp d8, d9, [x0, #(12 * 8)];\n ldp d10, d11, [x0, #(14 * 8)];\n ldp d12, d13, [x0, #(16 * 8)];\n ldp d14, d15, [x0, #(18 * 8)];\n\n // Load SP in to temporary then move\n ldr x0, [x0, #(20 * 8)];\n mov sp, x0;\n\n // Move value in to result register\n mov x0, x1;\n ret;\n )\" ::\n : \"memory\");\n}\n\nFEX_DEFAULT_VISIBILITY void ManuallyLoadJumpBuf(const JumpBuf& Buffer, uint64_t Value, uint64_t* GPRs, __uint128_t* FPRs, uint64_t* PC) {\n // First 12 values are registers [x19,x30].\n memcpy(&GPRs[19], &Buffer.Registers[0], sizeof(uint64_t) * 12);\n\n // Next 8 values are [D8,D15]\n // Retain upper 64-bits of the register, only modifying lower 64-bits.\n for (size_t i = 0; i < 8; ++i) {\n memcpy(&FPRs[8 + i], &Buffer.Registers[12 + i], sizeof(uint64_t));\n }\n\n // Last value is stack pointer\n memcpy(&GPRs[31], &Buffer.Registers[20], sizeof(uint64_t));\n\n // Load the expected value in to X0\n GPRs[0] = Value;\n\n // Load the PC with the current LR.\n *PC = GPRs[30];\n}\n\n#else\n[[nodiscard]]\nFEX_DEFAULT_VISIBILITY FEX_NAKED uint64_t SetJump(JumpBuf& Buffer) {\n __asm volatile(R\"(\n .intel_syntax noprefix;\n // rdi contains the jumpbuffer\n mov [rdi + (0 * 8)], rbx;\n mov [rdi + (1 * 8)], rsp;\n mov [rdi + (2 * 8)], rbp;\n mov [rdi + (3 * 8)], r12;\n mov [rdi + (4 * 8)], r13;\n mov [rdi + (5 * 8)], r14;\n mov [rdi + (6 * 8)], r15;\n\n // Return address is on the stack, load it and store\n mov rsi, [rsp];\n mov [rdi + (7 * 8)], rsi;\n\n // Return zero to signify this is the SetJump.\n mov rax, 0;\n ret;\n\n .att_syntax prefix;\n )\" ::\n : \"memory\");\n}\n\n[[noreturn]]\nFEX_DEFAULT_VISIBILITY FEX_NAKED void LongJump(const JumpBuf& Buffer, uint64_t Value) {\n __asm volatile(R\"(\n .intel_syntax noprefix;\n // rdi contains the jumpbuffer\n mov rbx, [rdi + (0 * 8)];\n mov rsp, [rdi + (1 * 8)];\n mov rbp, [rdi + (2 * 8)];\n mov r12, [rdi + (3 * 8)];\n mov r13, [rdi + (4 * 8)];\n mov r14, [rdi + (5 * 8)];\n mov r15, [rdi + (6 * 8)];\n\n // Move value in to result register\n mov rax, rsi;\n\n // Pop the dead return address off the stack\n pop rsi;\n\n // Load the original return address from the jumpbuffer\n mov rsi, [rdi + (7 * 8)];\n\n // Return using a jump\n jmp rsi;\n\n .att_syntax prefix;\n )\" ::\n : \"memory\");\n}\n\nFEX_DEFAULT_VISIBILITY void ManuallyLoadJumpBuf(JumpBuf& Buffer, uint64_t Value, uint64_t* GPRs, __uint128_t* FPRs, uint64_t* PC) {\n LOGMAN_MSG_A_FMT(\"This is unimplemented on x86-64\");\n}\n\n#endif\n} // namespace FEXCore::UncheckedLongJump\n" }, { "path": "FEXCore/Source/Utils/MemberFunctionToPointer.h", "content": "// SPDX-License-Identifier: MIT\n#pragma once\n\n#include \n\n#include \n\nnamespace FEXCore::Utils {\n\n/**\n * @brief Casts a class's member function pointer to a raw pointer that we can JIT\n *\n * Has additional validation to ensure we aren't casting a class member that is invalid\n */\ntemplate\nclass MemberFunctionToPointerCast final {\npublic:\n MemberFunctionToPointerCast(PointerToMemberType Function) {\n memcpy(&PMF, &Function, sizeof(PMF));\n }\n\n uintptr_t GetConvertedPointer() const {\n#ifdef ARCHITECTURE_x86_64\n // Itanium C++ ABI (https://itanium-cxx-abi.github.io/cxx-abi/abi.html#member-function-pointers)\n // Low bit of ptr specifies if this Member function pointer is virtual or not\n // Throw an assert if we were trying to cast a virtual member\n LOGMAN_THROW_A_FMT((PMF.ptr & 1) == 0, \"C++ Pointer-To-Member representation didn't have low bit set to 0. Are you trying to cast a \"\n \"virtual member?\");\n#elif defined(ARCHITECTURE_arm64)\n // C++ ABI for the Arm 64-bit Architecture (IHI 0059E)\n // 4.2.1 Representation of pointer to member function\n // Differs from Itanium specification\n LOGMAN_THROW_A_FMT(PMF.adj == 0, \"C++ Pointer-To-Member representation didn't have adj == 0. Are you trying to cast a virtual member?\");\n#else\n#error \"Don't know how to cast Member to function here. Likely just Itanium\"\n#endif\n return PMF.ptr;\n }\n\n // Gets the vtable entry position of a virtual member function.\n size_t GetVTableOffset() const {\n#ifdef ARCHITECTURE_x86_64\n // Itanium C++ ABI (https://itanium-cxx-abi.github.io/cxx-abi/abi.html#member-function-pointers)\n // Low bit of ptr specifies if this Member function pointer is virtual or not\n // Throw an assert if we are not loading a virtual member.\n LOGMAN_THROW_A_FMT((PMF.ptr & 1) == 1, \"C++ Pointer-To-Member representation didn't have low bit set to 1. This cast only works for \"\n \"virtual members.\");\n return PMF.ptr & ~1ULL;\n#elif defined(ARCHITECTURE_arm64)\n // C++ ABI for the Arm 64-bit Architecture (IHI 0059E)\n // 4.2.1 Representation of pointer to member function\n // Differs from Itanium specification\n LOGMAN_THROW_A_FMT((PMF.adj & 1) == 1, \"C++ Pointer-To-Member representation didn't have adj == 1. This cast only works for virtual \"\n \"members.\");\n return PMF.ptr;\n#else\n#error \"Don't know how to cast Member to function here. Likely just Itanium\"\n#endif\n }\n\n // Gets the pointer to the vtable entry for the object passed it.\n template\n uintptr_t GetVTableEntry(Class* VirtualClass) const {\n // VTable is always stored at the beginning of a class object.\n uintptr_t* VTable = *reinterpret_cast(VirtualClass);\n\n size_t Offset = GetVTableOffset() / sizeof(void*);\n return VTable[Offset];\n }\n\nprivate:\n struct PointerToMember {\n uintptr_t ptr;\n uintptr_t adj;\n };\n\n PointerToMember PMF;\n\n // Ensure the representation of PointerToMember matches\n static_assert(sizeof(PMF) == sizeof(PointerToMemberType));\n};\n} // namespace FEXCore::Utils\n" }, { "path": "FEXCore/Source/Utils/Profiler.cpp", "content": "// SPDX-License-Identifier: MIT\n#include \n#include \n#ifndef _WIN32\n#include \n#include \n#include \n#endif\n\n#include \n#include \n#include \n#include \n#include \n\n#ifdef ENABLE_FEXCORE_PROFILER\n#if FEXCORE_PROFILER_BACKEND == FEXCORE_PROFILER_BACKEND_GPUVIS\n#include \n#include \n#include \n#ifndef _WIN32\nstatic inline uint64_t GetTime() {\n // We want the time in the least amount of overhead possible\n // clock_gettime will do a VDSO call with the least amount of overhead\n struct timespec ts;\n clock_gettime(CLOCK_MONOTONIC, &ts);\n return ts.tv_sec * 1'000'000'000ULL + ts.tv_nsec;\n}\n#else\n\nstatic inline uint64_t GetTime() {\n // GetTime needs to return nanoseconds, query the interface.\n static uint64_t FrequencyScale = {};\n if (!FrequencyScale) [[unlikely]] {\n LARGE_INTEGER Frequency {};\n while (!QueryPerformanceFrequency(&Frequency))\n ;\n constexpr uint64_t NanosecondsInSecond = 1'000'000'000ULL;\n\n // On WINE this will always result in a scale of 100.\n FrequencyScale = NanosecondsInSecond / Frequency.QuadPart;\n }\n LARGE_INTEGER ticks;\n while (!QueryPerformanceCounter(&ticks))\n ;\n return ticks.QuadPart * FrequencyScale;\n}\n\n#endif\n\nnamespace FEXCore::Profiler {\nProfilerBlock::ProfilerBlock(const std::string_view Format)\n : DurationBegin {GetTime()}\n , Format {Format} {}\n\nProfilerBlock::~ProfilerBlock() {\n auto Duration = GetTime() - DurationBegin;\n TraceObject(Format, Duration);\n}\n} // namespace FEXCore::Profiler\n\nnamespace GPUVis {\n// ftrace FD for writing trace data.\n// Needs to be a raw FD since we hold this open for the entire application execution.\nstatic int TraceFD {-1};\n\n// Need to search the paths to find the real trace path\nstatic std::array TraceFSDirectories {\n \"/sys/kernel/tracing\",\n \"/sys/kernel/debug/tracing\",\n};\n\nvoid Init() {\n FEX_CONFIG_OPT(EnableGpuvisProfiling, ENABLEGPUVISPROFILING);\n if (!EnableGpuvisProfiling()) {\n return;\n }\n for (auto Path : TraceFSDirectories) {\n#ifdef _WIN32\n constexpr auto flags = O_WRONLY;\n#else\n constexpr auto flags = O_WRONLY | O_CLOEXEC;\n#endif\n fextl::string FilePath = fextl::fmt::format(\"{}/trace_marker\", Path);\n TraceFD = open(FilePath.c_str(), flags);\n if (TraceFD != -1) {\n // Opened TraceFD, early exit\n break;\n }\n }\n}\n\nvoid Shutdown() {\n if (TraceFD != -1) {\n close(TraceFD);\n TraceFD = -1;\n }\n}\n\nvoid TraceObject(const std::string_view Format, uint64_t Duration) {\n if (TraceFD != -1) {\n // Print the duration as something that began negative duration ago\n const auto StringSize = Format.size() + strlen(\" (lduration=-)\\n\") + 22;\n auto Event = reinterpret_cast(alloca(StringSize));\n auto Res = ::fmt::format_to_n(Event, StringSize, \"{} (lduration=-{})\\n\", Format, Duration);\n write(TraceFD, Event, Res.size);\n }\n}\n\nvoid TraceObject(const std::string_view Format) {\n if (TraceFD != -1) {\n const auto StringSize = Format.size() + 1;\n auto Event = reinterpret_cast(alloca(StringSize));\n auto Res = ::fmt::format_to_n(Event, StringSize, \"{}\\n\", Format);\n write(TraceFD, Event, Res.size);\n }\n}\n} // namespace GPUVis\n#elif FEXCORE_PROFILER_BACKEND == FEXCORE_PROFILER_BACKEND_TRACY\n#include \"tracy/Tracy.hpp\"\nnamespace Tracy {\nstatic int EnableAfterFork = 0;\nstatic bool Enable = false;\n\nvoid Init(std::string_view ProgramName, std::string_view ProgramPath) {\n const char* ProfileTargetName = getenv(\"FEX_PROFILE_TARGET_NAME\"); // Match by application name\n const char* ProfileTargetPath = getenv(\"FEX_PROFILE_TARGET_PATH\"); // Match by path suffix\n const char* WaitForFork = getenv(\"FEX_PROFILE_WAIT_FOR_FORK\"); // Don't enable profiling until the process forks N times\n bool Matched = (ProfileTargetName && ProgramName == ProfileTargetName) || (ProfileTargetPath && ProgramPath.ends_with(ProfileTargetPath));\n if (Matched && WaitForFork) {\n EnableAfterFork = std::atoi(WaitForFork);\n }\n Enable = Matched && !EnableAfterFork;\n if (Enable) {\n tracy::StartupProfiler();\n LogMan::Msg::IFmt(\"Tracy profiling started\");\n } else if (EnableAfterFork) {\n LogMan::Msg::IFmt(\"Tracy profiling will start after fork\");\n }\n}\n\nvoid PostForkAction(bool IsChild) {\n if (Enable) {\n // Tracy does not support multiprocess profiling\n LogMan::Msg::EFmt(\"Warning: Profiling a process with forks is not supported. Set the environment variable \"\n \"FEX_PROFILE_WAIT_FOR_FORK= to start profiling after the n-th fork.\");\n }\n\n if (IsChild) {\n Enable = false;\n return;\n }\n\n if (EnableAfterFork > 1) {\n --EnableAfterFork;\n LogMan::Msg::IFmt(\"Tracy profiling will start after {} forks\", EnableAfterFork);\n } else if (EnableAfterFork == 1) {\n Enable = true;\n EnableAfterFork = 0;\n tracy::StartupProfiler();\n LogMan::Msg::IFmt(\"Tracy profiling started\");\n }\n}\n\nvoid Shutdown() {\n if (Tracy::Enable) {\n LogMan::Msg::IFmt(\"Stopping Tracy profiling\");\n tracy::ShutdownProfiler();\n }\n}\n\nvoid TraceObject(const std::string_view Format, uint64_t Duration) {}\n\nvoid TraceObject(const std::string_view Format) {\n if (Tracy::Enable) {\n TracyMessage(Format.data(), Format.size());\n }\n}\n} // namespace Tracy\n#else\n#error Unknown profiler backend\n#endif\n#endif\n\nnamespace FEXCore::Profiler {\n\n#ifdef ENABLE_FEXCORE_PROFILER\nvoid Init(std::string_view ProgramName, std::string_view ProgramPath) {\n#if FEXCORE_PROFILER_BACKEND == FEXCORE_PROFILER_BACKEND_GPUVIS\n GPUVis::Init();\n#elif FEXCORE_PROFILER_BACKEND == FEXCORE_PROFILER_BACKEND_TRACY\n Tracy::Init(ProgramName, ProgramPath);\n#endif\n}\n\nvoid PostForkAction(bool IsChild) {\n#if FEXCORE_PROFILER_BACKEND == FEXCORE_PROFILER_BACKEND_TRACY\n Tracy::PostForkAction(IsChild);\n#endif\n}\n\nbool IsActive() {\n#if FEXCORE_PROFILER_BACKEND == FEXCORE_PROFILER_BACKEND_GPUVIS\n // Always active\n return true;\n#elif FEXCORE_PROFILER_BACKEND == FEXCORE_PROFILER_BACKEND_TRACY\n // Active if previously enabled\n return Tracy::Enable;\n#endif\n}\n\nvoid Shutdown() {\n#if FEXCORE_PROFILER_BACKEND == FEXCORE_PROFILER_BACKEND_GPUVIS\n GPUVis::Shutdown();\n#elif FEXCORE_PROFILER_BACKEND == FEXCORE_PROFILER_BACKEND_TRACY\n Tracy::Shutdown();\n#endif\n}\n\nvoid TraceObject(const std::string_view Format, uint64_t Duration) {\n#if FEXCORE_PROFILER_BACKEND == FEXCORE_PROFILER_BACKEND_GPUVIS\n GPUVis::TraceObject(Format, Duration);\n#elif FEXCORE_PROFILER_BACKEND == FEXCORE_PROFILER_BACKEND_TRACY\n Tracy::TraceObject(Format, Duration);\n#endif\n}\n\nvoid TraceObject(const std::string_view Format) {\n#if FEXCORE_PROFILER_BACKEND == FEXCORE_PROFILER_BACKEND_GPUVIS\n GPUVis::TraceObject(Format);\n#elif FEXCORE_PROFILER_BACKEND == FEXCORE_PROFILER_BACKEND_TRACY\n Tracy::TraceObject(Format);\n#endif\n}\n\n#endif\n} // namespace FEXCore::Profiler\n" }, { "path": "FEXCore/Source/Utils/SpinWaitLock.cpp", "content": "// SPDX-License-Identifier: MIT\n#include \"Utils/SpinWaitLock.h\"\n\nnamespace FEXCore::Utils::SpinWaitLock {\n#ifdef ARCHITECTURE_arm64\nconstexpr uint64_t NanosecondsInSecond = 1'000'000'000ULL;\n\nstatic uint64_t GetCycleCounterFrequency() {\n uint64_t Result {};\n __asm(\"mrs %[Res], CNTFRQ_EL0\" : [Res] \"=r\"(Result));\n return Result;\n}\n\nstatic uint64_t CalculateCyclesPerNanosecond() {\n // Snapdragon devices historically use a 19.2Mhz cycle counter frequency\n // This means that the number of cycles per nanosecond ends up being 52.0833...\n //\n // ARMv8.6 and ARMv9.1 requires the cycle counter frequency to be 1Ghz.\n // This means the number of cycles per nanosecond ends up being 1.\n uint64_t CounterFrequency = GetCycleCounterFrequency();\n return NanosecondsInSecond / CounterFrequency;\n}\n\nuint64_t CycleCounterFrequency = GetCycleCounterFrequency();\nuint64_t CyclesPerNanosecond = CalculateCyclesPerNanosecond();\n#endif\n} // namespace FEXCore::Utils::SpinWaitLock\n" }, { "path": "FEXCore/Source/Utils/SpinWaitLock.h", "content": "// SPDX-License-Identifier: MIT\n#pragma once\n\n#include \n#include \n#include \n#include \n\n#include \n#include \n\nnamespace FEXCore::Utils::SpinWaitLock {\n/**\n * @brief This provides routines to implement implement an \"efficient spin-loop\" using ARM's WFE and exclusive monitor interfaces.\n *\n * Spin-loops on mobile devices with a battery can be a bad idea as they burn a bunch of power. This attempts to mitigate some of the impact\n * by putting the CPU in to a lower-power state using WFE.\n * On platforms tested, WFE will put the CPU in to a lower power state for upwards of 0.11ms(!) per WFE. Which isn't a significant amount of\n * time but should still have power savings. Ideally WFE would be able to keep the CPU in a lower power state for longer. This also has the\n * added benefit that atomics aren't abusing the caches when spinning on a cacheline, which has knock-on powersaving benefits.\n *\n * This short timeout is because the Linux kernel has a 100 microsecond architecture timer which wakes up WFE and WFI. Nothing can be\n * improved beyond that period.\n *\n * FEAT_WFxT adds a new instruction with a timeout, but since the spurious wake-up is so aggressive it isn't worth using.\n *\n * It should be noted that this implementation has a few dozen cycles of start-up time. Which means the overhead for invoking this\n * implementation is slightly higher than a true spin-loop. The hot loop body itself is only three instructions so it is quite efficient.\n *\n * On non-ARM platforms it is truly a spin-loop, which is okay for debugging only.\n */\n#ifdef ARCHITECTURE_arm64\n\n#define LOADEXCLUSIVE(LoadExclusiveOp, RegSize) \\\n /* Prime the exclusive monitor with the passed in address. */ \\\n #LoadExclusiveOp \" %\" #RegSize \"[Result], [%[Futex]];\\n\"\n\n#define SPINLOOP_BODY(LoadAtomicOp, RegSize) \\\n /* WFE will wait for either the memory to change or spurious wake-up. */ \\\n \"wfe;\\n\" /* Load with acquire to get the result of memory. */ \\\n #LoadAtomicOp \" %\" #RegSize \"[Result], [%[Futex]];\\n\"\n\n#define SPINLOOP_WFE_LDX_8BIT LOADEXCLUSIVE(ldaxrb, w)\n#define SPINLOOP_WFE_LDX_16BIT LOADEXCLUSIVE(ldaxrh, w)\n#define SPINLOOP_WFE_LDX_32BIT LOADEXCLUSIVE(ldaxr, w)\n#define SPINLOOP_WFE_LDX_64BIT LOADEXCLUSIVE(ldaxr, x)\n\n#define SPINLOOP_8BIT SPINLOOP_BODY(ldarb, w)\n#define SPINLOOP_16BIT SPINLOOP_BODY(ldarh, w)\n#define SPINLOOP_32BIT SPINLOOP_BODY(ldar, w)\n#define SPINLOOP_64BIT SPINLOOP_BODY(ldar, x)\n\nextern uint64_t CycleCounterFrequency;\nextern uint64_t CyclesPerNanosecond;\n\n///< Get the raw cycle counter which is synchronizing.\n/// `CNTVCTSS_EL0` also does the same thing, but requires the FEAT_ECV feature.\nstatic inline uint64_t GetCycleCounter() {\n uint64_t Result {};\n __asm volatile(R\"(\n isb;\n mrs %[Res], CNTVCT_EL0;\n )\"\n : [Res] \"=r\"(Result));\n return Result;\n}\n\n///< Converts nanoseconds to number of cycles.\n/// If the cycle counter is 1Ghz then this is a direct 1:1 map.\nstatic inline uint64_t ConvertNanosecondsToCycles(const std::chrono::nanoseconds& Nanoseconds) {\n const auto NanosecondCount = Nanoseconds.count();\n return NanosecondCount / CyclesPerNanosecond;\n}\n\nstatic inline uint8_t LoadExclusive(uint8_t* Futex) {\n uint8_t Result {};\n __asm volatile(SPINLOOP_WFE_LDX_8BIT : [Result] \"=r\"(Result), [Futex] \"+r\"(Futex)::\"memory\");\n\n return Result;\n}\n\nstatic inline uint16_t LoadExclusive(uint16_t* Futex) {\n uint16_t Result {};\n __asm volatile(SPINLOOP_WFE_LDX_16BIT : [Result] \"=r\"(Result), [Futex] \"+r\"(Futex)::\"memory\");\n\n return Result;\n}\n\nstatic inline uint32_t LoadExclusive(uint32_t* Futex) {\n uint32_t Result {};\n __asm volatile(SPINLOOP_WFE_LDX_32BIT : [Result] \"=r\"(Result), [Futex] \"+r\"(Futex)::\"memory\");\n\n return Result;\n}\n\nstatic inline uint64_t LoadExclusive(uint64_t* Futex) {\n uint64_t Result {};\n __asm volatile(SPINLOOP_WFE_LDX_64BIT : [Result] \"=r\"(Result), [Futex] \"+r\"(Futex)::\"memory\");\n\n return Result;\n}\n\nstatic inline uint8_t WFELoadAtomic(uint8_t* Futex) {\n uint8_t Result {};\n __asm volatile(SPINLOOP_8BIT : [Result] \"=r\"(Result), [Futex] \"+r\"(Futex)::\"memory\");\n\n return Result;\n}\n\nstatic inline uint16_t WFELoadAtomic(uint16_t* Futex) {\n uint16_t Result {};\n __asm volatile(SPINLOOP_16BIT : [Result] \"=r\"(Result), [Futex] \"+r\"(Futex)::\"memory\");\n\n return Result;\n}\n\nstatic inline uint32_t WFELoadAtomic(uint32_t* Futex) {\n uint32_t Result {};\n __asm volatile(SPINLOOP_32BIT : [Result] \"=r\"(Result), [Futex] \"+r\"(Futex)::\"memory\");\n\n return Result;\n}\n\nstatic inline uint64_t WFELoadAtomic(uint64_t* Futex) {\n uint64_t Result {};\n __asm volatile(SPINLOOP_64BIT : [Result] \"=r\"(Result), [Futex] \"+r\"(Futex)::\"memory\");\n\n return Result;\n}\n\ntemplate\nstatic inline void WaitPred(T* Futex, T ComparisonValue) {\n auto AtomicFutex = std::atomic_ref(*Futex);\n T Result = AtomicFutex.load();\n\n while (!Pred {}(Result, ComparisonValue)) {\n Result = LoadExclusive(Futex);\n if (Pred {}(Result, ComparisonValue)) {\n return;\n }\n\n Result = WFELoadAtomic(Futex);\n }\n}\n\ntemplate\nstatic inline bool Wait(T* Futex, TT ExpectedValue, const std::chrono::nanoseconds& Timeout) {\n auto AtomicFutex = std::atomic_ref(*Futex);\n\n T Result = AtomicFutex.load();\n\n // Early exit if possible.\n if (Result == ExpectedValue) {\n return true;\n }\n\n const auto TimeoutCycles = ConvertNanosecondsToCycles(Timeout);\n const auto Begin = GetCycleCounter();\n\n do {\n Result = LoadExclusive(Futex);\n if (Result == ExpectedValue) {\n return true;\n }\n Result = WFELoadAtomic(Futex);\n\n const auto CurrentCycleCounter = GetCycleCounter();\n if ((CurrentCycleCounter - Begin) >= TimeoutCycles) {\n // Couldn't get value before timeout.\n return false;\n }\n } while (Result != ExpectedValue);\n\n // We got our result.\n return true;\n}\n\ntemplate bool Wait(uint8_t*, uint8_t, const std::chrono::nanoseconds&);\ntemplate bool Wait(uint16_t*, uint16_t, const std::chrono::nanoseconds&);\ntemplate bool Wait(uint32_t*, uint32_t, const std::chrono::nanoseconds&);\ntemplate bool Wait(uint64_t*, uint64_t, const std::chrono::nanoseconds&);\n\ntemplate\nstatic inline T OneShotWFEBitComparison(T* Futex, T Mask, T Comp) {\n auto AtomicFutex = std::atomic_ref(*Futex);\n T Result = AtomicFutex.load();\n\n // Early exit if possible.\n if ((Result & Mask) == Comp) {\n return Result;\n }\n\n Result = LoadExclusive(Futex);\n if ((Result & Mask) == Comp) {\n return Result;\n }\n\n // Waits for write and returns result.\n Result = WFELoadAtomic(Futex);\n return Result;\n}\n\n#else\n\ntemplate\nstatic inline void WaitPred(T* Futex, T ComparisonValue) {\n auto AtomicFutex = std::atomic_ref(*Futex);\n T Result = AtomicFutex.load();\n\n while (!Pred {}(Result, ComparisonValue)) {\n Result = AtomicFutex.load();\n }\n}\n\ntemplate\nstatic inline bool Wait(T* Futex, TT ExpectedValue, const std::chrono::nanoseconds& Timeout) {\n auto AtomicFutex = std::atomic_ref(*Futex);\n\n T Result = AtomicFutex.load();\n\n // Early exit if possible.\n if (Result == ExpectedValue) {\n return true;\n }\n\n const auto Begin = std::chrono::high_resolution_clock::now();\n\n do {\n Result = AtomicFutex.load();\n\n const auto CurrentCycleCounter = std::chrono::high_resolution_clock::now();\n if ((CurrentCycleCounter - Begin) >= Timeout) {\n // Couldn't get value before timeout.\n return false;\n }\n } while (Result != ExpectedValue);\n\n // We got our result.\n return true;\n}\n#endif\n\ntemplate\nstatic inline void Wait(T* Futex, TT ExpectedValue) {\n WaitPred, T>(Futex, ExpectedValue);\n}\n\ntemplate void Wait(uint8_t*, uint8_t);\ntemplate void Wait(uint16_t*, uint16_t);\ntemplate void Wait(uint32_t*, uint32_t);\ntemplate void Wait(uint64_t*, uint64_t);\n\ntemplate\nstatic inline void lock(T* Futex) {\n auto AtomicFutex = std::atomic_ref(*Futex);\n T Expected {};\n T Desired {1};\n\n // Try to CAS immediately.\n if (AtomicFutex.compare_exchange_strong(Expected, Desired)) {\n return;\n }\n\n do {\n // Wait until the futex is unlocked.\n Wait(Futex, 0);\n Expected = 0;\n } while (!AtomicFutex.compare_exchange_strong(Expected, Desired));\n}\n\ntemplate\nstatic inline bool try_lock(T* Futex) {\n auto AtomicFutex = std::atomic_ref(*Futex);\n T Expected {};\n T Desired {1};\n\n // Try to CAS immediately.\n if (AtomicFutex.compare_exchange_strong(Expected, Desired)) {\n return true;\n }\n\n return false;\n}\n\ntemplate\nstatic inline void unlock(T* Futex) {\n auto AtomicFutex = std::atomic_ref(*Futex);\n AtomicFutex.store(0);\n}\n\n#undef SPINLOOP_8BIT\n#undef SPINLOOP_16BIT\n#undef SPINLOOP_32BIT\n#undef SPINLOOP_64BIT\ntemplate\nclass UniqueSpinMutex final {\npublic:\n // Move-only type\n UniqueSpinMutex(const UniqueSpinMutex&) = delete;\n UniqueSpinMutex& operator=(const UniqueSpinMutex&) = delete;\n UniqueSpinMutex(UniqueSpinMutex&& rhs) = default;\n UniqueSpinMutex& operator=(UniqueSpinMutex&&) = default;\n\n UniqueSpinMutex(T* Futex)\n : Futex {Futex} {\n FEXCore::Utils::SpinWaitLock::lock(Futex);\n }\n\n ~UniqueSpinMutex() {\n FEXCore::Utils::SpinWaitLock::unlock(Futex);\n }\nprivate:\n T* Futex;\n};\n} // namespace FEXCore::Utils::SpinWaitLock\n" }, { "path": "FEXCore/Source/Utils/Telemetry.cpp", "content": "// SPDX-License-Identifier: MIT\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n#include \"Utils/Config.h\"\n\n#include \n#include \n#include \n#include \n\nnamespace FEXCore::Telemetry {\n#ifndef FEX_DISABLE_TELEMETRY\nstd::array TelemetryValues = {{}};\nconst std::array TelemetryNames {\n \"64byte Split Locks\",\n \"16byte Split atomics\",\n \"EVEX instructions (AVX512)\",\n \"16bit CAS Tear\",\n \"32bit CAS Tear\",\n \"64bit CAS Tear\",\n \"128bit CAS Tear\",\n \"Crash mask\",\n \"Write 32-bit Segment ES\",\n \"Write 32-bit Segment SS\",\n \"Write 32-bit Segment CS\",\n \"Write 32-bit Segment DS\",\n \"Uses 32-bit Segment ES\",\n \"Uses 32-bit Segment SS\",\n \"Uses 32-bit Segment CS\",\n \"Uses 32-bit Segment DS\",\n \"Non-Canonical 64-bit address access\",\n};\n\nstatic bool Enabled {true};\nvoid Initialize() {\n FEX_CONFIG_OPT(DisableTelemetry, DISABLETELEMETRY);\n if (DisableTelemetry) {\n Enabled = false;\n return;\n }\n\n const auto& DataDirectory = Config::GetTelemetryDirectory();\n\n // Ensure the folder structure is created for our configuration\n if (!FHU::Filesystem::Exists(DataDirectory) && !FHU::Filesystem::CreateDirectories(DataDirectory)) {\n LogMan::Msg::IFmt(\"Couldn't create telemetry Folder\");\n }\n}\n\nvoid Shutdown(const fextl::string& ApplicationName) {\n if (!Enabled) {\n return;\n }\n\n auto DataDirectory = Config::GetTelemetryDirectory() + ApplicationName + \".telem\";\n\n // Retain a single backup if the telemetry already existed.\n auto Backup = DataDirectory + \".bck\";\n\n // Failure on rename is okay.\n (void)FHU::Filesystem::RenameFile(DataDirectory, Backup);\n\n auto File = FEXCore::File::File(DataDirectory.c_str(),\n FEXCore::File::FileModes::WRITE | FEXCore::File::FileModes::CREATE | FEXCore::File::FileModes::TRUNCATE);\n\n if (File.IsValid()) {\n for (size_t i = 0; i < TelemetryType::TYPE_LAST; ++i) {\n auto& Name = TelemetryNames.at(i);\n auto& Data = TelemetryValues.at(i);\n fextl::fmt::print(File, \"{}: {}\\n\", Name, Data.load());\n }\n File.Flush();\n }\n}\n\n#endif\n} // namespace FEXCore::Telemetry\n" }, { "path": "FEXCore/Source/Utils/Threads.cpp", "content": "// SPDX-License-Identifier: MIT\n#include \n#include \n#include \n\n#include \n#include \n\nnamespace FEXCore::Threads {\nstatic fextl::unique_ptr CreateThread_Default(ThreadFunc Func, void* Arg) {\n ERROR_AND_DIE_FMT(\"Frontend didn't setup thread creation!\");\n}\n\nstatic void CleanupAfterFork_Default() {\n ERROR_AND_DIE_FMT(\"Frontend didn't setup thread creation!\");\n}\n\nstatic FEXCore::Threads::Pointers Ptrs = {\n .CreateThread = CreateThread_Default,\n .CleanupAfterFork = CleanupAfterFork_Default,\n};\n\nfextl::unique_ptr FEXCore::Threads::Thread::Create(ThreadFunc Func, void* Arg) {\n return Ptrs.CreateThread(Func, Arg);\n}\n\nvoid FEXCore::Threads::Thread::CleanupAfterFork() {\n return Ptrs.CleanupAfterFork();\n}\n\nvoid FEXCore::Threads::Thread::SetInternalPointers(const Pointers& _Ptrs) {\n Ptrs = _Ptrs;\n}\n} // namespace FEXCore::Threads\n" }, { "path": "FEXCore/Source/Utils/WritePriorityMutex.h", "content": "// SPDX-License-Identifier: MIT\n#pragma once\n#include \n#include \n\n#if !defined(_WIN32)\n#include /* Definition of FUTEX_* constants */\n#include /* Definition of SYS_* constants */\n#include \n#else\n#include \n#endif\n\n#include \n\n#include \"Utils/SpinWaitLock.h\"\n\nnamespace FEXCore::Utils::WritePriorityMutex {\n\n// A custom mutex that prioritizes exclusive locks.\n// In highly contested scenarios, this can help minimize overall contention time.\n//\n// Features:\n// - Up to 32767 pending exclusive locks (\"writers\")\n// - Up to 32767 pending shared_locks (\"readers\")\n// - Low-overhead waiting via WFE with a fallback to futex on timeout\n// - Direct writer->reader hand-off and vice-versa to further reduce overhead\n//\n// Trade-offs:\n// - No guaranteed order of wake-ups besides prioritizing writers\n// - No support for recursive locking\n// - We can't use FUTEX_LOCK_PI to enable priority inheritance\nclass Mutex final {\npublic:\n Mutex() = default;\n\n // Move-only type\n Mutex(const Mutex&) = delete;\n Mutex& operator=(const Mutex&) = delete;\n Mutex(Mutex&& rhs) = delete;\n Mutex& operator=(Mutex&&) = delete;\n\n void lock() {\n // Try a non-blocking lock first.\n if (try_lock()) {\n return;\n }\n\n // Try a quick WFE write-lock.\n if (Attempt_WFE_WriteLock()) {\n return;\n }\n\n // Still couldn't get it. Start waiting.\n auto AtomicFutex = std::atomic_ref(Futex);\n\n uint32_t Expected {};\n uint32_t Desired {};\n#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED\n Expected = AtomicFutex.load(std::memory_order_relaxed);\n do {\n // Increment the number of write waiters.\n Desired = Expected + WRITE_WAITER_INCREMENT;\n\n LOGMAN_THROW_A_FMT((Desired & WRITE_WAITER_COUNT_MASK) != 0, \"Overflow in write-waiters!\");\n } while (AtomicFutex.compare_exchange_strong(Expected, Desired, std::memory_order_acq_rel, std::memory_order_acquire) == false);\n#else\n // Increment the number of writers waiting. The following loop will attempt to acquire the write-lock while decrementing the waiter count.\n Expected = AtomicFutex.fetch_add(WRITE_WAITER_INCREMENT);\n Desired = Expected + WRITE_WAITER_INCREMENT;\n#endif\n\n // Thread added to waiter list.\n Expected = Desired;\n\n while (true) {\n bool Sleep = false;\n\n do {\n if ((Expected & WRITE_OWNED_BIT) == 0 && (Expected & READ_OWNER_COUNT_MASK) == 0) {\n // If not write-owned, and no read-owners, try to acquire.\n LOGMAN_THROW_A_FMT((Expected & WRITE_WAITER_COUNT_MASK) != 0, \"Underflow in write-waiters!\");\n\n // Add write-owned bit.\n Desired = Expected | WRITE_OWNED_BIT;\n\n // Remove ourselves from the wait list.\n Desired -= WRITE_WAITER_INCREMENT;\n\n Sleep = false;\n } else {\n // Already write-owned or read-locked. Go to sleep.\n Desired = Expected;\n Sleep = true;\n break;\n }\n\n } while (AtomicFutex.compare_exchange_strong(Expected, Desired, std::memory_order_acq_rel, std::memory_order_acquire) == false);\n\n if (!Sleep) {\n // Acquired early.\n LOGMAN_THROW_A_FMT((Desired & WRITE_OWNED_BIT) == WRITE_OWNED_BIT, \"Somehow acquired a write-lock without it being set!\");\n return;\n }\n\n // Two paths to get here.\n // Desired[31] = 1 (WRITE_OWNED_BIT)\n // OR\n // Desired[15:0] != 0 (READ_OWNER_COUNT_MASK)\n // Meaning that there was already a writer that owned the lock, or reads were owning it.\n // This thread already incremented `WRITE_WAITER_INCREMENT` before this loop.\n // - Linux waits for the full 32-bits to change (With bitset wakeup).\n // - Win32 also waits for the full 32-bits to change (with offset addr on the reader side to reduce stampeding).\n FutexWaitForWriteAvailable(Desired);\n\n Expected = AtomicFutex.load(std::memory_order_relaxed);\n }\n }\n\n void lock_shared() {\n // Try an uncontended lock first.\n if (try_lock_shared()) {\n return;\n }\n\n // Try a quick WFE read-lock.\n if (Attempt_WFE_ReadLock()) {\n return;\n }\n\n auto AtomicFutex = std::atomic_ref(Futex);\n\n uint32_t Expected = AtomicFutex.load(std::memory_order_relaxed);\n uint32_t Desired {};\n\n while (true) {\n bool Sleep = false;\n do {\n if ((Expected & WRITE_OWNED_BIT) == 0 && (Expected & WRITE_WAITER_COUNT_MASK) == 0) {\n // If no write-owner and no write-waiting, try and acquire.\n\n Desired = Expected + READ_OWNER_INCREMENT;\n LOGMAN_THROW_A_FMT((Desired & READ_OWNER_COUNT_MASK) != 0, \"Overflow in read-owners!\");\n Sleep = false;\n } else {\n // Waiting for lock to become available. Add to waiters.\n Desired = Expected | READ_WAITER_BIT;\n Sleep = true;\n }\n } while (AtomicFutex.compare_exchange_strong(Expected, Desired, std::memory_order_acq_rel, std::memory_order_acquire) == false);\n\n if (!Sleep) {\n // Acquired early.\n LOGMAN_THROW_A_FMT((Desired & WRITE_OWNED_BIT) != WRITE_OWNED_BIT, \"Somehow read-locked and got a write lock!\");\n return;\n }\n\n // Only one path to get here.\n // Desired[31][29:16] != 0 (Either writer-owned, or writer-waiting)\n // Desired[30][15:0] == READ_WAIT_BIT and number of read-owners (draining to zero as write-side is set)\n // - Linux waits for full 32-bit futex.\n // - Win32 waits for upper 16-bits to not match (Either zero writer owned, writer-wait is draining, and `READ_WAITER_BIT` changed).\n // Can get some spurious wake-ups which will `or` the `READ_WAITER_BIT` again, which does nothing.\n FutexWaitForReadAvailable(Desired);\n\n Expected = AtomicFutex.load(std::memory_order_relaxed);\n }\n }\n\n void unlock() {\n auto AtomicFutex = std::atomic_ref(Futex);\n\n uint32_t Expected = AtomicFutex.load(std::memory_order_relaxed);\n uint32_t Desired {};\n do {\n LOGMAN_THROW_A_FMT((Expected & WRITE_OWNED_BIT) == WRITE_OWNED_BIT, \"Trying to write-unlock something not write-locked!\");\n // Remove the exclusive lock bit.\n Desired = Expected & ~WRITE_OWNED_BIT;\n\n // If no more writers, then make sure to clear the read-waiters bit as well.\n if ((Desired & WRITE_WAITER_COUNT_MASK) == 0) {\n Desired &= ~READ_WAITER_BIT;\n }\n } while (AtomicFutex.compare_exchange_strong(Expected, Desired, std::memory_order_acq_rel, std::memory_order_acquire) == false);\n\n // `Expected` has old value. Containing `READ_WAITER_BIT` which was just masked off, and also `WRITE_WAITER_COUNT_MASK`.\n //\n // Two paths here to be careful about dead-locking other waiters:\n // - If there are any writers waiting, those get priority to wake.\n // - If there are zero writers waiting, and there are read waiters then make sure to wake them all.\n // Failure to send wake events can cause readers to \"infinitely\" hang! (ignoring spurious wake-up).\n if ((Expected & WRITE_WAITER_COUNT_MASK)) {\n // Handle write-write handoff.\n FutexWakeWriter();\n } else if ((Expected & READ_WAITER_BIT)) {\n // Handle write-reader handoff.\n FutexWakeReaders();\n }\n }\n\n void unlock_shared() {\n auto AtomicFutex = std::atomic_ref(Futex);\n\n uint32_t Desired {};\n#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED\n uint32_t Expected = AtomicFutex.load(std::memory_order_relaxed);\n do {\n LOGMAN_THROW_A_FMT((Expected & WRITE_OWNED_BIT) != WRITE_OWNED_BIT, \"Trying to read-unlock something write-locked!\");\n LOGMAN_THROW_A_FMT((Expected & READ_OWNER_COUNT_MASK) != 0, \"Trying to read-unlock something not read-locked!\");\n\n // Decrement the shared counter.\n Desired = Expected - READ_OWNER_INCREMENT;\n } while (AtomicFutex.compare_exchange_strong(Expected, Desired, std::memory_order_acq_rel, std::memory_order_acquire) == false);\n#else\n Desired = AtomicFutex.fetch_sub(READ_OWNER_INCREMENT) - READ_OWNER_INCREMENT;\n#endif\n\n // Handle read->write handoff if there are any waiting writers, and no readers left.\n // Only one path here but still need to be careful to not dead-lock waiting writers.\n // - If there are waiters /but/ this is not the final unlock_shared, then don't wake writer.\n // - Writer would wake and immediately sleep again if we woke on every unlock_shared.\n // - If there are waiters and this is the final unlock_shared, then wake a /single/ writer.\n // - We ignore any reader-waiters here as they must wait their turn for writers that are waiting.\n if ((Desired & WRITE_WAITER_COUNT_MASK) && (Desired & READ_OWNER_COUNT_MASK) == 0) {\n FutexWakeWriter();\n }\n }\n\n bool try_lock() {\n auto AtomicFutex = std::atomic_ref(Futex);\n\n uint32_t Expected = 0;\n\n // Try and grab the owned bit.\n uint32_t Desired = WRITE_OWNED_BIT;\n\n // try to CAS immediately.\n return AtomicFutex.compare_exchange_strong(Expected, Desired, std::memory_order_acq_rel, std::memory_order_acquire);\n }\n\n // Can race with other threads trying to lock shared!\n bool try_lock_shared() {\n auto AtomicFutex = std::atomic_ref(Futex);\n uint32_t Expected = AtomicFutex.load(std::memory_order_relaxed);\n\n // Exclusively owned or has a list of waiting owners. Can't pass.\n if ((Expected & WRITE_OWNED_BIT) || (Expected & WRITE_WAITER_COUNT_MASK)) {\n return false;\n }\n\n // Try to add reader.\n uint32_t Desired = Expected + READ_OWNER_INCREMENT;\n LOGMAN_THROW_A_FMT((Desired & READ_OWNER_COUNT_MASK) != 0, \"Overflow in read-owners!\");\n\n // Uncontended mutex check\n return AtomicFutex.compare_exchange_strong(Expected, Desired, std::memory_order_acq_rel, std::memory_order_acquire);\n }\n\n#if !defined(_WIN32)\n // Initialize the internal mutex object to its default initializer state.\n // Should only ever be used in the child process when a Linux fork() has occured.\n void StealAndDropActiveLocks() {\n Futex = 0;\n }\n#endif\n\nprivate:\n\n#if !defined(_WIN32)\n void FutexWaitForWriteAvailable(uint32_t Expected) {\n ::syscall(SYS_futex, &Futex, FUTEX_PRIVATE_FLAG | FUTEX_WAIT_BITSET, Expected, nullptr, nullptr, FUTEX_BITSET_WAIT_WRITERS);\n }\n\n // Read-lock waiting for writers to drain out.\n void FutexWaitForReadAvailable(uint32_t Expected) {\n ::syscall(SYS_futex, &Futex, FUTEX_PRIVATE_FLAG | FUTEX_WAIT_BITSET, Expected, nullptr, nullptr, FUTEX_BITSET_WAIT_READERS);\n }\n\n // Read-Lock or Write-lock unlocked, wake one writer.\n // - Read->Write handoff.\n // - Write->Write handoff.\n void FutexWakeWriter() {\n ::syscall(SYS_futex, &Futex, FUTEX_PRIVATE_FLAG | FUTEX_WAKE_BITSET, 1, nullptr, nullptr, FUTEX_BITSET_WAIT_WRITERS);\n }\n\n // Write-lock unlocked, wake read-locks waiting.\n void FutexWakeReaders() {\n // Wake all readers.\n ::syscall(SYS_futex, &Futex, FUTEX_PRIVATE_FLAG | FUTEX_WAKE_BITSET, INT_MAX, nullptr, nullptr, FUTEX_BITSET_WAIT_READERS);\n }\n#else\n // Writers wait for the full 32-bit futex.\n void FutexWaitForWriteAvailable(uint32_t Expected) {\n WaitOnAddress(&Futex, &Expected, sizeof(Futex), INFINITE);\n }\n\n // Readers wait for Futex bits [31:16] to be zero.\n void FutexWaitForReadAvailable(uint32_t Expected) {\n auto ReadWaiterAddress = reinterpret_cast(&Futex) + 2;\n uint16_t smol_Expected = Expected >> 16;\n WaitOnAddress(ReadWaiterAddress, &smol_Expected, sizeof(smol_Expected), INFINITE);\n }\n\n void FutexWakeWriter() {\n WakeByAddressSingle(&Futex);\n }\n\n void FutexWakeReaders() {\n auto ReadWaiterAddress = reinterpret_cast(&Futex) + 2;\n WakeByAddressAll(ReadWaiterAddress);\n }\n#endif\n\n // Reuse the SpinWaitLock WFE implementations for read/write lock acquiring with WFE.\n // Can't reuse the spin-lock directly as some bit-representations are different.\n // WFE-write-lock is less likely to occur the more read-lock threads are participating. Can still occur so good to try.\n // WFE-read-lock is actually quite likely to succeed.\n // Return: true if the lock was acquired.\n bool Attempt_WFE_WriteLock() {\n#ifdef ARCHITECTURE_arm64\n const auto Begin = FEXCore::Utils::SpinWaitLock::GetCycleCounter();\n auto Now = Begin;\n const auto Duration = FEXCore::Utils::SpinWaitLock::CycleCounterFrequency / CYCLECOUNT_DIVISOR;\n\n auto AtomicFutex = std::atomic_ref(Futex);\n uint32_t Expected = AtomicFutex.load(std::memory_order_relaxed);\n\n while ((Now - Begin) < Duration) {\n if (Expected == 0) {\n // Try and grab the owned bit.\n uint32_t Desired = WRITE_OWNED_BIT;\n\n if (AtomicFutex.compare_exchange_strong(Expected, Desired, std::memory_order_acq_rel, std::memory_order_acquire)) {\n return true;\n }\n }\n\n // One-shot attempt to wait for mask to be zero.\n Expected = FEXCore::Utils::SpinWaitLock::OneShotWFEBitComparison(&Futex, ~0U, 0U);\n Now = FEXCore::Utils::SpinWaitLock::GetCycleCounter();\n }\n#endif\n\n return false;\n }\n\n // Return: true if the lock was acquired.\n bool Attempt_WFE_ReadLock() {\n#ifdef ARCHITECTURE_arm64\n // Spin on a WFE for a short-amount of time, waiting for write-owned and writer-count to be zero.\n // - Attempt to acquire read-lock at that point.\n // - Don't add read-waiters bit on failure, return false.\n const auto Begin = FEXCore::Utils::SpinWaitLock::GetCycleCounter();\n auto Now = Begin;\n const auto Duration = FEXCore::Utils::SpinWaitLock::CycleCounterFrequency / CYCLECOUNT_DIVISOR;\n\n auto AtomicFutex = std::atomic_ref(Futex);\n uint32_t Expected = AtomicFutex.load(std::memory_order_relaxed);\n uint32_t Desired {};\n\n while ((Now - Begin) < Duration) {\n if ((Expected & WRITE_OWNED_BIT) == 0 && (Expected & WRITE_WAITER_COUNT_MASK) == 0) {\n // If no write-owner and no write-waiting, try and acquire.\n\n Desired = Expected + READ_OWNER_INCREMENT;\n LOGMAN_THROW_A_FMT((Desired & READ_OWNER_COUNT_MASK) != 0, \"Overflow in read-owners!\");\n if (AtomicFutex.compare_exchange_strong(Expected, Desired, std::memory_order_acq_rel, std::memory_order_acquire)) {\n return true;\n }\n }\n\n // One-shot attempt to wait for mask to be zero.\n Expected = FEXCore::Utils::SpinWaitLock::OneShotWFEBitComparison(&Futex, WRITE_OWNED_BIT | WRITE_WAITER_COUNT_MASK, 0U);\n Now = FEXCore::Utils::SpinWaitLock::GetCycleCounter();\n }\n#endif\n\n return false;\n }\n\n constexpr static uint32_t WRITE_OWNED_BIT = 1U << 31;\n constexpr static uint32_t READ_WAITER_BIT = 1U << 30;\n constexpr static uint32_t WRITE_WAITER_OFFSET = 16;\n constexpr static uint32_t WRITE_WAITER_INCREMENT = 1U << WRITE_WAITER_OFFSET;\n constexpr static uint32_t READ_OWNER_INCREMENT = 1;\n\n // Count masks\n constexpr static uint32_t WRITE_WAITER_COUNT_MASK = 0x3FFFU << WRITE_WAITER_OFFSET;\n constexpr static uint32_t READ_OWNER_COUNT_MASK = 0xFFFFU;\n\n // Independent futex bit-set masks.\n // Wait for readers to drain.\n constexpr static uint32_t FUTEX_BITSET_WAIT_READERS = 1U << 0;\n // Wait for writers to drain.\n constexpr static uint32_t FUTEX_BITSET_WAIT_WRITERS = 1U << 1;\n\n // Only spin on WFE for 0.01ms (10k ns).\n constexpr static uint64_t CYCLECOUNT_DIVISOR = 1'000'000'000ULL / 10'000U;\n\n // Layout:\n // Bits[31]: Write-lock bit.\n // Bits[30]: Read-waiter bit.\n // Bits[29:16]: Write-waiter count.\n // Bits[15:0]: Read-owner count.\n uint32_t Futex {};\n};\n} // namespace FEXCore::Utils::WritePriorityMutex\n" }, { "path": "FEXCore/Source/Utils/variable_length_integer.h", "content": "// SPDX-License-Identifier: MIT\n#pragma once\n\n#include \n\n#include \n#include \n#include \n#include \n\nnamespace FEXCore::Utils {\n// Variable length signed integer\n// The most common encoded size is 8-bit positive, but other values can occur\n//\n// 8-bit:\n// bit[7] = 0 - 8-bit\n// bit[6:0] = 7-bit encoding\n//\n// 16-bit:\n// byte1[7:6] = 0b10 - 16-bit\n// byte1[5:0] = top 6-bits\n// byte2[7:0] = Bottom 8-bits bits\n//\n// 32-bit\n// byte1[7:5] = 0b110 - 32-bit\n// byte1[4:0] = \n// word[31:0] = signed word\n//\n// 64-bit\n// byte1[7:5] = 0b111 - 64-bit\n// byte1[4:0] = \n// dword[63:0] = signed dword\nstruct vl64 final {\n static size_t EncodedSize(int64_t Data) {\n if (Data >= vl8_min && Data <= vl8_max) {\n return sizeof(vl8_enc);\n } else if (Data >= vl16_min && Data <= vl16_max) {\n return sizeof(vl16_enc);\n } else if (Data >= vl32_min && Data <= vl32_max) {\n return sizeof(vl32_enc);\n }\n return sizeof(vl64_enc);\n }\n\n struct Decoded {\n int64_t Integer;\n size_t Size;\n };\n\n static Decoded Decode(const uint8_t* data) {\n auto vl8_type = reinterpret_cast(data);\n auto vl16_type = reinterpret_cast(data);\n auto vl32_type = reinterpret_cast(data);\n auto vl64_type = reinterpret_cast(data);\n\n if (vl8_type->Type == vl8_type_header) {\n return {vl8_type->Integer, sizeof(vl8_enc)};\n } else if (vl16_type->HighBits.Type == vl16_type_header) {\n return {vl16_type->Integer(), sizeof(vl16_enc)};\n } else if (vl32_type->Type == vl32_type_header) {\n return {vl32_type->Integer, sizeof(vl32_enc)};\n }\n return {vl64_type->Integer, sizeof(vl64_enc)};\n }\n\n static size_t Encode(uint8_t* dst, int64_t Data) {\n auto vl8_type = reinterpret_cast(dst);\n auto vl16_type = reinterpret_cast(dst);\n auto vl32_type = reinterpret_cast(dst);\n auto vl64_type = reinterpret_cast(dst);\n\n if (Data >= vl8_min && Data <= vl8_max) {\n *vl8_type = {\n .Integer = static_cast(Data),\n .Type = vl8_type_header,\n };\n return sizeof(vl8_enc);\n } else if (Data >= vl16_min && Data <= vl16_max) {\n *vl16_type = {\n .HighBits {\n .Top = static_cast((Data >> 8) & 0xFF),\n .Type = vl16_type_header,\n },\n .LowBits = static_cast(Data & 0xFF),\n };\n return sizeof(vl16_enc);\n } else if (Data >= vl32_min && Data <= vl32_max) {\n *vl32_type = {\n .Type = vl32_type_header,\n .Integer = static_cast(Data),\n };\n return sizeof(vl32_enc);\n }\n\n *vl64_type = {\n .Type = vl64_type_header,\n .Integer = Data,\n };\n return sizeof(vl64_enc);\n }\n\nprivate:\n\n struct vl8_enc {\n int8_t Integer : 7;\n uint8_t Type : 1;\n };\n static_assert(sizeof(vl8_enc) == 1);\n\n struct vl16_enc {\n struct {\n int8_t Top : 6;\n uint8_t Type : 2;\n } HighBits;\n uint8_t LowBits;\n\n int64_t Integer() const {\n int16_t Value {};\n Value |= (HighBits.Top << 8);\n Value |= LowBits;\n return (Value << 2) >> 2;\n }\n };\n static_assert(sizeof(vl16_enc) == 2);\n\n struct FEX_PACKED vl32_enc {\n uint8_t Type;\n int32_t Integer;\n };\n static_assert(sizeof(vl32_enc) == 5);\n\n struct FEX_PACKED vl64_enc {\n uint8_t Type;\n int64_t Integer;\n };\n static_assert(sizeof(vl64_enc) == 9);\n\n // Maximum ranges for encodings.\n\n // vl8 can hold a signed 7-bit integer.\n // Encoded in one 8-bit value.\n constexpr static int64_t vl8_encoded_bits = 7;\n constexpr static int64_t vl8_type_header = 0;\n constexpr static int64_t vl8_min = std::numeric_limits::min() >> ((sizeof(int64_t) * 8) - vl8_encoded_bits);\n constexpr static int64_t vl8_max = std::numeric_limits::max() >> ((sizeof(int64_t) * 8) - vl8_encoded_bits);\n\n // vl16 can hold a signed 14-bit integer.\n // Encoded in one 16-bit value.\n constexpr static int64_t vl16_encoded_bits = 14;\n constexpr static int64_t vl16_type_header = 0b10;\n constexpr static int64_t vl16_min = std::numeric_limits::min() >> ((sizeof(int64_t) * 8) - vl16_encoded_bits);\n constexpr static int64_t vl16_max = std::numeric_limits::max() >> ((sizeof(int64_t) * 8) - vl16_encoded_bits);\n\n // vl32 can hold a signed 32-bit integer.\n // Encoded in 8-bit and 32-bit value;\n constexpr static int64_t vl32_encoded_bits = 32;\n constexpr static int64_t vl32_type_header = 0b1100'0000;\n constexpr static int64_t vl32_min = std::numeric_limits::min();\n constexpr static int64_t vl32_max = std::numeric_limits::max();\n\n // vl64 can hold a signed 32-bit integer.\n // Encoded in 8-bit and 64-bit value.\n constexpr static int64_t vl64_encoded_bits = 64;\n constexpr static int64_t vl64_type_header = 0b1110'0000;\n constexpr static int64_t vl64_min = std::numeric_limits::min();\n constexpr static int64_t vl64_max = std::numeric_limits::max();\n};\n\n// Variable length pair that optimizes around FEXCore's JITRIPReconstruction.\n//\n// 8-bit:\n// bit[7] = 0 - 8-bit\n// bit[6:4] = 3-bit unsigned - 1. [1 - 8] range.\n// bit[3:0] = 4-bit unsigned divided by 4 - 1. [4 - 64 byte] range.\n//\n// 16-bit:\n// byte1[7:6] = 0b10 - 16-bit\n// byte1[5:0] = 6-bit signed value [-32 - 31] range\n// byte2[7:0] = 8-bit signed value divided by 4. [-512 - 508] byte range.\n//\n// 32-bit and 64-bit don't attempt to do any compression beyond range checks.\n// 32-bit\n// byte1[7:5] = 0b110 - 32-bit\n// byte1[4:0] = \n// word1[31:0] = signed word\n// word2[31:0] = signed word\n//\n// 64-bit\n// byte1[7:5] = 0b111 - 64-bit\n// byte1[4:0] = \n// dword1[63:0] = signed dword\n// dword2[63:0] = signed dword\n\nstruct vl64pair final {\npublic:\n static size_t EncodedSize(uint64_t data_arm, uint64_t data_rip) {\n if (can_encode_vl8(data_arm, data_rip)) {\n return sizeof(vl8_enc);\n } else if (can_encode_vl16(data_arm, data_rip)) {\n return sizeof(vl16_enc);\n } else if (can_encode_vl32(data_arm, data_rip)) {\n return sizeof(vl32_enc);\n }\n return sizeof(vl64_enc);\n }\n\n struct Decoded {\n uint64_t IntegerARMPC;\n uint64_t IntegerX86RIP;\n size_t Size;\n };\n\n static Decoded Decode(const uint8_t* data) {\n auto vl8_type = reinterpret_cast(data);\n auto vl16_type = reinterpret_cast(data);\n auto vl32_type = reinterpret_cast(data);\n auto vl64_type = reinterpret_cast(data);\n\n if (vl8_type->Type == vl8_type_header) {\n return Decode(vl8_type);\n } else if (vl16_type->HighBits.Type == vl16_type_header) {\n return Decode(vl16_type);\n } else if (vl32_type->Type == vl32_type_header) {\n return Decode(vl32_type);\n }\n return {vl64_type->IntegerARMPC, vl64_type->IntegerX86RIP, sizeof(vl64_enc)};\n }\n\n static size_t Encode(uint8_t* dst, uint64_t data_arm, uint64_t data_rip) {\n auto vl8_type = reinterpret_cast(dst);\n auto vl16_type = reinterpret_cast(dst);\n auto vl32_type = reinterpret_cast(dst);\n auto vl64_type = reinterpret_cast(dst);\n\n if (can_encode_vl8(data_arm, data_rip)) {\n *vl8_type = {\n .IntegerARMPC = static_cast((data_arm - 1) >> vl8_arm_align_bits),\n .IntegerX86RIP = static_cast(data_rip - 1),\n .Type = vl8_type_header,\n };\n return sizeof(vl8_enc);\n } else if (can_encode_vl16(data_arm, data_rip)) {\n *vl16_type = {\n .HighBits {\n .IntegerX86RIP = static_cast(static_cast(data_rip)),\n .Type = vl16_type_header,\n },\n .IntegerARMPC = static_cast(static_cast(data_arm) >> vl8_arm_align_bits),\n };\n return sizeof(vl16_enc);\n } else if (can_encode_vl32(data_arm, data_rip)) {\n *vl32_type = {\n .Type = vl32_type_header,\n .IntegerARMPC = static_cast(data_arm),\n .IntegerX86RIP = static_cast(data_rip),\n };\n return sizeof(vl32_enc);\n }\n\n *vl64_type = {\n .Type = vl64_type_header,\n .IntegerARMPC = data_arm,\n .IntegerX86RIP = data_rip,\n };\n return sizeof(vl64_enc);\n }\n\nprivate:\n struct vl8_enc {\n uint8_t IntegerARMPC : 4;\n uint8_t IntegerX86RIP : 3;\n uint8_t Type : 1;\n };\n static_assert(sizeof(vl8_enc) == 1);\n\n static inline Decoded Decode(const vl8_enc* enc) {\n const uint64_t data_arm = enc->IntegerARMPC;\n const uint64_t data_rip = enc->IntegerX86RIP;\n return {(data_arm + 1) << vl8_arm_align_bits, data_rip + 1, sizeof(vl8_enc)};\n }\n\n struct vl16_enc {\n struct {\n int8_t IntegerX86RIP : 6;\n uint8_t Type : 2;\n } HighBits;\n int8_t IntegerARMPC;\n };\n static_assert(sizeof(vl16_enc) == 2);\n\n static inline Decoded Decode(const vl16_enc* enc) {\n int64_t arm_pc = enc->IntegerARMPC << vl8_arm_align_bits;\n int64_t x86_rip = enc->HighBits.IntegerX86RIP;\n return {static_cast(arm_pc), static_cast(x86_rip), sizeof(vl16_enc)};\n }\n\n struct FEX_PACKED vl32_enc {\n uint8_t Type;\n int32_t IntegerARMPC;\n int32_t IntegerX86RIP;\n };\n static_assert(sizeof(vl32_enc) == 9);\n\n static inline Decoded Decode(const vl32_enc* enc) {\n int64_t arm_pc = enc->IntegerARMPC;\n int64_t x86_rip = enc->IntegerX86RIP;\n return {static_cast(arm_pc), static_cast(x86_rip), sizeof(vl32_enc)};\n }\n\n struct FEX_PACKED vl64_enc {\n uint8_t Type;\n uint64_t IntegerARMPC;\n uint64_t IntegerX86RIP;\n };\n static_assert(sizeof(vl64_enc) == 17);\n\n // vl8 can hold a two small unsigned integers.\n // Encoded in 8-bit.\n constexpr static int64_t vl8_type_header = 0;\n constexpr static int64_t vl8_arm_min = 1;\n constexpr static int64_t vl8_arm_max = 16;\n constexpr static int64_t vl8_arm_align_bits = 2;\n constexpr static int64_t vl8_arm_shift_mask = (1U << vl8_arm_align_bits) - 1;\n constexpr static int64_t vl8_pc_min = 1;\n constexpr static int64_t vl8_pc_max = 8;\n static bool can_encode_vl8(uint64_t data_arm, uint64_t data_rip) {\n // GuestPC can only be [1,8] bytes.\n if (data_rip < vl8_pc_min || data_rip > vl8_pc_max) {\n return false;\n }\n // Unaligned doesn't fit at all.\n if (data_arm & vl8_arm_shift_mask) {\n return false;\n }\n\n // HostPC can only be [1,16] instructions.\n int64_t ShiftedHostPC = data_arm >> vl8_arm_align_bits;\n if (ShiftedHostPC < vl8_arm_min || ShiftedHostPC > vl8_arm_max) {\n return false;\n }\n\n return true;\n }\n\n // vl16 can hold a two small signed integers\n // Encoded in one 16-bit value.\n constexpr static int64_t vl16_type_header = 0b10;\n constexpr static int64_t vl16_arm_min = -128;\n constexpr static int64_t vl16_arm_max = 127;\n constexpr static int64_t vl16_arm_align_bits = 2;\n constexpr static int64_t vl16_arm_shift_mask = (1U << vl16_arm_align_bits) - 1;\n constexpr static int64_t vl16_pc_min = -32;\n constexpr static int64_t vl16_pc_max = 31;\n static bool can_encode_vl16(int64_t data_arm, int64_t data_rip) {\n // GuestPC can only be [-32,31] bytes.\n if (data_rip < vl16_pc_min || data_rip > vl16_pc_max) {\n return false;\n }\n\n // Unaligned doesn't fit at all.\n if (data_arm & vl16_arm_shift_mask) {\n return false;\n }\n\n // HostPC can only be [-128,127] instructions.\n int64_t ShiftedHostPC = data_arm >> vl16_arm_align_bits;\n if (ShiftedHostPC < vl16_arm_min || ShiftedHostPC > vl16_arm_max) {\n return false;\n }\n\n return true;\n }\n\n // vl32 can hold a two 32-bit integers.\n // Encoded in 8-bit and two 32-bit values.\n constexpr static int64_t vl32_type_header = 0b1100'0000;\n constexpr static int64_t vl32_min = std::numeric_limits::min();\n constexpr static int64_t vl32_max = std::numeric_limits::max();\n static bool can_encode_vl32(int64_t data_arm, int64_t data_rip) {\n if (data_rip < vl32_min || data_rip > vl32_max) {\n return false;\n }\n if (data_arm < vl32_min || data_arm > vl32_max) {\n return false;\n }\n\n return true;\n }\n\n // vl64 can hold a two 64-bit integers.\n // Encoded in 8-bit and two 64-bit values.\n constexpr static int64_t vl64_type_header = 0b1110'0000;\n};\n\n} // namespace FEXCore::Utils\n" }, { "path": "FEXCore/docs/CPUBackends.md", "content": "# FEXCore CPU Backends\n---\nFEXCore supports multiple CPU emulation backends. All of which ingest the IR that we have been generating.\n\n## IR Interpreter\nThe first one is the easiest. This just walks the IR list and interprets the IR as it goes through it. It isn't meant to be fast and is for debugging purposes.\nThis is used to easily inspect what is going on with the code generation and making sure logic is sound. Will most likely last in to perpetuity since it isn't exactly difficult to maintain and it is useful to have around\n\n## IR JIT\n**Not yet implemented**\nThis is meant to be our first JIT of call and will serve multiple purposes. It'll be the JIT that is used for our runtime compilation of code.\nThis means it needs to be fast during compilation and have decent runtime performance.\nGood chance that we will need to implement multiple of these depending on host architecture with some code reuse between them.\nThis JIT will also be what we use for gathering sampling data for passing off to another JIT for tiered recompilation and offline compilation later.\nShould use xbyak for our x86-64 host and Vixl for our AArch64 host. For other targets in the future we will see what is available\n\n# Future ideas\n---\n* Create an inline ASM or JIT'd dispatcher loop. Will allow our JITs to be more optimal by reserving more registers for guest state.\n* WebAssmembly or other browser language?\n * Might allow decent runtime performance of things emulated in a browser. Could be interesting.\n" }, { "path": "FEXCore/docs/CustomCPUBackend.md", "content": "# FEXCore custom CPU backends\n---\nCustom CPU backends can be useful for testing purposes or wanting to support situations that FEXCore doesn't currently understand.\nThe FEXCore::Context namespace provides a `SetCustomCPUBackendFactory` function for providing a factory function pointer to the core. This function will be used if the `DEFAULTCORE` configuration option is set to `CUSTOM`.\nIf the guest code creates more threads then the CPU factory function will be invoked for creating a CPUBackend per thread. If you don't want a unique CPUBackend object per thread then that needs to be handled by the user.\n\nIt's recommended to store the pointers provided to the factory function for later use.\n`FEXCore::Context::Context*` - Is a pointer to previously generated context object\n`FEXCore::Core::ThreadState*` - Is a pointer to a thread's state. Lives for as long as the guest thread is alive.\nTo use this factory, one must override the provided `FEXCore::CPU::CPUBackend` class with a custom one. This factory function then should return a newly allocated class.\n\n`FEXCore::CPU::CPUBackend::GetName` - Returns an `std::string` for the name of this core\n`FEXCore::CPU::CPUBackend::CompileCode` - Provides the CPUBackend with potentially an IR and DebugData for compiling code. Returns a pointer that needs to be long lasting to a piece of code that will be executed for the particular RIP.\n`FEXCore::CPU::CPUBackend::Initialize` - Called after the guest memory is initialized and all state is ready for the code to start initializing. Gets called just before the CPUBackend starts executing code for the first time.\n" }, { "path": "FEXCore/docs/Frontend.md", "content": "# FEXCore Frontend\n---\nThe FEXCore frontend's job is to translate an incoming x86-64 instruction stream in to a more easily digested version of x86.\nThis effectively expands x86-64 instruction encodings to be more easily ingested later on in the process.\nThis ends up being essential to allowing our IR translation step to be less strenuous. It can decode a \"common\" expanded instruction format rather than various things that x86-supports.\nFor a simple example, x86-64's primary op table has ALU ops that duplicate themselves at least six times with minor differences between each. The frontend is able to decode a large amount of these ops to the \"same\" op that the IR translation understands more readily.\nThis works for most instructions that follow a common decoding scheme, although there are instructions that don't follow the rules and must be handled explicitly elsewhere.\n\nAn example of decoded instructions:\n```\n00 C0: add al,al\n04 01: add al, 0x1\n```\nThese two instructions have a different encoding scheme but they are just an add.\nThey end up decoding to a generic format with the same destination operand but different sources.\nMay look subtle but there end up being far more complex cases and we don't want to handle hundreds of instructions differently.\nAfter the frontend is done decoding the instruction stream, it passes the output over to the OpDispatcher for translating to our IR.\n\n## Multiblock\n---\nThe Frontend has an additional duty. Since it is the main piece of code that understands the guest x86-64 code; It is also what does analysis of control flow to determine if we can end up compiling multiple blocks of guest code.\nThe Frontend already has to determine if it has hit a block ending instruction. This is anything that changes control flow. This feeds in to the analysis system to look at conditional branches to see if we can keep compiling code at the target location in the same functional unit.\n\nShort example:\n```\ntest eax, eax\njne .Continue\nret <--- We can continue past this instruction, which is an unconditional block ender\n.Continue:\n```\n\nThese sorts of patterns crop up extensively in compiled code. A large amount of traditional JITs will end up ending the block at any sort of conditional branch instruction.\nIf the analysis can determine the target conditional branch location, we can then know that the code can keep compiling past an unconditional block ender instruction.\nThis works for both backwards branches and forward branches.\n\n### Additional reading\n---\nThere are other emulators out there that implement multiblock JIT compilation with some success.\nThe best example of this that I know of is the [Dolphin GameCube and Wii Emulator](https://github.com/dolphin-emu/dolphin) Where I implemented the initial multiblock implementation.\nOne of the major limitations with a console emulator is that you can run in to infinite loops on backedges when using multiblock compilation. This is due to console emulation being able to run an infinite loop and let Interrupts or some other state cause it to break out.\nLuckily since we are a userspace emulator we don't have to deal with this problem. If an application has written an infinite loop, then without another thread running, it'll be a true infinite loop.\nAdditionally luckily is that we are going to emulate the strong memory model of x86-64 and also support true threads, this will mean that we don't need to do any manual thread scheduling in our emulator and switch between virtual threads.\n\n" }, { "path": "FEXCore/docs/IR.md", "content": "# FEXCore IR\n---\nThe IR for the FEXCore is an SSA based IR that is generated from the incoming x86-64 assembly.\nSSA is quite nice to work with when translating the x86-64 code to the IR, when optimizing that code with custom optimization passes, and also passing that IR to our CPU backends.\n\n## Emulation IR considerations\n* We have explicitly sized IR variables\n * Supports traditional element sizes of 1,2,4,8 bytes and some 16byte ops\n * Supports arbitrary number of vector elements\n * The op determines if something is float or integer based.\n* Clear separation of scalar IR ops and vector IR ops\n * ex, MUL versus VMUL\n* We have explicit Load/Store context IR ops\n * This allows us to have a clear separation between guest memory and tracked x86-64 state\n* We have an explicit CPUID IR op\n * This allows us to return fairly complex data (4 registers of data) and also having an easier optimization for constant CPUID functions\n * So if we const-prop the CPUID function then it'll just const-prop further along\n* We have an explicit syscall op\n * The syscall op is fairly complex as well, same with CPUID that if the syscall function is const-prop then we can directly call the syscall handler\n * Can save overhead by removing call overheads\n* The IR supports branching from one block to another\n * Has a conditional branch instruction that either branches to the target branch or falls through to the next block\n * Has an unconditional branch to explicitly jump to a block instead of falling through\n * **There is a desire to follow LLVM semantics around block limitations but it isn't currently strictly enforced**\n* Supports a debug `Print` Op for printing out values for debug viewing\n* Supports explicit Load/Store memory IR ops\n * This is for accessing guest memory and will do the memory offset translation in to the VM's memory space\n * This is done by just adding the VM memory base to the 64bit address passed in\n * This is done in a manner that the application **can** escape from the VM and isn't meant to be safe\n * There is an option for JITs to validate the memory region prior to accessing for ensuring correctness\n* IR is generated from a JSON file, fairly straightforward to extend.\n * Read the python generation file to determine the extent of what it can do\n\n## IR function considerations\nThe first SSA node is a special case node that is considered invalid. This means %0 will always be invalid for \"null\" node checks\nThe first real SSA node also has to be a IRHeader node. This means it is safe to assume that %1 will always be an IRHeader.\n\n\n```(%%1) IRHeader 0x41a9a0, %%2, 5```\n\nThe header provides information about that function like the entry point address.\nAdditionally it also points to the first `CodeBlock` IROp\n\n\n```(%%2) CodeBlock %%7, %%168, %%3```\n\n\n* The `CodeBlock` Op is a jump target and must be treated as if it'll be jumped to from other blocks\n * It contains pointers to the starting op and ending op and they are inclusive\n * It also contains a pointer to the next CodeBlock in a singly linked list\n * The last CodeBlock will point to the InvalidNode as the next block\n\n\n### Example code block\n\n```\n(%%3) CodeBlock %%169, %%173, %%4\n\t(%%169) BeginBlock %3\n\t%170 i64 = Constant 0x41a9e1\n\t(%%171) StoreContext %170 i64, 0x8, 0x0\n\t(%%172) ExitFunction\n\t(%%173) EndBlock %3\n```\n\n* BeginBlock points back to the CodeBlock SSA which helps with iterating across multiple blocks\n* EndBlock the ending op of a CodeBlock and also points back to the CodeBlock SSA.\n* ExitFunction will leave the function immediately and return back to the dispatcher\n* Every IR Op has an SSA value associated with it used for tracking the op itself\n\t* If the IROp doesn't have a real destination then it is invalid to use it as an argument in most other ops\n\n## In-memory representation\n\nThe in-memory representation of the IR may be a bit confusing when initially viewed and once dealing with optimizations then it may be confusing as well.\nCurrently the IR Generation is tied to the `OpDispatchBuilder` class. This class handles translating decoded x86 to our IR representation.\nWhen generating IR inside of the `OpDispatchBuilder` it is straight forward, just call the IR generation ops.\n\n### FEXCore::IR::IntrusiveAllocator\nThis is an intrusive allocator that is used by the `OpDispatchBuilder` for storing IR data. It is a simple linear arena allocator without resizing capabilities.\n\n### OpDispatchBuilder\nOpDispatchBuilder provides `IRListView ViewIR()` for handling the IR outside of the class:\n* Returns a wrapper container class the allows you to view the IR. This doesn't take ownership of the IR data.\n* If the OpDispatcherBuilder changes its IR then changes are also visible to this class\n\nThis class uses two IntrusiveAllocator objects for tracking IR data. `ListData` and `Data` are the object names.\n* `ListData` is for tracking the doubly linked list of nodes\n\t* This ONLY allocates `FEXCore::IR::OrderedNode` objects\n\t* When an OrderedNode is allocated its allocation location (NodeOffset) is just the offset from the base pointer\n\t* This allows us to only use uint32_t memory offsets to compact the IR\n\t* Additionally using offsets allows us the freedom to freely move our IR in memory without costly pointer adjustment\n\t* This means everything is fixed size allocated (SSA Node number calculation is just `AllocationOffset / sizeof(OrderedNode)`\n\t* OrderedNodes are what the SSA arguments are pointing to in the end\n\n\n### OrderedNode\nThis is a doubly linked list of all of our IR nodes. This allows us to walk forward or backward over the IR and they must be ordered correctly to ensure dominance of SSA values.\n* Contains `OrderedNodeHeader`\n\t* Contains `OpNodeWrapper Value`\n\t\t* Points to the `IROp_Header` backing op for this SSA node\n\t* Contains `OrderedNodeWrapper Next`\n\t\t* Points to the next `OrderedNode`\n\t* Contains `OrderedNodeWrapper Previous`\n\t\t* Points to the previous `OrderedNode`\n* Contains the NumUses\n\t* This allows us to easily walk to the list backwards and DCE the ops that have NumUses == 0\n* `IROp_Header *Op(uintptr_t Base)`\n\t* Allows you to get the backing IR data for this SSA value\n\n### NodeWrapperBase - Type for `OrderedNodeHeader` and `OpNodeWrapper`\n* `using OpNodeWrapper = NodeWrapperBase`\n* `using OrderedNodeWrapper = NodeWrapperBase`\n* This is a class to let you more easily convert NodeOffsets in to their real backing pointer\n* `GetNode(uintptr_t Base)` allows you to pass in the base pointer from the backing Intrusive allocator and get the object\n\t* **This can be confusing**\n\t* A good rule of thumb is to only ever use `GetNode(ListDataBegin)` with OrderedNodeWrapper\n\t* Then once you have the `OrderedNode*` from GetNode, Use the `Op(IRDataBegin)` function to get the IR data.\n\t* I do **NOT** recommend using `GetNode` directly from `OpNodeWrapper` as it is VERY easy to mess it up\n\n### NodeIterator\nProvides a fairly straightforward interface that allows easily walking the IR nodes with C++ increment and decrement operations.\nOnly iterates over a single block\n\n#### Example usage\n```cpp\n\tIR::NodeIterator After = ...;\n\tIR::NodeIterator End = ...;\n\n\twhile (After != End) {\n\t\t// NodeIterator() returns a pair of pointers to the OrderedNode and IROp data\n\t\t// You can unpack the result with structured bindings\n\t\tauto [CodeNode, IROp] = After();\n\n\t\t// IROp_Header contains a bunch of information about the IR object\n\t\t// We can convert it with the object's C or CW functions\n\n\t\tswitch(IROp->Op) {\n\t\t\tcase IR::OP_ADD: {\n\t\t\t\tFEXCore::IR::IROp_Add const *Op = IROp->C();\n\t\t\t\t/* We can now access members inside of IROp_Add that were previously unavailable\n\t\t\t\t\t You can still access the header definitions from Op->Header */\n\t\t\t\tbreak;\n\t\t\t}\n\t\t\t/* ... */\n\t\t}\n\t\t// Go to the next IR Op\n\t\t++After;\n\t}\n\n```\n\n### AllNodesIterator\nThis is like NodeIterator, except that it will cross block boundaries.\n\n### IRListView.GetBlocks()\nProvides a range for easy iterating over all the blocks in a multi-block with NodeIterator\n\n#### Example usage\n```c++\n\tfor (auto [BlockNode, BlockHeader] : CurrentIR.GetBlocks()) {\n\t\t// Do stuff for each block\n\t}\n```\n\n### IRListView.GetCode(BlockNode)\nProvides a range for easy iterating over all the code in a block\n\n#### Example usage\n```c++\n\tfor (auto [CodeNode, IROp] : CurrentIR.GetCode(BlockNode)) {\n\t\t// Do stuff for each op\n\n\t\tswitch(IROp->Op) {\n\t\t\tcase IR::OP_ADD: {\n\t\t\t\tFEXCore::IR::IROp_Add const *Op = IROp->C();\n\t\t\t\t// Do stuff for each Add op.\n\n\t\t\t\tbreak;\n\t\t\t}\n\t\t}\n\t}\n```\n\n### IRListView.GetAllCode()\nLike GetCode, except it uses AllNodesIterator to allow easy iterating over every single op in the entire Multiblock\n\n#### Example usage\n```c++\n\tfor (auto [CodeNode, IROp] : CurrentIR.GetAllCode()) {\n\t\t// Do stuff for each op\n\t}\n```\n\n## JSON file\nAn example of what the IR json looks like\n```\n\"StoreContext\": {\n \"SSAArgs\": \"1\",\n \"Args\": [\n \"uint8_t\", \"Size\",\n \"uint32_t\", \"Offset\"\n ]\n},\n```\nThe json entry name will be the name of the IR op and the dispatcher function.\nThis means you'll get a `_Add(...)` dispatcher function generated\n\n### JSON IR element options\n* `HasDest`\n * This is used on ops that return a value. Used for tracking of if ops return data\n* `SSAArgs`\n * These are the number of arguments that the op consumes that are SSA based\n * Needs to come from previous ops that had a destination\n* `SSANames`\n * Allows you to name the SSA arguments in an op\n * Otherwise the Op names will only be able to be accessed from the Header of the IR through its arguments array\n* `Args`\n * These are defined arguments that are stored in the IR encoding that aren't SSA based\n * Useful for things that are constant encoded and won't change after the fact\n* `FixedDestSize`\n * This allows you to override the op's destination size in bytes\n * Most ops with implicitly calculate their destination size through the maximum sizes of the IR arguments passed in\n* `DestSize`\n * This allows an IR size override that isn't just a size in bytes\n * This can let the size of the op be another argument or something more extensive\n* `RAOverride`\n * This allows an op to take regular SSA arguments (So optimization passes will still be aware of them) but also not have them be register allocated\n * Useful for block handling ops, where blocks aren't something that get register allocated but still need to have their uses tracked\n* `HelperGen`\n * If there is a complex IR Op that needs to be defined but you don't want an automatic dispatcher generated then this disables the generation of the\n dispatcher\n* `Last`\n * This is a special element only used for the last element in the list\n" }, { "path": "FEXCore/docs/MemoryModelEmulation.md", "content": "# What is x86-TSO and what is different compared to ARM's weak memory model?\nx86's memory model is a very strictly coherent memory model that effectively mandates that all memory accesses are \"atomic\". While atomicity is\nactually a bit more strict, we actually need to emulate it in ARM using atomic instructions. We are also required to emulate this strictness with\nunaligned accesses, which is due to x86 CPUs allowing unaligned atomics for \"free\" within a cacheline. Intel also takes this a step more and allowing\nfull atomics with a feature called \"split-locks\", AMD gains this same feature in Zen 5.\n\n# Emulating loads\nDue to x86 SIB addressing, this can happen on most instructions. FEX emulates these in a variety of ways depending on features.\nMost instructions are emulated with an atomic instruction but we also implement a feature called \"half-barrier\" atomics for unaligned atomics.\n\n## Base ARMv8.0\n- Addressing limitations\n - Register only\nThis is emulated using an atomic load instruction plus a nop.\n- On unaligned access the code gets backpatched to a non-atomic load plus a memory barrier\n\n## FEAT_LRCPC\n- Addressing limitations\n - Register only\nThis matches the base ARMv8.0 implementation but adds new instructions that match x86-TSO behaviour, making the emulation slightly quicker.\n- On unaligned access it still gets backpatched to non-atomic load plus a memory barrier.\n\n## FEAT_LRCPC2\n- Addressing limitations\n - Register plus 9-bit signed immediate (-256, 255)\nAdds some new instructions that allow immediate encoding inside of the previous LRCPC instructions\n\n## FEAT_LRCPC3\nAdds a handful of GPR instructions that aren't super interesting\n\nFEX doesn't currently implement these since no hardware supports it.\n\n- ldapr - Post-index load for stack\n- ldiapr - Post-index load pair for stack\n- stilp - pre-index store pair for stack\n- stlr - pre-index store for stack\n\n# Emulating stores\nAgain due to x86 SIB addressing, this can also happen on most instructions. There are less options for FEX with this extension, so in most cases this\njust turns in to an atomic store with half-barrier backpatching for unaligned accesses\n\n## FEAT_LRCPC, FEAT_LRCPC2\nAdds nothing for emulating stores\n\n## FEAT_LRCPC3\n\n# Emulating atomic instructions\nx86 has atomic memory operations that can do a variety of operations. For unaligned atomic operations FEX will emulate the operation inside the signal\nhandler if it happens to be unaligned.\n\n## CASPair - cmpxchg\n\n## Base ARMv8.0\n- Addressing limitations\n - Register only\nThis is emulated with a ldaxp+stlxp pair of instructions.\n\n## FEAT_LSE\n- Addressing limitations\n - Register only\nAdds a new caspal instruction that does the operation almost exactly like x86.\n\n## CAS - cmpxchg8b/cmpxchg16b\n\n## Base ARMv8.0\n- Addressing limitations\n - Register only\nSimilar to CASPair but now only uses a ldaxr+stlxr pair\n\n## FEAT_LSE\n- Addressing limitations\n - Register only\nSimilar to CASPair adds a new casal instruction that operates basically like x86\n\n# AtomicFetch\n## Op from the following list\n- Add\n- Sub\n- And\n- CLR\n- Or\n- Xor\n- Neg\n- Swap\n\n## Base ARMv8.0\n- Addressing limitations\n - Register only\nAll operations get emulated with an ldaxr+stlxr+ instruction\n\n## FEAT_LSE\n- Addressing limitations\n - Register only\nAlmost all operations now have a native atomic memory operation instruction. The only outlier is atomicNeg which doesn't have an LSE equivalent and\nuses the ARMv8.0 implementation.\n\n# Vector loads\nSince almost all memory accesses on x86 are TSO, this includes vector operations.\n\n## Base ARMv8.0\n- Addressing limitations\n - Register plus 9-bit signed immediate (-256, 255)\n - Register plus 12-bit unsigned scaled immediate (Scaled by access size)\nEmulated using half-barriers, which means a load+dmb\n\n## FEAT_LRCPC3\n- LDAP1 added for element loads. Register only address encoding\n- LDAPUR added for vector register loads, supports 9-bit simm offset\n\n# Vector stores\nJust like loads, these are emulated using half-barriers\n\n## Base ARMv8.0\n- Addressing limitations\n - Register plus 9-bit signed immediate (-256, 255)\n - Register plus 12-bit unsigned scaled immediate (Scaled by access size)\nEmulated using half-barriers, which means a dmb+str\n\n\n## FEAT_LRCPC3\n- STL1 added for element stores. Register only address encoding\n- STLUR added for vector register stores, supports 9-bit simm offset\n\n# Addressing limitations depending on operating mode\n## GPR loadstores\n### TSO Emulation disabled\n- Register only (ldr/str)\n- Register + Register + scale (ldr/str)\n- Register + 9-bit simm (ldur/stru)\n- Register + 12-bit unsigned scaled imm (ldr/str)\n\n### TSO Emulation enabled\n- Register only (ldar/stlr)\n- Register only (ldapr/stlr) - FEAT_LRCPC\n- Register + 9-bit simm (ldapr/stlur) - FEAT_LRCPC2\n\n## Vector loadstores\n### TSO Emulation disabled\n- Register only (ldr/str)\n- Register + Register + scale (ldr/str)\n- Register + 9-bit simm (ldur/stru)\n- Register + 12-bit unsigned scaled imm (ldr/str)\n\n### TSO Emulation enabled\n- Same as TSO emulation disabled due to half-barrier implementation\n\n### TSO Emulation enabled (FEAT_LRCPC3)\n- Register only (ldap1/stl1) - Element loadstore\n- Register + 9-bit simm (ldapur/stlur)\n\n## Atomic memory operations\nAlways TSO emulation enabled, always register only.\n" }, { "path": "FEXCore/docs/OpDispatcher.md", "content": "# FEXCore OpDispatcher\n---\nThe OpDispatcher is the step of the recompiler that takes the output from the Frontend and translates it to our IR.\nSince the x86-64 instruction set is so large (>1000 instructions in the current FEXCore tables) we need to reduce this down to something more manageable.\nWe will ingest our decoded x86-64 instructions and translate them down to more basic IR operations. The number of IR ops are currently in the dozens which is a lot easier to handle.\nOnce we have translated to the IR then we need to pass the IR over to optimization passes or our JIT cores.\n\nEx:\n```\n mov rax,0x1\n mov rdi,0x1\n mov rsi,0x20\n mov rdx,0x1\n syscall \n hlt\n ```\n Translates to the IR of:\n ```\nBeginBlock\n %8 i32 = Constant 0x1\n StoreContext 0x8, 0x8, %8\n %64 i32 = Constant 0x1\n StoreContext 0x8, 0x30, %64\n %120 i32 = Constant 0x1f\n StoreContext 0x8, 0x28, %120\n %176 i32 = Constant 0x1\n StoreContext 0x8, 0x20, %176\n %232 i64 = LoadContext 0x8, 0x8\n %264 i64 = LoadContext 0x8, 0x30\n %296 i64 = LoadContext 0x8, 0x28\n %328 i64 = LoadContext 0x8, 0x20\n %360 i64 = LoadContext 0x8, 0x58\n %392 i64 = LoadContext 0x8, 0x48\n %424 i64 = LoadContext 0x8, 0x50\n %456 i64 = Syscall %232, %264, %296, %328, %360, %392, %424\n StoreContext 0x8, 0x8, %456\n BeginBlock\n EndBlock 0x1e\n ExitFunction\n```\n### Multiblock\n---\nAn additional duty of the OpDispatcher is to handle the metadata that the Frontend provides for supporting multiblock.\nThe IR provides most of the functionality required for supporting robust branching and function creation required for generating large blocks of code translated from x86-64 emulation.\nThis is required since in the ideal situation we will be doing function level translation of x86-64 guest code to our IR.\nThe IR is currently lacking any idea of flags or PHI nodes, which can be problematic when optimizing branch heavy code. The good thing is that the LLVM JIT can use a mem to reg pass to minimize a large number of this code.\nIt **will** be required to improve the IR further once the runtime JIT becomes a higher priority\n" }, { "path": "FEXCore/include/FEXCore/Config/Config.h", "content": "// SPDX-License-Identifier: MIT\n#pragma once\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\nnamespace FEXCore::Config {\nnamespace Handler {\n static inline std::optional SMCCheckHandler(std::string_view Value) {\n if (Value == \"none\") {\n return \"0\";\n } else if (Value == \"mtrack\") {\n return \"1\";\n } else if (Value == \"full\") {\n return \"2\";\n }\n return \"0\";\n }\n} // namespace Handler\n\nenum ConfigOption {\n#define OPT_BASE(type, group, enum, json, default) CONFIG_##enum,\n#include \n};\n\n#define ENUMDEFINES\n#include \n\nenum ConfigSMCChecks {\n CONFIG_SMC_NONE,\n CONFIG_SMC_MTRACK,\n CONFIG_SMC_FULL,\n};\n\nenum class LayerType {\n LAYER_GLOBAL_MAIN, ///< /usr/share/fex-emu/Config.json by default\n LAYER_MAIN,\n LAYER_ARGUMENTS,\n LAYER_GLOBAL_STEAM_APP,\n LAYER_GLOBAL_APP,\n LAYER_LOCAL_STEAM_APP,\n LAYER_LOCAL_APP,\n LAYER_USER_OVERRIDE,\n LAYER_ENVIRONMENT,\n LAYER_TOP,\n};\n\ntemplate\nstatic inline std::optional EnumParser(const ArrayPairType& EnumPairs, const std::string_view View) {\n uint64_t EnumMask {};\n auto Results = std::from_chars(View.data(), View.data() + View.size(), EnumMask);\n if (Results.ec == std::errc()) {\n // If the data is a valid number, just pass it through.\n return std::nullopt;\n }\n\n auto Begin = 0;\n auto End = View.find_first_of(',');\n std::string_view Option = View.substr(Begin, End);\n while (Option.size() != 0) {\n auto EnumValue =\n std::find_if(EnumPairs.begin(), EnumPairs.end(), [Option](const PairTypes& Value) -> bool { return Value.first == Option; });\n\n if (EnumValue == EnumPairs.end()) {\n LogMan::Msg::IFmt(\"Skipping Unknown option: {}\", Option);\n } else {\n EnumMask |= FEXCore::ToUnderlying(EnumValue->second);\n }\n\n if (End == std::string::npos) {\n break;\n }\n Begin = End + 1;\n End = View.find_first_of(',', Begin);\n Option = View.substr(Begin, End - Begin);\n }\n\n return fextl::fmt::format(\"{}\", EnumMask);\n}\n\nusing StringArrayType = fextl::list;\n\nnamespace detail {\n template\n struct ConfigOptionInfo;\n#define DEFINE_METAINFO(type, enum, default) \\\n template<> \\\n struct ConfigOptionInfo { \\\n using Type = type; \\\n static auto Default() { \\\n extern default; \\\n return enum; \\\n } \\\n };\n#define OPT_BASE(type, group, enum, json, default) DEFINE_METAINFO(type, enum, const type enum)\n#define OPT_STR(group, enum, json, default) DEFINE_METAINFO(fextl::string, enum, const std::string_view enum)\n#define OPT_STRARRAY(group, enum, json, default) DEFINE_METAINFO(StringArrayType, enum, const std::string_view enum)\n#include \n} // namespace detail\n\nFEX_DEFAULT_VISIBILITY void SetDataDirectory(std::string_view Path, bool Global);\nFEX_DEFAULT_VISIBILITY void SetConfigDirectory(const std::string_view Path, bool Global);\nFEX_DEFAULT_VISIBILITY void SetConfigFileLocation(std::string_view Path, bool Global);\n\nFEX_DEFAULT_VISIBILITY const fextl::string& GetDataDirectory(bool Global = false);\nFEX_DEFAULT_VISIBILITY const fextl::string& GetConfigDirectory(bool Global);\nFEX_DEFAULT_VISIBILITY const fextl::string& GetConfigFileLocation(bool Global = false);\nFEX_DEFAULT_VISIBILITY fextl::string GetApplicationConfig(const std::string_view Program, bool Global);\n\nusing LayerValue = std::variant< fextl::string, StringArrayType, uint8_t, int8_t, uint16_t, int16_t, uint32_t, int32_t, uint64_t, int64_t, bool >;\n\nusing LayerOptions = fextl::unordered_map;\n\nclass FEX_DEFAULT_VISIBILITY Layer {\npublic:\n explicit Layer(const LayerType _Type);\n virtual ~Layer();\n\n virtual void Load() = 0;\n\n bool OptionExists(ConfigOption Option) const {\n return OptionMap.find(Option) != OptionMap.end();\n }\n\n std::optional All(ConfigOption Option) {\n const auto it = OptionMap.find(Option);\n if (it == OptionMap.end()) {\n return std::nullopt;\n }\n\n auto& Value = it->second;\n LOGMAN_THROW_A_FMT(std::holds_alternative(Value), \"Tried to get config of invalid type!\");\n\n return &std::get(Value);\n }\n\n std::optional Get(ConfigOption Option) {\n const auto it = OptionMap.find(Option);\n if (it == OptionMap.end()) {\n return std::nullopt;\n }\n\n auto& Value = it->second;\n LOGMAN_THROW_A_FMT(std::holds_alternative(Value), \"Tried to get config of invalid type!\");\n\n return &std::get(Value);\n }\n\n // Set will overwrite the object with a fextl::string without tests.\n void Set(ConfigOption Option, const char* Data) {\n LOGMAN_THROW_A_FMT(Data != nullptr, \"Data can't be null\");\n OptionMap[Option].emplace(fextl::string(Data));\n }\n\n void Set(ConfigOption Option, std::string_view Data) {\n OptionMap[Option].emplace(fextl::string(Data));\n }\n\n void Set(ConfigOption Option, fextl::string Data) {\n OptionMap[Option].emplace(std::move(Data));\n }\n\n void Set(ConfigOption Option, std::optional Data) {\n if (Data) {\n OptionMap[Option].emplace(std::move(*Data));\n }\n }\n\n // AppendStrArrayValue will append strings to its StringArrayType.\n // If the value was previously a different type, then throw an assert.\n void AppendStrArrayValue(ConfigOption Option, std::string_view Data) {\n auto it = OptionMap.find(Option);\n if (it == OptionMap.end()) {\n // If the option didn't exist as a StringArrayType yet, emplace it.\n it = OptionMap.emplace(Option, StringArrayType {}).first;\n }\n\n auto& Value = it->second;\n LOGMAN_THROW_A_FMT(std::holds_alternative(Value), \"Tried to get config of invalid type!\");\n std::get(Value).emplace_back(Data);\n }\n\n void Erase(ConfigOption Option) {\n OptionMap.erase(Option);\n }\n\n LayerType GetLayerType() const {\n return Type;\n }\n const LayerOptions& GetOptionMap() const {\n return OptionMap;\n }\n\nprotected:\n const LayerType Type;\n LayerOptions OptionMap;\n};\n\nFEX_DEFAULT_VISIBILITY void Initialize();\nFEX_DEFAULT_VISIBILITY void Shutdown();\n\nFEX_DEFAULT_VISIBILITY void Load();\nFEX_DEFAULT_VISIBILITY void ReloadMetaLayer();\nFEX_DEFAULT_VISIBILITY fextl::string FindContainer();\nFEX_DEFAULT_VISIBILITY fextl::string FindContainerPrefix();\n\nFEX_DEFAULT_VISIBILITY void AddLayer(fextl::unique_ptr _Layer);\n\nFEX_DEFAULT_VISIBILITY bool Exists(ConfigOption Option);\nFEX_DEFAULT_VISIBILITY std::optional All(ConfigOption Option);\ntemplate\nFEX_DEFAULT_VISIBILITY std::optional GetConv(ConfigOption Option);\nFEX_DEFAULT_VISIBILITY std::optional Get(ConfigOption Option);\nFEX_DEFAULT_VISIBILITY void Set(ConfigOption Option, std::string_view Data);\nFEX_DEFAULT_VISIBILITY void Erase(ConfigOption Option);\n\ntemplate\nclass FEX_DEFAULT_VISIBILITY Value {\npublic:\n // Single value type.\n template\n requires (std::is_fundamental_v || std::is_same_v)\n Value(FEXCore::Config::ConfigOption Option, TT Default) {\n ValueData = GetIfExists(Option, Default);\n }\n\n template\n requires (std::is_fundamental_v || std::is_same_v)\n Value(FEXCore::Config::ConfigOption Option, std::string_view Default) {\n ValueData = GetIfExists(Option, Default);\n }\n\n operator T() const {\n return ValueData;\n }\n\n T operator()() const requires (std::is_fundamental_v)\n {\n return ValueData;\n }\n\n const fextl::string& operator()() const requires (std::is_same_v)\n {\n return ValueData;\n }\n\n Value(T Value) requires (!std::is_same_v)\n {\n ValueData = std::move(Value);\n }\n\n // Array value types.\n Value(FEXCore::Config::ConfigOption Option, std::string_view) requires (std::is_same_v)\n {\n GetListIfExists(Option, &ValueData);\n }\n\n StringArrayType& All() requires (std::is_same_v)\n {\n return ValueData;\n }\n\nprivate:\n T ValueData {};\n\n static T GetIfExists(FEXCore::Config::ConfigOption Option, T Default);\n static T GetIfExists(FEXCore::Config::ConfigOption Option, std::string_view Default);\n\n static void GetListIfExists(FEXCore::Config::ConfigOption Option, StringArrayType* List);\n};\n\n/**\n * Wrapper around Value that automatically picks the default for the given ConfigOption\n */\ntemplate\nstruct FEX_DEFAULT_VISIBILITY Getter : public Value::Type> {\n using OptionInfo = detail::ConfigOptionInfo