Repository: YuAo/MetalLibraryArchive Branch: master Commit: de573ba4a7b9 Files: 26 Total size: 107.3 KB Directory structure: gitextract_009183w_/ ├── .github/ │ └── workflows/ │ └── swift.yml ├── .gitignore ├── LICENSE ├── Package.resolved ├── Package.swift ├── README.md ├── Sources/ │ ├── Explorer/ │ │ ├── AppIconView.swift │ │ ├── MetalLibraryAchiveUnpacker.swift │ │ ├── MetalLibraryArchiveDocument.swift │ │ ├── MetalLibraryArchiveView.swift │ │ ├── Toast.swift │ │ └── main.swift │ └── MetalLibraryArchive/ │ ├── Archive.swift │ ├── DeploymentTarget.swift │ ├── Function.swift │ ├── LanguageVersion.swift │ ├── LibraryType.swift │ ├── MetalDataType.swift │ ├── Platform.swift │ ├── SourceArchive.swift │ └── Tag.swift ├── Tests/ │ └── MetalLibraryArchiveTests/ │ └── Tests.swift └── Utilities/ ├── Package.resolved ├── Package.swift └── Sources/ └── MetalDataTypeTools/ ├── Command.swift └── MetalDataTypes.swift ================================================ FILE CONTENTS ================================================ ================================================ FILE: .github/workflows/swift.yml ================================================ name: Swift on: push: branches: [ master ] pull_request: branches: [ master ] jobs: build: runs-on: macos-12 steps: - uses: actions/checkout@v3 - name: Build run: swift build -v - name: Run tests run: swift test -v ================================================ FILE: .gitignore ================================================ .docc-build/ .swiftpm/ .DS_Store .build /Packages /*.xcodeproj xcuserdata/ ================================================ FILE: LICENSE ================================================ MIT License Copyright (c) 2022 Yu Ao Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: Package.resolved ================================================ { "object": { "pins": [ { "package": "swift-crypto", "repositoryURL": "https://github.com/apple/swift-crypto.git", "state": { "branch": null, "revision": "a8911e0fadc25aef1071d582355bd1037a176060", "version": "2.0.4" } } ] }, "version": 1 } ================================================ FILE: Package.swift ================================================ // swift-tools-version:5.5 // The swift-tools-version declares the minimum version of Swift required to build this package. import PackageDescription import Foundation let isUsingSwiftWASMToolchain = (ProcessInfo.processInfo.environment["TOOLCHAINS"] == "swiftwasm") let packageDependencies: [Package.Dependency] if isUsingSwiftWASMToolchain { packageDependencies = [] } else { packageDependencies = [.package(url: "https://github.com/apple/swift-crypto.git", "1.0.0" ..< "3.0.0")] } let metalLibraryArchiveTargetDependencies: [Target.Dependency] if isUsingSwiftWASMToolchain { metalLibraryArchiveTargetDependencies = [] } else { metalLibraryArchiveTargetDependencies = [.product(name: "Crypto", package: "swift-crypto")] } let package = Package( name: "MetalLibraryArchive", platforms: [ .macOS(.v10_15), .iOS(.v13), .watchOS(.v6), .tvOS(.v13), .macCatalyst(.v13) ], products: [ .library( name: "MetalLibraryArchive", targets: ["MetalLibraryArchive"]), ], dependencies: packageDependencies, targets: [ .executableTarget(name: "Explorer", dependencies: [ "MetalLibraryArchive" ]), .target(name: "MetalLibraryArchive", dependencies: metalLibraryArchiveTargetDependencies), .testTarget( name: "MetalLibraryArchiveTests", dependencies: [ "MetalLibraryArchive" ]), ] ) ================================================ FILE: README.md ================================================ # Metal Library Archive ![](https://github.com/YuAo/MetalLibraryArchive/workflows/Swift/badge.svg) `MetalLibraryArchive` is a product of reverse-engineering Apple's `metallib` file format. You can use `MetalLibraryArchive` to get the library type, target platform, Metal functions, etc., from a `metallib` file. The extracted information of a Metal function includes: - Function name. - Function type - vertex, fragment, kernel, extern, etc. - Metal Shading Language version of the function. - Bitcode of the function which can be converted into human-readable LLVM assembly language using [llvm-dis](https://llvm.org/docs/CommandGuide/llvm-dis.html). - Source code of the function if the `metallib` is configured to include source code. ## 🎈 Usage ### Web App Available at: https://yuao.github.io/MetalLibraryExplorer [Learn more](https://github.com/YuAo/MetalLibraryExplorer) ### Explorer App An executable target called "Explorer" is included in the package. "Explorer" is a GUI app that can open, unpack and disassemble (with the help of `llvm-dis`) `metallib` files. **Note** `llvm-dis` is not included, you can get a copy of the binary at https://github.com/llvm/llvm-project/releases Use the "Disassembler" menu in the app to locate the `llvm-dis` executable file. ![Screenshot](Assets/Explorer-Screenshot.png) ### Library You can also use `MetalLibraryArchive` as a library: ```Swift import MetalLibraryArchive let archive = try Archive(data: Data(contentsOf: metallibURL)) let libraryType = archive.libraryType let functions = archive.functions ``` ## 🚧 Metal Library Archive Binary Layout ### Header | Byte Range | Type | Content | |------------|--------|-------------------------------| | 0...3 | FourCharCode | MTLB | | 4...5 | UInt16 | Target platform | | 6...9 | (UInt16, UInt16) | Version of the metallib file (major, minor) | | 10 | UInt8 | Type of the metallib file | | 11 | UInt8 | Target OS | | 12...15 | (UInt16, UInt16) | Version of the target OS (major, minor) | | 16...23 | UInt64 | Size of the metallib file | | 24...39 | (UInt64, UInt64) | Offset and size of the function list | | 40...55 | (UInt64, UInt64) | Offset and size of the public metadata section | | 56...71 | (UInt64, UInt64) | Offset and size of the private metadata section | | 72...87 | (UInt64, UInt64) | Offset and size of the bitcode section | | Target Platform | Value | |----|-------| | macOS | 0x8001 (0x01,0x80)| | iOS | 0x0001 (0x01,0x00) | | metallib Type | Value | |----|-------| | Executable | 0x00 | | Core Image | 0x01 | | Dynamic | 0x02 | | Symbol Companion | 0x03 | | Target OS | Value | |----|-------| | Unknown | 0x00 | | macOS | 0x81 | | iOS | 0x82 | | tvOS | 0x83 | | watchOS | 0x84 | | bridgeOS [(Probably)](https://github.com/apple-oss-distributions/dyld/blob/419f8cbca6fb3420a248f158714a9d322af2aa5a/cache-builder/mrm_shared_cache_builder.h#L45) | 0x85 | | macCatalyst | 0x86 | | iOS Simulator | 0x87 | | tvOS Simulator | 0x88 | | watchOS Simulator | 0x89 | ### Function List | Byte Range | Type | Content | |------------|------------|---------------------------------------| | 0...3 | UInt32 | Entry count (the number of functions) | | 4... | Tag Groups | Each tag group holds some information about a Metal function | The number of tag groups equals the number of functions. ### Tag Group | Byte Range | Type | Content | |------------|--------|------------------------| | 0...3 | UInt32 | Size of the tag group | | 4... | Tags | | ### Tag | Byte Range | Type | Content | |------------|--------|--------------------| | 0...3 | FourCharCode | Name of the tag | | 4...5 | UInt16 | Size of the tag | | 6... | Bytes | Content of the tag | ### Function Information Tags | Name | Content Data Type | Content | |------|----------------------------------|--------------------------------------------------------------------------------------------| | NAME | NULL-terminated C-style string | Name of the function | | MDSZ | UInt64 | Size of the bitcode | | TYPE | UInt8 | Type of the function | | HASH | SHA256 Digest | Hash of the bitcode data (SHA256) | | OFFT | (UInt64, UInt64, UInt64) | Offsets of the information about this function in the public metadata section, private metadata section, and bitcode section | | SOFF | UInt64 | Offset of the source code archive of the function in the embedded source code section | | VERS | (UInt16, UInt16, UInt16, UInt16) | Bitcode and language versions (air.major, air.minor, language.major, language.minor) | | LAYR | UInt8 | [Metal type](#metal-data-type-table) of the `render_target_array_index` (for [layered rendering](https://developer.apple.com/documentation/metal/render_passes/rendering_to_multiple_texture_slices_in_a_draw_command)) | | TESS | UInt8 | Patch type and number of control points per-patch (for post-tessellation vertex function) | | ENDT | | End of the tag group | | Function Type | Value | Note | |---------------|-------|-------------------------------------------------------------| | Vertex | 0x00 | | | Fragment | 0x01 | | | Kernel | 0x02 | | | Unqualified | 0x03 | Functions in Metal dynamic library | | Visible | 0x04 | Functions with `[[visible]]` or `[[stitchable]]` attributes | | Extern | 0x05 | Extern functions complied with `-fcikernel` option | | Intersection | 0x06 | | Content of the `TESS` tag: ```swift // Patch types: // - triangle: 1 // - quad: 2 let content: UInt8 = controlPointCount << 2 | patchType ``` ### Public Metadata Contains information about function constants, tessellation patches, return types, etc. Tags: `CNST`, `VATT`, `VATY`, `RETR`, `ARGR`, etc. ### Private Metadata Contains paths to the shader source (`DEBI` tag) and `.air` (`DEPF` tag) files. ### Header Extension Only exists if `FunctionListOffset + FunctionListSize + 4 != PublicMetadataOffset` | Byte Range | Type | Content | |------------|--------|-------------------------------| | `FunctionListOffset + FunctionListSize + 4`... | Tags | Header extension tags | ### Header Extension Tags | Name | Type | Content | |------|----------------------------------|------------------------| | HDYN | (UInt64, UInt64) | Offset and size of the dynamic header section | | VLST | (UInt64, UInt64) | Offset and size of the exported variable list | | ILST | (UInt64, UInt64) | Offset and size of the imported symbol list | | HSRD/HSRC | (UInt64, UInt64) | Offset and size of the embedded source code section | | UUID | UUID | UUID of the Metal library. | | ENDT | | End of the header extension | ### Dynamic Header Section Tags | Name | Content Data Type | Content | |------|----------------------------------|------------------------| | NAME | NULL-terminated C-style string | Install name of the library | | DYNL | NULL-terminated C-style string | Linked dynamic library | ### Variable List & Imported Symbol List Variable list and imported symbol list have structures that are similar to that of the function list. ### Embedded Source Code Section Only exists if the `metallib` build process is configured to [include source code](https://developer.apple.com/documentation/metal/developing_and_debugging_metal_shaders). | Byte Range | Type | Content | |------------|--------|--------------------| | 0...1 | UInt16 | Number of items in this section | | 2...n | NULL-terminated C-style string | Link options of the `metallib` file | | n...m | NULL-terminated C-style string | Working directory | | m... | Tag Group | `SARC` tag | **Note** "Working directory" only exists in `HSRD`. **Note** `SARC` tag uses 4-bytes (`UInt32`) content size. Content of the `SARC` tag: | Byte Range | Type | Content | |------------|--------|--------------------| | 0...n | NULL-terminated C-style string | ID of the source code archive | | n... | BZh | Bzip2 compressed source code archive | ### Metal Data Type Table | Value | Type | Value | Type | | ----- | ---- | ----- | ---- | | 0x00 | None | 0x01 | Struct | | 0x02 | Array | 0x03 | Float | | 0x04 | Float2 | 0x05 | Float3 | | 0x06 | Float4 | 0x07 | Float2x2 | | 0x08 | Float2x3 | 0x09 | Float2x4 | | 0x0A | Float3x2 | 0x0B | Float3x3 | | 0x0C | Float3x4 | 0x0D | Float4x2 | | 0x0E | Float4x3 | 0x0F | Float4x4 | | 0x10 | Half | 0x11 | Half2 | | 0x12 | Half3 | 0x13 | Half4 | | 0x14 | Half2x2 | 0x15 | Half2x3 | | 0x16 | Half2x4 | 0x17 | Half3x2 | | 0x18 | Half3x3 | 0x19 | Half3x4 | | 0x1A | Half4x2 | 0x1B | Half4x3 | | 0x1C | Half4x4 | 0x1D | Int | | 0x1E | Int2 | 0x1F | Int3 | | 0x20 | Int4 | 0x21 | UInt | | 0x22 | UInt2 | 0x23 | UInt3 | | 0x24 | UInt4 | 0x25 | Short | | 0x26 | Short2 | 0x27 | Short3 | | 0x28 | Short4 | 0x29 | UShort | | 0x2A | UShort2 | 0x2B | UShort3 | | 0x2C | UShort4 | 0x2D | Char | | 0x2E | Char2 | 0x2F | Char3 | | 0x30 | Char4 | 0x31 | UChar | | 0x32 | UChar2 | 0x33 | UChar3 | | 0x34 | UChar4 | 0x35 | Bool | | 0x36 | Bool2 | 0x37 | Bool3 | | 0x38 | Bool4 | 0x3A | Texture | | 0x3B | Sampler | 0x3C | Pointer | | 0x3E | R8Unorm | 0x3F | R8Snorm | | 0x40 | R16Unorm | 0x41 | R16Snorm | | 0x42 | RG8Unorm | 0x43 | RG8Snorm | | 0x44 | RG16Unorm | 0x45 | RG16Snorm | | 0x46 | RGBA8Unorm | 0x47 | RGBA8Unorm_sRGB | | 0x48 | RGBA8Snorm | 0x49 | RGBA16Unorm | | 0x4A | RGBA16Snorm | 0x4B | RGB10A2Unorm | | 0x4C | RG11B10Float | 0x4D | RGB9E5Float | | 0x4E | RenderPipeline | 0x4F | ComputePipeline | | 0x50 | IndirectCommandBuffer | 0x51 | Long | | 0x52 | Long2 | 0x53 | Long3 | | 0x54 | Long4 | 0x55 | ULong | | 0x56 | ULong2 | 0x57 | ULong3 | | 0x58 | ULong4 | 0x59 | Double | | 0x5A | Double2 | 0x5B | Double3 | | 0x5C | Double4 | 0x5D | Float8 | | 0x5E | Float16 | 0x5F | Half8 | | 0x60 | Half16 | 0x61 | Int8 | | 0x62 | Int16 | 0x63 | UInt8 | | 0x64 | UInt16 | 0x65 | Short8 | | 0x66 | Short16 | 0x67 | UShort8 | | 0x68 | UShort16 | 0x69 | Char8 | | 0x6A | Char16 | 0x6B | UChar8 | | 0x6C | UChar16 | 0x6D | Long8 | | 0x6E | Long16 | 0x6F | ULong8 | | 0x70 | ULong16 | 0x71 | Double8 | | 0x72 | Double16 | 0x73 | VisibleFunctionTable | | 0x74 | IntersectionFunctionTable | 0x75 | PrimitiveAccelerationStructure | | 0x76 | InstanceAccelerationStructure | 0x77 | Bool8 | | 0x78 | Bool16 | | | ## ❤️ Contributing If you think there's a mistake, please open an issue. You can also choose to open a pull request with the failure test included. ## 👾 The Story This project would not have started without [zhuowei's research](https://worthdoingbadly.com/metalbitcode/) which revealed the basic binary layout of a `metallib` file, the function list as well as the bitcode section. Thanks, [@zhuowei](https://github.com/zhuowei)! ### What the assembly can tell I tried to continue the research to get a complete structure of the `metallib` file, but found it too hard to move forward based on guesswork alone. So I turned my attention to the `Metal.framework` hoping to find out how the framework loads a `metallib` file. Fortunately, it's not too hard after dragging `Metal.framework/Metal` to [Hopper Disassembler](https://www.hopperapp.com/). `Metal.framework` uses `MTLLibraryDataWithArchive::parseArchiveSync(...)` to load `metallib` files. There is a lot of information hidden in the assembly of `MTLLibraryDataWithArchive`. For example: - The file starts with `0x424c544d`(MTLB); The size of the file is recorded at offset `0x10`. ```cpp int __ZN25MTLLibraryDataWithArchive16parseArchiveSyncEPP7NSErrorb(void * * arg0, bool arg1) { r12 = rdx; r14 = arg1; r13 = arg0; (*(*arg0 + 0xb8))(arg0, 0x0); //LibraryWithFile::setPosition(...) r15 = r13 + 0x78; rbx = (*(*r13 + 0xc0))(r13, r15, 0x58); //LibraryWithFile::readBytes(...) rax = *r13; rax = (*(rax + 0xc8))(r13); //LibraryWithFile::getFileSize(...) // 0x424c544d - MTLB // File size field offset: 0x88 - 0x78 = 0x10 if (((rbx != 0x58) || (*(int32_t *)(r13 + 0x78) != 0x424c544d)) || (*(r13 + 0x88) != rax)) goto loc_6a65b; ... loc_6a65b: if (r14 == 0x0) goto loc_6a6c5; loc_6a660: rdx = @"Invalid library file"; ... } ``` - An `Int16` value at offset `0x4` is related to the target platform. ```cpp loc_6a610: // 0x7c - 0x78 = 0x4 rax = *(int16_t *)(r13 + 0x7c) & 0xffff; if ((rax >= 0x0) || (r12 == 0x0)) goto loc_6a6ea; loc_6a627: if (r14 == 0x0) goto loc_6a6c5; loc_6a630: rdx = @"This library format is not supported on this platform (or was built with an old version of the tools)"; goto loc_6a689; ``` - There is a "Header Extension Section" that contains information about the "Dynamic Header Section", "Imported Symbol List" and "Variable List": ```cpp if (MTLLibraryDataWithArchive::parseHeaderExtension(r13, r13 + 0x100, r14) != 0x0) { if (MTLLibraryDataWithArchive::parseDynamicHeaderSection(r13) != 0x0) { if (MTLLibraryDataWithArchive::parseImportedSymbolListSection(r13) != 0x0) { rax = MTLLibraryDataWithArchive::parseVariableListSection(r13); } else { rax = 0x0; } } else { rax = 0x0; } } else { rax = 0x0; } ``` - The bitcode is validated using SHA256. ```cpp int ____ZN25MTLLibraryDataWithArchive15validateBitCodeEmmPK6NSDataRK12MTLUINT256_t_block_invoke(int arg0) { ... CC_SHA256_Init(&var_B0); CC_SHA256_Update(&var_B0, r14, *(int32_t *)(r15 + 0x38)); CC_SHA256_Final(&var_48, &var_B0); ... } ``` - A lot of FourCC codes: ```cpp // 0x454e4454 - ENDT loc_6a8bc: if (rax == 0x454e4454) goto loc_6a871; ... // 0x54595045 - TYPE loc_6a984: if (rax == 0x54595045) goto loc_6a9dc; ... // 0x44594e4c - DYNL loc_6ae5b: if (rax != 0x44594e4c) goto loc_6b002; ... // 0x56455253 - VERS loc_6b731: if (rax == 0x56455253) goto loc_6b81c; ``` After some digging around I was able to get an overview of the `metallib` file's structure: - The file has a 88 bytes header that contains the file version, target platform, library type, section indices, etc. - There are 4 sections recorded in the file header: - Function list - Public metadata - Private metadata - Bitcode modules Each section is recorded with an offset and a size. This means sections can be non-contiguous, which allows Apple to introduce new sections in between without breaking the compatibility. And Apple did that exactly for the "header extension" section - it lies between the function list and the public metadata section. - Most of the sections (except the bitcode section) resemble a "tag" based structure: - [FourCharCode](https://en.wikipedia.org/wiki/FourCC) is used as the tag's name/type. - An `UInt16` (in most cases) value of size follows the tag's name. The source archive data tag `SARC` unsurprisingly uses an `UInt32` value for its size - a source archive can easily exceed 65KB. - Tags are grouped: - Each group represents a set of properties of an item. - The tag group ends with an `ENDT` tag. ### TDG - "Test Driven Guessing" Next, I need to figure out what information each tag/field holds. This can be hard to get from the assembly of the `Metal.framework` because: - Some fields may be designed purely for tooling or debugging, so `MTLLibraryDataWithArchive` may just ignore them. - The assembly is platform dependent. For example, the iOS version of `MTLLibraryDataWithArchive` may only check whether the `metallib` is built for iOS and cannot tell if the library is built for macOS. - Some fields are just hard to analyze and follow. Examples: - There are 3 offsets in the `OFFT` tag of the function, where are they pointing to? and how are they finally used? - What are the possible values of the function type? What does each value mean? It seems that the quickest way to get this information is through experiments. I started by manually compiling `metal` files with different shaders, options, and SDKs, then inspecting each field I was interested in. My desktop was quickly flooded with `metallib` files and [HexFiend](https://hexfiend.com/) windows, but I didn't find much useful information. I need something that can automatically build `metallib` and presents me only the field that I'm interested in. I came up with the "Test Driven Guessing": 1. Write a `metallib` parser based on the binary structure overview at hand. 2. In the parser, log the value of a field/tag (or some related fields) that is currently unknown. 3. Create tests that produce `metallib` files using different kinds of shaders and compile options that may affect the value of the field, and use the parser to parse the file data. 4. Run tests and analyze the log to make hypotheses. 5. Update the parser based on hypotheses. 6. Run tests again to verify. After a few rounds, I was able to get the function type table, target OS table, and the meaning of 3 offsets in the `OFFT` tag. I also found a few things interesting in this process: - Metal does not support watchOS, however, it is possible to build a `metallib` targeting watchOS. And Apple does include some `metallib`s in the watchOS SDK. (e.g. `Xcode.app/Contents/Developer/Platforms/WatchOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/watchOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/CoreImage.framework/ci_filters.metallib`) - Empty `metallib`s targeting old versions of iOS are [mistakenly marked as targeting macOS](https://github.com/YuAo/MetalLibraryArchive/blob/da16437b0549c7b21408e51b210627f73e323cbf/Tests/MetalLibraryArchiveTests/Tests.swift#L559). - I cannot build a `metallib` that has the target OS value `0x85`. At first I thought it might be reserved for the concealed [realityOS](https://github.com/apple-oss-distributions/dyld/blob/5c9192436bb195e7a8fe61f22a229ee3d30d8222/common/MachOFile.cpp#L578), but later found out it is [more likely for the bridgeOS](https://github.com/apple-oss-distributions/dyld/blob/419f8cbca6fb3420a248f158714a9d322af2aa5a/cache-builder/mrm_shared_cache_builder.h#L45). ### Updates **Apr 10, 2022** Tags like `LAYR`, `VATY`, `CNST`, etc., contain `UInt8` values of Metal data types. The corresponding description for each data type value can be retrieved using a private class in Metal.framework - [MTLTypeInternal](https://github.com/nst/iOS-Runtime-Headers/blob/fbb634c78269b0169efdead80955ba64eaaa2f21/Frameworks/Metal.framework/MTLTypeInternal.h) ```objective-c id value = [[NSClassFromString(@"MTLTypeInternal") alloc] initWithDataType:0x06]; NSLog(@"%@", value.description); // MTLDataTypeFloat4 ``` I created a command line tool to generate the Metal data type table. ```shell cd Utilities swift run metal-data-type-tools gen-markdown --columns 2 # generate a markdown table swift run metal-data-type-tools gen-swift # generate a Swift enum for Metal data types. ``` **Mar 31, 2022** The `air-lld` (`Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/metal/ios/bin/air-lld`) also provides a lot of information about how the `metallib` file is built. Some section names and descriptions are updated. ```cpp int __ZN4llvm3air20MetalLibObjectWriter5writeEv() { r14 = rdi; rax = llvm::air::MetalLibObjectWriter::writeHeader(); if (rax != 0x0) goto loc_1000351b9; loc_100035135: rax = llvm::air::MetalLibObjectWriter::writeFunctionList(); if (rax != 0x0) goto loc_1000351b9; loc_100035141: rax = llvm::air::MetalLibObjectWriter::writeHeaderExtension(); if (rax != 0x0) goto loc_1000351b9; loc_10003514d: rax = llvm::air::MetalLibObjectWriter::writePublicMetadata(); if (rax != 0x0) goto loc_1000351b9; loc_100035159: rax = llvm::air::MetalLibObjectWriter::writePrivateMetadata(); if (rax != 0x0) goto loc_1000351b9; loc_100035165: rax = llvm::air::MetalLibObjectWriter::writeModuleList(); if (rax != 0x0) goto loc_1000351b9; loc_100035171: rax = llvm::air::MetalLibObjectWriter::writeSources(); if (rax != 0x0) goto loc_1000351b9; loc_10003517d: rax = llvm::air::MetalLibObjectWriter::writeDynamicHeader(); if (rax != 0x0) goto loc_1000351b9; loc_100035189: rax = llvm::air::MetalLibObjectWriter::writeVariableList(); if (rax != 0x0) goto loc_1000351b9; loc_100035195: rax = llvm::air::MetalLibObjectWriter::writeImportedSymbolList(); if (rax != 0x0) goto loc_1000351b9; loc_1000351a1: rax = llvm::air::MetalLibObjectWriter::computeUUID(); if (rax != 0x0) goto loc_1000351b9; loc_1000351ad: rax = llvm::air::MetalLibObjectWriter::backpatchAllLocations(); if (rax == 0x0) goto loc_1000351c2; loc_1000351b9: rbx = rax; goto loc_1000351bb; loc_1000351bb: rax = rbx; return rax; loc_1000351c2: rbx = 0x0; std::__1::system_category(); goto loc_1000351bb; } ``` ================================================ FILE: Sources/Explorer/AppIconView.swift ================================================ // // File.swift // // // Created by YuAo on 2022/3/18. // import Foundation import AppKit public class LinearGradientView: NSView { override init(frame frameRect: NSRect) { super.init(frame: frameRect) self.setup() } required init?(coder: NSCoder) { super.init(coder: coder) self.setup() } public var colors: [NSColor] = [] { didSet { self.needsLayout = true } } public var locations: [CGFloat] = [] { didSet { self.needsLayout = true } } public var startPoint: CGPoint = .zero { didSet { self.needsLayout = true } } public var endPoint: CGPoint = CGPoint(x: 1, y: 1) { didSet { self.needsLayout = true } } private func setup() { //layer hosting view layer = CAGradientLayer() wantsLayer = true } private var gradientLayer: CAGradientLayer { return self.layer as! CAGradientLayer } public override func layout() { super.layout() self.gradientLayer.colors = self.colors.compactMap({ $0.cgColor }) self.gradientLayer.locations = self.locations.count > 0 ? self.locations.map({ NSNumber(value: Float($0)) }) : nil self.gradientLayer.startPoint = self.startPoint self.gradientLayer.endPoint = self.endPoint } } @available(macOS 11.0, *) class AppIconView: NSView { override init(frame frameRect: NSRect) { super.init(frame: frameRect) self.setup() } required init?(coder: NSCoder) { fatalError("init(coder:) has not been implemented") } private weak var gradientView: LinearGradientView! private weak var imageView: NSImageView! private func setup() { let gradientView = LinearGradientView() gradientView.colors = [NSColor.darkGray, NSColor.black] gradientView.locations = [0, 1] gradientView.startPoint = CGPoint(x: 0.5, y: 0) gradientView.endPoint = CGPoint(x: 0.5, y: 1) self.addSubview(gradientView) self.gradientView = gradientView let imageView = NSImageView(image: NSImage(systemSymbolName: "square.stack.3d.forward.dottedline.fill", accessibilityDescription: nil)!) imageView.contentTintColor = .white imageView.rotate(byDegrees: 90) imageView.imageScaling = .scaleProportionallyUpOrDown self.addSubview(imageView) self.imageView = imageView } override func layout() { super.layout() self.imageView.frame = self.bounds.insetBy(dx: self.bounds.width/6, dy: self.bounds.height/6) self.gradientView.frame = self.bounds.insetBy(dx: self.bounds.width/11, dy: self.bounds.height/11) self.gradientView.layer?.cornerCurve = .continuous self.gradientView.layer?.cornerRadius = self.bounds.height/5 let shadow = NSShadow() shadow.shadowColor = NSColor.black.withAlphaComponent(0.3) shadow.shadowBlurRadius = self.bounds.width/40 self.gradientView.shadow = shadow } } ================================================ FILE: Sources/Explorer/MetalLibraryAchiveUnpacker.swift ================================================ // // File.swift // // // Created by YuAo on 2022/3/18. // import Foundation import MetalLibraryArchive @available(macOS 11.0, *) struct Unpacker { static func unpack(_ archive: Archive, to url: URL, disassembler: URL?) throws { let fileManager = FileManager() try fileManager.createDirectory(at: url, withIntermediateDirectories: true, attributes: nil) var savedData = Set() for function in archive.functions { if savedData.contains(function.bitcode) { continue } let airFileURL = url.appendingPathComponent(function.name).appendingPathExtension("air") try function.bitcode.write(to: airFileURL) if let disassembler = disassembler { let process = Process() process.executableURL = disassembler process.arguments = [airFileURL.path] try process.run() process.waitUntilExit() } savedData.insert(function.bitcode) } for sourceArchive in archive.sourceArchives { let sourceArchiveURL = url.appendingPathComponent("SourceArchive-\(sourceArchive.id)").appendingPathExtension("bz2") try sourceArchive.data.write(to: sourceArchiveURL) } } } ================================================ FILE: Sources/Explorer/MetalLibraryArchiveDocument.swift ================================================ // // File.swift // // // Created by YuAo on 2022/3/17. // import SwiftUI import MetalLibraryArchive import UniformTypeIdentifiers @available(macOS 11.0, *) struct MetalLibraryArchiveDocument: FileDocument { enum Error: LocalizedError { case cannotLoadFile var errorDescription: String? { switch self { case .cannotLoadFile: return "Error loading file content." } } var failureReason: String? { errorDescription } } static let readableContentTypes: [UTType] = [.data] let archive: Archive let filename: String init(configuration: ReadConfiguration) throws { guard let content = configuration.file.regularFileContents else { throw Error.cannotLoadFile } archive = try Archive(data: content) filename = configuration.file.filename ?? "default.metallib" } func fileWrapper(configuration: WriteConfiguration) throws -> FileWrapper { throw CocoaError(.fileWriteNoPermission) } } ================================================ FILE: Sources/Explorer/MetalLibraryArchiveView.swift ================================================ // // File.swift // // // Created by YuAo on 2022/3/17. // import SwiftUI import MetalLibraryArchive @available(macOS 11.0, *) struct MetalLibraryView: View { private let archive: Archive private let filename: String private let functionBitcodeID: [String: Int] @Environment(\.disassemblerURL) private var disassemblerURL @Environment(\.colorScheme) private var colorScheme @Environment(\.controlActiveState) private var controlActiveState @ObservedObject private var toastController: ToastController = ToastController() init(archive: Archive, filename: String) { self.archive = archive self.filename = filename self.functionBitcodeID = { var currentID: Int = 0 var functionBitcodeID: [String: Int] = [:] var bitcodeID: [Data: Int] = [:] for function in archive.functions { if let id = bitcodeID[function.bitcode] { functionBitcodeID[function.name] = id } else { bitcodeID[function.bitcode] = currentID functionBitcodeID[function.name] = currentID currentID += 1 } } return functionBitcodeID }() } struct LanguageVersionView: View { private let text: String init(version: LanguageVersion) { text = "MSL \(version.major).\(version.minor)" } var body: some View { BadgeView(text: text, foregroundColor: .white, backgroundColor: .gray) } } struct BadgeView: View { @Environment(\.controlActiveState) private var controlActiveState let text: String let foregroundColor: Color let backgroundColor: Color var body: some View { Text(text) .font(Font.system(.footnote, design: .monospaced).weight(.medium)) .foregroundColor(foregroundColor) .padding(6) .background(RoundedRectangle(cornerRadius: 8).foregroundColor(controlActiveState == .inactive ? .gray.opacity(0.5) : backgroundColor)) } } struct BitcodeSizeView: View { private let text: String init(bytes: Int) { let formatter = ByteCountFormatter() text = formatter.string(fromByteCount: Int64(bytes)) } var body: some View { BadgeView(text: text, foregroundColor: .white, backgroundColor: .gray.opacity(0.75)) } } struct BitcodeIDView: View { private let text: String init(id: Int) { text = "BC \(id)" } var body: some View { BadgeView(text: text, foregroundColor: .white, backgroundColor: .gray.opacity(0.75)) } } struct SourceBadge: View { @Environment(\.controlActiveState) private var controlActiveState private var color: Color { controlActiveState == .inactive ? .gray.opacity(0.5) : .green.opacity(0.75) } var body: some View { Text("SRC") .font(Font.system(.caption, design: .monospaced).smallCaps()) .foregroundColor(color) .padding(EdgeInsets(top: 1, leading: 4, bottom: 1, trailing: 4)) .background(GeometryReader(content: { proxy in RoundedRectangle(cornerRadius: proxy.size.height/2).strokeBorder(color, lineWidth: 1, antialiased: true) })) } } struct FunctionTypeView: View { private struct TypeInfo { var symbol: String var textColor: Color var backgroundColor: Color } private let typeInfo: TypeInfo? init(type: FunctionType?) { typeInfo = { let colors: [Color] = [.green, .blue, .orange, .red, .pink, .purple, .yellow] precondition(colors.count == FunctionType.allCases.count) if let type = type { let colorIndex = FunctionType.allCases.firstIndex(of: type)! return TypeInfo(symbol: type.description, textColor: .white, backgroundColor: colors[colorIndex]) } else { return TypeInfo(symbol: "Unknown", textColor: .white, backgroundColor: .gray) } }() } var body: some View { if let info = typeInfo { BadgeView(text: info.symbol, foregroundColor: info.textColor, backgroundColor: info.backgroundColor) } } } struct TargetPlatformBadge: View { @Environment(\.colorScheme) private var colorScheme @Environment(\.controlActiveState) private var controlActiveState private var backgroundColor: Color { if controlActiveState == .inactive { return .gray.opacity(0.25) } else { if colorScheme == .dark { return .gray.opacity(0.25) } else { return .gray } } } let archive: Archive var body: some View { Group { if let deploymentTarget = archive.deploymentTarget { Text("\(archive.targetPlatform.description) (\(deploymentTarget.operatingSystem.description) \(deploymentTarget.operatingSystemVersion.description)) - \(archive.libraryType.description)") } else { Text("\(archive.targetPlatform.description) - \(archive.libraryType.description)") } } .font(Font.system(.footnote).weight(.medium)) .foregroundColor(.white) .padding(6) .background(RoundedRectangle(cornerRadius: 6).foregroundColor(backgroundColor)) } } @State private var functionType: FunctionType? var filteredEntries: [Function] { archive.functions.filter({ if let filter = functionType { if $0.type != filter { return false } } return true }) } var body: some View { List { HStack { if filteredEntries.count != archive.functions.count { Text("\(filteredEntries.count)/\(archive.functions.count) functions") } else { Text("\(archive.functions.count) functions") } Spacer() Picker("", selection: $functionType, content: { Text("All").tag(Optional.none) ForEach(FunctionType.allCases) { type in Text(type.description).tag(Optional.some(type)) } }) .scaledToFit() } ForEach(filteredEntries, id: \.name) { entry in HStack { Group { if #available(macOS 12.0, *) { Text(entry.name).textSelection(.enabled) } else { Text(entry.name) } }.font(Font.system(.headline, design: .monospaced).weight(.bold)) if entry.isSourceIncluded { SourceBadge() } Spacer() FunctionTypeView(type: entry.type) LanguageVersionView(version: entry.languageVersion) BitcodeIDView(id: functionBitcodeID[entry.name]!) } .lineLimit(nil) .padding() .background( RoundedRectangle(cornerRadius: 12) .foregroundColor(colorScheme == .dark ? Color.black.opacity(0.35) : Color.white) ) } } .presenter(for: toastController) .toolbar(content: { ToolbarItem(placement: .navigation, content: { TargetPlatformBadge(archive: archive) }) ToolbarItem(placement: .primaryAction, content: { Menu(content: { Button("Unpack", action: { unpack(disassemble: false) }) Button("Unpack and Disassemble", action: { unpack(disassemble: true) }) }, label: { Image(systemName: "tray.and.arrow.up.fill") }) }) }) .listStyle(.sidebar) } func unpack(disassemble: Bool) { let disassemblerURL: URL? if disassemble { if let url = self.disassemblerURL { disassemblerURL = url } else { class AlertDelegate: NSObject, NSAlertDelegate { func alertShowHelp(_ alert: NSAlert) -> Bool { NSWorkspace.shared.open(URL(string: "https://github.com/YuAo/MetalLibraryArchive")!) return true } } let delegate = AlertDelegate() let alert = NSAlert() alert.messageText = "In order to disassemble .air files. llvm-dis is required. Use the \"Disassembler\" menu to locate llvm-dis." alert.showsHelp = true alert.delegate = delegate alert.icon = nil alert.runModal() withExtendedLifetime(delegate, {}) return } } else { disassemblerURL = nil } let savePanel = NSSavePanel() savePanel.canCreateDirectories = true savePanel.nameFieldStringValue = "\(filename).unpacked" let response = savePanel.runModal() guard let url = savePanel.url, response == .OK else { return } do { try Unpacker.unpack(archive, to: url, disassembler: disassemblerURL) toastController.showToast(title: "Unpack Succeeded", type: .success) } catch { NSAlert(error: error).runModal() } } } extension FunctionType: Identifiable { public var id: Int { self.rawValue } } ================================================ FILE: Sources/Explorer/Toast.swift ================================================ // // File.swift // // // Created by YuAo on 2022/4/6. // import Foundation import SwiftUI enum ToastType { case `default` case success } struct Toast { var title: String var type: ToastType } class ToastController: ObservableObject { @Published private(set) var shouldShowToast: Bool = false @Published private(set) var toast: Toast = Toast(title: "", type: .default) private var timer: Timer? func showToast(title: String, type: ToastType = .default, duration: TimeInterval = 2.0) { timer?.invalidate() timer = Timer.scheduledTimer(withTimeInterval: duration, repeats: false, block: { [weak self] _ in self?.shouldShowToast = false }) toast = Toast(title: title, type: type) shouldShowToast = true } } @available(macOS 11.0, *) fileprivate struct ToastPresenter: ViewModifier { @ObservedObject var controller: ToastController @State private var showsToast: Bool = false @Environment(\.colorScheme) private var colorScheme func body(content: Content) -> some View { return content.overlay(VStack { if showsToast { Group { switch controller.toast.type { case .`default`: Text(controller.toast.title) case .success: Label(title: { Text(controller.toast.title) }, icon: { Image(systemName: "checkmark").foregroundColor(Color.green) }) } } .padding(EdgeInsets(top: 10, leading: 12, bottom: 10, trailing: 12)) .background(GeometryReader(content: { proxy in RoundedRectangle(cornerRadius: proxy.size.height / 2).fill(colorScheme == .dark ? Color(.sRGB, white: 0.25, opacity: 1) : Color.white) .shadow(color: .black.opacity(0.1), radius: 10, x: 0, y: 0) })) .padding() .transition(.move(edge: .top)) } Spacer() }).onReceive(controller.$shouldShowToast, perform: { value in withAnimation(.spring(), { self.showsToast = value }) }).clipped() } } @available(macOS 11.0, *) extension View { func presenter(for controller: ToastController) -> some View { self.modifier(ToastPresenter(controller: controller)) } } ================================================ FILE: Sources/Explorer/main.swift ================================================ // // File.swift // // // Created by YuAo on 2022/3/17. // import UniformTypeIdentifiers import SwiftUI // MARK: - Application Support @available(macOS 11.0, *) class AppDelegate: NSObject, NSApplicationDelegate { func applicationDidFinishLaunching(_ notification: Notification) { NSApp.setActivationPolicy(.regular) NSApp.activate(ignoringOtherApps: true) NSApplication.shared.dockTile.contentView = AppIconView() NSApplication.shared.dockTile.display() } } private struct DisassemblerURLEnvironmentKey: EnvironmentKey { static let defaultValue: URL? = nil } @available(macOS 11.0, *) extension EnvironmentValues { var disassemblerURL: URL? { get { self[DisassemblerURLEnvironmentKey.self] } set { self[DisassemblerURLEnvironmentKey.self] = newValue } } } @available(macOS 11.0, *) struct Main: App { @NSApplicationDelegateAdaptor var appDelegate: AppDelegate @AppStorage("llvm-disassembler-url", store: UserDefaults(suiteName: "com.imyuao.MetalLibraryArchive.app")) var disassemblerURL: URL? var body: some Scene { DocumentGroup(viewing: MetalLibraryArchiveDocument.self, viewer: { configuration in MetalLibraryView(archive: configuration.document.archive, filename: configuration.document.filename).environment(\.disassemblerURL, disassemblerURL) }).commands(content: { CommandMenu("Disassembler", content: { if let url = disassemblerURL, FileManager.default.fileExists(atPath: url.path) { Text("Disassembler: \(url.lastPathComponent) at \(url.path)") Divider() Button("Reset", action: { disassemblerURL = nil }) } else { Button("Locate llvm-dis", action: { let openPanel = NSOpenPanel() openPanel.allowedContentTypes = [.unixExecutable] let response = openPanel.runModal() if let url = openPanel.url, response == .OK { disassemblerURL = url } }) } }) }) } } if #available(macOS 11.0, *) { DispatchQueue.main.async { NSDocumentController.shared.openDocument(nil) } Main.main() } else { fatalError() } ================================================ FILE: Sources/MetalLibraryArchive/Archive.swift ================================================ import Foundation #if canImport(Crypto) import Crypto #endif public struct Archive: Hashable { public class DataScanner { public enum Error: LocalizedError { case indexOutOfBounds case invalidStringData public var errorDescription: String? { switch self { case .indexOutOfBounds: return "DataScanner: Index out of bounds." case .invalidStringData: return "DataScanner: Invalid string data." } } public var failureReason: String? { errorDescription } } private(set) var offset: Int let data: Data init(data: Data) { self.data = data self.offset = data.startIndex } private func checkBoundsForReading(byteCount: Int) throws { if offset + byteCount > data.endIndex { throw Error.indexOutOfBounds } } func scanFourCharCode() throws -> String { let byteCount = 4 try checkBoundsForReading(byteCount: byteCount) guard let string = String(data: data[offset..<(offset + byteCount)], encoding: .utf8) else { throw Error.invalidStringData } offset += byteCount assert(string.allSatisfy({ $0.isASCII })) return string } func scanData(byteCount: Int) throws -> Data { try checkBoundsForReading(byteCount: byteCount) let scanned = data[offset..<(offset + byteCount)] offset += byteCount return scanned } func scanDataToEnd() throws -> Data { try checkBoundsForReading(byteCount: 1) return data[offset...] } func scanCString() throws -> String { let string: String = try data[offset...].withUnsafeBytes({ pointer in guard let index = pointer.firstIndex(of: 0) else { throw Error.invalidStringData } let string = String(cString: pointer.bindMemory(to: UInt8.self).baseAddress!) guard index == string.lengthOfBytes(using: .utf8) else { throw Error.invalidStringData } return string }) offset += (string.lengthOfBytes(using: .utf8) + 1) return string } func scan(_ type: T.Type) throws -> Int { let size = MemoryLayout.size try checkBoundsForReading(byteCount: size) let value = data[offset..<(offset + size)].withUnsafeBytes({ pointer in return pointer.bindMemory(to: T.self)[0] }) offset += size return Int(value.littleEndian) } func scanTags(contentSizeType: T.Type) throws -> [Tag] { var tags: [Tag] = [] while true { let tagName = try scanFourCharCode() if tagName == "ENDT" { break } let tagSize = try scan(T.self) let content = try scanData(byteCount: tagSize) tags.append(Tag(name: tagName, content: content)) } return tags } func seek(to offset: Int) throws { if offset > data.count || offset < 0 { throw Error.indexOutOfBounds } self.offset = data.startIndex + offset } } public enum Error: LocalizedError { case invalidHeader case invalidFunctionListOffset case invalidTagGroupSize case incompleteFunctionInfo case incompleteSourceArchiveInfo case invalidBitcodeHash case unexpectedTagContentSize(tagName: String) case unexpectedFunctionListEnding case unexpectedLibraryType(Int) case unexpectedTargetPlatform(Int) case unexpectedBitcodeSize case unexpectedOperatingSystemType(Int) case unexpectedFileSize public var errorDescription: String? { switch self { case .invalidHeader: return "Invalid file header." case .invalidFunctionListOffset: return "Invalid function list offset." case .invalidTagGroupSize: return "Invalid tag group size." case .incompleteFunctionInfo: return "Incomplete function info." case .incompleteSourceArchiveInfo: return "Incomplete source archive info." case .invalidBitcodeHash: return "Invalid bitcode hash." case .unexpectedTagContentSize(let name): return "Unexpected size for tag \"\(name)\"." case .unexpectedFunctionListEnding: return "Unexpected function list ending." case .unexpectedLibraryType(let value): return "Unexpected library type: \(value)." case .unexpectedTargetPlatform(let value): return "Unexpected target platform: \(value)." case .unexpectedBitcodeSize: return "Unexpected bitcode size." case .unexpectedOperatingSystemType(let value): return "Unexpected OS type: \(value)." case .unexpectedFileSize: return "Unexpected file size." } } public var failureReason: String? { errorDescription } } public struct Version: Hashable, CustomStringConvertible { public let major: Int public let minor: Int public var description: String { return "\(major).\(minor)" } } public let functions: [Function] public let headerExtensionTags: [Tag] public let libraryType: LibraryType public let targetPlatform: Platform public let deploymentTarget: DeploymentTarget? public let sourceArchives: [SourceArchive] public let version: Version public init(data: Data) throws { let dataScanner = DataScanner(data: data) do { guard try dataScanner.scanFourCharCode() == "MTLB" else { throw Error.invalidHeader } } catch { throw Error.invalidHeader } //4...5 let targetPlatform = try dataScanner.scan(UInt16.self) if targetPlatform == 0x8001 { self.targetPlatform = .macOS } else if targetPlatform == 0x0001 { self.targetPlatform = .iOS } else { throw Error.unexpectedTargetPlatform(targetPlatform) } //6,7,8,9 let libraryMajorVersion = try dataScanner.scan(UInt16.self) let libraryMinorVersion = try dataScanner.scan(UInt16.self) version = Version(major: libraryMajorVersion, minor: libraryMinorVersion) //10 let libraryTypeValue = try dataScanner.scan(UInt8.self) if let type = LibraryType(rawValue: libraryTypeValue) { libraryType = type } else { throw Error.unexpectedLibraryType(libraryTypeValue) } // 11: Target OS let targetOSValue = try dataScanner.scan(UInt8.self) // 12...13: Target OS version, major let targetOSMajorVersion = try dataScanner.scan(UInt16.self) // 14...15: Target OS version, minor let targetOSMinorVersion = try dataScanner.scan(UInt16.self) if targetOSValue != 0 { guard let os = DeploymentTarget.OperatingSystem(rawValue: targetOSValue) else { throw Error.unexpectedOperatingSystemType(targetOSValue) } self.deploymentTarget = DeploymentTarget(operatingSystem: os, operatingSystemVersion: DeploymentTarget.OperatingSystem.Version(major: targetOSMajorVersion, minor: targetOSMinorVersion)) } else { self.deploymentTarget = nil } // 16...23 let fileSize = try dataScanner.scan(UInt64.self) guard fileSize == data.count else { throw Error.unexpectedFileSize } let functionListOffset = try dataScanner.scan(UInt64.self) //24...31 let functionListSize = try dataScanner.scan(UInt64.self) //32...39 // Public metadata offset let publicMetadataOffset = try dataScanner.scan(UInt64.self) //40...47 // Public metadata size _ = try dataScanner.scan(UInt64.self) //48...55 // Private metadata offset let privateMetadataOffset = try dataScanner.scan(UInt64.self) //56...63 // Private metadata size _ = try dataScanner.scan(UInt64.self) //64...71 let bitcodeOffset = try dataScanner.scan(UInt64.self) //72...79 let bitcodeSize = try dataScanner.scan(UInt64.self) //80...87 // Validations guard functionListOffset + functionListSize < fileSize, functionListOffset > 0 else { throw Error.invalidFunctionListOffset } if functionListSize > 0 { try dataScanner.seek(to: functionListOffset + functionListSize) let functionListEndMark = try dataScanner.scanFourCharCode() guard functionListEndMark == "ENDT" else { throw Error.unexpectedFunctionListEnding } } // Read header extension tags, 4 is for `ENDT` or `0x00000000` if functionListOffset + functionListSize + 4 != publicMetadataOffset { try dataScanner.seek(to: functionListOffset + functionListSize + 4) headerExtensionTags = try dataScanner.scanTags(contentSizeType: UInt16.self) } else { headerExtensionTags = [] } guard functionListSize > 0 else { functions = [] sourceArchives = [] return } guard bitcodeSize > 0 else { throw Error.unexpectedBitcodeSize } // Read functions try dataScanner.seek(to: functionListOffset) let numberOfFunctions = try dataScanner.scan(UInt32.self) let functionInfos: [FunctionInfo] = try { var infos: [FunctionInfo] = [] for _ in 0.. 0 else { throw Error.invalidTagGroupSize } infos.append(try Self.scanFunctionInfo(using: dataScanner)) } return infos }() let functions: [Function] = try { var entries: [Function] = [] for (index, info) in functionInfos.enumerated() { try dataScanner.seek(to: bitcodeOffset + Int(info.offsets.bitcode)) let functionBitcodeSize: UInt64 if let size = info.bitcodeSize { functionBitcodeSize = size } else { let nextOffset: UInt64 = index < functionInfos.count - 1 ? functionInfos[index + 1].offsets.bitcode : UInt64(bitcodeSize); functionBitcodeSize = nextOffset - info.offsets.bitcode; } let data = try dataScanner.scanData(byteCount: Int(functionBitcodeSize)) #if canImport(Crypto) guard SHA256.hash(data: data) == info.hash else { throw Error.invalidBitcodeHash } #endif try dataScanner.seek(to: publicMetadataOffset + Int(info.offsets.publicMetadata)) let publicMetadataTagSize = try dataScanner.scan(UInt32.self) guard publicMetadataTagSize > 0 else { throw Error.invalidTagGroupSize } let publicMetadataTags = try dataScanner.scanTags(contentSizeType: UInt16.self) try dataScanner.seek(to: privateMetadataOffset + Int(info.offsets.privateMetadata)) let privateMetadataTagsSize = try dataScanner.scan(UInt32.self) guard privateMetadataTagsSize > 0 else { throw Error.invalidTagGroupSize } let privateMetadataTags = try dataScanner.scanTags(contentSizeType: UInt16.self) entries.append(Function(name: info.name, type: info.type, languageVersion: info.languageVersion, tags: info.tags, publicMetadataTags: publicMetadataTags, privateMetadataTags: privateMetadataTags, bitcode: data, bitcodeHash: info.hash)) } return entries }() self.functions = functions if let embededSourceTag = headerExtensionTags.first(where: { $0.name == "HSRD" || $0.name == "HSRC" }) { guard embededSourceTag.content.count == MemoryLayout.size * 2 else { throw Error.unexpectedTagContentSize(tagName: embededSourceTag.name) } let offset = embededSourceTag.content.withUnsafeBytes({ ptr in ptr.bindMemory(to: UInt64.self)[0] }) try dataScanner.seek(to: Int(offset)) let archiveCount = try dataScanner.scan(UInt32.self) _ = try dataScanner.scanCString() //Link options if embededSourceTag.name == "HSRD" { _ = try dataScanner.scanCString() //Working dir } var archives: [SourceArchive] = [] for _ in 0.. 0 else { throw Error.invalidTagGroupSize } guard let sourceArchiveTag = try dataScanner.scanTags(contentSizeType: UInt32.self).first(where: { $0.name == "SARC" }) else { throw Error.incompleteSourceArchiveInfo } let tagContentScanner = DataScanner(data: sourceArchiveTag.content) let archiveID = try tagContentScanner.scanCString() let archiveData = try tagContentScanner.scanDataToEnd() precondition(archiveData.count + archiveID.lengthOfBytes(using: .utf8) + 1 == sourceArchiveTag.content.count) archives.append(SourceArchive(id: archiveID, data: archiveData)) } sourceArchives = archives } else { sourceArchives = [] } } } extension Archive { private struct FunctionInfo { var name: String var bitcodeSize: UInt64? var offsets: (publicMetadata: UInt64, privateMetadata: UInt64, bitcode: UInt64) var type: FunctionType? var languageVersion: LanguageVersion var hash: Data var tags: [Tag] } private static func scanFunctionInfo(using scanner: DataScanner) throws -> FunctionInfo { let tags: [Tag] = try scanner.scanTags(contentSizeType: UInt16.self) var name: String? var bitcodeSize: UInt64? var offsets: (publicMetadata: UInt64, privateMetadata: UInt64, bitcode: UInt64)? var type: FunctionType? var hash: Data? var languageVersion: LanguageVersion? for tag in tags { switch tag.name { case "NAME": name = String(data: tag.content.dropLast(), encoding: .utf8) case "MDSZ": guard tag.content.count == MemoryLayout.size else { throw Error.unexpectedTagContentSize(tagName: tag.name) } bitcodeSize = tag.content.withUnsafeBytes({ pointer in pointer.bindMemory(to: UInt64.self)[0] }) case "TYPE": guard tag.content.count == MemoryLayout.size else { throw Error.unexpectedTagContentSize(tagName: tag.name) } let rawType = tag.content.withUnsafeBytes({ pointer in pointer.bindMemory(to: UInt8.self)[0] }) type = FunctionType(rawValue: Int(rawType)) case "HASH": guard tag.content.count == 32 else { throw Error.unexpectedTagContentSize(tagName: tag.name) } hash = tag.content case "OFFT": guard tag.content.count == MemoryLayout.size * 3 else { throw Error.unexpectedTagContentSize(tagName: tag.name) } offsets = tag.content.withUnsafeBytes({ pointer in let offsetValues = pointer.bindMemory(to: UInt64.self) return (publicMetadata: offsetValues[0], privateMetadata: offsetValues[1], bitcode: offsetValues[2]) }) case "VERS": guard tag.content.count == MemoryLayout.size * 4 else { throw Error.unexpectedTagContentSize(tagName: tag.name) } let versionNumbers = tag.content.withUnsafeBytes({ pointer in Array(pointer.bindMemory(to: UInt16.self)) }) //air version _ = (Int(versionNumbers[0]), Int(versionNumbers[1])) languageVersion = LanguageVersion(major: Int(versionNumbers[2]), minor: Int(versionNumbers[3])) default: break } } guard let name = name, let offsets = offsets, let hash = hash, let languageVersion = languageVersion else { throw Error.incompleteFunctionInfo } return FunctionInfo(name: name, bitcodeSize: bitcodeSize, offsets: offsets, type: type, languageVersion: languageVersion, hash: hash, tags: tags) } } ================================================ FILE: Sources/MetalLibraryArchive/DeploymentTarget.swift ================================================ // // File.swift // // // Created by YuAo on 2022/3/26. // import Foundation public struct DeploymentTarget: Hashable { public enum OperatingSystem: Int, CustomStringConvertible, CaseIterable, Hashable { case macOS = 0x81 case iOS = 0x82 case tvOS = 0x83 case watchOS = 0x84 case bridgeOS = 0x85 case macCatalyst = 0x86 case iOSSimulator = 0x87 case tvOSSimulator = 0x88 case watchOSSimulator = 0x89 public var description: String { switch self { case .macOS: return "macOS" case .iOS: return "iOS" case .tvOS: return "tvOS" case .watchOS: return "watchOS" case .bridgeOS: return "bridgeOS" case .macCatalyst: return "Mac Catalyst" case .iOSSimulator: return "iOS Simulator" case .tvOSSimulator: return "tvOS Simulator" case .watchOSSimulator: return "watchOS Simulator" } } public struct Version: Hashable, CustomStringConvertible { public let major: Int public let minor: Int public var description: String { return "\(major).\(minor)" } } } public let operatingSystem: OperatingSystem public let operatingSystemVersion: OperatingSystem.Version } ================================================ FILE: Sources/MetalLibraryArchive/Function.swift ================================================ // // File.swift // // // Created by YuAo on 2022/3/18. // import Foundation public enum FunctionType: Int, CaseIterable, CustomStringConvertible, Hashable { case vertex = 0 case fragment = 1 case kernel = 2 case unqualified = 3 case visible = 4 case extern = 5 case intersection = 6 public var description: String { switch self { case .vertex: return "Vertex" case .fragment: return "Fragment" case .kernel: return "Kernel" case .unqualified: return "Unqualified" case .visible: return "Visible" case .extern: return "Extern" case .intersection: return "Intersection" } } } public struct Function: Hashable { public let name: String public let type: FunctionType? public let languageVersion: LanguageVersion public let tags: [Tag] public let publicMetadataTags: [Tag] public let privateMetadataTags: [Tag] public let bitcode: Data public let bitcodeHash: Data } extension Function { public var isSourceIncluded: Bool { return tags.contains(where: { $0.name == "SOFF" }) } } ================================================ FILE: Sources/MetalLibraryArchive/LanguageVersion.swift ================================================ // // File.swift // // // Created by YuAo on 2022/3/22. // import Foundation public struct LanguageVersion: Hashable, CustomStringConvertible { public init(major: Int, minor: Int) { self.major = major self.minor = minor } public let major: Int public let minor: Int public var description: String { return "\(major).\(minor)" } } ================================================ FILE: Sources/MetalLibraryArchive/LibraryType.swift ================================================ // // File.swift // // // Created by YuAo on 2022/3/22. // import Foundation public enum LibraryType: Int, CustomStringConvertible, CaseIterable, Hashable { case executable = 0 case coreImage = 1 case dynamic = 2 case symbolCompanion = 3 public var description: String { switch self { case .executable: return "Executable" case .coreImage: return "Core Image" case .dynamic: return "Dynamic" case .symbolCompanion: return "Symbol Companion" } } } ================================================ FILE: Sources/MetalLibraryArchive/MetalDataType.swift ================================================ // // MetalDataType.swift // Generated on 2022-04-10 15:38:24 +0000 // public enum MetalDataType: UInt8, Hashable, CaseIterable { case none = 0x00 case `struct` = 0x01 case array = 0x02 case float = 0x03 case float2 = 0x04 case float3 = 0x05 case float4 = 0x06 case float2x2 = 0x07 case float2x3 = 0x08 case float2x4 = 0x09 case float3x2 = 0x0A case float3x3 = 0x0B case float3x4 = 0x0C case float4x2 = 0x0D case float4x3 = 0x0E case float4x4 = 0x0F case half = 0x10 case half2 = 0x11 case half3 = 0x12 case half4 = 0x13 case half2x2 = 0x14 case half2x3 = 0x15 case half2x4 = 0x16 case half3x2 = 0x17 case half3x3 = 0x18 case half3x4 = 0x19 case half4x2 = 0x1A case half4x3 = 0x1B case half4x4 = 0x1C case int = 0x1D case int2 = 0x1E case int3 = 0x1F case int4 = 0x20 case uint = 0x21 case uint2 = 0x22 case uint3 = 0x23 case uint4 = 0x24 case short = 0x25 case short2 = 0x26 case short3 = 0x27 case short4 = 0x28 case ushort = 0x29 case ushort2 = 0x2A case ushort3 = 0x2B case ushort4 = 0x2C case char = 0x2D case char2 = 0x2E case char3 = 0x2F case char4 = 0x30 case uchar = 0x31 case uchar2 = 0x32 case uchar3 = 0x33 case uchar4 = 0x34 case bool = 0x35 case bool2 = 0x36 case bool3 = 0x37 case bool4 = 0x38 case texture = 0x3A case sampler = 0x3B case pointer = 0x3C case r8Unorm = 0x3E case r8Snorm = 0x3F case r16Unorm = 0x40 case r16Snorm = 0x41 case rg8Unorm = 0x42 case rg8Snorm = 0x43 case rg16Unorm = 0x44 case rg16Snorm = 0x45 case rgba8Unorm = 0x46 case rgba8Unorm_sRGB = 0x47 case rgba8Snorm = 0x48 case rgba16Unorm = 0x49 case rgba16Snorm = 0x4A case rgb10A2Unorm = 0x4B case rg11B10Float = 0x4C case rgb9E5Float = 0x4D case renderPipeline = 0x4E case computePipeline = 0x4F case indirectCommandBuffer = 0x50 case long = 0x51 case long2 = 0x52 case long3 = 0x53 case long4 = 0x54 case ulong = 0x55 case ulong2 = 0x56 case ulong3 = 0x57 case ulong4 = 0x58 case double = 0x59 case double2 = 0x5A case double3 = 0x5B case double4 = 0x5C case float8 = 0x5D case float16 = 0x5E case half8 = 0x5F case half16 = 0x60 case int8 = 0x61 case int16 = 0x62 case uint8 = 0x63 case uint16 = 0x64 case short8 = 0x65 case short16 = 0x66 case ushort8 = 0x67 case ushort16 = 0x68 case char8 = 0x69 case char16 = 0x6A case uchar8 = 0x6B case uchar16 = 0x6C case long8 = 0x6D case long16 = 0x6E case ulong8 = 0x6F case ulong16 = 0x70 case double8 = 0x71 case double16 = 0x72 case visibleFunctionTable = 0x73 case intersectionFunctionTable = 0x74 case primitiveAccelerationStructure = 0x75 case instanceAccelerationStructure = 0x76 case bool8 = 0x77 case bool16 = 0x78 } ================================================ FILE: Sources/MetalLibraryArchive/Platform.swift ================================================ // // File.swift // // // Created by YuAo on 2022/3/23. // import Foundation public enum Platform: CustomStringConvertible, CaseIterable, Hashable { case iOS case macOS public var description: String { switch self { case .iOS: return "iOS" case .macOS: return "macOS" } } } ================================================ FILE: Sources/MetalLibraryArchive/SourceArchive.swift ================================================ // // File.swift // // // Created by YuAo on 2022/3/25. // import Foundation public struct SourceArchive: Hashable { public let id: String public let data: Data } ================================================ FILE: Sources/MetalLibraryArchive/Tag.swift ================================================ // // File.swift // // // Created by YuAo on 2022/3/22. // import Foundation public struct Tag: Hashable { public let name: String public let content: Data } ================================================ FILE: Tests/MetalLibraryArchiveTests/Tests.swift ================================================ import XCTest import MetalLibraryArchive struct SDK { var name: String var metalPlatform: String static let macOS = SDK(name: "macosx", metalPlatform: "macos") static let iOS = SDK(name: "iphoneos", metalPlatform: "ios") static let tvOS = SDK(name: "appletvos", metalPlatform: "ios") static let tvOSSimulator = SDK(name: "appletvsimulator", metalPlatform: "ios") static let iOSSimulator = SDK(name: "iphonesimulator", metalPlatform: "ios") static let watchOS = SDK(name: "watchos", metalPlatform: "ios") var targetPlatform: Platform { switch self.metalPlatform { case "macos": return .macOS case "ios": return .iOS default: fatalError() } } } struct ComplieOptions { enum SourceRecordingOption { case none case embeded case separated } var target: String? var coreImageSupportEnabled: Bool = false var libraryType: LibraryType = .executable var languageVersion: LanguageVersion? var installName: String? var sourceRecordingOption: SourceRecordingOption = .none } struct MetalLibraryArchiveTestUtilities { struct Library { var data: Data var symbolCompanionData: Data? } static var temporaryDirectory: URL { let tempDirectoryURL = FileManager.default.temporaryDirectory.appendingPathComponent("MetalLibraryArchiveTests") try? FileManager.default.createDirectory(at: tempDirectoryURL, withIntermediateDirectories: true) return tempDirectoryURL } static func makeLibrary(source: String, sdk: SDK, options: ComplieOptions = ComplieOptions()) throws -> Library { let complierFlags: [String] = try { var flags: [String] = [] if let target = options.target { flags.append(contentsOf: ["-target", target]) } if let version = options.languageVersion { flags.append("-std=\(sdk.metalPlatform)-metal\(version.major).\(version.minor)") } if options.coreImageSupportEnabled { flags.append("-fcikernel") } if options.libraryType == .dynamic { flags.append("-dynamiclib") if let installName = options.installName { flags.append("-install_name") flags.append(installName) } } switch options.sourceRecordingOption { case .none: break case .embeded: flags.append("-frecord-sources") case .separated: if sdk.targetPlatform == .macOS { guard ProcessInfo.processInfo.operatingSystemVersion.majorVersion >= 12 else { throw XCTSkip("The target platform does not support companion metallib.") } } flags.append("-frecord-sources=flat") } return flags }() let tempDirectoryURL = self.temporaryDirectory let workUUID = UUID() let sourceURL = tempDirectoryURL.appendingPathComponent(workUUID.uuidString).appendingPathExtension("metal") let libraryURL = tempDirectoryURL.appendingPathComponent(workUUID.uuidString).appendingPathExtension("metallib") defer { do { try FileManager.default.removeItem(at: sourceURL) try FileManager.default.removeItem(at: libraryURL) } catch {} } try source.write(to: sourceURL, atomically: true, encoding: .utf8) let compileProcess = Process() compileProcess.executableURL = URL(fileURLWithPath: "/usr/bin/xcrun") compileProcess.arguments = ["--sdk", sdk.name, "metal"] + complierFlags + ["-o", libraryURL.path, sourceURL.path] try compileProcess.run() compileProcess.waitUntilExit() XCTAssert(compileProcess.terminationStatus == 0) let data = try Data(contentsOf: libraryURL) var library = Library(data: data) if options.sourceRecordingOption == .separated { library.symbolCompanionData = try Data(contentsOf: libraryURL.deletingPathExtension().appendingPathExtension("metallibsym")) } return library } } class MetalLibraryArchiveTests_macOSSDK: XCTestCase { var sdk: SDK = .macOS private func makeLibrary(source: String, options: ComplieOptions = ComplieOptions()) throws -> Data { return try MetalLibraryArchiveTestUtilities.makeLibrary(source: source, sdk: sdk, options: options).data } func testEmptyLibrary() throws { let data = try self.makeLibrary(source: "") let archive = try Archive(data: data) XCTAssertEqual(archive.targetPlatform, sdk.targetPlatform) XCTAssertEqual(archive.functions.count, 0) } func testVertexFunction() throws { let source = """ #include struct VertexOut { float4 position [[position]]; }; vertex VertexOut testVertex() { VertexOut out = {0}; return out; } """ let data = try self.makeLibrary(source: source) let archive = try Archive(data: data) XCTAssertEqual(archive.targetPlatform, sdk.targetPlatform) XCTAssertEqual(archive.functions.count, 1) let function = try XCTUnwrap(archive.functions.first) XCTAssertEqual(function.name, "testVertex") XCTAssertEqual(function.type, .vertex) XCTAssert(function.bitcode.count > 0) } func testFragmentFunction() throws { let source = """ #include fragment float4 testFragment() { return float4(0); } """ let data = try self.makeLibrary(source: source) let archive = try Archive(data: data) XCTAssertEqual(archive.targetPlatform, sdk.targetPlatform) XCTAssertEqual(archive.functions.count, 1) let function = try XCTUnwrap(archive.functions.first) XCTAssertEqual(function.name, "testFragment") XCTAssertEqual(function.type, .fragment) XCTAssert(function.bitcode.count > 0) } func testKernelFunction() throws { let source = """ #include kernel void testKernel() {} """ let data = try self.makeLibrary(source: source) let archive = try Archive(data: data) XCTAssertEqual(archive.targetPlatform, sdk.targetPlatform) XCTAssertEqual(archive.functions.count, 1) let function = try XCTUnwrap(archive.functions.first) XCTAssertEqual(function.name, "testKernel") XCTAssertEqual(function.type, .kernel) XCTAssert(function.bitcode.count > 0) } func testVisibleFunction() throws { let source = """ #include [[visible]] void test() {} """ let data = try self.makeLibrary(source: source) let archive = try Archive(data: data) XCTAssertEqual(archive.targetPlatform, sdk.targetPlatform) XCTAssertEqual(archive.functions.count, 1) let function = try XCTUnwrap(archive.functions.first) XCTAssertEqual(function.name, "test") XCTAssertEqual(function.type, .visible) XCTAssert(function.bitcode.count > 0) } func testIntersectionFunction() throws { let source = """ #include [[intersection(triangle)]] bool testIntersectionFunction() { return true; } """ let data = try self.makeLibrary(source: source) let archive = try Archive(data: data) XCTAssertEqual(archive.targetPlatform, sdk.targetPlatform) XCTAssertEqual(archive.functions.count, 1) let function = try XCTUnwrap(archive.functions.first) XCTAssertEqual(function.name, "testIntersectionFunction") XCTAssertEqual(function.type, .intersection) XCTAssert(function.bitcode.count > 0) } func testUnqualifiedFunction() throws { let source = """ #include namespace test { void action() {} } """ let data = try self.makeLibrary(source: source, options: ComplieOptions(libraryType: .dynamic, installName: "@executable_path/libtest.metallib")) let archive = try Archive(data: data) XCTAssertEqual(archive.targetPlatform, sdk.targetPlatform) XCTAssertEqual(archive.functions.count, 1) let function = try XCTUnwrap(archive.functions.first) XCTAssertEqual(function.name, "_ZN4test6actionEv") XCTAssertEqual(function.type, .unqualified) XCTAssert(function.bitcode.count > 0) } func testUnqualifiedFunction_globalNamespace() throws { let source = """ #include void test() {} """ let data = try self.makeLibrary(source: source, options: ComplieOptions(libraryType: .dynamic, installName: "@executable_path/libtest.metallib")) let archive = try Archive(data: data) XCTAssertEqual(archive.targetPlatform, sdk.targetPlatform) XCTAssertEqual(archive.functions.count, 1) let function = try XCTUnwrap(archive.functions.first) XCTAssertEqual(function.name, "_Z4testv") XCTAssertEqual(function.type, .unqualified) XCTAssert(function.bitcode.count > 0) } func testUnqualifiedFunction_externC() throws { let source = """ #include extern "C" void test() {} """ let data = try self.makeLibrary(source: source, options: ComplieOptions(libraryType: .dynamic, installName: "@executable_path/libtest.metallib")) let archive = try Archive(data: data) XCTAssertEqual(archive.targetPlatform, sdk.targetPlatform) XCTAssertEqual(archive.functions.count, 1) let function = try XCTUnwrap(archive.functions.first) XCTAssertEqual(function.name, "test") XCTAssertEqual(function.type, .unqualified) XCTAssert(function.bitcode.count > 0) } func testExternFunction() throws { let source = """ #include using namespace metal; #include extern "C" { namespace coreimage { float4 test(sample_t s) { return s.rgba; } }} """ let data = try self.makeLibrary(source: source, options: ComplieOptions(coreImageSupportEnabled: true)) let archive = try Archive(data: data) XCTAssertEqual(archive.targetPlatform, sdk.targetPlatform) XCTAssertEqual(archive.functions.count, 1) XCTAssertEqual(archive.libraryType, .coreImage) let function = try XCTUnwrap(archive.functions.first) XCTAssertEqual(function.name, "test") XCTAssertEqual(function.type, .extern) XCTAssert(function.bitcode.count > 0) XCTAssert(archive.functions[0].publicMetadataTags.contains(where: { $0.name == "RETR" })) XCTAssert(archive.functions[0].publicMetadataTags.contains(where: { $0.name == "ARGR" })) } func testLanguageVersion_2_0() throws { let source = """ #include kernel void testKernel() {} """ let mslVersion = LanguageVersion(major: 2, minor: 0) let data = try self.makeLibrary(source: source, options: ComplieOptions(languageVersion: mslVersion)) let archive = try Archive(data: data) XCTAssertEqual(archive.targetPlatform, sdk.targetPlatform) XCTAssertEqual(archive.functions.count, 1) let function = try XCTUnwrap(archive.functions.first) XCTAssertEqual(function.name, "testKernel") XCTAssertEqual(function.type, .kernel) XCTAssertEqual(function.languageVersion, mslVersion) XCTAssert(function.bitcode.count > 0) } func testLanguageVersion_1_2() throws { let source = """ #include kernel void testKernel() {} """ let mslVersion = LanguageVersion(major: 1, minor: 2) let data = try self.makeLibrary(source: source, options: ComplieOptions(languageVersion: mslVersion)) let archive = try Archive(data: data) XCTAssertEqual(archive.targetPlatform, sdk.targetPlatform) XCTAssertEqual(archive.functions.count, 1) let function = try XCTUnwrap(archive.functions.first) XCTAssertEqual(function.name, "testKernel") XCTAssertEqual(function.type, .kernel) XCTAssertEqual(function.languageVersion, mslVersion) XCTAssert(function.bitcode.count > 0) } func testMultipleFunctions() throws { let source = """ #include struct VertexOut { float4 position [[position]]; }; vertex VertexOut testVertex() { VertexOut out = {0}; return out; } kernel void testKernel() {} fragment float4 testFragment() { return float4(0); } """ let data = try self.makeLibrary(source: source) let archive = try Archive(data: data) XCTAssertEqual(archive.targetPlatform, sdk.targetPlatform) XCTAssertEqual(archive.functions.count, 3) } func testMultipleExternFunctions() throws { let source = """ #include using namespace metal; #include extern "C" { namespace coreimage { float4 test1(sample_t s) { return s.rgba; } void test2() { } void test3() { } }} """ let data = try self.makeLibrary(source: source, options: ComplieOptions(coreImageSupportEnabled: true)) let archive = try Archive(data: data) XCTAssertEqual(archive.targetPlatform, sdk.targetPlatform) XCTAssertEqual(archive.functions.count, 3) } func testQualifiedFunctionNames() throws { let source = """ #include namespace test { struct VertexOut { float4 position [[position]]; }; vertex VertexOut testVertex() { VertexOut out = {0}; return out; } kernel void testKernel() {} fragment float4 testFragment() { return float4(0); } } """ let data = try self.makeLibrary(source: source) let archive = try Archive(data: data) XCTAssertEqual(archive.targetPlatform, sdk.targetPlatform) XCTAssertEqual(archive.functions.count, 3) for function in archive.functions { XCTAssert(function.name.hasPrefix("test::")) } } func testLinkedLibraries() throws { let tempDirectoryURL = MetalLibraryArchiveTestUtilities.temporaryDirectory let workUUID = UUID() let workingDirectory = tempDirectoryURL.appendingPathComponent(workUUID.uuidString) try FileManager.default.createDirectory(at: workingDirectory, withIntermediateDirectories: true) defer { try? FileManager.default.removeItem(at: workingDirectory) } do { let source = """ #include namespace test { float4 blackColor() { return float4(0); } } """ let sourceURL = workingDirectory.appendingPathComponent("dylib.metal") let libraryURL = workingDirectory.appendingPathComponent("libTest.metallib") try source.write(to: sourceURL, atomically: true, encoding: .utf8) let compileProcess = Process() compileProcess.executableURL = URL(fileURLWithPath: "/usr/bin/xcrun") compileProcess.arguments = ["--sdk", sdk.name, "metal", "-dynamiclib", "-install_name", "@executable_path/libTest.metallib", "-o", libraryURL.path, sourceURL.path] try compileProcess.run() compileProcess.waitUntilExit() XCTAssert(compileProcess.terminationStatus == 0) let archive = try Archive(data: Data(contentsOf: libraryURL)) XCTAssertEqual(archive.targetPlatform, sdk.targetPlatform) XCTAssertEqual(archive.libraryType, .dynamic) XCTAssertEqual(archive.functions.count, 1) } do { let source = """ #include namespace test { float4 blackColor(); } kernel void testKernel() { test::blackColor(); } """ let sourceURL = workingDirectory.appendingPathComponent("exe.metal") let libraryURL = workingDirectory.appendingPathComponent("default.metallib") try source.write(to: sourceURL, atomically: true, encoding: .utf8) let compileProcess = Process() compileProcess.executableURL = URL(fileURLWithPath: "/usr/bin/xcrun") compileProcess.arguments = ["--sdk", sdk.name, "metal", "-L", workingDirectory.path, "-lTest", "-o", libraryURL.path, sourceURL.path] try compileProcess.run() compileProcess.waitUntilExit() XCTAssert(compileProcess.terminationStatus == 0) let archive = try Archive(data: Data(contentsOf: libraryURL)) XCTAssertEqual(archive.targetPlatform, sdk.targetPlatform) XCTAssertEqual(archive.libraryType, .executable) } } func testLibraryType() throws { let source = """ #include namespace test { void action() {} } """ do { let data = try self.makeLibrary(source: source, options: ComplieOptions(libraryType: .dynamic, installName: "@executable_path/libtest.metallib")) let archive = try Archive(data: data) XCTAssertEqual(archive.libraryType, .dynamic) } do { let data = try self.makeLibrary(source: source, options: ComplieOptions(libraryType: .executable)) let archive = try Archive(data: data) XCTAssertEqual(archive.targetPlatform, sdk.targetPlatform) XCTAssertEqual(archive.libraryType, .executable) } } func testTessellationFuntion() throws { let source = """ #include using namespace metal; struct ControlPoint { float4 position [[attribute(0)]]; float3 normal [[attribute(1)]]; }; struct PatchIn { patch_control_point controlPoints; }; struct VertexOut { float4 position [[position]]; }; [[patch(quad, 4)]] vertex VertexOut vertex_subdiv_quad(PatchIn patch [[stage_in]], float2 positionInPatch [[position_in_patch]]) { VertexOut out = {0}; return out; } """ let data = try self.makeLibrary(source: source) let archive = try Archive(data: data) XCTAssertEqual(archive.functions.count, 1) XCTAssertEqual(archive.targetPlatform, sdk.targetPlatform) XCTAssertEqual(archive.libraryType, .executable) let tessellationTag = try XCTUnwrap(archive.functions.first?.tags.first(where: { $0.name == "TESS" })) XCTAssertEqual(tessellationTag.content.withUnsafeBytes({ $0.bindMemory(to: UInt8.self)[0] }), 4 << 2 | 2) XCTAssert(archive.functions[0].publicMetadataTags.contains(where: { $0.name == "VATT" })) XCTAssert(archive.functions[0].publicMetadataTags.contains(where: { $0.name == "VATY" })) let vatyTag = try XCTUnwrap(archive.functions.first?.publicMetadataTags.first(where: { $0.name == "VATY" })) XCTAssertEqual(vatyTag.content.withUnsafeBytes({ $0.bindMemory(to: UInt16.self)[0] }), 2) // count XCTAssertEqual(vatyTag.content.withUnsafeBytes({ $0.bindMemory(to: UInt8.self)[2] }), MetalDataType.float4.rawValue) //position XCTAssertEqual(vatyTag.content.withUnsafeBytes({ $0.bindMemory(to: UInt8.self)[3] }), MetalDataType.float3.rawValue) //normal } func testFuntionConstants() throws { let source = """ #include constant int constantValueA [[function_constant(0)]]; kernel void testKernel(device float *io) { for (int i = 0; i < constantValueA; i += 1) { io[i] = 0; } } """ let data = try self.makeLibrary(source: source) let archive = try Archive(data: data) XCTAssertEqual(archive.functions.count, 1) XCTAssertEqual(archive.targetPlatform, sdk.targetPlatform) XCTAssertEqual(archive.libraryType, .executable) XCTAssert(archive.functions[0].publicMetadataTags.contains(where: { $0.name == "CNST" })) } func testFuntionConstants_unused() throws { let source = """ #include constant int constantValueA [[function_constant(0)]]; kernel void testKernel(device float *io) { for (int i = 0; i < constantValueA; i += 1) { break; } } """ let data = try self.makeLibrary(source: source) let archive = try Archive(data: data) XCTAssertEqual(archive.functions.count, 1) XCTAssertEqual(archive.targetPlatform, sdk.targetPlatform) XCTAssertEqual(archive.libraryType, .executable) XCTAssertEqual(archive.functions[0].publicMetadataTags.contains(where: { $0.name == "CNST" }), false) } func testLayeredRendering_uint() throws { let source = """ typedef struct { uint layer [[render_target_array_index]]; float4 position [[position]]; } ColorInOut; vertex ColorInOut vertexTransform(){ ColorInOut out; out.layer = 0; out.position = float4(0); return out; } """ let data = try self.makeLibrary(source: source) let archive = try Archive(data: data) XCTAssertEqual(archive.functions.count, 1) XCTAssertEqual(archive.targetPlatform, sdk.targetPlatform) XCTAssertEqual(archive.libraryType, .executable) let layerTag = try XCTUnwrap(archive.functions.first?.tags.first(where: { $0.name == "LAYR" })) XCTAssertEqual(layerTag.content.withUnsafeBytes({ $0.bindMemory(to: UInt8.self)[0] }), MetalDataType.uint.rawValue) } func testLayeredRendering_ushort() throws { let source = """ typedef struct { ushort layer [[render_target_array_index]]; float4 position [[position]]; } ColorInOut; vertex ColorInOut vertexTransform(){ ColorInOut out; out.layer = 0; out.position = float4(0); return out; } """ let data = try self.makeLibrary(source: source) let archive = try Archive(data: data) XCTAssertEqual(archive.functions.count, 1) XCTAssertEqual(archive.targetPlatform, sdk.targetPlatform) XCTAssertEqual(archive.libraryType, .executable) let layerTag = try XCTUnwrap(archive.functions.first?.tags.first(where: { $0.name == "LAYR" })) XCTAssertEqual(layerTag.content.withUnsafeBytes({ $0.bindMemory(to: UInt8.self)[0] }), MetalDataType.ushort.rawValue) } func testSourceArchives_executable() throws { let source = """ #include kernel void testKernel() {} """ let library = try MetalLibraryArchiveTestUtilities.makeLibrary(source: source, sdk: sdk, options: ComplieOptions(sourceRecordingOption: .embeded)) let archive = try Archive(data: library.data) XCTAssertEqual(archive.functions.count, 1) XCTAssertEqual(archive.targetPlatform, sdk.targetPlatform) XCTAssertEqual(archive.libraryType, .executable) XCTAssert(archive.sourceArchives.count > 0) XCTAssert(archive.functions[0].privateMetadataTags.contains(where: { $0.name == "DEPF" })) } func testSourceArchives_dynamic() throws { let source = """ #include namespace test { void action() {} } """ let library = try MetalLibraryArchiveTestUtilities.makeLibrary(source: source, sdk: sdk, options: ComplieOptions(libraryType: .dynamic, installName: "@executable_path/libtest.metallib", sourceRecordingOption: .embeded)) let archive = try Archive(data: library.data) XCTAssertEqual(archive.functions.count, 1) XCTAssertEqual(archive.targetPlatform, sdk.targetPlatform) XCTAssertEqual(archive.libraryType, .dynamic) XCTAssert(archive.sourceArchives.count > 0) } func testSourceArchives_executable_sym() throws { let source = """ #include kernel void testKernel() {} """ let library = try MetalLibraryArchiveTestUtilities.makeLibrary(source: source, sdk: sdk, options: ComplieOptions(sourceRecordingOption: .separated)) let archive = try Archive(data: library.data) XCTAssertEqual(archive.functions.count, 1) XCTAssertEqual(archive.targetPlatform, sdk.targetPlatform) XCTAssertEqual(archive.libraryType, .executable) XCTAssertEqual(archive.sourceArchives.count, 0) let symArchive = try Archive(data: XCTUnwrap(library.symbolCompanionData)) XCTAssertEqual(symArchive.functions.count, 1) XCTAssertEqual(symArchive.targetPlatform, sdk.targetPlatform) XCTAssertEqual(symArchive.libraryType, .symbolCompanion) XCTAssert(symArchive.sourceArchives.count > 0) } func testSourceArchives_dynamic_sym() throws { let source = """ #include namespace test { void action() {} } """ let library = try MetalLibraryArchiveTestUtilities.makeLibrary(source: source, sdk: sdk, options: ComplieOptions(libraryType: .dynamic, installName: "@executable_path/libtest.metallib", sourceRecordingOption: .separated)) let archive = try Archive(data: library.data) XCTAssertEqual(archive.functions.count, 1) XCTAssertEqual(archive.targetPlatform, sdk.targetPlatform) XCTAssertEqual(archive.libraryType, .dynamic) XCTAssertEqual(archive.sourceArchives.count, 0) let symArchive = try Archive(data: XCTUnwrap(library.symbolCompanionData)) XCTAssertEqual(symArchive.functions.count, 1) XCTAssertEqual(symArchive.targetPlatform, sdk.targetPlatform) XCTAssertEqual(symArchive.libraryType, .symbolCompanion) XCTAssert(symArchive.sourceArchives.count > 0) } } class MetalLibraryArchiveTests_iOSSDK: MetalLibraryArchiveTests_macOSSDK { override func setUp() { self.sdk = .iOS super.setUp() } } class MetalLibraryArchiveTests_tvOSSDK: MetalLibraryArchiveTests_macOSSDK { override func setUp() { self.sdk = .tvOS super.setUp() } } class MetalLibraryArchiveTests_iOSSimulatorSDK: MetalLibraryArchiveTests_macOSSDK { override func setUp() { self.sdk = .iOSSimulator super.setUp() } } class MetalLibraryArchiveTests_tvOSSimulatorSDK: MetalLibraryArchiveTests_macOSSDK { override func setUp() { self.sdk = .tvOSSimulator super.setUp() } } class MetalLibraryArchiveTests: XCTestCase { func testInvalidLibraryData() throws { do { let data = Data() XCTAssertThrowsError(try Archive(data: data)) } do { let data = "MTLB".data(using: .utf8)! XCTAssertThrowsError(try Archive(data: data)) } } } class MetalLibraryArchiveTests_TargetPlatformBug: XCTestCase { func testEmptyLibraryTargetPlatform_expectFailure() throws { let library = try MetalLibraryArchiveTestUtilities.makeLibrary(source: "", sdk: .iOS, options: ComplieOptions(target: "air64-apple-ios14.0")) let archive = try Archive(data: library.data) XCTExpectFailure(failingBlock: { XCTAssertEqual(archive.targetPlatform, .iOS) }) } func testEmptyLibraryTargetPlatform_expectSuccess() throws { let library = try MetalLibraryArchiveTestUtilities.makeLibrary(source: "", sdk: .iOS, options: ComplieOptions(target: "air64-apple-ios15.0")) let archive = try Archive(data: library.data) XCTAssertEqual(archive.targetPlatform, .iOS) } } class MetalLibraryArchiveTests_DeploymentTarget: XCTestCase { private let source = """ #include kernel void testKernel() {} """ func testDeploymentTarget_macOS_12() throws { let library = try MetalLibraryArchiveTestUtilities.makeLibrary(source: source, sdk: .macOS, options: ComplieOptions(target: "air64-apple-macos12.0")) let archive = try Archive(data: library.data) XCTAssertEqual(archive.targetPlatform, .macOS) XCTAssertEqual(archive.deploymentTarget?.operatingSystem, .macOS) XCTAssertEqual(archive.deploymentTarget?.operatingSystemVersion.description, "12.0") } func testDeploymentTarget_iOS_15() throws { let library = try MetalLibraryArchiveTestUtilities.makeLibrary(source: source, sdk: .iOS, options: ComplieOptions(target: "air64-apple-ios15.0")) let archive = try Archive(data: library.data) XCTAssertEqual(archive.targetPlatform, .iOS) XCTAssertEqual(archive.deploymentTarget?.operatingSystem, .iOS) XCTAssertEqual(archive.deploymentTarget?.operatingSystemVersion.description, "15.0") } func testDeploymentTarget_iOS_15_1() throws { let library = try MetalLibraryArchiveTestUtilities.makeLibrary(source: source, sdk: .iOS, options: ComplieOptions(target: "air64-apple-ios15.1")) let archive = try Archive(data: library.data) XCTAssertEqual(archive.targetPlatform, .iOS) XCTAssertEqual(archive.deploymentTarget?.operatingSystem, .iOS) XCTAssertEqual(archive.deploymentTarget?.operatingSystemVersion.description, "15.1") } func testDeploymentTarget_iOS_11() throws { let library = try MetalLibraryArchiveTestUtilities.makeLibrary(source: source, sdk: .iOS, options: ComplieOptions(target: "air64-apple-ios11.0")) let archive = try Archive(data: library.data) XCTAssertEqual(archive.targetPlatform, .iOS) XCTAssertEqual(archive.deploymentTarget, nil) } func testDeploymentTarget_watchOS_8_5() throws { let library = try MetalLibraryArchiveTestUtilities.makeLibrary(source: source, sdk: .watchOS, options: ComplieOptions(target: "air64-apple-watchos8.5")) let archive = try Archive(data: library.data) XCTAssertEqual(archive.targetPlatform, .iOS) XCTAssertEqual(archive.deploymentTarget?.operatingSystem, .watchOS) XCTAssertEqual(archive.deploymentTarget?.operatingSystemVersion.description, "8.5") } func testDeploymentTarget_iOSSimulator_15_2() throws { let library = try MetalLibraryArchiveTestUtilities.makeLibrary(source: source, sdk: .iOSSimulator, options: ComplieOptions(target: "air64-apple-ios15.2-simulator")) let archive = try Archive(data: library.data) XCTAssertEqual(archive.targetPlatform, .iOS) XCTAssertEqual(archive.deploymentTarget?.operatingSystem, .iOSSimulator) XCTAssertEqual(archive.deploymentTarget?.operatingSystemVersion.description, "15.2") } func testDeploymentTarget_macCatalyst_15() throws { let library = try MetalLibraryArchiveTestUtilities.makeLibrary(source: source, sdk: .iOS, options: ComplieOptions(target: "air64-apple-ios15.0-macabi")) let archive = try Archive(data: library.data) XCTAssertEqual(archive.targetPlatform, .macOS) XCTAssertEqual(archive.deploymentTarget?.operatingSystem, .macCatalyst) XCTAssertEqual(archive.deploymentTarget?.operatingSystemVersion.description, "15.0") } } ================================================ FILE: Utilities/Package.resolved ================================================ { "object": { "pins": [ { "package": "swift-argument-parser", "repositoryURL": "https://github.com/apple/swift-argument-parser", "state": { "branch": null, "revision": "82905286cc3f0fa8adc4674bf49437cab65a8373", "version": "1.1.1" } } ] }, "version": 1 } ================================================ FILE: Utilities/Package.swift ================================================ // swift-tools-version:5.4 // The swift-tools-version declares the minimum version of Swift required to build this package. import PackageDescription let package = Package( name: "Utilities", platforms: [.macOS(.v10_13)], products: [ .executable(name: "metal-data-type-tools", targets: ["MetalDataTypeTools"]) ], dependencies: [ .package(url: "https://github.com/apple/swift-argument-parser", from: "1.0.0") ], targets: [ .executableTarget( name: "MetalDataTypeTools", dependencies: [ .product(name: "ArgumentParser", package: "swift-argument-parser") ]) ] ) ================================================ FILE: Utilities/Sources/MetalDataTypeTools/Command.swift ================================================ // // File.swift // // // Created by YuAo on 2020/3/16. // import Foundation import ArgumentParser extension URL: ExpressibleByArgument { public init?(argument: String) { if let url = URL(string: argument), url.scheme != nil { self.init(string: argument) } else { //Assuming it is a file url. self.init(fileURLWithPath: argument) } } } struct GenerateMetalDataTypeTableMarkdown: ParsableCommand { static let configuration: CommandConfiguration = CommandConfiguration(commandName: "gen-markdown") @Option var columns: Int = 1 func run() throws { let rows = (MetalDataType.allTypes.count + columns - 1) / columns var table: String = "" for _ in 0.. = ["struct", "enum", "class"] let code = """ // // MetalDataType.swift // Generated on \(Date()) // public enum MetalDataType: UInt8, Hashable, CaseIterable { \(MetalDataType.allTypes.map({ type in if keywords.contains(type.camelCaseDescription) { return " case `\(type.camelCaseDescription)` = \(type.hexID)" } else { return " case \(type.camelCaseDescription) = \(type.hexID)" } }).joined(separator: "\n")) } """ print(code) } } @main struct MetalDataTypeTools: ParsableCommand { static let configuration: CommandConfiguration = CommandConfiguration(subcommands: [GenerateMetalDataTypeDefinition.self, GenerateMetalDataTypeTableMarkdown.self]) } ================================================ FILE: Utilities/Sources/MetalDataTypeTools/MetalDataTypes.swift ================================================ // // File.swift // // // Created by YuAo on 2022/4/10. // import Foundation import Metal // https://github.com/nst/iOS-Runtime-Headers/blob/fbb634c78269b0169efdead80955ba64eaaa2f21/Frameworks/Metal.framework/MTLTypeInternal.h @objc private protocol MTLTypeInternalProtocol: NSObjectProtocol { init(dataType: UInt64) var dataType: UInt64 { get } } struct MetalDataType { var id: UInt8 var description: String } extension MetalDataType { static let allTypes: [MetalDataType] = (UInt8.min...UInt8.max).compactMap({ id in struct Static { static let MTLTypeInternal: MTLTypeInternalProtocol.Type = { let type: AnyClass = NSClassFromString("MTLTypeInternal")! class_addProtocol(type, MTLTypeInternalProtocol.self) return type as! MTLTypeInternalProtocol.Type }() } let type = Static.MTLTypeInternal.init(dataType: UInt64(id)) if !type.description.isEmpty && type.description != "Unknown" { let prefix = "MTLDataType" precondition(type.description.hasPrefix(prefix)) let description = String(type.description.dropFirst(prefix.count)) return MetalDataType(id: id, description: description) } else { return nil } }) } extension MetalDataType { var hexID: String { if id < 16 { return "0x0\(String(id, radix: 16, uppercase: true))" } else { return "0x\(String(id, radix: 16, uppercase: true))" } } var camelCaseDescription: String { let prefix = description.prefix(while: { $0.isUppercase }) return prefix.lowercased() + description.dropFirst(prefix.count) } }