[ { "path": ".gitignore", "content": "Examples/Official/maple-diffusion/bins/*.txt\nExamples/Official/maple-diffusion/bins/*.bin\nxcuserdata/\n.DS_Store\n" }, { "path": ".swiftpm/xcode/package.xcworkspace/contents.xcworkspacedata", "content": "\n\n \n \n\n" }, { "path": "Converter Script/native-convert.py", "content": "#!/usr/bin/env python3\nimport sys\nif len(sys.argv) < 2: raise ValueError(f\"Usage: {sys.argv[0]} path_to_ckpt\")\n\nfrom pathlib import Path\nimport torch as th\nimport numpy as np\n\nckpt = th.load(sys.argv[1], map_location=\"cpu\")\noutpath = Path(\"./bins\")\noutpath.mkdir(exist_ok=True)\n\n# vocab for clip\nvocab_url = \"https://openaipublic.blob.core.windows.net/clip/bpe_simple_vocab_16e6.txt\"\nvocab_dest = outpath / vocab_url.split(\"/\")[-1]\nif not vocab_dest.exists():\n print(\"downloading clip vocab\")\n import requests\n with requests.get(vocab_url, stream=True) as r:\n assert r.status_code == 200, f\"{vocab_url} failed to download. please copy it to {vocab_dest} manually.\"\n with vocab_dest.open('wb') as vf:\n for c in r.iter_content(chunk_size=8192):\n vf.write(c)\n print(\"downloaded clip vocab\")\n\n# model weights\nfor k, v in ckpt[\"state_dict\"].items():\n if not hasattr(v, \"numpy\"): continue\n v.numpy().astype('float16').tofile(outpath / (k + \".bin\"))\n print(\"exporting state_dict\", k, end=\"\\r\")\nprint(\"\\nexporting other stuff...\")\n\n# other stuff\nth.exp(-th.log(th.tensor([10000])) * th.arange(0, 160) / 160).numpy().tofile(outpath / \"temb_coefficients_fp32.bin\")\nnp.triu(np.ones((1,1,77,77), dtype=np.float16) * -65500.0, k=1).astype(np.float16).tofile(outpath / \"causal_mask.bin\")\nnp.array([0.14013671875, 0.0711669921875, -0.03271484375, -0.11407470703125, 0.126220703125, 0.10101318359375, 0.034515380859375, -0.1383056640625, 0.126220703125, 0.07733154296875, 0.042633056640625, -0.177978515625]).astype(np.float16).tofile(outpath / \"aux_output_conv.weight.bin\")\nnp.array([0.423828125, 0.471923828125, 0.473876953125]).astype(np.float16).tofile(outpath / \"aux_output_conv.bias.bin\")\nprint(f\"Done!\")\n" }, { "path": "Examples/Simple/SimpleDiffusion/SimpleDiffusion/Assets.xcassets/AccentColor.colorset/Contents.json", "content": "{\n \"colors\" : [\n {\n \"idiom\" : \"universal\"\n }\n ],\n \"info\" : {\n \"author\" : \"xcode\",\n \"version\" : 1\n }\n}\n" }, { "path": "Examples/Simple/SimpleDiffusion/SimpleDiffusion/Assets.xcassets/AppIcon.appiconset/Contents.json", "content": "{\n \"images\" : [\n {\n \"idiom\" : \"mac\",\n \"scale\" : \"1x\",\n \"size\" : \"16x16\"\n },\n {\n \"idiom\" : \"mac\",\n \"scale\" : \"2x\",\n \"size\" : \"16x16\"\n },\n {\n \"idiom\" : \"mac\",\n \"scale\" : \"1x\",\n \"size\" : \"32x32\"\n },\n {\n \"idiom\" : \"mac\",\n \"scale\" : \"2x\",\n \"size\" : \"32x32\"\n },\n {\n \"idiom\" : \"mac\",\n \"scale\" : \"1x\",\n \"size\" : \"128x128\"\n },\n {\n \"idiom\" : \"mac\",\n \"scale\" : \"2x\",\n \"size\" : \"128x128\"\n },\n {\n \"idiom\" : \"mac\",\n \"scale\" : \"1x\",\n \"size\" : \"256x256\"\n },\n {\n \"idiom\" : \"mac\",\n \"scale\" : \"2x\",\n \"size\" : \"256x256\"\n },\n {\n \"idiom\" : \"mac\",\n \"scale\" : \"1x\",\n \"size\" : \"512x512\"\n },\n {\n \"idiom\" : \"mac\",\n \"scale\" : \"2x\",\n \"size\" : \"512x512\"\n }\n ],\n \"info\" : {\n \"author\" : \"xcode\",\n \"version\" : 1\n }\n}\n" }, { "path": "Examples/Simple/SimpleDiffusion/SimpleDiffusion/Assets.xcassets/Contents.json", "content": "{\n \"info\" : {\n \"author\" : \"xcode\",\n \"version\" : 1\n }\n}\n" }, { "path": "Examples/Simple/SimpleDiffusion/SimpleDiffusion/ContentView.swift", "content": "//\n// ContentView.swift\n// SimpleDiffusion\n//\n// Created by Morten Just on 10/21/22.\n//\n\nimport SwiftUI\nimport MapleDiffusion\nimport Combine\n\nstruct ContentView: View {\n \n // 1\n @StateObject var sd = Diffusion()\n @State var prompt = \"\"\n @State var steps = 20\n @State var guidance : Double = 7.5\n @State var image : CGImage?\n @State var inputImage: CGImage?\n @State var imagePublisher = Diffusion.placeholderPublisher\n @State var progress : Double = 0\n \n var anyProgress : Double { sd.loadingProgress < 1 ? sd.loadingProgress : progress }\n\n var body: some View {\n VStack {\n DiffusionImage(image: $image, inputImage: $inputImage, progress: $progress)\n Spacer()\n TextField(\"Prompt\", text: $prompt)\n // 3\n .onSubmit {\n let input = SampleInput(prompt: prompt,\n initImage: inputImage,\n steps: steps)\n self.imagePublisher = sd.generate(input: input)\n \n }\n .disabled(!sd.isModelReady)\n \n HStack {\n TextField(\"Steps\", value: $steps, formatter: NumberFormatter())\n TextField(\"Guidance\", value: $guidance, formatter: NumberFormatter())\n }\n \n ProgressView(value: anyProgress)\n .opacity(anyProgress == 1 || anyProgress == 0 ? 0 : 1)\n }\n .task {\n // 2\n let path = URL(string: \"http://localhost:8080/Diffusion.zip\")!\n do {\n try await sd.prepModels(remoteURL: path)\n } catch {\n assertionFailure(\"Hi, developer. You most likely don't have a local webserver running that serves the zip file with the transformed model files. See README.md\")\n }\n }\n \n // 4\n .onReceive(imagePublisher) { r in\n self.image = r.image\n self.progress = r.progress\n }\n .frame(minWidth: 200, minHeight: 200)\n }\n}\n\nstruct ContentView_Previews: PreviewProvider {\n static var previews: some View {\n ContentView()\n }\n}\n\n\n\nextension URL {\n static var modelFolder = FileManager.default\n .urls(for: .applicationSupportDirectory, in: .userDomainMask)[0]\n .appendingPathComponent(\"Photato/bins\")\n}\n" }, { "path": "Examples/Simple/SimpleDiffusion/SimpleDiffusion/Preview Content/Preview Assets.xcassets/Contents.json", "content": "{\n \"info\" : {\n \"author\" : \"xcode\",\n \"version\" : 1\n }\n}\n" }, { "path": "Examples/Simple/SimpleDiffusion/SimpleDiffusion/SimpleDiffusion.entitlements", "content": "\n\n\n\n\n" }, { "path": "Examples/Simple/SimpleDiffusion/SimpleDiffusion/SimpleDiffusionApp.swift", "content": "//\n// SimpleDiffusionApp.swift\n// SimpleDiffusion\n//\n// Created by Morten Just on 10/21/22.\n//\n\nimport SwiftUI\n\n@main\nstruct SimpleDiffusionApp: App {\n var body: some Scene {\n WindowGroup {\n ContentView()\n .ignoresSafeArea()\n }.windowStyle(.hiddenTitleBar)\n }\n}\n" }, { "path": "Examples/Simple/SimpleDiffusion/SimpleDiffusion.xcodeproj/project.pbxproj", "content": "// !$*UTF8*$!\n{\n\tarchiveVersion = 1;\n\tclasses = {\n\t};\n\tobjectVersion = 56;\n\tobjects = {\n\n/* Begin PBXBuildFile section */\n\t\tC858BB302902A59000DF30AE /* SimpleDiffusionApp.swift in Sources */ = {isa = PBXBuildFile; fileRef = C858BB2F2902A59000DF30AE /* SimpleDiffusionApp.swift */; };\n\t\tC858BB322902A59000DF30AE /* ContentView.swift in Sources */ = {isa = PBXBuildFile; fileRef = C858BB312902A59000DF30AE /* ContentView.swift */; };\n\t\tC858BB342902A59100DF30AE /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = C858BB332902A59100DF30AE /* Assets.xcassets */; };\n\t\tC858BB372902A59100DF30AE /* Preview Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = C858BB362902A59100DF30AE /* Preview Assets.xcassets */; };\n\t\tC858BB412902A68200DF30AE /* MapleDiffusion in Frameworks */ = {isa = PBXBuildFile; productRef = C858BB402902A68200DF30AE /* MapleDiffusion */; };\n/* End PBXBuildFile section */\n\n/* Begin PBXFileReference section */\n\t\tC858BB2C2902A59000DF30AE /* SimpleDiffusion.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = SimpleDiffusion.app; sourceTree = BUILT_PRODUCTS_DIR; };\n\t\tC858BB2F2902A59000DF30AE /* SimpleDiffusionApp.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SimpleDiffusionApp.swift; sourceTree = \"\"; };\n\t\tC858BB312902A59000DF30AE /* ContentView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ContentView.swift; sourceTree = \"\"; };\n\t\tC858BB332902A59100DF30AE /* Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = Assets.xcassets; sourceTree = \"\"; };\n\t\tC858BB362902A59100DF30AE /* Preview Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = \"Preview Assets.xcassets\"; sourceTree = \"\"; };\n\t\tC858BB382902A59100DF30AE /* SimpleDiffusion.entitlements */ = {isa = PBXFileReference; lastKnownFileType = text.plist.entitlements; path = SimpleDiffusion.entitlements; sourceTree = \"\"; };\n\t\tC858BB3E2902A63C00DF30AE /* maple-diffusion-package */ = {isa = PBXFileReference; lastKnownFileType = wrapper; name = \"maple-diffusion-package\"; path = ../../..; sourceTree = \"\"; };\n/* End PBXFileReference section */\n\n/* Begin PBXFrameworksBuildPhase section */\n\t\tC858BB292902A59000DF30AE /* Frameworks */ = {\n\t\t\tisa = PBXFrameworksBuildPhase;\n\t\t\tbuildActionMask = 2147483647;\n\t\t\tfiles = (\n\t\t\t\tC858BB412902A68200DF30AE /* MapleDiffusion in Frameworks */,\n\t\t\t);\n\t\t\trunOnlyForDeploymentPostprocessing = 0;\n\t\t};\n/* End PBXFrameworksBuildPhase section */\n\n/* Begin PBXGroup section */\n\t\tC858BB232902A59000DF30AE = {\n\t\t\tisa = PBXGroup;\n\t\t\tchildren = (\n\t\t\t\tC858BB3E2902A63C00DF30AE /* maple-diffusion-package */,\n\t\t\t\tC858BB2E2902A59000DF30AE /* SimpleDiffusion */,\n\t\t\t\tC858BB2D2902A59000DF30AE /* Products */,\n\t\t\t\tC858BB3F2902A68200DF30AE /* Frameworks */,\n\t\t\t);\n\t\t\tsourceTree = \"\";\n\t\t};\n\t\tC858BB2D2902A59000DF30AE /* Products */ = {\n\t\t\tisa = PBXGroup;\n\t\t\tchildren = (\n\t\t\t\tC858BB2C2902A59000DF30AE /* SimpleDiffusion.app */,\n\t\t\t);\n\t\t\tname = Products;\n\t\t\tsourceTree = \"\";\n\t\t};\n\t\tC858BB2E2902A59000DF30AE /* SimpleDiffusion */ = {\n\t\t\tisa = PBXGroup;\n\t\t\tchildren = (\n\t\t\t\tC858BB2F2902A59000DF30AE /* SimpleDiffusionApp.swift */,\n\t\t\t\tC858BB312902A59000DF30AE /* ContentView.swift */,\n\t\t\t\tC858BB332902A59100DF30AE /* Assets.xcassets */,\n\t\t\t\tC858BB382902A59100DF30AE /* SimpleDiffusion.entitlements */,\n\t\t\t\tC858BB352902A59100DF30AE /* Preview Content */,\n\t\t\t);\n\t\t\tpath = SimpleDiffusion;\n\t\t\tsourceTree = \"\";\n\t\t};\n\t\tC858BB352902A59100DF30AE /* Preview Content */ = {\n\t\t\tisa = PBXGroup;\n\t\t\tchildren = (\n\t\t\t\tC858BB362902A59100DF30AE /* Preview Assets.xcassets */,\n\t\t\t);\n\t\t\tpath = \"Preview Content\";\n\t\t\tsourceTree = \"\";\n\t\t};\n\t\tC858BB3F2902A68200DF30AE /* Frameworks */ = {\n\t\t\tisa = PBXGroup;\n\t\t\tchildren = (\n\t\t\t);\n\t\t\tname = Frameworks;\n\t\t\tsourceTree = \"\";\n\t\t};\n/* End PBXGroup section */\n\n/* Begin PBXNativeTarget section */\n\t\tC858BB2B2902A59000DF30AE /* SimpleDiffusion */ = {\n\t\t\tisa = PBXNativeTarget;\n\t\t\tbuildConfigurationList = C858BB3B2902A59100DF30AE /* Build configuration list for PBXNativeTarget \"SimpleDiffusion\" */;\n\t\t\tbuildPhases = (\n\t\t\t\tC858BB282902A59000DF30AE /* Sources */,\n\t\t\t\tC858BB292902A59000DF30AE /* Frameworks */,\n\t\t\t\tC858BB2A2902A59000DF30AE /* Resources */,\n\t\t\t);\n\t\t\tbuildRules = (\n\t\t\t);\n\t\t\tdependencies = (\n\t\t\t);\n\t\t\tname = SimpleDiffusion;\n\t\t\tpackageProductDependencies = (\n\t\t\t\tC858BB402902A68200DF30AE /* MapleDiffusion */,\n\t\t\t);\n\t\t\tproductName = SimpleDiffusion;\n\t\t\tproductReference = C858BB2C2902A59000DF30AE /* SimpleDiffusion.app */;\n\t\t\tproductType = \"com.apple.product-type.application\";\n\t\t};\n/* End PBXNativeTarget section */\n\n/* Begin PBXProject section */\n\t\tC858BB242902A59000DF30AE /* Project object */ = {\n\t\t\tisa = PBXProject;\n\t\t\tattributes = {\n\t\t\t\tBuildIndependentTargetsInParallel = 1;\n\t\t\t\tLastSwiftUpdateCheck = 1400;\n\t\t\t\tLastUpgradeCheck = 1400;\n\t\t\t\tTargetAttributes = {\n\t\t\t\t\tC858BB2B2902A59000DF30AE = {\n\t\t\t\t\t\tCreatedOnToolsVersion = 14.0.1;\n\t\t\t\t\t};\n\t\t\t\t};\n\t\t\t};\n\t\t\tbuildConfigurationList = C858BB272902A59000DF30AE /* Build configuration list for PBXProject \"SimpleDiffusion\" */;\n\t\t\tcompatibilityVersion = \"Xcode 14.0\";\n\t\t\tdevelopmentRegion = en;\n\t\t\thasScannedForEncodings = 0;\n\t\t\tknownRegions = (\n\t\t\t\ten,\n\t\t\t\tBase,\n\t\t\t);\n\t\t\tmainGroup = C858BB232902A59000DF30AE;\n\t\t\tproductRefGroup = C858BB2D2902A59000DF30AE /* Products */;\n\t\t\tprojectDirPath = \"\";\n\t\t\tprojectRoot = \"\";\n\t\t\ttargets = (\n\t\t\t\tC858BB2B2902A59000DF30AE /* SimpleDiffusion */,\n\t\t\t);\n\t\t};\n/* End PBXProject section */\n\n/* Begin PBXResourcesBuildPhase section */\n\t\tC858BB2A2902A59000DF30AE /* Resources */ = {\n\t\t\tisa = PBXResourcesBuildPhase;\n\t\t\tbuildActionMask = 2147483647;\n\t\t\tfiles = (\n\t\t\t\tC858BB372902A59100DF30AE /* Preview Assets.xcassets in Resources */,\n\t\t\t\tC858BB342902A59100DF30AE /* Assets.xcassets in Resources */,\n\t\t\t);\n\t\t\trunOnlyForDeploymentPostprocessing = 0;\n\t\t};\n/* End PBXResourcesBuildPhase section */\n\n/* Begin PBXSourcesBuildPhase section */\n\t\tC858BB282902A59000DF30AE /* Sources */ = {\n\t\t\tisa = PBXSourcesBuildPhase;\n\t\t\tbuildActionMask = 2147483647;\n\t\t\tfiles = (\n\t\t\t\tC858BB322902A59000DF30AE /* ContentView.swift in Sources */,\n\t\t\t\tC858BB302902A59000DF30AE /* SimpleDiffusionApp.swift in Sources */,\n\t\t\t);\n\t\t\trunOnlyForDeploymentPostprocessing = 0;\n\t\t};\n/* End PBXSourcesBuildPhase section */\n\n/* Begin XCBuildConfiguration section */\n\t\tC858BB392902A59100DF30AE /* Debug */ = {\n\t\t\tisa = XCBuildConfiguration;\n\t\t\tbuildSettings = {\n\t\t\t\tALWAYS_SEARCH_USER_PATHS = NO;\n\t\t\t\tCLANG_ANALYZER_NONNULL = YES;\n\t\t\t\tCLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;\n\t\t\t\tCLANG_CXX_LANGUAGE_STANDARD = \"gnu++20\";\n\t\t\t\tCLANG_ENABLE_MODULES = YES;\n\t\t\t\tCLANG_ENABLE_OBJC_ARC = YES;\n\t\t\t\tCLANG_ENABLE_OBJC_WEAK = YES;\n\t\t\t\tCLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;\n\t\t\t\tCLANG_WARN_BOOL_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_COMMA = YES;\n\t\t\t\tCLANG_WARN_CONSTANT_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES;\n\t\t\t\tCLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;\n\t\t\t\tCLANG_WARN_DOCUMENTATION_COMMENTS = YES;\n\t\t\t\tCLANG_WARN_EMPTY_BODY = YES;\n\t\t\t\tCLANG_WARN_ENUM_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_INFINITE_RECURSION = YES;\n\t\t\t\tCLANG_WARN_INT_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES;\n\t\t\t\tCLANG_WARN_OBJC_LITERAL_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;\n\t\t\t\tCLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES;\n\t\t\t\tCLANG_WARN_RANGE_LOOP_ANALYSIS = YES;\n\t\t\t\tCLANG_WARN_STRICT_PROTOTYPES = YES;\n\t\t\t\tCLANG_WARN_SUSPICIOUS_MOVE = YES;\n\t\t\t\tCLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE;\n\t\t\t\tCLANG_WARN_UNREACHABLE_CODE = YES;\n\t\t\t\tCLANG_WARN__DUPLICATE_METHOD_MATCH = YES;\n\t\t\t\tCOPY_PHASE_STRIP = NO;\n\t\t\t\tDEBUG_INFORMATION_FORMAT = dwarf;\n\t\t\t\tENABLE_STRICT_OBJC_MSGSEND = YES;\n\t\t\t\tENABLE_TESTABILITY = YES;\n\t\t\t\tGCC_C_LANGUAGE_STANDARD = gnu11;\n\t\t\t\tGCC_DYNAMIC_NO_PIC = NO;\n\t\t\t\tGCC_NO_COMMON_BLOCKS = YES;\n\t\t\t\tGCC_OPTIMIZATION_LEVEL = 0;\n\t\t\t\tGCC_PREPROCESSOR_DEFINITIONS = (\n\t\t\t\t\t\"DEBUG=1\",\n\t\t\t\t\t\"$(inherited)\",\n\t\t\t\t);\n\t\t\t\tGCC_WARN_64_TO_32_BIT_CONVERSION = YES;\n\t\t\t\tGCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;\n\t\t\t\tGCC_WARN_UNDECLARED_SELECTOR = YES;\n\t\t\t\tGCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;\n\t\t\t\tGCC_WARN_UNUSED_FUNCTION = YES;\n\t\t\t\tGCC_WARN_UNUSED_VARIABLE = YES;\n\t\t\t\tMACOSX_DEPLOYMENT_TARGET = 12.3;\n\t\t\t\tMTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE;\n\t\t\t\tMTL_FAST_MATH = YES;\n\t\t\t\tONLY_ACTIVE_ARCH = YES;\n\t\t\t\tSDKROOT = macosx;\n\t\t\t\tSWIFT_ACTIVE_COMPILATION_CONDITIONS = DEBUG;\n\t\t\t\tSWIFT_OPTIMIZATION_LEVEL = \"-Onone\";\n\t\t\t};\n\t\t\tname = Debug;\n\t\t};\n\t\tC858BB3A2902A59100DF30AE /* Release */ = {\n\t\t\tisa = XCBuildConfiguration;\n\t\t\tbuildSettings = {\n\t\t\t\tALWAYS_SEARCH_USER_PATHS = NO;\n\t\t\t\tCLANG_ANALYZER_NONNULL = YES;\n\t\t\t\tCLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;\n\t\t\t\tCLANG_CXX_LANGUAGE_STANDARD = \"gnu++20\";\n\t\t\t\tCLANG_ENABLE_MODULES = YES;\n\t\t\t\tCLANG_ENABLE_OBJC_ARC = YES;\n\t\t\t\tCLANG_ENABLE_OBJC_WEAK = YES;\n\t\t\t\tCLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;\n\t\t\t\tCLANG_WARN_BOOL_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_COMMA = YES;\n\t\t\t\tCLANG_WARN_CONSTANT_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES;\n\t\t\t\tCLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;\n\t\t\t\tCLANG_WARN_DOCUMENTATION_COMMENTS = YES;\n\t\t\t\tCLANG_WARN_EMPTY_BODY = YES;\n\t\t\t\tCLANG_WARN_ENUM_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_INFINITE_RECURSION = YES;\n\t\t\t\tCLANG_WARN_INT_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES;\n\t\t\t\tCLANG_WARN_OBJC_LITERAL_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;\n\t\t\t\tCLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES;\n\t\t\t\tCLANG_WARN_RANGE_LOOP_ANALYSIS = YES;\n\t\t\t\tCLANG_WARN_STRICT_PROTOTYPES = YES;\n\t\t\t\tCLANG_WARN_SUSPICIOUS_MOVE = YES;\n\t\t\t\tCLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE;\n\t\t\t\tCLANG_WARN_UNREACHABLE_CODE = YES;\n\t\t\t\tCLANG_WARN__DUPLICATE_METHOD_MATCH = YES;\n\t\t\t\tCOPY_PHASE_STRIP = NO;\n\t\t\t\tDEBUG_INFORMATION_FORMAT = \"dwarf-with-dsym\";\n\t\t\t\tENABLE_NS_ASSERTIONS = NO;\n\t\t\t\tENABLE_STRICT_OBJC_MSGSEND = YES;\n\t\t\t\tGCC_C_LANGUAGE_STANDARD = gnu11;\n\t\t\t\tGCC_NO_COMMON_BLOCKS = YES;\n\t\t\t\tGCC_WARN_64_TO_32_BIT_CONVERSION = YES;\n\t\t\t\tGCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;\n\t\t\t\tGCC_WARN_UNDECLARED_SELECTOR = YES;\n\t\t\t\tGCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;\n\t\t\t\tGCC_WARN_UNUSED_FUNCTION = YES;\n\t\t\t\tGCC_WARN_UNUSED_VARIABLE = YES;\n\t\t\t\tMACOSX_DEPLOYMENT_TARGET = 12.3;\n\t\t\t\tMTL_ENABLE_DEBUG_INFO = NO;\n\t\t\t\tMTL_FAST_MATH = YES;\n\t\t\t\tSDKROOT = macosx;\n\t\t\t\tSWIFT_COMPILATION_MODE = wholemodule;\n\t\t\t\tSWIFT_OPTIMIZATION_LEVEL = \"-O\";\n\t\t\t};\n\t\t\tname = Release;\n\t\t};\n\t\tC858BB3C2902A59100DF30AE /* Debug */ = {\n\t\t\tisa = XCBuildConfiguration;\n\t\t\tbuildSettings = {\n\t\t\t\tASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;\n\t\t\t\tASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor;\n\t\t\t\tCODE_SIGN_ENTITLEMENTS = SimpleDiffusion/SimpleDiffusion.entitlements;\n\t\t\t\tCODE_SIGN_STYLE = Automatic;\n\t\t\t\tCOMBINE_HIDPI_IMAGES = YES;\n\t\t\t\tCURRENT_PROJECT_VERSION = 1;\n\t\t\t\tDEVELOPMENT_ASSET_PATHS = \"\\\"SimpleDiffusion/Preview Content\\\"\";\n\t\t\t\tDEVELOPMENT_TEAM = UDGLB23X37;\n\t\t\t\tENABLE_HARDENED_RUNTIME = YES;\n\t\t\t\tENABLE_PREVIEWS = YES;\n\t\t\t\tGENERATE_INFOPLIST_FILE = YES;\n\t\t\t\tINFOPLIST_KEY_NSHumanReadableCopyright = \"\";\n\t\t\t\tLD_RUNPATH_SEARCH_PATHS = (\n\t\t\t\t\t\"$(inherited)\",\n\t\t\t\t\t\"@executable_path/../Frameworks\",\n\t\t\t\t);\n\t\t\t\tMARKETING_VERSION = 1.0;\n\t\t\t\tPRODUCT_BUNDLE_IDENTIFIER = app.otato.SimpleDiffusion;\n\t\t\t\tPRODUCT_NAME = \"$(TARGET_NAME)\";\n\t\t\t\tSWIFT_EMIT_LOC_STRINGS = YES;\n\t\t\t\tSWIFT_VERSION = 5.0;\n\t\t\t};\n\t\t\tname = Debug;\n\t\t};\n\t\tC858BB3D2902A59100DF30AE /* Release */ = {\n\t\t\tisa = XCBuildConfiguration;\n\t\t\tbuildSettings = {\n\t\t\t\tASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;\n\t\t\t\tASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor;\n\t\t\t\tCODE_SIGN_ENTITLEMENTS = SimpleDiffusion/SimpleDiffusion.entitlements;\n\t\t\t\tCODE_SIGN_STYLE = Automatic;\n\t\t\t\tCOMBINE_HIDPI_IMAGES = YES;\n\t\t\t\tCURRENT_PROJECT_VERSION = 1;\n\t\t\t\tDEVELOPMENT_ASSET_PATHS = \"\\\"SimpleDiffusion/Preview Content\\\"\";\n\t\t\t\tDEVELOPMENT_TEAM = UDGLB23X37;\n\t\t\t\tENABLE_HARDENED_RUNTIME = YES;\n\t\t\t\tENABLE_PREVIEWS = YES;\n\t\t\t\tGENERATE_INFOPLIST_FILE = YES;\n\t\t\t\tINFOPLIST_KEY_NSHumanReadableCopyright = \"\";\n\t\t\t\tLD_RUNPATH_SEARCH_PATHS = (\n\t\t\t\t\t\"$(inherited)\",\n\t\t\t\t\t\"@executable_path/../Frameworks\",\n\t\t\t\t);\n\t\t\t\tMARKETING_VERSION = 1.0;\n\t\t\t\tPRODUCT_BUNDLE_IDENTIFIER = app.otato.SimpleDiffusion;\n\t\t\t\tPRODUCT_NAME = \"$(TARGET_NAME)\";\n\t\t\t\tSWIFT_EMIT_LOC_STRINGS = YES;\n\t\t\t\tSWIFT_VERSION = 5.0;\n\t\t\t};\n\t\t\tname = Release;\n\t\t};\n/* End XCBuildConfiguration section */\n\n/* Begin XCConfigurationList section */\n\t\tC858BB272902A59000DF30AE /* Build configuration list for PBXProject \"SimpleDiffusion\" */ = {\n\t\t\tisa = XCConfigurationList;\n\t\t\tbuildConfigurations = (\n\t\t\t\tC858BB392902A59100DF30AE /* Debug */,\n\t\t\t\tC858BB3A2902A59100DF30AE /* Release */,\n\t\t\t);\n\t\t\tdefaultConfigurationIsVisible = 0;\n\t\t\tdefaultConfigurationName = Release;\n\t\t};\n\t\tC858BB3B2902A59100DF30AE /* Build configuration list for PBXNativeTarget \"SimpleDiffusion\" */ = {\n\t\t\tisa = XCConfigurationList;\n\t\t\tbuildConfigurations = (\n\t\t\t\tC858BB3C2902A59100DF30AE /* Debug */,\n\t\t\t\tC858BB3D2902A59100DF30AE /* Release */,\n\t\t\t);\n\t\t\tdefaultConfigurationIsVisible = 0;\n\t\t\tdefaultConfigurationName = Release;\n\t\t};\n/* End XCConfigurationList section */\n\n/* Begin XCSwiftPackageProductDependency section */\n\t\tC858BB402902A68200DF30AE /* MapleDiffusion */ = {\n\t\t\tisa = XCSwiftPackageProductDependency;\n\t\t\tproductName = MapleDiffusion;\n\t\t};\n/* End XCSwiftPackageProductDependency section */\n\t};\n\trootObject = C858BB242902A59000DF30AE /* Project object */;\n}\n" }, { "path": "Examples/Simple/SimpleDiffusion/SimpleDiffusion.xcodeproj/project.xcworkspace/contents.xcworkspacedata", "content": "\n\n \n \n\n" }, { "path": "Examples/Simple/SimpleDiffusion/SimpleDiffusion.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist", "content": "\n\n\n\n\tIDEDidComputeMac32BitWarning\n\t\n\n\n" }, { "path": "Examples/Simple/SimpleDiffusion/SimpleDiffusion.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved", "content": "{\n \"pins\" : [\n {\n \"identity\" : \"zipfoundation\",\n \"kind\" : \"remoteSourceControl\",\n \"location\" : \"https://github.com/weichsel/ZIPFoundation.git\",\n \"state\" : {\n \"revision\" : \"f6a22e7da26314b38bf9befce34ae8e4b2543090\",\n \"version\" : \"0.9.15\"\n }\n }\n ],\n \"version\" : 2\n}\n" }, { "path": "Examples/Single Line Diffusion/Single Line Diffusion/Assets.xcassets/AccentColor.colorset/Contents.json", "content": "{\n \"colors\" : [\n {\n \"idiom\" : \"universal\"\n }\n ],\n \"info\" : {\n \"author\" : \"xcode\",\n \"version\" : 1\n }\n}\n" }, { "path": "Examples/Single Line Diffusion/Single Line Diffusion/Assets.xcassets/AppIcon.appiconset/Contents.json", "content": "{\n \"images\" : [\n {\n \"idiom\" : \"mac\",\n \"scale\" : \"1x\",\n \"size\" : \"16x16\"\n },\n {\n \"idiom\" : \"mac\",\n \"scale\" : \"2x\",\n \"size\" : \"16x16\"\n },\n {\n \"idiom\" : \"mac\",\n \"scale\" : \"1x\",\n \"size\" : \"32x32\"\n },\n {\n \"idiom\" : \"mac\",\n \"scale\" : \"2x\",\n \"size\" : \"32x32\"\n },\n {\n \"idiom\" : \"mac\",\n \"scale\" : \"1x\",\n \"size\" : \"128x128\"\n },\n {\n \"idiom\" : \"mac\",\n \"scale\" : \"2x\",\n \"size\" : \"128x128\"\n },\n {\n \"idiom\" : \"mac\",\n \"scale\" : \"1x\",\n \"size\" : \"256x256\"\n },\n {\n \"idiom\" : \"mac\",\n \"scale\" : \"2x\",\n \"size\" : \"256x256\"\n },\n {\n \"idiom\" : \"mac\",\n \"scale\" : \"1x\",\n \"size\" : \"512x512\"\n },\n {\n \"idiom\" : \"mac\",\n \"scale\" : \"2x\",\n \"size\" : \"512x512\"\n }\n ],\n \"info\" : {\n \"author\" : \"xcode\",\n \"version\" : 1\n }\n}\n" }, { "path": "Examples/Single Line Diffusion/Single Line Diffusion/Assets.xcassets/Contents.json", "content": "{\n \"info\" : {\n \"author\" : \"xcode\",\n \"version\" : 1\n }\n}\n" }, { "path": "Examples/Single Line Diffusion/Single Line Diffusion/ContentView.swift", "content": "//\n// ContentView.swift\n// Single Line Diffusion\n//\n// Created by Morten Just on 10/28/22.\n//\n\nimport SwiftUI\nimport MapleDiffusion\n\nstruct ContentView: View {\n @State var image : CGImage?\n\n var body: some View {\n VStack {\n if let image { Image(image, scale: 1, label: Text(\"Generated\")) } else { Text(\"Loading. See console.\")}\n }\n .onAppear {\n Task.detached {\n for _ in 0...10 {\n image = try? await Diffusion.generate(localOrRemote: modelUrl, prompt: \"cat astronaut\")\n }\n }\n }\n .frame(minWidth: 500, minHeight: 500)\n }\n}\n\n\nstruct ContentView_Previews: PreviewProvider {\n static var previews: some View {\n ContentView()\n }\n}\n\n\nlet modelUrl = FileManager.default.urls(for: .applicationSupportDirectory, in: .userDomainMask)[0].appendingPathComponent(\"Photato/bins\")\n" }, { "path": "Examples/Single Line Diffusion/Single Line Diffusion/Preview Content/Preview Assets.xcassets/Contents.json", "content": "{\n \"info\" : {\n \"author\" : \"xcode\",\n \"version\" : 1\n }\n}\n" }, { "path": "Examples/Single Line Diffusion/Single Line Diffusion/Single_Line_Diffusion.entitlements", "content": "\n\n\n\n\n" }, { "path": "Examples/Single Line Diffusion/Single Line Diffusion/Single_Line_DiffusionApp.swift", "content": "//\n// Single_Line_DiffusionApp.swift\n// Single Line Diffusion\n//\n// Created by Morten Just on 10/28/22.\n//\n\nimport SwiftUI\n\n@main\nstruct Single_Line_DiffusionApp: App {\n var body: some Scene {\n WindowGroup {\n ContentView()\n }\n }\n}\n" }, { "path": "Examples/Single Line Diffusion/Single Line Diffusion.xcodeproj/project.pbxproj", "content": "// !$*UTF8*$!\n{\n\tarchiveVersion = 1;\n\tclasses = {\n\t};\n\tobjectVersion = 56;\n\tobjects = {\n\n/* Begin PBXBuildFile section */\n\t\tC8ED50B8290C204800153A3B /* Single_Line_DiffusionApp.swift in Sources */ = {isa = PBXBuildFile; fileRef = C8ED50B7290C204800153A3B /* Single_Line_DiffusionApp.swift */; };\n\t\tC8ED50BA290C204800153A3B /* ContentView.swift in Sources */ = {isa = PBXBuildFile; fileRef = C8ED50B9290C204800153A3B /* ContentView.swift */; };\n\t\tC8ED50BC290C204800153A3B /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = C8ED50BB290C204800153A3B /* Assets.xcassets */; };\n\t\tC8ED50BF290C204800153A3B /* Preview Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = C8ED50BE290C204800153A3B /* Preview Assets.xcassets */; };\n\t\tC8ED50C9290C207200153A3B /* MapleDiffusion in Frameworks */ = {isa = PBXBuildFile; productRef = C8ED50C8290C207200153A3B /* MapleDiffusion */; };\n/* End PBXBuildFile section */\n\n/* Begin PBXFileReference section */\n\t\tC8ED50B4290C204800153A3B /* Single Line Diffusion.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = \"Single Line Diffusion.app\"; sourceTree = BUILT_PRODUCTS_DIR; };\n\t\tC8ED50B7290C204800153A3B /* Single_Line_DiffusionApp.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Single_Line_DiffusionApp.swift; sourceTree = \"\"; };\n\t\tC8ED50B9290C204800153A3B /* ContentView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ContentView.swift; sourceTree = \"\"; };\n\t\tC8ED50BB290C204800153A3B /* Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = Assets.xcassets; sourceTree = \"\"; };\n\t\tC8ED50BE290C204800153A3B /* Preview Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = \"Preview Assets.xcassets\"; sourceTree = \"\"; };\n\t\tC8ED50C0290C204800153A3B /* Single_Line_Diffusion.entitlements */ = {isa = PBXFileReference; lastKnownFileType = text.plist.entitlements; path = Single_Line_Diffusion.entitlements; sourceTree = \"\"; };\n\t\tC8ED50C6290C206600153A3B /* maple-diffusion-package */ = {isa = PBXFileReference; lastKnownFileType = wrapper; name = \"maple-diffusion-package\"; path = ../..; sourceTree = \"\"; };\n/* End PBXFileReference section */\n\n/* Begin PBXFrameworksBuildPhase section */\n\t\tC8ED50B1290C204800153A3B /* Frameworks */ = {\n\t\t\tisa = PBXFrameworksBuildPhase;\n\t\t\tbuildActionMask = 2147483647;\n\t\t\tfiles = (\n\t\t\t\tC8ED50C9290C207200153A3B /* MapleDiffusion in Frameworks */,\n\t\t\t);\n\t\t\trunOnlyForDeploymentPostprocessing = 0;\n\t\t};\n/* End PBXFrameworksBuildPhase section */\n\n/* Begin PBXGroup section */\n\t\tC8ED50AB290C204800153A3B = {\n\t\t\tisa = PBXGroup;\n\t\t\tchildren = (\n\t\t\t\tC8ED50C6290C206600153A3B /* maple-diffusion-package */,\n\t\t\t\tC8ED50B6290C204800153A3B /* Single Line Diffusion */,\n\t\t\t\tC8ED50B5290C204800153A3B /* Products */,\n\t\t\t\tC8ED50C7290C207200153A3B /* Frameworks */,\n\t\t\t);\n\t\t\tsourceTree = \"\";\n\t\t};\n\t\tC8ED50B5290C204800153A3B /* Products */ = {\n\t\t\tisa = PBXGroup;\n\t\t\tchildren = (\n\t\t\t\tC8ED50B4290C204800153A3B /* Single Line Diffusion.app */,\n\t\t\t);\n\t\t\tname = Products;\n\t\t\tsourceTree = \"\";\n\t\t};\n\t\tC8ED50B6290C204800153A3B /* Single Line Diffusion */ = {\n\t\t\tisa = PBXGroup;\n\t\t\tchildren = (\n\t\t\t\tC8ED50B7290C204800153A3B /* Single_Line_DiffusionApp.swift */,\n\t\t\t\tC8ED50B9290C204800153A3B /* ContentView.swift */,\n\t\t\t\tC8ED50BB290C204800153A3B /* Assets.xcassets */,\n\t\t\t\tC8ED50C0290C204800153A3B /* Single_Line_Diffusion.entitlements */,\n\t\t\t\tC8ED50BD290C204800153A3B /* Preview Content */,\n\t\t\t);\n\t\t\tpath = \"Single Line Diffusion\";\n\t\t\tsourceTree = \"\";\n\t\t};\n\t\tC8ED50BD290C204800153A3B /* Preview Content */ = {\n\t\t\tisa = PBXGroup;\n\t\t\tchildren = (\n\t\t\t\tC8ED50BE290C204800153A3B /* Preview Assets.xcassets */,\n\t\t\t);\n\t\t\tpath = \"Preview Content\";\n\t\t\tsourceTree = \"\";\n\t\t};\n\t\tC8ED50C7290C207200153A3B /* Frameworks */ = {\n\t\t\tisa = PBXGroup;\n\t\t\tchildren = (\n\t\t\t);\n\t\t\tname = Frameworks;\n\t\t\tsourceTree = \"\";\n\t\t};\n/* End PBXGroup section */\n\n/* Begin PBXNativeTarget section */\n\t\tC8ED50B3290C204800153A3B /* Single Line Diffusion */ = {\n\t\t\tisa = PBXNativeTarget;\n\t\t\tbuildConfigurationList = C8ED50C3290C204800153A3B /* Build configuration list for PBXNativeTarget \"Single Line Diffusion\" */;\n\t\t\tbuildPhases = (\n\t\t\t\tC8ED50B0290C204800153A3B /* Sources */,\n\t\t\t\tC8ED50B1290C204800153A3B /* Frameworks */,\n\t\t\t\tC8ED50B2290C204800153A3B /* Resources */,\n\t\t\t);\n\t\t\tbuildRules = (\n\t\t\t);\n\t\t\tdependencies = (\n\t\t\t);\n\t\t\tname = \"Single Line Diffusion\";\n\t\t\tpackageProductDependencies = (\n\t\t\t\tC8ED50C8290C207200153A3B /* MapleDiffusion */,\n\t\t\t);\n\t\t\tproductName = \"Single Line Diffusion\";\n\t\t\tproductReference = C8ED50B4290C204800153A3B /* Single Line Diffusion.app */;\n\t\t\tproductType = \"com.apple.product-type.application\";\n\t\t};\n/* End PBXNativeTarget section */\n\n/* Begin PBXProject section */\n\t\tC8ED50AC290C204800153A3B /* Project object */ = {\n\t\t\tisa = PBXProject;\n\t\t\tattributes = {\n\t\t\t\tBuildIndependentTargetsInParallel = 1;\n\t\t\t\tLastSwiftUpdateCheck = 1400;\n\t\t\t\tLastUpgradeCheck = 1400;\n\t\t\t\tTargetAttributes = {\n\t\t\t\t\tC8ED50B3290C204800153A3B = {\n\t\t\t\t\t\tCreatedOnToolsVersion = 14.0.1;\n\t\t\t\t\t};\n\t\t\t\t};\n\t\t\t};\n\t\t\tbuildConfigurationList = C8ED50AF290C204800153A3B /* Build configuration list for PBXProject \"Single Line Diffusion\" */;\n\t\t\tcompatibilityVersion = \"Xcode 14.0\";\n\t\t\tdevelopmentRegion = en;\n\t\t\thasScannedForEncodings = 0;\n\t\t\tknownRegions = (\n\t\t\t\ten,\n\t\t\t\tBase,\n\t\t\t);\n\t\t\tmainGroup = C8ED50AB290C204800153A3B;\n\t\t\tproductRefGroup = C8ED50B5290C204800153A3B /* Products */;\n\t\t\tprojectDirPath = \"\";\n\t\t\tprojectRoot = \"\";\n\t\t\ttargets = (\n\t\t\t\tC8ED50B3290C204800153A3B /* Single Line Diffusion */,\n\t\t\t);\n\t\t};\n/* End PBXProject section */\n\n/* Begin PBXResourcesBuildPhase section */\n\t\tC8ED50B2290C204800153A3B /* Resources */ = {\n\t\t\tisa = PBXResourcesBuildPhase;\n\t\t\tbuildActionMask = 2147483647;\n\t\t\tfiles = (\n\t\t\t\tC8ED50BF290C204800153A3B /* Preview Assets.xcassets in Resources */,\n\t\t\t\tC8ED50BC290C204800153A3B /* Assets.xcassets in Resources */,\n\t\t\t);\n\t\t\trunOnlyForDeploymentPostprocessing = 0;\n\t\t};\n/* End PBXResourcesBuildPhase section */\n\n/* Begin PBXSourcesBuildPhase section */\n\t\tC8ED50B0290C204800153A3B /* Sources */ = {\n\t\t\tisa = PBXSourcesBuildPhase;\n\t\t\tbuildActionMask = 2147483647;\n\t\t\tfiles = (\n\t\t\t\tC8ED50BA290C204800153A3B /* ContentView.swift in Sources */,\n\t\t\t\tC8ED50B8290C204800153A3B /* Single_Line_DiffusionApp.swift in Sources */,\n\t\t\t);\n\t\t\trunOnlyForDeploymentPostprocessing = 0;\n\t\t};\n/* End PBXSourcesBuildPhase section */\n\n/* Begin XCBuildConfiguration section */\n\t\tC8ED50C1290C204800153A3B /* Debug */ = {\n\t\t\tisa = XCBuildConfiguration;\n\t\t\tbuildSettings = {\n\t\t\t\tALWAYS_SEARCH_USER_PATHS = NO;\n\t\t\t\tCLANG_ANALYZER_NONNULL = YES;\n\t\t\t\tCLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;\n\t\t\t\tCLANG_CXX_LANGUAGE_STANDARD = \"gnu++20\";\n\t\t\t\tCLANG_ENABLE_MODULES = YES;\n\t\t\t\tCLANG_ENABLE_OBJC_ARC = YES;\n\t\t\t\tCLANG_ENABLE_OBJC_WEAK = YES;\n\t\t\t\tCLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;\n\t\t\t\tCLANG_WARN_BOOL_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_COMMA = YES;\n\t\t\t\tCLANG_WARN_CONSTANT_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES;\n\t\t\t\tCLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;\n\t\t\t\tCLANG_WARN_DOCUMENTATION_COMMENTS = YES;\n\t\t\t\tCLANG_WARN_EMPTY_BODY = YES;\n\t\t\t\tCLANG_WARN_ENUM_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_INFINITE_RECURSION = YES;\n\t\t\t\tCLANG_WARN_INT_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES;\n\t\t\t\tCLANG_WARN_OBJC_LITERAL_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;\n\t\t\t\tCLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES;\n\t\t\t\tCLANG_WARN_RANGE_LOOP_ANALYSIS = YES;\n\t\t\t\tCLANG_WARN_STRICT_PROTOTYPES = YES;\n\t\t\t\tCLANG_WARN_SUSPICIOUS_MOVE = YES;\n\t\t\t\tCLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE;\n\t\t\t\tCLANG_WARN_UNREACHABLE_CODE = YES;\n\t\t\t\tCLANG_WARN__DUPLICATE_METHOD_MATCH = YES;\n\t\t\t\tCOPY_PHASE_STRIP = NO;\n\t\t\t\tDEBUG_INFORMATION_FORMAT = dwarf;\n\t\t\t\tENABLE_STRICT_OBJC_MSGSEND = YES;\n\t\t\t\tENABLE_TESTABILITY = YES;\n\t\t\t\tGCC_C_LANGUAGE_STANDARD = gnu11;\n\t\t\t\tGCC_DYNAMIC_NO_PIC = NO;\n\t\t\t\tGCC_NO_COMMON_BLOCKS = YES;\n\t\t\t\tGCC_OPTIMIZATION_LEVEL = 0;\n\t\t\t\tGCC_PREPROCESSOR_DEFINITIONS = (\n\t\t\t\t\t\"DEBUG=1\",\n\t\t\t\t\t\"$(inherited)\",\n\t\t\t\t);\n\t\t\t\tGCC_WARN_64_TO_32_BIT_CONVERSION = YES;\n\t\t\t\tGCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;\n\t\t\t\tGCC_WARN_UNDECLARED_SELECTOR = YES;\n\t\t\t\tGCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;\n\t\t\t\tGCC_WARN_UNUSED_FUNCTION = YES;\n\t\t\t\tGCC_WARN_UNUSED_VARIABLE = YES;\n\t\t\t\tMACOSX_DEPLOYMENT_TARGET = 12.3;\n\t\t\t\tMTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE;\n\t\t\t\tMTL_FAST_MATH = YES;\n\t\t\t\tONLY_ACTIVE_ARCH = YES;\n\t\t\t\tSDKROOT = macosx;\n\t\t\t\tSWIFT_ACTIVE_COMPILATION_CONDITIONS = DEBUG;\n\t\t\t\tSWIFT_OPTIMIZATION_LEVEL = \"-Onone\";\n\t\t\t};\n\t\t\tname = Debug;\n\t\t};\n\t\tC8ED50C2290C204800153A3B /* Release */ = {\n\t\t\tisa = XCBuildConfiguration;\n\t\t\tbuildSettings = {\n\t\t\t\tALWAYS_SEARCH_USER_PATHS = NO;\n\t\t\t\tCLANG_ANALYZER_NONNULL = YES;\n\t\t\t\tCLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;\n\t\t\t\tCLANG_CXX_LANGUAGE_STANDARD = \"gnu++20\";\n\t\t\t\tCLANG_ENABLE_MODULES = YES;\n\t\t\t\tCLANG_ENABLE_OBJC_ARC = YES;\n\t\t\t\tCLANG_ENABLE_OBJC_WEAK = YES;\n\t\t\t\tCLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;\n\t\t\t\tCLANG_WARN_BOOL_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_COMMA = YES;\n\t\t\t\tCLANG_WARN_CONSTANT_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES;\n\t\t\t\tCLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;\n\t\t\t\tCLANG_WARN_DOCUMENTATION_COMMENTS = YES;\n\t\t\t\tCLANG_WARN_EMPTY_BODY = YES;\n\t\t\t\tCLANG_WARN_ENUM_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_INFINITE_RECURSION = YES;\n\t\t\t\tCLANG_WARN_INT_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES;\n\t\t\t\tCLANG_WARN_OBJC_LITERAL_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;\n\t\t\t\tCLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES;\n\t\t\t\tCLANG_WARN_RANGE_LOOP_ANALYSIS = YES;\n\t\t\t\tCLANG_WARN_STRICT_PROTOTYPES = YES;\n\t\t\t\tCLANG_WARN_SUSPICIOUS_MOVE = YES;\n\t\t\t\tCLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE;\n\t\t\t\tCLANG_WARN_UNREACHABLE_CODE = YES;\n\t\t\t\tCLANG_WARN__DUPLICATE_METHOD_MATCH = YES;\n\t\t\t\tCOPY_PHASE_STRIP = NO;\n\t\t\t\tDEBUG_INFORMATION_FORMAT = \"dwarf-with-dsym\";\n\t\t\t\tENABLE_NS_ASSERTIONS = NO;\n\t\t\t\tENABLE_STRICT_OBJC_MSGSEND = YES;\n\t\t\t\tGCC_C_LANGUAGE_STANDARD = gnu11;\n\t\t\t\tGCC_NO_COMMON_BLOCKS = YES;\n\t\t\t\tGCC_WARN_64_TO_32_BIT_CONVERSION = YES;\n\t\t\t\tGCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;\n\t\t\t\tGCC_WARN_UNDECLARED_SELECTOR = YES;\n\t\t\t\tGCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;\n\t\t\t\tGCC_WARN_UNUSED_FUNCTION = YES;\n\t\t\t\tGCC_WARN_UNUSED_VARIABLE = YES;\n\t\t\t\tMACOSX_DEPLOYMENT_TARGET = 12.3;\n\t\t\t\tMTL_ENABLE_DEBUG_INFO = NO;\n\t\t\t\tMTL_FAST_MATH = YES;\n\t\t\t\tSDKROOT = macosx;\n\t\t\t\tSWIFT_COMPILATION_MODE = wholemodule;\n\t\t\t\tSWIFT_OPTIMIZATION_LEVEL = \"-O\";\n\t\t\t};\n\t\t\tname = Release;\n\t\t};\n\t\tC8ED50C4290C204800153A3B /* Debug */ = {\n\t\t\tisa = XCBuildConfiguration;\n\t\t\tbuildSettings = {\n\t\t\t\tASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;\n\t\t\t\tASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor;\n\t\t\t\tCODE_SIGN_ENTITLEMENTS = \"Single Line Diffusion/Single_Line_Diffusion.entitlements\";\n\t\t\t\tCODE_SIGN_STYLE = Automatic;\n\t\t\t\tCOMBINE_HIDPI_IMAGES = YES;\n\t\t\t\tCURRENT_PROJECT_VERSION = 1;\n\t\t\t\tDEVELOPMENT_ASSET_PATHS = \"\\\"Single Line Diffusion/Preview Content\\\"\";\n\t\t\t\tDEVELOPMENT_TEAM = UDGLB23X37;\n\t\t\t\tENABLE_HARDENED_RUNTIME = YES;\n\t\t\t\tENABLE_PREVIEWS = YES;\n\t\t\t\tGENERATE_INFOPLIST_FILE = YES;\n\t\t\t\tINFOPLIST_KEY_NSHumanReadableCopyright = \"\";\n\t\t\t\tLD_RUNPATH_SEARCH_PATHS = (\n\t\t\t\t\t\"$(inherited)\",\n\t\t\t\t\t\"@executable_path/../Frameworks\",\n\t\t\t\t);\n\t\t\t\tMARKETING_VERSION = 1.0;\n\t\t\t\tPRODUCT_BUNDLE_IDENTIFIER = \"app.otato.Single-Line-Diffusion\";\n\t\t\t\tPRODUCT_NAME = \"$(TARGET_NAME)\";\n\t\t\t\tSWIFT_EMIT_LOC_STRINGS = YES;\n\t\t\t\tSWIFT_VERSION = 5.0;\n\t\t\t};\n\t\t\tname = Debug;\n\t\t};\n\t\tC8ED50C5290C204800153A3B /* Release */ = {\n\t\t\tisa = XCBuildConfiguration;\n\t\t\tbuildSettings = {\n\t\t\t\tASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;\n\t\t\t\tASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor;\n\t\t\t\tCODE_SIGN_ENTITLEMENTS = \"Single Line Diffusion/Single_Line_Diffusion.entitlements\";\n\t\t\t\tCODE_SIGN_STYLE = Automatic;\n\t\t\t\tCOMBINE_HIDPI_IMAGES = YES;\n\t\t\t\tCURRENT_PROJECT_VERSION = 1;\n\t\t\t\tDEVELOPMENT_ASSET_PATHS = \"\\\"Single Line Diffusion/Preview Content\\\"\";\n\t\t\t\tDEVELOPMENT_TEAM = UDGLB23X37;\n\t\t\t\tENABLE_HARDENED_RUNTIME = YES;\n\t\t\t\tENABLE_PREVIEWS = YES;\n\t\t\t\tGENERATE_INFOPLIST_FILE = YES;\n\t\t\t\tINFOPLIST_KEY_NSHumanReadableCopyright = \"\";\n\t\t\t\tLD_RUNPATH_SEARCH_PATHS = (\n\t\t\t\t\t\"$(inherited)\",\n\t\t\t\t\t\"@executable_path/../Frameworks\",\n\t\t\t\t);\n\t\t\t\tMARKETING_VERSION = 1.0;\n\t\t\t\tPRODUCT_BUNDLE_IDENTIFIER = \"app.otato.Single-Line-Diffusion\";\n\t\t\t\tPRODUCT_NAME = \"$(TARGET_NAME)\";\n\t\t\t\tSWIFT_EMIT_LOC_STRINGS = YES;\n\t\t\t\tSWIFT_VERSION = 5.0;\n\t\t\t};\n\t\t\tname = Release;\n\t\t};\n/* End XCBuildConfiguration section */\n\n/* Begin XCConfigurationList section */\n\t\tC8ED50AF290C204800153A3B /* Build configuration list for PBXProject \"Single Line Diffusion\" */ = {\n\t\t\tisa = XCConfigurationList;\n\t\t\tbuildConfigurations = (\n\t\t\t\tC8ED50C1290C204800153A3B /* Debug */,\n\t\t\t\tC8ED50C2290C204800153A3B /* Release */,\n\t\t\t);\n\t\t\tdefaultConfigurationIsVisible = 0;\n\t\t\tdefaultConfigurationName = Release;\n\t\t};\n\t\tC8ED50C3290C204800153A3B /* Build configuration list for PBXNativeTarget \"Single Line Diffusion\" */ = {\n\t\t\tisa = XCConfigurationList;\n\t\t\tbuildConfigurations = (\n\t\t\t\tC8ED50C4290C204800153A3B /* Debug */,\n\t\t\t\tC8ED50C5290C204800153A3B /* Release */,\n\t\t\t);\n\t\t\tdefaultConfigurationIsVisible = 0;\n\t\t\tdefaultConfigurationName = Release;\n\t\t};\n/* End XCConfigurationList section */\n\n/* Begin XCSwiftPackageProductDependency section */\n\t\tC8ED50C8290C207200153A3B /* MapleDiffusion */ = {\n\t\t\tisa = XCSwiftPackageProductDependency;\n\t\t\tproductName = MapleDiffusion;\n\t\t};\n/* End XCSwiftPackageProductDependency section */\n\t};\n\trootObject = C8ED50AC290C204800153A3B /* Project object */;\n}\n" }, { "path": "Examples/Single Line Diffusion/Single Line Diffusion.xcodeproj/project.xcworkspace/contents.xcworkspacedata", "content": "\n\n \n \n\n" }, { "path": "Examples/Single Line Diffusion/Single Line Diffusion.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist", "content": "\n\n\n\n\tIDEDidComputeMac32BitWarning\n\t\n\n\n" }, { "path": "Examples/Single Line Diffusion/Single Line Diffusion.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved", "content": "{\n \"pins\" : [\n {\n \"identity\" : \"zipfoundation\",\n \"kind\" : \"remoteSourceControl\",\n \"location\" : \"https://github.com/weichsel/ZIPFoundation.git\",\n \"state\" : {\n \"revision\" : \"f6a22e7da26314b38bf9befce34ae8e4b2543090\",\n \"version\" : \"0.9.15\"\n }\n }\n ],\n \"version\" : 2\n}\n" }, { "path": "LICENSE", "content": "MIT License\n\nCopyright (c) 2022 Ollin Boer Bohan\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n" }, { "path": "Package.swift", "content": "// swift-tools-version: 5.7\n// The swift-tools-version declares the minimum version of Swift required to build this package.\n\nimport PackageDescription\n\nlet package = Package(\n name: \"MapleDiffusion\",\n platforms: [ .macOS(\"12.3\"), .iOS(\"15.4\") ],\n \n products: [\n // Products define the executables and libraries a package produces, and make them visible to other packages.\n .library(\n name: \"MapleDiffusion\",\n targets: [\"MapleDiffusion\"]),\n ],\n dependencies: [\n // Dependencies declare other packages that this package depends on.\n // .package(url: /* package url */, from: \"1.0.0\"),\n .package(url: \"https://github.com/weichsel/ZIPFoundation.git\", .upToNextMajor(from: \"0.9.0\"))\n\n \n ],\n targets: [\n // Targets are the basic building blocks of a package. A target can define a module or a test suite.\n // Targets can depend on other targets in this package, and on products in packages this package depends on.\n .target(\n name: \"MapleDiffusion\", dependencies: [\"ZIPFoundation\"]),\n .testTarget(\n name: \"MapleDiffusionTests\",\n dependencies: [\"MapleDiffusion\", \"ZIPFoundation\"]),\n ]\n)\n" }, { "path": "README.md", "content": "# Native Diffusion Swift Package\n\n[Join us on Discord](https://discord.gg/XNsw7x667a)\n\nNative Diffusion runs Stable Diffusion models **locally** on macOS / iOS devices, in Swift, using the MPSGraph framework (not Python).\n\nThis is the Swift Package Manager wrapper of [Maple Diffusion](https://github.com/madebyollin/maple-diffusion). It adds image-to-image, Swift Package Manager package, and convenient ways to use the code, like Combine publishers and async/await versions. It also supports downloading weights from any local or remote URL, including the app bundle itself. \n\nWould not be possible without\n* [@madebyollin](https://github.com/madebyollin/) who wrote the Metal Performance Shader Graph pipeline\n* [@GuiyeC](https://github.com/GuiyeC) who wrote the image-to-image implementation\n\n# Features\nGet started in 10 minutes\n* Extremely simple API. Generate an image in one line of code. \n\nMake it do what you want\n* Flexible API. Pass in prompt, guidance scale, steps, seed, and an image. \n* One-off conversion script from .ckpt to Native Diffusion's own memory-optimized format\n* Supports Dreambooth models. \n\nBuilt to be fun to code with\n* Supports async/await, Combine publisher and classic callbacks.\n* Optimized for SwiftUI, but can be used in any kind of project, including command line, UIKit, or AppKit\n\nBuilt for end-user speed and great user experience\n* 100% native. No Python, no environments, your user don't need to install anything first.\n* Model download built in. Point it to a web address with the model files in a zip archive. The package will download and install the model for later use. \n* As fast or faster than a server in the cloud on newer Macs\n\nCommercial use allowed\n* MIT Licensed (code). We'd love attribution, but it's not needed legally.\n* Generated images are licensed under the [CreativeML Open RAIL-M](https://github.com/CompVis/stable-diffusion/blob/main/LICENSE) license, meaning you can use the images for virtually anything, including commercial use.\n\n\n# Usage\n## One-line diffusion\nIn its simplest form it's as simple as one line:\n\n```swift\nlet image = try? await Diffusion.generate(localOrRemote: modelUrl, prompt: \"cat astronaut\")\n```\n\nYou can give it a local or remote URL or both. If remote, the downloaded weights are saved for later.\n\nThe single line version is currently limited in terms of parameters.\n\nSee `examples/SingleLineDiffusion` for a working example. \n\n## As an observable object\nLet's add some UI. Here's an entire working image generator app in a single SwiftUI view:\n\n![GIF demo](https://github.com/mortenjust/maple-diffusion/blob/main/Examples/Demos/simple-diffusion.gif)\n\n\n```swift\nstruct ContentView: View {\n \n // 1\n @StateObject var sd = Diffusion()\n @State var prompt = \"\"\n @State var image : CGImage?\n @State var imagePublisher = Diffusion.placeholderPublisher\n @State var progress : Double = 0\n \n var anyProgress : Double { sd.loadingProgress < 1 ? sd.loadingProgress : progress }\n\n var body: some View {\n VStack {\n \n DiffusionImage(image: $image, progress: $progress)\n Spacer()\n TextField(\"Prompt\", text: $prompt)\n // 3\n .onSubmit { self.imagePublisher = sd.generate(prompt: prompt) }\n .disabled(!sd.isModelReady)\n ProgressView(value: anyProgress)\n .opacity(anyProgress == 1 || anyProgress == 0 ? 0 : 1)\n }\n .task {\n // 2\n let path = URL(string: \"http://localhost:8080/Diffusion.zip\")!\n try! await sd.prepModels(remoteURL: path)\n }\n \n // 4\n .onReceive(imagePublisher) { r in\n self.image = r.image\n self.progress = r.progress\n }\n .frame(minWidth: 200, minHeight: 200)\n }\n}\n```\n\nHere's what it does\n1. Instantiate a `Diffusion` object\n2. Prepare the models, download if needed\n3. Submit a prompt for generation\n4. Receive updates during generation\n\nSee `examples/SimpleDiffusion` for a working example. \n\n## `DiffusionImage`\nAn optional SwiftUI view that is specialized for diffusion: \n- Receives drag and drop of an image from e.g. Finder and sends it back to you via a binding (macOS)\n- Automatically resizes the image to 512x512 (macOS)\n- Lets users drag the image to Finder or other apps (macOS)\n- Blurs the internmediate image while generating (macOS and iOS)\n\n# Install\nAdd `https://github.com/mortenjust/native-diffusion` in the [\"Swift Package Manager\" tab in Xcode](https://developer.apple.com/documentation/xcode/adding_package_dependencies_to_your_app)\n\n# Preparing the weights\nNative Diffusion splits the weights into a binary format that is different from the typical CKPT format. It uses many small files which it then (optionally) swaps in and out of memory, enabling it to run on both macOS and iOS. You can use the converter script in the package to convert your own CKPT file. \n\n## Option 1: Pre-converted Standard Stable Diffusion v1.5\nBy downloading this zip file, you accept the [creative license from StabilityAI](https://github.com/CompVis/stable-diffusion/blob/main/LICENSE). \n\n[Download ZIP](https://drive.google.com/file/d/1EU_qxlF-p6XrxsoACvcI2CAmu45NphAC/view?usp=share_link). Please don't use this URL in your software. \n\nWe'll get back to what to do with it in a second. \n\n## Option 2: Preparing your own `ckpt` file\n
If you want to use your own CKPT file (like a Dreambooth fine-tuning), you can convert it into Maple Diffusion format\n\n1. Download a Stable Diffusion model checkpoint to a folder, e.g. `~/Downloads/sd` ([`sd-v1-5.ckpt`](https://huggingface.co/runwayml/stable-diffusion-v1-5), or some derivation thereof)\n\n2. Setup & install Python with PyTorch, if you haven't already. \n\n```\n# Grab the converter script\ncd ~/Downloads/sd\ncurl https://raw.githubusercontent.com/mortenjust/maple-diffusion/main/Converter%20Script/maple-convert.py > maple-convert.py\n\n# may need to install conda first https://github.com/conda-forge/miniforge#homebrew\nconda deactivate\nconda remove -n native-diffusion --all\nconda create -n native-diffusion python=3.10\nconda activate native-diffusion\npip install torch typing_extensions numpy Pillow requests pytorch_lightning\n./native-convert.py ~/Downloads/sd-v1-4.ckpt\n```\nThe script will create a new folder called `bins`. We'll get back to what to do with it in a second.\n
\n\n# FAQ\n\n\n## Can I use a Dreambooth model?\nYes. Just copy the `alpha*` files from the standard conversion. This repo will include these files in the future. See [this issue](https://github.com/madebyollin/maple-diffusion/issues/22).\n\n## Does it support image to image prompting?\nYes. Simply pass in an `initImage` to your `SampleInput` when generating. \n\n## It crashes\nYou may need to regenerate the model files with the python script in the repo. This happens if you converted your ckpt model file before we added image2image.\n\n## Can I contribute? What's next?\nYes! A rough roadmap:\n\n- [ ] Stable Diffusion 2.0: - new larger output images, upscaling, depth-to-image\n- [ ] Add in-painting and out-painting\n- [ ] Generate other sizes and aspects than 512x512\n- [ ] Upscaling\n- [ ] Dreambooth training on-device\n- [x] Tighten up code quality overall. Most is proof of concept. \n- [x] Add image-to-image \n\nSee Issues for smaller contributions.\n\nIf you're making changes to the MPSGraph part of the codebase, consider making your contributions to the single-file repo and then integrate the changes in the wrapped file in this repo. \n\n## How fast is it? \nOn my MacBook Pro M1 Max, I get ~0.3s/step, which is significantly faster than any Python/PyTorch/Tensorflow installation I've tried. \n\nOn an iPhone it should take a minute or two. \n\nTo attain usable performance without tripping over iOS's 4GB memory limit, Native Diffusion relies internally on FP16 (NHWC) tensors, operator fusion from MPSGraph, and a truly pitiable degree of swapping models to device storage.\n\n## Does it support Stable Diffusion 2.0?\nNot yet. Would love some help on this. See above.\n\n## I have a question, comment or suggestion\nFeel free to post an issue!\n" }, { "path": "Sources/MapleDiffusion/Diffusion.swift", "content": "import Foundation\nimport Combine\nimport CoreGraphics\nimport CoreImage\n\n/**\n \n This is the package's wrapper for the `MapleDiffusion` class. It adds a few convenient ways to use @madebyollin's code.\n \n */\n\npublic class Diffusion : ObservableObject {\n \n /// Current state of the models. Only supports states for loading currently, updated via `initModels`\n public var state = PassthroughSubject()\n \n @Published public var isModelReady = false\n @Published public var loadingProgress : Double = 0.0\n \n var modelIsCold : Bool {\n print(\"coldness: \", isModelReady, loadingProgress)\n return !isModelReady && loadingProgress == 0 }\n \n var mapleDiffusion : MapleDiffusion!\n \n // Local, offline Stable Diffusion generation in Swift, no Python. Download + init + generate = 1 line of code.\n \n \n public static var shared = Diffusion()\n \n \n // TODO: Move to + generate\n // TODO: Add steps, guiadance etc as optional params\n public static func generate(localOrRemote modelURL: URL, prompt: String) async throws -> CGImage? {\n \n if shared.modelIsCold {\n if modelURL.isFileURL {\n try await shared.prepModels(localUrl: modelURL)\n } else {\n try await shared.prepModels(remoteURL: modelURL)\n }\n }\n return await shared.generate(input: SampleInput(prompt: prompt))\n }\n \n private var saveMemory = false\n\n public init(saveMemoryButBeSlower: Bool = false) {\n self.saveMemory = saveMemoryButBeSlower\n state.send(.notStarted)\n }\n\n /// Empty publisher for convenience in SwiftUI (since you can't listen to a nil)\n public static var placeholderPublisher : AnyPublisher { PassthroughSubject().eraseToAnyPublisher() }\n \n\n \n // Prep models\n \n \n /// Tuple type for easier access to the progress while also getting the full state.\n public typealias LoaderUpdate = (progress: Double, state: GeneratorState)\n \n /// Init models and update publishers. Run this off the main actor.\n /// TODO: 3 overloads: local only, remote only, both - build into arguments what they do\n var combinedProgress : Double = 0\n \n \n public func prepModels(localUrl: URL, progress:((Double)->Void)? = nil) async throws {\n let fetcher = ModelFetcher(local: localUrl)\n try await initModels(fetcher: fetcher, progress: progress)\n }\n \n public func prepModels(remoteURL: URL, progress:((Double)->Void)? = nil) async throws {\n let fetcher = ModelFetcher(remote: remoteURL)\n try await initModels(fetcher: fetcher, progress: progress)\n }\n \n public func prepModels(localUrl: URL, remoteUrl: URL, progress:((Double)->Void)? = nil) async throws {\n let fetcher = ModelFetcher(local: localUrl, remote: remoteUrl)\n try await initModels(fetcher: fetcher, progress: progress)\n }\n \n \n // Init Models\n \n \n private func initModels(\n fetcher: ModelFetcher,\n progress: ((Double)->Void)?\n ) async throws {\n let combinedSteps : Double = 2\n var combinedProgress : Double = 0\n \n await MainActor.run {\n self.loadingProgress = 0.05\n }\n \n /// 1. Fetch the model\n let modelLocation: URL = try await fetcher.fetch { p in\n combinedProgress = (p/combinedSteps)\n self.updateLoadingProgress(progress: combinedProgress, message: \"Fetching models\")\n }\n \n /// 2. instantiate MD, which has light side effects\n self.mapleDiffusion = MapleDiffusion(modelLocation: modelLocation, saveMemoryButBeSlower: saveMemory)\n \n// let earlierProgress = combinedProgress\n \n /// 3. Initialize models on a background thread\n// try await initModels() { p in\n// combinedProgress = (p/combinedSteps) + earlierProgress\n// self.updateLoadingProgress(progress: combinedProgress, message: \"Loading models\")\n// progress?(combinedProgress)\n// }\n \n /// 4. Done. Set published main status to true\n await MainActor.run {\n print(\"Model is ready\")\n self.state.send(.ready)\n self.loadingProgress = 1\n self.isModelReady = true\n }\n }\n \n private func updateLoadingProgress(progress: Double, message:String) {\n Task {\n await MainActor.run {\n self.state.send(GeneratorState.modelIsLoading(progress: progress, message: message))\n self.loadingProgress = progress\n }\n }\n }\n}\n" }, { "path": "Sources/MapleDiffusion/MPS/BPETokenizer.swift", "content": "//\n// BPETokenizer.swift\n// \n//\n// Created by Guillermo Cique Fernández on 9/11/22.\n//\n\nimport Foundation\nimport MetalPerformanceShadersGraph\n\nclass BPETokenizer {\n // why didn't they just byte-encode\n func whitespaceClean(s: String) -> String {\n return s.components(separatedBy: .whitespacesAndNewlines)\n .filter { !$0.isEmpty }\n .joined(separator: \" \")\n .trimmingCharacters(in: .whitespacesAndNewlines)\n }\n \n func getPairs(s: [String]) -> Set {\n return Set((1..\"})\n// `var vocabFileURL = modelFolder.appendingPathComponent(\"bpe_simple_vocab_16e6\").appendingPathExtension(\"txt\")`\n// let vocabFile = try! String(contentsOf: Bundle.main.url(forResource: \"bins/bpe_simple_vocab_16e6\", withExtension: \"txt\")!)\n \n let vocabFile = try! String(\n contentsOf: modelLocation\n .appendingPathComponent(\"bpe_simple_vocab_16e6\")\n .appendingPathExtension(\"txt\")\n )\n \n for (i, m) in vocabFile.split(separator: \"\\n\")[1..<48_895].enumerated() {\n ranks[String(m)] = i\n vocabList.append(m.split(separator: \" \").joined(separator: \"\"))\n }\n vocab = vocabList.enumerated().reduce(into: [:], {$0[$1.element] = $1.offset})\n }\n \n func encodeToken(s: String) -> [Int] {\n let token = String(s.utf8.map{bytesToUnicode[Int($0)]!})\n var word = token[..\"]\n var pairs = getPairs(s: Array(word))\n var mergedWordTokens = [token + \"\"]\n var count = 0\n if (!pairs.isEmpty) {\n while (true) {\n count += 1\n assert(count < 8192, \"encodeToken is trapped in a token factory for input \\(s)\")\n let highestRankedBigram = pairs.min(by: {ranks[$0, default: Int.max] < ranks[$1, default: Int.max]})!\n if (ranks[highestRankedBigram] == nil) { break }\n let fs = highestRankedBigram.split(separator: \" \")\n let (first, second) = (String(fs[0]), String(fs[1]))\n var (newWord, i) = ([String](), 0)\n while (i < word.count) {\n let j = word[i.. [Int] {\n let ns = NSString(string: whitespaceClean(s: s.lowercased()))\n var bpe: [Int] = []\n for match in pat.matches(in: String(ns), range: NSRange(location: 0, length: ns.length)) {\n bpe.append(contentsOf: encodeToken(s: ns.substring(with: match.range)))\n }\n if (bpe.count > 75) {\n print(\"Prompt of \\(bpe.count) bpe tokens will be truncated: \\(s)\")\n }\n return [49406] + bpe[.. MPSGraphTensorData {\n return graph.run(\n with: queue,\n feeds: [input: xIn],\n targetTensors: [output],\n targetOperations: nil\n )[output]!\n }\n}\n\nextension MPSGraph {\n func makeDecoder(at folder: URL, xIn: MPSGraphTensor) -> MPSGraphTensor {\n var x = xIn\n let name = \"first_stage_model.decoder\"\n x = multiplication(x, constant(1 / 0.18215, dataType: MPSDataType.float16), name: \"rescale\")\n x = makeConv(at: folder, xIn: x, name: \"first_stage_model.post_quant_conv\", outChannels: 4, khw: 1)\n x = makeConv(at: folder, xIn: x, name: name + \".conv_in\", outChannels: 512, khw: 3)\n \n // middle\n x = makeCoderResBlock(at: folder, xIn: x, name: name + \".mid.block_1\", outChannels: 512)\n x = makeCoderAttention(at: folder, xIn: x, name: name + \".mid.attn_1\")\n x = makeCoderResBlock(at: folder, xIn: x, name: name + \".mid.block_2\", outChannels: 512)\n \n // block 3\n x = makeCoderResBlock(at: folder, xIn: x, name: name + \".up.3.block.0\", outChannels: 512)\n x = makeCoderResBlock(at: folder, xIn: x, name: name + \".up.3.block.1\", outChannels: 512)\n x = makeCoderResBlock(at: folder, xIn: x, name: name + \".up.3.block.2\", outChannels: 512)\n x = upsampleNearest(xIn: x)\n x = makeConv(at: folder, xIn: x, name: name + \".up.3.upsample.conv\", outChannels: 512, khw: 3)\n \n // block 2\n x = makeCoderResBlock(at: folder, xIn: x, name: name + \".up.2.block.0\", outChannels: 512)\n x = makeCoderResBlock(at: folder, xIn: x, name: name + \".up.2.block.1\", outChannels: 512)\n x = makeCoderResBlock(at: folder, xIn: x, name: name + \".up.2.block.2\", outChannels: 512)\n x = upsampleNearest(xIn: x)\n x = makeConv(at: folder, xIn: x, name: name + \".up.2.upsample.conv\", outChannels: 512, khw: 3)\n \n // block 1\n x = makeCoderResBlock(at: folder, xIn: x, name: name + \".up.1.block.0\", outChannels: 256)\n x = makeCoderResBlock(at: folder, xIn: x, name: name + \".up.1.block.1\", outChannels: 256)\n x = makeCoderResBlock(at: folder, xIn: x, name: name + \".up.1.block.2\", outChannels: 256)\n x = upsampleNearest(xIn: x)\n x = makeConv(at: folder, xIn: x, name: name + \".up.1.upsample.conv\", outChannels: 256, khw: 3)\n \n // block 0\n x = makeCoderResBlock(at: folder, xIn: x, name: name + \".up.0.block.0\", outChannels: 128)\n x = makeCoderResBlock(at: folder, xIn: x, name: name + \".up.0.block.1\", outChannels: 128)\n x = makeCoderResBlock(at: folder, xIn: x, name: name + \".up.0.block.2\", outChannels: 128)\n \n x = makeGroupNormSwish(at: folder, xIn: x, name: name + \".norm_out\")\n x = makeConv(at: folder, xIn: x, name: name + \".conv_out\", outChannels: 3, khw: 3)\n x = addition(x, constant(1.0, dataType: MPSDataType.float16), name: nil)\n x = multiplication(x, constant(0.5, dataType: MPSDataType.float16), name: nil)\n return makeByteConverter(xIn: x)\n }\n}\n" }, { "path": "Sources/MapleDiffusion/MPS/Coder/Encoder.swift", "content": "//\n// Encoder.swift\n//\n//\n// Created by Guillermo Cique Fernández on 9/11/22.\n//\n\nimport Foundation\nimport MetalPerformanceShadersGraph\n\nclass Encoder {\n let device: MPSGraphDevice\n private let graph: MPSGraph\n private let encoderIn: MPSGraphTensor!\n private let encoderOut: MPSGraphTensor!\n private let noise: MPSGraphTensor!\n private let gaussianOut: MPSGraphTensor!\n private let scaled: MPSGraphTensor!\n private let stochasticEncode: MPSGraphTensor!\n private let stepIn: MPSGraphTensor!\n private let timestepsIn: MPSGraphTensor!\n \n init(synchronize: Bool,\n modelLocation: URL,\n device: MPSGraphDevice,\n inputShape: [NSNumber],\n outputShape: [NSNumber],\n timestepsShape: [NSNumber],\n seed: Int\n ) {\n self.device = device\n graph = MPSGraph(synchronize: synchronize)\n \n encoderIn = graph.placeholder(shape: inputShape, dataType: MPSDataType.uInt8, name: nil)\n encoderOut = graph.makeEncoder(at: modelLocation, xIn: encoderIn)\n \n noise = graph.randomTensor(\n withShape: outputShape,\n descriptor: MPSGraphRandomOpDescriptor(distribution: .normal, dataType: .float16)!,\n seed: seed,\n name: nil\n )\n gaussianOut = graph.diagonalGaussianDistribution(encoderOut, noise: noise)\n scaled = graph.multiplication(gaussianOut, graph.constant(0.18215, dataType: MPSDataType.float16), name: \"rescale\")\n \n stepIn = graph.placeholder(shape: [1], dataType: MPSDataType.int32, name: nil)\n timestepsIn = graph.placeholder(shape: timestepsShape, dataType: MPSDataType.int32, name: nil)\n stochasticEncode = graph.stochasticEncode(at: modelLocation, stepIn: stepIn, timestepsIn: timestepsIn, imageIn: scaled, noiseIn: noise)\n }\n \n func run(with queue: MTLCommandQueue, image: MPSGraphTensorData, step: Int, timesteps: MPSGraphTensorData) -> MPSGraphTensorData {\n let stepData = step.tensorData(device: device)\n \n return graph.run(\n with: queue,\n feeds: [\n encoderIn: image,\n stepIn: stepData,\n timestepsIn: timesteps\n ], targetTensors: [\n noise, encoderOut, gaussianOut, scaled, stochasticEncode\n ], targetOperations: nil\n )[stochasticEncode]!\n }\n}\n\nextension MPSGraph {\n func makeEncoder(at folder: URL, xIn: MPSGraphTensor) -> MPSGraphTensor {\n var x = xIn\n // Split into RBGA\n let xParts = split(x, numSplits: 4, axis: 2, name: nil)\n // Drop alpha channel\n x = concatTensors(xParts.dropLast(), dimension: 2, name: nil)\n x = cast(x, to: .float16, name: nil)\n x = division(x, constant(255.0, shape: [1], dataType: .float16), name: nil)\n x = expandDims(x, axis: 0, name: nil)\n x = multiplication(x, constant(2.0, shape: [1], dataType: .float16), name: nil)\n x = subtraction(x, constant(1.0, shape: [1], dataType: .float16), name: nil)\n \n let name = \"first_stage_model.encoder\"\n x = makeConv(at: folder, xIn: x, name: name + \".conv_in\", outChannels: 128, khw: 3)\n \n // block 0\n x = makeCoderResBlock(at: folder, xIn: x, name: name + \".down.0.block.0\", outChannels: 128)\n x = makeCoderResBlock(at: folder, xIn: x, name: name + \".down.0.block.1\", outChannels: 128)\n x = downsampleNearest(xIn: x)\n x = makeConv(at: folder, xIn: x, name: name + \".down.0.downsample.conv\", outChannels: 128, khw: 3)\n \n // block 1\n x = makeCoderResBlock(at: folder, xIn: x, name: name + \".down.1.block.0\", outChannels: 256)\n x = makeCoderResBlock(at: folder, xIn: x, name: name + \".down.1.block.1\", outChannels: 256)\n x = downsampleNearest(xIn: x)\n x = makeConv(at: folder, xIn: x, name: name + \".down.1.downsample.conv\", outChannels: 256, khw: 3)\n \n // block 2\n x = makeCoderResBlock(at: folder, xIn: x, name: name + \".down.2.block.0\", outChannels: 512)\n x = makeCoderResBlock(at: folder, xIn: x, name: name + \".down.2.block.1\", outChannels: 512)\n x = downsampleNearest(xIn: x)\n x = makeConv(at: folder, xIn: x, name: name + \".down.2.downsample.conv\", outChannels: 512, khw: 3)\n \n // block 3\n x = makeCoderResBlock(at: folder, xIn: x, name: name + \".down.3.block.0\", outChannels: 512)\n x = makeCoderResBlock(at: folder, xIn: x, name: name + \".down.3.block.1\", outChannels: 512)\n \n // middle\n x = makeCoderResBlock(at: folder, xIn: x, name: name + \".mid.block_1\", outChannels: 512)\n x = makeCoderAttention(at: folder, xIn: x, name: name + \".mid.attn_1\")\n x = makeCoderResBlock(at: folder, xIn: x, name: name + \".mid.block_2\", outChannels: 512)\n \n x = makeGroupNormSwish(at: folder, xIn: x, name: name + \".norm_out\")\n x = makeConv(at: folder, xIn: x, name: name + \".conv_out\", outChannels: 8, khw: 3)\n \n return makeConv(at: folder, xIn: x, name: \"first_stage_model.quant_conv\", outChannels: 8, khw: 1)\n }\n}\n" }, { "path": "Sources/MapleDiffusion/MPS/Coder/MPSGraph+Coder.swift", "content": "//\n// MPSGraph+Coder.swift\n//\n//\n// Created by Guillermo Cique Fernández on 9/11/22.\n//\n\nimport Foundation\nimport MetalPerformanceShadersGraph\n\nextension MPSGraph {\n func makeCoderResBlock(at folder: URL, xIn: MPSGraphTensor, name: String, outChannels: NSNumber) -> MPSGraphTensor {\n var x = xIn\n x = makeGroupNormSwish(at: folder, xIn: x, name: name + \".norm1\")\n x = makeConv(at: folder, xIn: x, name: name + \".conv1\", outChannels: outChannels, khw: 3)\n x = makeGroupNormSwish(at: folder, xIn: x, name: name + \".norm2\")\n x = makeConv(at: folder, xIn: x, name: name + \".conv2\", outChannels: outChannels, khw: 3)\n if (xIn.shape![3] != outChannels) {\n let ninShortcut = makeConv(at: folder, xIn: xIn, name: name + \".nin_shortcut\", outChannels: outChannels, khw: 1)\n return addition(x, ninShortcut, name: \"skip\")\n }\n return addition(x, xIn, name: \"skip\")\n }\n\n func makeCoderAttention(at folder: URL, xIn: MPSGraphTensor, name: String) -> MPSGraphTensor {\n var x = makeGroupNorm(at: folder, xIn: xIn, name: name + \".norm\")\n let c = x.shape![3]\n x = reshape(x, shape: [x.shape![0], NSNumber(value:x.shape![1].intValue * x.shape![2].intValue), c], name: nil)\n let q = makeLinear(at: folder, xIn: x, name: name + \".q\", outChannels: c, bias: false)\n var k = makeLinear(at: folder, xIn: x, name: name + \".k\", outChannels: c, bias: false)\n k = multiplication(k, constant(1.0 / sqrt(c.doubleValue), dataType: MPSDataType.float16), name: nil)\n k = transposeTensor(k, dimension: 1, withDimension: 2, name: nil)\n let v = makeLinear(at: folder, xIn: x, name: name + \".v\", outChannels: c, bias: false)\n var att = matrixMultiplication(primary: q, secondary: k, name: nil)\n att = softMax(with: att, axis: 2, name: nil)\n att = matrixMultiplication(primary: att, secondary: v, name: nil)\n x = makeLinear(at: folder, xIn: att, name: name + \".proj_out\", outChannels: c)\n x = reshape(x, shape: xIn.shape!, name: nil)\n return addition(x, xIn, name: nil)\n }\n}\n" }, { "path": "Sources/MapleDiffusion/MPS/Diffuser.swift", "content": "//\n// Diffuser.swift\n// \n//\n// Created by Guillermo Cique Fernández on 14/11/22.\n//\n\nimport Foundation\nimport MetalPerformanceShadersGraph\n\nclass Diffuser {\n private let device: MPSGraphDevice\n private let graph: MPSGraph\n private let xIn: MPSGraphTensor\n private let etaUncondIn: MPSGraphTensor\n private let etaCondIn: MPSGraphTensor\n private let timestepIn: MPSGraphTensor\n private let timestepSizeIn: MPSGraphTensor\n private let guidanceScaleIn: MPSGraphTensor\n private let out: MPSGraphTensor\n private let auxOut: MPSGraphTensor\n \n init(synchronize: Bool, modelLocation: URL, device: MPSGraphDevice, shape: [NSNumber]) {\n self.device = device\n graph = MPSGraph(synchronize: synchronize)\n xIn = graph.placeholder(shape: shape, dataType: MPSDataType.float16, name: nil)\n etaUncondIn = graph.placeholder(shape: shape, dataType: MPSDataType.float16, name: nil)\n etaCondIn = graph.placeholder(shape: shape, dataType: MPSDataType.float16, name: nil)\n timestepIn = graph.placeholder(shape: [1], dataType: MPSDataType.int32, name: nil)\n timestepSizeIn = graph.placeholder(shape: [1], dataType: MPSDataType.int32, name: nil)\n guidanceScaleIn = graph.placeholder(shape: [1], dataType: MPSDataType.float32, name: nil)\n out = graph.makeDiffusionStep(\n at: modelLocation,\n xIn: xIn,\n etaUncondIn: etaUncondIn,\n etaCondIn: etaCondIn,\n timestepIn: timestepIn,\n timestepSizeIn: timestepSizeIn,\n guidanceScaleIn: graph.cast(guidanceScaleIn, to: MPSDataType.float16, name: \"this string must not be the empty string\")\n )\n auxOut = graph.makeAuxUpsampler(at: modelLocation, xIn: out)\n }\n \n func run(\n with queue: MTLCommandQueue,\n latent: MPSGraphTensorData,\n timestep: Int,\n timestepSize: Int,\n etaUncond: MPSGraphTensorData,\n etaCond: MPSGraphTensorData,\n guidanceScale: MPSGraphTensorData\n ) -> (MPSGraphTensorData, MPSGraphTensorData?) {\n let timestepData = timestep.tensorData(device: device)\n let timestepSizeData = timestepSize.tensorData(device: device)\n let outputs = graph.run(\n with: queue,\n feeds: [\n xIn: latent,\n etaUncondIn: etaUncond,\n etaCondIn: etaCond,\n timestepIn: timestepData,\n timestepSizeIn: timestepSizeData,\n guidanceScaleIn: guidanceScale\n ],\n targetTensors: [out, auxOut],\n targetOperations: nil\n )\n return (outputs[out]!, outputs[auxOut])\n }\n}\n\nfileprivate extension MPSGraph {\n func makeDiffusionStep(\n at folder: URL,\n xIn: MPSGraphTensor,\n etaUncondIn: MPSGraphTensor,\n etaCondIn: MPSGraphTensor,\n timestepIn: MPSGraphTensor,\n timestepSizeIn: MPSGraphTensor,\n guidanceScaleIn: MPSGraphTensor\n ) -> MPSGraphTensor {\n \n // superconditioning\n var deltaCond = multiplication(subtraction(etaCondIn, etaUncondIn, name: nil), guidanceScaleIn, name: nil)\n deltaCond = tanh(with: deltaCond, name: nil) // NOTE: normal SD doesn't clamp here iirc\n let eta = addition(etaUncondIn, deltaCond, name: nil)\n \n // scheduler conditioning\n let alphasCumprod = loadConstant(at: folder, name: \"alphas_cumprod\", shape: [1000])\n let alphaIn = gatherAlongAxis(0, updates: alphasCumprod, indices: timestepIn, name: nil)\n let prevTimestep = maximum(\n constant(0, dataType: MPSDataType.int32),\n subtraction(timestepIn, timestepSizeIn, name: nil),\n name: nil\n )\n let alphaPrevIn = gatherAlongAxis(0, updates: alphasCumprod, indices: prevTimestep, name: nil)\n \n // scheduler step\n let deltaX0 = multiplication(squareRootOfOneMinus(alphaIn), eta, name: nil)\n let predX0Unscaled = subtraction(xIn, deltaX0, name: nil)\n let predX0 = division(predX0Unscaled, squareRoot(with: alphaIn, name: nil), name: nil)\n let dirX = multiplication(squareRootOfOneMinus(alphaPrevIn), eta, name: nil)\n let xPrevBase = multiplication(squareRoot(with: alphaPrevIn, name: nil), predX0, name:nil)\n return addition(xPrevBase, dirX, name: nil)\n }\n \n func makeAuxUpsampler(at folder: URL, xIn: MPSGraphTensor) -> MPSGraphTensor {\n var x = xIn\n x = makeConv(at: folder, xIn: xIn, name: \"aux_output_conv\", outChannels: 3, khw: 1)\n x = upsampleNearest(xIn: x, scaleFactor: 8)\n return makeByteConverter(xIn: x)\n }\n}\n" }, { "path": "Sources/MapleDiffusion/MPS/MPSGraph+Transformer.swift", "content": "//\n// MPSGraph+Transformer.swift\n// \n//\n// Created by Guillermo Cique Fernández on 9/11/22.\n//\n\nimport Foundation\nimport MetalPerformanceShadersGraph\n\nextension MPSGraph {\n func makeSpatialTransformerBlock(at folder: URL, xIn: MPSGraphTensor, name: String, contextIn: MPSGraphTensor, saveMemory: Bool) -> MPSGraphTensor {\n let n, h, w, c: NSNumber\n (n, h, w, c) = (xIn.shape![0], xIn.shape![1], xIn.shape![2], xIn.shape![3])\n var x = xIn\n x = makeGroupNorm(at: folder, xIn: x, name: name + \".norm\")\n x = makeConv(at: folder, xIn: x, name: name + \".proj_in\", outChannels: c, khw: 1)\n x = reshape(x, shape: [n, (h.intValue * w.intValue) as NSNumber, c], name: nil)\n x = makeBasicTransformerBlock(at: folder, xIn: x, name: name + \".transformer_blocks.0\", contextIn: contextIn, saveMemory: saveMemory)\n x = reshape(x, shape: [n, h, w, c], name: nil)\n x = makeConv(at: folder, xIn: x, name: name + \".proj_out\", outChannels: c, khw: 1)\n return addition(x, xIn, name: nil)\n }\n \n fileprivate func makeBasicTransformerBlock(at folder: URL, xIn: MPSGraphTensor, name: String, contextIn: MPSGraphTensor, saveMemory: Bool) -> MPSGraphTensor {\n var x = xIn\n var attn1 = makeLayerNorm(at: folder, xIn: x, name: name + \".norm1\")\n attn1 = makeCrossAttention(at: folder, xIn: attn1, name: name + \".attn1\", context: nil, saveMemory: saveMemory)\n x = addition(attn1, x, name: nil)\n var attn2 = makeLayerNorm(at: folder, xIn: x, name: name + \".norm2\")\n attn2 = makeCrossAttention(at: folder, xIn: attn2, name: name + \".attn2\", context: contextIn, saveMemory: saveMemory)\n x = addition(attn2, x, name: nil)\n var ff = makeLayerNorm(at: folder, xIn: x, name: name + \".norm3\")\n ff = makeFeedForward(at: folder, xIn: ff, name: name + \".ff.net\")\n return addition(ff, x, name: nil)\n }\n \n fileprivate func makeFeedForward(at folder: URL, xIn: MPSGraphTensor, name: String) -> MPSGraphTensor {\n assert(xIn.shape!.count == 3)\n let dim = xIn.shape![2]\n let dimMult = dim.intValue * 4\n let dimProj = NSNumber(value: dimMult * 2)\n let proj = makeLinear(at: folder, xIn: xIn, name: name + \".0.proj\", outChannels: dimProj)\n var x = sliceTensor(proj, dimension: 2, start: 0, length: dimMult, name: nil)\n var gate = sliceTensor(proj, dimension: 2, start: dimMult, length: dimMult, name: nil)\n gate = gelu(gate)\n x = multiplication(x, gate, name: nil)\n return makeLinear(at: folder, xIn: x, name: name + \".2\", outChannels: dim)\n }\n \n fileprivate func makeCrossAttention(at folder: URL, xIn: MPSGraphTensor, name: String, context: MPSGraphTensor?, saveMemory: Bool) -> MPSGraphTensor {\n let c = xIn.shape![2]\n let (nHeads, dHead) = (NSNumber(8), NSNumber(value: c.intValue / 8))\n var q = makeLinear(at: folder, xIn: xIn, name: name + \".to_q\", outChannels: c, bias: false)\n let context = context ?? xIn\n var k = makeLinear(at: folder, xIn: context, name: name + \".to_k\", outChannels: c, bias: false)\n var v = makeLinear(at: folder, xIn: context, name: name + \".to_v\", outChannels: c, bias: false)\n let n = xIn.shape![0]\n let hw = xIn.shape![1]\n let t = context.shape![1]\n q = reshape(q, shape: [n, hw, nHeads, dHead], name: nil)\n k = reshape(k, shape: [n, t, nHeads, dHead], name: nil)\n v = reshape(v, shape: [n, t, nHeads, dHead], name: nil)\n \n q = transposeTensor(q, dimension: 1, withDimension: 2, name: nil)\n k = transposeTensor(k, dimension: 1, withDimension: 2, name: nil)\n k = transposeTensor(k, dimension: 2, withDimension: 3, name: nil)\n k = multiplication(k, constant(1.0 / sqrt(dHead.doubleValue), dataType: MPSDataType.float16), name: nil)\n v = transposeTensor(v, dimension: 1, withDimension: 2, name: nil)\n \n var att: MPSGraphTensor\n if (saveMemory) {\n // MEM-HACK - silly graph seems to use less peak memory\n var attRes = [MPSGraphTensor]()\n let sliceSize = 1\n for i in 0.. MPSGraphTensor {\n let numels = shape.reduce(into: 1, { accumulator, value in\n accumulator *= value.intValue\n })\n let fileUrl: URL = folder.appendingPathComponent(name + (fp32 ? \"_fp32\" : \"\")).appendingPathExtension(\"bin\")\n let data: Data = try! Data(contentsOf: fileUrl, options: Data.ReadingOptions.alwaysMapped)\n let expectedCount = numels * (fp32 ? 4 : 2)\n assert(data.count == expectedCount, \"Mismatch between byte count of data \\(data.count) and expected size \\(expectedCount) for \\(numels) els in \\(fileUrl)\")\n return constant(data, shape: shape, dataType: fp32 ? MPSDataType.float32 : MPSDataType.float16)\n }\n \n \n func makeConv(at folder: URL, xIn: MPSGraphTensor, name: String, outChannels: NSNumber, khw: NSNumber, stride: Int = 1, bias: Bool = true) -> MPSGraphTensor {\n let w = loadConstant(at: folder, name: name + \".weight\", shape: [outChannels, xIn.shape![3], khw, khw])\n let p: Int = khw.intValue / 2;\n let convDesc = MPSGraphConvolution2DOpDescriptor(\n strideInX: stride,\n strideInY: stride,\n dilationRateInX: 1,\n dilationRateInY: 1,\n groups: 1,\n paddingLeft: p,\n paddingRight: p,\n paddingTop: p,\n paddingBottom: p,\n paddingStyle: MPSGraphPaddingStyle.explicit,\n dataLayout: MPSGraphTensorNamedDataLayout.NHWC,\n weightsLayout: MPSGraphTensorNamedDataLayout.OIHW\n )!\n let conv = convolution2D(xIn, weights: w, descriptor: convDesc, name: nil)\n if (bias) {\n let b = loadConstant(at: folder, name: name + \".bias\", shape: [1, 1, 1, outChannels])\n return addition(conv, b, name: nil)\n }\n return conv\n }\n \n func makeLinear(at folder: URL, xIn: MPSGraphTensor, name: String, outChannels: NSNumber, bias: Bool = true) -> MPSGraphTensor {\n if (xIn.shape!.count == 2) {\n var x = reshape(xIn, shape: [xIn.shape![0], 1, 1, xIn.shape![1]], name: nil)\n x = makeConv(at: folder, xIn: x, name: name, outChannels: outChannels, khw: 1, bias: bias)\n return reshape(x, shape: [xIn.shape![0], outChannels], name: nil)\n }\n var x = reshape(xIn, shape: [xIn.shape![0], 1, xIn.shape![1], xIn.shape![2]], name: nil)\n x = makeConv(at: folder, xIn: x, name: name, outChannels: outChannels, khw: 1, bias: bias)\n return reshape(x, shape: [xIn.shape![0], xIn.shape![1], outChannels], name: nil)\n }\n \n \n func makeLayerNorm(at folder: URL, xIn: MPSGraphTensor, name: String) -> MPSGraphTensor {\n assert(xIn.shape!.count == 3, \"layernorm requires NTC\")\n let gamma = loadConstant(at: folder, name: name + \".weight\", shape: [1, 1, xIn.shape![2]])\n let beta = loadConstant(at: folder, name: name + \".bias\", shape: [1, 1, xIn.shape![2]])\n let mean = mean(of: xIn, axes: [2], name: nil)\n let variance = variance(of: xIn, axes: [2], name: nil)\n let x = normalize(xIn, mean: mean, variance: variance, gamma: gamma, beta: beta, epsilon: 1e-5, name: nil)\n return reshape(x, shape: xIn.shape!, name: nil)\n }\n \n func makeGroupNorm(at folder: URL, xIn: MPSGraphTensor, name: String) -> MPSGraphTensor {\n var x = xIn\n if (xIn.shape!.count == 3) {\n x = expandDims(x, axes: [1], name: nil)\n }\n let shape = x.shape!\n let nGroups: NSNumber = 32\n let nGrouped: NSNumber = shape[3].floatValue / nGroups.floatValue as NSNumber\n let gamma = loadConstant(at: folder, name: name + \".weight\", shape: [1, 1, 1, nGroups, nGrouped])\n let beta = loadConstant(at: folder, name: name + \".bias\", shape: [1, 1, 1, nGroups, nGrouped])\n x = reshape(x, shape: [shape[0], shape[1], shape[2], nGroups, nGrouped], name: nil)\n let mean = mean(of: x, axes: [1, 2, 4], name: nil)\n let variance = variance(of: x, axes: [1, 2, 4], name: nil)\n x = normalize(x, mean: mean, variance: variance, gamma: gamma, beta: beta, epsilon: 1e-5, name: nil)\n return reshape(x, shape: xIn.shape!, name: nil)\n }\n\n func makeGroupNormSwish(at folder: URL, xIn: MPSGraphTensor, name: String) -> MPSGraphTensor {\n return swish(makeGroupNorm(at: folder, xIn: xIn, name: name))\n }\n \n func makeByteConverter(xIn: MPSGraphTensor) -> MPSGraphTensor {\n var x = xIn\n x = clamp(x, min: constant(0, shape: [1], dataType: MPSDataType.float16), max: constant(1.0, shape: [1], dataType: MPSDataType.float16), name: nil)\n x = multiplication(x, constant(255, shape: [1], dataType: MPSDataType.float16), name: nil)\n x = round(with: x, name: nil)\n x = cast(x, to: MPSDataType.uInt8, name: \"cast to uint8 rgba\")\n let alpha = constant(255, shape: [1, x.shape![1], x.shape![2], 1], dataType: MPSDataType.uInt8)\n return concatTensors([x, alpha], dimension: 3, name: nil)\n }\n \n func stochasticEncode(at folder: URL, stepIn: MPSGraphTensor, timestepsIn: MPSGraphTensor, imageIn: MPSGraphTensor, noiseIn: MPSGraphTensor) -> MPSGraphTensor {\n let alphasCumprod = loadConstant(at: folder, name: \"alphas_cumprod\", shape: [1000])\n let alphas = gatherAlongAxis(0, updates: alphasCumprod, indices: timestepsIn, name: nil)\n let sqrtAlphasCumprod = squareRoot(with: alphas, name: nil)\n let sqrtOneMinusAlphasCumprod = squareRootOfOneMinus(alphas)\n \n let imageAlphas = multiplication(extractIntoTensor(a: sqrtAlphasCumprod, t: stepIn, shape: imageIn.shape!), imageIn, name: nil)\n let noiseAlphas = multiplication(extractIntoTensor(a: sqrtOneMinusAlphasCumprod, t: stepIn, shape: imageIn.shape!), noiseIn, name: nil)\n return addition(imageAlphas, noiseAlphas, name: nil)\n }\n}\n\n// MARK: Operations\n\nextension MPSGraph {\n func swish(_ tensor: MPSGraphTensor) -> MPSGraphTensor {\n return multiplication(tensor, sigmoid(with: tensor, name: nil), name: nil)\n }\n \n func upsampleNearest(xIn: MPSGraphTensor, scaleFactor: Int = 2) -> MPSGraphTensor {\n return resize(\n xIn,\n size: [\n NSNumber(value:xIn.shape![1].intValue * scaleFactor),\n NSNumber(value:xIn.shape![2].intValue * scaleFactor)\n ],\n mode: MPSGraphResizeMode.nearest,\n centerResult: true,\n alignCorners: false,\n layout: MPSGraphTensorNamedDataLayout.NHWC,\n name: nil\n )\n }\n \n func downsampleNearest(xIn: MPSGraphTensor, scaleFactor: Int = 2) -> MPSGraphTensor {\n return resize(\n xIn,\n size: [\n NSNumber(value:xIn.shape![1].intValue / scaleFactor),\n NSNumber(value:xIn.shape![2].intValue / scaleFactor)\n ],\n mode: MPSGraphResizeMode.nearest,\n centerResult: true,\n alignCorners: false,\n layout: MPSGraphTensorNamedDataLayout.NHWC,\n name: nil\n )\n }\n \n func squareRootOfOneMinus(_ tensor: MPSGraphTensor) -> MPSGraphTensor {\n return squareRoot(with: subtraction(constant(1.0, dataType: MPSDataType.float16), tensor, name: nil), name: nil)\n }\n \n // Gaussian Error Linear Units\n func gelu(_ tensor: MPSGraphTensor) -> MPSGraphTensor {\n var x = tensor\n x = multiplication(x, constant(1/sqrt(2), dataType: MPSDataType.float16), name: nil)\n x = erf(with: x, name: nil)\n x = addition(x, constant(1, dataType: MPSDataType.float16), name: nil)\n x = multiplication(x, constant(0.5, dataType: MPSDataType.float16), name: nil)\n return multiplication(tensor, x, name: nil)\n }\n \n func diagonalGaussianDistribution(_ tensor: MPSGraphTensor, noise: MPSGraphTensor) -> MPSGraphTensor {\n let chunks = split(tensor, numSplits: 2, axis: 3, name: nil)\n let mean = chunks[0]\n let logvar = clamp(chunks[1],\n min: constant(-30, shape: [1], dataType: MPSDataType.float16),\n max: constant(20, shape: [1], dataType: MPSDataType.float16),\n name: nil)\n let std = exponent(with: multiplication(constant(0.5, shape: [1], dataType: MPSDataType.float16), logvar, name: nil), name: nil)\n return addition(mean, multiplication(std, noise, name: nil), name: nil)\n }\n \n func extractIntoTensor(a: MPSGraphTensor, t: MPSGraphTensor, shape: [NSNumber]) -> MPSGraphTensor {\n let out = gatherAlongAxis(-1, updates: a, indices: t, name: nil)\n return reshape(out, shape: [t.shape!.first!] + [NSNumber](repeating: 1, count: shape.count-1), name: nil)\n }\n}\n" }, { "path": "Sources/MapleDiffusion/MPS/MapleDiffusion.swift", "content": "import MetalPerformanceShadersGraph\nimport Foundation\n\n// Maple Diffusion implements stable diffusion (original v1.4 model)\n// inference via MPSGraph. iOS has a hard memory limit of 4GB (with\n// a special entitlement), so this implementation trades off latency\n// for memory usage in many places (tagged with MEM-HACK) in order to\n// stay under the limit and minimize probability of oom.\n\n/**\n \n When updating\n 1. Paste in the entire new file\n 2. Replace references to Bundle.main like this\n \n `let fileUrl: URL = Bundle.main.url(forResource: \"bins/\" + name + (fp32 ? \"_fp32\" : \"\"), withExtension: \".bin\")!`\n to\n`let fileUrl: URL = modelFolder.appendingPathComponent(name + (fp32 ? \"_fp32\" : \"\")).appendingPathExtension(\"bin\")`\n \n and also\n ```\n let vocabFile = try! String(\n contentsOf: modelFolder\n .appendingPathComponent(\"bpe_simple_vocab_16e6\")\n .appendingPathExtension(\"txt\")\n )\n ```\n \n */\n\n// madebyollin's code starts here:\n\nclass MapleDiffusion {\n let device: MTLDevice\n let graphDevice: MPSGraphDevice\n let commandQueue: MTLCommandQueue\n let saveMemory: Bool\n let synchronize: Bool\n let modelLocation: URL\n \n private var _textGuidance: TextGuidance?\n var textGuidance: TextGuidance {\n if let _textGuidance {\n return _textGuidance\n }\n let textGuidance = TextGuidance(\n synchronize: synchronize,\n modelLocation: modelLocation,\n device: graphDevice\n )\n _textGuidance = textGuidance\n return textGuidance\n }\n \n private var _uNet: UNet?\n var uNet: UNet {\n if let _uNet {\n return _uNet\n }\n let uNet = UNet(\n synchronize: synchronize,\n modelLocation: modelLocation,\n saveMemory: saveMemory,\n device: graphDevice,\n shape: [1, height, width, 4]\n )\n _uNet = uNet\n return uNet\n }\n \n lazy var diffuser: Diffuser = {\n Diffuser(\n synchronize: synchronize,\n modelLocation: modelLocation,\n device: graphDevice,\n shape: [1, height, width, 4]\n )\n }()\n \n private var _decoder: Decoder?\n var decoder: Decoder {\n if let _decoder {\n return _decoder\n }\n let decoder = Decoder(\n synchronize: synchronize,\n modelLocation: modelLocation,\n device: graphDevice,\n shape: [1, height, width, 4]\n )\n _decoder = decoder\n return decoder\n }\n \n var width: NSNumber = 64\n var height: NSNumber = 64\n \n public init(modelLocation: URL, saveMemoryButBeSlower: Bool = true) {\n self.modelLocation = modelLocation\n saveMemory = saveMemoryButBeSlower\n device = MTLCreateSystemDefaultDevice()!\n graphDevice = MPSGraphDevice(mtlDevice: device)\n commandQueue = device.makeCommandQueue()!\n synchronize = !device.hasUnifiedMemory\n }\n \n private func runTextGuidance(prompt: String, negativePrompt: String) -> (MPSGraphTensorData, MPSGraphTensorData) {\n let guidance = textGuidance.run(with: commandQueue, prompt: prompt, negativePrompt: negativePrompt)\n if saveMemory {\n // MEM-HACK unload the text guidance to fit the unet\n _textGuidance = nil\n }\n return guidance\n }\n \n private func initLatent(input: SampleInput, scheduler: Scheduler) -> MPSGraphTensorData {\n if let image = input.initImage, let strength = input.strength {\n let imageData = MPSGraphTensorData(device: graphDevice, cgImage: image)\n let timestepsData = scheduler.timestepsData\n let startStep = Int(Float(input.steps) * strength)\n let encoder = Encoder(\n synchronize: synchronize,\n modelLocation: modelLocation,\n device: graphDevice,\n inputShape: imageData.shape,\n outputShape: [1, height, width, 4],\n timestepsShape: timestepsData.shape,\n seed: input.seed\n )\n return encoder.run(with: commandQueue, image: imageData, step: startStep, timesteps: timestepsData)\n } else {\n let graph = MPSGraph(synchronize: synchronize)\n let out = graph.randomTensor(\n withShape: [1, height, width, 4],\n descriptor: MPSGraphRandomOpDescriptor(distribution: .normal, dataType: .float16)!,\n seed: input.seed,\n name: nil\n )\n return graph.run(with: commandQueue, feeds: [:], targetTensors: [out], targetOperations: nil)[out]!\n }\n }\n \n private func sample(\n latent: inout MPSGraphTensorData,\n input: SampleInput,\n baseGuidance: MPSGraphTensorData,\n textGuidance: MPSGraphTensorData,\n scheduler: Scheduler,\n completion: @escaping (CGImage?, Float, String) -> ()\n ) {\n let guidanceScaleData = input.guidanceScale.tensorData(device: graphDevice)\n let actualTimesteps = scheduler.timesteps(strength: input.strength)\n for (index, timestep) in actualTimesteps.enumerated() {\n let tick = CFAbsoluteTimeGetCurrent()\n \n let temb = scheduler.run(with: commandQueue, timestep: timestep)\n let (etaUncond, etaCond) = uNet.run(\n with: commandQueue,\n latent: latent,\n baseGuidance: baseGuidance,\n textGuidance: textGuidance,\n temb: temb\n )\n let (newLatent, auxOut) = diffuser.run(\n with: commandQueue,\n latent: latent,\n timestep: timestep,\n timestepSize: scheduler.timestepSize,\n etaUncond: etaUncond,\n etaCond: etaCond,\n guidanceScale: guidanceScaleData\n )\n latent = newLatent\n \n // update ui\n let tock = CFAbsoluteTimeGetCurrent()\n let stepRuntime = String(format:\"%.2fs\", tock - tick)\n let progressDesc = index == 0 ? \"Decoding...\" : \"Step \\(index) / \\(actualTimesteps.count) (\\(stepRuntime) / step)\"\n let outImage = auxOut?.cgImage\n let progress = 0.1 + (Float(index) / Float(actualTimesteps.count)) * 0.8\n completion(outImage, progress, progressDesc)\n }\n \n if saveMemory {\n // MEM-HACK: unload the unet to fit the decoder\n _uNet = nil\n }\n }\n \n private func runDecoder(latent: MPSGraphTensorData) -> CGImage? {\n let decodedLatent = decoder.run(with: commandQueue, xIn: latent)\n if saveMemory {\n // MEM-HACK unload the decoder\n _decoder = nil\n }\n return decodedLatent.cgImage\n }\n \n public func generate(\n input: SampleInput,\n completion: @escaping (CGImage?, Float, String) -> ()\n ) {\n let mainTick = CFAbsoluteTimeGetCurrent()\n \n // 1. String -> Embedding\n completion(input.initImage, 0, \"Tokenizing...\")\n let (baseGuidance, textGuidance) = runTextGuidance(prompt: input.prompt, negativePrompt: input.negativePrompt)\n \n // 2. Noise generation\n completion(input.initImage, 0.05, \"Generating noise...\")\n let scheduler = Scheduler(synchronize: synchronize, modelLocation: modelLocation, device: graphDevice, steps: input.steps)\n var latent = initLatent(input: input, scheduler: scheduler)\n \n // 3. Diffusion\n let startImage: CGImage?\n if saveMemory {\n startImage = input.initImage\n } else {\n startImage = runDecoder(latent: latent)\n }\n completion(startImage, 0.1, \"Starting diffusion...\")\n sample(\n latent: &latent,\n input: input,\n baseGuidance: baseGuidance,\n textGuidance: textGuidance,\n scheduler: scheduler,\n completion: completion\n )\n \n // 4. Decoder\n let finalImage = runDecoder(latent: latent)\n completion(finalImage, 1.0, \"Cooling down...\")\n let mainTock = CFAbsoluteTimeGetCurrent()\n let runtime = String(format:\"%.2fs\", mainTock - mainTick)\n print(\"Time\", runtime)\n }\n}\n" }, { "path": "Sources/MapleDiffusion/MPS/Scheduler.swift", "content": "//\n// Scheduler.swift\n// \n//\n// Created by Guillermo Cique Fernández on 13/11/22.\n//\n\nimport Foundation\nimport MetalPerformanceShadersGraph\n\nclass Scheduler {\n let count: Int\n private let timesteps: [Int]\n let timestepSize: Int\n var timestepsData: MPSGraphTensorData {\n let data = timesteps.map { Int32($0) }.withUnsafeBufferPointer { Data(buffer: $0) }\n return MPSGraphTensorData(\n device: device,\n data: data,\n shape: [NSNumber(value: timesteps.count)],\n dataType: MPSDataType.int32\n )\n }\n \n private let device: MPSGraphDevice\n private let graph: MPSGraph\n private let timestepIn: MPSGraphTensor\n private let tembOut: MPSGraphTensor\n \n init(synchronize: Bool, modelLocation: URL, device: MPSGraphDevice, steps: Int) {\n self.device = device\n count = steps\n \n timestepSize = 1000 / steps\n timesteps = Array(stride(from: 1, to: 1000, by: timestepSize))\n graph = MPSGraph(synchronize: synchronize)\n timestepIn = graph.placeholder(shape: [1], dataType: MPSDataType.int32, name: nil)\n tembOut = graph.makeTimeFeatures(at: modelLocation, tIn: timestepIn)\n }\n \n func timesteps(strength: Float?) -> [Int] {\n guard let strength else { return timesteps.reversed() }\n let startStep = Int(Float(count) * strength)\n return timesteps[0.. MPSGraphTensorData {\n let timestepData = [Int32(timestep)].withUnsafeBufferPointer { Data(buffer: $0) }\n let data = MPSGraphTensorData(device: device, data: timestepData, shape: [1], dataType: MPSDataType.int32)\n return graph.run(\n with: queue,\n feeds: [timestepIn: data],\n targetTensors: [tembOut],\n targetOperations: nil\n )[tembOut]!\n }\n}\n\nextension MPSGraph {\n func makeTimeFeatures(at folder: URL, tIn: MPSGraphTensor) -> MPSGraphTensor {\n var temb = cast(tIn, to: MPSDataType.float32, name: \"temb\")\n var coeffs = loadConstant(at: folder, name: \"temb_coefficients\", shape: [160], fp32: true)\n coeffs = cast(coeffs, to: MPSDataType.float32, name: \"coeffs\")\n temb = multiplication(temb, coeffs, name: nil)\n temb = concatTensors([cos(with: temb, name: nil), sin(with: temb, name: nil)], dimension: 0, name: nil)\n temb = reshape(temb, shape: [1, 320], name: nil)\n return cast(temb, to: MPSDataType.float16, name: \"temb fp16\")\n }\n}\n" }, { "path": "Sources/MapleDiffusion/MPS/TextGuidance.swift", "content": "//\n// TextGuidance.swift\n// \n//\n// Created by Guillermo Cique Fernández on 13/11/22.\n//\n\nimport Foundation\nimport MetalPerformanceShadersGraph\n\nclass TextGuidance {\n private let device: MPSGraphDevice\n private let tokenizer: BPETokenizer\n private let executable: MPSGraphExecutable\n \n init(synchronize: Bool, modelLocation: URL, device: MPSGraphDevice) {\n self.device = device\n self.tokenizer = BPETokenizer(modelLocation: modelLocation)\n \n let graph = MPSGraph(synchronize: synchronize)\n let textGuidanceIn = graph.placeholder(shape: [2, 77], dataType: MPSDataType.int32, name: nil)\n let textGuidanceOut = graph.makeTextGuidance(at: modelLocation, xIn: textGuidanceIn, name: \"cond_stage_model.transformer.text_model\")\n let textGuidanceOut0 = graph.sliceTensor(textGuidanceOut, dimension: 0, start: 0, length: 1, name: nil)\n let textGuidanceOut1 = graph.sliceTensor(textGuidanceOut, dimension: 0, start: 1, length: 1, name: nil)\n self.executable = graph.compile(\n with: device,\n feeds: [\n textGuidanceIn: MPSGraphShapedType(shape: textGuidanceIn.shape, dataType: MPSDataType.int32)\n ],\n targetTensors: [textGuidanceOut0, textGuidanceOut1],\n targetOperations: nil,\n compilationDescriptor: nil\n )\n }\n \n func run(with queue: MTLCommandQueue, prompt: String, negativePrompt: String) -> (MPSGraphTensorData, MPSGraphTensorData) {\n let baseTokens = tokenizer.encode(s: negativePrompt)\n let tokens = tokenizer.encode(s: prompt)\n \n let data = (baseTokens + tokens).map {Int32($0)}\n .withUnsafeBufferPointer { Data(buffer: $0) }\n let tensorData = MPSGraphTensorData(device: device, data: data, shape: [2, 77], dataType: MPSDataType.int32)\n let res = executable.run(with: queue, inputs: [tensorData], results: nil, executionDescriptor: nil)\n return (res[0], res[1])\n }\n}\n\nfileprivate extension MPSGraph {\n func makeTextGuidance(at folder: URL, xIn: MPSGraphTensor, name: String) -> MPSGraphTensor {\n var x = makeTextEmbeddings(at: folder, xIn: xIn, name: name + \".embeddings\")\n x = makeTextEncoder(at: folder, xIn: x, name: name + \".encoder\")\n return makeLayerNorm(at: folder, xIn: x, name: name + \".final_layer_norm\")\n }\n \n func makeTextEmbeddings(at folder: URL, xIn: MPSGraphTensor, name: String) -> MPSGraphTensor {\n var tokenEmbeddings = loadConstant(at: folder, name: name + \".token_embedding.weight\", shape: [1, 49408, 768])\n tokenEmbeddings = broadcast(tokenEmbeddings, shape: [2, 49408, 768], name: nil)\n let positionEmbeddings = loadConstant(at: folder, name: name + \".position_embedding.weight\", shape: [1, 77, 768])\n var embeddings = broadcast(expandDims(xIn, axes: [2], name: nil), shape: [2, 77, 768], name: nil)\n embeddings = gatherAlongAxis(1, updates: tokenEmbeddings, indices: embeddings, name: nil)\n return addition(embeddings, positionEmbeddings, name: nil)\n }\n \n func makeTextAttention(at folder: URL, xIn: MPSGraphTensor, name: String) -> MPSGraphTensor {\n let nHeads: NSNumber = 12\n let dHead: NSNumber = 64\n let c: NSNumber = 768\n var q = makeLinear(at: folder, xIn: xIn, name: name + \".q_proj\", outChannels: c)\n var k = makeLinear(at: folder, xIn: xIn, name: name + \".k_proj\", outChannels: c)\n var v = makeLinear(at: folder, xIn: xIn, name: name + \".v_proj\", outChannels: c)\n \n let n = xIn.shape![0]\n let t = xIn.shape![1]\n q = reshape(q, shape: [n, t, nHeads, dHead], name: nil)\n k = reshape(k, shape: [n, t, nHeads, dHead], name: nil)\n v = reshape(v, shape: [n, t, nHeads, dHead], name: nil)\n \n q = transposeTensor(q, dimension: 1, withDimension: 2, name: nil)\n k = transposeTensor(k, dimension: 1, withDimension: 2, name: nil)\n v = transposeTensor(v, dimension: 1, withDimension: 2, name: nil)\n \n var att = matrixMultiplication(primary: q, secondary: transposeTensor(k, dimension: 2, withDimension: 3, name: nil), name: nil)\n att = multiplication(att, constant(1.0 / sqrt(dHead.doubleValue), dataType: MPSDataType.float16), name: nil)\n att = addition(att, loadConstant(at: folder, name: \"causal_mask\", shape: [1, 1, 77, 77]), name: nil)\n att = softMax(with: att, axis: 3, name: nil)\n att = matrixMultiplication(primary: att, secondary: v, name: nil)\n att = transposeTensor(att, dimension: 1, withDimension: 2, name: nil)\n att = reshape(att, shape: [n, t, c], name: nil)\n return makeLinear(at: folder, xIn: att, name: name + \".out_proj\", outChannels: c)\n }\n\n func makeTextEncoderLayer(at folder: URL, xIn: MPSGraphTensor, name: String) -> MPSGraphTensor {\n var x = xIn\n x = makeLayerNorm(at: folder, xIn: x, name: name + \".layer_norm1\")\n x = makeTextAttention(at: folder, xIn: x, name: name + \".self_attn\")\n x = addition(x, xIn, name: nil)\n let skip = x\n x = makeLayerNorm(at: folder, xIn: x, name: name + \".layer_norm2\")\n x = makeLinear(at: folder, xIn: x, name: name + \".mlp.fc1\", outChannels: 3072)\n x = gelu(x)\n x = makeLinear(at: folder, xIn: x, name: name + \".mlp.fc2\", outChannels: 768)\n return addition(x, skip, name: nil)\n }\n\n func makeTextEncoder(at folder: URL, xIn: MPSGraphTensor, name: String) -> MPSGraphTensor {\n var x = xIn\n for i in 0..<12 {\n x = makeTextEncoderLayer(at: folder, xIn: x, name: name + \".layers.\\(i)\")\n }\n return x\n }\n}\n" }, { "path": "Sources/MapleDiffusion/MPS/UNet.swift", "content": "//\n// UNet.swift\n// \n//\n// Created by Guillermo Cique Fernández on 9/11/22.\n//\n\nimport Foundation\nimport MetalPerformanceShadersGraph\n\nclass UNet {\n let synchronize: Bool\n let modelLocation: URL\n let saveMemory: Bool\n let device: MPSGraphDevice\n \n // MEM-HACK: split into subgraphs\n private var unetAnUnexpectedJourneyExecutable: MPSGraphExecutable?\n private var anUnexpectedJourneyShapes = [[NSNumber]]()\n private var unetTheDesolationOfSmaugExecutable: MPSGraphExecutable?\n private var theDesolationOfSmaugShapes = [[NSNumber]]()\n private var theDesolationOfSmaugIndices = [MPSGraphTensor: Int]()\n private var unetTheBattleOfTheFiveArmiesExecutable: MPSGraphExecutable?\n private var theBattleOfTheFiveArmiesIndices = [MPSGraphTensor: Int]()\n \n init(synchronize: Bool, modelLocation: URL, saveMemory: Bool, device: MPSGraphDevice, shape: [NSNumber]) {\n self.synchronize = synchronize\n self.modelLocation = modelLocation\n self.saveMemory = saveMemory\n self.device = device\n \n loadAnUnexpectedJourney(shape: shape)\n loadTheDesolationOfSmaug()\n loadTheBattleOfTheFiveArmies()\n }\n \n private func loadAnUnexpectedJourney(shape: [NSNumber]) {\n let graph = MPSGraph(synchronize: synchronize)\n let xIn = graph.placeholder(shape: shape, dataType: MPSDataType.float16, name: nil)\n let condIn = graph.placeholder(shape: [saveMemory ? 1 : 2, 77, 768], dataType: MPSDataType.float16, name: nil)\n let tembIn = graph.placeholder(shape: [1, 320], dataType: MPSDataType.float16, name: nil)\n let unetOuts = graph.makeUNetAnUnexpectedJourney(\n at: modelLocation,\n xIn: xIn,\n tembIn: tembIn,\n condIn: condIn,\n name: \"model.diffusion_model\",\n saveMemory: saveMemory\n )\n let unetFeeds = [xIn, condIn, tembIn].reduce(into: [:], {$0[$1] = MPSGraphShapedType(shape: $1.shape!, dataType: $1.dataType)})\n unetAnUnexpectedJourneyExecutable = graph.compile(\n with: device,\n feeds: unetFeeds,\n targetTensors: unetOuts,\n targetOperations: nil,\n compilationDescriptor: nil\n )\n anUnexpectedJourneyShapes = unetOuts.map{$0.shape!}\n }\n \n private func loadTheDesolationOfSmaug() {\n let graph = MPSGraph(synchronize: synchronize)\n let condIn = graph.placeholder(shape: [saveMemory ? 1 : 2, 77, 768], dataType: MPSDataType.float16, name: nil)\n let placeholders = anUnexpectedJourneyShapes.map{graph.placeholder(shape: $0, dataType: MPSDataType.float16, name: nil)} + [condIn]\n theDesolationOfSmaugIndices.removeAll()\n for i in 0.. [MPSGraphTensorData] {\n var out = [MPSGraphTensorData]()\n for r in unetAnUnexpectedJourneyExecutable!.feedTensors! {\n for i in x {\n if (i.shape == r.shape) {\n out.append(i)\n }\n }\n }\n return out\n }\n \n private func reorderTheDesolationOfSmaug(x: [MPSGraphTensorData]) -> [MPSGraphTensorData] {\n var out = [MPSGraphTensorData]()\n for r in unetTheDesolationOfSmaugExecutable!.feedTensors! {\n out.append(x[theDesolationOfSmaugIndices[r]!])\n }\n return out\n }\n \n private func reorderTheBattleOfTheFiveArmies(x: [MPSGraphTensorData]) -> [MPSGraphTensorData] {\n var out = [MPSGraphTensorData]()\n for r in unetTheBattleOfTheFiveArmiesExecutable!.feedTensors! {\n out.append(x[theBattleOfTheFiveArmiesIndices[r]!])\n }\n return out\n }\n \n private func runUNet(\n with queue: MTLCommandQueue,\n latent: MPSGraphTensorData,\n guidance: MPSGraphTensorData,\n temb: MPSGraphTensorData\n ) -> MPSGraphTensorData {\n var x = unetAnUnexpectedJourneyExecutable!.run(\n with: queue,\n inputs: reorderAnUnexpectedJourney(x: [latent, guidance, temb]),\n results: nil,\n executionDescriptor: nil\n )\n x = unetTheDesolationOfSmaugExecutable!.run(\n with: queue,\n inputs: reorderTheDesolationOfSmaug(x: x + [guidance]),\n results: nil,\n executionDescriptor: nil\n )\n return unetTheBattleOfTheFiveArmiesExecutable!.run(\n with: queue,\n inputs: reorderTheBattleOfTheFiveArmies(x: x + [guidance]),\n results: nil,\n executionDescriptor: nil\n )[0]\n }\n \n private func runBatchedUNet(\n with queue: MTLCommandQueue,\n latent: MPSGraphTensorData,\n baseGuidance: MPSGraphTensorData,\n textGuidance: MPSGraphTensorData,\n temb: MPSGraphTensorData\n ) -> (MPSGraphTensorData, MPSGraphTensorData) {\n // concat\n var graph = MPSGraph(synchronize: synchronize)\n let bg = graph.placeholder(shape: baseGuidance.shape, dataType: MPSDataType.float16, name: nil)\n let tg = graph.placeholder(shape: textGuidance.shape, dataType: MPSDataType.float16, name: nil)\n let concatGuidance = graph.concatTensors([bg, tg], dimension: 0, name: nil)\n let concatGuidanceData = graph.run(\n with: queue,\n feeds: [\n bg : baseGuidance,\n tg: textGuidance\n ],\n targetTensors: [concatGuidance], targetOperations\n : nil\n )[concatGuidance]!\n // run\n let concatEtaData = runUNet(with: queue, latent: latent, guidance: concatGuidanceData, temb: temb)\n // split\n graph = MPSGraph(synchronize: synchronize)\n let etas = graph.placeholder(shape: concatEtaData.shape, dataType: concatEtaData.dataType, name: nil)\n let eta0 = graph.sliceTensor(etas, dimension: 0, start: 0, length: 1, name: nil)\n let eta1 = graph.sliceTensor(etas, dimension: 0, start: 1, length: 1, name: nil)\n let etaRes = graph.run(\n with: queue,\n feeds: [etas: concatEtaData],\n targetTensors: [eta0, eta1],\n targetOperations: nil\n )\n return (etaRes[eta0]!, etaRes[eta1]!)\n }\n \n \n func run(\n with queue: MTLCommandQueue,\n latent: MPSGraphTensorData,\n baseGuidance: MPSGraphTensorData,\n textGuidance: MPSGraphTensorData,\n temb: MPSGraphTensorData\n ) -> (MPSGraphTensorData, MPSGraphTensorData) {\n if (saveMemory) {\n // MEM-HACK: un/neg-conditional and text-conditional are run in two separate passes (not batched) to save memory\n let etaUncond = runUNet(with: queue, latent: latent, guidance: baseGuidance, temb: temb)\n let etaCond = runUNet(with: queue, latent: latent, guidance: textGuidance, temb: temb)\n return (etaUncond, etaCond)\n } else {\n return runBatchedUNet(with: queue, latent: latent, baseGuidance: baseGuidance, textGuidance: textGuidance, temb: temb)\n }\n }\n}\n\nextension MPSGraph {\n func makeTimeEmbed(at folder: URL, xIn: MPSGraphTensor, name: String) -> MPSGraphTensor {\n var x = xIn\n x = makeLinear(at: folder, xIn: x, name: name + \".0\", outChannels: 1280)\n x = swish(x)\n return makeLinear(at: folder, xIn: x, name: name + \".2\", outChannels: 1280)\n }\n\n func makeUNetResBlock(at folder: URL, xIn: MPSGraphTensor, embIn: MPSGraphTensor, name: String, inChannels: NSNumber, outChannels: NSNumber) -> MPSGraphTensor {\n var x = xIn\n x = makeGroupNormSwish(at: folder, xIn: x, name: name + \".in_layers.0\")\n x = makeConv(at: folder, xIn: x, name: name + \".in_layers.2\", outChannels: outChannels, khw: 3)\n var emb = embIn\n emb = swish(emb)\n emb = makeLinear(at: folder, xIn: emb, name: name + \".emb_layers.1\", outChannels: outChannels)\n emb = expandDims(emb, axes: [1, 2], name: nil)\n x = addition(x, emb, name: nil)\n x = makeGroupNormSwish(at: folder, xIn: x, name: name + \".out_layers.0\")\n x = makeConv(at: folder, xIn: x, name: name + \".out_layers.3\", outChannels: outChannels, khw: 3)\n \n var skip = xIn\n if (inChannels != outChannels) {\n skip = makeConv(at: folder, xIn: xIn, name: name + \".skip_connection\", outChannels: outChannels, khw: 1)\n }\n return addition(x, skip, name: nil)\n }\n\n\n func makeOutputBlock(at folder: URL, xIn: MPSGraphTensor, embIn: MPSGraphTensor, condIn: MPSGraphTensor, inChannels: NSNumber, outChannels: NSNumber, dHead: NSNumber, name: String, saveMemory: Bool, spatialTransformer: Bool = true, upsample: Bool = false) -> MPSGraphTensor {\n var x = xIn\n x = makeUNetResBlock(at: folder, xIn: x, embIn: embIn, name: name + \".0\", inChannels: inChannels, outChannels: outChannels)\n if (spatialTransformer) {\n x = makeSpatialTransformerBlock(at: folder, xIn: x, name: name + \".1\", contextIn: condIn, saveMemory: saveMemory)\n }\n if (upsample) {\n x = upsampleNearest(xIn: x)\n x = makeConv(at: folder, xIn: x, name: name + (spatialTransformer ? \".2\" : \".1\") + \".conv\", outChannels: outChannels, khw: 3)\n }\n return x\n }\n\n\n func makeUNetAnUnexpectedJourney(at folder: URL, xIn: MPSGraphTensor, tembIn: MPSGraphTensor, condIn: MPSGraphTensor, name: String, saveMemory: Bool = true) -> [MPSGraphTensor] {\n let emb = makeTimeEmbed(at: folder, xIn: tembIn, name: name + \".time_embed\")\n \n var savedInputs = [MPSGraphTensor]()\n var x = xIn\n \n if (!saveMemory) {\n // need to explicitly batch to avoid shape errors later iirc\n // TODO: did we actually need this\n x = broadcast(x, shape: [condIn.shape![0], x.shape![1], x.shape![2], x.shape![3]], name: nil)\n }\n \n // input blocks\n x = makeConv(at: folder, xIn: x, name: name + \".input_blocks.0.0\", outChannels: 320, khw: 3)\n savedInputs.append(x)\n \n x = makeUNetResBlock(at: folder, xIn: x, embIn: emb, name: name + \".input_blocks.1.0\", inChannels: 320, outChannels: 320)\n x = makeSpatialTransformerBlock(at: folder, xIn: x, name: name + \".input_blocks.1.1\", contextIn: condIn, saveMemory: saveMemory)\n savedInputs.append(x)\n \n x = makeUNetResBlock(at: folder, xIn: x, embIn: emb, name: name + \".input_blocks.2.0\", inChannels: 320, outChannels: 320)\n x = makeSpatialTransformerBlock(at: folder, xIn: x, name: name + \".input_blocks.2.1\", contextIn: condIn, saveMemory: saveMemory)\n savedInputs.append(x)\n \n // downsample\n x = makeConv(at: folder, xIn: x, name: name + \".input_blocks.3.0.op\", outChannels: 320, khw: 3, stride: 2)\n savedInputs.append(x)\n \n x = makeUNetResBlock(at: folder, xIn: x, embIn: emb, name: name + \".input_blocks.4.0\", inChannels: 320, outChannels: 640)\n x = makeSpatialTransformerBlock(at: folder, xIn: x, name: name + \".input_blocks.4.1\", contextIn: condIn, saveMemory: saveMemory)\n savedInputs.append(x)\n \n x = makeUNetResBlock(at: folder, xIn: x, embIn: emb, name: name + \".input_blocks.5.0\", inChannels: 640, outChannels: 640)\n x = makeSpatialTransformerBlock(at: folder, xIn: x, name: name + \".input_blocks.5.1\", contextIn: condIn, saveMemory: saveMemory)\n savedInputs.append(x)\n \n // downsample\n x = makeConv(at: folder, xIn: x, name: name + \".input_blocks.6.0.op\", outChannels: 640, khw: 3, stride: 2)\n savedInputs.append(x)\n \n x = makeUNetResBlock(at: folder, xIn: x, embIn: emb, name: name + \".input_blocks.7.0\", inChannels: 640, outChannels: 1280)\n x = makeSpatialTransformerBlock(at: folder, xIn: x, name: name + \".input_blocks.7.1\", contextIn: condIn, saveMemory: saveMemory)\n savedInputs.append(x)\n \n x = makeUNetResBlock(at: folder, xIn: x, embIn: emb, name: name + \".input_blocks.8.0\", inChannels: 1280, outChannels: 1280)\n x = makeSpatialTransformerBlock(at: folder, xIn: x, name: name + \".input_blocks.8.1\", contextIn: condIn, saveMemory: saveMemory)\n savedInputs.append(x)\n \n // downsample\n x = makeConv(at: folder, xIn: x, name: name + \".input_blocks.9.0.op\", outChannels: 1280, khw: 3, stride: 2)\n savedInputs.append(x)\n \n x = makeUNetResBlock(at: folder, xIn: x, embIn: emb, name: name + \".input_blocks.10.0\", inChannels: 1280, outChannels: 1280)\n savedInputs.append(x)\n \n x = makeUNetResBlock(at: folder, xIn: x, embIn: emb, name: name + \".input_blocks.11.0\", inChannels: 1280, outChannels: 1280)\n savedInputs.append(x)\n \n // middle blocks\n x = makeUNetResBlock(at: folder, xIn: x, embIn: emb, name: name + \".middle_block.0\", inChannels: 1280, outChannels: 1280)\n x = makeSpatialTransformerBlock(at: folder, xIn: x, name: name + \".middle_block.1\", contextIn: condIn, saveMemory: saveMemory)\n x = makeUNetResBlock(at: folder, xIn: x, embIn: emb, name: name + \".middle_block.2\", inChannels: 1280, outChannels: 1280)\n \n return savedInputs + [emb] + [x]\n }\n\n func makeUNetTheDesolationOfSmaug(at folder: URL, savedInputsIn: [MPSGraphTensor], name: String, saveMemory: Bool = true) -> [MPSGraphTensor] {\n var savedInputs = savedInputsIn\n let condIn = savedInputs.popLast()!\n var x = savedInputs.popLast()!\n let emb = savedInputs.popLast()!\n // output blocks\n x = concatTensors([x, savedInputs.popLast()!], dimension: 3, name: nil)\n x = makeOutputBlock(at: folder, xIn: x, embIn: emb, condIn: condIn, inChannels: 2560, outChannels: 1280, dHead: 160, name: name + \".output_blocks.0\", saveMemory: saveMemory, spatialTransformer: false, upsample: false)\n \n x = concatTensors([x, savedInputs.popLast()!], dimension: 3, name: nil)\n x = makeOutputBlock(at: folder, xIn: x, embIn: emb, condIn: condIn, inChannels: 2560, outChannels: 1280, dHead: 160, name: name + \".output_blocks.1\", saveMemory: saveMemory, spatialTransformer: false, upsample: false)\n \n // upsample\n x = concatTensors([x, savedInputs.popLast()!], dimension: 3, name: nil)\n x = makeOutputBlock(at: folder, xIn: x, embIn: emb, condIn: condIn, inChannels: 2560, outChannels: 1280, dHead: 160, name: name + \".output_blocks.2\", saveMemory: saveMemory, spatialTransformer: false, upsample: true)\n \n x = concatTensors([x, savedInputs.popLast()!], dimension: 3, name: nil)\n x = makeOutputBlock(at: folder, xIn: x, embIn: emb, condIn: condIn, inChannels: 2560, outChannels: 1280, dHead: 160, name: name + \".output_blocks.3\", saveMemory: saveMemory, spatialTransformer: true, upsample: false)\n \n x = concatTensors([x, savedInputs.popLast()!], dimension: 3, name: nil)\n x = makeOutputBlock(at: folder, xIn: x, embIn: emb, condIn: condIn, inChannels: 2560, outChannels: 1280, dHead: 160, name: name + \".output_blocks.4\", saveMemory: saveMemory, spatialTransformer: true, upsample: false)\n \n return savedInputs + [emb] + [x]\n }\n\n func makeUNetTheBattleOfTheFiveArmies(at folder: URL, savedInputsIn: [MPSGraphTensor], name: String, saveMemory: Bool = true) -> MPSGraphTensor {\n var savedInputs = savedInputsIn\n let condIn = savedInputs.popLast()!\n var x = savedInputs.popLast()!\n let emb = savedInputs.popLast()!\n // upsample\n x = concatTensors([x, savedInputs.popLast()!], dimension: 3, name: nil)\n x = makeOutputBlock(at: folder, xIn: x, embIn: emb, condIn: condIn, inChannels: 1920, outChannels: 1280, dHead: 160, name: name + \".output_blocks.5\", saveMemory: saveMemory, spatialTransformer: true, upsample: true)\n \n x = concatTensors([x, savedInputs.popLast()!], dimension: 3, name: nil)\n x = makeOutputBlock(at: folder, xIn: x, embIn: emb, condIn: condIn, inChannels: 1920, outChannels: 640, dHead: 80, name: name + \".output_blocks.6\", saveMemory: saveMemory, spatialTransformer: true, upsample: false)\n \n x = concatTensors([x, savedInputs.popLast()!], dimension: 3, name: nil)\n x = makeOutputBlock(at: folder, xIn: x, embIn: emb, condIn: condIn, inChannels: 1280, outChannels: 640, dHead: 80, name: name + \".output_blocks.7\", saveMemory: saveMemory, spatialTransformer: true, upsample: false)\n \n // upsample\n x = concatTensors([x, savedInputs.popLast()!], dimension: 3, name: nil)\n x = makeOutputBlock(at: folder, xIn: x, embIn: emb, condIn: condIn, inChannels: 960, outChannels: 640, dHead: 80, name: name + \".output_blocks.8\", saveMemory: saveMemory, spatialTransformer: true, upsample: true)\n \n x = concatTensors([x, savedInputs.popLast()!], dimension: 3, name: nil)\n x = makeOutputBlock(at: folder, xIn: x, embIn: emb, condIn: condIn, inChannels: 960, outChannels: 320, dHead: 40, name: name + \".output_blocks.9\", saveMemory: saveMemory, spatialTransformer: true, upsample: false)\n \n x = concatTensors([x, savedInputs.popLast()!], dimension: 3, name: nil)\n x = makeOutputBlock(at: folder, xIn: x, embIn: emb, condIn: condIn, inChannels: 640, outChannels: 320, dHead: 40, name: name + \".output_blocks.10\", saveMemory: saveMemory, spatialTransformer: true, upsample: false)\n \n x = concatTensors([x, savedInputs.popLast()!], dimension: 3, name: nil)\n x = makeOutputBlock(at: folder, xIn: x, embIn: emb, condIn: condIn, inChannels: 640, outChannels: 320, dHead: 40, name: name + \".output_blocks.11\", saveMemory: saveMemory, spatialTransformer: true, upsample: false)\n \n // out\n x = makeGroupNormSwish(at: folder, xIn: x, name: \"model.diffusion_model.out.0\")\n return makeConv(at: folder, xIn: x, name: \"model.diffusion_model.out.2\", outChannels: 4, khw: 3)\n }\n}\n" }, { "path": "Sources/MapleDiffusion/Model/GenResult.swift", "content": "\nimport Foundation\n//import AppKit\nimport CoreImage\nimport CoreGraphics\n\npublic struct GenResult {\n internal init(image: CGImage?, progress: Double, stage: String) {\n self.image = image\n self.progress = progress\n self.stage = stage\n }\n \n public let image : CGImage?\n public let progress : Double\n public let stage : String\n}\n" }, { "path": "Sources/MapleDiffusion/Model/GeneratorState.swift", "content": "\n// Created by Morten Just on 10/20/22.\n//\n\nimport Foundation\n\npublic enum GeneratorState: Equatable {\n public static func == (lhs: GeneratorState, rhs: GeneratorState) -> Bool {\n switch(lhs, rhs) {\n case (.ready, .ready):\n return true\n// case (.modelIsLoading(progress: <#T##Double#>, message: <#T##String#>))\n case (.notStarted, .notStarted):\n return true\n default:\n return false\n }\n }\n \n case notStarted\n case modelIsLoading(progress: Double, message: String)\n case ready\n}\n\n\n" }, { "path": "Sources/MapleDiffusion/Model/SampleInput.swift", "content": "//\n// SampleInput.swift\n// \n//\n// Created by Guillermo Cique Fernández on 14/11/22.\n//\n\nimport Foundation\nimport CoreGraphics\n\npublic struct SampleInput {\n var prompt: String\n var negativePrompt: String\n var initImage: CGImage? { didSet { checkSize() }}\n var strength: Float?\n var seed: Int\n var steps: Int\n var guidanceScale: Float\n \n public init(\n prompt: String,\n negativePrompt: String = \"\",\n seed: Int = Int.random(in: 0...Int.max),\n steps: Int = 20,\n guidanceScale: Float = 7.5\n ) {\n self.prompt = prompt\n self.negativePrompt = negativePrompt\n self.initImage = nil\n self.strength = nil\n self.seed = seed\n self.steps = steps\n self.guidanceScale = guidanceScale\n }\n \n public init(\n prompt: String,\n negativePrompt: String = \"\",\n initImage: CGImage?,\n strength: Float = 0.75,\n seed: Int = Int.random(in: 0...Int.max),\n steps: Int = 20,\n guidanceScale: Float = 5.0\n ) {\n self.prompt = prompt\n self.negativePrompt = negativePrompt\n self.initImage = initImage\n self.strength = strength\n self.seed = seed\n self.steps = steps\n self.guidanceScale = guidanceScale\n }\n \n private func checkSize() {\n guard let initImage else { return }\n if initImage.width != 512 || initImage.height != 512 {\n assertionFailure(\"Please make sure your input image is exactly 512x512. You can use your own cropping mechanism, or the extension in NSImage:crop:to (macOS). Feel free to contribute a general solution. See Github issues for ideas.\")\n }\n }\n}\n\n#if os(iOS)\nimport UIKit\n\npublic extension SampleInput {\n init(\n prompt: String,\n negativePrompt: String = \"\",\n initImage: UIImage,\n strength: Float = 0.75,\n seed: Int = Int.random(in: 0...Int.max),\n steps: Int = 50,\n guidanceScale: Float = 5.0\n ) {\n self.prompt = prompt\n self.negativePrompt = negativePrompt\n self.initImage = initImage.cgImage!\n self.strength = strength\n self.seed = seed\n self.steps = steps\n self.guidanceScale = guidanceScale\n }\n}\n#endif\n\n#if os(macOS)\nimport AppKit\n\npublic extension SampleInput {\n init(\n prompt: String,\n negativePrompt: String = \"\",\n initImage: NSImage,\n strength: Float = 0.75,\n seed: Int = Int.random(in: 0...Int.max),\n steps: Int = 50,\n guidanceScale: Float = 5.0\n ) {\n self.prompt = prompt\n self.negativePrompt = negativePrompt\n var imageRect = CGRect(x: 0, y: 0, width: initImage.size.width, height: initImage.size.height)\n self.initImage = initImage.cgImage(forProposedRect: &imageRect, context: nil, hints: nil)!\n self.strength = strength\n self.seed = seed\n self.steps = steps\n self.guidanceScale = guidanceScale\n }\n}\n#endif\n" }, { "path": "Sources/MapleDiffusion/Model/SampleOutput.swift", "content": "//\n// SampleOutput.swift\n// \n//\n// Created by Guillermo Cique Fernández on 14/11/22.\n//\n\nimport Foundation\nimport CoreGraphics\n\npublic struct SampleOutput {\n let image: CGImage?\n let input: SampleInput\n}\n" }, { "path": "Sources/MapleDiffusion/Support/CGImage + ItemProvider.swift", "content": "//\n// File.swift\n// \n//\n// Created by Morten Just on 10/25/22.\n//\n\n\n\n#if os(macOS)\nimport Foundation\nimport AppKit\nimport CoreGraphics\nimport CoreImage\nimport UniformTypeIdentifiers\n\n\nextension CGImage {\n func itemProvider(filename: String? = nil) -> NSItemProvider? {\n let rep = NSBitmapImageRep(cgImage: self)\n guard let data = rep.representation(using: .png, properties: [:]) else { return nil }\n \n let filename = filename ?? \"Generated Image \\(UUID().uuidString.prefix(4))\"\n \n let tempUrl = FileManager.default.temporaryDirectory.appendingPathComponent(filename).appendingPathExtension(\"png\")\n do {\n try data.write(to: tempUrl)\n \n let item = NSItemProvider(item: tempUrl as NSSecureCoding, typeIdentifier: UTType.fileURL.identifier)\n item.suggestedName = \"\\(filename).png\"\n \n return item\n } catch {\n return nil\n }\n }\n}\n#endif\n" }, { "path": "Sources/MapleDiffusion/Support/Diffusion + Generate.swift", "content": "//\n// File.swift\n// \n//\n// Created by Morten Just on 10/27/22.\n//\n\nimport Foundation\nimport Combine\nimport CoreGraphics\n\n/**\n Generator functions. Mostly convenience wrappers around generate:input[..]\n \n */\n\npublic extension Diffusion {\n \n \n /// Generate an image async. Optional callback with progress and intermediate image.\n func generate(\n input: SampleInput,\n progress: ((GenResult) -> Void)? = nil,\n remoteUrl: String? = nil\n ) async -> CGImage? {\n \n var combinedSteps : Double = 1\n var combinedProgress : Double = 0\n \n // maybe load model first\n if modelIsCold, let remoteUrl, let url = URL(string: remoteUrl) {\n combinedSteps += 1\n print(\"async gen: cold model, loading\")\n try! await prepModels(remoteURL: url) { p in\n combinedProgress = (p/combinedSteps)\n let res = GenResult(image: nil, progress: combinedProgress, stage: \"Loading Model\")\n progress?(res)\n }\n }\n \n // generate image\n return await withCheckedContinuation { continuation in\n print(\"async gen: generating\", input.prompt)\n self.generate(input: input) { (image, progressFloat, stage) in\n let genResult = GenResult(image: image, progress: Double(progressFloat), stage: stage)\n progress?(genResult)\n \n print(\"async gen: \", genResult.stage)\n \n if progressFloat == 1, let finalImage = genResult.image {\n continuation.resume(returning: finalImage)\n }\n \n if progressFloat == 1, genResult.image == nil {\n continuation.resume(returning: nil)\n }\n }\n }\n \n }\n \n \n /// Generate an image asynchronously. Optional callback with progress and intermediate image. Run inside a Task.detached to run in background.\n /// ```\n /// // without progress reporting\n /// let image = await diffusion.generate(\"astronaut in the ocean\")\n ///\n /// // with progress\n /// let image = await diffusion.generate(\"astronaut in the ocean\") { progress in\n /// print(\"progress: \\(progress.progress)\") }\n func generate(prompt: String,\n negativePrompt: String = \"\",\n inputImage: CGImage? = nil,\n seed: Int = Int.random(in: 0...Int.max),\n steps:Int = 20,\n guidanceScale:Float = 7.5,\n progress: ((GenResult) -> Void)? = nil,\n remoteUrl: String? = nil\n ) async -> CGImage? {\n /// This is a just a wrapper that adds `SampleInput`, mainly for simplicity and to not break pre-SampleInput builds.\n let sampleInput = SampleInput(prompt: prompt, negativePrompt: negativePrompt, seed: seed, steps: steps, guidanceScale: guidanceScale)\n return await generate(input: sampleInput, progress: progress)\n }\n \n \n\n \n /// Generate an image and get a publisher for progress and intermediate image. Runs detached with \"initiated\" priority.\n /// ```\n /// diffusion.generate(\"Astronaut in the ocean\")\n /// .sink { result in\n /// print(\"progress: \\(result.progress)\") // result also contains the intermediate image\n /// }.store(in: ...)\n ///\n func generate(input: SampleInput) -> AnyPublisher {\n let publisher = PassthroughSubject()\n \n Task.detached(priority: .userInitiated) { // TODO: Test other priorities and consider making it an optional argument\n self.generate(input: input) { (cgImage, progress, stage) in\n Task {\n print(\"RAW progress\", progress)\n await MainActor.run {\n let result = GenResult(image: cgImage, progress: Double(progress), stage: stage)\n publisher.send(result)\n if progress >= 1 { publisher.send(completion: .finished)}\n }\n }\n }\n }\n \n return publisher.eraseToAnyPublisher()\n }\n \n func generate(prompt: String, negativePrompt: String = \"\", seed: Int = Int.random(in: 0...Int.max), steps:Int = 20, guidanceScale:Float = 7.5) -> AnyPublisher {\n \n let sampleInput = SampleInput(prompt: prompt, negativePrompt: negativePrompt, seed: seed, steps: steps, guidanceScale: guidanceScale)\n \n return generate(input: sampleInput)\n \n }\n \n \n \n \n \n /// Generate and image with a callback on current thread.\n ///\n func generate(\n input: SampleInput,\n completion: @escaping (CGImage?, Float, String)->()\n ) {\n mapleDiffusion.generate(input: input) { cgImage, progress, stage in\n // penultimate step is also marked progress 1 in MD currently, working around it\n var realProgress = progress\n if progress == 1 && stage.contains(\"Decoding\") {\n realProgress = 0.97\n }\n \n completion(cgImage, realProgress, stage)\n }\n }\n}\n\n\n" }, { "path": "Sources/MapleDiffusion/Support/Drag + Drop Helpers.swift", "content": "//\n// File.swift\n// \n//\n// Created by Morten Just on 11/21/22.\n//\n\nimport Foundation\nimport CoreGraphics\nimport SwiftUI\n\n#if os(macOS)\n\n/// Allow safe and optional item dragging\nstruct FinderDraggable : ViewModifier {\n let image: CGImage\n let enabled: Bool\n \n func body(content: Content) -> some View {\n if enabled, let provider = image.itemProvider() {\n content.onDrag {\n provider\n }\n } else {\n content\n }\n }\n}\n\nstruct FinderImageDroppable : ViewModifier {\n var isTargeted: Binding?\n let forceSize: CGSize?\n var onDrop : ((CGImage) -> Void)\n \n func loadImage(url:URL) {\n let nsImage = NSImage(contentsOf: url)\n let cgImage = nsImage?.cgImage(forProposedRect: nil, context: nil, hints: nil)\n \n // resize here if forcesize is there\n \n \n \n if let cgImage {\n onDrop(cgImage)\n }\n }\n \n func body(content: Content) -> some View {\n content\n .droppable(isTargeted: isTargeted) { url in\n loadImage(url: url)\n }\n }\n}\n\nstruct FinderDroppable : ViewModifier {\n var isTargeted : Binding?\n var onDrop : (URL) -> Void\n \n func body(content: Content) -> some View {\n content\n .onDrop(of: [.fileURL], isTargeted: isTargeted) { providers in\n if let provider = (providers.first { $0.canLoadObject(ofClass: URL.self ) }) {\n let _ = provider.loadObject(ofClass: URL.self) { reading, error in\n if let reading {\n onDrop(reading)\n }\n if let error {\n print(\"error\", error)\n }\n }\n return true\n }\n return false\n }\n }\n}\n\nextension View {\n public func draggable(enabled: Bool,\n image: CGImage) -> some View {\n self.modifier(FinderDraggable(image: image, enabled: enabled))\n }\n \n public func droppable(isTargeted:Binding? = nil,\n onDrop: @escaping (URL)->Void) -> some View {\n self.modifier(FinderDroppable(isTargeted: isTargeted, onDrop: onDrop))\n }\n \n public func imageDroppable(isTargeted: Binding? = nil,\n forceSize: CGSize? = nil,\n onDrop: @escaping (CGImage)->Void) -> some View {\n self.modifier(FinderImageDroppable(isTargeted: isTargeted,\n forceSize:forceSize,\n onDrop: onDrop))\n }\n}\n#endif\n" }, { "path": "Sources/MapleDiffusion/Support/MPSGraphTensorData.swift", "content": "//\n// MPSGraphTensorData.swift\n// \n//\n// Created by Guillermo Cique Fernández on 9/11/22.\n//\n\nimport Foundation\nimport MetalPerformanceShadersGraph\n\nextension MPSGraphTensorData {\n var cgImage: CGImage? {\n let shape = self.shape.map{ $0.intValue }\n var imageArrayCPUBytes = [UInt8](repeating: 0, count: shape.reduce(1, *))\n self.mpsndarray().readBytes(&imageArrayCPUBytes, strideBytes: nil)\n return CGImage(\n width: shape[2],\n height: shape[1],\n bitsPerComponent: 8,\n bitsPerPixel: 32,\n bytesPerRow: shape[2]*shape[3],\n space: CGColorSpaceCreateDeviceRGB(),\n bitmapInfo: CGBitmapInfo(rawValue: CGBitmapInfo.byteOrder32Big.rawValue | CGImageAlphaInfo.noneSkipLast.rawValue),\n provider: CGDataProvider(data: NSData(bytes: &imageArrayCPUBytes, length: imageArrayCPUBytes.count))!,\n decode: nil,\n shouldInterpolate: true,\n intent: CGColorRenderingIntent.defaultIntent\n )\n }\n \n public convenience init(device: MPSGraphDevice, cgImage: CGImage) {\n let shape: [NSNumber] = [NSNumber(value: cgImage.height), NSNumber(value: cgImage.width), 4]\n let data = cgImage.dataProvider!.data! as Data\n self.init(device: device, data: data, shape: shape, dataType: .uInt8)\n }\n}\n\n/*\n bitsPerPixel 32\n bytesPerRow 2048\n byteOrderInfo CGImageByteOrderInfo\n colorSpace Optional( (kCGColorSpaceICCBased; kCGColorSpaceModelRGB; sRGB IEC61966-2.1))\n */\n\n\nextension Int {\n func tensorData(device: MPSGraphDevice) -> MPSGraphTensorData {\n let data = [Int32(self)].withUnsafeBufferPointer { Data(buffer: $0) }\n return MPSGraphTensorData(device: device, data: data, shape: [1], dataType: MPSDataType.int32)\n }\n}\n\nextension Float {\n func tensorData(device: MPSGraphDevice) -> MPSGraphTensorData {\n let data = [Float32(self)].withUnsafeBufferPointer { Data(buffer: $0) }\n return MPSGraphTensorData(device: device, data: data, shape: [1], dataType: MPSDataType.float32)\n }\n}\n" }, { "path": "Sources/MapleDiffusion/Support/ModelFetcher.swift", "content": "//\n// File.swift\n// \n//\n// Created by Morten Just on 10/22/22.\n//\n\nimport Foundation\nimport Combine\nimport ZIPFoundation\n\nclass ModelFetcher {\n let local: URL?\n let remote: URL?\n \n private var bin = Set()\n \n // Initializer overloads\n \n /// We saved our model locally, e.g. in the bundle.\n init(local: URL) {\n self.local = local\n self.remote = nil\n }\n \n /// (Recommended) Use the models at the default location. Download it first if not there.\n init(remote: URL) {\n self.remote = remote\n self.local = nil\n }\n \n /// Use the models at the given local location. Download to this destination if not there.\n init(local: URL, remote: URL) {\n self.local = local\n self.remote = remote\n }\n \n struct FileDownload {\n let url : URL?\n let progress: Double\n let stage:String\n }\n \n // Helpers\n \n \n private var bundleId : String {\n Bundle.main.bundleIdentifier ?? \"app.otato.diffusion\"\n }\n \n var finalLocalUrl: URL {\n local ?? fallbackModelFolder\n }\n \n /// Used if no local URL is provided\n var fallbackModelFolder : URL {\n // TODO: If on iOS use the bundle\n \n FileManager.default\n .urls(for: .applicationSupportDirectory, in: .userDomainMask)[0]\n .appendingPathComponent(bundleId)\n .appendingPathComponent(\"bins\")\n }\n \n /// Checks final destination and returns true if it's not empty\n var isModelPresentLocally : Bool {\n do {\n let files = try FileManager.default\n .contentsOfDirectory(at: finalLocalUrl, includingPropertiesForKeys: nil)\n return !files.isEmpty\n } catch {\n return false\n }\n }\n \n // methods\n \n enum ModelFetcherError : Error {\n case emptyFolderAndNoRemoteURL\n }\n\n func fetch(progress: ProgressClosure?) async throws -> URL\n {\n \n let combinedSteps = 2.0 // Download and unzip\n var combinedProgress = 0.0\n \n // is the calculated local folder, and it's not empty? go!\n if isModelPresentLocally {\n progress?(1.0)\n return finalLocalUrl }\n\n // create the folder if it doesn't exist\n try FileManager.default.createDirectory(at: finalLocalUrl, withIntermediateDirectories: true)\n \n // if the local folder is empty and we have a remote, start downloading and return a stream\n if !isModelPresentLocally, let remote {\n \n // 1 download\n let downloadedURL = try await URLSession\n .shared\n .downloadWithProgress(url: remote) { p in\n combinedProgress += (p/combinedSteps)\n progress?(combinedProgress)\n }\n \n // 2. unzip\n try await moveAndUnzip(downloadedURL) { p in\n combinedProgress += (p/combinedSteps)\n progress?(combinedSteps)\n }\n \n // 3 done!\n return finalLocalUrl\n }\n \n // with the overloads we shouldn't end up here\n assertionFailure()\n throw ModelFetcherError.emptyFolderAndNoRemoteURL\n// fatalError(\"The model folder is empty or not there, and no remote URL was provided\")\n }\n \n \n func moveAndUnzip(_ url : URL, completion: ProgressClosure?) async throws {\n let progress = Progress()\n var bin = Set()\n progress.publisher(for: \\.fractionCompleted)\n .map { Double(integerLiteral: $0) }\n .sink { value in\n completion?(value)\n }.store(in: &bin)\n try FileManager.default.unzipItem(at: url, to: finalLocalUrl, progress: progress)\n }\n \n \n\n}\n\n\npublic typealias ProgressClosure = (Double) -> Void\n" }, { "path": "Sources/MapleDiffusion/Support/NSImage + resizing.swift", "content": "//\n// Image resizing and cropping for input images\n// \n//\n// Created by Morten Just on 11/22/22.\n//\n\n\n#if os(macOS)\nimport Foundation\nimport AppKit\n\nextension NSImage {\n var isLandscape : Bool { size.height < size.width }\n func scale(factor: CGFloat) -> NSImage {\n let newWidth = self.size.width * factor\n let newHeight = self.size.height * factor\n let newSize = NSSize(width: newWidth, height: newHeight)\n \n // Draw self into a new image with the new size\n let newImage = NSImage(size: newSize, flipped: false) { rect in\n self.draw(in: .init(x: 0, y: 0, width: newWidth, height: newHeight))\n return true\n }\n return newImage\n }\n \n @objc var cgImage: CGImage? {\n get {\n guard let imageData = self.tiffRepresentation else {\n return nil }\n guard let sourceData = CGImageSourceCreateWithData(imageData as CFData, nil) else { return nil }\n return CGImageSourceCreateImageAtIndex(sourceData, 0, nil)\n }\n }\n \n func crop(to newSize:NSSize) -> NSImage {\n \n // scale\n var factor : CGFloat = 1\n if isLandscape {\n factor = newSize.height / self.size.height\n } else {\n factor = newSize.width / self.size.width\n }\n let scaledImage = scale(factor: factor)\n \n /// Find the center crop rect\n var fromRect = NSRect(x: 0, y: 0, width: newSize.width, height: newSize.height)\n if self.isLandscape {\n fromRect.origin.x = (0.5 * scaledImage.size.width) - (0.5 * newSize.width)\n } else {\n fromRect.origin.y = (0.5 * scaledImage.size.height) - (0.5 * newSize.height)\n }\n \n guard let rep = NSBitmapImageRep(\n bitmapDataPlanes: nil,\n pixelsWide: Int(newSize.width),\n pixelsHigh: Int(newSize.height),\n bitsPerSample: 8,\n samplesPerPixel: 4,\n hasAlpha: true,\n isPlanar: false,\n colorSpaceName: .deviceRGB,\n bytesPerRow: 0,\n bitsPerPixel: 0\n ) else {\n preconditionFailure()\n }\n \n /// Get rid of retina pixels\n NSGraphicsContext.saveGraphicsState()\n NSGraphicsContext.current = NSGraphicsContext(bitmapImageRep: rep)\n scaledImage.draw(at: .zero, from: fromRect, operation: .sourceOver, fraction: 1.0)\n NSGraphicsContext.restoreGraphicsState()\n \n let data = rep.representation(using: .tiff, properties: [:])\n let newImage = NSImage(data: data!)\n \n return newImage!\n \n }\n}\n\n\n\n#endif\n" }, { "path": "Sources/MapleDiffusion/Support/URLSession + async download.swift", "content": "//\n// File.swift\n// \n//\n// Created by Morten Just on 10/24/22.\n//\n\nimport Foundation\nimport Combine\n\nextension URLSession {\n func downloadWithProgress(url: URL, progress:((Double)->Void)? = nil) async throws -> URL {\n var downloadTask: URLSessionDownloadTask?\n var bin = Set()\n \n return try await withCheckedThrowingContinuation({ continuation in\n let request = URLRequest(url: url)\n \n downloadTask = URLSession.shared.downloadTask(with: request) { url, response, error in\n if let url {\n continuation.resume(returning: url)\n }\n \n if let error {\n continuation.resume(throwing: error)\n }\n }\n \n /// Send progress to closure if we've got one'\n if let progress, let downloadTask {\n downloadTask.publisher(for: \\.progress.fractionCompleted)\n .sink(receiveValue: { p in\n progress(p)\n }).store(in: &bin)\n }\n downloadTask?.resume()\n })\n }\n}\n" }, { "path": "Sources/MapleDiffusion/Views/DiffusionImage.swift", "content": "//\n// File.swift\n// \n//\n// Created by Morten Just on 10/21/22.\n//\n\nimport Foundation\nimport SwiftUI\nimport UniformTypeIdentifiers\n/// Displays a CGImage and blurs it according to its generation progress\npublic struct DiffusionImage : View {\n @Binding var image : CGImage?\n @Binding var progress : Double\n var inputImage : Binding?\n @State var isTargeted = false\n \n let label : String\n let draggable : Bool\n \n public init(image: Binding,\n inputImage: Binding? = nil,\n progress: Binding,c\n label: String = \"Generated Image\",\n draggable: Bool = true)\n {\n self._image = image\n self._progress = progress\n self.inputImage = inputImage\n self.label = label\n self.draggable = draggable\n }\n \n var enableDrag : Bool {\n guard draggable else { return false }\n return progress == 1 ? true : false\n }\n \n public var body: some View {\n \n ZStack {\n if let i = inputImage?.wrappedValue {\n Image(i, scale: 1, label: Text(\"Input image\"))\n .resizable()\n .aspectRatio(contentMode: .fit)\n }\n if let image {\n Image(image, scale: 1, label: Text(label))\n .resizable()\n .aspectRatio(contentMode: .fit)\n .animation(nil)\n .blur(radius: (1 - sqrt(progress)) * 100 )\n .blendMode(progress < 1 ? .sourceAtop : .normal)\n .animation(.linear(duration: 1), value: progress)\n .clipShape(Rectangle())\n#if os(macOS)\n .draggable(enabled: enableDrag, image: image)\n#endif\n }\n }\n \n#if os(macOS)\n .frame(maxWidth: .infinity, maxHeight: .infinity)\n .imageDroppable() { image in\n let ns = NSImage(cgImage: image, size: .init(width: image.width, height: image.height))\n let cropped = ns.crop(to: .init(width: 512, height: 512))\n let cg = cropped.cgImage\n self.inputImage?.wrappedValue = cg\n self.image = nil\n }\n#endif\n }\n}\n\n" }, { "path": "Tests/MapleDiffusionTests/ModelFetcherTests.swift", "content": "//\n// ModelFetcherTests.swift\n// \n//\n// Created by Morten Just on 10/24/22.\n//\n\nimport XCTest\n@testable import MapleDiffusion\n\n/**\n \n Warning:\n These tests are very much work in progress, and far from pure. They require set up and behaviors change from time to time. For example, if you run `testRemoteOnly` with no downloaded folders, it will download. On the second run, it wili not download. I guess the next steps would be to mainipulate the file system to create the same test context every time.\n \n */\n\n\nfinal class ModelFetcherTests: XCTestCase {\n \n /// To test, start a web server in the folder you're hosting Diffusion.zip (a zip of all the converted model files)\n let remoteUrl = URL(string: \"http://localhost:8080/Diffusion.zip\")!\n let localUrl = FileManager.default.urls(for: .applicationSupportDirectory, in: .userDomainMask)[0].appendingPathComponent(\"com.example.myapp/bins\")\n \n \n override func setUpWithError() throws {\n }\n\n override func tearDownWithError() throws {\n }\n \n \n \n /// In this case, the developer handles the downloadnig or is embedding in the bundle.\n func testFetchLocalOnly() async throws {\n \n /// Here, we're given only a file URL. We'll return an async stream that only outputs one element\n let fetcher = ModelFetcher(local: localUrl)\n \n var url : URL?\n for await status in fetcher.fetch() {\n print(\"status: \", status)\n if let u = status.url { url = u }\n }\n XCTAssertNotNil(url)\n print(\"--> got \", url!)\n \n let fileCount = try! FileManager.default.contentsOfDirectory(at: url!, includingPropertiesForKeys: nil).count\n \n XCTAssert(fileCount > 0)\n }\n \n func testRemoteOnly() async throws {\n \n let fetcher = ModelFetcher(remote: remoteUrl)\n \n var url: URL?\n for await status in fetcher.fetch() {\n print(\"status: \", status)\n if let u = status.url { url = u }\n }\n \n XCTAssertNotNil(url)\n print(\"--> got\", url!)\n \n let fileCount = try! FileManager.default.contentsOfDirectory(at: url!, includingPropertiesForKeys: nil).count\n \n XCTAssert(fileCount > 0)\n }\n \n func testLocalAndRemoteOnCleanSlate() async throws {\n try FileManager.default.contentsOfDirectory(at: localUrl, includingPropertiesForKeys: nil)\n .forEach { url in\n print(\"deleting\", url)\n try FileManager.default.removeItem(at: url)\n }\n try await testLocalAndRemote()\n }\n \n func testLocalAndRemote() async throws {\n \n let fetcher = ModelFetcher(\n local: localUrl,\n remote: remoteUrl)\n \n var url: URL?\n \n for await status in fetcher.fetch() {\n print(\"status: \", status)\n if let u = status.url { url = u }\n }\n \n XCTAssertNotNil(url)\n \n print(\"--> Final URL\", url!)\n \n let fileCount = try! FileManager.default.contentsOfDirectory(at: url!, includingPropertiesForKeys: nil).count\n \n XCTAssert(fileCount > 0)\n }\n \n\n func testPerformanceExample() throws {\n // This is an example of a performance test case.\n self.measure {\n // Put the code you want to measure the time of here.\n }\n }\n\n}\n" } ]