Repository: carson-katri/dream-textures Branch: main Commit: c2622a8a9f1a Files: 120 Total size: 551.0 KB Directory structure: gitextract_7tv49soa/ ├── .github/ │ ├── FUNDING.yml │ ├── ISSUE_TEMPLATE/ │ │ ├── bug_report.yml │ │ └── feature_request.md │ └── workflows/ │ ├── package-release.yml │ └── stale.yml ├── .gitignore ├── .python_dependencies/ │ └── .gitignore ├── LICENSE ├── README.md ├── __init__.py ├── absolute_path.py ├── api/ │ ├── __init__.py │ ├── backend/ │ │ ├── __init__.py │ │ └── backend.py │ └── models/ │ ├── __init__.py │ ├── control_net.py │ ├── fix_it_error.py │ ├── generation_arguments.py │ ├── generation_result.py │ ├── model.py │ ├── prompt.py │ ├── seamless_axes.py │ ├── step_preview_mode.py │ └── task.py ├── builtin_presets/ │ ├── Debug.py │ ├── Final.py │ └── Preview.py ├── classes.py ├── community_backends/ │ └── test.py ├── diffusers_backend.py ├── docs/ │ ├── AI_UPSCALING.md │ ├── DEVELOPMENT_ENVIRONMENT.md │ ├── HISTORY.md │ ├── IMAGE_GENERATION.md │ ├── INPAINT_OUTPAINT.md │ ├── RENDER_PASS.md │ ├── SETUP.md │ ├── TEXTURE_PROJECTION.md │ └── assets/ │ └── banner_image_prompt.json ├── engine/ │ ├── __init__.py │ ├── annotations/ │ │ ├── ade20k.py │ │ ├── compat.py │ │ ├── depth.py │ │ ├── normal.py │ │ ├── openpose.py │ │ └── viewport.py │ ├── engine.py │ ├── node.py │ ├── node_executor.py │ ├── node_tree.py │ └── nodes/ │ ├── annotation_nodes.py │ ├── input_nodes.py │ ├── pipeline_nodes.py │ └── utility_nodes.py ├── generator_process/ │ ├── __init__.py │ ├── actions/ │ │ ├── choose_device.py │ │ ├── control_net.py │ │ ├── controlnet_aux.py │ │ ├── convert_original_stable_diffusion_to_diffusers.py │ │ ├── depth_to_image.py │ │ ├── detect_seamless/ │ │ │ ├── __init__.py │ │ │ └── model.npz │ │ ├── huggingface_hub.py │ │ ├── image_to_image.py │ │ ├── inpaint.py │ │ ├── load_model.py │ │ ├── outpaint.py │ │ ├── prompt_to_image.py │ │ └── upscale.py │ ├── actor.py │ ├── block_in_use.py │ ├── directml_patches.py │ ├── future.py │ └── models/ │ ├── __init__.py │ ├── checkpoint.py │ ├── image_generation_result.py │ ├── model_config.py │ ├── model_type.py │ ├── optimizations.py │ ├── scheduler.py │ └── upscale_tiler.py ├── image_utils.py ├── operators/ │ ├── dream_texture.py │ ├── inpaint_area_brush.py │ ├── install_dependencies.py │ ├── notify_result.py │ ├── open_latest_version.py │ ├── project.py │ ├── upscale.py │ └── view_history.py ├── preferences.py ├── prompt_engineering.py ├── property_groups/ │ ├── control_net.py │ ├── dream_prompt.py │ └── seamless_result.py ├── realtime_viewport.py ├── render_pass.py ├── requirements/ │ ├── linux-rocm.txt │ ├── mac-mps-cpu.txt │ ├── win-dml.txt │ └── win-linux-cuda.txt ├── scripts/ │ ├── train_detect_seamless.py │ └── zip_dependencies.py ├── sd_configs/ │ ├── cldm_v15.yaml │ ├── cldm_v21.yaml │ ├── sd_xl_base.yaml │ ├── sd_xl_refiner.yaml │ ├── v1-inference.yaml │ ├── v2-inference-v.yaml │ ├── v2-inference.yaml │ ├── v2-inpainting-inference.yaml │ └── v2-midas-inference.yaml ├── tools.py ├── ui/ │ ├── panels/ │ │ ├── dream_texture.py │ │ ├── history.py │ │ ├── render_properties.py │ │ └── upscaling.py │ ├── presets.py │ └── space_types.py └── version.py ================================================ FILE CONTENTS ================================================ ================================================ FILE: .github/FUNDING.yml ================================================ # These are supported funding model platforms github: carson-katri # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] patreon: # Replace with a single Patreon username open_collective: # Replace with a single Open Collective username ko_fi: # Replace with a single Ko-fi username tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry liberapay: # Replace with a single Liberapay username issuehunt: # Replace with a single IssueHunt username otechie: # Replace with a single Otechie username lfx_crowdfunding: # Replace with a single LFX Crowdfunding project-name e.g., cloud-foundry custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2'] ================================================ FILE: .github/ISSUE_TEMPLATE/bug_report.yml ================================================ name: Bug Report description: File a bug report title: "" labels: ["bug"] body: - type: markdown attributes: value: | Before filing a bug report, [search for an existing issue](https://github.com/carson-katri/dream-textures/issues?q=is%3Aissue). Also, ensure you are running the [latest version](https://github.com/carson-katri/dream-textures/releases/latest). - type: textarea id: description attributes: label: Description description: Provide a clear and concise description of what the bug is. placeholder: Description validations: required: true - type: textarea id: steps attributes: label: Steps to Reproduce description: List the steps needed to reproduce the issue. placeholder: | 1. Go to '...' 2. Click on '...' validations: required: true - type: textarea id: expected-behavior attributes: label: Expected Behavior description: Describe what you expected to happen. placeholder: | The 'action' would do 'some amazing thing'. validations: required: true - type: markdown attributes: value: | Unless you are running on an unsupported platform, ensure you downloaded a [packaged release](https://github.com/carson-katri/dream-textures/releases/latest) and not the source code. - type: dropdown id: addon-version attributes: label: Addon Version multiple: false options: - Windows (CUDA) - Windows (DirectML) - macOS (Apple Silicon) - Other (Built from source) validations: required: true - type: dropdown id: blender-version attributes: label: Blender Version multiple: false options: - Blender 4.1+ - Blender 3.6 - 4.0 validations: required: true - type: dropdown id: hardware attributes: label: GPU description: NVIDIA 16 series cards are known to have a difficult time running Stable Diffusion. Please see the other issues regarding these cards. multiple: false options: - NVIDIA - NVIDIA 16 Series - AMD - Apple Silicon - Other validations: required: true ================================================ FILE: .github/ISSUE_TEMPLATE/feature_request.md ================================================ --- name: Feature request about: Suggest an idea for this project title: '' labels: enhancement assignees: '' --- **Is your feature request related to a problem? Please describe.** A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] **Describe the solution you'd like** A clear and concise description of what you want to happen. **Describe alternatives you've considered** A clear and concise description of any alternative solutions or features you've considered. **Additional context** Add any other context or screenshots about the feature request here. ================================================ FILE: .github/workflows/package-release.yml ================================================ name: Package Release on: push: branches: - "releases/**" workflow_dispatch: jobs: package-release: strategy: matrix: platform: - requirements: win-linux-cuda.txt os: windows-latest filename: windows-cuda - requirements: win-dml.txt os: windows-latest filename: windows-directml - requirements: mac-mps-cpu.txt os: macos-14 filename: macos-arm version: - python: '3.10' filename_suffix: '' - python: '3.11' filename_suffix: '-4-1' runs-on: ${{ matrix.platform.os }} steps: - name: Checkout repository uses: actions/checkout@v3 with: path: dream_textures - name: Setup Python uses: actions/setup-python@v4 with: python-version: ${{ matrix.version.python }} cache: 'pip' cache-dependency-path: '**/${{ matrix.platform.requirements }}' - name: Install dependencies into target shell: bash run: 'python -m pip install -r requirements/${{ matrix.platform.requirements }} --no-cache-dir --target .python_dependencies' working-directory: dream_textures - name: Zip dependencies with long paths shell: bash run: 'python ./dream_textures/scripts/zip_dependencies.py' - name: Archive Release uses: thedoctor0/zip-release@main with: type: zip filename: dream_textures-${{ matrix.platform.filename }}${{ matrix.version.filename_suffix }}.zip exclusions: '*.git*' - name: Archive and upload artifact uses: actions/upload-artifact@v3 with: name: dream_textures-${{ matrix.platform.filename }}${{ matrix.version.filename_suffix }} path: dream_textures-${{ matrix.platform.filename }}${{ matrix.version.filename_suffix }}.zip ================================================ FILE: .github/workflows/stale.yml ================================================ name: Close inactive issues on: schedule: - cron: "30 1 * * *" workflow_dispatch: jobs: close-issues: runs-on: ubuntu-latest permissions: issues: write pull-requests: write steps: - uses: actions/stale@v5 with: days-before-issue-stale: 60 days-before-issue-close: 7 stale-issue-label: "stale" stale-issue-message: "This issue is stale because it has been open for 60 days with no activity." close-issue-message: "This issue was closed because it has been inactive for 7 days since being marked as stale." days-before-pr-stale: -1 days-before-pr-close: -1 repo-token: ${{ secrets.GITHUB_TOKEN }} only-labels: bug ================================================ FILE: .gitignore ================================================ .DS_Store # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] *$py.class # C extensions *.so # Distribution / packaging .python_dependencies.zip .Python build/ develop-eggs/ dist/ downloads/ eggs/ .eggs/ lib/ lib64/ parts/ sdist/ var/ wheels/ share/python-wheels/ *.egg-info/ .installed.cfg *.egg MANIFEST # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. *.manifest *.spec # Installer logs pip-log.txt pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ .tox/ .nox/ .coverage .coverage.* .cache nosetests.xml coverage.xml *.cover *.py,cover .hypothesis/ .pytest_cache/ cover/ # Translations *.mo *.pot # Django stuff: *.log local_settings.py db.sqlite3 db.sqlite3-journal # Flask stuff: instance/ .webassets-cache # Scrapy stuff: .scrapy # Sphinx documentation docs/_build/ # PyBuilder .pybuilder/ target/ # Jupyter Notebook .ipynb_checkpoints # IPython profile_default/ ipython_config.py # pyenv # For a library or package, you might want to ignore these files since the code is # intended to run in multiple environments; otherwise, check them in: # .python-version # pipenv # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. # However, in case of collaboration, if having platform-specific dependencies or dependencies # having no cross-platform support, pipenv may install dependencies that don't work, or not # install all needed dependencies. #Pipfile.lock # poetry # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. # This is especially recommended for binary packages to ensure reproducibility, and is more # commonly ignored for libraries. # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control #poetry.lock # pdm # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. #pdm.lock # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it # in version control. # https://pdm.fming.dev/#use-with-ide .pdm.toml # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm __pypackages__/ # Celery stuff celerybeat-schedule celerybeat.pid # SageMath parsed files *.sage.py # Environments .env .venv env/ venv/ ENV/ env.bak/ venv.bak/ # Spyder project settings .spyderproject .spyproject # Rope project settings .ropeproject # mkdocs documentation /site # mypy .mypy_cache/ .dmypy.json dmypy.json # Pyre type checker .pyre/ # pytype static type analyzer .pytype/ # Cython debug symbols cython_debug/ # PyCharm # JetBrains specific template is maintained in a separate JetBrains.gitignore that can # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ ================================================ FILE: .python_dependencies/.gitignore ================================================ * !.gitignore ================================================ FILE: LICENSE ================================================ GNU GENERAL PUBLIC LICENSE Version 3, 29 June 2007 Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/> Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. Preamble The GNU General Public License is a free, copyleft license for software and other kinds of works. The licenses for most software and other practical works are designed to take away your freedom to share and change the works. By contrast, the GNU General Public License is intended to guarantee your freedom to share and change all versions of a program--to make sure it remains free software for all its users. We, the Free Software Foundation, use the GNU General Public License for most of our software; it applies also to any other work released this way by its authors. You can apply it to your programs, too. When we speak of free software, we are referring to freedom, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for them if you wish), that you receive source code or can get it if you want it, that you can change the software or use pieces of it in new free programs, and that you know you can do these things. To protect your rights, we need to prevent others from denying you these rights or asking you to surrender the rights. Therefore, you have certain responsibilities if you distribute copies of the software, or if you modify it: responsibilities to respect the freedom of others. For example, if you distribute copies of such a program, whether gratis or for a fee, you must pass on to the recipients the same freedoms that you received. You must make sure that they, too, receive or can get the source code. And you must show them these terms so they know their rights. Developers that use the GNU GPL protect your rights with two steps: (1) assert copyright on the software, and (2) offer you this License giving you legal permission to copy, distribute and/or modify it. For the developers' and authors' protection, the GPL clearly explains that there is no warranty for this free software. For both users' and authors' sake, the GPL requires that modified versions be marked as changed, so that their problems will not be attributed erroneously to authors of previous versions. Some devices are designed to deny users access to install or run modified versions of the software inside them, although the manufacturer can do so. This is fundamentally incompatible with the aim of protecting users' freedom to change the software. The systematic pattern of such abuse occurs in the area of products for individuals to use, which is precisely where it is most unacceptable. Therefore, we have designed this version of the GPL to prohibit the practice for those products. If such problems arise substantially in other domains, we stand ready to extend this provision to those domains in future versions of the GPL, as needed to protect the freedom of users. Finally, every program is threatened constantly by software patents. States should not allow patents to restrict development and use of software on general-purpose computers, but in those that do, we wish to avoid the special danger that patents applied to a free program could make it effectively proprietary. To prevent this, the GPL assures that patents cannot be used to render the program non-free. The precise terms and conditions for copying, distribution and modification follow. TERMS AND CONDITIONS 0. Definitions. "This License" refers to version 3 of the GNU General Public License. "Copyright" also means copyright-like laws that apply to other kinds of works, such as semiconductor masks. "The Program" refers to any copyrightable work licensed under this License. Each licensee is addressed as "you". "Licensees" and "recipients" may be individuals or organizations. To "modify" a work means to copy from or adapt all or part of the work in a fashion requiring copyright permission, other than the making of an exact copy. The resulting work is called a "modified version" of the earlier work or a work "based on" the earlier work. A "covered work" means either the unmodified Program or a work based on the Program. To "propagate" a work means to do anything with it that, without permission, would make you directly or secondarily liable for infringement under applicable copyright law, except executing it on a computer or modifying a private copy. Propagation includes copying, distribution (with or without modification), making available to the public, and in some countries other activities as well. To "convey" a work means any kind of propagation that enables other parties to make or receive copies. Mere interaction with a user through a computer network, with no transfer of a copy, is not conveying. An interactive user interface displays "Appropriate Legal Notices" to the extent that it includes a convenient and prominently visible feature that (1) displays an appropriate copyright notice, and (2) tells the user that there is no warranty for the work (except to the extent that warranties are provided), that licensees may convey the work under this License, and how to view a copy of this License. If the interface presents a list of user commands or options, such as a menu, a prominent item in the list meets this criterion. 1. Source Code. The "source code" for a work means the preferred form of the work for making modifications to it. "Object code" means any non-source form of a work. A "Standard Interface" means an interface that either is an official standard defined by a recognized standards body, or, in the case of interfaces specified for a particular programming language, one that is widely used among developers working in that language. The "System Libraries" of an executable work include anything, other than the work as a whole, that (a) is included in the normal form of packaging a Major Component, but which is not part of that Major Component, and (b) serves only to enable use of the work with that Major Component, or to implement a Standard Interface for which an implementation is available to the public in source code form. A "Major Component", in this context, means a major essential component (kernel, window system, and so on) of the specific operating system (if any) on which the executable work runs, or a compiler used to produce the work, or an object code interpreter used to run it. The "Corresponding Source" for a work in object code form means all the source code needed to generate, install, and (for an executable work) run the object code and to modify the work, including scripts to control those activities. However, it does not include the work's System Libraries, or general-purpose tools or generally available free programs which are used unmodified in performing those activities but which are not part of the work. For example, Corresponding Source includes interface definition files associated with source files for the work, and the source code for shared libraries and dynamically linked subprograms that the work is specifically designed to require, such as by intimate data communication or control flow between those subprograms and other parts of the work. The Corresponding Source need not include anything that users can regenerate automatically from other parts of the Corresponding Source. The Corresponding Source for a work in source code form is that same work. 2. Basic Permissions. All rights granted under this License are granted for the term of copyright on the Program, and are irrevocable provided the stated conditions are met. This License explicitly affirms your unlimited permission to run the unmodified Program. The output from running a covered work is covered by this License only if the output, given its content, constitutes a covered work. This License acknowledges your rights of fair use or other equivalent, as provided by copyright law. You may make, run and propagate covered works that you do not convey, without conditions so long as your license otherwise remains in force. You may convey covered works to others for the sole purpose of having them make modifications exclusively for you, or provide you with facilities for running those works, provided that you comply with the terms of this License in conveying all material for which you do not control copyright. Those thus making or running the covered works for you must do so exclusively on your behalf, under your direction and control, on terms that prohibit them from making any copies of your copyrighted material outside their relationship with you. Conveying under any other circumstances is permitted solely under the conditions stated below. Sublicensing is not allowed; section 10 makes it unnecessary. 3. Protecting Users' Legal Rights From Anti-Circumvention Law. No covered work shall be deemed part of an effective technological measure under any applicable law fulfilling obligations under article 11 of the WIPO copyright treaty adopted on 20 December 1996, or similar laws prohibiting or restricting circumvention of such measures. When you convey a covered work, you waive any legal power to forbid circumvention of technological measures to the extent such circumvention is effected by exercising rights under this License with respect to the covered work, and you disclaim any intention to limit operation or modification of the work as a means of enforcing, against the work's users, your or third parties' legal rights to forbid circumvention of technological measures. 4. Conveying Verbatim Copies. You may convey verbatim copies of the Program's source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice; keep intact all notices stating that this License and any non-permissive terms added in accord with section 7 apply to the code; keep intact all notices of the absence of any warranty; and give all recipients a copy of this License along with the Program. You may charge any price or no price for each copy that you convey, and you may offer support or warranty protection for a fee. 5. Conveying Modified Source Versions. You may convey a work based on the Program, or the modifications to produce it from the Program, in the form of source code under the terms of section 4, provided that you also meet all of these conditions: a) The work must carry prominent notices stating that you modified it, and giving a relevant date. b) The work must carry prominent notices stating that it is released under this License and any conditions added under section 7. This requirement modifies the requirement in section 4 to "keep intact all notices". c) You must license the entire work, as a whole, under this License to anyone who comes into possession of a copy. This License will therefore apply, along with any applicable section 7 additional terms, to the whole of the work, and all its parts, regardless of how they are packaged. This License gives no permission to license the work in any other way, but it does not invalidate such permission if you have separately received it. d) If the work has interactive user interfaces, each must display Appropriate Legal Notices; however, if the Program has interactive interfaces that do not display Appropriate Legal Notices, your work need not make them do so. A compilation of a covered work with other separate and independent works, which are not by their nature extensions of the covered work, and which are not combined with it such as to form a larger program, in or on a volume of a storage or distribution medium, is called an "aggregate" if the compilation and its resulting copyright are not used to limit the access or legal rights of the compilation's users beyond what the individual works permit. Inclusion of a covered work in an aggregate does not cause this License to apply to the other parts of the aggregate. 6. Conveying Non-Source Forms. You may convey a covered work in object code form under the terms of sections 4 and 5, provided that you also convey the machine-readable Corresponding Source under the terms of this License, in one of these ways: a) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by the Corresponding Source fixed on a durable physical medium customarily used for software interchange. b) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by a written offer, valid for at least three years and valid for as long as you offer spare parts or customer support for that product model, to give anyone who possesses the object code either (1) a copy of the Corresponding Source for all the software in the product that is covered by this License, on a durable physical medium customarily used for software interchange, for a price no more than your reasonable cost of physically performing this conveying of source, or (2) access to copy the Corresponding Source from a network server at no charge. c) Convey individual copies of the object code with a copy of the written offer to provide the Corresponding Source. This alternative is allowed only occasionally and noncommercially, and only if you received the object code with such an offer, in accord with subsection 6b. d) Convey the object code by offering access from a designated place (gratis or for a charge), and offer equivalent access to the Corresponding Source in the same way through the same place at no further charge. You need not require recipients to copy the Corresponding Source along with the object code. If the place to copy the object code is a network server, the Corresponding Source may be on a different server (operated by you or a third party) that supports equivalent copying facilities, provided you maintain clear directions next to the object code saying where to find the Corresponding Source. Regardless of what server hosts the Corresponding Source, you remain obligated to ensure that it is available for as long as needed to satisfy these requirements. e) Convey the object code using peer-to-peer transmission, provided you inform other peers where the object code and Corresponding Source of the work are being offered to the general public at no charge under subsection 6d. A separable portion of the object code, whose source code is excluded from the Corresponding Source as a System Library, need not be included in conveying the object code work. A "User Product" is either (1) a "consumer product", which means any tangible personal property which is normally used for personal, family, or household purposes, or (2) anything designed or sold for incorporation into a dwelling. In determining whether a product is a consumer product, doubtful cases shall be resolved in favor of coverage. For a particular product received by a particular user, "normally used" refers to a typical or common use of that class of product, regardless of the status of the particular user or of the way in which the particular user actually uses, or expects or is expected to use, the product. A product is a consumer product regardless of whether the product has substantial commercial, industrial or non-consumer uses, unless such uses represent the only significant mode of use of the product. "Installation Information" for a User Product means any methods, procedures, authorization keys, or other information required to install and execute modified versions of a covered work in that User Product from a modified version of its Corresponding Source. The information must suffice to ensure that the continued functioning of the modified object code is in no case prevented or interfered with solely because modification has been made. If you convey an object code work under this section in, or with, or specifically for use in, a User Product, and the conveying occurs as part of a transaction in which the right of possession and use of the User Product is transferred to the recipient in perpetuity or for a fixed term (regardless of how the transaction is characterized), the Corresponding Source conveyed under this section must be accompanied by the Installation Information. But this requirement does not apply if neither you nor any third party retains the ability to install modified object code on the User Product (for example, the work has been installed in ROM). The requirement to provide Installation Information does not include a requirement to continue to provide support service, warranty, or updates for a work that has been modified or installed by the recipient, or for the User Product in which it has been modified or installed. Access to a network may be denied when the modification itself materially and adversely affects the operation of the network or violates the rules and protocols for communication across the network. Corresponding Source conveyed, and Installation Information provided, in accord with this section must be in a format that is publicly documented (and with an implementation available to the public in source code form), and must require no special password or key for unpacking, reading or copying. 7. Additional Terms. "Additional permissions" are terms that supplement the terms of this License by making exceptions from one or more of its conditions. Additional permissions that are applicable to the entire Program shall be treated as though they were included in this License, to the extent that they are valid under applicable law. If additional permissions apply only to part of the Program, that part may be used separately under those permissions, but the entire Program remains governed by this License without regard to the additional permissions. When you convey a copy of a covered work, you may at your option remove any additional permissions from that copy, or from any part of it. (Additional permissions may be written to require their own removal in certain cases when you modify the work.) You may place additional permissions on material, added by you to a covered work, for which you have or can give appropriate copyright permission. Notwithstanding any other provision of this License, for material you add to a covered work, you may (if authorized by the copyright holders of that material) supplement the terms of this License with terms: a) Disclaiming warranty or limiting liability differently from the terms of sections 15 and 16 of this License; or b) Requiring preservation of specified reasonable legal notices or author attributions in that material or in the Appropriate Legal Notices displayed by works containing it; or c) Prohibiting misrepresentation of the origin of that material, or requiring that modified versions of such material be marked in reasonable ways as different from the original version; or d) Limiting the use for publicity purposes of names of licensors or authors of the material; or e) Declining to grant rights under trademark law for use of some trade names, trademarks, or service marks; or f) Requiring indemnification of licensors and authors of that material by anyone who conveys the material (or modified versions of it) with contractual assumptions of liability to the recipient, for any liability that these contractual assumptions directly impose on those licensors and authors. All other non-permissive additional terms are considered "further restrictions" within the meaning of section 10. If the Program as you received it, or any part of it, contains a notice stating that it is governed by this License along with a term that is a further restriction, you may remove that term. If a license document contains a further restriction but permits relicensing or conveying under this License, you may add to a covered work material governed by the terms of that license document, provided that the further restriction does not survive such relicensing or conveying. If you add terms to a covered work in accord with this section, you must place, in the relevant source files, a statement of the additional terms that apply to those files, or a notice indicating where to find the applicable terms. Additional terms, permissive or non-permissive, may be stated in the form of a separately written license, or stated as exceptions; the above requirements apply either way. 8. Termination. You may not propagate or modify a covered work except as expressly provided under this License. Any attempt otherwise to propagate or modify it is void, and will automatically terminate your rights under this License (including any patent licenses granted under the third paragraph of section 11). However, if you cease all violation of this License, then your license from a particular copyright holder is reinstated (a) provisionally, unless and until the copyright holder explicitly and finally terminates your license, and (b) permanently, if the copyright holder fails to notify you of the violation by some reasonable means prior to 60 days after the cessation. Moreover, your license from a particular copyright holder is reinstated permanently if the copyright holder notifies you of the violation by some reasonable means, this is the first time you have received notice of violation of this License (for any work) from that copyright holder, and you cure the violation prior to 30 days after your receipt of the notice. Termination of your rights under this section does not terminate the licenses of parties who have received copies or rights from you under this License. If your rights have been terminated and not permanently reinstated, you do not qualify to receive new licenses for the same material under section 10. 9. Acceptance Not Required for Having Copies. You are not required to accept this License in order to receive or run a copy of the Program. Ancillary propagation of a covered work occurring solely as a consequence of using peer-to-peer transmission to receive a copy likewise does not require acceptance. However, nothing other than this License grants you permission to propagate or modify any covered work. These actions infringe copyright if you do not accept this License. Therefore, by modifying or propagating a covered work, you indicate your acceptance of this License to do so. 10. Automatic Licensing of Downstream Recipients. Each time you convey a covered work, the recipient automatically receives a license from the original licensors, to run, modify and propagate that work, subject to this License. You are not responsible for enforcing compliance by third parties with this License. An "entity transaction" is a transaction transferring control of an organization, or substantially all assets of one, or subdividing an organization, or merging organizations. If propagation of a covered work results from an entity transaction, each party to that transaction who receives a copy of the work also receives whatever licenses to the work the party's predecessor in interest had or could give under the previous paragraph, plus a right to possession of the Corresponding Source of the work from the predecessor in interest, if the predecessor has it or can get it with reasonable efforts. You may not impose any further restrictions on the exercise of the rights granted or affirmed under this License. For example, you may not impose a license fee, royalty, or other charge for exercise of rights granted under this License, and you may not initiate litigation (including a cross-claim or counterclaim in a lawsuit) alleging that any patent claim is infringed by making, using, selling, offering for sale, or importing the Program or any portion of it. 11. Patents. A "contributor" is a copyright holder who authorizes use under this License of the Program or a work on which the Program is based. The work thus licensed is called the contributor's "contributor version". A contributor's "essential patent claims" are all patent claims owned or controlled by the contributor, whether already acquired or hereafter acquired, that would be infringed by some manner, permitted by this License, of making, using, or selling its contributor version, but do not include claims that would be infringed only as a consequence of further modification of the contributor version. For purposes of this definition, "control" includes the right to grant patent sublicenses in a manner consistent with the requirements of this License. Each contributor grants you a non-exclusive, worldwide, royalty-free patent license under the contributor's essential patent claims, to make, use, sell, offer for sale, import and otherwise run, modify and propagate the contents of its contributor version. In the following three paragraphs, a "patent license" is any express agreement or commitment, however denominated, not to enforce a patent (such as an express permission to practice a patent or covenant not to sue for patent infringement). To "grant" such a patent license to a party means to make such an agreement or commitment not to enforce a patent against the party. If you convey a covered work, knowingly relying on a patent license, and the Corresponding Source of the work is not available for anyone to copy, free of charge and under the terms of this License, through a publicly available network server or other readily accessible means, then you must either (1) cause the Corresponding Source to be so available, or (2) arrange to deprive yourself of the benefit of the patent license for this particular work, or (3) arrange, in a manner consistent with the requirements of this License, to extend the patent license to downstream recipients. "Knowingly relying" means you have actual knowledge that, but for the patent license, your conveying the covered work in a country, or your recipient's use of the covered work in a country, would infringe one or more identifiable patents in that country that you have reason to believe are valid. If, pursuant to or in connection with a single transaction or arrangement, you convey, or propagate by procuring conveyance of, a covered work, and grant a patent license to some of the parties receiving the covered work authorizing them to use, propagate, modify or convey a specific copy of the covered work, then the patent license you grant is automatically extended to all recipients of the covered work and works based on it. A patent license is "discriminatory" if it does not include within the scope of its coverage, prohibits the exercise of, or is conditioned on the non-exercise of one or more of the rights that are specifically granted under this License. You may not convey a covered work if you are a party to an arrangement with a third party that is in the business of distributing software, under which you make payment to the third party based on the extent of your activity of conveying the work, and under which the third party grants, to any of the parties who would receive the covered work from you, a discriminatory patent license (a) in connection with copies of the covered work conveyed by you (or copies made from those copies), or (b) primarily for and in connection with specific products or compilations that contain the covered work, unless you entered into that arrangement, or that patent license was granted, prior to 28 March 2007. Nothing in this License shall be construed as excluding or limiting any implied license or other defenses to infringement that may otherwise be available to you under applicable patent law. 12. No Surrender of Others' Freedom. If conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not excuse you from the conditions of this License. If you cannot convey a covered work so as to satisfy simultaneously your obligations under this License and any other pertinent obligations, then as a consequence you may not convey it at all. For example, if you agree to terms that obligate you to collect a royalty for further conveying from those to whom you convey the Program, the only way you could satisfy both those terms and this License would be to refrain entirely from conveying the Program. 13. Use with the GNU Affero General Public License. Notwithstanding any other provision of this License, you have permission to link or combine any covered work with a work licensed under version 3 of the GNU Affero General Public License into a single combined work, and to convey the resulting work. The terms of this License will continue to apply to the part which is the covered work, but the special requirements of the GNU Affero General Public License, section 13, concerning interaction through a network will apply to the combination as such. 14. Revised Versions of this License. The Free Software Foundation may publish revised and/or new versions of the GNU General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns. Each version is given a distinguishing version number. If the Program specifies that a certain numbered version of the GNU General Public License "or any later version" applies to it, you have the option of following the terms and conditions either of that numbered version or of any later version published by the Free Software Foundation. If the Program does not specify a version number of the GNU General Public License, you may choose any version ever published by the Free Software Foundation. If the Program specifies that a proxy can decide which future versions of the GNU General Public License can be used, that proxy's public statement of acceptance of a version permanently authorizes you to choose that version for the Program. Later license versions may give you additional or different permissions. However, no additional obligations are imposed on any author or copyright holder as a result of your choosing to follow a later version. 15. Disclaimer of Warranty. THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 16. Limitation of Liability. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. 17. Interpretation of Sections 15 and 16. If the disclaimer of warranty and limitation of liability provided above cannot be given local legal effect according to their terms, reviewing courts shall apply local law that most closely approximates an absolute waiver of all civil liability in connection with the Program, unless a warranty or assumption of liability accompanies a copy of the Program in return for a fee. END OF TERMS AND CONDITIONS How to Apply These Terms to Your New Programs If you develop a new program, and you want it to be of the greatest possible use to the public, the best way to achieve this is to make it free software which everyone can redistribute and change under these terms. To do so, attach the following notices to the program. It is safest to attach them to the start of each source file to most effectively state the exclusion of warranty; and each file should have at least the "copyright" line and a pointer to where the full notice is found. <one line to give the program's name and a brief idea of what it does.> Copyright (C) <year> <name of author> This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see <https://www.gnu.org/licenses/>. Also add information on how to contact you by electronic and paper mail. If the program does terminal interaction, make it output a short notice like this when it starts in an interactive mode: <program> Copyright (C) <year> <name of author> This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. This is free software, and you are welcome to redistribute it under certain conditions; type `show c' for details. The hypothetical commands `show w' and `show c' should show the appropriate parts of the General Public License. Of course, your program's commands might be different; for a GUI interface, you would use an "about box". You should also get your employer (if you work as a programmer) or school, if any, to sign a "copyright disclaimer" for the program, if necessary. For more information on this, and how to apply and follow the GNU GPL, see <https://www.gnu.org/licenses/>. The GNU General Public License does not permit incorporating your program into proprietary programs. If your program is a subroutine library, you may consider it more useful to permit linking proprietary applications with the library. If this is what you want to do, use the GNU Lesser General Public License instead of this License. But first, please read <https://www.gnu.org/licenses/why-not-lgpl.html>. ================================================ FILE: README.md ================================================ ![Dream Textures, subtitle: Stable Diffusion built-in to Blender](docs/assets/banner.png) [![Latest Release](https://flat.badgen.net/github/release/carson-katri/dream-textures)](https://github.com/carson-katri/dream-textures/releases/latest) [![Join the Discord](https://flat.badgen.net/badge/icon/discord?icon=discord&label)](https://discord.gg/EmDJ8CaWZ7) [![Total Downloads](https://img.shields.io/github/downloads/carson-katri/dream-textures/total?style=flat-square)](https://github.com/carson-katri/dream-textures/releases/latest) [![Buy on Blender Market](https://flat.badgen.net/badge/buy/blender%20market/orange)](https://www.blendermarket.com/products/dream-textures) * Create textures, concept art, background assets, and more with a simple text prompt * Use the 'Seamless' option to create textures that tile perfectly with no visible seam * Texture entire scenes with 'Project Dream Texture' and depth to image * Re-style animations with the Cycles render pass * Run the models on your machine to iterate without slowdowns from a service # Installation Download the [latest release](https://github.com/carson-katri/dream-textures/releases/latest) and follow the instructions there to get up and running. > On macOS, it is possible you will run into a quarantine issue with the dependencies. To work around this, run the following command in the app `Terminal`: `xattr -r -d com.apple.quarantine ~/Library/Application\ Support/Blender/3.3/scripts/addons/dream_textures/.python_dependencies`. This will allow the PyTorch `.dylib`s and `.so`s to load without having to manually allow each one in System Preferences. If you want a visual guide to installation, see this video tutorial from Ashlee Martino-Tarr: https://youtu.be/kEcr8cNmqZk > Ensure you always install the [latest version](https://github.com/carson-katri/dream-textures/releases/latest) of the add-on if any guides become out of date. # Usage Here's a few quick guides: ## [Setting Up](https://github.com/carson-katri/dream-textures/wiki/Setup) Setup instructions for various platforms and configurations. ## [Image Generation](https://github.com/carson-katri/dream-textures/wiki/Image-Generation) Create textures, concept art, and more with text prompts. Learn how to use the various configuration options to get exactly what you're looking for. ![A graphic showing each step of the image generation process](docs/assets/image_generation.png) ## [Texture Projection](https://github.com/carson-katri/dream-textures/wiki/Texture-Projection) Texture entire models and scenes with depth to image. ![A graphic showing each step of the texture projection process](docs/assets/texture_projection.png) ## [Inpaint/Outpaint](https://github.com/carson-katri/dream-textures/wiki/Inpaint-and-Outpaint) Inpaint to fix up images and convert existing textures into seamless ones automatically. Outpaint to increase the size of an image by extending it in any direction. ![A graphic showing each step of the outpainting process](docs/assets/inpaint_outpaint.png) ## [Render Engine](https://github.com/carson-katri/dream-textures/wiki/Render-Engine) Use the Dream Textures node system to create complex effects. ![A graphic showing each frame of a render, split with the scene and generated result](docs/assets/render_pass.png) ## [AI Upscaling](https://github.com/carson-katri/dream-textures/wiki/AI-Upscaling) Upscale your low-res generations 4x. ![A graphic showing each step of the upscaling process](docs/assets/upscale.png) ## [History](https://github.com/carson-katri/dream-textures/wiki/History) Recall, export, and import history entries for later use. # Compatibility Dream Textures has been tested with CUDA and Apple Silicon GPUs. Over 4GB of VRAM is recommended. If you have an issue with a supported GPU, please create an issue. ### Cloud Processing If your hardware is unsupported, you can use DreamStudio to process in the cloud. Follow the instructions in the release notes to setup with DreamStudio. # Contributing For detailed instructions on installing from source, see the guide on [setting up a development environment](https://github.com/carson-katri/dream-textures/wiki/Setting-Up-a-Development-Environment). # Troubleshooting If you are experiencing trouble getting Dream Textures running, check Blender's system console (in the top left under the "Window" dropdown next to "File" and "Edit") for any error messages. Then [search in the issues list](https://github.com/carson-katri/dream-textures/issues?q=is%3Aissue) with your error message and symptoms. > **Note** On macOS there is no option to open the system console. Instead, you can get logs by opening the app *Terminal*, entering the command `/Applications/Blender.app/Contents/MacOS/Blender` and pressing the Enter key. This will launch Blender and any error messages will show up in the Terminal app. ![A screenshot of the "Window" > "Toggle System Console" menu action in Blender](docs/assets/readme-toggle-console.png) Features and feedback are also accepted on the issues page. If you have any issues that aren't listed, feel free to add them there! The [Dream Textures Discord server](https://discord.gg/EmDJ8CaWZ7) also has a common issues list and strong community of helpful people, so feel free to come by for some help there as well. ================================================ FILE: __init__.py ================================================ # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, but # WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTIBILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. bl_info = { "name": "Dream Textures", "author": "Dream Textures contributors", "description": "Use Stable Diffusion to generate unique textures straight from the shader editor.", "blender": (3, 1, 0), "version": (0, 4, 1), "location": "Image Editor -> Sidebar -> Dream", "category": "Paint" } from multiprocessing import current_process if current_process().name != "__actor__": import bpy from bpy.props import IntProperty, PointerProperty, EnumProperty, BoolProperty, CollectionProperty import sys import os module_name = os.path.basename(os.path.dirname(__file__)) def clear_modules(): for name in list(sys.modules.keys()): if name.startswith(module_name) and name != module_name: del sys.modules[name] clear_modules() # keep before all addon imports from .render_pass import register_render_pass, unregister_render_pass, pass_inputs from .prompt_engineering import * from .operators.open_latest_version import check_for_updates from .operators.project import framebuffer_arguments from .classes import CLASSES, PREFERENCE_CLASSES from .tools import TOOLS from .operators.dream_texture import DreamTexture, kill_generator from .property_groups.dream_prompt import DreamPrompt from .property_groups.seamless_result import SeamlessResult from .ui.presets import register_default_presets from . import engine from .diffusers_backend import DiffusersBackend requirements_path_items = ( ('requirements/win-linux-cuda.txt', 'Linux/Windows (CUDA)', 'Linux or Windows with NVIDIA GPU'), ('requirements/mac-mps-cpu.txt', 'Apple Silicon', 'Apple M1/M2'), ('requirements/linux-rocm.txt', 'Linux (AMD)', 'Linux with AMD GPU'), ('requirements/win-dml.txt', 'Windows (DirectML)', 'Windows with DirectX 12 GPU'), ('requirements/dreamstudio.txt', 'DreamStudio', 'Cloud Compute Service') ) def register(): dt_op = bpy.ops for name in DreamTexture.bl_idname.split("."): dt_op = getattr(dt_op, name) if hasattr(bpy.types, dt_op.idname()): # objects under bpy.ops are created on the fly, have to check that it actually exists a little differently raise RuntimeError("Another instance of Dream Textures is already running.") bpy.types.Scene.dream_textures_requirements_path = EnumProperty(name="Platform", items=requirements_path_items, description="Specifies which set of dependencies to install", default='requirements/mac-mps-cpu.txt' if sys.platform == 'darwin' else 'requirements/win-linux-cuda.txt') for cls in PREFERENCE_CLASSES: bpy.utils.register_class(cls) bpy.types.Scene.dream_textures_history = CollectionProperty(type=DreamPrompt) check_for_updates() bpy.types.Scene.dream_textures_prompt = PointerProperty(type=DreamPrompt) bpy.types.Scene.dream_textures_prompt_file = PointerProperty(type=bpy.types.Text) bpy.types.Scene.init_img = PointerProperty(name="Init Image", type=bpy.types.Image) bpy.types.Scene.init_mask = PointerProperty(name="Init Mask", type=bpy.types.Image) bpy.types.Scene.init_depth = PointerProperty(name="Init Depth", type=bpy.types.Image, description="Use an existing depth map. Leave blank to generate one from the init image") bpy.types.Scene.seamless_result = PointerProperty(type=SeamlessResult) def get_selection_preview(self): history = bpy.context.scene.dream_textures_history if self.dream_textures_history_selection > 0 and self.dream_textures_history_selection < len(history): return history[self.dream_textures_history_selection].generate_prompt() return "" bpy.types.Scene.dream_textures_history_selection = IntProperty(default=1) bpy.types.Scene.dream_textures_history_selection_preview = bpy.props.StringProperty(name="", default="", get=get_selection_preview, set=lambda _, __: None) bpy.types.Scene.dream_textures_progress = bpy.props.IntProperty(name="", default=0, min=0, max=0) bpy.types.Scene.dream_textures_info = bpy.props.StringProperty(name="Info") bpy.types.Scene.dream_textures_last_execution_time = bpy.props.StringProperty(name="Last Execution Time", default="") bpy.types.Scene.dream_textures_viewport_enabled = BoolProperty(name="Viewport Enabled", default=False) bpy.types.Scene.dream_textures_render_properties_enabled = BoolProperty(default=False) bpy.types.Scene.dream_textures_render_properties_prompt = PointerProperty(type=DreamPrompt) bpy.types.Scene.dream_textures_render_properties_pass_inputs = EnumProperty(name="Pass Inputs", items=pass_inputs) bpy.types.Scene.dream_textures_upscale_prompt = PointerProperty(type=DreamPrompt) bpy.types.Scene.dream_textures_upscale_tile_size = IntProperty(name="Tile Size", default=128, step=64, min=64, max=512) bpy.types.Scene.dream_textures_upscale_blend = IntProperty(name="Blend", default=32, step=8, min=0, max=512) bpy.types.Scene.dream_textures_upscale_seamless_result = PointerProperty(type=SeamlessResult) bpy.types.Scene.dream_textures_project_prompt = PointerProperty(type=DreamPrompt) bpy.types.Scene.dream_textures_project_framebuffer_arguments = EnumProperty(name="Inputs", items=framebuffer_arguments) bpy.types.Scene.dream_textures_project_bake = BoolProperty(name="Bake", default=False, description="Re-maps the generated texture onto the specified UV map") def project_use_controlnet(self, context): if self.dream_textures_project_use_control_net: if len(self.dream_textures_project_prompt.control_nets) < 1: self.dream_textures_project_prompt.control_nets.add() else: self.dream_textures_project_prompt.control_nets.clear() bpy.types.Scene.dream_textures_project_use_control_net = BoolProperty(name="Use ControlNet", default=False, description="Use a depth ControlNet instead of a depth model", update=project_use_controlnet) engine.register() for cls in CLASSES: bpy.utils.register_class(cls) for tool in TOOLS: bpy.utils.register_tool(tool) bpy.types.Scene.dream_textures_render_engine = PointerProperty(type=engine.DreamTexturesRenderEngineProperties) bpy.types.RENDER_PT_context.append(engine.draw_device) # Monkey patch cycles render passes register_render_pass() register_default_presets() # Register the default backend. bpy.utils.register_class(DiffusersBackend) def unregister(): for cls in PREFERENCE_CLASSES: bpy.utils.unregister_class(cls) for cls in CLASSES: bpy.utils.unregister_class(cls) for tool in TOOLS: bpy.utils.unregister_tool(tool) bpy.types.RENDER_PT_context.remove(engine.draw_device) engine.unregister() unregister_render_pass() # Unregister the default backend bpy.utils.unregister_class(DiffusersBackend) kill_generator() ================================================ FILE: absolute_path.py ================================================ import os def absolute_path(component: str): """ Returns the absolute path to a file in the addon directory. Alternative to `os.abspath` that works the same on macOS and Windows. """ return os.path.join(os.path.dirname(os.path.realpath(__file__)), component) REAL_ESRGAN_WEIGHTS_PATH = absolute_path("weights/realesrgan/realesr-general-x4v3.pth") CLIPSEG_WEIGHTS_PATH = absolute_path("weights/clipseg/rd64-uni.pth") ================================================ FILE: api/__init__.py ================================================ from .models import * from .backend import * ================================================ FILE: api/backend/__init__.py ================================================ from .backend import * ================================================ FILE: api/backend/backend.py ================================================ try: import bpy from typing import Callable, List, Tuple from ..models.generation_arguments import GenerationArguments from ..models.generation_result import GenerationResult from ..models.model import Model StepCallback = Callable[[List[GenerationResult]], bool] Callback = Callable[[List[GenerationResult] | Exception], None] class Backend(bpy.types.PropertyGroup): """A backend for Dream Textures. Provide the following methods to create a valid backend. ```python def list_models(self) -> List[Model] def generate( self, arguments: GenerationArguments, step_callback: StepCallback, callback: Callback ) ``` """ @classmethod def register(cls): from ...property_groups.dream_prompt import DreamPrompt setattr(DreamPrompt, cls._attribute(), bpy.props.PointerProperty(type=cls)) @classmethod def unregister(cls): from ...property_groups.dream_prompt import DreamPrompt delattr(DreamPrompt, cls._attribute()) @classmethod def _id(cls) -> str: return f"{cls.__module__}.{cls.__name__}" @classmethod def _attribute(cls) -> str: return cls._id().replace('.', '_') @classmethod def _lookup(cls, id): return next( (backend for backend in cls._list_backends() if backend._id() == id), next(iter(cls._list_backends()), None) ) @classmethod def _list_backends(cls): return cls.__subclasses__() def list_models(self, context) -> List[Model]: """Provide a list of available models. The `id` of the model will be provided. """ ... def list_controlnet_models(self, context) -> List[Model]: """Provide a list of available ControlNet models. The `id` of the model will be provided. """ return [] def list_schedulers(self, context) -> List[str]: """Provide a list of available schedulers.""" ... def draw_prompt(self, layout, context): """Draw additional UI in the 'Prompt' panel""" ... def draw_advanced(self, layout, context): """Draw additional UI in the 'Advanced' panel""" ... def draw_speed_optimizations(self, layout, context): """Draw additional UI in the 'Speed Optimizations' panel""" ... def draw_memory_optimizations(self, layout, context): """Draw additional UI in the 'Memory Optimizations' panel""" ... def draw_extra(self, layout, context): """Draw additional UI in the panel""" ... def get_batch_size(self, context) -> int: """Return the selected batch size for the backend (if applicable). A default implementation is provided that returns `1`. """ return 1 def generate( self, arguments: GenerationArguments, step_callback: StepCallback, callback: Callback ): """ A request to generate an image. If the `step_callback` returns `False`, the generation should be cancelled. After cancelling, `callback` should be called with an `InterruptedError`. """ ... def validate( self, arguments: GenerationArguments ): """Validates the given arguments in the UI without generating. This validation should occur as quickly as possible. To report problems with the inputs, raise a `ValueError`. Use the `FixItError` to provide a solution to the problem as well. ```python if arguments.steps % 2 == 0: throw FixItError( "The number of steps is even", solution=FixItError.UpdateGenerationArgumentsSolution( title="Add 1 more step", arguments=dataclasses.replace( arguments, steps=arguments.steps + 1 ) ) ) ``` """ ... except: pass ================================================ FILE: api/models/__init__.py ================================================ from .generation_result import * from .model import * from .prompt import * from .seamless_axes import * from .step_preview_mode import * from .task import * from .fix_it_error import * ================================================ FILE: api/models/control_net.py ================================================ from dataclasses import dataclass from typing import Tuple, List from numpy.typing import NDArray @dataclass class ControlNet: model: str """The selected ControlNet model used for generation""" image: NDArray """The control image""" strength: float """The strength of the ControlNet's influence""" ================================================ FILE: api/models/fix_it_error.py ================================================ from typing import Callable, Any from .generation_arguments import GenerationArguments from dataclasses import dataclass class FixItError(Exception): """An exception with a solution. Call the `draw` method to render the UI elements responsible for resolving this error. """ def __init__(self, message, solution: 'Solution'): super().__init__(message) self._solution = solution def _draw(self, dream_prompt, context, layout): self._solution._draw(dream_prompt, context, layout) @dataclass class Solution: def _draw(self, dream_prompt, context, layout): ... @dataclass class ChangeProperty(Solution): """Prompts the user to change the given `property` of the `GenerationArguments`.""" property: str def _draw(self, dream_prompt, context, layout): layout.prop(dream_prompt, self.property) @dataclass class RunOperator(Solution): """Runs the given operator""" title: str operator: str modify_operator: Callable[[Any], None] def _draw(self, dream_prompt, context, layout): self.modify_operator( layout.operator(self.operator, text=self.title) ) ================================================ FILE: api/models/generation_arguments.py ================================================ from dataclasses import dataclass from typing import Tuple, List from ..models.task import Task from ..models.model import Model from ..models.prompt import Prompt from ..models.seamless_axes import SeamlessAxes from ..models.step_preview_mode import StepPreviewMode from ..models.control_net import ControlNet @dataclass class GenerationArguments: task: Task """The type of generation to perform. Use a match statement to perform different actions based on the selected task. ```python match task: case PromptToImage(): ... case ImageToImage(image=image, strength=strength, fit=fit): ... case Inpaint(image=image, fit=fit, strength=strength, mask_source=mask_source, mask_prompt=mask_prompt, confidence=confidence): ... case DepthToImage(depth=depth, image=image, strength=strength): ... case Outpaint(image=image, origin=origin): ... case _: raise NotImplementedError() ``` """ model: Model """The selected model. This is one of the options provided by `Backend.list_models`. """ prompt: Prompt """The positive and (optionally) negative prompt. If `prompt.negative` is `None`, then the 'Negative Prompt' panel was disabled by the user. """ size: Tuple[int, int] | None """The target size of the image, or `None` to use the native size of the model.""" seed: int """The random or user-provided seed to use.""" steps: int """The number of inference steps to perform.""" guidance_scale: float """The selected classifier-free guidance scale.""" scheduler: str """The selected scheduler. This is one of the options provided by `Backend.list_schedulers`. """ seamless_axes: SeamlessAxes """Which axes to tile seamlessly.""" step_preview_mode: StepPreviewMode """The style of preview to display at each step.""" iterations: int """The number of images to generate. The value sent to `callback` should contain the same number of `GenerationResult` instances in a list. """ control_nets: List[ControlNet] @staticmethod def _map_property_name(name: str) -> str | List[str] | None: """Converts a property name from `GenerationArguments` to the corresponding property of a `DreamPrompt`.""" match name: case "model": return "model" case "prompt": return ["prompt", "use_negative_prompt", "negative_prompt"] case "prompt.positive": return "prompt" case "prompt.negative": return ["use_negative_prompt", "negative_prompt"] case "size": return ["use_size", "width", "height"] case "seed": return "seed" case "steps": return "steps" case "guidance_scale": return "cfg_scale" case "scheduler": return "scheduler" case "seamless_axes": return "seamless_axes" case "step_preview_mode": return "step_preview_mode" case "iterations": return "iterations" case _: return None ================================================ FILE: api/models/generation_result.py ================================================ from dataclasses import dataclass from numpy.typing import NDArray import numpy as np import math @dataclass class GenerationResult: """The output of a `Backend`. Create a result with an `image` and a `seed`. ```python result = GenerationResult( progress=3, total=5, image=np.zeros((512, 512, 3)), seed=42 ) ``` Alternatively, create a result with just a `title` and progress values. ```python result = GenerationResult( progress=3, total=5, title="Loading model" ) ``` """ progress: int """The amount out of `total` that has been completed""" total: int """The number of steps to complete""" seed: int """The seed used to generate the image.""" title: str | None = None """The name of the currently executing task""" image: NDArray | None = None """The generated image as a Numpy array. The shape should be `(height, width, channels)`, where `channels` is 3 or 4. """ @staticmethod def tile_images(results: list['GenerationResult']) -> NDArray: images = [result.image for result in results] if len(images) == 0: return None elif len(images) == 1: return images[0] width = images[0].shape[1] height = images[0].shape[0] tiles_x = math.ceil(math.sqrt(len(images))) tiles_y = math.ceil(len(images) / tiles_x) tiles = np.zeros((height * tiles_y, width * tiles_x, images[0].shape[2]), dtype=images[0].dtype) bottom_offset = (tiles_x*tiles_y-len(images)) * width // 2 bottom = (tiles_y - 1) * height for i, image in enumerate(images): x = i % tiles_x y = int((i - x) / tiles_x) x *= width y *= height if y == bottom: x += bottom_offset tiles[y: y + height, x: x + width] = image return tiles ================================================ FILE: api/models/model.py ================================================ from dataclasses import dataclass @dataclass class Model: name: str description: str id: str ================================================ FILE: api/models/prompt.py ================================================ from dataclasses import dataclass from typing import List @dataclass class Prompt: positive: str | List[str] negative: str | List[str] | None ================================================ FILE: api/models/seamless_axes.py ================================================ from enum import Enum class SeamlessAxes(Enum): """Unified handling of seamless axes. Can be converted from str (id or text) or bool tuple/list (x, y). Each enum is equal to their respective convertible values. Special cases: AUTO: None OFF: False, empty str BOTH: True """ AUTO = 'auto', 'Auto-detect', None, None OFF = 'off', 'Off', False, False HORIZONTAL = 'x', 'X', True, False VERTICAL = 'y', 'Y', False, True BOTH = 'xy', 'Both', True, True def __init__(self, id, text, x, y): self.id = id self.text = text self.x = x self.y = y def __eq__(self, other): if isinstance(other, type(self)): return self is other if isinstance(other, str): return self.id == other or self.text == other or (other == '' and self is self.OFF) if isinstance(other, (tuple, list)) and len(other) == 2: return self.x == other[0] and self.y == other[1] if other is True and self is self.BOTH: return True if other is False and self is self.OFF: return True if other is None and self is self.AUTO: return True return False def __and__(self, other): return SeamlessAxes((self.x and other.x, self.y and other.y)) def __or__(self, other): return SeamlessAxes((self.x or other.x, self.y or other.y)) def __xor__(self, other): return SeamlessAxes((self.x != other.x, self.y != other.y)) def __invert__(self): return SeamlessAxes((not self.x, not self.y)) @classmethod def _missing_(cls, value): if isinstance(value, str): if value == '': return cls.OFF for e in cls: if e.id == value or e.text == value: return e raise ValueError(f'no {cls.__name__} with id {repr(id)}') elif isinstance(value, (tuple, list)) and len(value) == 2: for e in cls: if e.x == value[0] and e.y == value[1]: return e raise ValueError(f'no {cls.__name__} with x {value[0]} and y {value[1]}') elif value is True: return cls.BOTH elif value is False: return cls.OFF elif value is None: return cls.AUTO raise TypeError(f'expected str, bool, tuple[bool, bool], or None, got {repr(value)}') def bpy_enum(self, *args): return self.id, self.text, *args ================================================ FILE: api/models/step_preview_mode.py ================================================ import enum class StepPreviewMode(enum.Enum): NONE = "None" FAST = "Fast" FAST_BATCH = "Fast (Batch Tiled)" ACCURATE = "Accurate" ACCURATE_BATCH = "Accurate (Batch Tiled)" ================================================ FILE: api/models/task.py ================================================ from dataclasses import dataclass from typing import Tuple from numpy.typing import NDArray from enum import IntEnum class Task: """A specific task type. Access the properties of the task using dot notation. ```python # Task.ImageToImage task.image task.strength task.fit ``` Switch over the task to perform the correct actions. ```python match type(task): case PromptToImage: ... case ImageToImage: ... case Inpaint: ... case DepthToImage: ... case Outpaint: ... ``` """ @classmethod def name(cls) -> str: "unknown" """A human readable name for this task.""" @dataclass class PromptToImage(Task): @classmethod def name(cls): return "prompt to image" @dataclass class ImageToImage(Task): image: NDArray strength: float fit: bool @classmethod def name(cls): return "image to image" @dataclass class Inpaint(ImageToImage): class MaskSource(IntEnum): ALPHA = 0 PROMPT = 1 mask_source: MaskSource mask_prompt: str confidence: float @classmethod def name(cls): return "inpainting" @dataclass class DepthToImage(Task): depth: NDArray | None image: NDArray | None strength: float @classmethod def name(cls): return "depth to image" @dataclass class Outpaint(Task): image: NDArray origin: Tuple[int, int] @classmethod def name(cls): return "outpainting" @dataclass class Upscale(Task): image: NDArray tile_size: int blend: int @classmethod def name(cls): return "upscaling" ================================================ FILE: builtin_presets/Debug.py ================================================ import bpy prompt = bpy.context.scene.dream_textures_prompt prompt.steps = 20 prompt.cfg_scale = 7.5 prompt.scheduler = 'DPM Solver Multistep' prompt.step_preview_mode = 'Accurate' prompt.optimizations_attention_slicing = True prompt.optimizations_attention_slice_size_src = 'auto' prompt.optimizations_attention_slice_size = 1 prompt.optimizations_cudnn_benchmark = False prompt.optimizations_tf32 = False prompt.optimizations_amp = False prompt.optimizations_half_precision = True prompt.optimizations_sequential_cpu_offload = False prompt.optimizations_channels_last_memory_format = False prompt.optimizations_batch_size = 1 prompt.optimizations_vae_slicing = True prompt.optimizations_cpu_only = False ================================================ FILE: builtin_presets/Final.py ================================================ import bpy prompt = bpy.context.scene.dream_textures_prompt prompt.steps = 50 prompt.cfg_scale = 7.5 prompt.scheduler = 'DPM Solver Multistep' prompt.step_preview_mode = 'Fast' prompt.optimizations_attention_slicing = True prompt.optimizations_attention_slice_size_src = 'auto' prompt.optimizations_attention_slice_size = 1 prompt.optimizations_cudnn_benchmark = False prompt.optimizations_tf32 = False prompt.optimizations_amp = False prompt.optimizations_half_precision = True prompt.optimizations_sequential_cpu_offload = False prompt.optimizations_channels_last_memory_format = False prompt.optimizations_batch_size = 1 prompt.optimizations_vae_slicing = True prompt.optimizations_cpu_only = False ================================================ FILE: builtin_presets/Preview.py ================================================ import bpy prompt = bpy.context.scene.dream_textures_prompt prompt.steps = 20 prompt.cfg_scale = 7.5 prompt.scheduler = 'DPM Solver Multistep' prompt.step_preview_mode = 'Fast' prompt.optimizations_attention_slicing = True prompt.optimizations_attention_slice_size_src = 'auto' prompt.optimizations_attention_slice_size = 1 prompt.optimizations_cudnn_benchmark = False prompt.optimizations_tf32 = False prompt.optimizations_amp = False prompt.optimizations_half_precision = True prompt.optimizations_sequential_cpu_offload = False prompt.optimizations_channels_last_memory_format = False prompt.optimizations_batch_size = 1 prompt.optimizations_vae_slicing = True prompt.optimizations_cpu_only = False ================================================ FILE: classes.py ================================================ from .operators.install_dependencies import InstallDependencies, UninstallDependencies from .operators.open_latest_version import OpenLatestVersion from .operators.dream_texture import DreamTexture, ReleaseGenerator, CancelGenerator from .operators.view_history import SCENE_UL_HistoryList, RecallHistoryEntry, ClearHistory, RemoveHistorySelection, ExportHistorySelection, ImportPromptFile from .operators.inpaint_area_brush import InpaintAreaBrushActivated from .operators.upscale import Upscale from .operators.project import ProjectDreamTexture, dream_texture_projection_panels from .operators.notify_result import NotifyResult from .property_groups.control_net import ControlNet, ControlNetsAdd, ControlNetsRemove, ControlNetsAddMenu, BakeControlNetImage from .property_groups.dream_prompt import DreamPrompt from .property_groups.seamless_result import SeamlessResult from .ui.panels import dream_texture, history, upscaling, render_properties from .preferences import OpenURL, StableDiffusionPreferences,\ ImportWeights, Model, ModelSearch, InstallModel, PREFERENCES_UL_ModelList,\ CheckpointGroup, LinkCheckpoint, UnlinkCheckpoint, PREFERENCES_UL_CheckpointList from .ui.presets import DREAM_PT_AdvancedPresets, DREAM_MT_AdvancedPresets, AddAdvancedPreset, RestoreDefaultPresets from . import engine CLASSES = ( *render_properties.render_properties_panels(), DreamTexture, ReleaseGenerator, CancelGenerator, OpenLatestVersion, SCENE_UL_HistoryList, RecallHistoryEntry, ClearHistory, RemoveHistorySelection, ExportHistorySelection, ImportPromptFile, InpaintAreaBrushActivated, Upscale, ProjectDreamTexture, ControlNetsAddMenu, ControlNetsAdd, ControlNetsRemove, BakeControlNetImage, DREAM_PT_AdvancedPresets, DREAM_MT_AdvancedPresets, AddAdvancedPreset, NotifyResult, engine.DreamTexturesRenderEngineProperties, engine.DreamTexturesRenderEngine, engine.NewEngineNodeTree, *engine.engine_panels(), # The order these are registered in matters *dream_texture.dream_texture_panels(), *upscaling.upscaling_panels(), *history.history_panels(), *dream_texture_projection_panels(), ) PREFERENCE_CLASSES = ( PREFERENCES_UL_ModelList, ModelSearch, InstallModel, Model, ControlNet, DreamPrompt, SeamlessResult, UninstallDependencies, InstallDependencies, OpenURL, ImportWeights, RestoreDefaultPresets, CheckpointGroup, LinkCheckpoint, UnlinkCheckpoint, PREFERENCES_UL_CheckpointList, StableDiffusionPreferences, ) ================================================ FILE: community_backends/test.py ================================================ bl_info = { "name": "Test Backend", "blender": (3, 1, 0), "category": "Paint", } import bpy from typing import List, Tuple from dream_textures.api import * class TestBackend(Backend): name = "Test" description = "A short description of this backend" custom_optimization: bpy.props.BoolProperty(name="My Custom Optimization") def list_models(self, context) -> List[Model]: return [] def list_schedulers(self, context) -> List[str]: return [] def generate(self, task: Task, model: Model, prompt: Prompt, size: Tuple[int, int] | None, seed: int, steps: int, guidance_scale: float, scheduler: str, seamless_axes: SeamlessAxes, step_preview_mode: StepPreviewMode, iterations: int, step_callback: StepCallback, callback: Callback): raise NotImplementedError() def draw_speed_optimizations(self, layout, context): layout.prop(self, "custom_optimization") def register(): bpy.utils.register_class(TestBackend) def unregister(): bpy.utils.unregister_class(TestBackend) ================================================ FILE: diffusers_backend.py ================================================ import bpy from bpy.props import FloatProperty, IntProperty, EnumProperty, BoolProperty from typing import List from .api import Backend, StepCallback, Callback from .api.models import Model, GenerationArguments, GenerationResult from .api.models.task import PromptToImage, ImageToImage, Inpaint, DepthToImage, Outpaint, Upscale from .api.models.fix_it_error import FixItError from .generator_process import Generator from .generator_process.future import Future from .generator_process.models import CPUOffload, ModelType, Optimizations, Scheduler from .preferences import checkpoint_lookup, StableDiffusionPreferences, _template_model_download_progress, InstallModel, model_lookup from functools import reduce def _convert_models(models): return [ None if model is None else (model.id, model.name, model.description) for model in models ] class DiffusersBackend(Backend): name = "HuggingFace Diffusers" description = "Local image generation inside of Blender" attention_slicing: BoolProperty(name="Attention Slicing", default=True, description="Computes attention in several steps. Saves some memory in exchange for a small speed decrease") attention_slice_size_src: EnumProperty( name="Attention Slice Size", items=( ("auto", "Automatic", "Computes attention in two steps", 1), ("manual", "Manual", "Computes attention in `attention_head_dim // size` steps. A smaller `size` saves more memory.\n" "`attention_head_dim` must be a multiple of `size`, otherwise the image won't generate properly.\n" "`attention_head_dim` can be found within the model snapshot's unet/config.json file", 2) ), default=1 ) attention_slice_size: IntProperty(name="Attention Slice Size", default=1, min=1) cudnn_benchmark: BoolProperty(name="cuDNN Benchmark", description="Allows cuDNN to benchmark multiple convolution algorithms and select the fastest", default=False) tf32: BoolProperty(name="TF32", description="Utilizes tensor cores on Ampere (RTX 30xx) or newer GPUs for matrix multiplications.\nHas no effect if half precision is enabled", default=False) half_precision: BoolProperty(name="Half Precision", description="Reduces memory usage and increases speed in exchange for a slight loss in image quality.\nHas no effect if CPU only is enabled or using a GTX 16xx GPU", default=True) cpu_offload: EnumProperty( name="CPU Offload", items=( ("off", "Off", "", 0), ("model", "Model", "Some memory savings with minimal speed penalty", 1), ("submodule", "Submodule", "Better memory savings with large speed penalty", 2) ), default=0, description="Dynamically moves models in and out of device memory for reduced memory usage with reduced speed" ) channels_last_memory_format: BoolProperty(name="Channels Last Memory Format", description="An alternative way of ordering NCHW tensors that may be faster or slower depending on the device", default=False) sdp_attention: BoolProperty( name="SDP Attention", description="Scaled dot product attention requires less memory and often comes with a good speed increase.\n" "Prompt recall may not produce the exact same image, but usually only minor noise differences.\n" "Overrides attention slicing", default=True ) batch_size: IntProperty(name="Batch Size", default=1, min=1, description="Improves speed when using iterations or upscaling in exchange for higher memory usage.\nHighly recommended to use with VAE slicing enabled") vae_slicing: BoolProperty(name="VAE Slicing", description="Reduces memory usage of batched VAE decoding. Has no effect if batch size is 1.\nMay have a small performance improvement with large batches", default=True) vae_tiling: EnumProperty( name="VAE Tiling", items=( ("off", "Off", "", 0), ("half", "Half", "Uses tiles of half the selected model's default size. Likely to cause noticeably inaccurate colors", 1), ("full", "Full", "Uses tiles of the selected model's default size, intended for use where image size is manually set higher. May cause slightly inaccurate colors", 2), ("manual", "Manual", "", 3) ), default=0, description="Decodes generated images in tiled regions to reduce memory usage in exchange for longer decode time and less accurate colors.\nCan allow for generating larger images that would otherwise run out of memory on the final step" ) vae_tile_size: IntProperty(name="VAE Tile Size", min=1, default=512, description="Width and height measurement of tiles. Smaller sizes are more likely to cause inaccurate colors and other undesired artifacts") vae_tile_blend: IntProperty(name="VAE Tile Blend", min=0, default=64, description="Minimum amount of how much each edge of a tile will intersect its adjacent tile") cfg_end: FloatProperty(name="CFG End", min=0, max=1, default=1, description="The percentage of steps to complete before disabling classifier-free guidance") cpu_only: BoolProperty(name="CPU Only", default=False, description="Disables GPU acceleration and is extremely slow") use_sdxl_refiner: BoolProperty(name="Use SDXL Refiner", default=False, description="Provide a refiner model to run automatically after the initial generation") sdxl_refiner_model: EnumProperty(name="SDXL Refiner Model", items=lambda self, context: _convert_models(self.list_models(context)), description="Specify which model to use as a refiner") def list_models(self, context): def model_case(model, i): return Model( name=model.model_base.replace('models--', '').replace('--', '/'), description=ModelType[model.model_type].name, id=model.model_base.replace('models--', '').replace('--', '/') ) models = {} for i, model in enumerate(context.preferences.addons[StableDiffusionPreferences.bl_idname].preferences.installed_models): if model.model_type in {ModelType.CONTROL_NET.name, ModelType.UNKNOWN.name}: continue if model.model_type not in models: models[model.model_type] = [model_case(model, i)] else: models[model.model_type].append(model_case(model, i)) return reduce( lambda a, b: a + [None] + sorted(b, key=lambda m: m.id), [ models[group] for group in sorted(models.keys()) ], [] ) def list_controlnet_models(self, context): return [ Model( name=model.model_base.replace('models--', '').replace('--', '/'), description="ControlNet", id=model.model_base.replace('models--', '').replace('--', '/') ) for model in context.preferences.addons[StableDiffusionPreferences.bl_idname].preferences.installed_models if model.model_type == ModelType.CONTROL_NET.name ] def list_schedulers(self, context) -> List[str]: return [scheduler.value for scheduler in Scheduler] def get_batch_size(self, context) -> int: return self.batch_size def optimizations(self) -> Optimizations: optimizations = Optimizations() for prop in dir(self): if hasattr(optimizations, prop) and not prop.startswith('__'): setattr(optimizations, prop, getattr(self, prop)) if self.attention_slice_size_src == 'auto': optimizations.attention_slice_size = 'auto' optimizations.cpu_offload = CPUOffload(optimizations.cpu_offload) return optimizations def generate(self, arguments: GenerationArguments, step_callback: StepCallback, callback: Callback): gen = Generator.shared() common_kwargs = { 'model': checkpoint_lookup.get(arguments.model.id), 'scheduler': Scheduler(arguments.scheduler), 'optimizations': self.optimizations(), 'prompt': arguments.prompt.positive, 'steps': arguments.steps, 'width': arguments.size[0] if arguments.size is not None else None, 'height': arguments.size[1] if arguments.size is not None else None, 'seed': arguments.seed, 'cfg_scale': arguments.guidance_scale, 'use_negative_prompt': arguments.prompt.negative is not None, 'negative_prompt': arguments.prompt.negative or "", 'seamless_axes': arguments.seamless_axes, 'iterations': arguments.iterations, 'step_preview_mode': arguments.step_preview_mode, 'sdxl_refiner_model': (checkpoint_lookup.get(self.sdxl_refiner_model) if self.use_sdxl_refiner else None), } future: Future match arguments.task: case PromptToImage(): if len(arguments.control_nets) > 0: future = gen.control_net( **common_kwargs, control_net=[checkpoint_lookup.get(c.model) for c in arguments.control_nets], control=[c.image for c in arguments.control_nets], controlnet_conditioning_scale=[c.strength for c in arguments.control_nets], image=None, inpaint=False, inpaint_mask_src='alpha', text_mask='', text_mask_confidence=1, strength=1 ) else: future = gen.prompt_to_image(**common_kwargs) case Inpaint(image=image, fit=fit, strength=strength, mask_source=mask_source, mask_prompt=mask_prompt, confidence=confidence): if len(arguments.control_nets) > 0: future = gen.control_net( **common_kwargs, control_net=[c.model for c in arguments.control_nets], control=[c.image for c in arguments.control_nets], controlnet_conditioning_scale=[c.strength for c in arguments.control_nets], image=image, inpaint=True, inpaint_mask_src='alpha' if mask_source == Inpaint.MaskSource.ALPHA else 'prompt', text_mask=mask_prompt, text_mask_confidence=confidence, strength=strength ) else: future = gen.inpaint( image=image, fit=fit, strength=strength, inpaint_mask_src='alpha' if mask_source == Inpaint.MaskSource.ALPHA else 'prompt', text_mask=mask_prompt, text_mask_confidence=confidence, **common_kwargs ) case ImageToImage(image=image, strength=strength, fit=fit): if len(arguments.control_nets) > 0: future = gen.control_net( **common_kwargs, control_net=[c.model for c in arguments.control_nets], control=[c.image for c in arguments.control_nets], controlnet_conditioning_scale=[c.strength for c in arguments.control_nets], image=image, inpaint=False, inpaint_mask_src='alpha', text_mask='', text_mask_confidence=1, strength=strength ) else: future = gen.image_to_image(image=image, fit=fit, strength=strength, **common_kwargs) case DepthToImage(depth=depth, image=image, strength=strength): future = gen.depth_to_image( depth=depth, image=image, strength=strength, **common_kwargs ) case Outpaint(image=image, origin=origin): future = gen.outpaint( image=image, outpaint_origin=origin, fit=False, strength=1, inpaint_mask_src='alpha', text_mask='', text_mask_confidence=1, **common_kwargs ) case Upscale(image=image, tile_size=tile_size, blend=blend): future = gen.upscale( image=image, tile_size=tile_size, blend=blend, **common_kwargs ) case _: raise NotImplementedError() def on_step(future: Future, step_image: [GenerationResult]): should_continue = step_callback(step_image) if not should_continue: future.cancel() callback(InterruptedError()) def on_done(future: Future): callback(future.result(last_only=True)) def on_exception(_, exception): callback(exception) future.add_response_callback(on_step) future.add_exception_callback(on_exception) future.add_done_callback(on_done) def validate(self, arguments: GenerationArguments): model = None if arguments.model is None else model_lookup.get(arguments.model.id) if model is None: raise FixItError("No model selected.", FixItError.ChangeProperty("model")) else: if not model.model_type.matches_task(arguments.task): class DownloadModel(FixItError.Solution): def _draw(self, dream_prompt, context, layout): if not _template_model_download_progress(context, layout): target_model_type = ModelType.from_task(arguments.task) if target_model_type is not None: install_model = layout.operator(InstallModel.bl_idname, text=f"Download {target_model_type.recommended_model()} (Recommended)", icon="IMPORT") install_model.model = target_model_type.recommended_model() install_model.prefer_fp16_revision = context.preferences.addons[StableDiffusionPreferences.bl_idname].preferences.prefer_fp16_revision model_task_description = f"""Incorrect model type selected for {type(arguments.task).name().replace('_', ' ').lower()} tasks. The selected model is for {model.model_type.name.replace('_', ' ').lower()}.""" if not any(m.model_type.matches_task(arguments.task) for m in model_lookup._models.values()): raise FixItError( message=model_task_description + "\nYou do not have any compatible models downloaded:", solution=DownloadModel() ) else: raise FixItError( message=model_task_description + "\nSelect a different model below.", solution=FixItError.ChangeProperty("model") ) def draw_advanced(self, layout, context): layout.prop(self, "use_sdxl_refiner") col = layout.column() col.enabled = self.use_sdxl_refiner col.prop(self, "sdxl_refiner_model") def draw_speed_optimizations(self, layout, context): inferred_device = Optimizations.infer_device() if self.cpu_only: inferred_device = "cpu" def optimization(prop): if Optimizations.device_supports(prop, inferred_device): layout.prop(self, prop) optimization("cudnn_benchmark") optimization("tf32") optimization("half_precision") optimization("channels_last_memory_format") optimization("batch_size") def draw_memory_optimizations(self, layout, context): inferred_device = Optimizations.infer_device() if self.cpu_only: inferred_device = "cpu" def optimization(prop): if Optimizations.device_supports(prop, inferred_device): layout.prop(self, prop) optimization("sdp_attention") optimization("attention_slicing") slice_size_row = layout.row() slice_size_row.prop(self, "attention_slice_size_src") if self.attention_slice_size_src == 'manual': slice_size_row.prop(self, "attention_slice_size", text="Size") optimization("cpu_offload") optimization("cpu_only") optimization("vae_slicing") optimization("vae_tiling") if self.vae_tiling == "manual": optimization("vae_tile_size") optimization("vae_tile_blend") ================================================ FILE: docs/AI_UPSCALING.md ================================================ # AI Upscaling Use the Stable Diffusion upscaler to increase images 4x in size while retaining detail. You can guide the upscaler with a text prompt. > Upscaling uses the model `stabilityai/stable-diffusion-4x-upscaler`. This model will automatically be downloaded when the operator is first run. Use the AI Upscaling panel to access this tool. 1. Open the image to upscale in an *Image Editor* space 2. Expand the *AI Upscaling* panel, located in the *Dream* sidebar tab 3. Type a prompt to subtly influence the generation. 4. Optionally configure the tile size, blend, and other advanced options. ![](assets/ai_upscaling/panel.png) The upscaled image will be opened in the *Image Editor*. The image will be named `Source Image Name (Upscaled)`. ## Tile Size Due to the large VRAM consumption of the `stabilityai/stable-diffusion-4x-upscaler` model, the input image is split into tiles with each tile being upscaled independently, then stitched back together. The default tile size is 128x128, which will result in an image of size 512x512. These 512x512 images are stitched back together to form the final image. You can increase or decrease the tile size depending on your GPU's capabilities. The *Blend* parameter controls how much overlap is included in the tiles to help reduce visible seams. ================================================ FILE: docs/DEVELOPMENT_ENVIRONMENT.md ================================================ # Setting Up a Development Environment With the following steps, you can start contributing to Dream Textures. These steps can also be used to setup the add-on on Linux. ## Cloning A basic knowledge of Git will be necessary to contribute. To start, clone the repository: ```sh git clone https://github.com/carson-katri/dream-textures.git dream_textures ``` > If you use SSH, clone with `git clone git@github.com:carson-katri/dream-textures.git dream_textures` This will clone the repository into the `dream_textures` folder. ## Installing to Blender You can install the add-on to Blender in multiple ways. The easiest way is to copy the folder into the add-ons directory. This directory is in different places on different systems. * Windows * `%USERPROFILE%\AppData\Roaming\Blender Foundation\Blender\3.4\scripts\addons` * macOS * `/Users/$USER/Library/Application Support/Blender/3.4/scripts/addons` * Linux * `$HOME/.config/blender/3.4/scripts/addons` > This path may be different depending on how you installed Blender. See [Blender's documentation](https://docs.blender.org/manual/en/latest/advanced/blender_directory_layout.html) for more information on the directory layout. If you can't find the add-on folder, you can look at another third-party add-on you already have in Blender preferences and see where it is located. ![A screenshot highlighting the add-on directory in Blender preferences](assets/development_environment/locating_addons.png) ### Using Visual Studio Code > This is not necessary if you won't be making any changes to Dream Textures or prefer a different IDE. You can also install and debug the add-on with the [Blender Development]() extension for Visual Studio Code. Open the `dream_textures` folder in VS Code, open the command palette (Windows: <kbd>Shift</kbd> + <kbd>Ctrl</kbd> + <kbd>P</kbd>, macOS: <kbd>Shift</kbd> + <kbd>Command</kbd> + <kbd>P</kbd>), and search for the command `Blender: Start`. ![](assets/development_environment/command_palette.png) Then choose which Blender installation to use. ![](assets/development_environment/choose_installation.png) Blender will now start up with the add-on installed. You can verify this by going to Blender's preferences and searching for *Dream Textures*. ## Installing Dependencies When installing from source, the dependencies are not included. You can install them from Blender's preferences. First, enable *Developer Extras* so Dream Textures' developer tools will be displayed. ![](assets/development_environment/developer_extras.png) Then, use the *Developer Tools* section to install the dependencies. ![](assets/development_environment/install_dependencies.png) ### Installing Dependencies Manually In some cases, the *Install Dependencies* tool may not work. In this case, you can install the dependencies from the command line. The best way to install dependencies is using the Python that ships with Blender. The command will differ depending on your operating system and Blender installation. On some platforms, Blender does not come with `pip` pre-installed. You can use `ensurepip` to install it if necessary. ```sh # Windows "C:\Program Files\Blender Foundation\Blender 3.4\3.4\python\bin\python.exe" -m ensurepip # macOS /Applications/Blender.app/Contents/Resources/3.4/python/bin/python3.10 -m ensurepip # Linux (via snap) /snap/blender/3132/3.4/python/bin/python3.10 -m ensurepip ``` Once you have `pip`, the dependencies can be installed. All of the packages *must* be installed to `dream_textures/.python_dependencies`. The following commands assume they are being run from inside the `dream_textures` folder. ```sh # Windows "C:\Program Files\Blender Foundation\Blender 3.4\3.4\python\bin\python.exe" -m pip install -r requirements/win-linux-cuda.txt --target .python_dependencies # macOS /Applications/Blender.app/Contents/Resources/3.4/python/bin/python3.10 -m pip install -r requirements/mac-mps-cpu.txt --target .python_dependencies # Linux (via snap) /snap/blender/3132/3.4/python/bin/python3.10 -m pip install -r requirements/win-linux-cuda.txt --target .python_dependencies ``` ## Using the Add-on Once you have the dependencies installed, the add-on will become fully usable. Continue setting up as described in the [setup guide](./SETUP.md). ## Common Issues ### macOS 1. On Apple Silicon, with the `requirements-dream-studio.txt` you may run into an error with gRPC using an incompatible binary. If so, please use the following command to install the correct gRPC version: ```sh pip install --no-binary :all: grpcio --ignore-installed --target .python_dependencies --upgrade ``` ================================================ FILE: docs/HISTORY.md ================================================ # History Each time you generate, the full configuration is saved to the *History* panel. You can recall any previous generation to re-run it by clicking *Recall Prompt*. ## Prompt Import/Export You can also export the selected prompt to JSON for later import. This is a more permanent way to backup prompts, and can be useful for sharing an exact image. ### Export 1. Select a history entry row 2. Click the export icon button 3. Save the JSON file to your computer ![A screenshot of the History panel with the Export icon button highlighted](assets/history/history-export.png) ### Import 1. Select the import icon button in the header of the *Dream Texture* panel 2. Open a valid prompt JSON file 3. Every configuration option will be loaded in ![A screenshot of the Dream Texture panel with the Import icon button highlighted](assets/history/history-import.png) ================================================ FILE: docs/IMAGE_GENERATION.md ================================================ # Image Generation 1. To open Dream Textures, go to an Image Editor or Shader Editor 1. Ensure the sidebar is visible by pressing *N* or checking *View* > *Sidebar* 2. Select the *Dream* panel to open the interface ![A screenshot showing the 'Dream' panel in an Image Editor space](assets/image_generation/opening-ui.png) Enter a prompt then click *Generate*. It can take anywhere from a few seconds to a few minutes to generate, depending on your GPU. ## Options ### Pipeline Two options are currently available: * Stable Diffusion - for local generation * DreamStudio - for cloud processing Only the options available for the version you installed and the keys provided in the add-on preferences will be available. ### Model Choose from any installed model. Some options require specific kinds of model. For example, [] ### Prompt A few presets are available to help you create great prompts. They work by asking you to fill in a few simple fields, then generate a full prompt string that is passed to Stable Diffusion. The default preset is *Texture*. It asks for a subject, and adds the word `texture` to the end. So if you enter `brick wall`, it will use the prompt `brick wall texture`. ### Seamless Checking seamless will use a circular convolution to create a perfectly seamless image, which works great for textures. You can also specify which axes should be seamless. ### Negative Enabling negative prompts gives you finer control over your image. For example, if you asked for a `cloud city`, but you wanted to remove the buildings it added, you could enter the negative prompt `building`. This would tell Stable Diffusion to avoid drawing buildings. You can add as much content you want to the negative prompt, and it will avoid everything entered. ### Size The target image dimensions. The width/height should be a multiple of 64, or it will round to the closest one for you. Most graphics cards with 4+GB of VRAM should be able to generate 512x512 images. However, if you are getting CUDA memory errors, try decreasing the size. > Stable Diffusion was trained on 512x512 images, so you will get the best results at this size (or at least when leaving one dimensions at 512). ### Source Image Choose an image from a specific *File* or use the *Open Image*. Three actions are available that work on a source image. #### Modify Mixes the image with the noise with the ratio specified by the *Noise Strength*. This will make Stable Diffusion match the style, composition, etc. from it. Stength specifies how much latent noise to mix with the image. A higher strength means more latent noise, and more deviation from the init image. If you want it to stick to the image more, decrease the strength. > Depending on the strength value, some steps will be skipped. For example, if you specified `10` steps and set strength to `0.5`, only `5` steps would be used. Fit to width/height will ensure the image is contained within the configured size. The *Image Type* option has a few options: 1. Color - Mixes the image with noise > The following options require a depth model to be selected, such as `stabilityai/stable-diffusion-2-depth`. Follow the instructions to [download a model](setup.md#download-a-model). 2. Color and Generated Depth - Uses MiDaS to infer the depth of the initial image and includes it in the conditioning. Can give results that more closely match the composition of the source image. 3. Color and Depth Map - Specify a secondary image to use as the depth map, instead of generating one with MiDaS. 4. Depth - Treats the intial image as a depth map, and ignores any color. The generated image will match the composition but not colors of the original. ### Advanced You can have more control over the generation by trying different values for these parameters: * Random Seed - When enabled, a seed will be selected for you * Seed - The value used to seed RNG, if text is input instead of a number its hash will be used * Steps - Number of sampler steps, higher steps will give the sampler more time to converge and clear up artifacts * CFG Scale - How strongly the prompt influences the output * Scheduler - Some schedulers take fewer steps to produce a good result than others. Try each one and see what you prefer. * Step Preview - Whether to show each step in the image editor. Defaults to 'Fast', which samples the latents without using the VAE. 'Accurrate' will run the latents through the VAE at each step and slow down generation significantly. * Speed Optimizations - Various optimizations to increase generation speed, some at the cost of VRAM. Recommended default is *Half Precision*. * Memory Optimizations - Various optimizations to reduce VRAM consumption, some at the cost of speed. Recommended default is *Attention Slicing* with *Automatic* slice size. ### Iterations How many images to generate. This is only particularly useful when *Random Seed* is enabled. ================================================ FILE: docs/INPAINT_OUTPAINT.md ================================================ # Inpaint/Outpaint This guide shows how to use both [inpainting](#inpainting) and [outpainting](#outpainting). > For both inpainting and outpainting you *must* use a model fine-tuned for inpainting, such as `stabilityai/stable-diffusion-2-inpainting`. Follow the instructions to [download a model](setup.md#download-a-model). # Inpainting Inpainting refers to filling in or replacing parts of an image. It can also be used to [make existing textures seamless](#making-textures-seamless). The quickest way to inpaint is with the *Mark Inpaint Area* brush. 1. Use the *Mark Inpaint Area* brush to remove the edges of the image 2. Enter a prompt for what should fill the erased area 3. Enable *Source Image*, select the *Open Image* source and the *Inpaint* action 4. Choose the *Alpha Channel* mask source 5. Click *Generate* ![](assets/inpaint_outpaint/inpaint.png) ## Making Textures Seamless Inpainting can also be used to make an existing texture seamless. 1. Use the *Mark Inpaint Area* brush to remove the edges of the image 2. Enter a prompt that describes the texture, and check *Seamless* 3. Enable *Source Image*, select the *Open Image* source and the *Inpaint* action 4. Click *Generate* ![](assets/inpaint_outpaint/seamless_inpaint.png) # Outpainting Outpainting refers to extending an image beyond its original size. Use an inpainting model such as `stabilityai/stable-diffusion-2-inpainting` for outpainting as well. 1. Select an image to outpaint and open it in an Image Editor 2. Choose a size, this is how large the outpaint will be 3. Enable *Source Image*, select the *Open Image* source and the *Outpaint* action 4. Set the origin of the outpaint. See [Choosing an Origin](#choosing-an-origin) for more info. ### Choosing an Origin The top left corner of the image is (0, 0), with the bottom right corner being the (width, height). You should always include overlap or the outpaint will be completely unrelated to the original. The add-on will warn you if you do not include any. Take the image below for example. We want to outpaint the bottom right side. Let's figure out the correct origin. Here's what we know: 1. We know our image is 512x960. You can find this in the sidebar on the *Image* tab. 2. We set the size of the outpaint to 512x512 in the *Dream* tab With this information we can calculate: 1. The X origin will be the width of the image minus some overlap. The width is 512px, and we want 64px of overlap. So the X origin will be set to `512 - 64` or `448`. 2. The Y origin will be the height of the image minus the height of the outpaint size. The height of the image is 960px, and the height of the outpaint is 512px. So the Y origin will be set to `960 - 512` or `448`. > Tip: You can enter math expressions into any Blender input field. ![](assets/inpaint_outpaint/outpaint_origin.png) After selecting this origin, we can outpaint the bottom right side. ![](assets/inpaint_outpaint/outpaint.gif) Here are other values we could have used for other parts of the image: * Bottom Left: `(-512 + 64, 512 - 64)` or `(-448, 448)` * Top Right: `(512 - 64, 0)` or `(448, 0)` * Top Left: `(-512 + 64, 0)` or `(-448, 0)` * Top: `(0, 0)` * Bottom: `(0, 512 - 64)` or `(0, 448)` ================================================ FILE: docs/RENDER_PASS.md ================================================ # Render Pass A custom 'Dream Textures' render pass is available for Cycles. This allows you to run Dream Textures on the render result each time the scene is rendered. It works with animations as well, and can be used to perform style transfer on each frame of an animation. > The render pass and Cycles will both use significant amounts of VRAM depending on the scene. You can use the CPU to render on Cycles to save resources for Dream Textures. 1. In the *Render Properties* panel, switch to the *Cycles* render engine > In the *Output Properties* panel, ensure the image size is reasonable for your GPU and Stable Diffusion. 512x512 is a good place to start. ![A screenshot of the Render Properties panel with the Cycles render engine selected, and the Dream Textures render pass checked](assets/render_pass/cycles.png) 2. Enable the *Dream Textures* render pass, and enter a text prompt. You can also specify which pass inputs to use. Using depth information can help the rendered result match the scene geometry. > When using depth, you *must* select a depth model, such as `stabilityai/stable-diffusion-2-depth`. Follow the instructions to [download a model](setup.md#download-a-model). ![](assets/render_pass/pass_inputs.png) 3. To use the Dream Textures generated image as the final result, open the *Compositor* space 4. Enable *Use Nodes* 5. Connect the *Dream Textures* socket from the *Render Layers* node to the *Image* socket of the *Composite* node ![A screenshot of the Compositor space with Use Nodes checked and the Dream Textures socket from the Render Layers node connected to the Image socket of the Composite node](assets/render_pass/render-pass-compositor.png) Now whenever you render your scene, it will automatically run through Stable Diffusion and output the result. Here's an example using only the *Depth* pass as input. ![](assets/render_pass/spinning_cube_animation.gif) ## Controlling the Output ### Depth Using a depth model and choosing *Pass Inputs* such as *Depth* or *Color and Depth* can go a long way in making the result closely match the geometry of your scene. ### Noise Strength The noise strength parameter is very important when using this render pass. It is the same as *Noise Strength* described in [Image Generation](IMAGE_GENERATION.md#modify). If you want your scene composition, colors, etc. to be preserved, use a lower strength value. If you want Stable Diffusion to take more control, use a higher strength value. This does not apply when using the *Depth* pass input and no color. ### Seed Enabling *Random Seed* can give you some cool effects, and allow for more experimentation. However, if you are trying to do simple style transfer on an animation, using a consistent seed can help the animation be more coherent. ## Animation You can animate most of the properties when using the render pass. Simply create keyframes as you typically would in Blender, and the properties will automatically be updated for each frame. ================================================ FILE: docs/SETUP.md ================================================ # Setting Up Getting up and running is easy. Make sure you have several GBs of storage free, as the model weights and add-on consume a lot of storage space. In general, all of the instructions you need to setup will be given within Blender. However, if you want to see screenshots and more explanation this can be helpful. If you have any problems, you can get help in the [Dream Textures Discord server](https://discord.gg/EmDJ8CaWZ7). ## Installation See the [release notes](https://github.com/carson-katri/dream-textures/releases/latest) for the most recent version of Dream Textures. There you will find a section titled "Choose Your Installation". Use the dropdowns to find the right version for your system. > The add-on archive for Windows is compressed with 7-Zip to fit within GitHub's file size limits. Follow the instructions on the release notes page to extract the contained `.zip` file. After you have the add-on installed in Blender, check the box in Blender preferences to enable it. Then follow the steps below to complete setup. If you downloaded a DreamStudio-only version, you can skip to the [DreamStudio section](#dreamstudio). > **DO NOT** try to install dependencies. Tools for doing so are intended for development. *Always* [download a prebuilt version](https://github.com/carson-katri/dream-textures/releases/latest). ## Download a Model There are [hundreds of models](https://huggingface.co/sd-dreambooth-library) to choose from. A good model to start with is `stabilityai/stable-diffusion-2-1-base`. This is the latest 512x512 Stable Diffusion model. In the add-on preferences, search for this model. Then click the download button on the right. > Depending on your Internet speeds, it may take a few minutes to download. ![](assets/setup/stable_diffusion_2_1_base.png) ### Other Useful Models There are a few other models you may want to download as well: * `stabilityai/stable-diffusion-2-inpainting` - Fine-tuned for inpainting, and required to use the [inpaint and outpaint](INPAINT_OUTPAINT.md) features * `stabilityai/stable-diffusion-2-depth` - Uses depth information to guide generation, required for [texture projection](TEXTURE_PROJECTION.md), the [render pass](RENDER_PASS.md) when using depth pass input, and image to image when using depth * `stabilityai/stable-diffusion-x4-upscaler` - Upscales the input image 4x, used only for [upscaling](AI_UPSCALING.md) * `stabilityai/stable-diffusion-2-1` - Fine-tuned for 768x768 images ### Private Models Some models are gated or private to your account/organization. To download these models, generate a Hugging Face Hub token and paste it in the "Token" field. You can generate a token in your [account settings on Hugging Face](https://huggingface.co/settings/tokens). ![](assets/setup/hfh_token.png) ### Importing Checkpoints Dream Textures can also import `.ckpt` files, which you may be familiar with if you've used [AUTOMATIC1111/stable-diffusion-webui](https://github.com/AUTOMATIC1111/stable-diffusion-webui). Click *Import Checkpoint File* in add-on preferences, then select your checkpoint file. You should also specify what kind of model it is in the sidebar of the file picker. ![](assets/setup/checkpoint_import.png) If you don't see the sidebar, press *N* on your keyboard or click the gear icon in the top right. Here's what the various *Model Config* options are for: * `v1` - Weights from the original [CompVis/stable-diffusion](https://github.com/CompVis/stable-diffusion) code, such as `stable-diffusion-v1-x` * `v2 (512, epsilon)` - Weights from the updated [Stability-AI/stablediffusion](https://github.com/Stability-AI/stablediffusion) code, specifically models with `prediction_type="epsilon"` such as the 512x512 models * `v2 (768, v_prediction)` - Weights from the updated [Stability-AI/stablediffusion](https://github.com/Stability-AI/stablediffusion) code, specifically models with `prediction_type="v_prediction"` such as the 768x768 models After choosing the file, the model will be converted to the Diffusers format automatically and be available in your model list. ## DreamStudio To connect your DreamStudio account, enter your API key in the add-on preferences. ![](assets/setup/dream_studio_key.png) You can find your key in the [account settings page of DreamStudio](https://beta.dreamstudio.ai/membership?tab=apiKeys). ![](assets/setup/dreamstudio.png) ================================================ FILE: docs/TEXTURE_PROJECTION.md ================================================ # Texture Projection Using depth to image, Dream Textures is able to texture entire scenes automatically with a simple prompt. It's sort of like [Ian Hubert's method](https://www.youtube.com/watch?v=v_ikG-u_6r0) in reverse. Instead of starting with an image and building geometry around that, we start with the geometry and generate an image that projects perfectly onto it. > Make sure you download a depth model such as `stabilityai/stable-diffusion-2-depth`. Follow the instructions to [download a model](SETUP.md#download-a-model). Follow the steps below to project a texture onto your mesh. ## Selecting a Target In the 3D Viewport, select the objects you want to project onto. Then in edit mode, select all of the faces to target. Only the selected faces will be given the new texture. Every object in the viewport will be factored into the depth map. To only use the selected objects in the depth map, enter local view by pressing */* or selecting *View* > *Local View* > *Toggle Local View* in the viewport menu bar. > Tip: Large unbroken faces do not always project well. Try subdividing your mesh if the projection is warping. ![](assets/texture_projection/edit_mode.png) ## Prompting In the sidebar, select the "Dream" panel. Choose a depth model from the dropdown, and enter a prompt. All of the options available for image generation are available here as well. At the bottom you can choose what data to use for the projection. The default is "Depth". You can switch to "Depth and Color" to factor in the current color from the viewport. > The color data is retrieved from the current viewport. Switch to *Viewport Shading* or *Rendered* view to use the colors specified for EEVEE or Cycles. You can also adjust the size of the image from the default 512x512. Note that the depth data is in the same aspect ratio as the 3D Viewport window. If you have the Blender window landscape, you may want to adjust the size to be 768x512 or something similar. You could also shrink your window to make it square. ## Project Dream Texture After configuring everything, click the *Project Dream Texture* button. This will begin the depth to image process. You are free to move the viewport around as it generates. Switch to *Viewport Shading* mode to see each step as it generates live. A new material will be added named with the seed of the generated image. The UVs for each selected face are also updated to be projected from the angle the material was generated at. > Tip: In the sidebar under *View* you can adjust the *Focal Length* of the viewport. ![](assets/texture_projection/projection.gif) ================================================ FILE: docs/assets/banner_image_prompt.json ================================================ { "prompt_structure": "concept_art", "use_negative_prompt": true, "negative_prompt": "person dragon bird ocean buildings woman", "width": 1024, "height": 256, "seamless": false, "show_advanced": false, "random_seed": false, "seed": "4141004939", "precision": "auto", "iterations": 1, "steps": 25, "cfg_scale": 7.5, "sampler_name": "k_lms", "show_steps": false, "use_init_img": false, "use_inpainting": false, "strength": 0.75, "fit": true, "prompt_structure_token_subject": "clouds ethereal mid-air by greg rutkowski and artgerm", "prompt_structure_token_subject_enum": "custom", "prompt_structure_token_framing": "", "prompt_structure_token_framing_enum": "ecu", "prompt_structure_token_position": "", "prompt_structure_token_position_enum": "overhead", "prompt_structure_token_film_type": "", "prompt_structure_token_film_type_enum": "bw", "prompt_structure_token_camera_settings": "", "prompt_structure_token_camera_settings_enum": "high_speed", "prompt_structure_token_shooting_context": "", "prompt_structure_token_shooting_context_enum": "film_still", "prompt_structure_token_lighting": "", "prompt_structure_token_lighting_enum": "golden_hour", "prompt_structure_token_subject_type": "", "prompt_structure_token_subject_type_enum": "environment", "prompt_structure_token_genre": "", "prompt_structure_token_genre_enum": "fantasy", "prompt": "clouds ethereal mid-air by greg rutkowski and artgerm, Environment concept art, Fantasy digital painting, trending on ArtStation [person dragon bird ocean buildings woman]" } ================================================ FILE: engine/__init__.py ================================================ from .engine import * from .node_tree import * from .node_executor import * from .node import * from .nodes.input_nodes import * from .nodes.pipeline_nodes import * from .nodes.utility_nodes import * from .nodes.annotation_nodes import * from .annotations import openpose from .annotations import ade20k import bpy import nodeitems_utils class DreamTexturesNodeCategory(nodeitems_utils.NodeCategory): @classmethod def poll(cls, context): return context.space_data.tree_type == DreamTexturesNodeTree.__name__ pipeline_items = [ nodeitems_utils.NodeItem(NodeStableDiffusion.bl_idname), nodeitems_utils.NodeItem(NodeControlNet.bl_idname), ] input_items = [ nodeitems_utils.NodeItem(NodeInteger.bl_idname), nodeitems_utils.NodeItem(NodeString.bl_idname), nodeitems_utils.NodeItem(NodeImage.bl_idname), nodeitems_utils.NodeItem(NodeCollection.bl_idname), nodeitems_utils.NodeItem(NodeRenderProperties.bl_idname), ] if bpy.app.version >= (3, 5, 0): input_items.append(nodeitems_utils.NodeItem(NodeImageFile.bl_idname)) utility_items = [ nodeitems_utils.NodeItem(NodeMath.bl_idname), nodeitems_utils.NodeItem(NodeRandomValue.bl_idname), nodeitems_utils.NodeItem(NodeRandomSeed.bl_idname), nodeitems_utils.NodeItem(NodeSeed.bl_idname), nodeitems_utils.NodeItem(NodeClamp.bl_idname), nodeitems_utils.NodeItem(NodeFramePath.bl_idname), nodeitems_utils.NodeItem(NodeCropImage.bl_idname), nodeitems_utils.NodeItem(NodeJoinImages.bl_idname), nodeitems_utils.NodeItem(NodeColorCorrect.bl_idname), nodeitems_utils.NodeItem(NodeSeparateColor.bl_idname), nodeitems_utils.NodeItem(NodeCombineColor.bl_idname), nodeitems_utils.NodeItem(NodeSwitch.bl_idname), nodeitems_utils.NodeItem(NodeCompare.bl_idname), nodeitems_utils.NodeItem(NodeReplaceString.bl_idname), ] if bpy.app.version >= (3, 5, 0): utility_items.append(nodeitems_utils.NodeItem(NodeResizeImage.bl_idname)) annotations_items = [ nodeitems_utils.NodeItem(NodeAnnotationDepth.bl_idname), nodeitems_utils.NodeItem(NodeAnnotationOpenPose.bl_idname), nodeitems_utils.NodeItem(NodeAnnotationADE20K.bl_idname), nodeitems_utils.NodeItem(NodeAnnotationViewport.bl_idname), ] group_items = [ nodeitems_utils.NodeItem(bpy.types.NodeGroupOutput.__name__), ] categories = [ DreamTexturesNodeCategory("DREAM_TEXTURES_PIPELINE", "Pipeline", items=pipeline_items), DreamTexturesNodeCategory("DREAM_TEXTURES_INPUT", "Input", items=input_items), DreamTexturesNodeCategory("DREAM_TEXTURES_UTILITY", "Utilities", items=utility_items), DreamTexturesNodeCategory("DREAM_TEXTURES_ANNOTATIONS", "Annotations", items=annotations_items), DreamTexturesNodeCategory("DREAM_TEXTURES_GROUP", "Group", items=group_items), ] def register(): # Prompt bpy.types.Scene.dream_textures_engine_prompt = bpy.props.PointerProperty(type=DreamPrompt) # OpenPose bpy.utils.register_class(openpose.ArmatureOpenPoseData) bpy.types.Armature.dream_textures_openpose = bpy.props.PointerProperty( type=openpose.ArmatureOpenPoseData ) bpy.utils.register_class(openpose.BoneOpenPoseData) bpy.types.Bone.dream_textures_openpose = bpy.props.PointerProperty( type=openpose.BoneOpenPoseData ) # ADE20K bpy.utils.register_class(ade20k.ObjectADE20KData) bpy.types.Object.dream_textures_ade20k = bpy.props.PointerProperty( type=ade20k.ObjectADE20KData ) bpy.utils.register_class(DreamTexturesNodeTree) # Nodes bpy.utils.register_class(NodeSocketControlNet) bpy.utils.register_class(NodeStableDiffusion) bpy.utils.register_class(NodeControlNet) bpy.utils.register_class(NodeInteger) bpy.utils.register_class(NodeString) bpy.utils.register_class(NodeCollection) bpy.utils.register_class(NodeImage) bpy.utils.register_class(NodeImageFile) bpy.utils.register_class(NodeRenderProperties) bpy.utils.register_class(NodeAnnotationDepth) bpy.utils.register_class(NodeAnnotationNormal) bpy.utils.register_class(NodeAnnotationOpenPose) bpy.utils.register_class(NodeAnnotationADE20K) bpy.utils.register_class(NodeAnnotationViewport) bpy.utils.register_class(NodeMath) bpy.utils.register_class(NodeRandomValue) bpy.utils.register_class(NodeRandomSeed) bpy.utils.register_class(NodeSeed) bpy.utils.register_class(NodeClamp) bpy.utils.register_class(NodeFramePath) bpy.utils.register_class(NodeCropImage) bpy.utils.register_class(NodeResizeImage) bpy.utils.register_class(NodeJoinImages) bpy.utils.register_class(NodeColorCorrect) bpy.utils.register_class(NodeSeparateColor) bpy.utils.register_class(NodeCombineColor) bpy.utils.register_class(NodeSwitch) bpy.utils.register_class(NodeCompare) bpy.utils.register_class(NodeReplaceString) nodeitems_utils.register_node_categories("DREAM_TEXTURES_CATEGORIES", categories) def unregister(): # OpenPose del bpy.types.Armature.dream_textures_openpose bpy.utils.unregister_class(openpose.ArmatureOpenPoseData) del bpy.types.Bone.dream_textures_openpose bpy.utils.unregister_class(openpose.BoneOpenPoseData) # ADE20K del bpy.types.Object.dream_textures_ade20k bpy.utils.unregister_class(ade20k.ObjectADE20KData) bpy.utils.unregister_class(DreamTexturesNodeTree) # Nodes bpy.utils.unregister_class(NodeSocketControlNet) bpy.utils.unregister_class(NodeStableDiffusion) bpy.utils.unregister_class(NodeControlNet) bpy.utils.unregister_class(NodeInteger) bpy.utils.unregister_class(NodeString) bpy.utils.unregister_class(NodeCollection) bpy.utils.unregister_class(NodeImage) bpy.utils.unregister_class(NodeImageFile) bpy.utils.unregister_class(NodeRenderProperties) bpy.utils.unregister_class(NodeAnnotationDepth) bpy.utils.unregister_class(NodeAnnotationNormal) bpy.utils.unregister_class(NodeAnnotationOpenPose) bpy.utils.unregister_class(NodeAnnotationADE20K) bpy.utils.unregister_class(NodeAnnotationViewport) bpy.utils.unregister_class(NodeMath) bpy.utils.unregister_class(NodeRandomValue) bpy.utils.unregister_class(NodeRandomSeed) bpy.utils.unregister_class(NodeSeed) bpy.utils.unregister_class(NodeClamp) bpy.utils.unregister_class(NodeFramePath) bpy.utils.unregister_class(NodeCropImage) bpy.utils.unregister_class(NodeResizeImage) bpy.utils.unregister_class(NodeJoinImages) bpy.utils.unregister_class(NodeColorCorrect) bpy.utils.unregister_class(NodeSeparateColor) bpy.utils.unregister_class(NodeCombineColor) bpy.utils.unregister_class(NodeSwitch) bpy.utils.unregister_class(NodeCompare) bpy.utils.unregister_class(NodeReplaceString) nodeitems_utils.unregister_node_categories("DREAM_TEXTURES_CATEGORIES") ================================================ FILE: engine/annotations/ade20k.py ================================================ import bpy import gpu from gpu_extras.batch import batch_for_shader import numpy as np import threading from .compat import UNIFORM_COLOR annotation_enum_cases = [('91', 'airplane', 'airplane;aeroplane;plane'), ('127', 'animal', 'animal;animate;being;beast;brute;creature;fauna'), ('93', 'apparel', 'apparel;wearing;apparel;dress;clothes'), ('79', 'arcade machine', 'arcade;machine'), ('31', 'armchair', 'armchair'), ('139', 'ashcan', 'ashcan;trash;can;garbage;can;wastebin;ash;bin;ash-bin;ashbin;dustbin;trash;barrel;trash;bin'), ('87', 'awning', 'awning;sunshade;sunblind'), ('116', 'bag', 'bag'), ('120', 'ball', 'ball'), ('96', 'bannister', 'bannister;banister;balustrade;balusters;handrail'), ('78', 'bar', 'bar'), ('112', 'barrel', 'barrel;cask'), ('41', 'base', 'base;pedestal;stand'), ('113', 'basket', 'basket;handbasket'), ('38', 'bathtub', 'bathtub;bathing;tub;bath;tub'), ('8', 'bed', 'bed'), ('70', 'bench', 'bench'), ('128', 'bicycle', 'bicycle;bike;wheel;cycle'), ('132', 'blanket', 'blanket;cover'), ('64', 'blind', 'blind;screen'), ('77', 'boat', 'boat'), ('63', 'book', 'bookcase'), ('63', 'bookcase', 'bookcase'), ('89', 'booth', 'booth;cubicle;stall;kiosk'), ('99', 'bottle', 'bottle'), ('42', 'box', 'box'), ('62', 'bridge', 'bridge;span'), ('100', 'buffet', 'buffet;counter;sideboard'), ('2', 'building', 'building;edifice'), ('145', 'bulletin board', 'bulletin;board;notice;board'), ('81', 'bus', 'bus;autobus;coach;charabanc;double-decker;jitney;motorbus;motorcoach;omnibus;passenger;vehicle'), ('11', 'cabinet', 'cabinet'), ('107', 'canopy', 'canopy'), ('21', 'car', 'car;auto;automobile;machine;motorcar'), ('56', 'case', 'case;display;case;showcase;vitrine'), ('6', 'ceiling', 'ceiling'), ('20', 'chair', 'chair'), ('86', 'chandelier', 'chandelier;pendant;pendent'), ('45', 'chest of drawers', 'chest;of;drawers;chest;bureau;dresser'), ('149', 'clock', 'clock'), ('65', 'coffee table', 'coffee;table;cocktail;table'), ('43', 'column', 'column;pillar'), ('75', 'computer', 'computer;computing;machine;computing;device;data;processor;electronic;computer;information;processing;system'), ('106', 'conveyer belt', 'conveyer;belt;conveyor;belt;conveyer;conveyor;transporter'), ('46', 'counter', 'counter'), ('71', 'countertop', 'countertop'), ('118', 'cradle', 'cradle'), ('142', 'crt screen', 'crt;screen'), ('19', 'curtain', 'curtain;drape;drapery;mantle;pall'), ('40', 'cushion', 'cushion'), ('34', 'desk', 'desk'), ('92', 'dirt track', 'dirt;track'), ('130', 'dishwasher', 'dishwasher;dish;washer;dishwashing;machine'), ('15', 'door', 'door;double;door'), ('14', 'earth', 'earth;ground'), ('97', 'escalator', 'escalator;moving;staircase;moving;stairway'), ('140', 'fan', 'fan'), ('33', 'fence', 'fence;fencing'), ('30', 'field', 'field'), ('50', 'fireplace', 'fireplace;hearth;open;fireplace'), ('150', 'flag', 'flag'), ('4', 'floor', 'floor;flooring'), ('67', 'flower', 'flower'), ('121', 'food', 'food;solid;food'), ('105', 'fountain', 'fountain'), ('148', 'glass', 'glass;drinking;glass'), ('52', 'grandstand', 'grandstand;covered;stand'), ('10', 'grass', 'grass'), ('69', 'hill', 'hill'), ('134', 'hood', 'hood;exhaust;hood'), ('26', 'house', 'house'), ('80', 'hovel', 'hovel;hut;hutch;shack;shanty'), ('74', 'land', 'kitchen;island'), ('74', 'kitchen island', 'kitchen;island'), ('129', 'lake', 'lake'), ('37', 'lamp', 'lamp'), ('83', 'light', 'light;light;source'), ('125', 'microwave', 'microwave;microwave;oven'), ('117', 'minibike', 'minibike;motorbike'), ('28', 'mirror', 'mirror'), ('144', 'monitor', 'monitor;monitoring;device'), ('17', 'mountain', 'mountain;mount'), ('98', 'ottoman', 'ottoman;pouf;pouffe;puff;hassock'), ('119', 'oven', 'oven'), ('23', 'painting', 'painting;picture'), ('73', 'palm', 'palm;palm;tree'), ('53', 'path', 'path'), ('13', 'person', 'person;individual;someone;somebody;mortal;soul'), ('141', 'pier', 'pier;wharf;wharfage;dock'), ('58', 'pillow', 'pillow'), ('18', 'plant', 'plant;flora;plant;life'), ('143', 'plate', 'plate'), ('109', 'plaything', 'plaything;toy'), ('94', 'pole', 'pole'), ('57', 'pool table', 'pool;table;billiard;table;snooker;table'), ('101', 'poster', 'poster;posting;placard;notice;bill;card'), ('147', 'radiator', 'radiator'), ('39', 'railing', 'railing;rail'), ('51', 'refrigerator', 'refrigerator;icebox'), ('61', 'river', 'river'), ('7', 'road', 'road;route'), ('35', 'rock', 'rock;stone'), ('29', 'rug', 'rug;carpet;carpeting'), ('55', 'runway', 'runway'), ('47', 'sand', 'sand'), ('135', 'sconce', 'sconce'), ('59', 'screen door', 'screen;door;screen'), ('59', 'screen', 'screen;door;screen'), ('133', 'sculpture', 'sculpture'), ('27', 'sea', 'sea'), ('32', 'seat', 'seat'), ('25', 'shelf', 'shelf'), ('104', 'ship', 'ship'), ('146', 'shower', 'shower'), ('12', 'sidewalk', 'sidewalk;pavement'), ('44', 'signboard', 'signboard;sign'), ('48', 'sink', 'sink'), ('3', 'sky', 'sky'), ('49', 'skyscraper', 'skyscraper'), ('24', 'sofa', 'sofa;couch;lounge'), ('102', 'stage', 'stage'), ('54', 'step', 'stairs;steps'), ('54', 'stairs', 'stairs;steps'), ('60', 'stairway', 'stairway;staircase'), ('72', 'stove', 'stove;kitchen;stove;range;kitchen;range;cooking;stove'), ('88', 'streetlight', 'streetlight;street;lamp'), ('110', 'swimming pool', 'swimming;pool;swimming;bath;natatorium'), ('76', 'swivel chair', 'swivel;chair'), ('16', 'table', 'table'), ('123', 'tank', 'tank;storage;tank'), ('90', 'television receiver', 'television;television;receiver;television;set;tv;tv;set;idiot;box;boob;tube;telly;goggle;box'), ('115', 'tent', 'tent;collapsible;shelter'), ('66', 'stool', 'toilet;can;commode;crapper;pot;potty;stool;throne'), ('66', 'pot', 'toilet;can;commode;crapper;pot;potty;stool;throne'), ('66', 'toilet', 'toilet;can;commode;crapper;pot;potty;stool;throne'), ('82', 'towel', 'towel'), ('85', 'tower', 'tower'), ('124', 'trade name', 'trade;name;brand;name;brand;marque'), ('137', 'traffic light', 'traffic;light;traffic;signal;stoplight'), ('138', 'tray', 'tray'), ('5', 'tree', 'tree'), ('84', 'truck', 'truck;motortruck'), ('103', 'van', 'van'), ('136', 'vase', 'vase'), ('1', 'wall', 'wall'), ('36', 'wardrobe', 'wardrobe;closet;press'), ('108', 'washer', 'washer;automatic;washer;washing;machine'), ('22', 'water', 'water'), ('114', 'waterfall', 'waterfall;falls'), ('9', 'windowpane', 'windowpane;window')] annotation_colors = {'80': (1.0, 0.0, 0.996078431372549, 1.0), '140': (0.00392156862745098, 0.9607843137254902, 1.0, 1.0), '115': (0.43529411764705883, 0.8784313725490196, 0.996078431372549, 1.0), '33': (1.0, 0.7215686274509804, 0.023529411764705882, 1.0), '77': (0.6784313725490196, 1.0, 0.0, 1.0), '58': (0.0, 0.9215686274509803, 1.0, 1.0), '98': (0.996078431372549, 0.6, 0.0, 1.0), '74': (0.0, 1.0, 0.1607843137254902, 1.0), '150': (0.3568627450980392, 0.0, 0.996078431372549, 1.0), '117': (0.6431372549019608, 0.0, 1.0, 1.0), '18': (0.8, 1.0, 0.01568627450980392, 1.0), '145': (0.7215686274509804, 1.0, 0.0, 1.0), '63': (0.0, 1.0, 0.9607843137254902, 1.0), '9': (0.9019607843137255, 0.9019607843137255, 0.9019607843137255, 1.0), '34': (0.0392156862745098, 1.0, 0.2784313725490196, 1.0), '120': (1.0, 0.00392156862745098, 0.6352941176470588, 1.0), '61': (0.0392156862745098, 0.7843137254901961, 0.7843137254901961, 1.0), '32': (0.027450980392156862, 0.996078431372549, 0.8745098039215686, 1.0), '124': (0.5215686274509804, 0.996078431372549, 0.0, 1.0), '6': (0.4666666666666667, 0.47058823529411764, 0.3137254901960784, 1.0), '104': (1.0, 0.9215686274509803, 0.0, 1.0), '38': (0.4, 0.03137254901960784, 1.0, 1.0), '106': (0.5215686274509804, 0.0, 1.0, 1.0), '56': (0.00392156862745098, 0.0, 0.996078431372549, 1.0), '12': (0.9215686274509803, 1.0, 0.027450980392156862, 1.0), '19': (1.0, 0.2, 0.03137254901960784, 1.0), '70': (0.7607843137254902, 1.0, 0.00392156862745098, 1.0), '88': (0.00392156862745098, 0.2784313725490196, 1.0, 1.0), '62': (1.0, 0.3254901960784314, 0.00392156862745098, 1.0), '7': (0.5490196078431373, 0.5490196078431373, 0.5490196078431373, 1.0), '29': (1.0, 0.03529411764705882, 0.3607843137254902, 1.0), '82': (1.0, 0.0, 0.4, 1.0), '142': (0.47843137254901963, 0.00392156862745098, 1.0, 1.0), '11': (0.8784313725490196, 0.0196078431372549, 1.0, 1.0), '147': (1.0, 0.8392156862745098, 0.00784313725490196, 1.0), '76': (0.0392156862745098, 0.0, 1.0, 1.0), '146': (0.0, 0.5215686274509804, 0.996078431372549, 1.0), '54': (1.0, 0.8784313725490196, 0.00392156862745098, 1.0), '94': (0.2, 0.0, 1.0, 1.0), '59': (0.0, 0.8, 1.0, 1.0), '118': (0.6039215686274509, 0.0, 1.0, 1.0), '39': (1.0, 0.24313725490196078, 0.027450980392156862, 1.0), '101': (0.5607843137254902, 0.996078431372549, 0.00392156862745098, 1.0), '46': (0.9254901960784314, 0.043137254901960784, 1.0, 1.0), '110': (0.0, 0.7215686274509804, 1.0, 1.0), '45': (0.023529411764705882, 0.2, 1.0, 1.0), '93': (0.0, 0.44313725490196076, 0.996078431372549, 1.0), '13': (0.5882352941176471, 0.0196078431372549, 0.24313725490196078, 1.0), '50': (0.9803921568627451, 0.03529411764705882, 0.058823529411764705, 1.0), '64': (0.00392156862745098, 0.23921568627450981, 1.0, 1.0), '30': (0.43529411764705883, 0.03529411764705882, 1.0, 1.0), '99': (0.0, 1.0, 0.043137254901960784, 1.0), '35': (1.0, 0.1568627450980392, 0.03529411764705882, 1.0), '125': (0.996078431372549, 0.0, 0.9176470588235294, 1.0), '105': (0.03137254901960784, 0.7215686274509804, 0.6705882352941176, 1.0), '135': (0.0, 0.1607843137254902, 1.0, 1.0), '53': (1.0, 0.11764705882352941, 0.0, 1.0), '66': (0.0, 1.0, 0.5215686274509804, 1.0), '8': (0.8, 0.0196078431372549, 0.996078431372549, 1.0), '26': (1.0, 0.03137254901960784, 0.8666666666666667, 1.0), '14': (0.47058823529411764, 0.47058823529411764, 0.27450980392156865, 1.0), '73': (0.0, 0.3215686274509804, 0.996078431372549, 1.0), '23': (1.0, 0.023529411764705882, 0.2, 1.0), '24': (0.043137254901960784, 0.4, 1.0, 1.0), '97': (0.0, 1.0, 0.6392156862745098, 1.0), '132': (0.0784313725490196, 0.0, 1.0, 1.0), '100': (1.0, 0.4392156862745098, 0.0, 1.0), '83': (1.0, 0.6784313725490196, 0.00392156862745098, 1.0), '85': (0.996078431372549, 0.7215686274509804, 0.7215686274509804, 1.0), '79': (1.0, 0.3607843137254902, 0.0, 1.0), '144': (0.0, 0.3607843137254902, 1.0, 1.0), '113': (0.3568627450980392, 1.0, 0.0, 1.0), '133': (1.0, 1.0, 0.0, 1.0), '52': (0.12156862745098039, 1.0, 0.0, 1.0), '28': (0.8627450980392157, 0.8627450980392157, 0.8627450980392157, 1.0), '10': (0.0196078431372549, 0.9803921568627451, 0.027450980392156862, 1.0), '2': (0.7058823529411765, 0.47058823529411764, 0.47058823529411764, 1.0), '103': (0.6392156862745098, 0.996078431372549, 0.0, 1.0), '91': (0.00392156862745098, 0.996078431372549, 0.3254901960784314, 1.0), '121': (1.0, 0.8, 0.0, 1.0), '86': (0.0, 0.12156862745098039, 0.996078431372549, 1.0), '27': (0.03529411764705882, 0.027450980392156862, 0.9058823529411765, 1.0), '114': (0.0, 0.8823529411764706, 1.0, 1.0), '20': (0.796078431372549, 0.27450980392156865, 0.011764705882352941, 1.0), '134': (0.0, 0.6, 1.0, 1.0), '17': (0.5568627450980392, 1.0, 0.5450980392156862, 1.0), '3': (0.023529411764705882, 0.9019607843137255, 0.9019607843137255, 1.0), '65': (0.00392156862745098, 0.996078431372549, 0.4392156862745098, 1.0), '139': (0.6823529411764706, 0.0, 1.0, 1.0), '130': (0.8392156862745098, 1.0, 0.0, 1.0), '136': (0.00392156862745098, 1.0, 0.803921568627451, 1.0), '51': (0.0784313725490196, 1.0, 0.0, 1.0), '15': (0.03137254901960784, 1.0, 0.20392156862745098, 1.0), '55': (0.6, 1.0, 0.0, 1.0), '5': (0.01568627450980392, 0.7843137254901961, 0.023529411764705882, 1.0), '22': (0.23921568627450981, 0.9019607843137255, 0.984313725490196, 1.0), '72': (0.2, 1.0, 0.0, 1.0), '41': (1.0, 0.4823529411764706, 0.0392156862745098, 1.0), '137': (0.1607843137254902, 0.0, 1.0, 1.0), '92': (0.0, 0.0392156862745098, 1.0, 1.0), '21': (0.0, 0.4, 0.7843137254901961, 1.0), '109': (1.0, 0.0, 0.11764705882352941, 1.0), '1': (0.47058823529411764, 0.47058823529411764, 0.47058823529411764, 1.0), '67': (1.0, 0.00392156862745098, 0.011764705882352941, 1.0), '60': (0.12156862745098039, 0.0, 0.996078431372549, 1.0), '44': (1.0, 0.0196078431372549, 0.6039215686274509, 1.0), '69': (1.0, 0.4, 0.0, 1.0), '107': (0.0, 0.996078431372549, 0.36470588235294116, 1.0), '4': (0.3137254901960784, 0.19607843137254902, 0.20392156862745098, 1.0), '90': (0.0, 1.0, 0.7647058823529411, 1.0), '149': (0.396078431372549, 1.0, 0.0, 1.0), '129': (0.0392156862745098, 0.7490196078431373, 0.8313725490196079, 1.0), '75': (0.0, 1.0, 0.6784313725490196, 1.0), '141': (0.2823529411764706, 0.0, 1.0, 1.0), '128': (1.0, 0.9607843137254902, 0.0, 1.0), '57': (1.0, 0.2784313725490196, 0.0, 1.0), '49': (0.5490196078431373, 0.5490196078431373, 0.5490196078431373, 1.0), '108': (0.7215686274509804, 0.0, 0.996078431372549, 1.0), '87': (0.0, 1.0, 0.23921568627450981, 1.0), '43': (1.0, 0.03137254901960784, 0.16470588235294117, 1.0), '71': (0.0, 0.5607843137254902, 1.0, 1.0), '31': (0.03529411764705882, 0.996078431372549, 0.8352941176470589, 1.0), '81': (1.0, 0.0, 0.9568627450980393, 1.0), '36': (0.027450980392156862, 1.0, 1.0, 1.0), '143': (0.0, 1.0, 0.7254901960784313, 1.0), '148': (0.09411764705882353, 0.7607843137254902, 0.7607843137254902, 1.0), '102': (0.3215686274509804, 0.0, 1.0, 1.0), '84': (1.0, 0.0, 0.08235294117647059, 1.0), '47': (0.6274509803921569, 0.5882352941176471, 0.07450980392156863, 1.0), '127': (1.0, 0.0, 0.47843137254901963, 1.0), '16': (0.996078431372549, 0.023529411764705882, 0.3215686274509804, 1.0), '78': (0.0, 0.996078431372549, 0.6078431372549019, 1.0), '25': (1.0, 0.023529411764705882, 0.27450980392156865, 1.0), '138': (0.16862745098039217, 0.996078431372549, 0.011764705882352941, 1.0), '42': (0.00392156862745098, 1.0, 0.07450980392156863, 1.0), '119': (0.2784313725490196, 1.0, 0.00392156862745098, 1.0), '112': (0.996078431372549, 0.0, 0.4392156862745098, 1.0), '89': (0.996078431372549, 0.0, 0.796078431372549, 1.0), '96': (0.0, 0.47843137254901963, 1.0, 1.0), '37': (0.8823529411764706, 1.0, 0.03529411764705882, 1.0), '48': (0.0, 0.6392156862745098, 0.996078431372549, 1.0), '116': (0.27058823529411763, 0.7176470588235294, 0.6196078431372549, 1.0), '40': (1.0, 0.7607843137254902, 0.027450980392156862, 1.0), '123': (0.0, 1.0, 0.9176470588235294, 1.0)} def annotation_update(self, context): self.color = annotation_colors[self.annotation][:3] class ObjectADE20KData(bpy.types.PropertyGroup): bl_label = "ADE20K Segmentation" bl_idname = "dream_textures.ObjectADE20KData" enabled: bpy.props.BoolProperty(name="Enabled", default=False) annotation: bpy.props.EnumProperty( name="Class", items=annotation_enum_cases, update=annotation_update ) # for visualization only color: bpy.props.FloatVectorProperty(name="", subtype='COLOR', default=annotation_colors[annotation_enum_cases[0][0]][:3]) def render_ade20k_map(context, collection=None, invert=True): e = threading.Event() result = None def _execute(): nonlocal result width, height = context.scene.render.resolution_x, context.scene.render.resolution_y matrix = context.scene.camera.matrix_world.inverted() projection_matrix = context.scene.camera.calc_matrix_camera( context, x=width, y=height ) offscreen = gpu.types.GPUOffScreen(width, height) with offscreen.bind(): fb = gpu.state.active_framebuffer_get() fb.clear(color=(0.0, 0.0, 0.0, 0.0), depth=1) gpu.state.depth_test_set('LESS_EQUAL') gpu.state.depth_mask_set(True) with gpu.matrix.push_pop(): gpu.matrix.load_matrix(matrix) gpu.matrix.load_projection_matrix(projection_matrix) def render_mesh(mesh, transform, color): mesh.transform(transform) mesh.calc_loop_triangles() vertices = np.empty((len(mesh.vertices), 3), 'f') indices = np.empty((len(mesh.loop_triangles), 3), 'i') mesh.vertices.foreach_get("co", np.reshape(vertices, len(mesh.vertices) * 3)) mesh.loop_triangles.foreach_get("vertices", np.reshape(indices, len(mesh.loop_triangles) * 3)) draw_annotation(vertices, indices, color) if collection is None: for object in context.object_instances: if not hasattr(object.object, 'dream_textures_ade20k') or not object.object.dream_textures_ade20k.enabled: continue try: mesh = object.object.to_mesh() if mesh is not None: render_mesh(mesh, object.matrix_world, annotation_colors[object.object.dream_textures_ade20k.annotation]) object.object.to_mesh_clear() except: continue else: for object in collection.objects: if not hasattr(object, 'dream_textures_ade20k') or not object.dream_textures_ade20k.enabled: continue try: mesh = object.to_mesh(depsgraph=context) if mesh is not None: render_mesh(mesh, object.matrix_world, annotation_colors[object.dream_textures_ade20k.annotation]) object.to_mesh_clear() except: continue result = np.array(fb.read_color(0, 0, width, height, 4, 0, 'FLOAT').to_list()) result[:, :, 3] = 1 gpu.state.depth_test_set('NONE') offscreen.free() e.set() if threading.current_thread() == threading.main_thread(): _execute() return result else: bpy.app.timers.register(_execute, first_interval=0) e.wait() return result def draw_annotation(vertices, indices, color): shader = gpu.shader.from_builtin(UNIFORM_COLOR) batch = batch_for_shader( shader, 'TRIS', {"pos": vertices}, indices=indices, ) shader.bind() shader.uniform_float("color", color) batch.draw(shader) ================================================ FILE: engine/annotations/compat.py ================================================ import bpy UNIFORM_COLOR = 'UNIFORM_COLOR' if bpy.app.version >= (3, 4, 0) else '3D_UNIFORM_COLOR' """ 2D_ and 3D_ prefixed built-in shaders were deprecated in Blender 3.4 and removed in Blender 4.0 """ ================================================ FILE: engine/annotations/depth.py ================================================ import bpy import gpu from gpu_extras.batch import batch_for_shader import numpy as np import threading from .compat import UNIFORM_COLOR def render_depth_map(context, collection=None, invert=True, width=None, height=None, matrix=None, projection_matrix=None, main_thread=False): e = threading.Event() result = None width, height = width or context.scene.render.resolution_x, height or context.scene.render.resolution_y matrix = matrix or context.scene.camera.matrix_world.inverted() projection_matrix = projection_matrix or context.scene.camera.calc_matrix_camera( context, x=width, y=height ) def _execute(): nonlocal result offscreen = gpu.types.GPUOffScreen(width, height) with offscreen.bind(): fb = gpu.state.active_framebuffer_get() fb.clear(color=(0.0, 0.0, 0.0, 0.0), depth=1) gpu.state.depth_test_set('LESS_EQUAL') gpu.state.depth_mask_set(True) with gpu.matrix.push_pop(): gpu.matrix.load_matrix(matrix) gpu.matrix.load_projection_matrix(projection_matrix) shader = gpu.shader.from_builtin(UNIFORM_COLOR) def render_mesh(mesh, transform): mesh.transform(transform) mesh.calc_loop_triangles() vertices = np.empty((len(mesh.vertices), 3), 'f') indices = np.empty((len(mesh.loop_triangles), 3), 'i') mesh.vertices.foreach_get("co", np.reshape(vertices, len(mesh.vertices) * 3)) mesh.loop_triangles.foreach_get("vertices", np.reshape(indices, len(mesh.loop_triangles) * 3)) batch = batch_for_shader( shader, 'TRIS', {"pos": vertices}, indices=indices, ) batch.draw(shader) if collection is None: for object in context.object_instances: try: mesh = object.object.to_mesh() if mesh is not None: render_mesh(mesh, object.matrix_world) object.object.to_mesh_clear() except: continue else: for object in collection.objects: try: mesh = object.to_mesh(depsgraph=context) if mesh is not None: render_mesh(mesh, object.matrix_world) object.to_mesh_clear() except: continue depth = np.array(fb.read_depth(0, 0, width, height).to_list()) if invert: depth = 1 - depth mask = np.array(fb.read_color(0, 0, width, height, 4, 0, 'UBYTE').to_list())[:, :, 3] depth *= mask depth = np.interp(depth, [np.ma.masked_equal(depth, 0, copy=False).min(), depth.max()], [0, 1]).clip(0, 1) gpu.state.depth_test_set('NONE') offscreen.free() result = depth e.set() if main_thread or threading.current_thread() == threading.main_thread(): _execute() return result else: bpy.app.timers.register(_execute, first_interval=0) e.wait() return result ================================================ FILE: engine/annotations/normal.py ================================================ import bpy import gpu from gpu_extras.batch import batch_for_shader import bmesh import numpy as np import threading def render_normal_map(context, collection=None, width=None, height=None, matrix=None, projection_matrix=None, main_thread=False): e = threading.Event() result = None width, height = width or context.scene.render.resolution_x, height or context.scene.render.resolution_y matrix = matrix or context.scene.camera.matrix_world.inverted() projection_matrix = projection_matrix or context.scene.camera.calc_matrix_camera( context, x=width, y=height ) def normals_shader(smooth): vert_out = gpu.types.GPUStageInterfaceInfo("my_interface") if smooth: vert_out.smooth('VEC3', "color") else: vert_out.flat('VEC3', "color") shader_info = gpu.types.GPUShaderCreateInfo() shader_info.push_constant('MAT4', "ModelViewProjectionMatrix") shader_info.push_constant('MAT4', "ModelViewMatrix") shader_info.vertex_in(0, 'VEC3', "position") shader_info.vertex_in(1, 'VEC3', "normal") shader_info.vertex_out(vert_out) shader_info.fragment_out(0, 'VEC4', "fragColor") shader_info.vertex_source(""" void main() { gl_Position = ModelViewProjectionMatrix * vec4(position, 1.0f); color = normalize(mat3(ModelViewMatrix) * normal); } """) shader_info.fragment_source(""" void main() { vec4 c = vec4((color + 1) / 2, 1.0f); fragColor = vec4(1 - c.r, c.g, c.b, c.a); } """) return gpu.shader.create_from_info(shader_info) def _execute(): nonlocal result offscreen = gpu.types.GPUOffScreen(width, height) with offscreen.bind(): fb = gpu.state.active_framebuffer_get() fb.clear(color=(0.5, 0.5, 1.0, 1.0), depth=1) gpu.state.depth_test_set('LESS_EQUAL') gpu.state.depth_mask_set(True) with gpu.matrix.push_pop(): gpu.matrix.load_matrix(matrix) gpu.matrix.load_projection_matrix(projection_matrix) def render_mesh(mesh, transform): smooth_shader = normals_shader(smooth=True) smooth_shader.uniform_float("ModelViewMatrix", gpu.matrix.get_model_view_matrix()) flat_shader = normals_shader(smooth=False) flat_shader.uniform_float("ModelViewMatrix", gpu.matrix.get_model_view_matrix()) mesh.transform(transform) bm = bmesh.new() bm.from_mesh(mesh) smooth_mesh = bm.copy() flat_mesh = bm.copy() bmesh.ops.delete(smooth_mesh, geom=[f for f in smooth_mesh.faces if not f.smooth], context='FACES') bmesh.ops.delete(flat_mesh, geom=[f for f in flat_mesh.faces if f.smooth], context='FACES') def draw(mesh, smooth): vertices = [v.co for v in mesh.verts] normals = [v.normal for v in mesh.verts] indices = [[loop.vert.index for loop in looptris] for looptris in mesh.calc_loop_triangles()] shader = smooth_shader if smooth else flat_shader shader.bind() batch = batch_for_shader( shader, 'TRIS', {"position": vertices, "normal": normals}, indices=indices, ) batch.draw(shader) if len(smooth_mesh.verts) > 0: draw(smooth_mesh, smooth=True) if len(flat_mesh.verts) > 0: draw(flat_mesh, smooth=False) if collection is None: for object in context.object_instances: try: mesh = object.object.to_mesh() if mesh is not None: render_mesh(mesh, object.matrix_world) object.object.to_mesh_clear() except: continue else: for object in collection.objects: try: mesh = object.to_mesh(depsgraph=context) if mesh is not None: render_mesh(mesh, object.matrix_world) object.to_mesh_clear() except: continue normal_map = np.array(fb.read_color(0, 0, width, height, 4, 0, 'FLOAT').to_list()) gpu.state.depth_test_set('NONE') offscreen.free() result = normal_map e.set() if main_thread or threading.current_thread() == threading.main_thread(): _execute() return result else: bpy.app.timers.register(_execute, first_interval=0) e.wait() return result ================================================ FILE: engine/annotations/openpose.py ================================================ import bpy import bpy_extras import gpu from gpu_extras.batch import batch_for_shader import mathutils import numpy as np import enum import math import threading from .compat import UNIFORM_COLOR class Side(enum.IntEnum): HEAD = 0 TAIL = 1 class Bone(enum.IntEnum): NOSE = 0 CHEST = 1 SHOULDER_L = 2 SHOULDER_R = 3 ELBOW_L = 4 ELBOW_R = 5 HAND_L = 6 HAND_R = 7 HIP_L = 8 HIP_R = 9 KNEE_L = 10 KNEE_R = 11 FOOT_L = 12 FOOT_R = 13 EYE_L = 14 EYE_R = 15 EAR_L = 16 EAR_R = 17 def identify(self, armature, pose): if not getattr(armature.dream_textures_openpose, self.name): return None, None for bone in pose.bones: if bone.bone.dream_textures_openpose.enabled: if bone.bone.dream_textures_openpose.bone == str(self.value): return bone, Side(int(bone.bone.dream_textures_openpose.side)) options = self.name_detection_options() for option in options: if (result := pose.bones.get(option[0], None)) is not None: return result, option[1] return None, None def name_detection_options(self): match self: case Bone.NOSE: return [('nose_master', Side.TAIL), ('nose.001', Side.TAIL), ('Head', Side.TAIL)] case Bone.CHEST: return [('spine_fk.003', Side.TAIL), ('spine.003', Side.TAIL), ('Spine4', Side.TAIL)] case Bone.SHOULDER_L: return [('shoulder_ik.L', Side.TAIL), ('shoulder.L', Side.TAIL), ('LeftShoulder', Side.TAIL)] case Bone.SHOULDER_R: return [('shoulder_ik.R', Side.TAIL), ('shoulder.R', Side.TAIL), ('RightShoulder', Side.TAIL)] case Bone.ELBOW_L: return [('upper_arm_ik.L', Side.TAIL), ('upper_arm.L', Side.TAIL), ('LeftArm', Side.TAIL)] case Bone.ELBOW_R: return [('upper_arm_ik.R', Side.TAIL), ('upper_arm.R', Side.TAIL), ('RightArm', Side.TAIL)] case Bone.HAND_L: return [('hand_ik.L', Side.TAIL), ('forearm.L', Side.TAIL), ('LeftForeArm', Side.TAIL)] case Bone.HAND_R: return [('hand_ik.R', Side.TAIL), ('forearm.R', Side.TAIL), ('RightForeArm', Side.TAIL)] case Bone.HIP_L: return [('thigh_ik.L', Side.HEAD), ('thigh.L', Side.HEAD), ('LeftThigh', Side.HEAD)] case Bone.HIP_R: return [('thigh_ik.R', Side.HEAD), ('thigh.R', Side.HEAD), ('RightThigh', Side.HEAD)] case Bone.KNEE_L: return [('thigh_ik.L', Side.TAIL), ('thigh.L', Side.TAIL), ('LeftShin', Side.HEAD)] case Bone.KNEE_R: return [('thigh_ik.R', Side.TAIL), ('thigh.R', Side.TAIL), ('RightShin', Side.HEAD)] case Bone.FOOT_L: return [('foot_ik.L', Side.TAIL), ('shin.L', Side.TAIL), ('LeftFoot', Side.HEAD)] case Bone.FOOT_R: return [('foot_ik.R', Side.TAIL), ('shin.R', Side.TAIL), ('RightFoot', Side.HEAD)] case Bone.EYE_L: return [('master_eye.L', Side.TAIL), ('eye.L', Side.TAIL)] case Bone.EYE_R: return [('master_eye.R', Side.TAIL), ('eye.R', Side.TAIL)] case Bone.EAR_L: return [('ear.L', Side.TAIL), ('ear.L.001', Side.TAIL)] case Bone.EAR_R: return [('ear.R', Side.TAIL), ('ear.R.001', Side.TAIL)] def color(self): match self: case Bone.NOSE: return (255, 0, 0) case Bone.CHEST: return (255, 85, 0) case Bone.SHOULDER_L: return (85, 255, 0) case Bone.SHOULDER_R: return (255, 170, 0) case Bone.ELBOW_L: return (0, 255, 0) case Bone.ELBOW_R: return (255, 255, 0) case Bone.HAND_L: return (0, 255, 85) case Bone.HAND_R: return (170, 255, 0) case Bone.HIP_L: return (0, 85, 255) case Bone.HIP_R: return (0, 255, 170) case Bone.KNEE_L: return (0, 0, 255) case Bone.KNEE_R: return (0, 255, 255) case Bone.FOOT_L: return (85, 0, 255) case Bone.FOOT_R: return (0, 170, 255) case Bone.EYE_L: return (255, 0, 255) case Bone.EYE_R: return (170, 0, 255) case Bone.EAR_L: return (255, 0, 85) case Bone.EAR_R: return (255, 0, 170) openpose_bones = ((str(b.value), b.name.title(), '') for b in Bone) openpose_sides = ((str(s.value), s.name.title(), '') for s in Side) class BoneOpenPoseData(bpy.types.PropertyGroup): bl_label = "OpenPose" bl_idname = "dream_textures.BoneOpenPoseData" enabled: bpy.props.BoolProperty(name="Enabled", default=False) bone: bpy.props.EnumProperty( name="OpenPose Bone", items=openpose_bones ) side: bpy.props.EnumProperty( name="Bone Side", items=openpose_sides ) ArmatureOpenPoseData = type('ArmatureOpenPoseData', (bpy.types.PropertyGroup,), { "bl_label": "OpenPose", "bl_idname": "dream_textures.ArmatureOpenPoseData", "__annotations__": { b.name: bpy.props.BoolProperty(name=b.name.title(), default=True) for b in Bone }, }) def render_openpose_map(context, collection=None): e = threading.Event() result = None def _execute(): nonlocal result width, height = context.scene.render.resolution_x, context.scene.render.resolution_y offscreen = gpu.types.GPUOffScreen(width, height) with offscreen.bind(): fb = gpu.state.active_framebuffer_get() fb.clear(color=(0.0, 0.0, 0.0, 0.0), depth=1) gpu.state.depth_test_set('LESS_EQUAL') gpu.state.depth_mask_set(True) lines = { (Bone.NOSE, Bone.CHEST): (0, 0, 255), (Bone.CHEST, Bone.SHOULDER_L): (255, 85, 0), (Bone.CHEST, Bone.SHOULDER_R): (255, 0, 0), (Bone.SHOULDER_L, Bone.ELBOW_L): (170, 255, 0), (Bone.SHOULDER_R, Bone.ELBOW_R): (255, 170, 0), (Bone.ELBOW_L, Bone.HAND_L): (85, 255, 0), (Bone.ELBOW_R, Bone.HAND_R): (255, 255, 0), (Bone.CHEST, Bone.HIP_L): (0, 255, 255), (Bone.CHEST, Bone.HIP_R): (0, 255, 0), (Bone.HIP_L, Bone.KNEE_L): (0, 170, 255), (Bone.HIP_R, Bone.KNEE_R): (0, 255, 85), (Bone.KNEE_L, Bone.FOOT_L): (0, 85, 255), (Bone.KNEE_R, Bone.FOOT_R): (0, 255, 170), (Bone.NOSE, Bone.EYE_L): (255, 0, 255), (Bone.NOSE, Bone.EYE_R): (85, 0, 255), (Bone.EYE_L, Bone.EAR_L): (255, 0, 170), (Bone.EYE_R, Bone.EAR_R): (170, 0, 255), } with gpu.matrix.push_pop(): ratio = width / height projection_matrix = mathutils.Matrix(( (1 / ratio, 0, 0, 0), (0, 1, 0, 0), (0, 0, -1, 0), (0, 0, 0, 1) )) gpu.matrix.load_matrix(mathutils.Matrix.Identity(4)) gpu.matrix.load_projection_matrix(projection_matrix) gpu.state.blend_set('ALPHA') def transform(x, y): return ( (x - 0.5) * 2 * ratio, (y - 0.5) * 2 ) shader = gpu.shader.from_builtin(UNIFORM_COLOR) batch = batch_for_shader(shader, 'TRI_STRIP', {"pos": [(-ratio, -1, 0), (-ratio, 1, 0), (ratio, -1, 0), (ratio, 1, 0)]}) shader.bind() shader.uniform_float("color", (0, 0, 0, 1)) batch.draw(shader) for object in (context.scene.objects if collection is None else collection.objects): object = object.evaluated_get(context) if object.hide_render: continue if object.pose is None: continue for connection, color in lines.items(): a, a_side = connection[0].identify(object.data, object.pose) b, b_side = connection[1].identify(object.data, object.pose) if a is None or b is None: continue a = bpy_extras.object_utils.world_to_camera_view(context.scene, context.scene.camera, object.matrix_world @ (a.tail if a_side == Side.TAIL else a.head)) b = bpy_extras.object_utils.world_to_camera_view(context.scene, context.scene.camera, object.matrix_world @ (b.tail if b_side == Side.TAIL else b.head)) draw_ellipse_2d(transform(a[0], a[1]), transform(b[0], b[1]), .015, 32, (color[0] / 255.0, color[1] / 255.0, color[2] / 255.0, 0.5)) for b in Bone: bone, side = b.identify(object.data, object.pose) color = b.color() if bone is None: continue tail = bpy_extras.object_utils.world_to_camera_view(context.scene, context.scene.camera, object.matrix_world @ (bone.tail if side == Side.TAIL else bone.head)) draw_circle_2d(transform(tail[0], tail[1]), .015, 16, (color[0] / 255.0, color[1] / 255.0, color[2] / 255.0, 0.5)) depth = np.array(fb.read_color(0, 0, width, height, 4, 0, 'FLOAT').to_list()) gpu.state.depth_test_set('NONE') offscreen.free() result = depth e.set() if threading.current_thread() == threading.main_thread(): _execute() return result else: bpy.app.timers.register(_execute, first_interval=0) e.wait() return result def draw_circle_2d(center, radius, segments, color): m = (1.0 / (segments - 1)) * (math.pi * 2) coords = [ ( center[0] + math.cos(m * p) * radius, center[1] + math.sin(m * p) * radius, 0 ) for p in range(segments) ] shader = gpu.shader.from_builtin(UNIFORM_COLOR) batch = batch_for_shader(shader, 'TRI_FAN', {"pos": coords}) shader.uniform_float("color", color) batch.draw(shader) def draw_ellipse_2d(start, end, thickness, segments, color): length = math.sqrt((end[0] - start[0]) ** 2 + (end[1] - start[1]) ** 2) theta = math.atan2(end[1] - start[1], end[0] - start[0]) center = ( (start[0] + end[0]) / 2, (start[1] + end[1]) / 2 ) major, minor = length / 2, thickness m = (1.0 / (segments - 1)) * (math.pi * 2) coords = [ ( center[0] + major * math.cos(m * p) * math.cos(theta) - minor * math.sin(m * p) * math.sin(theta), center[1] + major * math.cos(m * p) * math.sin(theta) + minor * math.sin(m * p) * math.cos(theta), 0 ) for p in range(segments) ] shader = gpu.shader.from_builtin(UNIFORM_COLOR) batch = batch_for_shader(shader, 'TRI_FAN', {"pos": coords}) shader.uniform_float("color", color) batch.draw(shader) ================================================ FILE: engine/annotations/viewport.py ================================================ import bpy import gpu import numpy as np import threading def render_viewport_color(context, width=None, height=None, matrix=None, projection_matrix=None, main_thread=False): e = threading.Event() result = None width, height = width or context.scene.render.resolution_x, height or context.scene.render.resolution_y matrix = matrix or context.scene.camera.matrix_world.inverted() projection_matrix = projection_matrix or context.scene.camera.calc_matrix_camera( context, x=width, y=height ) def _execute(): nonlocal result offscreen = gpu.types.GPUOffScreen(width, height) with offscreen.bind(): fb = gpu.state.active_framebuffer_get() fb.clear(color=(0.0, 0.0, 0.0, 0.0), depth=1) gpu.state.depth_test_set('LESS_EQUAL') gpu.state.depth_mask_set(True) with gpu.matrix.push_pop(): gpu.matrix.load_matrix(matrix) gpu.matrix.load_projection_matrix(projection_matrix) area = next(a for a in bpy.context.screen.areas if a.type == 'VIEW_3D') offscreen.draw_view3d( context.scene, context.view_layer, next(s for s in area.spaces), next(r for r in area.regions if r.type == 'WINDOW'), matrix, projection_matrix, do_color_management=False ) color = np.array(fb.read_color(0, 0, width, height, 4, 0, 'FLOAT').to_list()) gpu.state.depth_test_set('NONE') offscreen.free() result = color e.set() if main_thread or threading.current_thread() == threading.main_thread(): _execute() return result else: bpy.app.timers.register(_execute, first_interval=0) e.wait() return result ================================================ FILE: engine/engine.py ================================================ import bpy import gpu from bl_ui.properties_render import RenderButtonsPanel from bl_ui.properties_output import RenderOutputButtonsPanel from bl_ui.properties_view_layer import ViewLayerButtonsPanel import numpy as np from ..ui.panels.dream_texture import optimization_panels from .node_tree import DreamTexturesNodeTree from ..engine import node_executor from .annotations import depth from ..property_groups.dream_prompt import backend_options from .nodes.pipeline_nodes import NodeStableDiffusion from .nodes.input_nodes import NodeRenderProperties from ..generator_process import actor from .. import image_utils class DreamTexturesRenderEngine(bpy.types.RenderEngine): """A custom Dream Textures render engine, that uses Stable Diffusion and scene data to render images, instead of as a pass on top of Cycles.""" bl_idname = "DREAM_TEXTURES" bl_label = "Dream Textures" bl_use_preview = False bl_use_postprocess = True bl_use_gpu_context = actor.main_thread_rendering def __init__(self): pass def __del__(self): pass def render(self, depsgraph): scene = depsgraph.scene result = self.begin_result(0, 0, scene.render.resolution_x, scene.render.resolution_y) layer = result.layers[0].passes["Combined"] self.update_result(result) try: progress = 0 def node_begin(node): self.update_stats("Node", node.label or node.name) def node_update(response): if isinstance(response, np.ndarray): try: image_utils.np_to_render_pass(response, layer, top_to_bottom=False) self.update_result(result) except: pass def node_end(_): nonlocal progress progress += 1 self.update_progress(progress / len(scene.dream_textures_render_engine.node_tree.nodes)) group_outputs = node_executor.execute(scene.dream_textures_render_engine.node_tree, depsgraph, node_begin=node_begin, node_update=node_update, node_end=node_end, test_break=self.test_break) node_result = group_outputs[0][1] for k, v in group_outputs: if type(v) == int or type(v) == str or type(v) == float: self.get_result().stamp_data_add_field(k, str(v)) except Exception as error: self.report({'ERROR'}, str(error)) raise error image_utils.np_to_render_pass(node_result, layer, top_to_bottom=False) if "Depth" in result.layers[0].passes: z = depth.render_depth_map(depsgraph, invert=True) image_utils.np_to_render_pass(z, result.layers[0].passes["Depth"], top_to_bottom=False) self.end_result(result) def update_render_passes(self, scene=None, renderlayer=None): self.register_pass(scene, renderlayer, "Combined", 4, "RGBA", 'COLOR') self.register_pass(scene, renderlayer, "Depth", 1, "Z", 'VALUE') class NewEngineNodeTree(bpy.types.Operator): bl_idname = "dream_textures.new_engine_node_tree" bl_label = "New Node Tree" def execute(self, context): # TODO: Get the name of the default node tree from a config file # TODO: type is deprecated https://docs.blender.org/api/current/bpy.types.NodeTree.html # When testing the type of the resulting node tree using bpy.data.node_groups['Dream Textures Node Tree'].type it is '' because of that bpy.ops.node.new_node_tree(type=DreamTexturesNodeTree.bl_idname, name="Dream Textures Node Editor") # Get the newly generated node tree node_tree = bpy.data.node_groups[-1] node_sd = node_tree.nodes.new(type=NodeStableDiffusion.bl_idname) node_sd.location = (200, 200) node_out = node_tree.nodes.new(type="NodeGroupOutput") # Blender 4.0 uses a new API for in- and outputs if bpy.app.version[0] > 3: node_tree.interface.new_socket('Image', description="Output of the final image.", in_out='OUTPUT', socket_type='NodeSocketColor') else: node_tree.outputs.new('NodeSocketColor','Image') node_out.location = (400, 200) node_tree.links.new(node_sd.outputs['Image'], node_out.inputs['Image']) node_props = node_tree.nodes.new(type=NodeRenderProperties.bl_idname) node_props.location = (0,200) node_tree.links.new(node_props.outputs['Resolution X'], node_sd.inputs['Width']) node_tree.links.new(node_props.outputs['Resolution Y'], node_sd.inputs['Height']) # in case the node editor is open, synchronize the open node trees: for area in context.screen.areas: if area.type == 'NODE_EDITOR': if area.spaces.active.tree_type == DreamTexturesNodeTree.bl_idname: area.spaces.active.node_tree = node_tree return {'FINISHED'} def draw_device(self, context): scene = context.scene layout = self.layout layout.use_property_split = True layout.use_property_decorate = False if context.engine == DreamTexturesRenderEngine.bl_idname: layout.template_ID(scene.dream_textures_render_engine, "node_tree", text="Node Tree", new=NewEngineNodeTree.bl_idname) layout.prop(scene.dream_textures_render_engine, "backend") def _poll_node_tree(self, value): return value.bl_idname == "DreamTexturesNodeTree" def _update_engine_backend(self, context): if self.node_tree is not None: for node in self.node_tree.nodes: if node.bl_idname == NodeStableDiffusion.bl_idname: node.prompt.backend = self.backend context.scene.dream_textures_engine_prompt.backend = self.backend class DreamTexturesRenderEngineProperties(bpy.types.PropertyGroup): node_tree: bpy.props.PointerProperty(type=DreamTexturesNodeTree, name="Node Tree", poll=_poll_node_tree) backend: bpy.props.EnumProperty(name="Backend", items=backend_options, default=0, description="The backend to use for all pipeline nodes", update=_update_engine_backend) def engine_panels(): bpy.types.RENDER_PT_output.COMPAT_ENGINES.add(DreamTexturesRenderEngine.bl_idname) bpy.types.RENDER_PT_color_management.COMPAT_ENGINES.add(DreamTexturesRenderEngine.bl_idname) bpy.types.RENDER_PT_stamp.COMPAT_ENGINES.add(DreamTexturesRenderEngine.bl_idname) bpy.types.RENDER_PT_format.COMPAT_ENGINES.add(DreamTexturesRenderEngine.bl_idname) bpy.types.DATA_PT_lens.COMPAT_ENGINES.add(DreamTexturesRenderEngine.bl_idname) def get_prompt(context): return context.scene.dream_textures_engine_prompt class RenderPanel(bpy.types.Panel, RenderButtonsPanel): COMPAT_ENGINES = {DreamTexturesRenderEngine.bl_idname} def draw(self, context): self.layout.use_property_decorate = True class OutputPanel(bpy.types.Panel, RenderOutputButtonsPanel): COMPAT_ENGINES = {DreamTexturesRenderEngine.bl_idname} def draw(self, context): self.layout.use_property_decorate = True class ViewLayerPanel(bpy.types.Panel, ViewLayerButtonsPanel): COMPAT_ENGINES = {DreamTexturesRenderEngine.bl_idname} def draw(self, context): pass # Render Properties yield from optimization_panels(RenderPanel, 'engine', get_prompt, "") class NodeTreeInputsPanel(RenderPanel): """Create a subpanel for format options""" bl_idname = f"DREAM_PT_dream_panel_node_tree_inputs_engine" bl_label = "Inputs" def draw(self, context): super().draw(context) layout = self.layout layout.use_property_split = True node_tree = context.scene.dream_textures_render_engine.node_tree if node_tree is not None and hasattr(node_tree, "inputs"): for input in node_tree.inputs: layout.prop(input, "default_value", text=input.name) yield NodeTreeInputsPanel # View Layer class ViewLayerPassesPanel(ViewLayerPanel): bl_idname = "DREAM_PT_dream_panel_view_layer_passes" bl_label = "Passes" def draw(self, context): layout = self.layout layout.use_property_split = True layout.use_property_decorate = False view_layer = context.view_layer col = layout.column() col.prop(view_layer, "use_pass_combined") col.prop(view_layer, "use_pass_z") col.prop(view_layer, "use_pass_normal") yield ViewLayerPassesPanel # Bone properties class OpenPoseArmaturePanel(bpy.types.Panel): bl_idname = "DREAM_PT_dream_textures_armature_openpose" bl_label = "OpenPose" bl_space_type = 'PROPERTIES' bl_region_type = 'WINDOW' bl_context = "data" @classmethod def poll(cls, context): return context.armature def draw_header(self, context): bone = context.bone or context.edit_bone if bone: self.layout.prop(bone.dream_textures_openpose, "enabled", text="") def draw(self, context): layout = self.layout armature = context.armature p = armature.dream_textures_openpose row = layout.row() row.prop(p, "EAR_L", toggle=True) row.prop(p, "EYE_L", toggle=True) row.prop(p, "EYE_R", toggle=True) row.prop(p, "EAR_R", toggle=True) layout.prop(p, "NOSE", toggle=True) row = layout.row() row.prop(p, "SHOULDER_L", toggle=True) row.prop(p, "CHEST", toggle=True) row.prop(p, "SHOULDER_R", toggle=True) row = layout.row() row.prop(p, "ELBOW_L", toggle=True) row.separator() row.prop(p, "HIP_L", toggle=True) row.prop(p, "HIP_R", toggle=True) row.separator() row.prop(p, "ELBOW_R", toggle=True) row = layout.row() row.prop(p, "HAND_L", toggle=True) row.separator() row.prop(p, "KNEE_L", toggle=True) row.prop(p, "KNEE_R", toggle=True) row.separator() row.prop(p, "HAND_R", toggle=True) row = layout.row() row.prop(p, "FOOT_L", toggle=True) row.prop(p, "FOOT_R", toggle=True) yield OpenPoseArmaturePanel class OpenPoseBonePanel(bpy.types.Panel): bl_idname = "DREAM_PT_dream_textures_bone_openpose" bl_label = "OpenPose" bl_space_type = 'PROPERTIES' bl_region_type = 'WINDOW' bl_context = "bone" @classmethod def poll(cls, context): return context.bone and context.scene.render.engine == 'DREAM_TEXTURES' def draw_header(self, context): bone = context.bone if bone: self.layout.prop(bone.dream_textures_openpose, "enabled", text="") def draw(self, context): layout = self.layout layout.use_property_split = True bone = context.bone layout.enabled = bone.dream_textures_openpose.enabled layout.prop(bone.dream_textures_openpose, "bone") layout.prop(bone.dream_textures_openpose, "side") yield OpenPoseBonePanel class ADE20KObjectPanel(bpy.types.Panel): bl_idname = "DREAM_PT_dream_textures_object_ade20k" bl_label = "ADE20K Segmentation" bl_space_type = 'PROPERTIES' bl_region_type = 'WINDOW' bl_context = "object" @classmethod def poll(cls, context): return context.object and context.scene.render.engine == 'DREAM_TEXTURES' def draw_header(self, context): object = context.object if object: self.layout.prop(object.dream_textures_ade20k, "enabled", text="") def draw(self, context): layout = self.layout layout.use_property_split = True object = context.object layout.enabled = object.dream_textures_ade20k.enabled r = layout.split(factor=0.9) r.prop(object.dream_textures_ade20k, "annotation") c = r.column() c.enabled = False c.prop(object.dream_textures_ade20k, "color") yield ADE20KObjectPanel ================================================ FILE: engine/node.py ================================================ import bpy from .node_tree import DreamTexturesNodeTree class DreamTexturesNode(bpy.types.Node): @classmethod def poll(cls, tree): return tree.bl_idname == DreamTexturesNodeTree.bl_idname ================================================ FILE: engine/node_executor.py ================================================ import bpy # from dream_textures.engine import node_executor # node_executor.execute(bpy.data.node_groups["NodeTree"], bpy.context.evaluated_depsgraph_get()) class NodeExecutionContext: def __init__(self, depsgraph, start, update, end, test_break, cache={}): self.depsgraph = depsgraph self.start = start self.update = update self.end = end self.test_break = test_break self.cache = {} self.preferences = bpy.context.preferences def _evaluate_input(self, input): if input.is_linked: if len(input.links) > 1: return [ self.execute(link.from_socket.node)[link.from_socket.name] for link in input.links ] else: return self.execute(input.links[0].from_socket.node)[input.links[0].from_socket.name] else: return getattr(input, 'default_value', None) def execute(self, node): if self.test_break(): return None result = None match node.bl_idname: case 'dream_textures.node_switch': kwargs = { 'switch': self._evaluate_input(node.inputs[0]), 'false': lambda: self._evaluate_input(node.inputs[1]), 'true': lambda: self._evaluate_input(node.inputs[2]) } self.start(node) result = node.execute(self, **kwargs) case _: match node.type: case 'GROUP_INPUT': self.start(node) result = { input.name: input.default_value for input in self.depsgraph.scene.dream_textures_render_engine.node_tree.inputs } case 'GROUP_OUTPUT': self.start(node) result = [ (input.name, self.execute(input.links[0].from_socket.node)[input.links[0].from_socket.name]) for input in node.inputs if len(input.links) > 0 ] case 'FRAME': self.start(node) result = None case _: if node in self.cache: self.start(node) result = self.cache[node] else: kwargs = { input.name.lower().replace(' ', '_'): self._evaluate_input(input) for input in node.inputs } self.start(node) result = node.execute(self, **kwargs) self.end(node) return result def execute(node_tree, depsgraph, node_begin=lambda node: None, node_update=lambda result: None, node_end=lambda node: None, test_break=lambda: False): output = next(n for n in node_tree.nodes if n.type == 'GROUP_OUTPUT') context = NodeExecutionContext(depsgraph, node_begin, node_update, node_end, test_break) return context.execute(output) ================================================ FILE: engine/node_tree.py ================================================ import bpy class DreamTexturesNodeTree(bpy.types.NodeTree): bl_idname = "DreamTexturesNodeTree" bl_label = "Dream Textures Node Editor" bl_description = "Nodes for the Dream Textures Render Engine" bl_icon = 'NODETREE' @classmethod def poll(cls, context): return context.scene.render.engine == 'DREAM_TEXTURES' ================================================ FILE: engine/nodes/annotation_nodes.py ================================================ import bpy from ..node import DreamTexturesNode from ..annotations import depth from ..annotations import normal from ..annotations import openpose from ..annotations import ade20k from ..annotations import viewport import numpy as np annotation_src = ( ('collection', 'Collection', 'Render the annotation for a specific collection'), ('scene', 'Scene', 'Render the annotation for the entire scene'), ) def _update_annotation_inputs(self, context): inputs = {socket.name: socket for socket in self.inputs} inputs['Collection'].enabled = self.src == 'collection' class NodeAnnotationDepth(DreamTexturesNode): bl_idname = "dream_textures.node_annotation_depth" bl_label = "Depth Map" src: bpy.props.EnumProperty(name="", items=annotation_src, update=_update_annotation_inputs) def init(self, context): self.inputs.new("NodeSocketCollection", "Collection") self.inputs.new("NodeSocketBool", "Invert") self.outputs.new("NodeSocketColor", "Depth Map") def draw_buttons(self, context, layout): layout.prop(self, "src") def execute(self, context, collection, invert): depth_map = depth.render_depth_map(context.depsgraph, collection=collection if self.src == 'collection' else None, invert=invert) context.update(depth_map) return { 'Depth Map': depth_map, } class NodeAnnotationNormal(DreamTexturesNode): bl_idname = "dream_textures.node_annotation_normal" bl_label = "Normal Map" src: bpy.props.EnumProperty(name="", items=annotation_src, update=_update_annotation_inputs) def init(self, context): self.inputs.new("NodeSocketCollection", "Collection") self.outputs.new("NodeSocketColor", "Normal Map") def draw_buttons(self, context, layout): layout.prop(self, "src") def execute(self, context, collection): normal_map = normal.render_normal_map(context.depsgraph, collection=collection if self.src == 'collection' else None) context.update(normal_map) return { 'Normal Map': normal_map, } class NodeAnnotationOpenPose(DreamTexturesNode): bl_idname = "dream_textures.node_annotation_openpose" bl_label = "OpenPose Map" src: bpy.props.EnumProperty(name="", items=annotation_src, update=_update_annotation_inputs) def init(self, context): self.inputs.new("NodeSocketCollection", "Collection") self.outputs.new("NodeSocketColor", "OpenPose Map") def draw_buttons(self, context, layout): layout.prop(self, "src") def execute(self, context, collection): openpose_map = openpose.render_openpose_map(context.depsgraph, collection=collection if self.src == 'collection' else None) context.update(openpose_map) return { 'OpenPose Map': openpose_map } class NodeAnnotationADE20K(DreamTexturesNode): bl_idname = "dream_textures.node_annotation_ade20k" bl_label = "ADE20K Segmentation Map" src: bpy.props.EnumProperty(name="", items=annotation_src, update=_update_annotation_inputs) def init(self, context): self.inputs.new("NodeSocketCollection", "Collection") self.outputs.new("NodeSocketColor", "Segmentation Map") def draw_buttons(self, context, layout): layout.prop(self, "src") def execute(self, context, collection): ade20k_map = ade20k.render_ade20k_map(context.depsgraph, collection=collection if self.src == 'collection' else None) context.update(ade20k_map) return { 'Segmentation Map': ade20k_map } class NodeAnnotationViewport(DreamTexturesNode): bl_idname = "dream_textures.node_annotation_viewport" bl_label = "Viewport Color" def init(self, context): self.outputs.new("NodeSocketColor", "Viewport Color") def draw_buttons(self, context, layout): pass def execute(self, context): color = viewport.render_viewport_color(context.depsgraph) context.update(color) return { 'Viewport Color': color } ================================================ FILE: engine/nodes/input_nodes.py ================================================ import bpy import gpu from gpu_extras.batch import batch_for_shader import numpy as np from ..node import DreamTexturesNode from ..annotations import openpose from ..annotations import depth from ... import image_utils class NodeString(DreamTexturesNode): bl_idname = "dream_textures.node_string" bl_label = "String" value: bpy.props.StringProperty(name="") def init(self, context): self.outputs.new("NodeSocketString", "String") def draw_buttons(self, context, layout): layout.prop(self, "value") def execute(self, context): return { 'String': self.value } class NodeInteger(DreamTexturesNode): bl_idname = "dream_textures.node_integer" bl_label = "Integer" value: bpy.props.IntProperty(name="") def init(self, context): self.outputs.new("NodeSocketInt", "Integer") def draw_buttons(self, context, layout): layout.prop(self, "value") def execute(self, context): return { 'Integer': self.value } class NodeCollection(DreamTexturesNode): bl_idname = "dream_textures.node_collection" bl_label = "Collection" value: bpy.props.PointerProperty(type=bpy.types.Collection, name="") def init(self, context): self.outputs.new("NodeSocketCollection", "Collection") def draw_buttons(self, context, layout): layout.prop(self, "value") def execute(self, context): return { 'Collection': self.value } class NodeImage(DreamTexturesNode): bl_idname = "dream_textures.node_image" bl_label = "Image" value: bpy.props.PointerProperty(type=bpy.types.Image) def init(self, context): self.outputs.new("NodeSocketColor", "Image") def draw_buttons(self, context, layout): layout.template_ID(self, "value", open="image.open") if self.value is not None: layout.prop(self.value.colorspace_settings, "name", text="Color Space") def execute(self, context): result = image_utils.bpy_to_np(self.value, color_space="Linear", top_to_bottom=False) context.update(result) return { 'Image': result } class NodeImageFile(DreamTexturesNode): bl_idname = "dream_textures.node_image_file" bl_label = "Image File" def init(self, context): self.inputs.new("NodeSocketString", "Path") self.outputs.new("NodeSocketColor", "Image") def draw_buttons(self, context, layout): pass def execute(self, context, path): pixels = image_utils.image_to_np(path, default_color_space="sRGB", to_color_space="Linear", top_to_bottom=False) context.update(pixels) return { 'Image': pixels } class NodeRenderProperties(DreamTexturesNode): bl_idname = "dream_textures.node_render_properties" bl_label = "Render Properties" def init(self, context): self.outputs.new("NodeSocketInt", "Resolution X") self.outputs.new("NodeSocketInt", "Resolution Y") self.outputs.new("NodeSocketString", "Output Filepath") self.outputs.new("NodeSocketInt", "Frame") def draw_buttons(self, context, layout): pass def execute(self, context): return { 'Resolution X': context.depsgraph.scene.render.resolution_x, 'Resolution Y': context.depsgraph.scene.render.resolution_y, 'Output Filepath': context.depsgraph.scene.render.filepath, 'Frame': context.depsgraph.scene.frame_current } ================================================ FILE: engine/nodes/pipeline_nodes.py ================================================ import bpy import numpy as np from dataclasses import dataclass from typing import Any, List import enum from ..node import DreamTexturesNode from ...generator_process import Generator from ...property_groups.control_net import control_net_options from ...property_groups.dream_prompt import DreamPrompt from ..annotations import openpose from ..annotations import depth from ..annotations import normal from ..annotations import ade20k from ... import api from ...property_groups.seamless_result import SeamlessAxes import threading from ... import image_utils class NodeSocketControlNet(bpy.types.NodeSocket): bl_idname = "NodeSocketControlNet" bl_label = "ControlNet Socket" def __init__(self): self.link_limit = 0 def draw(self, context, layout, node, text): layout.label(text=text) def draw_color(self, context, node): return (0.63, 0.63, 0.63, 1) class ControlType(enum.IntEnum): DEPTH = 1 OPENPOSE = 2 NORMAL = 3 ADE20K_SEGMENTATION = 4 @dataclass class ControlNet: model: str image: Any collection: Any control_type: ControlType conditioning_scale: float def control(self, context): if self.image is not None: return np.flipud(self.image) else: match self.control_type: case ControlType.DEPTH: return np.flipud(depth.render_depth_map(context, collection=self.collection)) case ControlType.OPENPOSE: return np.flipud(openpose.render_openpose_map(context, collection=self.collection)) case ControlType.NORMAL: return np.flipud(normal.render_normal_map(context, collection=self.collection)) case ControlType.ADE20K_SEGMENTATION: return np.flipud(ade20k.render_ade20k_map(context, collection=self.collection)) def _update_stable_diffusion_sockets(self, context): inputs = {socket.name: socket for socket in self.inputs} inputs['Source Image'].enabled = self.task in {'image_to_image', 'depth_to_image', 'inpaint'} inputs['Noise Strength'].enabled = self.task in {'image_to_image', 'depth_to_image'} if self.task == 'depth_to_image': inputs['Noise Strength'].default_value = 1.0 inputs['Depth Map'].enabled = self.task == 'depth_to_image' inputs['ControlNets'].enabled = self.task != 'depth_to_image' class NodeStableDiffusion(DreamTexturesNode): bl_idname = "dream_textures.node_stable_diffusion" bl_label = "Stable Diffusion" prompt: bpy.props.PointerProperty(type=DreamPrompt) task: bpy.props.EnumProperty(name="", items=( ('prompt_to_image', 'Prompt to Image', '', 1), ('image_to_image', 'Image to Image', '', 2), ('depth_to_image', 'Depth to Image', '', 3), ('inpaint', 'Inpaint', '', 4), ), update=_update_stable_diffusion_sockets) def update(self): self.prompt.backend = bpy.context.scene.dream_textures_render_engine.backend def init(self, context): self.inputs.new("NodeSocketColor", "Depth Map") self.inputs.new("NodeSocketColor", "Source Image") self.inputs.new("NodeSocketFloat", "Noise Strength").default_value = 0.75 self.inputs.new("NodeSocketString", "Prompt") self.inputs.new("NodeSocketString", "Negative Prompt") self.inputs.new("NodeSocketInt", "Width").default_value = 512 self.inputs.new("NodeSocketInt", "Height").default_value = 512 self.inputs.new("NodeSocketInt", "Steps").default_value = 25 self.inputs.new("NodeSocketInt", "Seed") self.inputs.new("NodeSocketFloat", "CFG Scale").default_value = 7.50 self.inputs.new("NodeSocketControlNet", "ControlNets") self.outputs.new("NodeSocketColor", "Image") _update_stable_diffusion_sockets(self, context) def draw_buttons(self, context, layout): layout.prop(self, "task") prompt = self.prompt layout.prop(prompt, "model", text="") layout.prop(prompt, "scheduler", text="") layout.prop(prompt, "seamless_axes", text="") def execute(self, context, prompt, negative_prompt, width, height, steps, seed, cfg_scale, controlnets, depth_map, source_image, noise_strength): backend: api.Backend = self.prompt.get_backend() if np.array(source_image).shape == (4,): # the source image is a default color, ignore it. source_image = None else: source_image = image_utils.color_transform(np.flipud(source_image), "Linear", "sRGB") def get_task(): match self.task: case 'prompt_to_image': return api.PromptToImage() case 'image_to_image': return api.ImageToImage(source_image, noise_strength, fit=False) case 'depth_to_image': return api.DepthToImage(image_utils.grayscale(depth_map), source_image, noise_strength) case 'inpaint': return api.Inpaint(source_image, noise_strength, fit=False, mask_source=api.Inpaint.MaskSource.ALPHA, mask_prompt="", confidence=0) def map_controlnet(c): return api.models.control_net.ControlNet(c.model, c.control(context.depsgraph), c.conditioning_scale) args = api.GenerationArguments( get_task(), model=next(model for model in self.prompt.get_backend().list_models(context) if model is not None and model.id == self.prompt.model), prompt=api.Prompt( prompt, negative_prompt ), size=(width, height), seed=seed, steps=steps, guidance_scale=cfg_scale, scheduler=self.prompt.scheduler, seamless_axes=SeamlessAxes(self.prompt.seamless_axes), step_preview_mode=api.models.StepPreviewMode.FAST, iterations=1, control_nets=[map_controlnet(c) for c in controlnets] if isinstance(controlnets, list) else ([map_controlnet(controlnets)] if controlnets is not None else []) ) event = threading.Event() result = None exception = None def step_callback(progress: List[api.GenerationResult]) -> bool: context.update(image_utils.image_to_np(progress[-1].image, default_color_space="sRGB", to_color_space="Linear", top_to_bottom=False)) return True # if context.test_break(): # nonlocal result # result = [response] # event.set() def callback(results: List[api.GenerationResult] | Exception): if isinstance(results, Exception): nonlocal exception exception = results event.set() else: nonlocal result result = image_utils.image_to_np(results[-1].image, default_color_space="sRGB", to_color_space="Linear", top_to_bottom=False) event.set() backend = self.prompt.get_backend() backend.generate(args, step_callback=step_callback, callback=callback) event.wait() if exception is not None: raise exception return { 'Image': result } def _update_control_net_sockets(self, context): inputs = {socket.name: socket for socket in self.inputs} inputs['Collection'].enabled = self.input_type == 'collection' inputs['Image'].enabled = self.input_type == 'image' class NodeControlNet(DreamTexturesNode): bl_idname = "dream_textures.node_control_net" bl_label = "ControlNet" control_net: bpy.props.EnumProperty(name="", items=control_net_options) input_type: bpy.props.EnumProperty(name="", items=( ('collection', 'Collection', '', 1), ('image', 'Image', '', 2), ), update=_update_control_net_sockets) control_type: bpy.props.EnumProperty(name="", items=( ('DEPTH', 'Depth', '', 1), ('OPENPOSE', 'OpenPose', '', 2), ('NORMAL', 'Normal Map', '', 3), ('ADE20K_SEGMENTATION', 'ADE20K Segmentation', '', 4), )) def init(self, context): self.inputs.new("NodeSocketCollection", "Collection") self.inputs.new("NodeSocketColor", "Image") self.inputs.new("NodeSocketFloat", "Conditioning Scale").default_value = 1 self.outputs.new(NodeSocketControlNet.bl_idname, "Control") _update_control_net_sockets(self, context) def draw_buttons(self, context, layout): layout.prop(self, "control_net") layout.prop(self, "input_type") if self.input_type != 'image': layout.prop(self, "control_type") def execute(self, context, collection, image, conditioning_scale): return { 'Control': ControlNet( self.control_net, image if self.input_type == 'image' else None, collection if self.input_type == 'collection' else None, ControlType[self.control_type], conditioning_scale ) } ================================================ FILE: engine/nodes/utility_nodes.py ================================================ import bpy import numpy as np import random from ..node import DreamTexturesNode from ...property_groups.dream_prompt import seed_clamp from ... import image_utils class NodeMath(DreamTexturesNode): bl_idname = "dream_textures.node_math" bl_label = "Math" operation: bpy.props.EnumProperty( name="Operation", items=( ("add", "Add", ""), ("subtract", "Subtract", ""), ("multiply", "Multiply", ""), ("divide", "Divide", ""), ) ) def init(self, context): self.inputs.new("NodeSocketFloat", "A") self.inputs.new("NodeSocketFloat", "B") self.outputs.new("NodeSocketFloat", "Value") def draw_buttons(self, context, layout): layout.prop(self, "operation", text="") def perform(self, a, b): match self.operation: case 'add': return a + b case 'subtract': return a - b case 'multiply': return a * b case 'divide': return a / b def execute(self, context, a, b): return { 'Value': self.perform(a, b) } class NodeRandomValue(DreamTexturesNode): bl_idname = "dream_textures.node_random_value" bl_label = "Random Value" data_type: bpy.props.EnumProperty(name="", items=( ('integer', 'Integer', '', 1), )) def init(self, context): self.inputs.new("NodeSocketInt", "Min") self.inputs.new("NodeSocketInt", "Max").default_value = np.iinfo(np.int32).max self.outputs.new("NodeSocketInt", "Value") def draw_buttons(self, context, layout): layout.prop(self, "data_type") def execute(self, context, min, max): return { 'Value': random.randrange(min, max) } class NodeRandomSeed(DreamTexturesNode): bl_idname = "dream_textures.node_random_seed" bl_label = "Random Seed" def init(self, context): self.outputs.new("NodeSocketInt", "Value") def draw_buttons(self, context, layout): pass def execute(self, context): return { 'Value': random.randrange(0, np.iinfo(np.uint32).max) } class NodeSeed(DreamTexturesNode): bl_idname = "dream_textures.node_seed" bl_label = "Seed" seed: bpy.props.StringProperty(name="", default="", update=seed_clamp) def init(self, context): self.outputs.new("NodeSocketInt", "Value") def draw_buttons(self, context, layout): layout.prop(self, "seed") def execute(self, context): return { 'Value': int(self.seed) } class NodeClamp(DreamTexturesNode): bl_idname = "dream_textures.node_clamp" bl_label = "Clamp" def init(self, context): self.inputs.new("NodeSocketFloat", "Value") self.inputs.new("NodeSocketFloat", "Min") self.inputs.new("NodeSocketFloat", "Max") self.outputs.new("NodeSocketFloat", "Result") def draw_buttons(self, context, layout): pass def execute(self, context, value, min, max): return { 'Result': np.clip(value, min, max) } class NodeFramePath(DreamTexturesNode): bl_idname = "dream_textures.node_frame_path" bl_label = "Frame Path" def init(self, context): self.inputs.new("NodeSocketInt", "Frame") self.outputs.new("NodeSocketString", "Frame Path") def draw_buttons(self, context, layout): pass def execute(self, context, frame): return { 'Frame Path': context.depsgraph.scene.render.frame_path(frame=int(frame)), } class NodeCropImage(DreamTexturesNode): bl_idname = "dream_textures.node_crop_image" bl_label = "Crop Image" def init(self, context): self.inputs.new("NodeSocketColor", "Image") self.inputs.new("NodeSocketInt", "X") self.inputs.new("NodeSocketInt", "Y") self.inputs.new("NodeSocketInt", "Width") self.inputs.new("NodeSocketInt", "Height") self.outputs.new("NodeSocketColor", "Cropped Image") def draw_buttons(self, context, layout): pass def execute(self, context, image, x, y, width, height): x, y = int(x), int(y) width, height = int(width), int(height) result = image[y:y+height, x:x+width, ...] context.update(result) return { 'Cropped Image': result, } class NodeResizeImage(DreamTexturesNode): bl_idname = "dream_textures.node_resize_image" bl_label = "Resize Image" def init(self, context): self.inputs.new("NodeSocketColor", "Image") self.inputs.new("NodeSocketInt", "Width") self.inputs.new("NodeSocketInt", "Height") self.outputs.new("NodeSocketColor", "Resized Image") def draw_buttons(self, context, layout): pass def execute(self, context, image, width, height): result = image_utils.resize(image, (width, height)) context.update(result) return { 'Resized Image': result, } class NodeJoinImages(DreamTexturesNode): bl_idname = "dream_textures.node_join_images" bl_label = "Join Images" direction: bpy.props.EnumProperty(name="", items=( ('horizontal', 'Horizontal', ''), ('vertical', 'Vertical', ''), )) def init(self, context): self.inputs.new("NodeSocketColor", "A") self.inputs.new("NodeSocketColor", "B") self.outputs.new("NodeSocketColor", "Joined Images") def draw_buttons(self, context, layout): layout.prop(self, "direction") def execute(self, context, a, b): match self.direction: case 'horizontal': result = np.hstack([a, b]) case 'vertical': result = np.vstack([a, b]) context.update(result) return { 'Joined Images': result, } class NodeSeparateColor(DreamTexturesNode): bl_idname = "dream_textures.node_separate_color" bl_label = "Separate Color" def init(self, context): self.inputs.new("NodeSocketColor", "Color") self.outputs.new("NodeSocketFloat", "Red") self.outputs.new("NodeSocketFloat", "Green") self.outputs.new("NodeSocketFloat", "Blue") self.outputs.new("NodeSocketFloat", "Alpha") def draw_buttons(self, context, layout): pass def execute(self, context, color): return { 'Red': color[..., 0], 'Green': color[..., 1] if color.shape[-1] > 1 else 0, 'Blue': color[..., 2] if color.shape[-1] > 2 else 0, 'Alpha': color[..., 3] if color.shape[-1] > 3 else 0, } class NodeCombineColor(DreamTexturesNode): bl_idname = "dream_textures.node_combine_color" bl_label = "Combine Color" def init(self, context): self.inputs.new("NodeSocketFloat", "Red") self.inputs.new("NodeSocketFloat", "Green") self.inputs.new("NodeSocketFloat", "Blue") self.inputs.new("NodeSocketFloat", "Alpha") self.outputs.new("NodeSocketColor", "Color") def draw_buttons(self, context, layout): pass def execute(self, context, red, green, blue, alpha): return { 'Color': np.stack([red, green, blue, alpha], axis=-1) } class NodeColorCorrect(DreamTexturesNode): bl_idname = "dream_textures.node_color_correct" bl_label = "Color Correct" mode: bpy.props.EnumProperty(name="Mode", items=( ('histogram', 'Match Histograms', ''), )) def init(self, context): self.inputs.new("NodeSocketColor", "Image") self.inputs.new("NodeSocketColor", "Target") self.outputs.new("NodeSocketColor", "Image") def draw_buttons(self, context, layout): layout.prop(self, "mode", text="") def execute(self, context, image, target): match self.mode: case 'histogram': flat_image = image.ravel() flat_target = target.ravel() _, image_indices, image_counts = np.unique(flat_image, return_inverse=True, return_counts=True) target_values, target_counts = np.unique(flat_target, return_counts=True) image_quantiles = np.cumsum(image_counts).astype(np.float64) image_quantiles /= image_quantiles[-1] target_quantiles = np.cumsum(target_counts).astype(np.float64) target_quantiles /= target_quantiles[-1] result = np.interp(image_quantiles, target_quantiles, target_values)[image_indices].reshape(image.shape) return { 'Image': result } class NodeSwitch(DreamTexturesNode): bl_idname = "dream_textures.node_switch" bl_label = "Switch" def init(self, context): self.inputs.new("NodeSocketBool", "Switch") self.inputs.new("NodeSocketColor", "False") self.inputs.new("NodeSocketColor", "True") self.outputs.new("NodeSocketColor", "Output") def draw_buttons(self, context, layout): pass def execute(self, context, switch, false, true): return { 'Output': true() if switch else false() } class NodeCompare(DreamTexturesNode): bl_idname = "dream_textures.node_compare" bl_label = "Compare" operation: bpy.props.EnumProperty(name="", items=( ('<', 'Less Than', ''), ('<=', 'Less Than or Equal', ''), ('>', 'Greater Than', ''), ('>=', 'Greater Than or Equal', ''), ('==', 'Equal', ''), ('!=', 'Not Equal', ''), )) def init(self, context): self.inputs.new("NodeSocketFloat", "A") self.inputs.new("NodeSocketFloat", "B") self.outputs.new("NodeSocketBool", "Result") def draw_buttons(self, context, layout): layout.prop(self, "operation") def execute(self, context, a, b): match self.operation: case '<': result = a < b case '<=': result = a <= b case '>': result = a > b case '>=': result = a >= b case '==': result = a == b case '!=': result = a != b return { 'Result': result } class NodeReplaceString(DreamTexturesNode): bl_idname = "dream_textures.node_replace_string" bl_label = "Replace String" def init(self, context): self.inputs.new("NodeSocketString", "String") self.inputs.new("NodeSocketString", "Find") self.inputs.new("NodeSocketString", "Replace") self.outputs.new("NodeSocketString", "String") def draw_buttons(self, context, layout): pass def execute(self, context, string, find, replace): return { 'String': string.replace(find, replace) } ================================================ FILE: generator_process/__init__.py ================================================ from typing import Callable from .actor import Actor, is_actor_process class RunInSubprocess(Exception): """ Decorators to support running functions that are not defined under the Generator class in its subprocess. This is to reduce what would otherwise be duplicate function definitions that logically don't belong to the Generator, but require something in its subprocess (such as access to installed dependencies). """ def __new__(cls, func=None): if func is None: # support `raise RunInSubprocess` return super().__new__(cls) return cls.always(func) @staticmethod def always(func): if is_actor_process: return func def wrapper(*args, **kwargs): return Generator.shared().call(wrapper, *args, **kwargs).result() RunInSubprocess._copy_attributes(func, wrapper) return wrapper @staticmethod def when(condition: bool | Callable[..., bool]): if not isinstance(condition, Callable): if condition: return RunInSubprocess.always return lambda x: x def decorator(func): if is_actor_process: return func def wrapper(*args, **kwargs): if condition(*args, **kwargs): return Generator.shared().call(wrapper, *args, **kwargs).result() return func(*args, **kwargs) RunInSubprocess._copy_attributes(func, wrapper) return wrapper return decorator @staticmethod def when_raised(func): if is_actor_process: return func def wrapper(*args, **kwargs): try: return func(*args, **kwargs) except RunInSubprocess: return Generator.shared().call(wrapper, *args, **kwargs).result() RunInSubprocess._copy_attributes(func, wrapper) return wrapper @staticmethod def _copy_attributes(src, dst): for n in ["__annotations__", "__doc__", "__name__", "__module__", "__qualname__"]: if hasattr(src, n): setattr(dst, n, getattr(src, n)) class Generator(Actor): """ The actor used for all background processes. """ from .actions.choose_device import choose_device from .actions.load_model import load_model from .actions.prompt_to_image import prompt_to_image from .actions.image_to_image import image_to_image from .actions.inpaint import inpaint from .actions.outpaint import outpaint from .actions.upscale import upscale from .actions.depth_to_image import depth_to_image from .actions.control_net import control_net from .actions.huggingface_hub import hf_snapshot_download, hf_list_models, hf_list_installed_models from .actions.convert_original_stable_diffusion_to_diffusers import convert_original_stable_diffusion_to_diffusers from .actions.detect_seamless import detect_seamless from .actions.controlnet_aux import controlnet_aux @staticmethod def call(func, *args, **kwargs): return func(*args, **kwargs) ================================================ FILE: generator_process/actions/choose_device.py ================================================ import importlib.util import sys def choose_device(self, optimizations) -> str: """ Automatically select which PyTorch device to use. """ if optimizations.cpu_only: return "cpu" import torch if torch.cuda.is_available(): return "cuda" elif torch.backends.mps.is_available(): return "mps" elif importlib.util.find_spec("torch_directml"): import torch_directml if torch_directml.is_available(): torch.utils.rename_privateuse1_backend("dml") return "dml" return "cpu" ================================================ FILE: generator_process/actions/control_net.py ================================================ from typing import Union, Generator, Callable, List, Optional, Dict, Any from contextlib import nullcontext import numpy as np import logging import os import random from .prompt_to_image import Checkpoint, Scheduler, Optimizations, StepPreviewMode, step_latents, step_images, _configure_model_padding from ...api.models.seamless_axes import SeamlessAxes from ..future import Future from ...image_utils import image_to_np, rgb, resize, ImageOrPath def control_net( self, model: str | Checkpoint, scheduler: str | Scheduler, optimizations: Optimizations, control_net: list[str | Checkpoint], control: list[ImageOrPath] | None, controlnet_conditioning_scale: list[float], image: ImageOrPath | None, # image to image # inpaint inpaint: bool, inpaint_mask_src: str, text_mask: str, text_mask_confidence: float, strength: float, prompt: str | list[str], steps: int, seed: int, width: int | None, height: int | None, cfg_scale: float, use_negative_prompt: bool, negative_prompt: str, seamless_axes: SeamlessAxes | str | bool | tuple[bool, bool] | None, step_preview_mode: StepPreviewMode, **kwargs ) -> Generator[Future, None, None]: future = Future() yield future import diffusers import torch device = self.choose_device(optimizations) # StableDiffusionPipeline w/ caching if image is not None: if inpaint: pipe = self.load_model(diffusers.AutoPipelineForInpainting, model, optimizations, scheduler, controlnet=control_net) else: pipe = self.load_model(diffusers.AutoPipelineForImage2Image, model, optimizations, scheduler, controlnet=control_net) else: pipe = self.load_model(diffusers.AutoPipelineForText2Image, model, optimizations, scheduler, controlnet=control_net) # Optimizations pipe = optimizations.apply(pipe, device) # RNG batch_size = len(prompt) if isinstance(prompt, list) else 1 generator = [] for _ in range(batch_size): gen = torch.Generator(device="cpu" if device in ("mps", "dml") else device) # MPS and DML do not support the `Generator` API generator.append(gen.manual_seed(random.randrange(0, np.iinfo(np.uint32).max) if seed is None else seed)) if batch_size == 1: # Some schedulers don't handle a list of generators: https://github.com/huggingface/diffusers/issues/1909 generator = generator[0] # Init Image # FIXME: The `unet.config.sample_size` of the depth model is `32`, not `64`. For now, this will be hardcoded to `512`. height = height or 512 width = width or 512 rounded_size = ( int(8 * (width // 8)), int(8 * (height // 8)), ) # StableDiffusionControlNetPipeline.check_image() currently fails without adding batch dimension control_image = None if control is None else [image_to_np(c, mode="RGB", size=rounded_size)[np.newaxis] for c in control] image = image_to_np(image, size=rounded_size) if inpaint: match inpaint_mask_src: case 'alpha': mask_image = 1-image[..., -1] image = rgb(image) case 'prompt': image = rgb(image) from transformers import AutoProcessor, CLIPSegForImageSegmentation processor = AutoProcessor.from_pretrained("CIDAS/clipseg-rd64-refined", do_rescale=False) clipseg = CLIPSegForImageSegmentation.from_pretrained("CIDAS/clipseg-rd64-refined") inputs = processor(text=[text_mask], images=[image], return_tensors="pt", padding=True) outputs = clipseg(**inputs) mask_image = (torch.sigmoid(outputs.logits) >= text_mask_confidence).detach().numpy().astype(np.float32) mask_image = resize(mask_image, (width, height)) else: mask_image = None # Seamless if seamless_axes == SeamlessAxes.AUTO: init_sa = None if image is None else self.detect_seamless(image) control_sa = None if control_image is None else self.detect_seamless(control_image[0][0]) if init_sa is not None and control_sa is not None: seamless_axes = init_sa & control_sa elif init_sa is not None: seamless_axes = init_sa elif control_sa is not None: seamless_axes = control_sa _configure_model_padding(pipe.unet, seamless_axes) _configure_model_padding(pipe.vae, seamless_axes) # Inference with (torch.inference_mode() if device not in ('mps', "dml") else nullcontext()), \ (torch.autocast(device) if optimizations.can_use("amp", device) else nullcontext()): def callback(pipe, step, timestep, callback_kwargs): if future.check_cancelled(): raise InterruptedError() future.add_response(step_latents(pipe, step_preview_mode, callback_kwargs["latents"], generator, step, steps)) return callback_kwargs try: if image is not None: if mask_image is not None: result = pipe( prompt=prompt, negative_prompt=negative_prompt if use_negative_prompt else None, control_image=control_image, controlnet_conditioning_scale=controlnet_conditioning_scale, image=image, mask_image=mask_image, strength=strength, width=rounded_size[0], height=rounded_size[1], num_inference_steps=steps, guidance_scale=cfg_scale, generator=generator, callback_on_step_end=callback, callback_steps=1, output_type="np" ) else: result = pipe( prompt=prompt, negative_prompt=negative_prompt if use_negative_prompt else None, control_image=control_image, controlnet_conditioning_scale=controlnet_conditioning_scale, image=image, strength=strength, width=rounded_size[0], height=rounded_size[1], num_inference_steps=steps, guidance_scale=cfg_scale, generator=generator, callback_on_step_end=callback, callback_steps=1, output_type="np" ) else: result = pipe( prompt=prompt, negative_prompt=negative_prompt if use_negative_prompt else None, image=control_image, controlnet_conditioning_scale=controlnet_conditioning_scale, width=rounded_size[0], height=rounded_size[1], num_inference_steps=steps, guidance_scale=cfg_scale, generator=generator, callback_on_step_end=callback, callback_steps=1, output_type="np" ) future.add_response(step_images(result.images, generator, steps, steps)) except InterruptedError: pass future.set_done() ================================================ FILE: generator_process/actions/controlnet_aux.py ================================================ import numpy as np from numpy.typing import NDArray from ..models.optimizations import Optimizations from ...image_utils import np_to_pil def controlnet_aux( self, processor_id: str, image: NDArray, optimizations: Optimizations, **kwargs ) -> NDArray: if processor_id == "none": return image from controlnet_aux.processor import Processor processor = Processor(processor_id) device = self.choose_device(optimizations) try: processor.processor.to(device) except: # not all processors can run on the GPU pass processed_image = processor(np_to_pil(image)) return np.array(processed_image) / 255.0 ================================================ FILE: generator_process/actions/convert_original_stable_diffusion_to_diffusers.py ================================================ import os from .huggingface_hub import DownloadStatus from ..future import Future from ..models import ModelConfig def convert_original_stable_diffusion_to_diffusers( self, checkpoint_path: str, model_config: ModelConfig, half_precision: bool, ) -> str: import torch from huggingface_hub.constants import HF_HUB_CACHE from diffusers.pipelines.stable_diffusion.convert_from_ckpt import download_from_original_stable_diffusion_ckpt, download_controlnet_from_original_ckpt future = Future() yield future DownloadStatus.hook_download_tqdm(future) future.add_response(DownloadStatus(f"Reading {checkpoint_path}", 0, 1)) index = 0 def hook_save_pretrained(model, dirs_count, total): old_save_pretrained = model.save_pretrained def save_pretrained(self, save_directory, *args, **kwargs): nonlocal index dirs = [] directory = save_directory for _ in range(dirs_count): dirs.append(os.path.basename(directory)) directory = os.path.dirname(directory) dirs.reverse() future.add_response(DownloadStatus(f"Saving {os.path.join(*dirs)}", index, total)) index += 1 return old_save_pretrained(save_directory, *args, **kwargs) model.save_pretrained = save_pretrained.__get__(model) if model_config in [ModelConfig.CONTROL_NET_1_5, ModelConfig.CONTROL_NET_2_1]: pipe = download_controlnet_from_original_ckpt( checkpoint_path, original_config_file=model_config.original_config, from_safetensors=checkpoint_path.endswith(".safetensors"), ) if half_precision: pipe.to(dtype=torch.float16) index = 1 hook_save_pretrained(pipe, 1, 2) else: pipe = download_from_original_stable_diffusion_ckpt( checkpoint_path, original_config_file=model_config.original_config, from_safetensors=checkpoint_path.endswith(".safetensors"), pipeline_class=model_config.pipeline ) if half_precision: pipe.to(torch_dtype=torch.float16) models = [] for name in pipe._get_signature_keys(pipe)[0]: model = getattr(pipe, name, None) if model is not None and hasattr(model, "save_pretrained"): models.append(model) for i, model in enumerate(models): hook_save_pretrained(model, 2, len(models)) dump_path = os.path.join(HF_HUB_CACHE, os.path.splitext(os.path.basename(checkpoint_path))[0]) pipe.save_pretrained(dump_path, variant="fp16" if half_precision else None) future.set_done() ================================================ FILE: generator_process/actions/depth_to_image.py ================================================ from typing import Union, Generator, Callable, List, Optional import os from contextlib import nullcontext import numpy as np import random from .prompt_to_image import Checkpoint, Scheduler, Optimizations, StepPreviewMode, step_latents, step_images, _configure_model_padding from ...api.models.seamless_axes import SeamlessAxes from ..future import Future from ...image_utils import image_to_np, ImageOrPath def depth_to_image( self, model: str | Checkpoint, scheduler: str | Scheduler, optimizations: Optimizations, depth: ImageOrPath | None, image: ImageOrPath | None, strength: float, prompt: str | list[str], steps: int, seed: int, width: int | None, height: int | None, cfg_scale: float, use_negative_prompt: bool, negative_prompt: str, seamless_axes: SeamlessAxes | str | bool | tuple[bool, bool] | None, step_preview_mode: StepPreviewMode, **kwargs ) -> Generator[Future, None, None]: future = Future() yield future import diffusers import torch import PIL.Image class DreamTexturesDepth2ImgPipeline(diffusers.StableDiffusionInpaintPipeline): def prepare_depth(self, depth, image, dtype, device): device = torch.device('cpu' if device.type == 'mps' else device.type) if depth is None: from transformers import DPTFeatureExtractor, DPTForDepthEstimation import contextlib feature_extractor = DPTFeatureExtractor.from_pretrained("Intel/dpt-large") depth_estimator = DPTForDepthEstimation.from_pretrained("Intel/dpt-large") depth_estimator = depth_estimator.to(device) pixel_values = feature_extractor(images=image, return_tensors="pt", do_rescale=False).pixel_values pixel_values = pixel_values.to(device=device) # The DPT-Hybrid model uses batch-norm layers which are not compatible with fp16. # So we use `torch.autocast` here for half precision inference. context_manger = torch.autocast("cuda", dtype=dtype) if device.type == "cuda" else contextlib.nullcontext() with context_manger: depth_map = depth_estimator(pixel_values).predicted_depth depth_map = torch.nn.functional.interpolate( depth_map.unsqueeze(1), size=(height // self.vae_scale_factor, width // self.vae_scale_factor), mode="bicubic", align_corners=False, ) depth_min = torch.amin(depth_map, dim=[1, 2, 3], keepdim=True) depth_max = torch.amax(depth_map, dim=[1, 2, 3], keepdim=True) depth_map = 2.0 * (depth_map - depth_min) / (depth_max - depth_min) - 1.0 depth_map = depth_map.to(device) return depth_map else: if isinstance(depth, PIL.Image.Image): depth = np.array(depth.convert("L")) depth = depth.astype(np.float32) / 255.0 depth = depth[None, None] depth = torch.from_numpy(depth) return depth def prepare_depth_latents( self, depth, batch_size, height, width, dtype, device, generator, do_classifier_free_guidance ): # resize the mask to latents shape as we concatenate the mask to the latents # we do that before converting to dtype to avoid breaking in case we're using cpu_offload # and half precision depth = torch.nn.functional.interpolate( depth, size=(height // self.vae_scale_factor, width // self.vae_scale_factor) ) depth = depth.to(device=device, dtype=dtype) # duplicate mask and masked_image_latents for each generation per prompt, using mps friendly method depth = depth.repeat(batch_size, 1, 1, 1) depth = torch.cat([depth] * 2) if do_classifier_free_guidance else depth return depth def prepare_img2img_latents(self, batch_size, num_channels_latents, height, width, dtype, device, generator, latents=None, image=None, timestep=None): shape = (batch_size, num_channels_latents, height // self.vae_scale_factor, width // self.vae_scale_factor) if isinstance(generator, list) and len(generator) != batch_size: raise ValueError( f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" f" size of {batch_size}. Make sure the batch size matches the length of the generators." ) if latents is None: rand_device = "cpu" if device.type == "mps" else device if isinstance(generator, list): shape = (1,) + shape[1:] latents = [ torch.randn(shape, generator=generator[i], device=rand_device, dtype=dtype) for i in range(batch_size) ] latents = torch.cat(latents, dim=0).to(device) else: latents = torch.randn(shape, generator=generator, device=rand_device, dtype=dtype).to(device) else: if latents.shape != shape: raise ValueError(f"Unexpected latents shape, got {latents.shape}, expected {shape}") latents = latents.to(device) # scale the initial noise by the standard deviation required by the scheduler latents = latents * self.scheduler.init_noise_sigma if image is not None: image = image.to(device=device, dtype=dtype) if isinstance(generator, list): image_latents = [ self.vae.encode(image[0:1]).latent_dist.sample(generator[i]) for i in range(batch_size) ] image_latents = torch.cat(image_latents, dim=0) else: image_latents = self.vae.encode(image).latent_dist.sample(generator) image_latents = torch.nn.functional.interpolate( image_latents, size=(height // self.vae_scale_factor, width // self.vae_scale_factor) ) image_latents = 0.18215 * image_latents rand_device = "cpu" if device.type == "mps" else device shape = image_latents.shape if isinstance(generator, list): shape = (1,) + shape[1:] noise = [ torch.randn(shape, generator=generator[i], device=rand_device, dtype=dtype) for i in range(batch_size) ] noise = torch.cat(noise, dim=0).to(device) else: noise = torch.randn(shape, generator=generator, device=rand_device, dtype=dtype).to(device) latents = self.scheduler.add_noise(image_latents, noise, timestep) return latents def get_timesteps(self, num_inference_steps, strength, device): # get the original timestep using init_timestep offset = self.scheduler.config.get("steps_offset", 0) init_timestep = int(num_inference_steps * strength) + offset init_timestep = min(init_timestep, num_inference_steps) t_start = max(num_inference_steps - init_timestep + offset, 0) timesteps = self.scheduler.timesteps[t_start:] return timesteps, num_inference_steps - t_start @torch.no_grad() def __call__( self, prompt: Union[str, List[str]], depth_image: Union[torch.FloatTensor, PIL.Image.Image], image: Optional[Union[torch.FloatTensor, PIL.Image.Image]] = None, strength: float = 0.8, height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: int = 50, guidance_scale: float = 7.5, negative_prompt: Optional[Union[str, List[str]]] = None, num_images_per_prompt: Optional[int] = 1, eta: float = 0.0, generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, latents: Optional[torch.FloatTensor] = None, output_type: Optional[str] = "pil", return_dict: bool = True, callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None, callback_steps: Optional[int] = 1, **kwargs, ): # 0. Default height and width to unet height = height or self.unet.config.sample_size * self.vae_scale_factor width = width or self.unet.config.sample_size * self.vae_scale_factor # 1. Check inputs self.check_inputs(prompt=prompt, image=image, mask_image=depth_image, height=height, width=width, strength=strength, callback_steps=callback_steps, output_type=output_type) # 2. Define call parameters batch_size = 1 if isinstance(prompt, str) else len(prompt) device = self._execution_device # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` # corresponds to doing no classifier free guidance. do_classifier_free_guidance = guidance_scale > 1.0 # 3. Encode input prompt text_embeddings = self._encode_prompt( prompt, device, num_images_per_prompt, do_classifier_free_guidance, negative_prompt ) # 4. Prepare the depth image depth = self.prepare_depth(depth_image, image, text_embeddings.dtype, device) if image is not None: image = self.image_processor.preprocess(image) # 5. set timesteps self.scheduler.set_timesteps(num_inference_steps, device=device) timesteps = self.scheduler.timesteps if image is not None: timesteps, num_inference_steps = self.get_timesteps(num_inference_steps, strength, device) # 6. Prepare latent variables num_channels_latents = self.vae.config.latent_channels if image is not None: latent_timestep = timesteps[:1].repeat(batch_size * num_images_per_prompt) latents = self.prepare_img2img_latents( batch_size * num_images_per_prompt, num_channels_latents, height, width, text_embeddings.dtype, device, generator, latents, image, latent_timestep ) else: latents = self.prepare_latents( batch_size * num_images_per_prompt, num_channels_latents, height, width, text_embeddings.dtype, device, generator, latents, )[0] # 7. Prepare mask latent variables depth = self.prepare_depth_latents( depth, batch_size * num_images_per_prompt, height, width, text_embeddings.dtype, device, generator, do_classifier_free_guidance, ) # 8. Check that sizes of mask, masked image and latents match num_channels_depth = depth.shape[1] if num_channels_latents + num_channels_depth != self.unet.config.in_channels: raise ValueError( f"Select a depth model, such as 'stabilityai/stable-diffusion-2-depth'" ) # 9. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) # 10. Denoising loop num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order with self.progress_bar(total=num_inference_steps) as progress_bar: for i, t in enumerate(timesteps): # expand the latents if we are doing classifier free guidance latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents # concat latents, mask, masked_image_latents in the channel dimension latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) latent_model_input = torch.cat([latent_model_input, depth], dim=1) # predict the noise residual noise_pred = self.unet(latent_model_input, t, encoder_hidden_states=text_embeddings).sample # perform guidance if do_classifier_free_guidance: noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) # compute the previous noisy sample x_t -> x_t-1 latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs).prev_sample # call the callback, if provided if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): progress_bar.update() if callback is not None and i % callback_steps == 0: callback(i, t, latents) if not output_type == "latent": condition_kwargs = {} image = self.vae.decode(latents / self.vae.config.scaling_factor, return_dict=False, **condition_kwargs)[0] image, has_nsfw_concept = self.run_safety_checker(image, device, text_embeddings.dtype) else: image = latents has_nsfw_concept = None if has_nsfw_concept is None: do_denormalize = [True] * image.shape[0] else: do_denormalize = [not has_nsfw for has_nsfw in has_nsfw_concept] image = self.image_processor.postprocess(image, output_type=output_type, do_denormalize=do_denormalize) # Offload last model to CPU if hasattr(self, "final_offload_hook") and self.final_offload_hook is not None: self.final_offload_hook.offload() if not return_dict: return (image, has_nsfw_concept) return diffusers.pipelines.stable_diffusion.StableDiffusionPipelineOutput(images=image, nsfw_content_detected=has_nsfw_concept) device = self.choose_device(optimizations) # StableDiffusionPipeline w/ caching pipe = self.load_model(DreamTexturesDepth2ImgPipeline, model, optimizations, scheduler) # Optimizations pipe = optimizations.apply(pipe, device) # RNG batch_size = len(prompt) if isinstance(prompt, list) else 1 generator = [] for _ in range(batch_size): gen = torch.Generator(device="cpu" if device in ("mps", "dml") else device) # MPS and DML do not support the `Generator` API generator.append(gen.manual_seed(random.randrange(0, np.iinfo(np.uint32).max) if seed is None else seed)) if batch_size == 1: # Some schedulers don't handle a list of generators: https://github.com/huggingface/diffusers/issues/1909 generator = generator[0] # Init Image # FIXME: The `unet.config.sample_size` of the depth model is `32`, not `64`. For now, this will be hardcoded to `512`. height = height or 512 width = width or 512 rounded_size = ( int(8 * (width // 8)), int(8 * (height // 8)), ) depth = image_to_np(depth, mode="L", size=rounded_size, to_color_space=None) image = image_to_np(image, mode="RGB", size=rounded_size) # Seamless if seamless_axes == SeamlessAxes.AUTO: init_sa = None if image is None else self.detect_seamless(image) depth_sa = None if depth is None else self.detect_seamless(depth) if init_sa is not None and depth_sa is not None: seamless_axes = init_sa & depth_sa elif init_sa is not None: seamless_axes = init_sa elif depth_sa is not None: seamless_axes = depth_sa _configure_model_padding(pipe.unet, seamless_axes) _configure_model_padding(pipe.vae, seamless_axes) # Inference with torch.inference_mode() if device not in ('mps', "dml") else nullcontext(): def callback(step, _, latents): if future.check_cancelled(): raise InterruptedError() future.add_response(step_latents(pipe, step_preview_mode, latents, generator, step, steps)) try: result = pipe( prompt=prompt, negative_prompt=negative_prompt if use_negative_prompt else None, depth_image=depth, image=image, strength=strength, width=rounded_size[0], height=rounded_size[1], num_inference_steps=steps, guidance_scale=cfg_scale, generator=generator, callback=callback, callback_steps=1, output_type="np" ) future.add_response(step_images(result.images, generator, steps, steps)) except InterruptedError: pass future.set_done() ================================================ FILE: generator_process/actions/detect_seamless/__init__.py ================================================ from enum import Enum import numpy as np from numpy.typing import NDArray from ....api.models.seamless_axes import SeamlessAxes from .... import image_utils def detect_seamless(self, image: image_utils.ImageOrPath) -> SeamlessAxes: import os import torch from torch import nn if image.shape[0] < 8 or image.shape[1] < 8: return SeamlessAxes.OFF model = getattr(self, 'detect_seamless_model', None) if model is None: state_npz = np.load(os.path.join(os.path.dirname(__file__), 'model.npz')) state = {k: torch.tensor(v) for k, v in state_npz.items()} class SeamlessModel(nn.Module): def __init__(self): super(SeamlessModel, self).__init__() self.conv = nn.Sequential( nn.Conv2d(3, 64, kernel_size=3, stride=1, padding=1), nn.Dropout(.2), nn.PReLU(64), nn.Conv2d(64, 16, kernel_size=3, stride=1, padding=1), nn.Dropout(.2), nn.PReLU(16), nn.Conv2d(16, 64, kernel_size=8, stride=4, padding=0), nn.Dropout(.2), nn.PReLU(64), nn.Conv2d(64, 64, kernel_size=(1, 3), stride=1, padding=0), nn.Dropout(.2) ) self.gru = nn.GRU(64, 32, batch_first=True) self.fc = nn.Linear(32, 1) def forward(self, x: torch.Tensor): if len(x.size()) == 3: x = x.unsqueeze(0) x = self.conv(x) h = torch.zeros(self.gru.num_layers, x.size()[0], self.gru.hidden_size, dtype=x.dtype, device=x.device) x, h = self.gru(x.squeeze(3).transpose(2, 1), h) return torch.tanh(self.fc(x[:, -1])) model = SeamlessModel() model.load_state_dict(state) model.eval() setattr(self, 'detect_seamless_model', model) if torch.cuda.is_available(): device = 'cuda' elif torch.backends.mps.is_available(): device = 'cpu' else: device = 'cpu' image = image_utils.image_to_np(image, mode="RGB") # slice 8 pixels off each edge and combine opposing sides where the seam/seamless portion is in the middle # may trim up to 3 pixels off the length of each edge to make them a multiple of 4 # expects pixel values to be between 0-1 before this step edge_x = np.zeros((image.shape[0], 16, 3), dtype=np.float32) edge_x[:, :8] = image[:, -8:] edge_x[:, 8:] = image[:, :8] edge_x *= 2 edge_x -= 1 edge_x = edge_x[:image.shape[0] // 4 * 4].transpose(2, 0, 1) edge_y = np.zeros((16, image.shape[1], 3), dtype=np.float32) edge_y[:8] = image[-8:] edge_y[8:] = image[:8] edge_y *= 2 edge_y -= 1 edge_y = edge_y[:, :image.shape[1] // 4 * 4].transpose(2, 1, 0) @torch.no_grad() def infer(*inputs): try: model.to(device) results = [] for tensor in inputs: results.append(model(tensor)) return results finally: # swap model in and out of device rather than reloading from file model.to('cpu') if edge_x.shape == edge_y.shape: # both edges batched together edges = torch.tensor(np.array([edge_x, edge_y]), dtype=torch.float32, device=device) res = infer(edges) return SeamlessAxes((res[0][0].item() > 0, res[0][1].item() > 0)) else: edge_x = torch.tensor(edge_x, dtype=torch.float32, device=device) edge_y = torch.tensor(edge_y, dtype=torch.float32, device=device) res = infer(edge_x, edge_y) return SeamlessAxes((res[0].item() > 0, res[1].item() > 0)) ================================================ FILE: generator_process/actions/huggingface_hub.py ================================================ from dataclasses import dataclass import os from pathlib import Path from typing import Dict, List, Optional, Union, Generator, BinaryIO import copy import io import os import tempfile import warnings from contextlib import contextmanager from functools import partial from hashlib import sha256 from pathlib import Path import requests import json import enum from ..future import Future from ..models import ModelType @dataclass class Model: id: str author: str tags: list[str] likes: int downloads: int model_type: ModelType def hf_list_models( self, query: str, token: str, ) -> list[Model]: from huggingface_hub import HfApi if hasattr(self, "huggingface_hub_api"): api: HfApi = self.huggingface_hub_api else: api = HfApi() setattr(self, "huggingface_hub_api", api) models = api.list_models( tags="diffusers", search=query, token=token, ) return [ Model(m.id, m.author or "", m.tags, m.likes if hasattr(m, "likes") else 0, getattr(m, "downloads", -1), ModelType.UNKNOWN) for m in models if m.id is not None and m.tags is not None and 'diffusers' in (m.tags or {}) ] def hf_list_installed_models(self) -> list[Model]: from huggingface_hub.constants import HF_HUB_CACHE from diffusers.utils.hub_utils import old_diffusers_cache def list_dir(cache_dir): if not os.path.exists(cache_dir): return [] def detect_model_type(snapshot_folder): unet_config = os.path.join(snapshot_folder, 'unet', 'config.json') config = os.path.join(snapshot_folder, 'config.json') if os.path.exists(unet_config): with open(unet_config, 'r') as f: return ModelType(json.load(f)['in_channels']) elif os.path.exists(config): with open(config, 'r') as f: config_dict = json.load(f) if '_class_name' in config_dict and config_dict['_class_name'] == 'ControlNetModel': return ModelType.CONTROL_NET else: return ModelType.UNKNOWN else: return ModelType.UNKNOWN def _map_model(file): storage_folder = os.path.join(cache_dir, file) model_type = ModelType.UNKNOWN if os.path.exists(os.path.join(storage_folder, 'model_index.json')) or os.path.exists(os.path.join(storage_folder, 'config.json')): snapshot_folder = storage_folder model_type = detect_model_type(snapshot_folder) else: refs_folder = os.path.join(storage_folder, "refs") if not os.path.exists(refs_folder): return None for revision in os.listdir(refs_folder): ref_path = os.path.join(storage_folder, "refs", revision) with open(ref_path) as f: commit_hash = f.read() snapshot_folder = os.path.join(storage_folder, "snapshots", commit_hash) if (detected_type := detect_model_type(snapshot_folder)) != ModelType.UNKNOWN: model_type = detected_type break return Model( storage_folder, "", [], -1, -1, model_type ) return [ model for model in ( _map_model(file) for file in os.listdir(cache_dir) if os.path.isdir(os.path.join(cache_dir, file)) ) if model is not None ] new_cache_list = list_dir(HF_HUB_CACHE) model_ids = [os.path.basename(m.id) for m in new_cache_list] for model in list_dir(old_diffusers_cache): if os.path.basename(model.id) not in model_ids: new_cache_list.append(model) return new_cache_list @dataclass class DownloadStatus: file: str index: int total: int @classmethod def hook_download_tqdm(cls, future): from huggingface_hub import utils, file_download progresses = set() class future_tqdm(utils.tqdm): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.progress() def update(self, n=1): ret = super().update(n=n) self.progress() return ret def progress(self): nonlocal progresses progresses.add(self) ratio = self.n / self.total count = 0 for tqdm in progresses: r = tqdm.n / tqdm.total if r == 1: continue count += 1 if tqdm != self and ratio < r: # only show download status of most complete file return future.add_response(cls(f"{count} file{'' if count == 1 else 's'}: {self.desc}", self.n, self.total)) file_download.tqdm = future_tqdm def hf_snapshot_download( self, model: str, token: str, variant: str | None = None, resume_download=True ): from huggingface_hub import snapshot_download, repo_info from diffusers import StableDiffusionPipeline from diffusers.pipelines.pipeline_utils import variant_compatible_siblings future = Future() yield future DownloadStatus.hook_download_tqdm(future) info = repo_info(model, token=token) files = [file.rfilename for file in info.siblings] if "model_index.json" in files: # check if the variant files are available before trying to download them _, variant_files = variant_compatible_siblings(files, variant=variant) StableDiffusionPipeline.download( model, token=token, variant=variant if len(variant_files) > 0 else None, resume_download=resume_download, ) elif "config.json" in files: # individual model, such as controlnet or vae fp16_weights = ["diffusion_pytorch_model.fp16.safetensors", "diffusion_pytorch_model.fp16.bin"] fp32_weights = ["diffusion_pytorch_model.safetensors", "diffusion_pytorch_model.bin"] if variant == "fp16": weights_names = fp16_weights + fp32_weights else: weights_names = fp32_weights + fp16_weights weights = next((name for name in weights_names if name in files), None) if weights is None: raise FileNotFoundError(f"Can't find appropriate weights in {model}") snapshot_download( model, token=token, resume_download=resume_download, allow_patterns=["config.json", weights] ) else: raise ValueError(f"{model} doesn't appear to be a pipeline or model") future.set_done() ================================================ FILE: generator_process/actions/image_to_image.py ================================================ from typing import Union, Generator, Callable, List, Optional import os from contextlib import nullcontext from numpy.typing import NDArray import numpy as np import random from .prompt_to_image import Checkpoint, Scheduler, Optimizations, StepPreviewMode, step_latents, step_images, _configure_model_padding from ...api.models.seamless_axes import SeamlessAxes from ..future import Future from ...image_utils import image_to_np, size, resize, ImageOrPath def image_to_image( self, model: str | Checkpoint, scheduler: str | Scheduler, optimizations: Optimizations, image: ImageOrPath, fit: bool, strength: float, prompt: str | list[str], steps: int, width: int | None, height: int | None, seed: int, cfg_scale: float, use_negative_prompt: bool, negative_prompt: str, seamless_axes: SeamlessAxes | str | bool | tuple[bool, bool] | None, step_preview_mode: StepPreviewMode, # Stability SDK key: str | None = None, **kwargs ) -> Generator[Future, None, None]: future = Future() yield future import diffusers import torch device = self.choose_device(optimizations) # Stable Diffusion pipeline w/ caching pipe = self.load_model(diffusers.AutoPipelineForImage2Image, model, optimizations, scheduler) # Optimizations pipe = optimizations.apply(pipe, device) # RNG batch_size = len(prompt) if isinstance(prompt, list) else 1 generator = [] for _ in range(batch_size): gen = torch.Generator(device="cpu" if device in ("mps", "dml") else device) # MPS and DML do not support the `Generator` API generator.append(gen.manual_seed(random.randrange(0, np.iinfo(np.uint32).max) if seed is None else seed)) if batch_size == 1: # Some schedulers don't handle a list of generators: https://github.com/huggingface/diffusers/issues/1909 generator = generator[0] # Init Image image = image_to_np(image, mode="RGB") if fit: height = height or pipe.unet.config.sample_size * pipe.vae_scale_factor width = width or pipe.unet.config.sample_size * pipe.vae_scale_factor image = resize(image, (width, height)) else: width, height = size(image) # Seamless if seamless_axes == SeamlessAxes.AUTO: seamless_axes = self.detect_seamless(image) _configure_model_padding(pipe.unet, seamless_axes) _configure_model_padding(pipe.vae, seamless_axes) # Inference with torch.inference_mode() if device not in ('mps', "dml") else nullcontext(): def callback(pipe, step, timestep, callback_kwargs): if future.check_cancelled(): raise InterruptedError() future.add_response(step_latents(pipe, step_preview_mode, callback_kwargs["latents"], generator, step, steps)) return callback_kwargs try: result = pipe( prompt=prompt, negative_prompt=negative_prompt if use_negative_prompt else None, image=[image] * batch_size, strength=strength, num_inference_steps=steps, guidance_scale=cfg_scale, generator=generator, callback_on_step_end=callback, callback_steps=1, output_type="np" ) future.add_response(step_images(result.images, generator, steps, steps)) except InterruptedError: pass future.set_done() ================================================ FILE: generator_process/actions/inpaint.py ================================================ from typing import Union, Generator, Callable, List, Optional import os from contextlib import nullcontext import numpy as np import random from .prompt_to_image import Checkpoint, Scheduler, Optimizations, StepPreviewMode, step_latents, step_images, _configure_model_padding from ...api.models.seamless_axes import SeamlessAxes from ..future import Future from ...image_utils import image_to_np, size, resize, rgb, ImageOrPath def inpaint( self, model: str | Checkpoint, scheduler: str | Scheduler, optimizations: Optimizations, image: ImageOrPath, fit: bool, strength: float, prompt: str | list[str], steps: int, width: int | None, height: int | None, seed: int, cfg_scale: float, use_negative_prompt: bool, negative_prompt: str, seamless_axes: SeamlessAxes | str | bool | tuple[bool, bool] | None, iterations: int, step_preview_mode: StepPreviewMode, inpaint_mask_src: str, text_mask: str, text_mask_confidence: float, # Stability SDK key: str | None = None, **kwargs ) -> Generator[Future, None, None]: future = Future() yield future import diffusers import torch device = self.choose_device(optimizations) # StableDiffusionPipeline w/ caching pipe = self.load_model(diffusers.AutoPipelineForInpainting, model, optimizations, scheduler) height = height or pipe.unet.config.sample_size * pipe.vae_scale_factor width = width or pipe.unet.config.sample_size * pipe.vae_scale_factor # Optimizations pipe = optimizations.apply(pipe, device) # RNG batch_size = len(prompt) if isinstance(prompt, list) else 1 generator = [] for _ in range(batch_size): gen = torch.Generator(device="cpu" if device in ("mps", "dml") else device) # MPS and DML do not support the `Generator` API generator.append(gen.manual_seed(random.randrange(0, np.iinfo(np.uint32).max) if seed is None else seed)) if batch_size == 1: # Some schedulers don't handle a list of generators: https://github.com/huggingface/diffusers/issues/1909 generator = generator[0] # Init Image image = image_to_np(image) if fit: height = height or pipe.unet.config.sample_size * pipe.vae_scale_factor width = width or pipe.unet.config.sample_size * pipe.vae_scale_factor image = resize(image, (width, height)) else: width, height = size(image) # Seamless if seamless_axes == SeamlessAxes.AUTO: seamless_axes = self.detect_seamless(image) _configure_model_padding(pipe.unet, seamless_axes) _configure_model_padding(pipe.vae, seamless_axes) # Inference with torch.inference_mode() if device not in ('mps', "dml") else nullcontext(): match inpaint_mask_src: case 'alpha': mask_image = 1-image[..., -1] image = rgb(image) case 'prompt': image = rgb(image) from transformers import AutoProcessor, CLIPSegForImageSegmentation processor = AutoProcessor.from_pretrained("CIDAS/clipseg-rd64-refined", do_rescale=False) clipseg = CLIPSegForImageSegmentation.from_pretrained("CIDAS/clipseg-rd64-refined") inputs = processor(text=[text_mask], images=[image], return_tensors="pt", padding=True) outputs = clipseg(**inputs) mask_image = (torch.sigmoid(outputs.logits) >= text_mask_confidence).detach().numpy().astype(np.float32) mask_image = resize(mask_image, (width, height)) def callback(pipe, step, timestep, callback_kwargs): if future.check_cancelled(): raise InterruptedError() future.add_response(step_latents(pipe, step_preview_mode, callback_kwargs["latents"], generator, step, steps)) return callback_kwargs try: result = pipe( prompt=prompt, negative_prompt=negative_prompt if use_negative_prompt else None, image=[image] * batch_size, mask_image=[mask_image] * batch_size, strength=strength, height=height, width=width, num_inference_steps=steps, guidance_scale=cfg_scale, generator=generator, callback_on_step_end=callback, callback_steps=1, output_type="np" ) future.add_response(step_images(result.images, generator, steps, steps)) except InterruptedError: pass future.set_done() ================================================ FILE: generator_process/actions/load_model.py ================================================ import gc import logging import os from ..models import Checkpoint, ModelConfig, Scheduler logger = logging.getLogger(__name__) def revision_paths(model, config="model_index.json"): from huggingface_hub.constants import HF_HUB_CACHE is_repo = "/" in model if os.path.exists(os.path.join(model, config)): is_repo = False elif not is_repo and os.path.exists(os.path.join(HF_HUB_CACHE, model, config)): model = os.path.join(HF_HUB_CACHE, model) elif not is_repo: raise ValueError(f"{model} is not a valid repo, imported checkpoint, or path") if not is_repo: return {"main": model} model_path = os.path.join(HF_HUB_CACHE, "--".join(["models", *model.split("/")])) refs_path = os.path.join(model_path, "refs") revisions = {} if not os.path.isdir(refs_path): return revisions for ref in os.listdir(refs_path): with open(os.path.join(refs_path, ref)) as f: commit_hash = f.read() snapshot_path = os.path.join(model_path, "snapshots", commit_hash) if os.path.isdir(snapshot_path): revisions[ref] = snapshot_path return revisions def cache_check(*, exists_callback=None): def decorator(func): def wrapper(cache, model, *args, **kwargs): if model in cache: r = cache[model] if exists_callback is not None: r = cache[model] = exists_callback(cache, model, r, *args, **kwargs) else: r = cache[model] = func(cache, model, *args, **kwargs) return r return wrapper return decorator @cache_check() def _load_controlnet_model(cache, model, half_precision): from diffusers import ControlNetModel import torch if isinstance(model, str) and os.path.isfile(model): model = Checkpoint(model, None) if isinstance(model, Checkpoint): control_net_model = ControlNetModel.from_single_file( model.path, config_file=model.config.original_config if isinstance(model.config, ModelConfig) else model.config, ) if half_precision: control_net_model.to(torch.float16) return control_net_model revisions = revision_paths(model, "config.json") if "main" not in revisions: # controlnet models shouldn't have a fp16 revision to worry about raise FileNotFoundError(f"{model} does not contain a main revision") fp16_weights = ["diffusion_pytorch_model.fp16.safetensors", "diffusion_pytorch_model.fp16.bin"] fp32_weights = ["diffusion_pytorch_model.safetensors", "diffusion_pytorch_model.bin"] if half_precision: weights_names = fp16_weights + fp32_weights else: weights_names = fp32_weights + fp16_weights weights = next((name for name in weights_names if os.path.isfile(os.path.join(revisions["main"], name))), None) if weights is None: raise FileNotFoundError(f"Can't find appropriate weights in {model}") half_weights = weights in fp16_weights if not half_precision and half_weights: logger.warning(f"Can't load fp32 weights for model {model}, attempting to load fp16 instead") return ControlNetModel.from_pretrained( revisions["main"], torch_dtype=torch.float16 if half_precision else None, variant="fp16" if half_weights else None ) def _load_checkpoint(model_class, checkpoint, dtype, **kwargs): from diffusers.pipelines.stable_diffusion.convert_from_ckpt import download_from_original_stable_diffusion_ckpt if isinstance(checkpoint, Checkpoint): model = checkpoint.path config = checkpoint.config else: model = checkpoint config = ModelConfig.AUTO_DETECT if not os.path.exists(model): raise FileNotFoundError(f"Can't locate {model}") config_file = config.original_config if isinstance(config, ModelConfig) else config if hasattr(model_class, "from_single_file"): return model_class.from_single_file( model, torch_dtype=dtype, original_config_file=config_file, **kwargs ) else: # auto pipelines won't support from_single_file() https://github.com/huggingface/diffusers/issues/4367 from_pipe = hasattr(model_class, "from_pipe") if from_pipe: pipeline_class = config.pipeline if isinstance(config, ModelConfig) else None else: pipeline_class = model_class pipe = download_from_original_stable_diffusion_ckpt( model, from_safetensors=model.endswith(".safetensors"), original_config_file=config_file, pipeline_class=pipeline_class, controlnet=kwargs.get("controlnet", None) ) if dtype is not None: pipe.to(torch_dtype=dtype) if from_pipe: pipe = model_class.from_pipe(pipe, **kwargs) return pipe def _convert_pipe(cache, model, pipe, model_class, half_precision, scheduler, **kwargs): if model_class.__name__ not in { # some tasks are not supported by auto pipeline 'DreamTexturesDepth2ImgPipeline', 'StableDiffusionUpscalePipeline', }: pipe = model_class.from_pipe(pipe, **kwargs) scheduler.create(pipe) return pipe @cache_check(exists_callback=_convert_pipe) def _load_pipeline(cache, model, model_class, half_precision, scheduler, **kwargs): import torch dtype = torch.float16 if half_precision else None if isinstance(model, Checkpoint) or os.path.splitext(model)[1] in [".ckpt", ".safetensors"]: pipe = _load_checkpoint(model_class, model, dtype, **kwargs) scheduler.create(pipe) return pipe revisions = revision_paths(model) strategies = [] if "main" in revisions: strategies.append({"model_path": revisions["main"], "variant": "fp16" if half_precision else None}) if not half_precision: # fp16 variant can automatically use fp32 files, but fp32 won't automatically use fp16 files strategies.append({"model_path": revisions["main"], "variant": "fp16", "_warn_precision_fallback": True}) if "fp16" in revisions: strategies.append({"model_path": revisions["fp16"], "_warn_precision_fallback": not half_precision}) if len(strategies) == 0: raise FileNotFoundError(f"{model} does not contain a main or fp16 revision") exc = None for strat in strategies: if strat.pop("_warn_precision_fallback", False): logger.warning(f"Can't load fp32 weights for model {model}, attempting to load fp16 instead") try: pipe = model_class.from_pretrained(strat.pop("model_path"), torch_dtype=dtype, safety_checker=None, requires_safety_checker=False, **strat, **kwargs) pipe.scheduler = scheduler.create(pipe) return pipe except Exception as e: if exc is None: exc = e raise exc def load_model(self, model_class, model, optimizations, scheduler, controlnet=None, sdxl_refiner_model=None, **kwargs): import torch from diffusers import StableDiffusionXLPipeline, AutoPipelineForImage2Image from diffusers.pipelines.controlnet.multicontrolnet import MultiControlNetModel device = self.choose_device(optimizations) half_precision = optimizations.can_use_half(device) invalidation_properties = (device, half_precision, optimizations.cpu_offloading(device), controlnet is not None) # determine models to be removed from cache if not hasattr(self, "_pipe") or self._pipe is None or self._pipe[0] != invalidation_properties: model_cache = {} self._pipe = (invalidation_properties, model_cache) gc.collect() torch.cuda.empty_cache() else: model_cache = self._pipe[1] expected_models = {model} if sdxl_refiner_model is not None: expected_models.add(sdxl_refiner_model) if controlnet is not None: expected_models.update(name for name in controlnet) clear_models = set(model_cache).difference(expected_models) for name in clear_models: model_cache.pop(name) for pipe in model_cache.items(): if isinstance(getattr(pipe, "controlnet", None), MultiControlNetModel): # make sure no longer needed ControlNetModels are cleared # the MultiControlNetModel container will be remade pipe.controlnet = None if len(clear_models) > 0: gc.collect() torch.cuda.empty_cache() # load or obtain models from cache if controlnet is not None: kwargs["controlnet"] = MultiControlNetModel([ _load_controlnet_model(model_cache, name, half_precision) for name in controlnet ]) if not isinstance(scheduler, Scheduler): try: scheduler = Scheduler[scheduler] except KeyError: raise ValueError(f"scheduler expected one of {[s.name for s in Scheduler]}, got {repr(scheduler)}") pipe = _load_pipeline(model_cache, model, model_class, half_precision, scheduler, **kwargs) if isinstance(pipe, StableDiffusionXLPipeline) and sdxl_refiner_model is not None: return pipe, _load_pipeline(model_cache, sdxl_refiner_model, AutoPipelineForImage2Image, half_precision, scheduler, **kwargs) elif sdxl_refiner_model is not None: if model_cache.pop(sdxl_refiner_model, None) is not None: # refiner was previously used and left enabled but is not compatible with the now selected model gc.collect() torch.cuda.empty_cache() # the caller expects a tuple since refiner was defined return pipe, None return pipe ================================================ FILE: generator_process/actions/outpaint.py ================================================ from typing import Tuple, Generator import numpy as np from ..future import Future from ...api.models.generation_result import GenerationResult from ...image_utils import image_to_np, rgba, ImageOrPath def outpaint( self, image: ImageOrPath, width: int | None, height: int | None, outpaint_origin: Tuple[int, int], **kwargs ) -> Generator[Future, None, None]: future = Future() yield future width = width or 512 height = height or 512 image = image_to_np(image) if outpaint_origin[0] > image.shape[1] or outpaint_origin[0] < -width: raise ValueError(f"Outpaint origin X ({outpaint_origin[0]}) must be between {-width} and {image.shape[1]}") if outpaint_origin[1] > image.shape[0] or outpaint_origin[1] < -height: raise ValueError(f"Outpaint origin Y ({outpaint_origin[1]}) must be between {-height} and {image.shape[0]}") outpaint_bounds = np.zeros(( max(image.shape[0], outpaint_origin[1] + height) - min(0, outpaint_origin[1]), max(image.shape[1], outpaint_origin[0] + width) - min(0, outpaint_origin[0]), 4 ), dtype=np.float32) def paste(under, over, offset): under[offset[0]:offset[0] + over.shape[0], offset[1]:offset[1] + over.shape[1]] = over return under paste(outpaint_bounds, image, ( 0 if outpaint_origin[1] > 0 else -outpaint_origin[1], 0 if outpaint_origin[0] > 0 else -outpaint_origin[0] )) offset_origin = ( max(outpaint_origin[1], 0), # upper max(outpaint_origin[0], 0), # left ) # Crop out the area to generate inpaint_tile = outpaint_bounds[offset_origin[0]:offset_origin[0]+height, offset_origin[1]:offset_origin[1]+width] def process(_, step: [GenerationResult]): for res in step: res.image = paste(outpaint_bounds.copy(), rgba(res.image), offset_origin) future.add_response(step) inpaint_generator = self.inpaint( image=inpaint_tile, width=width, height=height, **kwargs ) inpaint_future = next(inpaint_generator) inpaint_future.check_cancelled = future.check_cancelled inpaint_future.add_response_callback(process) inpaint_future.add_exception_callback(future.set_exception) for _ in inpaint_generator: pass future.set_done() ================================================ FILE: generator_process/actions/prompt_to_image.py ================================================ import functools from typing import Generator from contextlib import nullcontext import numpy as np import random from ...api.models.seamless_axes import SeamlessAxes from ...api.models.step_preview_mode import StepPreviewMode from ..models import Checkpoint, Optimizations, Scheduler from ..models.image_generation_result import step_latents, step_images from ..future import Future def prompt_to_image( self, model: str | Checkpoint, scheduler: str | Scheduler, optimizations: Optimizations, prompt: str | list[str], steps: int, width: int | None, height: int | None, seed: int, cfg_scale: float, use_negative_prompt: bool, negative_prompt: str, seamless_axes: SeamlessAxes | str | bool | tuple[bool, bool] | None, iterations: int, step_preview_mode: StepPreviewMode, # Stability SDK key: str | None = None, sdxl_refiner_model: str | Checkpoint | None = None, **kwargs ) -> Generator[Future, None, None]: future = Future() yield future import diffusers import torch device = self.choose_device(optimizations) # Stable Diffusion pipeline w/ caching if sdxl_refiner_model is not None and device == "cuda" and (optimizations.cpu_offloading(device) or torch.cuda.mem_get_info()[1] > 20 * 1024**3 * (1 if optimizations.can_use_half(device) else 2)): pipe, refiner = self.load_model(diffusers.AutoPipelineForText2Image, model, optimizations, scheduler, sdxl_refiner_model=sdxl_refiner_model) else: pipe = self.load_model(diffusers.AutoPipelineForText2Image, model, optimizations, scheduler) refiner = None height = height or pipe.unet.config.sample_size * pipe.vae_scale_factor width = width or pipe.unet.config.sample_size * pipe.vae_scale_factor # Optimizations pipe = optimizations.apply(pipe, device) # RNG batch_size = len(prompt) if isinstance(prompt, list) else 1 generator = [] for _ in range(batch_size): gen = torch.Generator(device="cpu" if device in ("mps", "dml") else device) # MPS and DML do not support the `Generator` API generator.append(gen.manual_seed(random.randrange(0, np.iinfo(np.uint32).max) if seed is None else seed)) if batch_size == 1: # Some schedulers don't handle a list of generators: https://github.com/huggingface/diffusers/issues/1909 generator = generator[0] # Seamless _configure_model_padding(pipe.unet, seamless_axes) _configure_model_padding(pipe.vae, seamless_axes) # Inference with torch.inference_mode() if device not in ('mps', "dml") else nullcontext(): is_sdxl = isinstance(pipe, diffusers.StableDiffusionXLPipeline) output_type = "latent" if is_sdxl and sdxl_refiner_model is not None else "np" def callback(pipe, step, timestep, callback_kwargs): if future.check_cancelled(): raise InterruptedError() future.add_response(step_latents(pipe, step_preview_mode, callback_kwargs["latents"], generator, step, steps)) return callback_kwargs try: result = pipe( prompt=prompt, height=height, width=width, num_inference_steps=steps, guidance_scale=cfg_scale, negative_prompt=negative_prompt if use_negative_prompt else None, num_images_per_prompt=1, eta=0.0, generator=generator, latents=None, output_type=output_type, return_dict=True, callback_on_step_end=callback, callback_steps=1, #cfg_end=optimizations.cfg_end ) if is_sdxl and sdxl_refiner_model is not None and refiner is None: # allow load_model() to garbage collect pipe pipe = None refiner = self.load_model(diffusers.AutoPipelineForImage2Image, sdxl_refiner_model, optimizations, scheduler) if refiner is not None: refiner = optimizations.apply(refiner, device) result = refiner( prompt=prompt, negative_prompt=[""], callback_on_step_end=callback, callback_steps=1, num_inference_steps=steps, image=result.images, output_type="np" ) future.add_response(step_images(result.images, generator, steps, steps)) except InterruptedError: pass future.set_done() def _conv_forward_asymmetric(self, input, weight, bias): import torch.nn as nn """ Patch for Conv2d._conv_forward that supports asymmetric padding """ if input.device.type == "dml": # DML pad() will wrongly fill the tensor in constant mode with the supplied value # (default 0) when padding on both ends of a dimension, can't split to two calls. working = nn.functional.pad(input, self._reversed_padding_repeated_twice, mode='circular') pad_w0, pad_w1, pad_h0, pad_h1 = self._reversed_padding_repeated_twice if self.asymmetric_padding_mode[0] == 'constant': working[:, :, :, :pad_w0] = 0 if pad_w1 > 0: working[:, :, :, -pad_w1:] = 0 if self.asymmetric_padding_mode[1] == 'constant': working[:, :, :pad_h0] = 0 if pad_h1 > 0: working[:, :, -pad_h1:] = 0 else: working = nn.functional.pad(input, self.asymmetric_padding[0], mode=self.asymmetric_padding_mode[0]) working = nn.functional.pad(working, self.asymmetric_padding[1], mode=self.asymmetric_padding_mode[1]) return nn.functional.conv2d(working, weight, bias, self.stride, nn.modules.utils._pair(0), self.dilation, self.groups) def _lora_compatible_conv_forward(self, hidden_states, scale=1.0): return self._conv_forward(hidden_states, self.weight, self.bias) def _configure_model_padding(model, seamless_axes): import torch.nn as nn from diffusers.models.lora import LoRACompatibleConv """ Modifies the 2D convolution layers to use a circular padding mode based on the `seamless` and `seamless_axes` options. """ seamless_axes = SeamlessAxes(seamless_axes) if seamless_axes == SeamlessAxes.AUTO: seamless_axes = seamless_axes.OFF if getattr(model, "seamless_axes", SeamlessAxes.OFF) == seamless_axes: return model.seamless_axes = seamless_axes for m in model.modules(): if isinstance(m, (nn.Conv2d, nn.ConvTranspose2d, LoRACompatibleConv)): if seamless_axes.x or seamless_axes.y: if isinstance(m, LoRACompatibleConv): m.forward = _lora_compatible_conv_forward.__get__(m, LoRACompatibleConv) m.asymmetric_padding_mode = ( 'circular' if seamless_axes.x else 'constant', 'circular' if seamless_axes.y else 'constant' ) m.asymmetric_padding = ( (m._reversed_padding_repeated_twice[0], m._reversed_padding_repeated_twice[1], 0, 0), (0, 0, m._reversed_padding_repeated_twice[2], m._reversed_padding_repeated_twice[3]) ) m._conv_forward = _conv_forward_asymmetric.__get__(m, nn.Conv2d) else: if isinstance(m, LoRACompatibleConv): m.forward = LoRACompatibleConv.forward.__get__(m, LoRACompatibleConv) m._conv_forward = nn.Conv2d._conv_forward.__get__(m, nn.Conv2d) if hasattr(m, 'asymmetric_padding_mode'): del m.asymmetric_padding_mode if hasattr(m, 'asymmetric_padding'): del m.asymmetric_padding ================================================ FILE: generator_process/actions/upscale.py ================================================ import numpy as np from .prompt_to_image import Optimizations, Scheduler, StepPreviewMode, _configure_model_padding from ...api.models.seamless_axes import SeamlessAxes import random from numpy.typing import NDArray from ..models import Checkpoint, Optimizations, Scheduler, UpscaleTiler, step_images from ..future import Future from contextlib import nullcontext from ...image_utils import rgb, rgba def upscale( self, image: NDArray, model: str | Checkpoint, prompt: str, steps: int, seed: int, cfg_scale: float, scheduler: Scheduler, tile_size: int, blend: int, seamless_axes: SeamlessAxes | str | bool | tuple[bool, bool] | None, optimizations: Optimizations, step_preview_mode: StepPreviewMode, **kwargs ): future = Future() yield future import torch import diffusers device = self.choose_device(optimizations) pipe = self.load_model(diffusers.StableDiffusionUpscalePipeline, model, optimizations, scheduler) # Optimizations pipe = optimizations.apply(pipe, device) # RNG batch_size = len(prompt) if isinstance(prompt, list) else 1 generator = [] for _ in range(batch_size): gen = torch.Generator(device="cpu" if device in ("mps", "dml") else device) # MPS and DML do not support the `Generator` API generator.append(gen.manual_seed(random.randrange(0, np.iinfo(np.uint32).max) if seed is None else seed)) if batch_size == 1: # Some schedulers don't handle a list of generators: https://github.com/huggingface/diffusers/issues/1909 generator = generator[0] # Seamless tiler = UpscaleTiler(image, 4, tile_size, blend, seamless_axes) _configure_model_padding(pipe.unet, seamless_axes & ~tiler.seamless_axes) _configure_model_padding(pipe.vae, seamless_axes & ~tiler.seamless_axes) for i in range(0, len(tiler), optimizations.batch_size): if future.check_cancelled(): future.set_done() return batch_size = min(len(tiler)-i, optimizations.batch_size) ids = list(range(i, i+batch_size)) low_res_tiles = [rgb(tiler[id]) for id in ids] # Inference with torch.inference_mode() if device not in ('mps', "dml") else nullcontext(): high_res_tiles = pipe( prompt=[prompt[0] if isinstance(prompt, list) else prompt] * batch_size, image=low_res_tiles, num_inference_steps=steps, generator=generator, guidance_scale=cfg_scale, output_type="np" ).images for id, tile in zip(ids, high_res_tiles): tiler[id] = rgba(tile) if step_preview_mode != StepPreviewMode.NONE: future.add_response(step_images( [tiler.combined()], generator, i + batch_size, len(tiler), )) if step_preview_mode == StepPreviewMode.NONE: future.add_response(step_images( [tiler.combined()], generator, len(tiler), len(tiler) )) future.set_done() ================================================ FILE: generator_process/actor.py ================================================ from multiprocessing import Queue, Lock, current_process, get_context import multiprocessing.synchronize import enum import traceback import threading from typing import Type, TypeVar, Generator import site import sys import os from ..absolute_path import absolute_path from .future import Future def _patch_zip_direct_transformers_import(): # direct_transformers_import() implementation doesn't work when transformers is in a zip archive # since it relies on existing file paths. The function appears to ensure the correct root module # is obtained when there could be another loadable transformers module or it isn't in any sys.path # directory during development testing, both not being a concern in this environment. def direct_transformers_import(*_, **__): import transformers return transformers from transformers.utils import import_utils import_utils.direct_transformers_import = direct_transformers_import from transformers import utils utils.direct_transformers_import = direct_transformers_import def _load_dependencies(): site.addsitedir(absolute_path(".python_dependencies")) deps = sys.path.pop(-1) sys.path.insert(0, deps) if sys.platform == 'win32': # fix for ImportError: DLL load failed while importing cv2: The specified module could not be found. # cv2 needs python3.dll, which is stored in Blender's root directory instead of its python directory. python3_path = os.path.abspath(os.path.join(sys.executable, "..\\..\\..\\..\\python3.dll")) if os.path.exists(python3_path): os.add_dll_directory(os.path.dirname(python3_path)) # fix for OSError: [WinError 126] The specified module could not be found. Error loading "...\dream_textures\.python_dependencies\torch\lib\shm.dll" or one of its dependencies. # Allows for shm.dll from torch==2.3.0 to access dependencies from mkl==2021.4.0 # These DLL dependencies are not in the usual places that torch would look at due to being pip installed to a target directory. mkl_bin = absolute_path(".python_dependencies\\Library\\bin") if os.path.exists(mkl_bin): os.add_dll_directory(mkl_bin) if os.path.exists(absolute_path(".python_dependencies.zip")): sys.path.insert(1, absolute_path(".python_dependencies.zip")) _patch_zip_direct_transformers_import() main_thread_rendering = False is_actor_process = current_process().name == "__actor__" if is_actor_process: _load_dependencies() elif {"-b", "-f", "-a"}.intersection(sys.argv): main_thread_rendering = True import bpy def main_thread_rendering_finished(): # starting without -b will allow Blender to continue running with UI after rendering is complete global main_thread_rendering main_thread_rendering = False bpy.app.timers.register(main_thread_rendering_finished, persistent=True) class ActorContext(enum.IntEnum): """ The context of an `Actor` object. One `Actor` instance is the `FRONTEND`, while the other instance is the backend, which runs in a separate process. The `FRONTEND` sends messages to the `BACKEND`, which does work and returns a result. """ FRONTEND = 0 BACKEND = 1 class Message: """ Represents a function signature with a method name, positonal arguments, and keyword arguments. Note: All arguments must be picklable. """ def __init__(self, method_name, args, kwargs): self.method_name = method_name self.args = args self.kwargs = kwargs CANCEL = "__cancel__" END = "__end__" def _start_backend(cls, message_queue, response_queue): cls( ActorContext.BACKEND, message_queue=message_queue, response_queue=response_queue ).start() class TracedError(BaseException): def __init__(self, base: BaseException, trace: str): self.base = base self.trace = trace T = TypeVar('T', bound='Actor') class Actor: """ Base class for specialized actors. Uses queues to send actions to a background process and receive a response. Calls to any method declared by the frontend are automatically dispatched to the backend. All function arguments must be picklable. """ _message_queue: Queue _response_queue: Queue _lock: multiprocessing.synchronize.Lock _shared_instance = None # Methods that are not used for message passing, and should not be overridden in `_setup`. _protected_methods = { "start", "close", "is_alive", "can_use", "shared" } def __init__(self, context: ActorContext, message_queue: Queue = None, response_queue: Queue = None): self.context = context self._message_queue = message_queue if message_queue is not None else get_context('spawn').Queue(maxsize=1) self._response_queue = response_queue if response_queue is not None else get_context('spawn').Queue(maxsize=1) self._setup() self.__class__._shared_instance = self def _setup(self): """ Setup the Actor after initialization. """ match self.context: case ActorContext.FRONTEND: self._lock = Lock() for name in filter(lambda name: callable(getattr(self, name)) and not name.startswith("_") and name not in self._protected_methods, dir(self)): setattr(self, name, self._send(name)) case ActorContext.BACKEND: pass @classmethod def shared(cls: Type[T]) -> T: return cls._shared_instance or cls(ActorContext.FRONTEND).start() def start(self: T) -> T: """ Start the actor process. """ match self.context: case ActorContext.FRONTEND: self.process = get_context('spawn').Process(target=_start_backend, args=(self.__class__, self._message_queue, self._response_queue), name="__actor__", daemon=True) main_module = sys.modules["__main__"] main_file = getattr(main_module, "__file__", None) if main_file == "<blender string>": # Fix for Blender 4.0 not being able to start a subprocess # while previously installed addons are being initialized. try: main_module.__file__ = None self.process.start() finally: main_module.__file__ = main_file else: self.process.start() case ActorContext.BACKEND: os.environ["PYTORCH_ENABLE_MPS_FALLBACK"] = "1" self._backend_loop() return self def close(self): """ Stop the actor process. """ match self.context: case ActorContext.FRONTEND: self.process.terminate() self._message_queue.close() self._response_queue.close() case ActorContext.BACKEND: pass @classmethod def shared_close(cls: Type[T]): if cls._shared_instance is None: return cls._shared_instance.close() cls._shared_instance = None def is_alive(self): match self.context: case ActorContext.FRONTEND: return self.process.is_alive() case ActorContext.BACKEND: return True def can_use(self): if result := self._lock.acquire(block=False): self._lock.release() return result def _backend_loop(self): while True: self._receive(self._message_queue.get()) def _receive(self, message: Message): try: response = getattr(self, message.method_name)(*message.args, **message.kwargs) if isinstance(response, Generator): for res in iter(response): extra_message = None try: extra_message = self._message_queue.get(block=False) except: pass if extra_message == Message.CANCEL: break if isinstance(res, Future): def check_cancelled(): try: return self._message_queue.get(block=False) == Message.CANCEL except: return False res.check_cancelled = check_cancelled res.add_response_callback(lambda _, res: self._response_queue.put(res)) res.add_exception_callback(lambda _, e: self._response_queue.put(RuntimeError(repr(e)))) res.add_done_callback(lambda _: None) else: self._response_queue.put(res) else: self._response_queue.put(response) except Exception as e: trace = traceback.format_exc() try: if sys.modules[e.__module__].__file__.startswith(absolute_path(".python_dependencies")): e = RuntimeError(repr(e)) # might be more suitable to have specific substitute exceptions for cases # like torch.cuda.OutOfMemoryError for frontend handling in the future except (AttributeError, KeyError): pass self._response_queue.put(TracedError(e, trace)) self._response_queue.put(Message.END) def _send(self, name): def _send(*args, _block=False, **kwargs): if main_thread_rendering: _block = True future = Future() def _send_thread(future: Future): self._lock.acquire() self._message_queue.put(Message(name, args, kwargs)) while not future.done: if future.cancelled: self._message_queue.put(Message.CANCEL) response = self._response_queue.get() if response == Message.END: future.set_done() elif isinstance(response, TracedError): response.base.__cause__ = Exception(response.trace) future.set_exception(response.base) elif isinstance(response, Exception): future.set_exception(response) else: future.add_response(response) self._lock.release() if _block: _send_thread(future) else: thread = threading.Thread(target=_send_thread, args=(future,), daemon=True) thread.start() return future return _send def __del__(self): self.close() ================================================ FILE: generator_process/block_in_use.py ================================================ def block_in_use(func): def block(self, *args, **kwargs): if self.in_use: raise RuntimeError(f"Can't call {func.__qualname__} while process is in use") self.in_use = True # generator function is separate so in_use gets set immediately rather than waiting for first next() call def sub(): try: yield from func(self, *args, **kwargs) finally: self.in_use = False return sub() # Pass the name through so we can use it in `setattr` on `GeneratorProcess`. block.__name__ = func.__name__ return block ================================================ FILE: generator_process/directml_patches.py ================================================ import functools import gc import torch from torch import Tensor active_dml_patches: list | None = None def pad(input, pad, mode="constant", value=None, *, pre_patch): if input.device.type == "dml" and mode == "constant": pad_dims = torch.tensor(pad, dtype=torch.int32).view(-1, 2).flip(0) both_ends = False for pre, post in pad_dims: if pre != 0 and post != 0: both_ends = True break if both_ends: if value is None: value = 0 if pad_dims.size(0) < input.ndim: pad_dims = pre_patch(pad_dims, (0, 0, input.ndim-pad_dims.size(0), 0)) ret = torch.full(torch.Size(torch.tensor(input.size(), dtype=pad_dims.dtype) + pad_dims.sum(dim=1)), fill_value=value, dtype=input.dtype, device=input.device) assign_slices = [slice(max(0, int(pre)), None if post <= 0 else -max(0, int(post))) for pre, post in pad_dims] index_slices = [slice(max(0, -int(pre)), None if post >= 0 else -max(0, -int(post))) for pre, post in pad_dims] ret[assign_slices] = input[index_slices] return ret return pre_patch(input, pad, mode=mode, value=value) def layer_norm(input, normalized_shape, weight = None, bias = None, eps = 1e-05, *, pre_patch): if input.device.type == "dml": return pre_patch(input.contiguous(), normalized_shape, weight, bias, eps) return pre_patch(input, normalized_shape, weight, bias, eps) def retry_OOM(module): if hasattr(module, "_retry_OOM"): return forward = module.forward def is_OOM(e: RuntimeError): if hasattr(e, "_retry_OOM"): return False if len(e.args) == 0: return False if not isinstance(e.args[0], str): return False return ( e.args[0].startswith("Could not allocate tensor with") and e.args[0].endswith("bytes. There is not enough GPU video memory available!") ) def wrapper(*args, **kwargs): try: try: return forward(*args, **kwargs) except RuntimeError as e: if is_OOM(e): tb = e.__traceback__.tb_next while tb is not None: # clear locals from traceback so that intermediate tensors can be garbage collected # helps recover from Attention blocks more often tb.tb_frame.clear() tb = tb.tb_next # print("retrying!", type(module).__name__) gc.collect() return forward(*args, **kwargs) raise except RuntimeError as e: if is_OOM(e): # only retry leaf modules e._retry_OOM = True raise module.forward = wrapper module._retry_OOM = True def enable(pipe): for comp in pipe.components.values(): if not isinstance(comp, torch.nn.Module): continue for module in comp.modules(): retry_OOM(module) global active_dml_patches if active_dml_patches is not None: return active_dml_patches = [] def dml_patch(object, name, patched): original = getattr(object, name) setattr(object, name, functools.partial(patched, pre_patch=original)) active_dml_patches.append({"object": object, "name": name, "original": original}) def dml_patch_method(object, name, patched): original = getattr(object, name) setattr(object, name, functools.partialmethod(patched, pre_patch=original)) active_dml_patches.append({"object": object, "name": name, "original": original}) dml_patch(torch.nn.functional, "pad", pad) dml_patch(torch.nn.functional, "layer_norm", layer_norm) def decorate_forward(name, module): """Helper function to better find which modules DML fails in as it often does not raise an exception and immediately crashes the python interpreter.""" original = module.forward def func(self, *args, **kwargs): print(f"{name} in module {type(self)}") def nan_check(key, x): if isinstance(x, Tensor) and x.dtype in [torch.float16, torch.float32] and x.isnan().any(): raise RuntimeError(f"{key} got NaN!") for i, v in enumerate(args): nan_check(i, v) for k, v in kwargs.items(): nan_check(k, v) r = original(*args, **kwargs) nan_check("return", r) return r module.forward = func.__get__(module) # only enable when testing # for name, model in [("text_encoder", pipe.text_encoder), ("unet", pipe.unet), ("vae", pipe.vae)]: # for module in model.modules(): # decorate_forward(name, module) def disable(pipe): global active_dml_patches if active_dml_patches is None: return for patch in active_dml_patches: setattr(patch["object"], patch["name"], patch["original"]) active_dml_patches = None ================================================ FILE: generator_process/future.py ================================================ import functools import threading from typing import Callable, Any, MutableSet class Future: """ Object that represents a value that has not completed processing, but will in the future. Add callbacks to be notified when values become available, or use `.result()` and `.exception()` to wait for the value. """ _response_callbacks: MutableSet[Callable[['Future', Any], None]] = set() _exception_callbacks: MutableSet[Callable[['Future', BaseException], None]] = set() _done_callbacks: MutableSet[Callable[['Future'], None]] = set() _responses: list = [] _exception: BaseException | None = None _done_event: threading.Event done: bool = False cancelled: bool = False check_cancelled: Callable[[], bool] = lambda: False call_done_on_exception: bool = True def __init__(self): self._response_callbacks = set() self._exception_callbacks = set() self._done_callbacks = set() self._responses = [] self._exception = None self._done_event = threading.Event() self.done = False self.cancelled = False self.call_done_on_exception = True def result(self, last_only=False): """ Get the result value (blocking). """ def _response(): match len(self._responses): case 0: return None case 1: return self._responses[0] case _: return self._responses[-1] if last_only else self._responses if self._exception is not None: raise self._exception if self.done: return _response() else: self._done_event.wait() if self._exception is not None: raise self._exception return _response() def exception(self): if self.done: return self._exception else: self._done_event.wait() return self._exception def cancel(self): self.cancelled = True def _run_on_main_thread(self, func): if threading.current_thread() == threading.main_thread(): func() return try: import bpy bpy.app.timers.register(func, persistent=True) except: func() def add_response(self, response): """ Add a response value and notify all consumers. """ self._responses.append(response) def run_callbacks(): for response_callback in self._response_callbacks: response_callback(self, response) self._run_on_main_thread(run_callbacks) def set_exception(self, exception: BaseException): """ Set the exception. """ self._exception = exception def run_callbacks(): for exception_callback in self._exception_callbacks: exception_callback(self, exception) self._run_on_main_thread(run_callbacks) def set_done(self): """ Mark the future as done. """ assert not self.done self.done = True self._done_event.set() if self._exception is None or self.call_done_on_exception: def run_callbacks(): for done_callback in self._done_callbacks: done_callback(self) self._run_on_main_thread(run_callbacks) def add_response_callback(self, callback: Callable[['Future', Any], None]): """ Add a callback to run whenever a response is received. Will be called multiple times by generator functions. """ self._response_callbacks.add(callback) def add_exception_callback(self, callback: Callable[['Future', BaseException], None]): """ Add a callback to run when the future errors. Will only be called once at the first exception. """ self._exception_callbacks.add(callback) if self._exception is not None: self._run_on_main_thread(functools.partial(callback, self, self._exception)) def add_done_callback(self, callback: Callable[['Future'], None]): """ Add a callback to run when the future is marked as done. Will only be called once. """ self._done_callbacks.add(callback) if self.done: self._run_on_main_thread(functools.partial(callback, self)) ================================================ FILE: generator_process/models/__init__.py ================================================ from .checkpoint import * from .image_generation_result import * from .model_config import * from .model_type import * from .optimizations import * from .scheduler import * from .upscale_tiler import * ================================================ FILE: generator_process/models/checkpoint.py ================================================ from dataclasses import dataclass from .model_config import ModelConfig @dataclass(frozen=True) class Checkpoint: path: str config: ModelConfig | str | None ================================================ FILE: generator_process/models/image_generation_result.py ================================================ from ...api.models.step_preview_mode import StepPreviewMode from ...api.models.generation_result import GenerationResult def step_latents(pipe, mode, latents, generator, iteration, steps): seeds = [gen.initial_seed() for gen in generator] if isinstance(generator, list) else [generator.initial_seed()] scale = 2 ** (len(pipe.vae.config.block_out_channels) - 1) match mode: case StepPreviewMode.FAST: return [ GenerationResult( progress=iteration, total=steps, seed=seeds[-1], image=approximate_decoded_latents(latents[-1:], scale) ) ] case StepPreviewMode.FAST_BATCH: return [ GenerationResult( progress=iteration, total=steps, seed=seed, image=approximate_decoded_latents(latent, scale) ) for latent, seed in zip(latents[:, None], seeds) ] case StepPreviewMode.ACCURATE: return [ GenerationResult( progress=iteration, total=steps, seed=seeds[-1], image=decode_latents(pipe, latents[-1:]) ) ] case StepPreviewMode.ACCURATE_BATCH: return [ GenerationResult( progress=iteration, total=steps, seed=seed, image=decode_latents(pipe, latent) ) for latent, seed in zip(latents[:, None], seeds) ] return [ GenerationResult( progress=iteration, total=steps, seed=seeds[-1] ) ] def step_images(images, generator, iteration, steps): if not isinstance(images, list) and images.ndim == 3: images = images[None] seeds = [gen.initial_seed() for gen in generator] if isinstance(generator, list) else [generator.initial_seed()] return [ GenerationResult( progress=iteration, total=steps, seed=seed, image=image ) for image, seed in zip(images, seeds) ] def decode_latents(pipe, latents): return pipe.image_processor.postprocess(pipe.vae.decode(latents / pipe.vae.config.scaling_factor).sample, output_type="np") def approximate_decoded_latents(latents, scale=None): """ Approximate the decoded latents without using the VAE. """ import torch # origingally adapted from code by @erucipe and @keturn here: # https://discuss.huggingface.co/t/decoding-latents-to-rgb-without-upscaling/23204/7 # these updated numbers for v1.5 are from @torridgristle v1_5_latent_rgb_factors = torch.tensor([ # R G B [ 0.3444, 0.1385, 0.0670], # L1 [ 0.1247, 0.4027, 0.1494], # L2 [-0.3192, 0.2513, 0.2103], # L3 [-0.1307, -0.1874, -0.7445] # L4 ], dtype=latents.dtype, device=latents.device) latent_image = latents[0].permute(1, 2, 0) @ v1_5_latent_rgb_factors if scale is not None: latent_image = torch.nn.functional.interpolate( latent_image.permute(2, 0, 1).unsqueeze(0), scale_factor=scale, mode="nearest" ).squeeze(0).permute(1, 2, 0) latent_image = ((latent_image + 1) / 2).clamp(0, 1).cpu() return latent_image.numpy() ================================================ FILE: generator_process/models/model_config.py ================================================ import enum from ...absolute_path import absolute_path class ModelConfig(enum.Enum): AUTO_DETECT = "auto-detect" STABLE_DIFFUSION_1 = "v1" STABLE_DIFFUSION_2_BASE = "v2 (512, epsilon)" STABLE_DIFFUSION_2 = "v2 (768, v_prediction)" STABLE_DIFFUSION_2_DEPTH = "v2 (depth)" STABLE_DIFFUSION_2_INPAINTING = "v2 (inpainting)" STABLE_DIFFUSION_XL_BASE = "XL (base)" STABLE_DIFFUSION_XL_REFINER = "XL (refiner)" CONTROL_NET_1_5 = "1.5 (ControlNet)" CONTROL_NET_2_1 = "2.1 (ControlNet)" @property def original_config(self): match self: case ModelConfig.AUTO_DETECT: return None case ModelConfig.STABLE_DIFFUSION_1: return absolute_path("sd_configs/v1-inference.yaml") case ModelConfig.STABLE_DIFFUSION_2_BASE: return absolute_path("sd_configs/v2-inference.yaml") case ModelConfig.STABLE_DIFFUSION_2: return absolute_path("sd_configs/v2-inference-v.yaml") case ModelConfig.STABLE_DIFFUSION_2_DEPTH: return absolute_path("sd_configs/v2-midas-inference.yaml") case ModelConfig.STABLE_DIFFUSION_2_INPAINTING: return absolute_path("sd_configs/v2-inpainting-inference.yaml") case ModelConfig.STABLE_DIFFUSION_XL_BASE: return absolute_path("sd_configs/sd_xl_base.yaml") case ModelConfig.STABLE_DIFFUSION_XL_REFINER: return absolute_path("sd_configs/sd_xl_refiner.yaml") case ModelConfig.CONTROL_NET_1_5: return absolute_path("sd_configs/cldm_v15.yaml") case ModelConfig.CONTROL_NET_2_1: return absolute_path("sd_configs/cldm_v21.yaml") @property def pipeline(self): # allows for saving with correct _class_name in model_index.json and necessary for some models to import import diffusers match self: case ModelConfig.AUTO_DETECT: return None case ModelConfig.STABLE_DIFFUSION_2_DEPTH: return diffusers.StableDiffusionDepth2ImgPipeline case ModelConfig.STABLE_DIFFUSION_2_INPAINTING: return diffusers.StableDiffusionInpaintPipeline case ModelConfig.STABLE_DIFFUSION_XL_BASE: return diffusers.StableDiffusionXLPipeline case ModelConfig.STABLE_DIFFUSION_XL_REFINER: return diffusers.StableDiffusionXLImg2ImgPipeline case ModelConfig.CONTROL_NET_1_5 | ModelConfig.CONTROL_NET_2_1: return diffusers.ControlNetModel case _: return diffusers.StableDiffusionPipeline ================================================ FILE: generator_process/models/model_type.py ================================================ import enum from ...api.models.task import * from .model_config import ModelConfig class ModelType(enum.IntEnum): """ Inferred model type from the U-Net `in_channels`. """ UNKNOWN = 0 PROMPT_TO_IMAGE = 4 DEPTH = 5 UPSCALING = 7 INPAINTING = 9 CONTROL_NET = -1 UNSPECIFIED_CHECKPOINT = -2 @classmethod def _missing_(cls, _): return cls.UNKNOWN def recommended_model(self) -> str: """Provides a recommended model for a given task. This method has a bias towards the latest version of official Stability AI models. """ match self: case ModelType.PROMPT_TO_IMAGE: return "stabilityai/stable-diffusion-2-1" case ModelType.DEPTH: return "stabilityai/stable-diffusion-2-depth" case ModelType.UPSCALING: return "stabilityai/stable-diffusion-x4-upscaler" case ModelType.INPAINTING: return "stabilityai/stable-diffusion-2-inpainting" case _: return "stabilityai/stable-diffusion-2-1" def matches_task(self, task: Task) -> bool: """Indicates if the model type is correct for a given `Task`. If not an error should be shown to the user to select a different model. """ if self == ModelType.UNSPECIFIED_CHECKPOINT: return True match task: case PromptToImage(): return self == ModelType.PROMPT_TO_IMAGE case Inpaint(): return self == ModelType.INPAINTING case DepthToImage(): return self == ModelType.DEPTH case Outpaint(): return self == ModelType.INPAINTING case ImageToImage(): return self == ModelType.PROMPT_TO_IMAGE case _: return False @staticmethod def from_task(task: Task) -> 'ModelType | None': match task: case PromptToImage(): return ModelType.PROMPT_TO_IMAGE case Inpaint(): return ModelType.INPAINTING case DepthToImage(): return ModelType.DEPTH case Outpaint(): return ModelType.INPAINTING case ImageToImage(): return ModelType.PROMPT_TO_IMAGE case _: return None @staticmethod def from_config(config: ModelConfig): match config: case ModelConfig.AUTO_DETECT: return ModelType.UNSPECIFIED_CHECKPOINT case ModelConfig.STABLE_DIFFUSION_2_DEPTH: return ModelType.DEPTH case ModelConfig.STABLE_DIFFUSION_2_INPAINTING: return ModelType.INPAINTING case ModelConfig.CONTROL_NET_1_5 | ModelConfig.CONTROL_NET_2_1: return ModelType.CONTROL_NET case _: return ModelType.PROMPT_TO_IMAGE ================================================ FILE: generator_process/models/optimizations.py ================================================ from enum import Enum from typing import Annotated, Union, _AnnotatedAlias import functools import os import sys from dataclasses import dataclass from .upscale_tiler import tiled_decode_latents class CPUOffload(Enum): OFF = "off" MODEL = "model" SUBMODULE = "submodule" def __bool__(self): return self != CPUOffload.OFF @dataclass(eq=True) class Optimizations: attention_slicing: bool = True attention_slice_size: Union[str, int] = "auto" cudnn_benchmark: Annotated[bool, "cuda"] = False tf32: Annotated[bool, "cuda"] = False amp: Annotated[bool, "cuda"] = False half_precision: Annotated[bool, {"cuda", "dml"}] = True cpu_offload: Annotated[str, {"cuda", "dml"}] = CPUOffload.OFF channels_last_memory_format: bool = False sdp_attention: bool = True batch_size: int = 1 vae_slicing: bool = True vae_tiling: str = "off" vae_tile_size: int = 512 vae_tile_blend: int = 64 cfg_end: float = 1.0 cpu_only: bool = False @staticmethod def infer_device() -> str: from ...absolute_path import absolute_path if sys.platform == "darwin": return "mps" elif os.path.exists(absolute_path(".python_dependencies/torch_directml")): return "dml" else: return "cuda" @classmethod def device_supports(cls, property, device) -> bool: annotation = cls.__annotations__.get(property, None) if isinstance(annotation, _AnnotatedAlias): opt_dev = annotation.__metadata__[0] if isinstance(opt_dev, str): return opt_dev == device return device in opt_dev return annotation is not None def can_use(self, property, device) -> bool: return self.device_supports(property, device) and getattr(self, property) def can_use_half(self, device): if self.half_precision and device == "cuda": import torch name = torch.cuda.get_device_name() return not ("GTX 1650" in name or "GTX 1660" in name) return self.can_use("half_precision", device) def cpu_offloading(self, device): return self.cpu_offload if self.device_supports("cpu_offload", device) else CPUOffload.OFF def apply(self, pipeline, device): """ Apply the optimizations to a diffusers pipeline. All exceptions are ignored to make this more general purpose across different pipelines. """ import torch if not self.cpu_offloading(device): pipeline = pipeline.to(device) torch.backends.cudnn.benchmark = self.can_use("cudnn_benchmark", device) torch.backends.cuda.matmul.allow_tf32 = self.can_use("tf32", device) try: if self.can_use("sdp_attention", device): from diffusers.models.attention_processor import AttnProcessor2_0 pipeline.unet.set_attn_processor(AttnProcessor2_0()) elif self.can_use("attention_slicing", device): pipeline.enable_attention_slicing(self.attention_slice_size) else: pipeline.disable_attention_slicing() # will also disable AttnProcessor2_0 except: pass try: if pipeline.device != pipeline._execution_device: pass # pipeline is already offloaded, offloading again can cause `pipeline._execution_device` to be incorrect elif self.cpu_offloading(device) == CPUOffload.MODEL: # adapted from diffusers.StableDiffusionPipeline.enable_model_cpu_offload() to allow DirectML device and unimplemented pipelines from accelerate import cpu_offload_with_hook hook = None models = [] # text_encoder can be None in SDXL Pipeline but not text_encoder_2 if pipeline.text_encoder is not None: models.append(pipeline.text_encoder) if hasattr(pipeline, "text_encoder_2"): models.append(pipeline.text_encoder_2) models.extend([pipeline.unet, pipeline.vae]) if hasattr(pipeline, "controlnet"): models.append(pipeline.controlnet) for cpu_offloaded_model in models: _, hook = cpu_offload_with_hook(cpu_offloaded_model, device, prev_module_hook=hook) if getattr(pipeline, "safety_checker", None) is not None: _, hook = cpu_offload_with_hook(pipeline.safety_checker, device, prev_module_hook=hook) # We'll offload the last model manually. pipeline.final_offload_hook = hook elif self.cpu_offloading(device) == CPUOffload.SUBMODULE: # adapted from diffusers.StableDiffusionPipeline.enable_sequential_cpu_offload() to allow DirectML device and unimplemented pipelines from accelerate import cpu_offload models = [] # text_encoder can be None in SDXL Pipeline but not text_encoder_2 if pipeline.text_encoder is not None: models.append(pipeline.text_encoder) if hasattr(pipeline, "text_encoder_2"): models.append(pipeline.text_encoder_2) models.extend([pipeline.unet, pipeline.vae]) if hasattr(pipeline, "controlnet"): models.append(pipeline.controlnet) for cpu_offloaded_model in models: cpu_offload(cpu_offloaded_model, device) if getattr(pipeline, "safety_checker", None) is not None: cpu_offload(pipeline.safety_checker, device, offload_buffers=True) except: pass try: if self.can_use("channels_last_memory_format", device): pipeline.unet.to(memory_format=torch.channels_last) else: pipeline.unet.to(memory_format=torch.contiguous_format) except: pass try: if self.can_use("vae_slicing", device): # Not many pipelines implement the enable_vae_slicing()/disable_vae_slicing() # methods but all they do is forward their call to the vae anyway. pipeline.vae.enable_slicing() else: pipeline.vae.disable_slicing() except: pass try: if self.vae_tiling != "off": if not isinstance(pipeline.vae.decode, functools.partial): pipeline.vae.decode = functools.partial(tiled_decode_latents.__get__(pipeline), pre_patch=pipeline.vae.decode) pipeline.vae.decode.keywords['optimizations'] = self elif self.vae_tiling == "off" and isinstance(pipeline.vae.decode, functools.partial): pipeline.vae.decode = pipeline.vae.decode.keywords["pre_patch"] except: pass from .. import directml_patches if device == "dml": directml_patches.enable(pipeline) else: directml_patches.disable(pipeline) return pipeline ================================================ FILE: generator_process/models/scheduler.py ================================================ import enum class Scheduler(enum.Enum): DDIM = "DDIM" DDPM = "DDPM" DEIS_MULTISTEP = "DEIS Multistep" DPM_SOLVER_MULTISTEP = "DPM Solver Multistep" DPM_SOLVER_MULTISTEP_KARRAS = "DPM Solver Multistep Karras" DPM_SOLVER_SINGLESTEP = "DPM Solver Singlestep" DPM_SOLVER_SINGLESTEP_KARRAS = "DPM Solver Singlestep Karras" EULER_DISCRETE = "Euler Discrete" EULER_DISCRETE_KARRAS = "Euler Discrete Karras" EULER_ANCESTRAL_DISCRETE = "Euler Ancestral Discrete" HEUN_DISCRETE = "Heun Discrete" HEUN_DISCRETE_KARRAS = "Heun Discrete Karras" KDPM2_DISCRETE = "KDPM2 Discrete" # Non-functional on mps KDPM2_ANCESTRAL_DISCRETE = "KDPM2 Ancestral Discrete" LMS_DISCRETE = "LMS Discrete" LMS_DISCRETE_KARRAS = "LMS Discrete Karras" PNDM = "PNDM" UNIPC_MULTISTEP = "UniPC Multistep" def create(self, pipeline): import diffusers def scheduler_class(): match self: case Scheduler.DDIM: return diffusers.schedulers.DDIMScheduler case Scheduler.DDPM: return diffusers.schedulers.DDPMScheduler case Scheduler.DEIS_MULTISTEP: return diffusers.schedulers.DEISMultistepScheduler case Scheduler.DPM_SOLVER_MULTISTEP | Scheduler.DPM_SOLVER_MULTISTEP_KARRAS: return diffusers.schedulers.DPMSolverMultistepScheduler case Scheduler.DPM_SOLVER_SINGLESTEP | Scheduler.DPM_SOLVER_SINGLESTEP_KARRAS: return diffusers.schedulers.DPMSolverSinglestepScheduler case Scheduler.EULER_DISCRETE | Scheduler.EULER_DISCRETE_KARRAS: return diffusers.schedulers.EulerDiscreteScheduler case Scheduler.EULER_ANCESTRAL_DISCRETE: return diffusers.schedulers.EulerAncestralDiscreteScheduler case Scheduler.HEUN_DISCRETE | Scheduler.HEUN_DISCRETE_KARRAS: return diffusers.schedulers.HeunDiscreteScheduler case Scheduler.KDPM2_DISCRETE: return diffusers.schedulers.KDPM2DiscreteScheduler case Scheduler.KDPM2_ANCESTRAL_DISCRETE: return diffusers.schedulers.KDPM2AncestralDiscreteScheduler case Scheduler.LMS_DISCRETE | Scheduler.LMS_DISCRETE_KARRAS: return diffusers.schedulers.LMSDiscreteScheduler case Scheduler.PNDM: return diffusers.schedulers.PNDMScheduler case Scheduler.UNIPC_MULTISTEP: return diffusers.schedulers.UniPCMultistepScheduler original_config = getattr(pipeline.scheduler, "_original_config", pipeline.scheduler.config) scheduler = scheduler_class().from_config(original_config, use_karras_sigmas=self.name.endswith("KARRAS")) scheduler._original_config = original_config pipeline.scheduler = scheduler return scheduler ================================================ FILE: generator_process/models/upscale_tiler.py ================================================ import math from typing import Optional import numpy as np from ..actions.detect_seamless import SeamlessAxes from numpy.typing import NDArray class UpscaleTiler: def __init__( self, image: NDArray, scale: int, tile_size: int | tuple[int, int], blend: int | tuple[int, int], seamless_axes: SeamlessAxes, defer_seamless: bool = True, out_channels: Optional[int] = None ): height, width = image.shape[:2] if scale < 1: raise ValueError("scale must be 1 or higher") if isinstance(tile_size, int): tile_size = (tile_size, tile_size) if tile_size[0] <= 0 or tile_size[1] <= 0: raise ValueError("tile size must be 1 or higher") if isinstance(blend, int): blend = (blend, blend) if blend[0] < 0 or blend[1] < 0: raise ValueError("blend must be 0 or higher") seamless_axes = SeamlessAxes(seamless_axes) if defer_seamless: # Seamless handling may be deferred to upscaler model or VAE rather than using larger or multiple tiles seamless_axes = SeamlessAxes((seamless_axes.x and width > tile_size[0], seamless_axes.y and height > tile_size[1])) max_width = width*2 if seamless_axes.x else width max_height = height*2 if seamless_axes.y else height tile_size = (min(tile_size[0], max_width), min(tile_size[1], max_height)) blend = (min(blend[0], math.ceil(tile_size[0]/2)), min(blend[1], math.ceil(tile_size[1]/2))) self.image = image self.scale = scale self.tile_size = tile_size self.blend = blend self.seamless_axes = seamless_axes self.x_tiles = self.axis_tiles(width, tile_size[0], blend[0], seamless_axes.x) self.y_tiles = self.axis_tiles(height, tile_size[1], blend[1], seamless_axes.y) if out_channels is None: out_channels = image.shape[2] # combined image with last channel containing pixel weights self.canvas = np.zeros((image.shape[0] * scale, image.shape[1] * scale, out_channels + 1), dtype=np.float32) scaled_tile_size = (tile_size[0] * scale, tile_size[1] * scale) weight_gradient_y = [min(i + 1, scaled_tile_size[1] - i) for i in range(scaled_tile_size[1])] weight_gradient_x = [min(i + 1, scaled_tile_size[0] - i) for i in range(scaled_tile_size[0])] tile_weight = np.zeros(scaled_tile_size, dtype=np.float32) tile_weight[:] = weight_gradient_y # determines how much each pixel in a blended area influences the final color, basically a pyramid self.tile_weight = np.minimum(tile_weight, np.reshape(weight_gradient_x, (scaled_tile_size[0], 1))) @staticmethod def axis_tiles(axis_size: int, tile_size: int, blend: int, seamless: bool) -> list[int]: """ Returns a list of values where each tile starts on an axis. Blend is only guaranteed as a minimum and may vary by a pixel between tiles. """ if seamless: count = math.ceil(axis_size / (tile_size - blend)) blend_balance = math.ceil(tile_size - axis_size / count) final = min(axis_size - tile_size + blend_balance, axis_size * 2 - tile_size) else: count = math.ceil((axis_size - tile_size) / (tile_size - blend)) + 1 final = axis_size - tile_size if count == 1: return [0] return [i * final // (count - 1) for i in range(count)] def combined(self) -> NDArray: return self.canvas[:, :, :-1] def index_to_xy(self, index: int): key_y = index % len(self.y_tiles) key_x = (index - key_y) // len(self.y_tiles) return key_x, key_y def __getitem__(self, key: int | tuple[int, int]) -> NDArray: if isinstance(key, int): key = self.index_to_xy(key) image = self.image tile_size = self.tile_size x0 = self.x_tiles[key[0]] x1 = x0 + tile_size[0] x2 = image.shape[1] - x0 y0 = self.y_tiles[key[1]] y1 = y0 + tile_size[1] y2 = image.shape[0] - y0 if x2 >= tile_size[0] and y2 >= tile_size[1]: return image[y0:y1, x0:x1] # seamless axis wrapping if isinstance(image, np.ndarray): tile = np.empty((tile_size[0], tile_size[1], image.shape[2]), dtype=image.dtype) else: import torch tile = torch.empty((tile_size[0], tile_size[1], image.shape[2]), dtype=image.dtype, device=image.device) if x2 < tile_size[0]: if y2 < tile_size[1]: # wrap bottom/right to top/left tile[:y2, :x2] = image[y0:, x0:] tile[y2:, :x2] = image[:tile_size[1] - y2, x0:] tile[:y2, x2:] = image[y0:, :tile_size[0] - x2] tile[y2:, x2:] = image[:tile_size[1] - y2, :tile_size[0] - x2] else: # wrap right to left tile[:, :x2] = image[y0:y1, x0:] tile[:, x2:] = image[y0:y1, :tile_size[0] - x2] else: # wrap bottom to top tile[:y2] = image[y0:, x0:x1] tile[y2:] = image[:tile_size[1] - y2, x0:x1] return tile def __setitem__(self, key: int | tuple[int, int], tile: NDArray): if isinstance(key, int): key = self.index_to_xy(key) canvas = self.canvas scale = self.scale tile_size = (self.tile_size[0] * scale, self.tile_size[1] * scale) tile_weight = self.tile_weight x0 = self.x_tiles[key[0]] * scale x1 = x0 + tile_size[0] x2 = canvas.shape[1] - x0 y0 = self.y_tiles[key[1]] * scale y1 = y0 + tile_size[1] y2 = canvas.shape[0] - y0 def update(canvas_slice, tile_slice, weight_slice): weight_slice = weight_slice.reshape(weight_slice.shape[0], weight_slice.shape[1], 1) # undo weighted average, then add new tile with its weights applied and average again canvas_slice[:, :, :-1] *= canvas_slice[:, :, -1:] canvas_slice[:, :, :-1] += tile_slice * weight_slice canvas_slice[:, :, -1:] += weight_slice canvas_slice[:, :, :-1] /= canvas_slice[:, :, -1:] if x2 >= tile_size[0] and y2 >= tile_size[1]: update(canvas[y0:y1, x0:x1], tile, tile_weight) elif x2 < tile_size[0]: if y2 < tile_size[1]: update(canvas[y0:, x0:], tile[:y2, :x2], tile_weight[:y2, :x2]) update(canvas[:tile_size[1] - y2, x0:], tile[y2:, :x2], tile_weight[y2:, :x2]) update(canvas[y0:, :tile_size[0] - x2], tile[:y2, x2:], tile_weight[:y2, x2:]) update(canvas[:tile_size[1] - y2, :tile_size[0] - x2], tile[y2:, x2:], tile_weight[y2:, x2:]) else: update(canvas[y0:y1, x0:], tile[:, :x2], tile_weight[:, :x2]) update(canvas[y0:y1, :tile_size[0] - x2], tile[:, x2:], tile_weight[:, x2:]) else: update(canvas[y0:, x0:x1], tile[:y2], tile_weight[:y2]) update(canvas[:tile_size[1] - y2, x0:x1], tile[y2:], tile_weight[y2:]) def __iter__(self): for x in range(len(self.x_tiles)): for y in range(len(self.y_tiles)): yield (x, y), self[x, y] def __len__(self): return len(self.x_tiles) * len(self.y_tiles) def tiled_decode_latents(self, latents, return_dict=False, *, pre_patch, optimizations): # not all pipelines (namely upscale) have the vae_scale_factor attribute vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) default_size = self.unet.config.sample_size * vae_scale_factor match optimizations.vae_tiling: case "full": tile_size = default_size blend = math.ceil(tile_size / 8) case "half": tile_size = math.ceil(default_size / 2) blend = math.ceil(tile_size / 8) case "manual": tile_size = optimizations.vae_tile_size blend = optimizations.vae_tile_blend case _: return pre_patch(latents) seamless_axes = getattr(self.vae, "seamless_axes", SeamlessAxes.OFF) images = [] for image_latents in latents.split(1, dim=0): tiler = UpscaleTiler( image_latents.squeeze(0).permute(1, 2, 0), vae_scale_factor, math.ceil(tile_size / vae_scale_factor), math.ceil(blend / vae_scale_factor), seamless_axes, out_channels=self.vae.config.out_channels ) configure_model_padding(self.vae, seamless_axes & ~tiler.seamless_axes) for id, tile in tiler: tiler[id] = pre_patch(tile.permute(2, 0, 1).unsqueeze(0)).sample.squeeze(0).permute(1, 2, 0).cpu().numpy() images.append(np.expand_dims(tiler.combined(), 0).transpose(0, 3, 1, 2)) configure_model_padding(self.vae, seamless_axes) images = np.concatenate(images) import torch images = torch.from_numpy(images) if not return_dict: return (images,) from diffusers.models.vae import DecoderOutput return DecoderOutput(images) def configure_model_padding(model, seamless_axes): import torch.nn as nn """ Modifies the 2D convolution layers to use a circular padding mode based on the `seamless_axes` option. """ seamless_axes = SeamlessAxes(seamless_axes) if seamless_axes == SeamlessAxes.AUTO: seamless_axes = seamless_axes.OFF if getattr(model, "seamless_axes", SeamlessAxes.OFF) == seamless_axes: return model.seamless_axes = seamless_axes for m in model.modules(): if isinstance(m, (nn.Conv2d, nn.ConvTranspose2d)): if seamless_axes.x or seamless_axes.y: m.asymmetric_padding_mode = ( 'circular' if seamless_axes.x else 'constant', 'circular' if seamless_axes.y else 'constant' ) m.asymmetric_padding = ( (m._reversed_padding_repeated_twice[0], m._reversed_padding_repeated_twice[1], 0, 0), (0, 0, m._reversed_padding_repeated_twice[2], m._reversed_padding_repeated_twice[3]) ) m._conv_forward = _conv_forward_asymmetric.__get__(m, nn.Conv2d) else: m._conv_forward = nn.Conv2d._conv_forward.__get__(m, nn.Conv2d) if hasattr(m, 'asymmetric_padding_mode'): del m.asymmetric_padding_mode if hasattr(m, 'asymmetric_padding'): del m.asymmetric_padding def _conv_forward_asymmetric(self, input, weight, bias): import torch.nn as nn """ Patch for Conv2d._conv_forward that supports asymmetric padding """ if input.device.type == "dml": # DML pad() will wrongly fill the tensor in constant mode with the supplied value # (default 0) when padding on both ends of a dimension, can't split to two calls. working = nn.functional.pad(input, self._reversed_padding_repeated_twice, mode='circular') pad_w0, pad_w1, pad_h0, pad_h1 = self._reversed_padding_repeated_twice if self.asymmetric_padding_mode[0] == 'constant': working[:, :, :, :pad_w0] = 0 if pad_w1 > 0: working[:, :, :, -pad_w1:] = 0 if self.asymmetric_padding_mode[1] == 'constant': working[:, :, :pad_h0] = 0 if pad_h1 > 0: working[:, :, -pad_h1:] = 0 else: working = nn.functional.pad(input, self.asymmetric_padding[0], mode=self.asymmetric_padding_mode[0]) working = nn.functional.pad(working, self.asymmetric_padding[1], mode=self.asymmetric_padding_mode[1]) return nn.functional.conv2d(working, weight, bias, self.stride, nn.modules.utils._pair(0), self.dilation, self.groups) ================================================ FILE: image_utils.py ================================================ import importlib.util import os import sys from os import PathLike from typing import Tuple, Literal, Union, TYPE_CHECKING import numpy as np from numpy.typing import NDArray, DTypeLike from .generator_process import RunInSubprocess """ This module allows for simple handling of image data in numpy ndarrays in some common formats. Dimensions: 2: HW - L 3: HWC - L/LA/RGB/RGBA 4: NHWC - batched HWC Channels: 1: L 2: LA 3: RGB 4: RGBA """ def version_str(version): return ".".join(str(x) for x in version) # find_spec("bpy") will never return None has_bpy = sys.modules.get("bpy", None) is not None has_ocio = importlib.util.find_spec("PyOpenColorIO") is not None has_oiio = importlib.util.find_spec("OpenImageIO") is not None has_pil = importlib.util.find_spec("PIL") is not None if has_bpy: # frontend import bpy BLENDER_VERSION = bpy.app.version OCIO_CONFIG = os.path.join(bpy.utils.resource_path('LOCAL'), 'datafiles/colormanagement/config.ocio') # Easier to share via environment variables than to enforce backends with subprocesses to use their own methods of sharing. os.environ["BLENDER_VERSION"] = version_str(BLENDER_VERSION) os.environ["BLENDER_OCIO_CONFIG"] = OCIO_CONFIG else: # backend BLENDER_VERSION = tuple(int(x) for x in os.environ["BLENDER_VERSION"].split(".")) OCIO_CONFIG = os.environ["BLENDER_OCIO_CONFIG"] if TYPE_CHECKING: import bpy import PIL.Image def _bpy_version_error(required_version, feature, module): if BLENDER_VERSION >= required_version: return Exception(f"{module} is unexpectedly missing in Blender {version_str(BLENDER_VERSION)}") return Exception(f"{feature} requires Blender {version_str(required_version)} or higher, you are using {version_str(BLENDER_VERSION)}") def size(array: NDArray) -> Tuple[int, int]: if array.ndim == 2: return array.shape[1], array.shape[0] if array.ndim in [3, 4]: return array.shape[-2], array.shape[-3] raise ValueError(f"Can't determine size from {array.ndim} dimensions") def channels(array: NDArray) -> int: if array.ndim == 2: return 1 if array.ndim in [3, 4]: return array.shape[-1] raise ValueError(f"Can't determine channels from {array.ndim} dimensions") def ensure_alpha(array: NDArray, alpha=None) -> NDArray: """ Args: array: Image pixels values. alpha: Default alpha value if an alpha channel will be made. Will be inferred from `array.dtype` if None. Returns: The converted image or the original image if it already had alpha. """ c = channels(array) if c in [2, 4]: return array if c not in [1, 3]: raise ValueError(f"Can't ensure alpha from {c} channels") if alpha is None: alpha = 0 if np.issubdtype(array.dtype, np.floating): alpha = 1 elif np.issubdtype(array.dtype, np.integer): alpha = np.iinfo(array.dtype).max array = ensure_channel_dim(array) return np.pad(array, [*[(0, 0)]*(array.ndim-1), (0, 1)], constant_values=alpha) def ensure_opaque(array: NDArray) -> NDArray: """ Removes the alpha channel if it exists. """ if channels(array) in [2, 4]: return array[..., :-1] return array def ensure_channel_dim(array: NDArray) -> NDArray: """ Expands a HW grayscale image to HWC. """ if array.ndim == 2: return array[..., np.newaxis] return array def rgb(array: NDArray) -> NDArray: """ Converts a grayscale image to RGB or removes the alpha channel from an RGBA image. If the image was already RGB the original array will be returned. """ c = channels(array) match channels(array): case 1: return np.concatenate([ensure_channel_dim(array)] * 3, axis=-1) case 2: return np.concatenate([array[..., :1]] * 3, axis=-1) case 3: return array case 4: return array[..., :3] raise ValueError(f"Can't make {c} channels RGB") def rgba(array: NDArray, alpha=None) -> NDArray: """ Args: array: Image pixels values. alpha: Default alpha value if an alpha channel will be made. Will be inferred from `array.dtype` if None. Returns: The converted image or the original image if it already was RGBA. """ c = channels(array) if c == 4: return array if c == 2: l, a = np.split(array, 2, axis=-1) return np.concatenate([l, l, l, a], axis=-1) return ensure_alpha(rgb(array), alpha) def grayscale(array: NDArray) -> NDArray: """ Converts `array` into HW or NHWC grayscale. This is intended for converting an RGB image that is already visibly grayscale, such as a depth map. It will not make a good approximation of perceived lightness of an otherwise colored image. """ if array.ndim == 2: return array c = channels(array) if array.ndim == 3: if c in [1, 2]: return array[..., 0] elif c in [3, 4]: return np.max(array[..., :3], axis=-1) raise ValueError(f"Can't make {c} channels grayscale") elif array.ndim == 4: if c in [1, 2]: return array[..., :1] elif c in [3, 4]: return np.max(array[..., :3], axis=-1, keepdims=True) raise ValueError(f"Can't make {c} channels grayscale") raise ValueError(f"Can't make {array.ndim} dimensions grayscale") def _passthrough_alpha(from_array, to_array): if channels(from_array) not in [2, 4]: return to_array to_array = np.concatenate([ensure_channel_dim(to_array), from_array[..., -1:]], axis=-1) return to_array def linear_to_srgb(array: NDArray, clamp=True) -> NDArray: """ Args: array: Image to convert from linear to sRGB color space. Will be converted to float32 if it isn't already a float dtype. clamp: whether to restrict the result between 0..1 """ if not np.issubdtype(array.dtype, np.floating): array = to_dtype(array, np.float32) srgb = ensure_opaque(array) srgb = np.where( srgb <= 0.0031308, srgb * 12.92, (np.abs(srgb) ** (1/2.4) * 1.055) - 0.055 # abs() to suppress `RuntimeWarning: invalid value encountered in power` for negative values ) if clamp: # conversion may produce values outside standard range, usually >1 srgb = np.clip(srgb, 0, 1) srgb = _passthrough_alpha(array, srgb) return srgb def srgb_to_linear(array: NDArray) -> NDArray: """ Converts from sRGB to linear color space. Will be converted to float32 if it isn't already a float dtype. """ if not np.issubdtype(array.dtype, np.floating): array = to_dtype(array, np.float32) linear = ensure_opaque(array) linear = np.where( linear <= 0.04045, linear / 12.92, ((linear + 0.055) / 1.055) ** 2.4 ) linear = _passthrough_alpha(array, linear) return linear @RunInSubprocess.when_raised def color_transform(array: NDArray, from_color_space: str, to_color_space: str, *, clamp_srgb=True) -> NDArray: """ Args: array: Pixel values in `from_color_space` from_color_space: Color space of `array` to_color_space: Desired color space clamp_srgb: Restrict values inside the standard range when converting to sRGB. Returns: Pixel values in `to_color_space`. The image will be converted to RGB/RGBA float32 for most transforms. Transforms between linear and sRGB may remain grayscale and keep the original DType if it was floating point. """ # Blender handles Raw and Non-Color images as if they were in Linear color space. if from_color_space in ["Raw", "Non-Color"]: from_color_space = "Linear" if to_color_space in ["Raw", "Non-Color"]: to_color_space = "Linear" if from_color_space == to_color_space: return array elif from_color_space == "Linear" and to_color_space == "sRGB": return linear_to_srgb(array, clamp_srgb) elif from_color_space == "sRGB" and to_color_space == "Linear": return srgb_to_linear(array) if not has_ocio: raise RunInSubprocess import PyOpenColorIO as OCIO config = OCIO.Config.CreateFromFile(OCIO_CONFIG) proc = config.getProcessor(from_color_space, to_color_space).getDefaultCPUProcessor() # OCIO requires RGB/RGBA float32. # There is a channel agnostic apply(), but I can't seem to get it to work. # getOptimizedCPUProcessor() can handle different precisions, but I doubt it would have meaningful use. array = to_dtype(array, np.float32) c = channels(array) if c in [1, 3]: array = rgb(array) proc.applyRGB(array) if clamp_srgb and to_color_space == "sRGB": array = np.clip(array, 0, 1) return array elif c in [2, 4]: array = rgba(array) proc.applyRGBA(array) if clamp_srgb and to_color_space == "sRGB": array = np.clip(array, 0, 1) return array raise ValueError(f"Can't color transform {c} channels") # inverse=True is often crashing from EXCEPTION_ACCESS_VIOLATION while on frontend. # Normally this is caused by not running on the main thread or accessing a deleted # object, neither seem to be the issue here. Doesn't matter if the backend imports # its own OCIO or the one packaged with Blender. # Stack trace: # OpenColorIO_2_2.dll :0x00007FFDE8961160 OpenColorIO_v2_2::GradingTone::validate # OpenColorIO_2_2.dll :0x00007FFDE8A2BD40 OpenColorIO_v2_2::Processor::isNoOp # OpenColorIO_2_2.dll :0x00007FFDE882EA00 OpenColorIO_v2_2::CPUProcessor::apply # PyOpenColorIO.pyd :0x00007FFDEB0F0E40 pybind11::error_already_set::what # PyOpenColorIO.pyd :0x00007FFDEB0F0E40 pybind11::error_already_set::what # PyOpenColorIO.pyd :0x00007FFDEB0F0E40 pybind11::error_already_set::what # PyOpenColorIO.pyd :0x00007FFDEB0E7510 pybind11::error_already_set::discard_as_unraisable @RunInSubprocess.when(lambda *_, inverse=False, **__: inverse or not has_ocio) def render_color_transform( array: NDArray, exposure: float, gamma: float, view_transform: str, display_device: str, look: str, *, inverse: bool = False, color_space: str | None = None, clamp_srgb: bool = True, ) -> NDArray: import PyOpenColorIO as OCIO ocio_config = OCIO.Config.CreateFromFile(OCIO_CONFIG) # A reimplementation of `OCIOImpl::createDisplayProcessor` from the Blender source. # https://github.com/blender/blender/blob/3816fcd8611bc2836ee8b2a5225b378a02141ce4/intern/opencolorio/ocio_impl.cc#L666 # Modified to support a final color space transform. def create_display_processor( config, input_colorspace, view, display, look, scale, # Exposure exponent, # Gamma inverse, color_space ): group = OCIO.GroupTransform() # Exposure if scale != 1: # Always apply exposure in scene linear. color_space_transform = OCIO.ColorSpaceTransform() color_space_transform.setSrc(input_colorspace) color_space_transform.setDst(OCIO.ROLE_SCENE_LINEAR) group.appendTransform(color_space_transform) # Make further transforms aware of the color space change input_colorspace = OCIO.ROLE_SCENE_LINEAR # Apply scale matrix_transform = OCIO.MatrixTransform( [scale, 0.0, 0.0, 0.0, 0.0, scale, 0.0, 0.0, 0.0, 0.0, scale, 0.0, 0.0, 0.0, 0.0, 1.0]) group.appendTransform(matrix_transform) # Add look transform use_look = look is not None and len(look) > 0 if use_look: look_output = config.getLook(look).getProcessSpace() if look_output is not None and len(look_output) > 0: look_transform = OCIO.LookTransform() look_transform.setSrc(input_colorspace) look_transform.setDst(look_output) look_transform.setLooks(look) group.appendTransform(look_transform) # Make further transforms aware of the color space change. input_colorspace = look_output else: # For empty looks, no output color space is returned. use_look = False # Add view and display transform display_view_transform = OCIO.DisplayViewTransform() display_view_transform.setSrc(input_colorspace) display_view_transform.setLooksBypass(use_look) display_view_transform.setView(view) display_view_transform.setDisplay(display) group.appendTransform(display_view_transform) if color_space is not None: group.appendTransform(OCIO.ColorSpaceTransform(input_colorspace if display == "None" else display, color_space)) # Gamma if exponent != 1: exponent_transform = OCIO.ExponentTransform([exponent, exponent, exponent, 1.0]) group.appendTransform(exponent_transform) if inverse: group.setDirection(OCIO.TransformDirection.TRANSFORM_DIR_INVERSE) # Create processor from transform. This is the moment were OCIO validates # the entire transform, no need to check for the validity of inputs above. return config.getProcessor(group) # Exposure and gamma transformations derived from Blender source: # https://github.com/blender/blender/blob/3816fcd8611bc2836ee8b2a5225b378a02141ce4/source/blender/imbuf/intern/colormanagement.cc#L867 scale = 2 ** exposure exponent = 1 / max(gamma, np.finfo(np.float32).eps) processor = create_display_processor(ocio_config, OCIO.ROLE_SCENE_LINEAR, view_transform, display_device, look if look != 'None' else None, scale, exponent, inverse, color_space) array = to_dtype(array, np.float32) c = channels(array) if c in [1, 3]: array = rgb(array) processor.getDefaultCPUProcessor().applyRGB(array) elif c in [2, 4]: array = rgba(array) processor.getDefaultCPUProcessor().applyRGBA(array) else: raise ValueError(f"Can't color transform {c} channels") if clamp_srgb and (color_space == "sRGB" or (display_device == "sRGB" and color_space is None)) and not inverse: array = np.clip(array, 0, 1) return array def scene_color_transform(array: NDArray, scene: Union["bpy.types.Scene", None] = None, *, inverse: bool = False, color_space: str | None = None, clamp_srgb=True) -> NDArray: if scene is None: import bpy scene = bpy.context.scene view = scene.view_settings display = scene.display_settings.display_device return render_color_transform( array, view.exposure, view.gamma, view.view_transform, display, view.look, inverse=inverse, clamp_srgb=clamp_srgb, color_space=color_space ) def _unsigned(dtype: DTypeLike) -> DTypeLike: match bits := np.iinfo(dtype).bits: case 8: return np.uint8 case 16: return np.uint16 case 32: return np.uint32 case 64: return np.uint64 raise ValueError(f"unexpected bit depth {bits} from {repr(dtype)}") def to_dtype(array: NDArray, dtype: DTypeLike) -> NDArray: """ Remaps values with respect to ranges rather than simply casting for integer DTypes. `integer(0)=float(0)`, `integer.MAX=float(1)`, and signed `integer.MIN+1=float(-1)` """ dtype = np.dtype(dtype) from_dtype = array.dtype if dtype == from_dtype: return array from_floating = np.issubdtype(from_dtype, np.floating) from_integer = np.issubdtype(from_dtype, np.integer) to_floating = np.issubdtype(dtype, np.floating) to_integer = np.issubdtype(dtype, np.integer) if from_floating and to_floating: array = array.astype(dtype) if np.finfo(from_dtype).bits > np.finfo(dtype).bits: # prevent inf when lowering precision array = np.nan_to_num(array) elif from_floating and to_integer: iinfo = np.iinfo(dtype) array = (array.clip(-1 if iinfo.min < 0 else 0, 1) * iinfo.max).round().astype(dtype) elif from_integer and to_floating: iinfo = np.iinfo(from_dtype) array = (array / iinfo.max).astype(dtype) elif from_integer and to_integer: from_signed = np.issubdtype(from_dtype, np.signedinteger) to_signed = np.issubdtype(dtype, np.signedinteger) from_bits = np.iinfo(from_dtype).bits to_bits = np.iinfo(dtype).bits if from_signed: from_bits -= 1 if to_signed: to_bits -= 1 bit_diff = to_bits - from_bits if from_signed and not to_signed: # unsigned output does not support negative array = np.maximum(array, 0) if from_signed and to_signed: # simpler to handle bit manipulation in unsigned sign = np.sign(array) array = np.abs(array) if bit_diff > 0: # Repeat bits rather than using a single left shift # so that from_iinfo.max turns into to_iinfo.max # and all values remain equally spaced. # Example 8 to 16 bits: # (incorrect) 0x00FF << 8 = 0xFF00 # (correct) 0x00FF << 8 | 0x00FF = 0xFFFF # Implementation uses multiplication instead of potentially multiple left shifts and ors: # 0x00FF * 0x0101 = 0xFFFF base = array.astype(_unsigned(dtype)) m = 0 for i in range(bit_diff, -1, -from_bits): m += 2 ** i array = base * m remaining_bits = bit_diff % from_bits if remaining_bits > 0: # when changing between signed and unsigned bit_diff is not a multiple of from_bits array |= base >> (from_bits-remaining_bits) elif bit_diff < 0: array = array.astype(_unsigned(from_dtype), copy=False) >> -bit_diff if from_signed and to_signed: array = np.multiply(array, sign, dtype=dtype) array = array.astype(dtype, copy=False) else: raise TypeError(f"Unable to convert from {array.dtype} to {dtype}") return array @RunInSubprocess.when(not has_oiio) def resize(array: NDArray, size: Tuple[int, int], clamp=True): no_channels = array.ndim == 2 if no_channels: array = array[..., np.newaxis] no_batch = array.ndim < 4 if no_batch: array = array[np.newaxis, ...] if clamp: c_min = np.min(array, axis=(1, 2), keepdims=True) c_max = np.max(array, axis=(1, 2), keepdims=True) if has_oiio: import OpenImageIO as oiio resized = [] for unbatched in array: # OpenImageIO can have batched images, but doesn't support resizing them image_in = oiio.ImageBuf(unbatched) image_out = oiio.ImageBufAlgo.resize(image_in, roi=oiio.ROI(0, int(size[0]), 0, int(size[1]))) if image_out.has_error: raise Exception(image_out.geterror()) resized.append(image_out.get_pixels(image_in.spec().format)) array = np.stack(resized) else: original_dtype = array.dtype if np.issubdtype(original_dtype, np.floating): if original_dtype == np.float16: # interpolation not implemented for float16 on CPU array = to_dtype(array, np.float32) elif np.issubdtype(original_dtype, np.integer): # integer interpolation only supported for uint8 nearest, nearest-exact or bilinear bits = np.iinfo(original_dtype).bits array = to_dtype(array, np.float64 if bits >= 32 else np.float32) import torch array = torch.from_numpy(np.transpose(array, (0, 3, 1, 2))) array = torch.nn.functional.interpolate(array, size=(size[1], size[0]), mode="bilinear") array = np.transpose(array, (0, 2, 3, 1)).numpy() array = to_dtype(array, original_dtype) if clamp: array = np.clip(array, c_min, c_max) if no_batch: array = np.squeeze(array, 0) if no_channels: array = np.squeeze(array, -1) return array def bpy_to_np(image: "bpy.types.Image", *, color_space: str | None = "sRGB", clamp_srgb=True, top_to_bottom=True) -> NDArray: """ Args: image: Image to extract pixels values from. color_space: The color space to convert to. `None` will apply no color transform. Keep in mind that Raw/Non-Color images are handled as if they were in Linear color space. clamp_srgb: Restrict values inside the standard range when converting to sRGB. top_to_bottom: The y-axis is flipped to a more common standard of `top=0` to `bottom=height-1`. Returns: A ndarray copy of `image.pixels` in RGBA float32 format. """ if image.type == "RENDER_RESULT": # can't get pixels automatically without rendering again and freezing Blender until it finishes, or saving to disk raise ValueError(f"{image.name} image can't be used directly, alternatively use a compositor viewer node") array = np.empty((image.size[1], image.size[0], image.channels), dtype=np.float32) # foreach_get/set is extremely fast to read/write an entire image compared to alternatives # see https://projects.blender.org/blender/blender/commit/9075ec8269e7cb029f4fab6c1289eb2f1ae2858a image.pixels.foreach_get(array.ravel()) if color_space is not None: if image.type == "COMPOSITING": # Viewer Node array = scene_color_transform(array, color_space=color_space, clamp_srgb=clamp_srgb) else: array = color_transform(array, image.colorspace_settings.name, color_space, clamp_srgb=clamp_srgb) if top_to_bottom: array = np.flipud(array) return rgba(array) def np_to_bpy(array: NDArray, name=None, existing_image=None, float_buffer=None, color_space: str = "sRGB", top_to_bottom=True) -> "bpy.types.Image": """ Args: array: Image pixel values. The y-axis is expected to be ordered `top=0` to `bottom=height-1`. name: Name of the image data-block. If None it will be `existing_image.name` or "Untitled". existing_image: Image data-block to overwrite. float_buffer: Make Blender keep data in (`True`) 32-bit float values, or (`False`) 8-bit integer values. `None` won't invalidate `existing_image`, but if a new image is created it will be `False`. color_space: Color space of `array`. Returns: A new Blender image or `existing_image` if it didn't require replacement. """ if array.ndim == 4 and array.shape[0] > 1: raise ValueError(f"Can't convert a batched array of {array.shape[0]} images to a Blender image") # create or replace image import bpy width, height = size(array) if name is None: name = "Untitled" if existing_image is None else existing_image.name if existing_image is not None and existing_image.type in ["RENDER_RESULT", "COMPOSITING"]: existing_image = None elif existing_image is not None and ( existing_image.size[0] != width or existing_image.size[1] != height or (existing_image.channels != channels(array) and existing_image.channels != 4) or (existing_image.is_float != float_buffer and float_buffer is not None) ): bpy.data.images.remove(existing_image) existing_image = None if existing_image is None: image = bpy.data.images.new( name, width=width, height=height, alpha=channels(array) == 4, float_buffer=False if float_buffer is None else float_buffer ) else: image = existing_image image.name = name image.colorspace_settings.name = color_space # adjust array pixels to fit into image if array.ndim == 4: array = array[0] if top_to_bottom: array = np.flipud(array) array = to_dtype(array, np.float32) if image.channels == 4: array = rgba(array) elif image.channels == 3: # I believe image.channels only exists for backwards compatibility and modern versions of Blender # will always handle images as RGBA. I can't manage to make or import an image and end up with # anything but 4 channels. Support for images with 3 channels will be kept just in case. array = rgb(array) else: raise NotImplementedError(f"Blender image unexpectedly has {image.channels} channels") # apply pixels to image image.pixels.foreach_set(array.ravel()) image.pack() image.update() return image def render_pass_to_np( render_pass: "bpy.types.RenderPass", size: Tuple[int, int], *, color_management: bool = False, color_space: str | None = None, clamp_srgb: bool = True, top_to_bottom: bool = True ): array = np.empty((*reversed(size), render_pass.channels), dtype=np.float32) if BLENDER_VERSION >= (4, 1, 0): render_pass.rect.foreach_get(array.reshape(-1)) else: render_pass.rect.foreach_get(array.reshape(-1, render_pass.channels)) if color_management: array = scene_color_transform(array, color_space=color_space, clamp_srgb=clamp_srgb) elif color_space is not None: array = color_transform(array, "Linear", color_space, clamp_srgb=clamp_srgb) if top_to_bottom: array = np.flipud(array) return array def np_to_render_pass( array: NDArray, render_pass: "bpy.types.RenderPass", *, inverse_color_management: bool = False, color_space: str | None = None, dtype: DTypeLike = np.float32, top_to_bottom: bool = True ): if inverse_color_management: array = scene_color_transform(array, inverse=True, color_space=color_space) elif color_space is not None: array = color_transform(color_space, "Linear") if channels(array) != render_pass.channels: match render_pass.channels: case 1: array = grayscale(array) case 3: array = rgb(array) case 4: array = rgba(array) case _: raise NotImplementedError(f"Render pass {render_pass.name} unexpectedly requires {render_pass.channels} channels") if dtype is not None: array = to_dtype(array, dtype) if top_to_bottom: array = np.flipud(array) if BLENDER_VERSION >= (4, 1, 0): render_pass.rect.foreach_set(array.reshape(-1)) else: render_pass.rect.foreach_set(array.reshape(-1, render_pass.channels)) def _mode(array, mode): if mode is None: return array elif mode == "RGBA": return rgba(array) elif mode == "RGB": return rgb(array) elif mode == "L": return grayscale(array) elif mode == "LA": return ensure_alpha(_passthrough_alpha(array, grayscale(array))) raise ValueError(f"mode expected one of {['RGB', 'RGBA', 'L', 'LA', None]}, got {repr(mode)}") def pil_to_np(image, *, dtype: DTypeLike | None = np.float32, mode: Literal["RGB", "RGBA", "L", "LA"] | None = None) -> NDArray: # some modes don't require being converted to RGBA for proper handling in other module functions # see for other modes https://pillow.readthedocs.io/en/stable/handbook/concepts.html#concept-modes if image.mode not in ["RGB", "RGBA", "L", "LA", "I", "F", "I;16"]: image = image.convert("RGBA") array = np.array(image) if dtype is not None: array = to_dtype(array, dtype) array = _mode(array, mode) return array def np_to_pil(array: NDArray, *, mode: Literal["RGB", "RGBA", "L", "LA"] | None = None): from PIL import Image array = to_dtype(array, np.uint8) if mode is None: if channels(array) == 1 and array.ndim == 3: # PIL L mode can't have a channel dimension array = array[..., 1] else: array = _mode(array, mode) # PIL does support higher precision modes for a single channel, but I don't see a need for supporting them yet. # uint16="I;16", int32="I", float32="F" return Image.fromarray(array, mode=mode) def _dtype_to_type_desc(dtype): import OpenImageIO as oiio dtype = np.dtype(dtype) match dtype: case np.uint8: return oiio.TypeUInt8 case np.uint16: return oiio.TypeUInt16 case np.uint32: return oiio.TypeUInt32 case np.uint64: return oiio.TypeUInt64 case np.int8: return oiio.TypeInt8 case np.int16: return oiio.TypeInt16 case np.int32: return oiio.TypeInt32 case np.int64: return oiio.TypeInt64 case np.float16: return oiio.TypeHalf case np.float32: return oiio.TypeFloat case np.float64: # no oiio.TypeDouble return oiio.TypeDesc(oiio.BASETYPE.DOUBLE) raise TypeError(f"can't convert {dtype} to OpenImageIO.TypeDesc") @RunInSubprocess.when(not has_oiio) def path_to_np( path: str | PathLike, *, dtype: DTypeLike | None = np.float32, default_color_space: str | None = None, to_color_space: str | None = "sRGB" ) -> NDArray: """ Args: path: Path to an image file. dtype: Data type of the returned array. `None` won't change the data type. The data type may still change if a color transform occurs. default_color_space: The color space that `image_or_path` will be handled as when it can't be determined automatically. to_color_space: Color space of the returned array. `None` won't apply a color transform. """ if has_oiio: import OpenImageIO as oiio image = oiio.ImageInput.open(str(path)) if image is None: raise IOError(oiio.geterror()) type_desc = image.spec().format if dtype is not None: type_desc = _dtype_to_type_desc(dtype) array = image.read_image(type_desc) from_color_space = image.spec().get_string_attribute("oiio:ColorSpace", default_color_space) image.close() else: from PIL import Image array = pil_to_np(Image.open(path)) if dtype is not None: array = to_dtype(array, dtype) from_color_space = "sRGB" if from_color_space is not None and to_color_space is not None: array = color_transform(array, from_color_space, to_color_space) return array ImageOrPath = Union[NDArray, "PIL.Image.Image", str, PathLike] """Backend compatible image types""" def image_to_np( image_or_path: ImageOrPath | "bpy.types.Image" | None, *, dtype: DTypeLike | None = np.float32, mode: Literal["RGB", "RGBA", "L", "LA"] | None = "RGBA", default_color_space: str | None = None, to_color_space: str | None = "sRGB", size: Tuple[int, int] | None = None, top_to_bottom: bool = True ) -> NDArray: """ Opens an image from disk or takes an image object and converts it to `numpy.ndarray`. Usable for image argument sanitization when the source can vary in type or format. Args: image_or_path: Either a file path or an instance of `bpy.types.Image`, `PIL.Image.Image`, or `numpy.ndarray`. `None` will return `None`. dtype: Data type of the returned array. `None` won't change the data type. The data type may still change if a color transform occurs. mode: Channel mode of the returned array. `None` won't change the mode. The mode may still change if a color transform occurs. default_color_space: The color space that `image_or_path` will be handled as when it can't be determined automatically. to_color_space: Color space of the returned array. `None` won't apply a color transform. size: Resize to specific dimensions. `None` won't change the size. top_to_bottom: Flips the image like `bpy_to_np(top_to_bottom=True)` does when `True` and `image_or_path` is a Blender image. Other image sources will only be flipped when `False`. """ if image_or_path is None: return None # convert image_or_path to numpy.ndarray match image_or_path: case PathLike() | str(): array = path_to_np(image_or_path, dtype=dtype, default_color_space=default_color_space, to_color_space=to_color_space) from_color_space = None case object(__module__="PIL.Image", __class__=type(__name__="Image")): # abnormal class check because PIL cannot be imported on frontend array = pil_to_np(image_or_path) from_color_space = "sRGB" case object(__module__="bpy.types", __class__=type(__name__="Image")): # abnormal class check because bpy cannot be imported on backend array = bpy_to_np(image_or_path, color_space=to_color_space) from_color_space = None case np.ndarray(): array = image_or_path from_color_space = default_color_space case _: raise TypeError(f"not an image or path {repr(type(image_or_path))}") # apply image requirements if not top_to_bottom: array = np.flipud(array) if from_color_space is not None and to_color_space is not None: array = color_transform(array, from_color_space, to_color_space) if dtype is not None: array = to_dtype(array, dtype) array = _mode(array, mode) if size is not None: array = resize(array, size) return array ================================================ FILE: operators/dream_texture.py ================================================ import bpy import hashlib import numpy as np from typing import List, Literal from .notify_result import NotifyResult from ..prompt_engineering import * from ..generator_process import Generator from .. import api from .. import image_utils from ..generator_process.models.optimizations import Optimizations from ..diffusers_backend import DiffusersBackend import time import math def get_source_image(context, source: Literal['file', 'open_editor']): match source: case 'file': return context.scene.init_img case 'open_editor': if context.area.type == 'IMAGE_EDITOR': return context.area.spaces.active.image else: init_image = None for area in context.screen.areas: if area.type == 'IMAGE_EDITOR': if area.spaces.active.image is not None: init_image = area.spaces.active.image return init_image case _: raise ValueError(f"unsupported source {repr(source)}") class DreamTexture(bpy.types.Operator): bl_idname = "shade.dream_texture" bl_label = "Dream Texture" bl_description = "Generate a texture with AI" bl_options = {'REGISTER'} @classmethod def poll(cls, context): try: prompt = context.scene.dream_textures_prompt backend: api.Backend = prompt.get_backend() backend.validate(prompt.generate_args(context)) except: return False return Generator.shared().can_use() def execute(self, context): screen = context.screen scene = context.scene prompt = scene.dream_textures_prompt backend: api.Backend = prompt.get_backend() history_template = {prop: getattr(context.scene.dream_textures_prompt, prop) for prop in context.scene.dream_textures_prompt.__annotations__.keys()} history_template["iterations"] = 1 history_template["random_seed"] = False is_file_batch = context.scene.dream_textures_prompt.prompt_structure == file_batch_structure.id file_batch_lines = [] if is_file_batch: context.scene.dream_textures_prompt.iterations = 1 file_batch_lines = [line.body for line in context.scene.dream_textures_prompt_file.lines if len(line.body.strip()) > 0] history_template["prompt_structure"] = custom_structure.id node_tree = context.material.node_tree if hasattr(context, 'material') and hasattr(context.material, 'node_tree') else None node_tree_center = np.array(node_tree.view_center) if node_tree is not None else None node_tree_top_left = np.array(context.region.view2d.region_to_view(0, context.region.height)) if node_tree is not None else None screen = context.screen scene = context.scene generated_args = scene.dream_textures_prompt.generate_args(context) context.scene.seamless_result.update_args(generated_args) context.scene.seamless_result.update_args(history_template, as_id=True) def execute_backend(control_images): # Setup the progress indicator bpy.types.Scene.dream_textures_progress = bpy.props.IntProperty(name="", default=0, min=0, max=generated_args.steps) scene.dream_textures_info = "Starting..." # Get any init images try: init_image = get_source_image(context, prompt.init_img_src) except ValueError: init_image = None if init_image is not None: init_image_color_space = "sRGB" if scene.dream_textures_prompt.use_init_img and scene.dream_textures_prompt.modify_action_source_type in ['depth_map', 'depth']: init_image_color_space = None init_image = image_utils.bpy_to_np(init_image, color_space=init_image_color_space) # Callbacks last_data_block = None execution_start = time.time() def step_callback(progress: List[api.GenerationResult]) -> bool: nonlocal last_data_block scene.dream_textures_last_execution_time = f"{time.time() - execution_start:.2f} seconds" scene.dream_textures_progress = progress[-1].progress for area in context.screen.areas: for region in area.regions: if region.type == "UI": region.tag_redraw() image = api.GenerationResult.tile_images(progress) if image is None: return CancelGenerator.should_continue last_data_block = image_utils.np_to_bpy(image, f"Step {progress[-1].progress}/{progress[-1].total}", last_data_block) for area in screen.areas: if area.type == 'IMAGE_EDITOR' and not area.spaces.active.use_image_pin: area.spaces.active.image = last_data_block return CancelGenerator.should_continue iteration = 0 iteration_limit = len(file_batch_lines) if is_file_batch else generated_args.iterations iteration_square = math.ceil(math.sqrt(iteration_limit)) node_pad = np.array((20, 20)) node_size = np.array((240, 277)) + node_pad if node_tree is not None: # keep image nodes grid centered but don't go beyond top and left sides of nodes editor node_anchor = node_tree_center + node_size * 0.5 * (-iteration_square, (iteration_limit-1) // iteration_square + 1) node_anchor = np.array((np.maximum(node_tree_top_left[0], node_anchor[0]), np.minimum(node_tree_top_left[1], node_anchor[1]))) + node_pad * (0.5, -0.5) def callback(results: List[api.GenerationResult] | Exception): if isinstance(results, Exception): scene.dream_textures_info = "" scene.dream_textures_progress = 0 CancelGenerator.should_continue = None if not isinstance(results, InterruptedError): # this is a user-initiated cancellation eval('bpy.ops.' + NotifyResult.bl_idname)('INVOKE_DEFAULT', exception=repr(results)) raise results else: nonlocal last_data_block nonlocal iteration for result in results: if result.image is None or result.seed is None: continue # Create a trimmed image name prompt_string = context.scene.dream_textures_prompt.prompt_structure_token_subject seed_str_length = len(str(result.seed)) trim_aware_name = (prompt_string[:54 - seed_str_length] + '..') if len(prompt_string) > 54 else prompt_string name_with_trimmed_prompt = f"{trim_aware_name} ({result.seed})" image = image_utils.np_to_bpy(result.image, name_with_trimmed_prompt, last_data_block) last_data_block = None if node_tree is not None: nodes = node_tree.nodes texture_node = nodes.new("ShaderNodeTexImage") texture_node.image = image texture_node.location = node_anchor + node_size * ((iteration % iteration_square), -(iteration // iteration_square)) nodes.active = texture_node for area in screen.areas: if area.type == 'IMAGE_EDITOR' and not area.spaces.active.use_image_pin: area.spaces.active.image = image scene.dream_textures_prompt.seed = str(result.seed) # update property in case seed was sourced randomly or from hash # create a hash from the Blender image datablock to use as unique ID of said image and store it in the prompt history # and as custom property of the image. Needs to be a string because the int from the hash function is too large image_hash = hashlib.sha256((np.array(image.pixels) * 255).tobytes()).hexdigest() image['dream_textures_hash'] = image_hash scene.dream_textures_prompt.hash = image_hash history_entry = context.scene.dream_textures_history.add() for key, value in history_template.items(): match key: case 'control_nets': for net in value: n = history_entry.control_nets.add() for prop in n.__annotations__.keys(): setattr(n, prop, getattr(net, prop)) case _: setattr(history_entry, key, value) history_entry.seed = str(result.seed) history_entry.hash = image_hash history_entry.width = result.image.shape[1] history_entry.height = result.image.shape[0] if is_file_batch: history_entry.prompt_structure_token_subject = file_batch_lines[iteration] iteration += 1 if iteration < iteration_limit: generate_next() else: scene.dream_textures_info = "" scene.dream_textures_progress = 0 CancelGenerator.should_continue = None # Call the backend CancelGenerator.should_continue = True # reset global cancellation state def generate_next(): args = prompt.generate_args(context, iteration=iteration, init_image=init_image, control_images=control_images) backend.generate(args, step_callback=step_callback, callback=callback) generate_next() # Prepare ControlNet images if len(prompt.control_nets) > 0: bpy.types.Scene.dream_textures_progress = bpy.props.IntProperty(name="", default=0, min=0, max=len(prompt.control_nets)) scene.dream_textures_info = "Processing Control Images..." context.scene.dream_textures_progress = 0 gen = Generator.shared() optimizations = backend.optimizations() if isinstance(backend, DiffusersBackend) else Optimizations() control_images = [] def process_next(i): if i >= len(prompt.control_nets): execute_backend(control_images) return net = prompt.control_nets[i] future = gen.controlnet_aux( processor_id=net.processor_id, image=image_utils.bpy_to_np(net.control_image, color_space=None), optimizations=optimizations ) def on_response(future): control_images.append(future.result(last_only=True)) context.scene.dream_textures_progress = i + 1 process_next(i + 1) future.add_done_callback(on_response) process_next(0) else: execute_backend(None) return {"FINISHED"} def kill_generator(context=bpy.context): Generator.shared_close() try: context.scene.dream_textures_info = "" context.scene.dream_textures_progress = 0 CancelGenerator.should_continue = None except: pass class ReleaseGenerator(bpy.types.Operator): bl_idname = "shade.dream_textures_release_generator" bl_label = "Release Generator" bl_description = "Releases the generator class to free up VRAM" bl_options = {'REGISTER'} def execute(self, context): kill_generator(context) return {'FINISHED'} class CancelGenerator(bpy.types.Operator): bl_idname = "shade.dream_textures_stop_generator" bl_label = "Cancel Generator" bl_description = "Stops the generator without reloading everything next time" bl_options = {'REGISTER'} should_continue = None @classmethod def poll(cls, context): return cls.should_continue is not None def execute(self, context): CancelGenerator.should_continue = False return {'FINISHED'} ================================================ FILE: operators/inpaint_area_brush.py ================================================ import bpy reset_blend_mode = 'MIX' reset_curve_preset = 'CUSTOM' reset_strength = 1.0 class InpaintAreaBrushActivated(bpy.types.GizmoGroup): bl_idname = "dream_textures.inpaint_area_brush_activated" bl_label = "Inpaint Area Brush Activated" bl_space_type = 'IMAGE_EDITOR' bl_context_mode = 'PAINT' bl_region_type = 'WINDOW' def setup(self, context): global reset_blend_mode global reset_curve_preset global reset_strength reset_blend_mode = bpy.data.brushes["TexDraw"].blend reset_curve_preset = bpy.data.brushes["TexDraw"].curve_preset reset_strength = bpy.data.brushes["TexDraw"].strength def set_blend(): bpy.data.brushes["TexDraw"].blend = "ERASE_ALPHA" bpy.data.brushes["TexDraw"].curve_preset = "CONSTANT" bpy.data.brushes["TexDraw"].strength = 1.0 bpy.ops.paint.brush_select(image_tool='DRAW', toggle=False) bpy.app.timers.register(set_blend) def __del__(self): bpy.data.brushes["TexDraw"].blend = reset_blend_mode bpy.data.brushes["TexDraw"].curve_preset = reset_curve_preset bpy.data.brushes["TexDraw"].strength = reset_strength class InpaintAreaBrush(bpy.types.WorkSpaceTool): bl_space_type = 'IMAGE_EDITOR' bl_context_mode = 'PAINT' bl_idname = "dream_textures.inpaint_area_brush" bl_label = "Mark Inpaint Area" bl_description = "Mark an area for inpainting" bl_icon = "brush.gpencil_draw.tint" bl_widget = InpaintAreaBrushActivated.bl_idname def draw_settings(self, layout, tool): layout.prop(bpy.context.scene.tool_settings.unified_paint_settings, 'size') ================================================ FILE: operators/install_dependencies.py ================================================ import bpy import os import site import sys import sysconfig import subprocess import requests import tarfile from enum import IntEnum from ..absolute_path import absolute_path from ..generator_process import Generator class PipInstall(IntEnum): DEPENDENCIES = 1 STANDARD = 2 USER_SITE = 3 def install_pip(method = PipInstall.STANDARD): """ Installs pip if not already present. Please note that ensurepip.bootstrap() also calls pip, which adds the environment variable PIP_REQ_TRACKER. After ensurepip.bootstrap() finishes execution, the directory doesn't exist anymore. However, when subprocess is used to call pip, in order to install a package, the environment variables still contain PIP_REQ_TRACKER with the now nonexistent path. This is a problem since pip checks if PIP_REQ_TRACKER is set and if it is, attempts to use it as temp directory. This would result in an error because the directory can't be found. Therefore, PIP_REQ_TRACKER needs to be removed from environment variables. :return: """ import ensurepip if method == PipInstall.DEPENDENCIES: # ensurepip doesn't have a useful way of installing to a specific directory. # root parameter can be used, but it just concatenates that to the beginning of # where it decides to install to, causing a more complicated path to where it installs. wheels = {} for name, package in ensurepip._get_packages().items(): if package.wheel_name: whl = os.path.join(os.path.dirname(ensurepip.__file__), "_bundled", package.wheel_name) else: whl = package.wheel_path wheels[name] = whl pip_whl = os.path.join(wheels['pip'], 'pip') subprocess.run([sys.executable, pip_whl, "install", *wheels.values(), "--upgrade", "--no-index", "--no-deps", "--no-cache-dir", "--target", absolute_path(".python_dependencies")], check=True) return # STANDARD or USER_SITE no_user = os.environ.get("PYTHONNOUSERSITE", None) if method == PipInstall.STANDARD: os.environ["PYTHONNOUSERSITE"] = "1" else: os.environ.pop("PYTHONNOUSERSITE", None) try: ensurepip.bootstrap(user=method==PipInstall.USER_SITE) finally: os.environ.pop("PIP_REQ_TRACKER", None) if no_user: os.environ["PYTHONNOUSERSITE"] = no_user else: os.environ.pop("PYTHONNOUSERSITE", None) def install_pip_any(*methods): methods = methods or PipInstall for method in methods: print(f"Attempting to install pip: {PipInstall(method).name}") try: install_pip(method) return method except: import traceback traceback.print_exc() def get_pip_install(): def run(pip): if os.path.exists(pip): try: subprocess.run([sys.executable, pip, "--version"], check=True) return True except subprocess.CalledProcessError: pass return False if run(absolute_path(".python_dependencies/pip")): return PipInstall.DEPENDENCIES # This seems to not raise CalledProcessError while debugging in vscode, but works fine in normal use. # subprocess.run([sys.executable, "-s", "-m", "pip", "--version"], check=True) # Best to check if the module directory exists first. for path in site.getsitepackages(): if run(os.path.join(path,"pip")): return PipInstall.STANDARD if run(os.path.join(site.getusersitepackages(),"pip")): return PipInstall.USER_SITE def install_and_import_requirements(requirements_txt=None, pip_install=PipInstall.STANDARD): """ Installs all modules in the 'requirements.txt' file. """ environ_copy = dict(os.environ) if pip_install != PipInstall.USER_SITE: environ_copy["PYTHONNOUSERSITE"] = "1" if pip_install == PipInstall.DEPENDENCIES: environ_copy["PYTHONPATH"] = absolute_path(".python_dependencies") python_include_dir = sysconfig.get_paths()['include'] if not os.path.exists(python_include_dir): try: os.makedirs(python_include_dir) finally: pass if os.access(python_include_dir, os.W_OK): print("downloading additional include files") python_devel_tgz_path = absolute_path('python-devel.tgz') response = requests.get(f"https://www.python.org/ftp/python/{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}/Python-{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}.tgz") with open(python_devel_tgz_path, 'wb') as f: f.write(response.content) with tarfile.open(python_devel_tgz_path) as python_devel_tgz: def members(tf): prefix = f"Python-{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}/Include/" l = len(prefix) for member in tf.getmembers(): if member.path.startswith(prefix): member.path = member.path[l:] yield member python_devel_tgz.extractall(path=python_include_dir, members=members(python_devel_tgz)) os.remove(python_devel_tgz_path) else: print(f"skipping include files, can't write to {python_include_dir}",file=sys.stderr) subprocess.run([sys.executable, "-m", "pip", "install", "-r", absolute_path(requirements_txt), "--upgrade", "--no-cache-dir", "--target", absolute_path('.python_dependencies')], check=True, env=environ_copy, cwd=absolute_path("")) class InstallDependencies(bpy.types.Operator): bl_idname = "stable_diffusion.install_dependencies" bl_label = "Install Dependencies" bl_description = ("Downloads and installs the required python packages into the '.python_dependencies' directory of the addon.") bl_options = {"REGISTER", "INTERNAL"} def invoke(self, context, event): return context.window_manager.invoke_confirm(self, event) def execute(self, context): # Open the console so we can watch the progress. if sys.platform == 'win32': bpy.ops.wm.console_toggle() Generator.shared_close() try: pip_install = get_pip_install() if pip_install is None: pip_install = install_pip_any() if pip_install is None: raise ImportError(f'Pip could not be installed. You may have to manually install pip into {absolute_path(".python_dependencies")}') install_and_import_requirements(requirements_txt=context.scene.dream_textures_requirements_path, pip_install=pip_install) except (subprocess.CalledProcessError, ImportError) as err: self.report({"ERROR"}, str(err)) return {"CANCELLED"} return {"FINISHED"} class UninstallDependencies(bpy.types.Operator): bl_idname = "stable_diffusion.uninstall_dependencies" bl_label = "Uninstall Dependencies" bl_description = ("Uninstalls specific dependencies from Blender's site-packages") bl_options = {"REGISTER", "INTERNAL"} conflicts: bpy.props.StringProperty(name="Conflicts") def execute(self, context): # Open the console so we can watch the progress. if sys.platform == 'win32': bpy.ops.wm.console_toggle() environ_copy = dict(os.environ) environ_copy["PYTHONNOUSERSITE"] = "1" subprocess.run([sys.executable, "-m", "pip", "uninstall", "-y", *self.conflicts.split(' ')], check=True, env=environ_copy, cwd=absolute_path("")) return {"FINISHED"} ================================================ FILE: operators/notify_result.py ================================================ import bpy import os import sys class NotifyResult(bpy.types.Operator): bl_idname = "shade.dream_textures_notify_result" bl_label = "Notify Result" bl_description = "Notifies of a generation completion or any error messages" bl_options = {'REGISTER'} exception: bpy.props.StringProperty(name="Exception", default="") def modal(self, context, event): if self.exception != "": self.report({'ERROR'}, f"""An error occurred while generating. Check the issues tab on GitHub to see if this has been reported before: {self.exception}""") return {'CANCELLED'} else: return {'FINISHED'} def invoke(self, context, event): context.window_manager.modal_handler_add(self) return {'RUNNING_MODAL'} def execute(self, context): return {'FINISHED'} ================================================ FILE: operators/open_latest_version.py ================================================ import requests import bpy import webbrowser from ..version import VERSION, version_tag, version_tuple REPO_OWNER = "carson-katri" REPO_NAME = "dream-textures" latest_version = VERSION def check_for_updates(): try: global latest_version response = requests.get(f"https://api.github.com/repos/{REPO_OWNER}/{REPO_NAME}/releases") releases = response.json() latest_version = version_tuple(releases[0]['tag_name']) except: pass def new_version_available(): return not latest_version == VERSION force_show_download = False def do_force_show_download(): global force_show_download force_show_download = True def is_force_show_download(): return force_show_download class OpenLatestVersion(bpy.types.Operator): bl_idname = "stable_diffusion.open_latest_version" bl_label = f"Update Available..." bl_description = ("Opens a window to download the latest release from GitHub") bl_options = {"REGISTER", "INTERNAL"} @classmethod def poll(cls, context): return True def execute(self, context): webbrowser.open(f'https://github.com/carson-katri/dream-textures/releases/tag/{version_tag(latest_version)}') return {"FINISHED"} ================================================ FILE: operators/project.py ================================================ import bpy import gpu import gpu.texture from gpu_extras.batch import batch_for_shader import bmesh from bpy_extras import view3d_utils import mathutils import numpy as np from typing import List from .view_history import ImportPromptFile from .open_latest_version import OpenLatestVersion, is_force_show_download, new_version_available from ..ui.panels.dream_texture import advanced_panel, create_panel, prompt_panel, size_panel from .dream_texture import CancelGenerator, ReleaseGenerator from .notify_result import NotifyResult from ..generator_process import Generator from ..generator_process.models import ModelType from ..api.models import FixItError import tempfile from ..engine.annotations.depth import render_depth_map from .. import api from .. import image_utils framebuffer_arguments = [ ('depth', 'Depth', 'Only provide the scene depth as input'), ('color', 'Depth and Color', 'Provide the scene depth and color as input'), ] def _validate_projection(context): if len(context.selected_objects) == 0: def object_mode_operator(operator): operator.mode = 'OBJECT' def select_by_type_operator(operator): operator.type = 'MESH' raise FixItError( """No objects selected Select at least one object to project onto.""", FixItError.RunOperator("Switch to Object Mode", "object.mode_set", object_mode_operator) if context.object.mode != 'OBJECT' else FixItError.RunOperator("Select All Meshes", "object.select_by_type", select_by_type_operator) ) if context.object is not None and context.object.mode != 'EDIT': def fix_mode(operator): operator.mode = 'EDIT' raise FixItError( """Enter edit mode In edit mode, select the faces to project onto.""", FixItError.RunOperator("Switch to Edit Mode", "object.mode_set", fix_mode) ) has_selection = False for obj in context.selected_objects: if not hasattr(obj, "data"): continue mesh = bmesh.from_edit_mesh(obj.data) bm = mesh.copy() bm.select_mode = {'FACE'} for f in bm.faces: if f.select: has_selection = True break if not has_selection: raise FixItError( """No faces selected. Select at least one face to project onto.""", FixItError.RunOperator("Select All Faces", "mesh.select_all", lambda _: None) ) def dream_texture_projection_panels(): class DREAM_PT_dream_panel_projection(bpy.types.Panel): """Creates a Dream Textures panel for projection""" bl_label = "Dream Texture Projection" bl_idname = f"DREAM_PT_dream_panel_projection" bl_category = "Dream" bl_space_type = 'VIEW_3D' bl_region_type = 'UI' @classmethod def poll(cls, context): if cls.bl_space_type == 'NODE_EDITOR': return context.area.ui_type == "ShaderNodeTree" or context.area.ui_type == "CompositorNodeTree" else: return True def draw_header_preset(self, context): layout = self.layout layout.operator(ImportPromptFile.bl_idname, text="", icon="IMPORT") layout.separator() def draw(self, context): layout = self.layout layout.use_property_split = True layout.use_property_decorate = False if is_force_show_download(): layout.operator(OpenLatestVersion.bl_idname, icon="IMPORT", text="Download Latest Release") elif new_version_available(): layout.operator(OpenLatestVersion.bl_idname, icon="IMPORT") layout.prop(context.scene.dream_textures_project_prompt, "backend") layout.prop(context.scene.dream_textures_project_prompt, 'model') yield DREAM_PT_dream_panel_projection def get_prompt(context): return context.scene.dream_textures_project_prompt yield from create_panel('VIEW_3D', 'UI', DREAM_PT_dream_panel_projection.bl_idname, prompt_panel, get_prompt) yield create_panel('VIEW_3D', 'UI', DREAM_PT_dream_panel_projection.bl_idname, size_panel, get_prompt) yield from create_panel('VIEW_3D', 'UI', DREAM_PT_dream_panel_projection.bl_idname, advanced_panel, get_prompt) def actions_panel(sub_panel, space_type, get_prompt): class ActionsPanel(sub_panel): """Create a subpanel for actions""" bl_idname = f"DREAM_PT_dream_panel_projection_actions" bl_label = "Actions" bl_options = {'HIDE_HEADER'} def draw(self, context): super().draw(context) layout = self.layout layout.use_property_split = True prompt = get_prompt(context) layout.prop(context.scene, "dream_textures_project_framebuffer_arguments") if context.scene.dream_textures_project_framebuffer_arguments == 'color': layout.prop(prompt, "strength") col = layout.column() col.prop(context.scene, "dream_textures_project_use_control_net") if context.scene.dream_textures_project_use_control_net and len(prompt.control_nets) > 0: col.prop(prompt.control_nets[0], "control_net", text="Depth ControlNet") col.prop(prompt.control_nets[0], "conditioning_scale", text="ControlNet Conditioning Scale") col.prop(context.scene, "dream_textures_project_bake") if context.scene.dream_textures_project_bake: for obj in context.selected_objects: col.prop_search(obj.data.uv_layers, "active", obj.data, "uv_layers", text=f"{obj.name} Target UVs") row = layout.row(align=True) row.scale_y = 1.5 if CancelGenerator.poll(context): row.operator(CancelGenerator.bl_idname, icon="SNAP_FACE", text="") if context.scene.dream_textures_progress <= 0: if context.scene.dream_textures_info != "": disabled_row = row.row(align=True) disabled_row.operator(ProjectDreamTexture.bl_idname, text=context.scene.dream_textures_info, icon="INFO") disabled_row.enabled = False else: r = row.row(align=True) r.operator(ProjectDreamTexture.bl_idname, icon="MOD_UVPROJECT") r.enabled = context.object is not None and context.object.mode == 'EDIT' else: disabled_row = row.row(align=True) disabled_row.use_property_split = True disabled_row.prop(context.scene, 'dream_textures_progress', slider=True) disabled_row.enabled = False row.operator(ReleaseGenerator.bl_idname, icon="X", text="") # Validation try: _validate_projection(context) prompt = context.scene.dream_textures_project_prompt backend: api.Backend = prompt.get_backend() args = prompt.generate_args(context) args.task = api.task.PromptToImage() if context.scene.dream_textures_project_use_control_net else api.task.DepthToImage(None, None, 0) backend.validate(args) except FixItError as e: error_box = layout.box() error_box.use_property_split = False for i, line in enumerate(e.args[0].split('\n')): error_box.label(text=line, icon="ERROR" if i == 0 else "NONE") e._draw(context.scene.dream_textures_project_prompt, context, error_box) except Exception as e: print(e) return ActionsPanel yield create_panel('VIEW_3D', 'UI', DREAM_PT_dream_panel_projection.bl_idname, actions_panel, get_prompt) def bake(context, mesh, src, dest, src_uv, dest_uv): def bake_shader(): vert_out = gpu.types.GPUStageInterfaceInfo("my_interface") vert_out.smooth('VEC2', "uvInterp") shader_info = gpu.types.GPUShaderCreateInfo() shader_info.sampler(0, 'FLOAT_2D', "image") shader_info.vertex_in(0, 'VEC2', "src_uv") shader_info.vertex_in(1, 'VEC2', "dest_uv") shader_info.vertex_out(vert_out) shader_info.fragment_out(0, 'VEC4', "fragColor") shader_info.vertex_source(""" void main() { gl_Position = vec4(dest_uv * 2 - 1, 0.0, 1.0); uvInterp = src_uv; } """) shader_info.fragment_source(""" void main() { fragColor = texture(image, uvInterp); } """) return gpu.shader.create_from_info(shader_info) width, height = dest.size[0], dest.size[1] offscreen = gpu.types.GPUOffScreen(width, height) buffer = gpu.types.Buffer('FLOAT', width * height * 4, src) texture = gpu.types.GPUTexture(size=(width, height), data=buffer, format='RGBA16F') with offscreen.bind(): fb = gpu.state.active_framebuffer_get() fb.clear(color=(0.0, 0.0, 0.0, 0.0)) with gpu.matrix.push_pop(): gpu.matrix.load_matrix(mathutils.Matrix.Identity(4)) gpu.matrix.load_projection_matrix(mathutils.Matrix.Identity(4)) vertices = np.array([[l.vert.index for l in loop] for loop in mesh.calc_loop_triangles()], dtype='i') shader = bake_shader() batch = batch_for_shader( shader, 'TRIS', {"src_uv": src_uv, "dest_uv": dest_uv}, indices=vertices, ) shader.uniform_sampler("image", texture) batch.draw(shader) projected = np.array(fb.read_color(0, 0, width, height, 4, 0, 'FLOAT').to_list()) offscreen.free() dest.pixels[:] = projected.ravel() class ProjectDreamTexture(bpy.types.Operator): bl_idname = "shade.dream_texture_project" bl_label = "Project Dream Texture" bl_description = "Automatically texture all selected objects using the depth buffer and Stable Diffusion" bl_options = {'REGISTER'} @classmethod def poll(cls, context): try: _validate_projection(context) prompt = context.scene.dream_textures_project_prompt backend: api.Backend = prompt.get_backend() args = prompt.generate_args(context) args.task = api.task.PromptToImage() if context.scene.dream_textures_project_use_control_net else api.task.DepthToImage(None, None, 0) backend.validate(args) except: return False return Generator.shared().can_use() @classmethod def get_uv_layer(cls, mesh: bmesh.types.BMesh): for i in range(len(mesh.loops.layers.uv)): uv = mesh.loops.layers.uv[i] if uv.name.lower() == "projected uvs": return uv, i return mesh.loops.layers.uv.new("Projected UVs"), len(mesh.loops.layers.uv) - 1 def execute(self, context): # Setup the progress indicator def step_progress_update(self, context): if hasattr(context.area, "regions"): for region in context.area.regions: if region.type == "UI": region.tag_redraw() return None bpy.types.Scene.dream_textures_progress = bpy.props.IntProperty(name="", default=0, min=0, max=context.scene.dream_textures_project_prompt.steps, update=step_progress_update) context.scene.dream_textures_info = "Starting..." # Get region size region_width = region_height = None for area in context.screen.areas: if area.type == 'VIEW_3D': for region in area.regions: if region.type == 'WINDOW': region_width, region_height = region.width, region.height if region_width is None or region_height is None: self.report({'ERROR'}, "Could not determine region size.") # Render the viewport if context.scene.dream_textures_project_framebuffer_arguments == 'color': context.scene.dream_textures_info = "Rendering viewport color..." res_x, res_y = context.scene.render.resolution_x, context.scene.render.resolution_y view3d_spaces = [] for area in context.screen.areas: if area.type == 'VIEW_3D': for region in area.regions: if region.type == 'WINDOW': context.scene.render.resolution_x, context.scene.render.resolution_y = region.width, region.height for space in area.spaces: if space.type == 'VIEW_3D': if space.overlay.show_overlays: view3d_spaces.append(space) space.overlay.show_overlays = False init_img_path = tempfile.NamedTemporaryFile(suffix='.png').name render_filepath, file_format = context.scene.render.filepath, context.scene.render.image_settings.file_format context.scene.render.image_settings.file_format = 'PNG' context.scene.render.filepath = init_img_path bpy.ops.render.opengl(write_still=True, view_context=True) for space in view3d_spaces: space.overlay.show_overlays = True context.scene.render.resolution_x, context.scene.render.resolution_y = res_x, res_y context.scene.render.filepath, context.scene.render.image_settings.file_format = render_filepath, file_format else: init_img_path = None context.scene.dream_textures_info = "Generating UVs and materials..." material = bpy.data.materials.new(name="diffused-material") material.use_nodes = True image_texture_node = material.node_tree.nodes.new("ShaderNodeTexImage") principled_node = next((n for n in material.node_tree.nodes if n.type == 'BSDF_PRINCIPLED')) material.node_tree.links.new(image_texture_node.outputs[0], principled_node.inputs[0]) uv_map_node = material.node_tree.nodes.new("ShaderNodeUVMap") uv_map_node.uv_map = bpy.context.selected_objects[0].data.uv_layers.active.name if context.scene.dream_textures_project_bake else "Projected UVs" material.node_tree.links.new(uv_map_node.outputs[0], image_texture_node.inputs[0]) target_objects = [] for obj in bpy.context.selected_objects: if not hasattr(obj, "data") or not hasattr(obj.data, "materials"): continue material_index = len(obj.material_slots) obj.data.materials.append(material) mesh = bmesh.from_edit_mesh(obj.data) # Project from UVs view and update material index mesh.verts.ensure_lookup_table() mesh.verts.index_update() def vert_to_uv(v): screen_space = view3d_utils.location_3d_to_region_2d(context.region, context.space_data.region_3d, obj.matrix_world @ v.co) if screen_space is None: return None return (screen_space[0] / context.region.width, screen_space[1] / context.region.height) uv_layer, uv_layer_index = ProjectDreamTexture.get_uv_layer(mesh) bm = mesh.copy() bm.select_mode = {'FACE'} bmesh.ops.split_edges(bm, edges=bm.edges) bmesh.ops.delete(bm, geom=[f for f in bm.faces if not f.select], context='FACES') target_objects.append((bm, bm.loops.layers.uv[uv_layer_index])) mesh.faces.ensure_lookup_table() for face in mesh.faces: if face.select: for loop in face.loops: uv = vert_to_uv(mesh.verts[loop.vert.index]) if uv is None: continue loop[uv_layer].uv = uv face.material_index = material_index bmesh.update_edit_mesh(obj.data) context.scene.dream_textures_info = "Rendering viewport depth..." depth = np.flipud(render_depth_map( context.evaluated_depsgraph_get(), collection=None, width=region_width, height=region_height, matrix=context.space_data.region_3d.view_matrix, projection_matrix=context.space_data.region_3d.window_matrix, main_thread=True )) texture = None def step_callback(progress: List[api.GenerationResult]) -> bool: nonlocal texture context.scene.dream_textures_progress = progress[-1].progress image = api.GenerationResult.tile_images(progress) texture = image_utils.np_to_bpy(image, f"Step {progress[-1].progress}/{progress[-1].total}", texture) image_texture_node.image = texture return CancelGenerator.should_continue def callback(results: List[api.GenerationResult] | Exception): CancelGenerator.should_continue = None if isinstance(results, Exception): context.scene.dream_textures_info = "" context.scene.dream_textures_progress = 0 if not isinstance(results, InterruptedError): # this is a user-initiated cancellation eval('bpy.ops.' + NotifyResult.bl_idname)('INVOKE_DEFAULT', exception=repr(results)) raise results else: nonlocal texture context.scene.dream_textures_info = "" context.scene.dream_textures_progress = 0 result = results[-1] prompt_subject = context.scene.dream_textures_project_prompt.prompt_structure_token_subject seed_str_length = len(str(result.seed)) trim_aware_name = (prompt_subject[:54 - seed_str_length] + '..') if len(prompt_subject) > 54 else prompt_subject name_with_trimmed_prompt = f"{trim_aware_name} ({result.seed})" texture = image_utils.np_to_bpy(result.image, name_with_trimmed_prompt, texture) image_texture_node.image = texture if context.scene.dream_textures_project_bake: for bm, src_uv_layer in target_objects: dest = bpy.data.images.new(name=f"{texture.name} (Baked)", width=texture.size[0], height=texture.size[1]) dest_uv_layer = bm.loops.layers.uv.active src_uvs = np.empty((len(bm.verts), 2), dtype=np.float32) dest_uvs = np.empty((len(bm.verts), 2), dtype=np.float32) for face in bm.faces: for loop in face.loops: src_uvs[loop.vert.index] = loop[src_uv_layer].uv dest_uvs[loop.vert.index] = loop[dest_uv_layer].uv bake(context, bm, result.image.ravel(), dest, src_uvs, dest_uvs) dest.update() dest.pack() image_texture_node.image = dest backend: api.Backend = context.scene.dream_textures_project_prompt.get_backend() context.scene.dream_textures_info = "Starting..." CancelGenerator.should_continue = True # reset global cancellation state image_data = bpy.data.images.load(init_img_path) if init_img_path is not None else None image = np.asarray(image_data.pixels).reshape((*depth.shape, image_data.channels)) if image_data is not None else None if context.scene.dream_textures_project_use_control_net: generated_args: api.GenerationArguments = context.scene.dream_textures_project_prompt.generate_args(context, init_image=image, control_images=[image_utils.rgba(depth)]) backend.generate(generated_args, step_callback=step_callback, callback=callback) else: generated_args: api.GenerationArguments = context.scene.dream_textures_project_prompt.generate_args(context) generated_args.task = api.DepthToImage(depth, image, context.scene.dream_textures_project_prompt.strength) backend.generate(generated_args, step_callback=step_callback, callback=callback) for area in context.screen.areas: if area.type == 'VIEW_3D': area.tag_redraw() return {'FINISHED'} return {'FINISHED'} ================================================ FILE: operators/upscale.py ================================================ import bpy import numpy as np from typing import List, Literal from .. import api from ..prompt_engineering import custom_structure from ..generator_process import Generator from .dream_texture import CancelGenerator from .. import image_utils upscale_options = [ ("2", "2x", "", 2), ("4", "4x", "", 4), ("8", "8x", "", 8), ] def get_source_image(context): node_tree = context.material.node_tree if hasattr(context, 'material') else None active_node = next((node for node in node_tree.nodes if node.select and node.bl_idname == 'ShaderNodeTexImage'), None) if node_tree is not None else None if active_node is not None and active_node.image is not None: return active_node.image elif context.area.type == 'IMAGE_EDITOR': return context.area.spaces.active.image else: input_image = None for area in context.screen.areas: if area.type == 'IMAGE_EDITOR': if area.spaces.active.image is not None: input_image = area.spaces.active.image return input_image class Upscale(bpy.types.Operator): bl_idname = "shade.dream_textures_upscale" bl_label = "Upscale" bl_description = ("Upscale with Stable Diffusion x4 Upscaler") bl_options = {"REGISTER"} @classmethod def poll(cls, context): return Generator.shared().can_use() def execute(self, context): screen = context.screen scene = context.scene node_tree = context.material.node_tree if hasattr(context, 'material') else None active_node = next((node for node in node_tree.nodes if node.select and node.bl_idname == 'ShaderNodeTexImage'), None) if node_tree is not None else None def step_progress_update(self, context): if hasattr(context.area, "regions"): for region in context.area.regions: if region.type == "UI": region.tag_redraw() return None bpy.types.Scene.dream_textures_info = bpy.props.StringProperty(name="Info", update=step_progress_update) input_image = get_source_image(context) if input_image is None: self.report({"ERROR"}, "No open image in the Image Editor space, or selected Image Texture node.") return {"FINISHED"} image_pixels = image_utils.bpy_to_np(input_image) generated_args = context.scene.dream_textures_upscale_prompt.generate_args(context) context.scene.dream_textures_upscale_seamless_result.update_args(generated_args) # Setup the progress indicator def step_progress_update(self, context): if hasattr(context.area, "regions"): for region in context.area.regions: if region.type == "UI": region.tag_redraw() return None bpy.types.Scene.dream_textures_progress = bpy.props.IntProperty(name="", default=0, min=0, max=generated_args.steps, update=step_progress_update) scene.dream_textures_info = "Starting..." last_data_block = None def step_callback(progress: List[api.GenerationResult]) -> bool: nonlocal last_data_block if last_data_block is None: bpy.types.Scene.dream_textures_progress = bpy.props.IntProperty(name="", default=progress[-1].progress, min=0, max=progress[-1].total, update=step_progress_update) scene.dream_textures_progress = progress[-1].progress if progress[-1].image is not None: last_data_block = image_utils.np_to_bpy(progress[-1].image, f"Tile {progress[-1].progress}/{progress[-1].total}", last_data_block) for area in screen.areas: if area.type == 'IMAGE_EDITOR' and not area.spaces.active.use_image_pin: area.spaces.active.image = last_data_block return CancelGenerator.should_continue def callback(results: List[api.GenerationResult] | Exception): if isinstance(results, Exception): scene.dream_textures_info = "" scene.dream_textures_progress = 0 CancelGenerator.should_continue = None else: nonlocal last_data_block if last_data_block is not None: bpy.data.images.remove(last_data_block) last_data_block = None if results[-1].image is None: return image = image_utils.np_to_bpy(results[-1].image, f"{input_image.name} (Upscaled)", last_data_block) for area in screen.areas: if area.type == 'IMAGE_EDITOR' and not area.spaces.active.use_image_pin: area.spaces.active.image = image if active_node is not None: active_node.image = image scene.dream_textures_info = "" scene.dream_textures_progress = 0 CancelGenerator.should_continue = None prompt = context.scene.dream_textures_upscale_prompt prompt.prompt_structure = custom_structure.id backend: api.Backend = prompt.get_backend() generated_args.task = api.models.task.Upscale(image=image_pixels, tile_size=context.scene.dream_textures_upscale_tile_size, blend=context.scene.dream_textures_upscale_blend) CancelGenerator.should_continue = True backend.generate( generated_args, step_callback=step_callback, callback=callback ) return {"FINISHED"} ================================================ FILE: operators/view_history.py ================================================ import bpy from bpy_extras.io_utils import ImportHelper, ExportHelper import json import os from ..property_groups.dream_prompt import DreamPrompt, scheduler_options from ..preferences import StableDiffusionPreferences class SCENE_UL_HistoryList(bpy.types.UIList): def draw_item(self, context, layout, data, item, icon, active_data, active_propname): if self.layout_type in {'DEFAULT', 'COMPACT'}: layout.label(text=item.get_prompt_subject(), translate=False, icon_value=icon) layout.label(text=f"{item.seed}", translate=False) layout.label(text=f"{item.width}x{item.height}", translate=False) layout.label(text=f"{item.steps} steps", translate=False) layout.label(text=item.scheduler, translate=False) elif self.layout_type == 'GRID': layout.alignment = 'CENTER' layout.label(text="", icon_value=icon) class RecallHistoryEntry(bpy.types.Operator): bl_idname = "shade.dream_textures_history_recall" bl_label = "Recall Prompt" bl_description = "Open the Dream Textures dialog with the historical properties filled in" bl_options = {'REGISTER'} @classmethod def poll(self, context): return context.scene.dream_textures_history_selection is not None def execute(self, context): selection = context.scene.dream_textures_history[context.scene.dream_textures_history_selection] for prop in selection.__annotations__.keys(): if hasattr(context.scene.dream_textures_prompt, prop): match prop: case 'control_nets': context.scene.dream_textures_prompt.control_nets.clear() for net in selection.control_nets: n = context.scene.dream_textures_prompt.control_nets.add() for k in n.__annotations__.keys(): setattr(n, k, getattr(net, k)) case _: setattr(context.scene.dream_textures_prompt, prop, getattr(selection, prop)) # when the seed of the promt is found in the available image datablocks, use that one in the open image editor # note: when there is more than one image with the seed in it's name, do nothing. Same when no image with that seed is available. if prop == 'hash': hash_string = str(getattr(selection, prop)) existing_image = None # accessing custom properties for image datablocks in Blender is still a bit cumbersome for i in bpy.data.images: if i.get('dream_textures_hash', None) == hash_string: existing_image = i break if existing_image is not None: for area in context.screen.areas: if area.type != 'IMAGE_EDITOR': continue area.spaces.active.image = existing_image return {"FINISHED"} class ClearHistory(bpy.types.Operator): bl_idname = "shade.dream_textures_history_clear" bl_label = "Clear History" bl_description = "Removes all history entries" bl_options = {'REGISTER'} def execute(self, context): context.scene.dream_textures_history.clear() return {"FINISHED"} class RemoveHistorySelection(bpy.types.Operator): bl_idname = "shade.dream_textures_history_remove_selection" bl_label = "Remove History Selection" bl_description = "Removes the selected history entry" bl_options = {'REGISTER'} @classmethod def poll(self, context): return context.scene.dream_textures_history_selection is not None def execute(self, context): context.scene.dream_textures_history.remove(context.scene.dream_textures_history_selection) return {"FINISHED"} class ExportHistorySelection(bpy.types.Operator, ExportHelper): bl_idname = "shade.dream_textures_history_export" bl_label = "Export Prompt" bl_description = "Exports the selected history entry to a JSON file" filename_ext = ".json" filter_glob: bpy.props.StringProperty( default="*.json", options={'HIDDEN'}, maxlen=255, ) @classmethod def poll(self, context): return context.scene.dream_textures_history_selection is not None def invoke(self, context, event): selection = context.scene.dream_textures_history[context.scene.dream_textures_history_selection] self.filepath = "untitled" if selection is None else selection.get_prompt_subject() context.window_manager.fileselect_add(self) return {'RUNNING_MODAL'} def execute(self, context): selection = context.scene.dream_textures_history[context.scene.dream_textures_history_selection] if selection is None: self.report({"ERROR"}, "No valid selection to export.") return {"FINISHED"} with open(self.filepath, 'w', encoding='utf-8') as target: args = {key: getattr(selection, key) for key in DreamPrompt.__annotations__} args["outpaint_origin"] = list(args["outpaint_origin"]) json.dump(args, target, indent=4) return {"FINISHED"} class ImportPromptFile(bpy.types.Operator, ImportHelper): bl_idname = "shade.dream_textures_import_prompt" bl_label = "Import Prompt" bl_description = "Imports a JSON file as a prompt" filename_ext = ".json" filter_glob: bpy.props.StringProperty( default="*.json", options={'HIDDEN'}, maxlen=255, ) def execute(self, context): _, extension = os.path.splitext(self.filepath) if extension != ".json": self.report({"ERROR"}, "Invalid prompt JSON file selected.") return {"FINISHED"} with open(self.filepath, 'r', encoding='utf-8') as target: args = json.load(target) for key, value in args.items(): if hasattr(context.scene.dream_textures_prompt, key) and value is not None: setattr(context.scene.dream_textures_prompt, key, value) return {"FINISHED"} ================================================ FILE: preferences.py ================================================ import bpy from bpy.props import CollectionProperty, StringProperty from bpy_extras.io_utils import ImportHelper import os import webbrowser import importlib.util import site from .absolute_path import absolute_path from .operators.install_dependencies import InstallDependencies, UninstallDependencies from .operators.open_latest_version import OpenLatestVersion from .ui.presets import RestoreDefaultPresets, default_presets_missing from .generator_process import Generator from .generator_process.actions.huggingface_hub import DownloadStatus, Model as HubModel from .generator_process.models import Checkpoint, ModelConfig, ModelType is_downloading = False class OpenURL(bpy.types.Operator): bl_idname = "dream_textures.open_url" bl_label = "Get Access Token" bl_description = ("Opens huggingface.co to the tokens page") bl_options = {"REGISTER", "INTERNAL"} url: bpy.props.StringProperty(name="URL") def execute(self, context): webbrowser.open(self.url) return {"FINISHED"} _model_config_options = [(m.name, m.value, '') for m in ModelConfig] import_extensions = ['.ckpt', '.safetensors', '.pth'] import_extensions_glob = ";".join(import_extensions).replace(".", "*.") class ImportWeights(bpy.types.Operator, ImportHelper): bl_idname = "dream_textures.import_weights" bl_label = "Import Checkpoint File" filename_ext = ".ckpt" filter_glob: bpy.props.StringProperty( default=import_extensions_glob, options={'HIDDEN'}, maxlen=255, ) model_config: bpy.props.EnumProperty( name="Model Config", items=_model_config_options ) prefer_fp16_variant: bpy.props.BoolProperty( name="Save Half Precision Weights", default=True ) def execute(self, context): global is_downloading is_downloading = True f = Generator.shared().convert_original_stable_diffusion_to_diffusers(self.filepath, ModelConfig[self.model_config], self.prefer_fp16_variant) def on_progress(_, response: DownloadStatus): bpy.context.preferences.addons[__package__].preferences.download_file = response.file bpy.context.preferences.addons[__package__].preferences.download_progress = int((response.index / response.total) * 100) def on_done(future): global is_downloading is_downloading = False fetch_installed_models() def on_exception(_, exception): self.report({"ERROR"}, str(exception)) raise exception f.add_response_callback(on_progress) f.add_done_callback(on_done) f.add_exception_callback(on_exception) return {"FINISHED"} class Model(bpy.types.PropertyGroup): bl_label = "Model" bl_idname = "dream_textures.Model" model: bpy.props.StringProperty(name="Model") model_base: bpy.props.StringProperty() downloads: bpy.props.IntProperty(name="Downloads") likes: bpy.props.IntProperty(name="Likes") model_type: bpy.props.EnumProperty(name="Model Type", items=[(t.name, t.name, '') for t in ModelType]) class PREFERENCES_UL_ModelList(bpy.types.UIList): def draw_item(self, context, layout, data, item, icon, active_data, active_propname): model_name = item.model is_installed = False if os.path.exists(item.model): model_name = os.path.basename(item.model).replace('models--', '').replace('--', '/') is_installed = True split = layout.split(factor=0.75) split.label(text=model_name) if item.downloads != -1: split.label(text=str(item.downloads), icon="IMPORT") if item.downloads != -1: split.label(text=str(item.likes), icon="HEART") if ModelType[item.model_type] != ModelType.UNKNOWN: split.label(text=item.model_type.replace('_', ' ').title()) install_model = layout.operator(InstallModel.bl_idname, text="", icon="FILE_FOLDER" if is_installed else "IMPORT") install_model.model = item.model install_model.prefer_fp16_variant = data.prefer_fp16_variant install_model.resume_download = data.resume_download def set_model_list(model_list: str, models: list): getattr(bpy.context.preferences.addons[__package__].preferences, model_list).clear() for model in models: m = getattr(bpy.context.preferences.addons[__package__].preferences, model_list).add() m.model = model.id m.model_base = os.path.basename(model.id) m.downloads = model.downloads m.likes = model.likes try: m.model_type = model.model_type.name except: pass class checkpoint_lookup: _checkpoints = {} @classmethod def get(cls, item): return cls._checkpoints.get(item, item) class model_lookup: _models = {} @classmethod def get(cls, item): return cls._models.get(item, None) def fetch_installed_models(blocking=True): def on_done(future): model_list = future.result() model_lookup._models = { os.path.basename(model.id).replace('models--', '').replace('--', '/'): model for model in model_list } pref = bpy.context.preferences.addons[__package__].preferences checkpoint_links = ((link.path, ModelConfig[link.model_config]) for link in pref.linked_checkpoints) checkpoints = {} for path, config in checkpoint_links: if not os.path.exists(path): continue if os.path.isfile(path): checkpoints[os.path.basename(path)] = (path, config) continue for name in os.listdir(path): if os.path.splitext(name)[1] not in import_extensions: continue if name in checkpoints: # file linked config takes precedence over folder linked config continue checkpoints[name] = (os.path.join(path, name), config) checkpoint_lookup._checkpoints.clear() for path, config in checkpoints.values(): model = HubModel(path, "", [], -1, -1, ModelType.from_config(config)) model_list.append(model) checkpoint_lookup._checkpoints[os.path.basename(path)] = Checkpoint(path, config) model_lookup._models[os.path.basename(path)] = model set_model_list('installed_models', model_list) future = Generator.shared().hf_list_installed_models() if blocking: on_done(future) else: future.add_done_callback(on_done) class ModelSearch(bpy.types.Operator): bl_idname = "dream_textures.model_search" bl_label = "Search" bl_description = ("Searches Hugging Face Hub for models") bl_options = {"REGISTER", "INTERNAL"} def execute(self, context): return {"FINISHED"} class InstallModel(bpy.types.Operator): bl_idname = "dream_textures.install_model" bl_label = "Install or Open" bl_description = ("Install or open a model from the cache") bl_options = {"REGISTER", "INTERNAL"} model: StringProperty(name="Model ID") prefer_fp16_variant: bpy.props.BoolProperty(name="", default=True) resume_download: bpy.props.BoolProperty(name="", default=True) def execute(self, context): if os.path.exists(self.model): if os.path.isfile(self.model): webbrowser.open(f"file://{os.path.dirname(self.model)}") else: webbrowser.open(f"file://{self.model}") else: global is_downloading is_downloading = True f = Generator.shared().hf_snapshot_download( self.model, bpy.context.preferences.addons[__package__].preferences.hf_token, "fp16" if self.prefer_fp16_variant else None, self.resume_download ) def on_progress(_, response: DownloadStatus): bpy.context.preferences.addons[__package__].preferences.download_file = response.file bpy.context.preferences.addons[__package__].preferences.download_progress = int((response.index / response.total) * 100) def on_done(future): global is_downloading is_downloading = False fetch_installed_models() def on_exception(_, exception): self.report({"ERROR"}, str(exception)) raise exception f.add_response_callback(on_progress) f.add_done_callback(on_done) f.add_exception_callback(on_exception) return {"FINISHED"} def _model_search(self, context): def on_done(future): set_model_list('model_results', future.result()) Generator.shared().hf_list_models(self.model_query, self.hf_token).add_done_callback(on_done) def _update_ui(self, context): if hasattr(context.area, "regions"): for region in context.area.regions: if region.type == "UI": region.tag_redraw() return None def _template_model_download_progress(context, layout): global is_downloading preferences = context.preferences.addons[StableDiffusionPreferences.bl_idname].preferences if is_downloading: progress_col = layout.column() progress_col.label(text=preferences.download_file) progress_col.prop(preferences, "download_progress", slider=True) progress_col.enabled = False return is_downloading class CheckpointGroup(bpy.types.PropertyGroup): bl_label = "Model" bl_idname = "dream_textures.checkpoint" path: bpy.props.StringProperty(name="Checkpoint") model_config: bpy.props.EnumProperty( name="Model Config", items=_model_config_options ) class LinkCheckpoint(bpy.types.Operator, ImportHelper): bl_idname = "dream_textures.link_checkpoint" bl_label = "Link Checkpoint File or Folder" filename_ext = ".ckpt" files: CollectionProperty( type=bpy.types.OperatorFileListElement, options={'HIDDEN', 'SKIP_SAVE'} ) filter_glob: bpy.props.StringProperty( default=import_extensions_glob, options={'HIDDEN'}, maxlen=255, ) model_config: bpy.props.EnumProperty( name="Model Config", items=_model_config_options ) def invoke(self, context, _event): if os.path.isfile(self.filepath): # Reset to a directory, otherwise the filename remains populated and can cause issues to select a directory if gone unnoticed. self.filepath = os.path.dirname(self.filepath) + os.path.sep return super().invoke(context, _event) def execute(self, context): pref = context.preferences.addons[__package__].preferences for file in self.files: path = self.filepath if file.name != "": path = os.path.join(os.path.dirname(path), file.name) if not os.path.exists(path): self.report({"ERROR"}, f"{path} does not exist") continue if os.path.isfile(path) and os.path.splitext(path)[1] not in import_extensions: self.report({"ERROR"}, f"{os.path.basename(path)} is not a checkpoint") continue link = next((link for link in pref.linked_checkpoints if link.path == path), None) if link is None: link = pref.linked_checkpoints.add() link.path = path link.model_config = self.model_config fetch_installed_models() return {"FINISHED"} class UnlinkCheckpoint(bpy.types.Operator): bl_idname = "dream_textures.unlink_checkpoint" bl_label = "Unlink Checkpoint File" path: bpy.props.StringProperty() def execute(self, context): pref = context.preferences.addons[__package__].preferences index = next((i for i, link in enumerate(pref.linked_checkpoints) if link.path == self.path), -1) if index != -1: pref.linked_checkpoints.remove(index) fetch_installed_models() return {"FINISHED"} class PREFERENCES_UL_CheckpointList(bpy.types.UIList): def draw_item(self, context, layout, data, item, icon, active_data, active_propname): split = layout.split(factor=0.75) split.label(text=item.path) split.label(text=ModelConfig[item.model_config].value) install_model = layout.operator(InstallModel.bl_idname, text="", icon="FILE_FOLDER") install_model.model = item.path unlink = layout.operator(UnlinkCheckpoint.bl_idname, text="", icon="TRASH") unlink.path = item.path class StableDiffusionPreferences(bpy.types.AddonPreferences): bl_idname = __package__ dream_studio_key: StringProperty(name="DreamStudio Key") model_query: StringProperty(name="Search", update=_model_search) model_results: CollectionProperty(type=Model) active_model_result: bpy.props.IntProperty(name="Active Model", default=0) hf_token: StringProperty(name="HuggingFace Token") prefer_fp16_variant: bpy.props.BoolProperty(name="Prefer Half Precision Weights", description="Download fp16 weights if available for smaller file size. If you run with 'Half Precision' disabled, you should not use this setting", default=True) resume_download: bpy.props.BoolProperty(name="Resume Incomplete Download", description="Continue an in-progress download in case if Blender was closed or connection was interrupted, otherwise incomplete files will be entirely redownloaded", default=True) installed_models: CollectionProperty(type=Model) active_installed_model: bpy.props.IntProperty(name="Active Model", default=0) linked_checkpoints: CollectionProperty(type=CheckpointGroup) active_linked_checkpoint: bpy.props.IntProperty(name="Active Checkpoint", default=0) download_file: bpy.props.StringProperty(name="") download_progress: bpy.props.IntProperty(name="", min=0, max=100, subtype="PERCENTAGE", update=_update_ui) model_cache = [] @staticmethod def register(): fetch_installed_models(False) def draw(self, context): layout = self.layout weights_installed = len(self.installed_models) > 0 if not weights_installed: layout.label(text="Complete the following steps to finish setting up the addon:") has_dependencies = len(os.listdir(absolute_path(".python_dependencies"))) > 2 if has_dependencies: if not _template_model_download_progress(context, layout): conflicting_packages = ["wandb", "k_diffusion"] conflicting_package_specs = {} for package in conflicting_packages: spec = importlib.util.find_spec(package) if spec is not None: conflicting_package_specs[package] = spec if len(conflicting_package_specs) > 0: conflicts_box = layout.box() conflicts_box.label(text="WARNING", icon="ERROR") conflicts_box.label(text=f"The following packages conflict with Dream Textures: {', '.join(conflicting_packages)}") conflicts_box.label(text=f"You may need to run Blender as an administrator to remove these packages") conflicts_box.operator(UninstallDependencies.bl_idname, text="Uninstall Conflicting Packages", icon="CANCEL").conflicts = ' '.join(conflicting_packages) conflicts_box.label(text=f"If the button above fails, you can remove the following folders manually:") for package in conflicting_packages: if package not in conflicting_package_specs: continue location = conflicting_package_specs[package].submodule_search_locations[0] conflicts_box.operator(OpenURL.bl_idname, text=f"Open '{location}'").url = f"file://{location}" if not weights_installed: default_weights_box = layout.box() default_weights_box.label(text="You need to download at least one model.") install_model = default_weights_box.operator(InstallModel.bl_idname, text="Download Stable Diffusion v2.1 (Recommended)", icon="IMPORT") install_model.model = "stabilityai/stable-diffusion-2-1" install_model.prefer_fp16_variant = self.prefer_fp16_variant install_model.resume_download = self.resume_download search_box = layout.box() search_box.label(text="Find Models", icon="SETTINGS") search_box.label(text="Search Hugging Face Hub for more compatible models.") search_box.prop(self, "model_query", text="", icon="VIEWZOOM") if len(self.model_results) > 0: search_box.template_list(PREFERENCES_UL_ModelList.__name__, "dream_textures_model_results", self, "model_results", self, "active_model_result") search_box.label(text="Some models require authentication. Provide a token to download gated models.") auth_row = search_box.row() auth_row.prop(self, "hf_token", text="Token") auth_row.operator(OpenURL.bl_idname, text="Get Your Token", icon="KEYINGSET").url = "https://huggingface.co/settings/tokens" search_box.prop(self, "prefer_fp16_variant") search_box.prop(self, "resume_download") layout.template_list(PREFERENCES_UL_ModelList.__name__, "dream_textures_installed_models", self, "installed_models", self, "active_installed_model") import_weights = layout.operator(ImportWeights.bl_idname, icon='IMPORT') import_weights.prefer_fp16_variant = self.prefer_fp16_variant layout.template_list(PREFERENCES_UL_CheckpointList.__name__, "dream_textures_linked_checkpoints", self, "linked_checkpoints", self, "active_linked_checkpoint") layout.operator(LinkCheckpoint.bl_idname, icon='FOLDER_REDIRECT') if weights_installed or len(self.dream_studio_key) > 0: complete_box = layout.box() complete_box.label(text="Addon Setup Complete", icon="CHECKMARK") complete_box.label(text="To locate the interface:") complete_box.label(text="1. Open an Image Editor or Shader Editor space") complete_box.label(text="2. Enable 'View' > 'Sidebar'") complete_box.label(text="3. Select the 'Dream' tab") if default_presets_missing(): presets_box = layout.box() presets_box.label(text="Default Presets", icon="PRESET") presets_box.label(text="It looks like you removed some of the default presets.") presets_box.label(text="You can restore them here.") presets_box.operator(RestoreDefaultPresets.bl_idname, icon="RECOVER_LAST") else: missing_dependencies_box = layout.box() missing_dependencies_box.label(text="Dependencies Missing", icon="ERROR") missing_dependencies_box.label(text="You've likely downloaded source instead of release by accident.") missing_dependencies_box.label(text="Follow the instructions to install for your platform.") missing_dependencies_box.operator(OpenLatestVersion.bl_idname, text="Download Latest Release") contributors_box = layout.box() contributors_box.label(text="Contributors", icon="COMMUNITY") contributors_box.label(text="Dream Textures is made possible by the contributors on GitHub.") contributors_box.operator(OpenURL.bl_idname, text="See All Contributors", icon="URL").url = "https://github.com/carson-katri/dream-textures/graphs/contributors" if context.preferences.view.show_developer_ui: # If 'Developer Extras' is enabled, show addon development tools developer_box = layout.box() developer_box.label(text="Development Tools", icon="CONSOLE") warn_box = developer_box.box() warn_box.label(text="WARNING", icon="ERROR") warn_box.label(text="This section is for addon development only.") warn_box.label(text="Do not use any operators in this section unless you are setting up a development environment.") if has_dependencies: warn_box = developer_box.box() warn_box.label(text="Dependencies already installed. Only install below if you developing the addon", icon="CHECKMARK") developer_box.prop(context.scene, 'dream_textures_requirements_path') developer_box.operator_context = 'INVOKE_DEFAULT' developer_box.operator(InstallDependencies.bl_idname, icon="CONSOLE") ================================================ FILE: prompt_engineering.py ================================================ from collections import namedtuple PromptToken = namedtuple('PromptToken', ['id', 'label', 'values']) PromptStructure = namedtuple('PromptStructure', ['id', 'label', 'structure', 'generate']) framing_token = PromptToken('framing', 'Framing', ( ('ecu', 'Extreme Close-up'), ('cu', 'Close-up'), ('mcu', 'Medium Close Up'), ('ms', 'Medium Shot'), ('ls', 'Long Shot'), ('els', 'Extra Long Shot'), )) position_token = PromptToken('position', 'Position', ( ('overhead', 'Overhead View'), ('aerial', 'Aerial View'), ('low', 'Low Angle'), ('dutch', 'Dutch Angle'), ('ots', 'Over-the-shoulder shot'), )) film_type_token = PromptToken('film_type', 'Film Type', ( ('bw', 'Black & White'), ('fc', 'Full Color'), ('cine', 'Cinematic'), ('polaroid', 'Polaroid'), ('anaglyph', 'Anaglyph'), ('double', 'Double Exposure'), )) camera_settings_token = PromptToken('camera_settings', 'Camera Settings', ( ('high_speed', 'Fast Shutter Speed'), ('long_exposure', 'Long Exposure'), ('bokeh', 'Shallow Depth of Field'), ('deep_dof', 'Deep Depth of Field'), ('tilt_shift', 'Tilt Shift'), ('motion_blur', 'Motion Blur'), ('telephoto', 'Telephoto Lens'), ('macro', 'Macro Lens'), ('wide_angle', 'Wide Angle Lens'), ('fish_eye', 'Fish-Eye Lens'), )) shooting_context_token = PromptToken('shooting_context', 'Shooting Context', ( ('film_still', 'Film Still'), ('photograph', 'Photograph'), ('studio_portrait', 'Studio Portrait Photograph'), ('outdoor', 'Outdoor Photograph'), ('cctv', 'Surveillance Footage'), )) subject_token = PromptToken('subject', 'Subject', ()) lighting_token = PromptToken('lighting', 'Lighting', ( ('golden_hour', 'Golden Hour'), ('blur_hour', 'Blue Hour'), ('midday', 'Midday'), ('overcast', 'Overcast'), ('silhouette', 'Mostly Silhouetted'), ('warm', 'Warm Lighting, 2700K'), ('cold', 'Flourescent Lighting, 4800K'), ('flash', 'Harsh Flash'), ('ambient', 'Ambient Lighting'), ('dramatic', 'Dramatic Lighting'), ('backlit', 'Backlit'), ('studio', 'Studio Lighting'), ('above', 'Lit from Above'), ('below', 'Lit from Below'), ('left', 'Lit from the Left'), ('right', 'Lit from the Right'), )) def texture_prompt(tokens): return f"{tokens.subject} texture" texture_structure = PromptStructure( 'texture', 'Texture', [subject_token], texture_prompt ) def photography_prompt(tokens): return f"A {tokens.framing} {tokens.position} {tokens.film_type} {tokens.camera_settings} {tokens.shooting_context} of {tokens.subject}, {tokens.lighting}" photography_structure = PromptStructure( 'photography', 'Photography', (subject_token, framing_token, position_token, film_type_token, camera_settings_token, shooting_context_token, lighting_token), photography_prompt ) subject_type_token = PromptToken('subject_type', 'Subject Type', ( ('environment', 'Environment'), ('character', 'Character'), ('weapon', 'Weapon'), ('vehicle', 'Vehicle'), )) genre_token = PromptToken('genre', 'Genre', ( ('scifi', 'Sci-Fi'), ('fantasy', 'Fantasy'), ('cyberpunk', 'Cyberpunk'), ('cinematic', 'Cinematic'), )) def concept_art_prompt(tokens): return f"{tokens.subject}, {tokens.subject_type} concept art, {tokens.genre} digital painting, trending on ArtStation" concept_art_structure = PromptStructure( 'concept_art', 'Concept Art', (subject_token, subject_type_token, genre_token), concept_art_prompt ) def custom_prompt(tokens): return f"{tokens.subject}" custom_structure = PromptStructure( 'custom', 'Custom', [subject_token], custom_prompt ) def file_batch_prompt(tokens): return f"" file_batch_structure = PromptStructure( 'file_batch', "File Batch", [], file_batch_prompt ) prompt_structures = [ custom_structure, texture_structure, photography_structure, concept_art_structure, file_batch_structure ] def map_structure(x): return (x.id, x.label, '') prompt_structures_items = list(map(map_structure, prompt_structures)) ================================================ FILE: property_groups/control_net.py ================================================ import bpy from bpy.props import FloatProperty, EnumProperty, PointerProperty, IntProperty, BoolProperty from .. import api, image_utils from ..diffusers_backend import DiffusersBackend from ..generator_process import Generator from ..generator_process.models.optimizations import Optimizations def control_net_options(self, context): return [ None if model is None else (model.id, model.name, model.description) for model in context.scene.dream_textures_prompt.get_backend().list_controlnet_models(context) ] PROCESSOR_IDS = [ ("none", "None", "No pre-processing"), None, ("depth_leres", "Depth (LeRes)", ""), ("depth_leres++", "Depth (LeRes++)", ""), ("depth_midas", "Depth (MiDaS)", ""), ("depth_zoe", "Depth (Zoe)", ""), None, ("canny", "Canny", "Canny edge detection"), ("mlsd", "M-LSD", ""), ("softedge_hed", "Soft Edge (HED)", ""), ("softedge_hedsafe", "Soft Edge (HED-Safe)", ""), ("softedge_pidinet", "Soft Edge (PidiNet)", ""), ("softedge_pidsafe", "Soft Edge (Pidsafe)", ""), None, ("lineart_anime", "Lineart (Anime)", ""), ("lineart_coarse", "Lineart (Coarse)", ""), ("lineart_realistic", "Lineart (Realistic)", ""), None, ("normal_bae", "Normal (BAE)", ""), None, ("openpose", "OpenPose", ""), ("openpose_face", "OpenPose (Face)", ""), ("openpose_faceonly", "OpenPose (Face Only)", ""), ("openpose_full", "OpenPose (Full)", ""), ("openpose_hand", "OpenPose (Hand)", ""), # ("dwpose", "DWPose", ""), # requires additional dependencies # ("mediapipe_face", "MediaPipe Face", ""), # requires additional dependencies None, ("scribble_hed", "Scribble (HED)", ""), ("scribble_pidinet", "Scribble (PidiNet)", ""), None, ("shuffle", "Shuffle", ""), ] class ControlNet(bpy.types.PropertyGroup): control_net: EnumProperty(name="ControlNet", items=control_net_options, description="Specify which ControlNet to use") conditioning_scale: FloatProperty(name="Conditioning Scale", default=1.0, description="Increases the strength of the ControlNet's effect") control_image: PointerProperty(type=bpy.types.Image) processor_id: EnumProperty( name="Processor", items=PROCESSOR_IDS, description="Pre-process the control image" ) enabled: BoolProperty(name="Enabled", default=True) class ControlNetsAddMenu(bpy.types.Menu): bl_idname = "DREAM_MT_control_nets_add" bl_label = "Add ControlNet" def draw(self, context): layout = self.layout for model in control_net_options(self, context): if model is None: layout.separator() else: layout.operator("dream_textures.control_nets_add", text=model[1]).control_net = model[0] class ControlNetsAdd(bpy.types.Operator): bl_idname = "dream_textures.control_nets_add" bl_label = "Add ControlNet" control_net: EnumProperty(name="ControlNet", items=control_net_options) def execute(self, context): net = context.scene.dream_textures_prompt.control_nets.add() net.control_net = self.control_net return {'FINISHED'} class ControlNetsRemove(bpy.types.Operator): bl_idname = "dream_textures.control_nets_remove" bl_label = "Remove ControlNet" index: IntProperty(name="Index") def execute(self, context): context.scene.dream_textures_prompt.control_nets.remove(self.index) return {'FINISHED'} class BakeControlNetImage(bpy.types.Operator): bl_idname = "dream_textures.control_net_bake" bl_label = "Bake Control Image" bl_description = "Runs the selected processor, and bakes the result to an image datablock" index: IntProperty(name="Index") def execute(self, context): prompt = context.scene.dream_textures_prompt net = prompt.control_nets[self.index] gen = Generator.shared() backend: api.Backend = prompt.get_backend() optimizations = backend.optimizations() if isinstance(backend, DiffusersBackend) else Optimizations() future = gen.controlnet_aux( processor_id=net.processor_id, image=image_utils.bpy_to_np(net.control_image, color_space=None), optimizations=optimizations ) control_image = image_utils.np_to_bpy( future.result(last_only=True), f"{net.control_image.name} ({next(processor[1] for processor in PROCESSOR_IDS if processor != None and processor[0] == net.processor_id)})", ) net.control_image = control_image net.processor_id = "none" for area in context.screen.areas: if area.type == 'IMAGE_EDITOR' and not area.spaces.active.use_image_pin: area.spaces.active.image = control_image return {'FINISHED'} ================================================ FILE: property_groups/dream_prompt.py ================================================ import bpy from bpy.props import FloatProperty, IntProperty, EnumProperty, BoolProperty, StringProperty, IntVectorProperty, CollectionProperty import os import sys from typing import _AnnotatedAlias from ..generator_process.actions.detect_seamless import SeamlessAxes from ..generator_process.actions.prompt_to_image import Optimizations, Scheduler, StepPreviewMode from ..prompt_engineering import * from ..preferences import StableDiffusionPreferences from .control_net import ControlNet import numpy as np from functools import reduce from .. import api from ..image_utils import bpy_to_np, grayscale def scheduler_options(self, context): return [ (scheduler, scheduler, '') for scheduler in self.get_backend().list_schedulers(context) ] step_preview_mode_options = [(mode.value, mode.value, '') for mode in StepPreviewMode] precision_options = [ ('auto', 'Automatic', "", 1), ('float32', 'Full Precision (float32)', "", 2), ('autocast', 'Autocast', "", 3), ('float16', 'Half Precision (float16)', "", 4), ] init_image_sources = [ ('file', 'File', '', 'IMAGE_DATA', 1), ('open_editor', 'Open Image', '', 'TPAINT_HLT', 2), ] init_image_actions = [ ('modify', 'Modify', 'Combine the initial image with noise to influence the output', 'IMAGE', 1), ('inpaint', 'Inpaint', 'Fill in any masked areas', 'TPAINT_HLT', 2), ('outpaint', 'Outpaint', 'Extend the image in a specific direction', 'FULLSCREEN_ENTER', 3), ] def init_image_actions_filtered(self, context): available = ['modify', 'inpaint', 'outpaint'] return list(filter(lambda x: x[0] in available, init_image_actions)) inpaint_mask_sources = [ ('alpha', 'Alpha Channel', '', 1), ('prompt', 'Prompt', '', 2), ] def inpaint_mask_sources_filtered(self, context): available = ['alpha', 'prompt'] return list(filter(lambda x: x[0] in available, inpaint_mask_sources)) seamless_axes = [ SeamlessAxes.AUTO.bpy_enum('Detect from source image when modifying or inpainting, off otherwise', -1), SeamlessAxes.OFF.bpy_enum('', 0), SeamlessAxes.HORIZONTAL.bpy_enum('', 1), SeamlessAxes.VERTICAL.bpy_enum('', 2), SeamlessAxes.BOTH.bpy_enum('', 3), ] def modify_action_source_type(self, context): return [ ('color', 'Color', 'Use the color information from the image', 1), None, ('depth_generated', 'Color and Generated Depth', 'Use MiDaS to infer the depth of the initial image and include it in the conditioning. Can give results that more closely match the composition of the source image', 2), ('depth_map', 'Color and Depth Map', 'Specify a secondary image to use as the depth map. Can give results that closely match the composition of the depth map', 3), ('depth', 'Depth', 'Treat the initial image as a depth map, and ignore any color. Matches the composition of the source image without any color influence', 4), ] def model_options(self, context): return [ None if model is None else (model.id, model.name, model.description) for model in self.get_backend().list_models(context) ] def _model_update(self, context): options = [m for m in model_options(self, context) if m is not None] if self.model == '' and len(options) > 0: self.model = options[0] def backend_options(self, context): return [ (backend._id(), backend.name if hasattr(backend, "name") else backend.__name__, backend.description if hasattr(backend, "description") else "") for backend in api.Backend._list_backends() ] def seed_clamp(self, ctx): # clamp seed right after input to make it clear what the limits are try: s = str(max(0,min(int(float(self.seed)),2**32-1))) # float() first to make sure any seed that is a number gets clamped, not just ints if s != self.seed: self.seed = s except (ValueError, OverflowError): pass # will get hashed once generated attributes = { "backend": EnumProperty(name="Backend", items=backend_options, description="Specify which generation backend to use"), "model": EnumProperty(name="Model", items=model_options, description="Specify which model to use for inference", update=_model_update), "control_nets": CollectionProperty(type=ControlNet), "active_control_net": IntProperty(name="Active ControlNet"), # Prompt "prompt_structure": EnumProperty(name="Preset", items=prompt_structures_items, description="Fill in a few simple options to create interesting images quickly"), "use_negative_prompt": BoolProperty(name="Use Negative Prompt", default=False), "negative_prompt": StringProperty(name="Negative Prompt", description="The model will avoid aspects of the negative prompt"), # Size "use_size": BoolProperty(name="Manual Size", default=False), "width": IntProperty(name="Width", default=512, min=64, step=64), "height": IntProperty(name="Height", default=512, min=64, step=64), # Simple Options "seamless_axes": EnumProperty(name="Seamless Axes", items=seamless_axes, default=SeamlessAxes.AUTO.id, description="Specify which axes should be seamless/tilable"), # Advanced "show_advanced": BoolProperty(name="", default=False), "random_seed": BoolProperty(name="Random Seed", default=True, description="Randomly pick a seed"), "seed": StringProperty(name="Seed", default="0", description="Manually pick a seed", update=seed_clamp), "iterations": IntProperty(name="Iterations", default=1, min=1, description="How many images to generate"), "steps": IntProperty(name="Steps", default=25, min=1), "cfg_scale": FloatProperty(name="CFG Scale", default=7.5, min=0, description="How strongly the prompt influences the image"), "scheduler": EnumProperty(name="Scheduler", items=scheduler_options, default=3), # defaults to "DPM Solver Multistep" "step_preview_mode": EnumProperty(name="Step Preview", description="Displays intermediate steps in the Image Viewer. Disabling can speed up generation", items=step_preview_mode_options, default=1), # Init Image "use_init_img": BoolProperty(name="Use Init Image", default=False), "init_img_src": EnumProperty(name=" ", items=init_image_sources, default="file"), "init_img_action": EnumProperty(name="Action", items=init_image_actions_filtered, default=1), "strength": FloatProperty(name="Noise Strength", description="The ratio of noise:image. A higher value gives more 'creative' results", default=0.75, min=0, max=1, soft_min=0.01, soft_max=0.99), "fit": BoolProperty(name="Fit to width/height", description="Resize the source image to match the specified size", default=True), "use_init_img_color": BoolProperty(name="Color Correct", default=True), "modify_action_source_type": EnumProperty(name="Image Type", items=modify_action_source_type, default=1, description="What kind of data is the source image"), # Inpaint "inpaint_mask_src": EnumProperty(name="Mask Source", items=inpaint_mask_sources_filtered, default=1), "inpaint_replace": FloatProperty(name="Replace", description="Replaces the masked area with a specified amount of noise, can create more extreme changes. Values of 0 or 1 will give the best results", min=0, max=1, default=0), "text_mask": StringProperty(name="Mask Prompt"), "text_mask_confidence": FloatProperty(name="Confidence Threshold", description="How confident the segmentation model needs to be", default=0.5, min=0), # Outpaint "outpaint_origin": IntVectorProperty(name="Origin", default=(0, 448), size=2, description="The position of the outpaint area relative to the top left corner of the image. A value of (0, 512) will outpaint from the bottom of a 512x512 image"), # Resulting image "hash": StringProperty(name="Image Hash"), } def map_structure_token_items(value): return (value[0], value[1], '') for structure in prompt_structures: for token in structure.structure: if not isinstance(token, str): attributes['prompt_structure_token_' + token.id] = StringProperty(name=token.label) attributes['prompt_structure_token_' + token.id + '_enum'] = EnumProperty( name=token.label, items=[('custom', 'Custom', '')] + list(map(map_structure_token_items, token.values)), default='custom' if len(token.values) == 0 else token.values[0][0], ) DreamPrompt = type('DreamPrompt', (bpy.types.PropertyGroup,), { "bl_label": "DreamPrompt", "bl_idname": "dream_textures.DreamPrompt", "__annotations__": attributes, }) def generate_prompt(self): structure = next(x for x in prompt_structures if x.id == self.prompt_structure) class dotdict(dict): """dot.notation access to dictionary attributes""" __getattr__ = dict.get __setattr__ = dict.__setitem__ __delattr__ = dict.__delitem__ tokens = {} for segment in structure.structure: enum_value = getattr(self, 'prompt_structure_token_' + segment.id + '_enum') if enum_value == 'custom': tokens[segment.id] = getattr(self, 'prompt_structure_token_' + segment.id) else: tokens[segment.id] = next(x for x in segment.values if x[0] == enum_value)[1] return structure.generate(dotdict(tokens)) def get_prompt_subject(self): structure = next(x for x in prompt_structures if x.id == self.prompt_structure) for segment in structure.structure: if segment.id == 'subject': return getattr(self, 'prompt_structure_token_' + segment.id) return self.generate_prompt() def get_seed(self): import numpy numpy.random.randn() if self.random_seed: return None # let stable diffusion automatically pick one try: return max(0,min(int(float(self.seed)),2**32-1)) # clamp int except (ValueError, OverflowError): h = hash(self.seed) # not an int, let's hash it! if h < 0: h = ~h return (h & 0xFFFFFFFF) ^ (h >> 32) # 64 bit hash down to 32 bits def generate_args(self, context, iteration=0, init_image=None, control_images=None) -> api.GenerationArguments: is_file_batch = self.prompt_structure == file_batch_structure.id file_batch_lines = [] file_batch_lines_negative = [] if is_file_batch: file_batch_lines = [line.body for line in context.scene.dream_textures_prompt_file.lines if len(line.body.strip()) > 0] file_batch_lines_negative = [""] * len(file_batch_lines) backend: api.Backend = self.get_backend() batch_size = backend.get_batch_size(context) iteration_limit = len(file_batch_lines) if is_file_batch else self.iterations batch_size = min(batch_size, iteration_limit-iteration) task: api.Task = api.PromptToImage() if self.use_init_img: match self.init_img_action: case 'modify': match self.modify_action_source_type: case 'color': task = api.ImageToImage( image=init_image, strength=self.strength, fit=self.fit ) case 'depth_generated': task = api.DepthToImage( depth=None, image=init_image, strength=self.strength ) case 'depth_map': task = api.DepthToImage( depth=None if init_image is None else grayscale(bpy_to_np(context.scene.init_depth, color_space=None)), image=init_image, strength=self.strength ) case 'depth': task = api.DepthToImage( image=None, depth=None if init_image is None else grayscale(init_image), strength=self.strength ) case 'inpaint': task = api.Inpaint( image=init_image, strength=self.strength, fit=self.fit, mask_source=api.Inpaint.MaskSource.ALPHA if self.inpaint_mask_src == 'alpha' else api.Inpaint.MaskSource.PROMPT, mask_prompt=self.text_mask, confidence=self.text_mask_confidence ) case 'outpaint': task = api.Outpaint( image=init_image, origin=(self.outpaint_origin[0], self.outpaint_origin[1]) ) return api.GenerationArguments( task=task, model=next((model for model in self.get_backend().list_models(context) if model is not None and model.id == self.model), None), prompt=api.Prompt( file_batch_lines[iteration:iteration+batch_size] if is_file_batch else [self.generate_prompt()] * batch_size, file_batch_lines_negative[iteration:iteration+batch_size] if is_file_batch else ([self.negative_prompt] * batch_size if self.use_negative_prompt else None) ), size=(self.width, self.height) if self.use_size else None, seed=self.get_seed(), steps=self.steps, guidance_scale=self.cfg_scale, scheduler=self.scheduler, seamless_axes=SeamlessAxes(self.seamless_axes), step_preview_mode=StepPreviewMode(self.step_preview_mode), iterations=self.iterations, control_nets=[ api.models.control_net.ControlNet( net.control_net, control_images[i] if control_images is not None else None, net.conditioning_scale ) for i, net in enumerate(self.control_nets) if net.enabled ] ) def get_backend(self) -> api.Backend: return getattr(self, api.Backend._lookup(self.backend)._attribute()) DreamPrompt.generate_prompt = generate_prompt DreamPrompt.get_prompt_subject = get_prompt_subject DreamPrompt.get_seed = get_seed DreamPrompt.generate_args = generate_args DreamPrompt.get_backend = get_backend ================================================ FILE: property_groups/seamless_result.py ================================================ import bpy import numpy as np from ..generator_process.actions.detect_seamless import SeamlessAxes from ..generator_process import Generator from ..preferences import StableDiffusionPreferences from ..api.models import GenerationArguments from .. import image_utils def update(self, context): if hasattr(context.area, "regions"): for region in context.area.regions: if region.type == "UI": region.tag_redraw() class SeamlessResult(bpy.types.PropertyGroup): bl_label = "SeamlessResult" bl_idname = "dream_textures.SeamlessResult" image: bpy.props.PointerProperty(type=bpy.types.Image) result: bpy.props.StringProperty(name="Auto-detected", update=update, default=SeamlessAxes.OFF.text) def check(self, image): if image == self.image or not Generator.shared().can_use(): return if image is not None and (hash_string := image.get('dream_textures_hash', None)) is not None: res = None def hash_init(): self.image = image self.result = res for args in bpy.context.scene.dream_textures_history: if args.get('hash', None) == hash_string and args.seamless_axes != SeamlessAxes.AUTO: res = SeamlessAxes(args.seamless_axes).text bpy.app.timers.register(hash_init) return can_process = image is not None and image.size[0] >= 8 and image.size[1] >= 8 def init(): self.image = image self.result = 'Processing' if can_process else SeamlessAxes.OFF.text bpy.app.timers.register(init) if not can_process: return pixels = image_utils.bpy_to_np(image) def result(future): self.result = future.result().text Generator.shared().detect_seamless(pixels).add_done_callback(result) def update_args(self, args, as_id=False): if isinstance(args, GenerationArguments): if args.seamless_axes == SeamlessAxes.AUTO and self.result != 'Processing': if as_id: args.seamless_axes = SeamlessAxes(self.result).id else: args.seamless_axes = SeamlessAxes(self.result) else: if args['seamless_axes'] == SeamlessAxes.AUTO and self.result != 'Processing': if as_id: args['seamless_axes'] = SeamlessAxes(self.result).id else: args['seamless_axes'] = SeamlessAxes(self.result) ================================================ FILE: realtime_viewport.py ================================================ # Realtime Viewport is still under development, and is not currently used. import bpy import cycles import time import threading import gpu from gpu_extras.batch import batch_for_shader import numpy as np from multiprocessing.shared_memory import SharedMemory from .operators.dream_texture import dream_texture view_update_original = cycles.CyclesRender.view_update view_draw_original = cycles.CyclesRender.view_draw def debounce(wait_time): """ Decorator that will debounce a function so that it is called after wait_time seconds If it is called multiple times, will wait for the last call to be debounced and run only this one. """ def decorator(function): def debounced(*args, **kwargs): def call_function(): debounced._timer = None return function(*args, **kwargs) # if we already have a call to the function currently waiting to be executed, reset the timer if debounced._timer is not None: debounced._timer.cancel() # after wait_time, call the function provided to the decorator with its arguments debounced._timer = threading.Timer(wait_time, call_function) debounced._timer.start() debounced._timer = None return debounced return decorator def DREAMTEXTURES_HT_viewport_enabled(self, context): self.layout.prop(context.scene, "dream_textures_viewport_enabled", text="", icon="OUTLINER_OB_VOLUME" if context.scene.dream_textures_viewport_enabled else "VOLUME_DATA", toggle=True) is_rendering_viewport = False last_viewport_update = time.time() last_viewport_pixel_buffer_update = time.time() dream_viewport = None is_rendering_dream = False render_dream_flag = False viewport_pixel_buffer = None viewport_size = (0, 0) ignore_next = 0 def create_image(): print("Create image") global dream_viewport dream_viewport = bpy.data.images.new('Dream Viewport', width=32, height=32) def register_realtime_viewport(): bpy.app.timers.register(create_image) def view_update_decorator(original): def view_update(self, context, depsgraph): result = original(self, context, depsgraph) global last_viewport_update global ignore_next if ignore_next <= 0: last_viewport_update = time.time() print("View Update") ignore_next -= 1 return result return view_update cycles.CyclesRender.view_update = view_update_decorator(cycles.CyclesRender.view_update) def updates_stopped(): global last_viewport_update global is_rendering_viewport global is_rendering_dream threshold_reached = (time.time() - last_viewport_update) < 0.5 if threshold_reached != is_rendering_viewport: is_rendering_viewport = threshold_reached global viewport_pixel_buffer if not is_rendering_viewport and not is_rendering_dream and viewport_pixel_buffer is not None: print("Stopped rendering viewport") is_rendering_dream = True array = np.flipud((np.array(viewport_pixel_buffer) * 255).astype(np.int8)) pixels_memory = SharedMemory(create=True, size=array.nbytes) pixels_memory_array = np.ndarray(array.shape, dtype=array.dtype, buffer=pixels_memory.buf) pixels_memory_array[:] = array[:] def image_callback(shared_memory_name, seed, width, height, upscaled=False): if not upscaled: shared_memory = SharedMemory(shared_memory_name) pixels = np.frombuffer(shared_memory.buf, dtype=np.float32).copy() global ignore_next ignore_next = 5 global dream_viewport dream_viewport.scale(width, height) dream_viewport.pixels[:] = pixels shared_memory.close() pixels_memory.close() print("Done") global is_rendering_dream is_rendering_dream = False # for area in bpy.context.screen.areas: # if area.type == 'VIEW_3D': # area.tag_redraw() def step_callback(step, width=None, height=None, shared_memory_name=None): pass dream_texture(bpy.context.scene.dream_textures_render_properties_prompt, step_callback, image_callback, init_img_shared_memory=pixels_memory.name, init_img_shared_memory_width=viewport_size[0], init_img_shared_memory_height=viewport_size[1]) return 0.5 bpy.app.timers.register(updates_stopped) def draw(): global last_viewport_pixel_buffer_update if not bpy.context.scene.dream_textures_viewport_enabled: return if (time.time() - last_viewport_pixel_buffer_update) < 0.5: return last_viewport_pixel_buffer_update = time.time() # get currently bound framebuffer framebuffer = gpu.state.active_framebuffer_get() # get information on current viewport viewport_info = gpu.state.viewport_get() width = viewport_info[2] height = viewport_info[3] global viewport_pixel_buffer global viewport_size viewport_pixel_buffer = framebuffer.read_color(0, 0, width, height, 4, 0, 'FLOAT').to_list() viewport_size = (width, height) bpy.types.SpaceView3D.draw_handler_add(draw, (), 'WINDOW', 'PRE_VIEW') def draw_dream(): global is_rendering_dream global is_rendering_viewport global dream_viewport if not bpy.context.scene.dream_textures_viewport_enabled or is_rendering_viewport: return texture = gpu.texture.from_image(dream_viewport) viewport_info = gpu.state.viewport_get() width = viewport_info[2] height = viewport_info[3] shader = gpu.shader.from_builtin("2D_IMAGE") shader.bind() shader.uniform_sampler("image", texture) batch = batch_for_shader(shader, 'TRI_FAN', { 'pos': ((0, 0), (width, 0), (width, height), (0, height)), 'texCoord': ((0, 0), (1, 0), (1, 1), (0, 1)), }) batch.draw(shader) bpy.types.SpaceView3D.draw_handler_add(draw_dream, (), 'WINDOW', 'POST_PIXEL') bpy.types.VIEW3D_HT_header.append(DREAMTEXTURES_HT_viewport_enabled) def unregister_realtime_viewport(): global view_update_original cycles.CyclesRender.view_update = view_update_original global view_draw_original cycles.CyclesRender.view_draw = view_draw_original bpy.types.VIEW3D_HT_header.remove(DREAMTEXTURES_HT_viewport_enabled) ================================================ FILE: render_pass.py ================================================ import bpy import cycles import numpy as np import os from typing import List import threading from .generator_process import Generator from . import api from . import image_utils pass_inputs = [ ('color', 'Color', 'Provide the scene color as input'), ('depth', 'Depth', 'Provide the Z pass as depth input'), ('color_depth', 'Color and Depth', 'Provide the scene color and depth as input'), ] update_render_passes_original = cycles.CyclesRender.update_render_passes render_original = cycles.CyclesRender.render # del_original = cycles.CyclesRender.__del__ def register_render_pass(): def update_render_passes_decorator(original): def update_render_passes(self, scene=None, renderlayer=None): result = original(self, scene, renderlayer) self.register_pass(scene, renderlayer, "Dream Textures", 4, "RGBA", 'COLOR') return result return update_render_passes cycles.CyclesRender.update_render_passes = update_render_passes_decorator(cycles.CyclesRender.update_render_passes) def render_decorator(original): def render(self, depsgraph): scene = depsgraph.scene if hasattr(depsgraph, "scene") else depsgraph if not scene.dream_textures_render_properties_enabled: return original(self, depsgraph) result = original(self, depsgraph) try: original_result = self.get_result() self.add_pass("Dream Textures", 4, "RGBA") scale = scene.render.resolution_percentage / 100.0 size_x = int(scene.render.resolution_x * scale) size_y = int(scene.render.resolution_y * scale) if size_x % 64 != 0 or size_y % 64 != 0: self.report({"ERROR"}, f"Image dimensions must be multiples of 64 (e.x. 512x512, 512x768, ...) closest is {round(size_x/64)*64}x{round(size_y/64)*64}") return result render_result = self.begin_result(0, 0, size_x, size_y) for layer in render_result.layers: for render_pass in layer.passes: if render_pass.name == "Dream Textures": try: self._render_dream_textures_pass(layer, (size_x, size_y), scene, render_pass, render_result) except Exception as e: self.error_set(str(e)) else: source_pass = None for original_layer in original_result.layers: if layer.name == original_layer.name: for original_pass in original_layer.passes: if original_pass.name == render_pass.name: source_pass = original_pass pixels = image_utils.render_pass_to_np(source_pass, size=(size_x, size_y)) image_utils.np_to_render_pass(pixels, render_pass) self.end_result(render_result) except Exception as e: print(e) return result return render cycles.CyclesRender.render = render_decorator(cycles.CyclesRender.render) cycles.CyclesRender._render_dream_textures_pass = _render_dream_textures_pass # def del_decorator(original): # def del_patch(self): # result = original(self) # kill_generator() # return result # return del_patch # cycles.CyclesRender.__del__ = del_decorator(cycles.CyclesRender.__del__) def unregister_render_pass(): global update_render_passes_original cycles.CyclesRender.update_render_passes = update_render_passes_original global render_original cycles.CyclesRender.render = render_original del cycles.CyclesRender._render_dream_textures_pass # global del_original # cycles.CyclesRender.__del__ = del_original def _render_dream_textures_pass(self, layer, size, scene, render_pass, render_result): def combined(): self.update_stats("Dream Textures", "Applying color management transforms") return image_utils.render_pass_to_np(layer.passes["Combined"], size, color_management=True, color_space="sRGB") def depth(): d = image_utils.render_pass_to_np(layer.passes["Depth"], size).squeeze(2) return (1 - np.interp(d, [0, np.ma.masked_equal(d, d.max(), copy=False).max()], [0, 1])) self.update_stats("Dream Textures", "Starting") prompt = scene.dream_textures_render_properties_prompt match scene.dream_textures_render_properties_pass_inputs: case 'color': task = api.ImageToImage( combined(), prompt.strength, True ) case 'depth': task = api.DepthToImage( depth(), None, prompt.strength ) case 'color_depth': task = api.DepthToImage( depth(), combined(), prompt.strength ) event = threading.Event() dream_pixels = None def step_callback(progress: List[api.GenerationResult]) -> bool: self.update_progress(progress[-1].progress / progress[-1].total) image_utils.np_to_render_pass(progress[-1].image, render_pass) self.update_result(render_result) # This does not seem to have an effect. return True def callback(results: List[api.GenerationResult] | Exception): nonlocal dream_pixels dream_pixels = results[-1].image event.set() backend: api.Backend = prompt.get_backend() generated_args: api.GenerationArguments = prompt.generate_args(bpy.context) generated_args.task = task generated_args.size = size self.update_stats("Dream Textures", "Generating...") backend.generate( generated_args, step_callback=step_callback, callback=callback ) event.wait() # Perform an inverse transform so when Blender applies its transform everything looks correct. self.update_stats("Dream Textures", "Applying inverse color management transforms") image_utils.np_to_render_pass(dream_pixels, render_pass, inverse_color_management=True, color_space="sRGB") self.update_stats("Dream Textures", "Finished") ================================================ FILE: requirements/linux-rocm.txt ================================================ diffusers==0.27.2 invisible-watermark transformers accelerate huggingface_hub controlnet-aux==0.0.7 --extra-index-url https://download.pytorch.org/whl/rocm6.1/ torch==2.3.1 # Original SD checkpoint conversion pytorch-lightning tensorboard omegaconf scipy # LMSDiscreteScheduler opencolorio==2.3.2 # color management matplotlib ================================================ FILE: requirements/mac-mps-cpu.txt ================================================ diffusers==0.27.2 invisible-watermark transformers accelerate huggingface_hub>=0.19.3 controlnet-aux==0.0.7 torch==2.3.1 # Original SD checkpoint conversion pytorch-lightning tensorboard omegaconf scipy # LMSDiscreteScheduler opencolorio==2.3.2 # color management matplotlib ================================================ FILE: requirements/win-dml.txt ================================================ diffusers==0.27.2 invisible-watermark transformers accelerate huggingface_hub controlnet-aux==0.0.7 torch-directml torch==2.3.1 # Original SD checkpoint conversion pytorch-lightning tensorboard omegaconf scipy # LMSDiscreteScheduler opencolorio==2.3.2 # color management matplotlib ================================================ FILE: requirements/win-linux-cuda.txt ================================================ diffusers==0.27.2 invisible-watermark transformers accelerate huggingface_hub controlnet-aux==0.0.7 --extra-index-url https://download.pytorch.org/whl/cu118 torch==2.3.1 # Original SD checkpoint conversion pytorch-lightning tensorboard omegaconf scipy # LMSDiscreteScheduler opencolorio==2.3.2 # color management matplotlib # OpenPose ================================================ FILE: scripts/train_detect_seamless.py ================================================ """ It's recommended to copy this script to its own project folder to keep it with your own image samples and trained models. Each dataset should have images of the same square dimensions for batched training and validation. You can train with multiple datasets in the same session. dataset_layout/ imagesNone/ [sample_images] imagesX/ [sample_images] imagesY/ [sample_images] imagesXY/ [sample_images] """ # if torch, numpy, and cv2 are not installed to site-packages # import site # site.addsitedir(r"path/to/dream_textures/.python_dependencies") import itertools import os from datetime import datetime import cv2 import numpy as np import torch from numpy._typing import NDArray from torch import nn from torch.utils.data import Dataset, DataLoader EDGE_SLICE = 8 if torch.cuda.is_available(): DEVICE = 'cuda' elif torch.backends.mps.is_available(): DEVICE = 'mps' else: DEVICE = 'cpu' class SeamlessModel(nn.Module): def __init__(self): super(SeamlessModel, self).__init__() self.conv = nn.Sequential( nn.Conv2d(3, 64, kernel_size=3, stride=1, padding=1), nn.Dropout(.2), nn.PReLU(64), nn.Conv2d(64, 16, kernel_size=3, stride=1, padding=1), nn.Dropout(.2), nn.PReLU(16), nn.Conv2d(16, 64, kernel_size=8, stride=4, padding=0), nn.Dropout(.2), nn.PReLU(64), nn.Conv2d(64, 64, kernel_size=(1, 3), stride=1, padding=0), nn.Dropout(.2), ) self.gru = nn.GRU(64, 32, batch_first=True) self.fc = nn.Linear(32, 1) def forward(self, x: torch.Tensor): if len(x.size()) == 3: x = x.unsqueeze(0) # x[batch, channels, height, EDGE_SLICE*2] x = self.conv(x) # x[batch, features, height/4, 1] h = torch.zeros(self.gru.num_layers, x.size()[0], self.gru.hidden_size, dtype=x.dtype, device=x.device) x = x.squeeze(3).transpose(2, 1) # x[batch, height/4, features] x, h = self.gru(x, h) return torch.tanh(self.fc(x[:, -1])) def image_edges(path): image: NDArray = cv2.imread(path) # Pretty sure loading images is a bottleneck and makes the first epoch incredibly slow until fully in RAM. # Might be worth caching the edges in an easier to deserialize format with np.savez() edge_x = np.zeros((image.shape[0], EDGE_SLICE * 2, 3), dtype=np.float32) edge_x[:, :EDGE_SLICE] = image[:, -EDGE_SLICE:] edge_x[:, EDGE_SLICE:] = image[:, :EDGE_SLICE] edge_y = np.zeros((EDGE_SLICE * 2, image.shape[1], 3), dtype=np.float32) edge_y[:EDGE_SLICE] = image[-EDGE_SLICE:] edge_y[EDGE_SLICE:] = image[:EDGE_SLICE] return edge_x, edge_y def prepare_edge(edge: NDArray, axis: str) -> torch.Tensor: edge = (edge * 2 / 255 - 1) if axis == 'x': edge = edge.transpose(2, 0, 1) elif axis == 'y': edge = edge.transpose(2, 1, 0) else: raise ValueError('axis should be "x" or "y"') return torch.as_tensor(edge, dtype=torch.float32) def prepare_edges(edge_x: NDArray, edge_y: NDArray) -> tuple[torch.Tensor, torch.Tensor]: edge_x = edge_x * 2 / 255 - 1 edge_y = edge_y * 2 / 255 - 1 edge_x = edge_x.transpose(2, 0, 1) edge_y = edge_y.transpose(2, 1, 0) return torch.as_tensor(edge_x, dtype=torch.float32), torch.as_tensor(edge_y, dtype=torch.float32) def seamless_tensor(seamless): return torch.tensor([1 if seamless else -1], dtype=torch.float32) class EdgeDataset(Dataset): def __init__(self, path): self.data = [] self._load_dir(os.path.join(path, 'imagesNone'), (False, False)) self._load_dir(os.path.join(path, 'imagesX'), (True, False)) self._load_dir(os.path.join(path, 'imagesY'), (False, True)) self._load_dir(os.path.join(path, 'imagesXY'), (True, True)) print(f'dataset loaded {path} contains {len(self)}') def _load_dir(self, imdir, seamless): if not os.path.exists(imdir): print(f'skipping {imdir}, does not exist') return if not os.path.isdir(imdir): print(f'skipping {imdir}, not a directory') return print(f'loading {imdir}') for image in sorted(os.listdir(imdir)): path = os.path.join(imdir, image) if not os.path.isfile(path): continue self.data.append((seamless_tensor(seamless[0]), None, 'x', path)) self.data.append((seamless_tensor(seamless[1]), None, 'y', path)) def __len__(self): return len(self.data) def __getitem__(self, idx) -> tuple[torch.Tensor, torch.Tensor, str, str]: ret = self.data[idx] if ret[1] is not None: return ret path = ret[3] edge_x, edge_y = prepare_edges(*image_edges(path)) # Edges will be cached in cpu when first requested. Might not be desirable with a large enough dataset. if idx % 2 == 0: ret = (ret[0], edge_x, 'x', path) self.data[idx] = ret self.data[idx + 1] = (self.data[idx + 1][0], edge_y, 'y', path) else: self.data[idx - 1] = (self.data[idx - 1][0], edge_x, 'x', path) ret = (ret[0], edge_y, 'y', path) self.data[idx] = ret return ret CHANNEL_PERMUTATIONS = [*itertools.permutations((0, 1, 2))] class PermutedEdgeDataset(Dataset): """Permutes the channels to better generalize color data.""" def __init__(self, dataset: EdgeDataset | str): if isinstance(dataset, str): dataset = EdgeDataset(dataset) self.base_dataset = dataset def __len__(self): return len(self.base_dataset) * len(CHANNEL_PERMUTATIONS) def __getitem__(self, idx): perm = CHANNEL_PERMUTATIONS[idx % len(CHANNEL_PERMUTATIONS)] result, edge, edge_type, path = self.base_dataset[idx // len(CHANNEL_PERMUTATIONS)] edge_perm = torch.zeros(edge.size(), dtype=edge.dtype) edge_perm[0] = edge[perm[0]] edge_perm[1] = edge[perm[1]] edge_perm[2] = edge[perm[2]] return result, edge_perm, edge_type, path, perm def mix_iter(*iterables): """Iterates through multiple objects while attempting to balance by yielding from which one has the highest of remaining/length""" iterables = [x for x in iterables if len(x) > 0] lengths = [len(x) for x in iterables] counts = lengths.copy() ratios = [1.] * len(iterables) iters = [x.__iter__() for x in iterables] while True: idx = -1 max_ratio = 0 for i, ratio in enumerate(ratios): if ratio > max_ratio: idx = i max_ratio = ratio if idx == -1: return c = counts[idx] - 1 counts[idx] = c ratios[idx] = c / lengths[idx] yield next(iters[idx]) def train(model: nn.Module, train_datasets, valid_datasets, epochs=1000, training_rate=0.0001, batch=50): train_loaders = [DataLoader(PermutedEdgeDataset(ds), batch_size=batch, shuffle=True, num_workers=0, pin_memory=True) for ds in train_datasets] valid_loaders = [DataLoader(ds, batch_size=batch, num_workers=0, pin_memory=True) for ds in valid_datasets] criterion = nn.MSELoss() criterion.to(DEVICE) optimizer = torch.optim.SGD(model.parameters(), training_rate, .9) def train_one_epoch(): running_loss = 0. print_rate = 5000 print_after = print_rate for i, data in enumerate(mix_iter(*train_loaders)): seamless = data[0].to(DEVICE) edge = data[1].to(DEVICE) optimizer.zero_grad() output = model(edge) loss: torch.Tensor = criterion(output, seamless) loss.backward() torch.nn.utils.clip_grad_norm_(model.parameters(), 5) optimizer.step() running_loss += loss.item() if i * batch > print_after: print_after += print_rate print(f"LOSS train {running_loss / (i + 1)}") return running_loss / (i + 1) timestamp = datetime.now().strftime('%Y%m%d_%H%M%S') best_vloss = 1_000_000. for epoch in range(epochs): print(f'EPOCH {epoch}:') model.train(True) avg_loss = train_one_epoch() model.train(False) running_vloss = 0.0 with torch.no_grad(): for i, vdata in enumerate(mix_iter(valid_loaders)): expected_results = vdata[0].to(DEVICE) inputs = vdata[1].to(DEVICE) outputs = model(inputs) vloss = criterion(outputs, expected_results) running_vloss += vloss avg_vloss = running_vloss / (i + 1) print(f'LOSS train {avg_loss} valid {avg_vloss}') # Track best performance, and save the model's state if avg_vloss < best_vloss: best_vloss = avg_vloss model_path = f'model/model_{timestamp}_{epoch}_{int(avg_vloss * 1000)}.pt' torch.save(model.state_dict(), model_path) @torch.no_grad() def validate(model, datasets): # datasets here do not need images of equal sizes or to be square as there is no batching passes = 0 fails = 0 print_limit = 100 print_count = 0 def do_print(result, path, axis): nonlocal print_count if print_count < print_limit: print(f'{path} {axis} {result}') print_count += 1 for valid_dataset in datasets: for data in valid_dataset: expected_result = data[0] tensor = data[1] axis = data[2] path = data[3] result = model(tensor.to(DEVICE))[0].item() if expected_result.item() == 1: if result >= 0: passes += 1 else: fails += 1 do_print(result, path, axis) elif expected_result.item() == -1: if result < 0: passes += 1 else: fails += 1 do_print(result, path, axis) else: raise RuntimeError(f'Unexpected result target {expected_result.item()}') if print_count > print_limit: print(f"{print_count - print_limit} more") total = passes + fails print(f"Passed: {passes} | {passes / total * 100:.2f}%") # edge accuracy print(f"Failed: {fails} | {fails / total * 100:.2f}%") print(f"Passed²: {(passes / total) ** 2 * 100:.2f}%") # image accuracy # I prefer to not perpetuate the public distribution of torch.save() pickled files. def save_npz(path, state_dict): np.savez(path, **state_dict) def load_npz(path): state_dict_np: dict[str, NDArray] = np.load(path, allow_pickle=False) state_dict_torch = dict() for name, arr in state_dict_np.items(): state_dict_torch[name] = torch.from_numpy(arr) return state_dict_torch def main(): model = SeamlessModel() # resume training or validate a saved model # model.load_state_dict(load_npz("model.npz")) # model.load_state_dict(torch.load("model/model_20221203_162623_26_10.pt")) model.to(DEVICE) datasets = [ (EdgeDataset('train/samples'), EdgeDataset('valid/samples')), (EdgeDataset('train/samples2x'), EdgeDataset('valid/samples2x')), (EdgeDataset('train/samples4x'), EdgeDataset('valid/samples4x')) ] # Though it's possible to keep training and validation samples in the same dataset, you really shouldn't. # If you add new samples to a dataset that's being used like this DO NOT resume a previously trained model. # Training and validation samples will get reshuffled and your validation samples will likely be overfit. # gen = torch.Generator().manual_seed(132) # datasets = [ # torch.utils.data.random_split(EdgeDataset('samples'), [.8, .2], gen), # torch.utils.data.random_split(EdgeDataset('samples2x'), [.8, .2], gen), # torch.utils.data.random_split(EdgeDataset('samples4x'), [.8, .2], gen) # ] # If you're generating new samples it can be useful to modify generator_process/actions/prompt_to_image.py # to automatically save images to the dataset. It's best to keep them separate at first and run a previously # trained model on them to help find bad samples. Stable diffusion can at times add solid colored borders to # the edges of images that are not meant to be seamless. I recommend deleting all samples where an edge # appears seamless with scrutiny but was not generated to be, don't move it to another folder in the dataset. # datasets = [ # (None, EdgeDataset('tmp')) # ] # If you only want to validate a saved model. # datasets = [(None, valid) for _, valid in datasets] train_datasets = [] valid_datasets = [] for t, v in datasets: if t is not None: train_datasets.append(t) if v is not None: valid_datasets.append(v) try: if len(train_datasets) > 0: train(model, train_datasets, valid_datasets, epochs=50, training_rate=0.001) # It should easily converge in under 50 epochs. # Training rate is a little high, but I've never managed better # results with a lower rate and several times more epochs. except KeyboardInterrupt: pass # As long as your images have meaningful names you can get feedback on # what kind of images aren't detecting well to add similar samples. model.train(False) validate(model, valid_datasets) if __name__ == '__main__': main() ================================================ FILE: scripts/zip_dependencies.py ================================================ import shutil import zipfile from pathlib import Path def main(): root = Path(__file__).parent.parent deps = root / '.python_dependencies' deps_to_zip = [deps / 'transformers'] for dep in deps_to_zip: if not dep.exists(): raise FileNotFoundError(dep) elif not dep.is_dir(): raise EnvironmentError(f"not a directory {dep}") zip_deps_path = root / '.python_dependencies.zip' zip_deps_path.unlink(True) with zipfile.PyZipFile(str(zip_deps_path), mode='x') as zip_deps: for dep in deps_to_zip: zip_deps.writepy(str(dep)) shutil.rmtree(str(dep)) if __name__ == '__main__': main() ================================================ FILE: sd_configs/cldm_v15.yaml ================================================ model: target: cldm.cldm.ControlLDM params: linear_start: 0.00085 linear_end: 0.0120 num_timesteps_cond: 1 log_every_t: 200 timesteps: 1000 first_stage_key: "jpg" cond_stage_key: "txt" control_key: "hint" image_size: 64 channels: 4 cond_stage_trainable: false conditioning_key: crossattn monitor: val/loss_simple_ema scale_factor: 0.18215 use_ema: False only_mid_control: False control_stage_config: target: cldm.cldm.ControlNet params: image_size: 32 # unused in_channels: 4 hint_channels: 3 model_channels: 320 attention_resolutions: [ 4, 2, 1 ] num_res_blocks: 2 channel_mult: [ 1, 2, 4, 4 ] num_heads: 8 use_spatial_transformer: True transformer_depth: 1 context_dim: 768 use_checkpoint: True legacy: False unet_config: target: cldm.cldm.ControlledUnetModel params: image_size: 32 # unused in_channels: 4 out_channels: 4 model_channels: 320 attention_resolutions: [ 4, 2, 1 ] num_res_blocks: 2 channel_mult: [ 1, 2, 4, 4 ] num_heads: 8 use_spatial_transformer: True transformer_depth: 1 context_dim: 768 use_checkpoint: True legacy: False first_stage_config: target: ldm.models.autoencoder.AutoencoderKL params: embed_dim: 4 monitor: val/rec_loss ddconfig: double_z: true z_channels: 4 resolution: 256 in_channels: 3 out_ch: 3 ch: 128 ch_mult: - 1 - 2 - 4 - 4 num_res_blocks: 2 attn_resolutions: [] dropout: 0.0 lossconfig: target: torch.nn.Identity cond_stage_config: target: ldm.modules.encoders.modules.FrozenCLIPEmbedder ================================================ FILE: sd_configs/cldm_v21.yaml ================================================ model: target: cldm.cldm.ControlLDM params: linear_start: 0.00085 linear_end: 0.0120 num_timesteps_cond: 1 log_every_t: 200 timesteps: 1000 first_stage_key: "jpg" cond_stage_key: "txt" control_key: "hint" image_size: 64 channels: 4 cond_stage_trainable: false conditioning_key: crossattn monitor: val/loss_simple_ema scale_factor: 0.18215 use_ema: False only_mid_control: False control_stage_config: target: cldm.cldm.ControlNet params: use_checkpoint: True image_size: 32 # unused in_channels: 4 hint_channels: 3 model_channels: 320 attention_resolutions: [ 4, 2, 1 ] num_res_blocks: 2 channel_mult: [ 1, 2, 4, 4 ] num_head_channels: 64 # need to fix for flash-attn use_spatial_transformer: True use_linear_in_transformer: True transformer_depth: 1 context_dim: 1024 legacy: False unet_config: target: cldm.cldm.ControlledUnetModel params: use_checkpoint: True image_size: 32 # unused in_channels: 4 out_channels: 4 model_channels: 320 attention_resolutions: [ 4, 2, 1 ] num_res_blocks: 2 channel_mult: [ 1, 2, 4, 4 ] num_head_channels: 64 # need to fix for flash-attn use_spatial_transformer: True use_linear_in_transformer: True transformer_depth: 1 context_dim: 1024 legacy: False first_stage_config: target: ldm.models.autoencoder.AutoencoderKL params: embed_dim: 4 monitor: val/rec_loss ddconfig: #attn_type: "vanilla-xformers" double_z: true z_channels: 4 resolution: 256 in_channels: 3 out_ch: 3 ch: 128 ch_mult: - 1 - 2 - 4 - 4 num_res_blocks: 2 attn_resolutions: [] dropout: 0.0 lossconfig: target: torch.nn.Identity cond_stage_config: target: ldm.modules.encoders.modules.FrozenOpenCLIPEmbedder params: freeze: True layer: "penultimate" ================================================ FILE: sd_configs/sd_xl_base.yaml ================================================ model: target: sgm.models.diffusion.DiffusionEngine params: scale_factor: 0.13025 disable_first_stage_autocast: True denoiser_config: target: sgm.modules.diffusionmodules.denoiser.DiscreteDenoiser params: num_idx: 1000 weighting_config: target: sgm.modules.diffusionmodules.denoiser_weighting.EpsWeighting scaling_config: target: sgm.modules.diffusionmodules.denoiser_scaling.EpsScaling discretization_config: target: sgm.modules.diffusionmodules.discretizer.LegacyDDPMDiscretization network_config: target: sgm.modules.diffusionmodules.openaimodel.UNetModel params: adm_in_channels: 2816 num_classes: sequential use_checkpoint: True in_channels: 4 out_channels: 4 model_channels: 320 attention_resolutions: [4, 2] num_res_blocks: 2 channel_mult: [1, 2, 4] num_head_channels: 64 use_spatial_transformer: True use_linear_in_transformer: True transformer_depth: [1, 2, 10] # note: the first is unused (due to attn_res starting at 2) 32, 16, 8 --> 64, 32, 16 context_dim: 2048 spatial_transformer_attn_type: softmax-xformers legacy: False conditioner_config: target: sgm.modules.GeneralConditioner params: emb_models: # crossattn cond - is_trainable: False input_key: txt target: sgm.modules.encoders.modules.FrozenCLIPEmbedder params: layer: hidden layer_idx: 11 # crossattn and vector cond - is_trainable: False input_key: txt target: sgm.modules.encoders.modules.FrozenOpenCLIPEmbedder2 params: arch: ViT-bigG-14 version: laion2b_s39b_b160k freeze: True layer: penultimate always_return_pooled: True legacy: False # vector cond - is_trainable: False input_key: original_size_as_tuple target: sgm.modules.encoders.modules.ConcatTimestepEmbedderND params: outdim: 256 # multiplied by two # vector cond - is_trainable: False input_key: crop_coords_top_left target: sgm.modules.encoders.modules.ConcatTimestepEmbedderND params: outdim: 256 # multiplied by two # vector cond - is_trainable: False input_key: target_size_as_tuple target: sgm.modules.encoders.modules.ConcatTimestepEmbedderND params: outdim: 256 # multiplied by two first_stage_config: target: sgm.models.autoencoder.AutoencoderKLInferenceWrapper params: embed_dim: 4 monitor: val/rec_loss ddconfig: attn_type: vanilla-xformers double_z: true z_channels: 4 resolution: 256 in_channels: 3 out_ch: 3 ch: 128 ch_mult: [1, 2, 4, 4] num_res_blocks: 2 attn_resolutions: [] dropout: 0.0 lossconfig: target: torch.nn.Identity ================================================ FILE: sd_configs/sd_xl_refiner.yaml ================================================ model: target: sgm.models.diffusion.DiffusionEngine params: scale_factor: 0.13025 disable_first_stage_autocast: True denoiser_config: target: sgm.modules.diffusionmodules.denoiser.DiscreteDenoiser params: num_idx: 1000 weighting_config: target: sgm.modules.diffusionmodules.denoiser_weighting.EpsWeighting scaling_config: target: sgm.modules.diffusionmodules.denoiser_scaling.EpsScaling discretization_config: target: sgm.modules.diffusionmodules.discretizer.LegacyDDPMDiscretization network_config: target: sgm.modules.diffusionmodules.openaimodel.UNetModel params: adm_in_channels: 2560 num_classes: sequential use_checkpoint: True in_channels: 4 out_channels: 4 model_channels: 384 attention_resolutions: [4, 2] num_res_blocks: 2 channel_mult: [1, 2, 4, 4] num_head_channels: 64 use_spatial_transformer: True use_linear_in_transformer: True transformer_depth: 4 context_dim: [1280, 1280, 1280, 1280] # 1280 spatial_transformer_attn_type: softmax-xformers legacy: False conditioner_config: target: sgm.modules.GeneralConditioner params: emb_models: # crossattn and vector cond - is_trainable: False input_key: txt target: sgm.modules.encoders.modules.FrozenOpenCLIPEmbedder2 params: arch: ViT-bigG-14 version: laion2b_s39b_b160k legacy: False freeze: True layer: penultimate always_return_pooled: True # vector cond - is_trainable: False input_key: original_size_as_tuple target: sgm.modules.encoders.modules.ConcatTimestepEmbedderND params: outdim: 256 # multiplied by two # vector cond - is_trainable: False input_key: crop_coords_top_left target: sgm.modules.encoders.modules.ConcatTimestepEmbedderND params: outdim: 256 # multiplied by two # vector cond - is_trainable: False input_key: aesthetic_score target: sgm.modules.encoders.modules.ConcatTimestepEmbedderND params: outdim: 256 # multiplied by one first_stage_config: target: sgm.models.autoencoder.AutoencoderKLInferenceWrapper params: embed_dim: 4 monitor: val/rec_loss ddconfig: attn_type: vanilla-xformers double_z: true z_channels: 4 resolution: 256 in_channels: 3 out_ch: 3 ch: 128 ch_mult: [1, 2, 4, 4] num_res_blocks: 2 attn_resolutions: [] dropout: 0.0 lossconfig: target: torch.nn.Identity ================================================ FILE: sd_configs/v1-inference.yaml ================================================ model: base_learning_rate: 1.0e-04 target: ldm.models.diffusion.ddpm.LatentDiffusion params: linear_start: 0.00085 linear_end: 0.0120 num_timesteps_cond: 1 log_every_t: 200 timesteps: 1000 first_stage_key: "jpg" cond_stage_key: "txt" image_size: 64 channels: 4 cond_stage_trainable: false # Note: different from the one we trained before conditioning_key: crossattn monitor: val/loss_simple_ema scale_factor: 0.18215 use_ema: False scheduler_config: # 10000 warmup steps target: ldm.lr_scheduler.LambdaLinearScheduler params: warm_up_steps: [ 10000 ] cycle_lengths: [ 10000000000000 ] # incredibly large number to prevent corner cases f_start: [ 1.e-6 ] f_max: [ 1. ] f_min: [ 1. ] unet_config: target: ldm.modules.diffusionmodules.openaimodel.UNetModel params: image_size: 32 # unused in_channels: 4 out_channels: 4 model_channels: 320 attention_resolutions: [ 4, 2, 1 ] num_res_blocks: 2 channel_mult: [ 1, 2, 4, 4 ] num_heads: 8 use_spatial_transformer: True transformer_depth: 1 context_dim: 768 use_checkpoint: True legacy: False first_stage_config: target: ldm.models.autoencoder.AutoencoderKL params: embed_dim: 4 monitor: val/rec_loss ddconfig: double_z: true z_channels: 4 resolution: 256 in_channels: 3 out_ch: 3 ch: 128 ch_mult: - 1 - 2 - 4 - 4 num_res_blocks: 2 attn_resolutions: [] dropout: 0.0 lossconfig: target: torch.nn.Identity cond_stage_config: target: ldm.modules.encoders.modules.FrozenCLIPEmbedder ================================================ FILE: sd_configs/v2-inference-v.yaml ================================================ model: base_learning_rate: 1.0e-4 target: ldm.models.diffusion.ddpm.LatentDiffusion params: parameterization: "v" linear_start: 0.00085 linear_end: 0.0120 num_timesteps_cond: 1 log_every_t: 200 timesteps: 1000 first_stage_key: "jpg" cond_stage_key: "txt" image_size: 64 channels: 4 cond_stage_trainable: false conditioning_key: crossattn monitor: val/loss_simple_ema scale_factor: 0.18215 use_ema: False # we set this to false because this is an inference only config unet_config: target: ldm.modules.diffusionmodules.openaimodel.UNetModel params: use_checkpoint: True use_fp16: True image_size: 32 # unused in_channels: 4 out_channels: 4 model_channels: 320 attention_resolutions: [ 4, 2, 1 ] num_res_blocks: 2 channel_mult: [ 1, 2, 4, 4 ] num_head_channels: 64 # need to fix for flash-attn use_spatial_transformer: True use_linear_in_transformer: True transformer_depth: 1 context_dim: 1024 legacy: False first_stage_config: target: ldm.models.autoencoder.AutoencoderKL params: embed_dim: 4 monitor: val/rec_loss ddconfig: #attn_type: "vanilla-xformers" double_z: true z_channels: 4 resolution: 256 in_channels: 3 out_ch: 3 ch: 128 ch_mult: - 1 - 2 - 4 - 4 num_res_blocks: 2 attn_resolutions: [] dropout: 0.0 lossconfig: target: torch.nn.Identity cond_stage_config: target: ldm.modules.encoders.modules.FrozenOpenCLIPEmbedder params: freeze: True layer: "penultimate" ================================================ FILE: sd_configs/v2-inference.yaml ================================================ model: base_learning_rate: 1.0e-4 target: ldm.models.diffusion.ddpm.LatentDiffusion params: linear_start: 0.00085 linear_end: 0.0120 num_timesteps_cond: 1 log_every_t: 200 timesteps: 1000 first_stage_key: "jpg" cond_stage_key: "txt" image_size: 64 channels: 4 cond_stage_trainable: false conditioning_key: crossattn monitor: val/loss_simple_ema scale_factor: 0.18215 use_ema: False # we set this to false because this is an inference only config unet_config: target: ldm.modules.diffusionmodules.openaimodel.UNetModel params: use_checkpoint: True use_fp16: True image_size: 32 # unused in_channels: 4 out_channels: 4 model_channels: 320 attention_resolutions: [ 4, 2, 1 ] num_res_blocks: 2 channel_mult: [ 1, 2, 4, 4 ] num_head_channels: 64 # need to fix for flash-attn use_spatial_transformer: True use_linear_in_transformer: True transformer_depth: 1 context_dim: 1024 legacy: False first_stage_config: target: ldm.models.autoencoder.AutoencoderKL params: embed_dim: 4 monitor: val/rec_loss ddconfig: #attn_type: "vanilla-xformers" double_z: true z_channels: 4 resolution: 256 in_channels: 3 out_ch: 3 ch: 128 ch_mult: - 1 - 2 - 4 - 4 num_res_blocks: 2 attn_resolutions: [] dropout: 0.0 lossconfig: target: torch.nn.Identity cond_stage_config: target: ldm.modules.encoders.modules.FrozenOpenCLIPEmbedder params: freeze: True layer: "penultimate" ================================================ FILE: sd_configs/v2-inpainting-inference.yaml ================================================ model: base_learning_rate: 5.0e-05 target: ldm.models.diffusion.ddpm.LatentInpaintDiffusion params: linear_start: 0.00085 linear_end: 0.0120 num_timesteps_cond: 1 log_every_t: 200 timesteps: 1000 first_stage_key: "jpg" cond_stage_key: "txt" image_size: 64 channels: 4 cond_stage_trainable: false conditioning_key: hybrid scale_factor: 0.18215 monitor: val/loss_simple_ema finetune_keys: null use_ema: False unet_config: target: ldm.modules.diffusionmodules.openaimodel.UNetModel params: use_checkpoint: True image_size: 32 # unused in_channels: 9 out_channels: 4 model_channels: 320 attention_resolutions: [ 4, 2, 1 ] num_res_blocks: 2 channel_mult: [ 1, 2, 4, 4 ] num_head_channels: 64 # need to fix for flash-attn use_spatial_transformer: True use_linear_in_transformer: True transformer_depth: 1 context_dim: 1024 legacy: False first_stage_config: target: ldm.models.autoencoder.AutoencoderKL params: embed_dim: 4 monitor: val/rec_loss ddconfig: #attn_type: "vanilla-xformers" double_z: true z_channels: 4 resolution: 256 in_channels: 3 out_ch: 3 ch: 128 ch_mult: - 1 - 2 - 4 - 4 num_res_blocks: 2 attn_resolutions: [ ] dropout: 0.0 lossconfig: target: torch.nn.Identity cond_stage_config: target: ldm.modules.encoders.modules.FrozenOpenCLIPEmbedder params: freeze: True layer: "penultimate" data: target: ldm.data.laion.WebDataModuleFromConfig params: tar_base: null # for concat as in LAION-A p_unsafe_threshold: 0.1 filter_word_list: "data/filters.yaml" max_pwatermark: 0.45 batch_size: 8 num_workers: 6 multinode: True min_size: 512 train: shards: - "pipe:aws s3 cp s3://stability-aws/laion-a-native/part-0/{00000..18699}.tar -" - "pipe:aws s3 cp s3://stability-aws/laion-a-native/part-1/{00000..18699}.tar -" - "pipe:aws s3 cp s3://stability-aws/laion-a-native/part-2/{00000..18699}.tar -" - "pipe:aws s3 cp s3://stability-aws/laion-a-native/part-3/{00000..18699}.tar -" - "pipe:aws s3 cp s3://stability-aws/laion-a-native/part-4/{00000..18699}.tar -" #{00000-94333}.tar" shuffle: 10000 image_key: jpg image_transforms: - target: torchvision.transforms.Resize params: size: 512 interpolation: 3 - target: torchvision.transforms.RandomCrop params: size: 512 postprocess: target: ldm.data.laion.AddMask params: mode: "512train-large" p_drop: 0.25 # NOTE use enough shards to avoid empty validation loops in workers validation: shards: - "pipe:aws s3 cp s3://deep-floyd-s3/datasets/laion_cleaned-part5/{93001..94333}.tar - " shuffle: 0 image_key: jpg image_transforms: - target: torchvision.transforms.Resize params: size: 512 interpolation: 3 - target: torchvision.transforms.CenterCrop params: size: 512 postprocess: target: ldm.data.laion.AddMask params: mode: "512train-large" p_drop: 0.25 lightning: find_unused_parameters: True modelcheckpoint: params: every_n_train_steps: 5000 callbacks: metrics_over_trainsteps_checkpoint: params: every_n_train_steps: 10000 image_logger: target: main.ImageLogger params: enable_autocast: False disabled: False batch_frequency: 1000 max_images: 4 increase_log_steps: False log_first_step: False log_images_kwargs: use_ema_scope: False inpaint: False plot_progressive_rows: False plot_diffusion_rows: False N: 4 unconditional_guidance_scale: 5.0 unconditional_guidance_label: [""] ddim_steps: 50 # todo check these out for depth2img, ddim_eta: 0.0 # todo check these out for depth2img, trainer: benchmark: True val_check_interval: 5000000 num_sanity_val_steps: 0 accumulate_grad_batches: 1 ================================================ FILE: sd_configs/v2-midas-inference.yaml ================================================ model: base_learning_rate: 5.0e-07 target: ldm.models.diffusion.ddpm.LatentDepth2ImageDiffusion params: linear_start: 0.00085 linear_end: 0.0120 num_timesteps_cond: 1 log_every_t: 200 timesteps: 1000 first_stage_key: "jpg" cond_stage_key: "txt" image_size: 64 channels: 4 cond_stage_trainable: false conditioning_key: hybrid scale_factor: 0.18215 monitor: val/loss_simple_ema finetune_keys: null use_ema: False depth_stage_config: target: ldm.modules.midas.api.MiDaSInference params: model_type: "dpt_hybrid" unet_config: target: ldm.modules.diffusionmodules.openaimodel.UNetModel params: use_checkpoint: True image_size: 32 # unused in_channels: 5 out_channels: 4 model_channels: 320 attention_resolutions: [ 4, 2, 1 ] num_res_blocks: 2 channel_mult: [ 1, 2, 4, 4 ] num_head_channels: 64 # need to fix for flash-attn use_spatial_transformer: True use_linear_in_transformer: True transformer_depth: 1 context_dim: 1024 legacy: False first_stage_config: target: ldm.models.autoencoder.AutoencoderKL params: embed_dim: 4 monitor: val/rec_loss ddconfig: #attn_type: "vanilla-xformers" double_z: true z_channels: 4 resolution: 256 in_channels: 3 out_ch: 3 ch: 128 ch_mult: - 1 - 2 - 4 - 4 num_res_blocks: 2 attn_resolutions: [ ] dropout: 0.0 lossconfig: target: torch.nn.Identity cond_stage_config: target: ldm.modules.encoders.modules.FrozenOpenCLIPEmbedder params: freeze: True layer: "penultimate" ================================================ FILE: tools.py ================================================ from .operators.inpaint_area_brush import InpaintAreaBrush TOOLS = ( InpaintAreaBrush, ) ================================================ FILE: ui/panels/dream_texture.py ================================================ import bpy from bpy.types import Panel from ..presets import DREAM_PT_AdvancedPresets from ...prompt_engineering import * from ...operators.dream_texture import DreamTexture, ReleaseGenerator, CancelGenerator, get_source_image from ...operators.open_latest_version import OpenLatestVersion, is_force_show_download, new_version_available from ...operators.view_history import ImportPromptFile from ..space_types import SPACE_TYPES from ...generator_process.actions.detect_seamless import SeamlessAxes from ...api.models import FixItError from ...property_groups.dream_prompt import DreamPrompt from ...property_groups.control_net import BakeControlNetImage from ... import api def dream_texture_panels(): for space_type in SPACE_TYPES: class DreamTexturePanel(Panel): """Creates a Panel in the scene context of the properties editor""" bl_label = "Dream Texture" bl_idname = f"DREAM_PT_dream_panel_{space_type}" bl_category = "Dream" bl_space_type = space_type bl_region_type = 'UI' @classmethod def poll(cls, context): if cls.bl_space_type == 'NODE_EDITOR': return context.area.ui_type == "ShaderNodeTree" or context.area.ui_type == "CompositorNodeTree" else: return True def draw_header_preset(self, context): layout = self.layout layout.operator(ImportPromptFile.bl_idname, text="", icon="IMPORT") layout.separator() def draw(self, context): layout = self.layout layout.use_property_split = True layout.use_property_decorate = False if is_force_show_download(): layout.operator(OpenLatestVersion.bl_idname, icon="IMPORT", text="Download Latest Release") elif new_version_available(): layout.operator(OpenLatestVersion.bl_idname, icon="IMPORT") layout.prop(context.scene.dream_textures_prompt, "backend") layout.prop(context.scene.dream_textures_prompt, 'model') DreamTexturePanel.__name__ = f"DREAM_PT_dream_panel_{space_type}" yield DreamTexturePanel def get_prompt(context): return context.scene.dream_textures_prompt def get_seamless_result(context, prompt): init_image = None if prompt.use_init_img and prompt.init_img_action in ['modify', 'inpaint']: init_image = get_source_image(context, prompt.init_img_src) context.scene.seamless_result.check(init_image) return context.scene.seamless_result yield from create_panel(space_type, 'UI', DreamTexturePanel.bl_idname, prompt_panel, get_prompt, get_seamless_result=get_seamless_result) yield create_panel(space_type, 'UI', DreamTexturePanel.bl_idname, size_panel, get_prompt) yield from create_panel(space_type, 'UI', DreamTexturePanel.bl_idname, init_image_panels, get_prompt) yield create_panel(space_type, 'UI', DreamTexturePanel.bl_idname, control_net_panel, get_prompt) yield from create_panel(space_type, 'UI', DreamTexturePanel.bl_idname, advanced_panel, get_prompt) yield create_panel(space_type, 'UI', DreamTexturePanel.bl_idname, actions_panel, get_prompt) def create_panel(space_type, region_type, parent_id, ctor, get_prompt, use_property_decorate=False, **kwargs): class BasePanel(Panel): bl_category = "Dream" bl_space_type = space_type bl_region_type = region_type class SubPanel(BasePanel): bl_category = "Dream" bl_space_type = space_type bl_region_type = region_type bl_parent_id = parent_id def draw(self, context): self.layout.use_property_decorate = use_property_decorate return ctor(kwargs.pop('base_panel', SubPanel), space_type, get_prompt, **kwargs) def prompt_panel(sub_panel, space_type, get_prompt, get_seamless_result=None): class PromptPanel(sub_panel): """Create a subpanel for prompt input""" bl_label = "Prompt" bl_idname = f"DREAM_PT_dream_panel_prompt_{space_type}" def draw_header_preset(self, context): layout = self.layout layout.prop(get_prompt(context), "prompt_structure", text="") def draw(self, context): super().draw(context) layout = self.layout layout.use_property_split = True prompt = get_prompt(context) structure = next(x for x in prompt_structures if x.id == prompt.prompt_structure) for segment in structure.structure: segment_row = layout.row() enum_prop = 'prompt_structure_token_' + segment.id + '_enum' is_custom = getattr(prompt, enum_prop) == 'custom' if is_custom: segment_row.prop(prompt, 'prompt_structure_token_' + segment.id) enum_cases = DreamPrompt.__annotations__[enum_prop].keywords['items'] if len(enum_cases) != 1 or enum_cases[0][0] != 'custom': segment_row.prop(prompt, enum_prop, icon_only=is_custom) if prompt.prompt_structure == file_batch_structure.id: layout.template_ID(context.scene, "dream_textures_prompt_file", open="text.open") layout.prop(prompt, "seamless_axes") if prompt.seamless_axes == SeamlessAxes.AUTO and get_seamless_result is not None: auto_row = self.layout.row() auto_row.enabled = False auto_row.prop(get_seamless_result(context, prompt), "result") yield PromptPanel class NegativePromptPanel(sub_panel): """Create a subpanel for negative prompt input""" bl_idname = f"DREAM_PT_dream_panel_negative_prompt_{space_type}" bl_label = "Negative" bl_parent_id = PromptPanel.bl_idname @classmethod def poll(cls, context): return get_prompt(context).prompt_structure != file_batch_structure.id def draw_header(self, context): layout = self.layout layout.prop(get_prompt(context), "use_negative_prompt", text="") def draw(self, context): super().draw(context) layout = self.layout layout.use_property_split = True layout.enabled = layout.enabled and get_prompt(context).use_negative_prompt scene = context.scene layout.prop(get_prompt(context), "negative_prompt") yield NegativePromptPanel def size_panel(sub_panel, space_type, get_prompt): class SizePanel(sub_panel): """Create a subpanel for size options""" bl_idname = f"DREAM_PT_dream_panel_size_{space_type}" bl_label = "Size" bl_options = {'DEFAULT_CLOSED'} def draw_header(self, context): self.layout.prop(get_prompt(context), "use_size", text="") def draw(self, context): super().draw(context) layout = self.layout layout.use_property_split = True layout.enabled = layout.enabled and get_prompt(context).use_size layout.prop(get_prompt(context), "width") layout.prop(get_prompt(context), "height") return SizePanel def init_image_panels(sub_panel, space_type, get_prompt): class InitImagePanel(sub_panel): """Create a subpanel for init image options""" bl_idname = f"DREAM_PT_dream_panel_init_image_{space_type}" bl_label = "Source Image" bl_options = {'DEFAULT_CLOSED'} def draw_header(self, context): self.layout.prop(get_prompt(context), "use_init_img", text="") def draw(self, context): super().draw(context) layout = self.layout prompt = get_prompt(context) layout.enabled = prompt.use_init_img layout.prop(prompt, "init_img_src", expand=True) if prompt.init_img_src == 'file': layout.template_ID(context.scene, "init_img", open="image.open") layout.prop(prompt, "init_img_action", expand=True) layout.use_property_split = True if prompt.init_img_action == 'inpaint': layout.prop(prompt, "inpaint_mask_src") if prompt.inpaint_mask_src == 'prompt': layout.prop(prompt, "text_mask") layout.prop(prompt, "text_mask_confidence") layout.prop(prompt, "inpaint_replace") elif prompt.init_img_action == 'outpaint': layout.prop(prompt, "outpaint_origin") def _outpaint_warning_box(warning): box = layout.box() box.label(text=warning, icon="ERROR") if prompt.outpaint_origin[0] <= -prompt.width or prompt.outpaint_origin[1] <= -prompt.height: _outpaint_warning_box("Outpaint has no overlap, so the result will not blend") init_img = context.scene.init_img if prompt.init_img_src == 'file' else None if init_img is None: for area in context.screen.areas: if area.type == 'IMAGE_EDITOR': if area.spaces.active.image is not None: init_img = area.spaces.active.image if init_img is not None: if prompt.outpaint_origin[0] >= init_img.size[0] or \ prompt.outpaint_origin[1] >= init_img.size[1]: _outpaint_warning_box("Outpaint has no overlap, so the result will not blend") elif prompt.init_img_action == 'modify': layout.prop(prompt, "fit") if prompt.init_img_action != 'outpaint': layout.prop(prompt, "strength") layout.prop(prompt, "use_init_img_color") if prompt.init_img_action == 'modify': layout.prop(prompt, "modify_action_source_type") if prompt.modify_action_source_type == 'depth_map': layout.template_ID(context.scene, "init_depth", open="image.open") yield InitImagePanel def control_net_panel(sub_panel, space_type, get_prompt): class ControlNetPanel(sub_panel): """Create a subpanel for ControlNet options""" bl_idname = f"DREAM_PT_dream_panel_control_net_{space_type}" bl_label = "ControlNet" bl_options = {'DEFAULT_CLOSED'} def draw(self, context): layout = self.layout prompt = get_prompt(context) layout.operator("wm.call_menu", text="Add ControlNet", icon='ADD').name = "DREAM_MT_control_nets_add" for i, control_net in enumerate(prompt.control_nets): box = layout.box() box.use_property_split = False box.use_property_decorate = False row = box.row() row.prop(control_net, "enabled", icon="MODIFIER_ON" if control_net.enabled else "MODIFIER_OFF", icon_only=True, emboss=False) row.prop(control_net, "control_net", text="") row.operator("dream_textures.control_nets_remove", icon='X', emboss=False, text="").index = i col = box.column() col.use_property_split = True col.template_ID(control_net, "control_image", open="image.open", text="Image") processor_row = col.row() processor_row.prop(control_net, "processor_id") if control_net.processor_id != "none": processor_row.operator(BakeControlNetImage.bl_idname, icon='RENDER_STILL', text='').index = i col.prop(control_net, "conditioning_scale") return ControlNetPanel def advanced_panel(sub_panel, space_type, get_prompt): class AdvancedPanel(sub_panel): """Create a subpanel for advanced options""" bl_idname = f"DREAM_PT_dream_panel_advanced_{space_type}" bl_label = "Advanced" bl_options = {'DEFAULT_CLOSED'} def draw_header_preset(self, context): DREAM_PT_AdvancedPresets.draw_panel_header(self.layout) def draw(self, context): super().draw(context) layout = self.layout layout.use_property_split = True prompt = get_prompt(context) layout.prop(prompt, "random_seed") if not prompt.random_seed: layout.prop(prompt, "seed") # advanced_box.prop(self, "iterations") # Disabled until supported by the addon. layout.prop(prompt, "steps") layout.prop(prompt, "cfg_scale") layout.prop(prompt, "scheduler") layout.prop(prompt, "step_preview_mode") backend: api.Backend = prompt.get_backend() backend.draw_advanced(layout, context) yield AdvancedPanel yield from optimization_panels(sub_panel, space_type, get_prompt, AdvancedPanel.bl_idname) def optimization_panels(sub_panel, space_type, get_prompt, parent_id=""): class SpeedOptimizationPanel(sub_panel): """Create a subpanel for speed optimizations""" bl_idname = f"DREAM_PT_dream_panel_speed_optimizations_{space_type}" bl_label = "Speed Optimizations" bl_parent_id = parent_id def draw(self, context): super().draw(context) layout = self.layout layout.use_property_split = True prompt = get_prompt(context) backend: api.Backend = prompt.get_backend() backend.draw_speed_optimizations(layout, context) yield SpeedOptimizationPanel class MemoryOptimizationPanel(sub_panel): """Create a subpanel for memory optimizations""" bl_idname = f"DREAM_PT_dream_panel_memory_optimizations_{space_type}" bl_label = "Memory Optimizations" bl_parent_id = parent_id def draw(self, context): super().draw(context) layout = self.layout layout.use_property_split = True prompt = get_prompt(context) backend: api.Backend = prompt.get_backend() backend.draw_memory_optimizations(layout, context) yield MemoryOptimizationPanel def actions_panel(sub_panel, space_type, get_prompt): class ActionsPanel(sub_panel): """Create a subpanel for actions""" bl_idname = f"DREAM_PT_dream_panel_actions_{space_type}" bl_label = "Advanced" bl_options = {'HIDE_HEADER'} def draw(self, context): super().draw(context) layout = self.layout layout.use_property_split = True prompt = get_prompt(context) iterations_row = layout.row() iterations_row.enabled = prompt.prompt_structure != file_batch_structure.id iterations_row.prop(prompt, "iterations") row = layout.row(align=True) row.scale_y = 1.5 if CancelGenerator.poll(context): row.operator(CancelGenerator.bl_idname, icon="SNAP_FACE", text="") if context.scene.dream_textures_progress <= 0: if context.scene.dream_textures_info != "": disabled_row = row.row(align=True) disabled_row.operator(DreamTexture.bl_idname, text=context.scene.dream_textures_info, icon="INFO") disabled_row.enabled = False else: row.operator(DreamTexture.bl_idname, icon="PLAY", text="Generate") else: if bpy.app.version[0] >= 4: progress = context.scene.dream_textures_progress progress_max = bpy.types.Scene.dream_textures_progress.keywords['max'] row.progress(text=f"{progress} / {progress_max}", factor=progress / progress_max) else: disabled_row = row.row(align=True) disabled_row.use_property_split = True disabled_row.prop(context.scene, 'dream_textures_progress', slider=True) disabled_row.enabled = False row.operator(ReleaseGenerator.bl_idname, icon="X", text="") if context.scene.dream_textures_last_execution_time != "": r = layout.row() r.scale_x = 0.5 r.scale_y = 0.5 r.label(text=context.scene.dream_textures_last_execution_time, icon="SORTTIME") # Validation try: backend: api.Backend = prompt.get_backend() backend.validate(prompt.generate_args(context)) except FixItError as e: error_box = layout.box() error_box.use_property_split = False for i, line in enumerate(e.args[0].split('\n')): error_box.label(text=line, icon="ERROR" if i == 0 else "NONE") e._draw(prompt, context, error_box) return ActionsPanel ================================================ FILE: ui/panels/history.py ================================================ import bpy from bpy.types import Panel from ...prompt_engineering import * from ...operators.dream_texture import DreamTexture, ReleaseGenerator from ...operators.view_history import ExportHistorySelection, ImportPromptFile, RecallHistoryEntry, ClearHistory, RemoveHistorySelection from ...operators.open_latest_version import OpenLatestVersion, is_force_show_download, new_version_available from ...preferences import StableDiffusionPreferences from ..space_types import SPACE_TYPES def history_panels(): for space_type in SPACE_TYPES: class HistoryPanel(Panel): """Panel for Dream Textures History""" bl_label = "History" bl_category = "Dream" bl_idname = f"DREAM_PT_dream_history_panel_{space_type}" bl_space_type = space_type bl_region_type = 'UI' @classmethod def poll(cls, context): if cls.bl_space_type == 'NODE_EDITOR': return context.area.ui_type == "ShaderNodeTree" or context.area.ui_type == "CompositorNodeTree" else: return True def draw(self, context): self.layout.template_list("SCENE_UL_HistoryList", "", context.scene, "dream_textures_history", context.scene, "dream_textures_history_selection") row = self.layout.row() row.prop(context.scene, "dream_textures_history_selection_preview") row.operator(RemoveHistorySelection.bl_idname, text="", icon="X") row.operator(ExportHistorySelection.bl_idname, text="", icon="EXPORT") self.layout.operator(RecallHistoryEntry.bl_idname) self.layout.operator(ClearHistory.bl_idname) HistoryPanel.__name__ = f"DREAM_PT_dream_history_panel_{space_type}" yield HistoryPanel ================================================ FILE: ui/panels/render_properties.py ================================================ import bpy from .dream_texture import create_panel, prompt_panel, advanced_panel from ...property_groups.dream_prompt import backend_options from ...generator_process.models import ModelType from ...preferences import StableDiffusionPreferences class RenderPropertiesPanel(bpy.types.Panel): """Panel for Dream Textures render properties""" bl_label = "Dream Textures" bl_idname = "DREAM_PT_dream_render_properties_panel" bl_space_type = 'PROPERTIES' bl_region_type = 'WINDOW' bl_context = 'render' bl_options = {'DEFAULT_CLOSED'} @classmethod def poll(self, context): return context.scene.render.engine == 'CYCLES' def draw_header(self, context): self.layout.prop(context.scene, "dream_textures_render_properties_enabled", text="") def draw(self, context): layout = self.layout layout.use_property_split = True layout.use_property_decorate = False layout.active = context.scene.dream_textures_render_properties_enabled if len(backend_options(self, context)) > 1: layout.prop(context.scene.dream_textures_render_properties_prompt, "backend") layout.prop(context.scene.dream_textures_render_properties_prompt, 'model') layout.prop(context.scene.dream_textures_render_properties_prompt, "strength") layout.prop(context.scene, "dream_textures_render_properties_pass_inputs") if context.scene.dream_textures_render_properties_pass_inputs != 'color': if not bpy.context.view_layer.use_pass_z: box = layout.box() box.label(text="Z Pass Disabled", icon="ERROR") box.label(text="Enable the Z pass to use depth pass inputs") box.use_property_split = False box.prop(context.view_layer, "use_pass_z") models = list(filter( lambda m: m.model_base == context.scene.dream_textures_render_properties_prompt.model, context.preferences.addons[StableDiffusionPreferences.bl_idname].preferences.installed_models )) if len(models) > 0 and ModelType[models[0].model_type] != ModelType.DEPTH: box = layout.box() box.label(text="Unsupported model", icon="ERROR") box.label(text="Select a depth model, such as 'stabilityai/stable-diffusion-2-depth'") def render_properties_panels(): yield RenderPropertiesPanel def get_prompt(context): return context.scene.dream_textures_render_properties_prompt space_type = RenderPropertiesPanel.bl_space_type region_type = RenderPropertiesPanel.bl_region_type panels = [ *create_panel(space_type, region_type, RenderPropertiesPanel.bl_idname, prompt_panel, get_prompt, True), *create_panel(space_type, region_type, RenderPropertiesPanel.bl_idname, advanced_panel, get_prompt, True), ] for panel in panels: def draw_decorator(original): def draw(self, context): self.layout.enabled = context.scene.dream_textures_render_properties_enabled return original(self, context) return draw panel.draw = draw_decorator(panel.draw) if hasattr(panel, 'draw_header_preset'): panel.draw_header_preset = draw_decorator(panel.draw_header_preset) if hasattr(panel, 'draw_header'): panel.draw_header = draw_decorator(panel.draw_header) yield panel ================================================ FILE: ui/panels/upscaling.py ================================================ from bpy.types import Panel from ...prompt_engineering import * from ...operators.upscale import Upscale, get_source_image from ...operators.dream_texture import CancelGenerator, ReleaseGenerator from ...generator_process.actions.detect_seamless import SeamlessAxes from .dream_texture import create_panel, advanced_panel from ..space_types import SPACE_TYPES def upscaling_panels(): for space_type in SPACE_TYPES: class UpscalingPanel(Panel): """Panel for AI Upscaling""" bl_label = "AI Upscaling" bl_category = "Dream" bl_idname = f"DREAM_PT_dream_upscaling_panel_{space_type}" bl_space_type = space_type bl_region_type = 'UI' bl_options = {'DEFAULT_CLOSED'} @classmethod def poll(cls, context): if cls.bl_space_type == 'NODE_EDITOR': return context.area.ui_type == "ShaderNodeTree" or context.area.ui_type == "CompositorNodeTree" else: return True def draw(self, context): layout = self.layout layout.use_property_split = True layout.use_property_decorate = False prompt = context.scene.dream_textures_upscale_prompt layout.prop(prompt, "backend") layout.prop(prompt, "model") layout.prop(prompt, "prompt_structure_token_subject") layout.prop(context.scene, "dream_textures_upscale_tile_size") layout.prop(context.scene, "dream_textures_upscale_blend") layout.prop(prompt, "seamless_axes") if prompt.seamless_axes == SeamlessAxes.AUTO: node_tree = context.material.node_tree if hasattr(context, 'material') else None active_node = next((node for node in node_tree.nodes if node.select and node.bl_idname == 'ShaderNodeTexImage'), None) if node_tree is not None else None init_image = get_source_image(context) context.scene.dream_textures_upscale_seamless_result.check(init_image) auto_row = layout.row() auto_row.enabled = False auto_row.prop(context.scene.dream_textures_upscale_seamless_result, "result") if context.scene.dream_textures_upscale_tile_size > 128: warning_box = layout.box() warning_box.label(text="Warning", icon="ERROR") warning_box.label(text="Large tile sizes consume more VRAM.") UpscalingPanel.__name__ = UpscalingPanel.bl_idname class ActionsPanel(Panel): """Panel for AI Upscaling Actions""" bl_category = "Dream" bl_label = "Actions" bl_idname = f"DREAM_PT_dream_upscaling_actions_panel_{space_type}" bl_space_type = space_type bl_region_type = 'UI' bl_parent_id = UpscalingPanel.bl_idname bl_options = {'HIDE_HEADER'} @classmethod def poll(cls, context): if cls.bl_space_type == 'NODE_EDITOR': return context.area.ui_type == "ShaderNodeTree" or context.area.ui_type == "CompositorNodeTree" else: return True def draw(self, context): layout = self.layout layout.use_property_split = True layout.use_property_decorate = False image = get_source_image(context) row = layout.row(align=True) row.scale_y = 1.5 if CancelGenerator.poll(context): row.operator(CancelGenerator.bl_idname, icon="SNAP_FACE", text="") if context.scene.dream_textures_progress <= 0: if context.scene.dream_textures_info != "": disabled_row = row.row(align=True) disabled_row.operator(Upscale.bl_idname, text=context.scene.dream_textures_info, icon="INFO") disabled_row.enabled = False else: row.operator( Upscale.bl_idname, text=f"Upscale to {image.size[0] * 4}x{image.size[1] * 4}" if image is not None else "Upscale", icon="FULLSCREEN_ENTER" ) else: disabled_row = row.row(align=True) disabled_row.use_property_split = True disabled_row.prop(context.scene, 'dream_textures_progress', slider=True) disabled_row.enabled = False row.operator(ReleaseGenerator.bl_idname, icon="X", text="") yield UpscalingPanel advanced_panels = [*create_panel(space_type, 'UI', UpscalingPanel.bl_idname, advanced_panel, lambda context: context.scene.dream_textures_upscale_prompt)] outer_panel = advanced_panels[0] outer_original_idname = outer_panel.bl_idname outer_panel.bl_idname += "_upscaling" for panel in advanced_panels: panel.bl_idname += "_upscaling" if panel.bl_parent_id == outer_original_idname: panel.bl_parent_id = outer_panel.bl_idname yield panel yield ActionsPanel ================================================ FILE: ui/presets.py ================================================ import bpy from bpy.types import Panel, Operator, Menu from bl_operators.presets import AddPresetBase from bl_ui.utils import PresetPanel from typing import _AnnotatedAlias import os import shutil from ..absolute_path import absolute_path from ..generator_process.actions.prompt_to_image import Optimizations class DreamTexturesPresetPanel(PresetPanel, Panel): preset_operator = "script.execute_preset" class DREAM_PT_AdvancedPresets(DreamTexturesPresetPanel): bl_label = "Advanced Presets" preset_subdir = "dream_textures/advanced" preset_add_operator = "dream_textures.advanced_preset_add" class DREAM_MT_AdvancedPresets(Menu): bl_label = 'Advanced Presets' preset_subdir = 'dream_textures/advanced' preset_operator = 'script.execute_preset' draw = Menu.draw_preset class AddAdvancedPreset(AddPresetBase, Operator): bl_idname = 'dream_textures.advanced_preset_add' bl_label = 'Add Advanced Preset' preset_menu = 'DREAM_MT_AdvancedPresets' preset_subdir = 'dream_textures/advanced' preset_defines = ['prompt = bpy.context.scene.dream_textures_prompt'] preset_values = [ "prompt.steps", "prompt.cfg_scale", "prompt.scheduler", "prompt.step_preview_mode", "prompt.optimizations_attention_slicing", "prompt.optimizations_attention_slice_size_src", "prompt.optimizations_attention_slice_size", "prompt.optimizations_cudnn_benchmark", "prompt.optimizations_tf32", "prompt.optimizations_amp", "prompt.optimizations_half_precision", "prompt.optimizations_sequential_cpu_offload", "prompt.optimizations_channels_last_memory_format", "prompt.optimizations_batch_size", "prompt.optimizations_vae_slicing", "prompt.optimizations_cpu_only", ] class RestoreDefaultPresets(Operator): bl_idname = "dream_textures.restore_default_presets" bl_label = "Restore Default Presets" bl_description = ("Restores all default presets provided by the addon.") bl_options = {"REGISTER", "INTERNAL"} def execute(self, context): register_default_presets(force=True) return {"FINISHED"} PRESETS_PATH = os.path.join(bpy.utils.user_resource('SCRIPTS'), 'presets/dream_textures/advanced') DEFAULT_PRESETS_PATH = absolute_path('builtin_presets') def register_default_presets(force=False): presets_path_exists = os.path.isdir(PRESETS_PATH) if not presets_path_exists or force: if not presets_path_exists: os.makedirs(PRESETS_PATH) for default_preset in os.listdir(DEFAULT_PRESETS_PATH): if not os.path.exists(os.path.join(PRESETS_PATH, default_preset)): shutil.copy(os.path.join(DEFAULT_PRESETS_PATH, default_preset), PRESETS_PATH) def default_presets_missing(): if not os.path.isdir(PRESETS_PATH): return True for default_preset in os.listdir(DEFAULT_PRESETS_PATH): if not os.path.exists(os.path.join(PRESETS_PATH, default_preset)): return True ================================================ FILE: ui/space_types.py ================================================ SPACE_TYPES = {'IMAGE_EDITOR', 'NODE_EDITOR'} ================================================ FILE: version.py ================================================ VERSION = (0, 4, 1) def version_tag(version): return f"{version[0]}.{version[1]}.{version[2]}" def version_tuple(tag): return tuple(map(lambda x: int(x), tag.split('.')))