Repository: XmYx/deforum-comfy-nodes Branch: main Commit: 25c5e78b5c59 Files: 50 Total size: 564.5 KB Directory structure: gitextract_kwee4pen/ ├── .gitattributes ├── .github/ │ └── workflows/ │ └── publish_action.yml ├── .gitignore ├── LICENSE ├── README.md ├── __init__.py ├── deforum_nodes/ │ ├── __init__.py │ ├── exec_hijack.py │ ├── mapping.py │ ├── modules/ │ │ ├── __init__.py │ │ ├── better_resize/ │ │ │ ├── __init__.py │ │ │ ├── interp_methods.py │ │ │ └── resize_right.py │ │ ├── deforum_comfy_sampler.py │ │ ├── deforum_comfyui_helpers.py │ │ ├── deforum_constants.py │ │ ├── deforum_node_base.py │ │ ├── deforum_ui_data.py │ │ ├── interp.py │ │ └── standalone_cadence.py │ └── nodes/ │ ├── __init__.py │ ├── deforum_advnoise_node.py │ ├── deforum_audiosync_nodes.py │ ├── deforum_cache_nodes.py │ ├── deforum_cnet_nodes.py │ ├── deforum_cond_nodes.py │ ├── deforum_data_nodes.py │ ├── deforum_framewarp_node.py │ ├── deforum_hybrid_nodes.py │ ├── deforum_image_nodes.py │ ├── deforum_interpolation_nodes.py │ ├── deforum_iteration_nodes.py │ ├── deforum_legacy_nodes.py │ ├── deforum_logic_nodes.py │ ├── deforum_noise_nodes.py │ ├── deforum_prompt_nodes.py │ ├── deforum_sampler_nodes.py │ ├── deforum_schedule_visualizer.py │ ├── deforum_video_nodes.py │ └── redirect_console_node.py ├── examples/ │ ├── deforum_base.json │ ├── deforum_cadence.json │ ├── deforum_integrated.json │ ├── deforum_ip_adapter.json │ ├── deforum_simple.json │ ├── deforum_stablecascade.json │ └── deforum_stablecascade_legacy.json ├── install.py ├── node_list.json └── web/ └── js/ └── deforumIterateNode.js ================================================ FILE CONTENTS ================================================ ================================================ FILE: .gitattributes ================================================ # Auto detect text files and perform LF normalization * text=auto ================================================ FILE: .github/workflows/publish_action.yml ================================================ name: Publish to Comfy registry on: workflow_dispatch: push: branches: - main paths: - "pyproject.toml" jobs: publish-node: name: Publish Custom Node to registry runs-on: ubuntu-latest steps: - name: Check out code uses: actions/checkout@v4 - name: Publish Custom Node uses: Comfy-Org/publish-node-action@main with: ## Add your own personal access token to your Github Repository secrets and reference it here. personal_access_token: ${{ secrets.REGISTRY_ACCESS_TOKEN }} ================================================ FILE: .gitignore ================================================ # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] *$py.class # C extensions *.so # Distribution / packaging .Python build/ develop-eggs/ dist/ downloads/ eggs/ .eggs/ lib/ lib64/ parts/ sdist/ var/ wheels/ share/python-wheels/ *.egg-info/ .installed.cfg *.egg MANIFEST # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. *.manifest *.spec # Installer logs pip-log.txt pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ .tox/ .nox/ .coverage .coverage.* .cache nosetests.xml coverage.xml *.cover *.py,cover .hypothesis/ .pytest_cache/ cover/ # Translations *.mo *.pot # Django stuff: *.log local_settings.py db.sqlite3 db.sqlite3-journal # Flask stuff: instance/ .webassets-cache # Scrapy stuff: .scrapy # Sphinx documentation docs/_build/ # PyBuilder .pybuilder/ target/ # Jupyter Notebook .ipynb_checkpoints # IPython profile_default/ ipython_config.py # pyenv # For a library or package, you might want to ignore these files since the code is # intended to run in multiple environments; otherwise, check them in: # .python-version # pipenv # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. # However, in case of collaboration, if having platform-specific dependencies or dependencies # having no cross-platform support, pipenv may install dependencies that don't work, or not # install all needed dependencies. #Pipfile.lock # poetry # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. # This is especially recommended for binary packages to ensure reproducibility, and is more # commonly ignored for libraries. # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control #poetry.lock # PEP 582; used by e.g. github.com/David-OConnor/pyflow __pypackages__/ # Celery stuff celerybeat-schedule celerybeat.pid # SageMath parsed files *.sage.py # Environments .env .venv env/ venv/ ENV/ env.bak/ venv.bak/ # Spyder project settings .spyderproject .spyproject # Rope project settings .ropeproject # mkdocs documentation /site # mypy .mypy_cache/ .dmypy.json dmypy.json # Pyre type checker .pyre/ # pytype static type analyzer .pytype/ # Cython debug symbols cython_debug/ # PyCharm # JetBrains specific template is maintainted in a separate JetBrains.gitignore that can # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ ================================================ FILE: LICENSE ================================================ MIT License Copyright (c) 2024 Deforum LLC. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: README.md ================================================ # Deforum for ComfyUI Deforum integration for ComfyUI. ## Installation To get started with Deforum Comfy Nodes, please make sure ComfyUI is installed and you are using Python v3.10 or these nodes will not work. We recommend using a virtual environment. Follow the steps below depending on your method of preference. ### ComfyUI Manager Look for `Deforum Nodes` by `XmYx` ### Manual Install To install Deforum for ComfyUI we will clone this repo into the `custom_nodes` folder ```bash git clone https://github.com/XmYx/deforum-comfy-nodes.git ``` ## Recommended Custom Nodes Here is a list of extra custom nodes that greatly improves the experience of using Deforum. ```bash https://github.com/rgthree/rgthree-comfy https://github.com/a1lazydog/ComfyUI-AudioScheduler https://github.com/cubiq/ComfyUI_IPAdapter_plus https://github.com/Kosinkadink/ComfyUI-VideoHelperSuite https://github.com/Kosinkadink/ComfyUI-Advanced-ControlNet https://github.com/WASasquatch/was-node-suite-comfyui https://github.com/11cafe/comfyui-workspace-manager https://github.com/cubiq/ComfyUI_essentials https://github.com/FizzleDorf/ComfyUI_FizzNodes https://github.com/ltdrdata/ComfyUI-Impact-Pack https://github.com/Fannovel16/ComfyUI-Frame-Interpolation https://github.com/Fannovel16/ComfyUI-Video-Matting https://github.com/crystian/ComfyUI-Crystools ``` ## Usage 1. Launch ComfyUI 2. Load any of the example workflows from the examples folder. 3. Queue prompt, this will generate your first frame, you can enable Auto queueing, or batch as many images as long you'd like your animation to be. ## Contribution We welcome contributions from the community! If you're interested in improving Deforum Comfy Nodes or have ideas for new features, please follow these steps: 1. Fork the repository on GitHub. 2. Create a new branch for your feature or fix. 3. Commit your changes with clear, descriptive messages. 4. Push your changes to the branch and open a pull request. ## License Deforum Comfy Nodes is licensed under the MIT License. For more details, see the LICENSE file in the repository. ## Community and Support Join our Discord community to discuss Deforum Comfy Nodes, share your creations, and get help from the developers and other users: [Join Discord](https://discord.gg/deforum) [Visit our website](https://deforum.art) ![Deforum Website Logo](docs/logo.png) ================================================ FILE: __init__.py ================================================ from .deforum_nodes.mapping import NODE_CLASS_MAPPINGS, NODE_DISPLAY_NAME_MAPPINGS WEB_DIRECTORY = "./web" __all__ = ['NODE_CLASS_MAPPINGS', 'NODE_DISPLAY_NAME_MAPPINGS', "WEB_DIRECTORY"] ================================================ FILE: deforum_nodes/__init__.py ================================================ ================================================ FILE: deforum_nodes/exec_hijack.py ================================================ import execution, nodes orig_exec = execution.map_node_over_list def map_node_over_list(obj, input_data_all, func, allow_interrupt=False, execution_block_cb=None, pre_execute_cb=None): try: # check if node wants the lists input_is_list = getattr(obj, "INPUT_IS_LIST", False) if len(input_data_all) == 0: max_len_input = 0 else: max_len_input = max([len(x) for x in input_data_all.values() if x is not None]) # get a slice of inputs, repeat last input when list isn't long enough def slice_dict(d, i): d_new = dict() for k, v in d.items(): if v is None: # Skip if any of the values is None return None d_new[k] = v[i if len(v) > i else -1] return d_new results = [] for k, v in input_data_all.items(): if v == "skip": print("[deforum] Skipping execution of", obj) return [] def process_inputs(inputs, index=None): if allow_interrupt: nodes.before_node_execution() execution_block = None for k, v in inputs.items(): if isinstance(v, ExecutionBlocker): execution_block = execution_block_cb(v) if execution_block_cb else v break if execution_block is None: if pre_execute_cb is not None and index is not None: pre_execute_cb(index) results.append(getattr(obj, func)(**inputs)) else: results.append(execution_block) if input_is_list: process_inputs(input_data_all, 0) elif max_len_input == 0: process_inputs({}) else: for i in range(max_len_input): sliced_input = slice_dict(input_data_all, i) if sliced_input is None: # Skip this iteration if there's a None value after slicing continue if allow_interrupt: nodes.before_node_execution() for k, v in sliced_input.items(): if v == "skip": print("[deforum] Skipping execution of", obj) return [] process_inputs(sliced_input, i) return results except: print("[deforum] Executor HiJack Failed and was deactivated, please report the issue on GitHub") execution.map_node_over_list = orig_exec return orig_exec(obj, input_data_all, func, allow_interrupt, execution_block_cb, pre_execute_cb) execution.map_node_over_list = map_node_over_list print("[deforum] Execution HiJack Active") ================================================ FILE: deforum_nodes/mapping.py ================================================ import importlib import inspect import sys """ DEFORUM STORAGE IMPORT """ from .modules.deforum_constants import DeforumStorage gs = DeforumStorage() """ NODE CLASS IMPORTS """ from .nodes.deforum_audiosync_nodes import * from .nodes.deforum_cache_nodes import * from .nodes.deforum_cnet_nodes import * from .nodes.deforum_cond_nodes import * from .nodes.deforum_data_nodes import * from .nodes.deforum_framewarp_node import * from .nodes.deforum_hybrid_nodes import * from .nodes.deforum_interpolation_nodes import * from .nodes.deforum_image_nodes import * from .nodes.deforum_iteration_nodes import * from .nodes.deforum_legacy_nodes import * from .nodes.deforum_logic_nodes import * try: from .nodes.deforum_noise_nodes import AddCustomNoiseNode except: pass try: from .nodes.deforum_advnoise_node import AddAdvancedNoiseNode except: pass from .nodes.deforum_prompt_nodes import * from .nodes.redirect_console_node import DeforumRedirectConsole from .nodes.deforum_sampler_nodes import * from .nodes.deforum_schedule_visualizer import * from .nodes.deforum_video_nodes import * from . import exec_hijack # Create an empty dictionary for class mappings NODE_CLASS_MAPPINGS = {} NODE_DISPLAY_NAME_MAPPINGS = {} # Iterate through all classes defined in your code # Get a reference to the current module to inspect its imported node classes deforum_node_module = sys.modules[__name__] # Iterate through all classes defined in the current module for name, obj in inspect.getmembers(deforum_node_module): # Check if the member is a class if inspect.isclass(obj) and hasattr(obj, "INPUT_TYPES"): # Extract the class name and display name class_name = name display_name = getattr(obj, "display_name", name) # Use class attribute or default to class name # Add the class to the mappings NODE_CLASS_MAPPINGS[class_name] = obj NODE_DISPLAY_NAME_MAPPINGS[name] = "(deforum) " + display_name ================================================ FILE: deforum_nodes/modules/__init__.py ================================================ ================================================ FILE: deforum_nodes/modules/better_resize/__init__.py ================================================ ================================================ FILE: deforum_nodes/modules/better_resize/interp_methods.py ================================================ from math import pi try: import torch except ImportError: torch = None try: import numpy except ImportError: numpy = None if numpy is None and torch is None: raise ImportError("Must have either Numpy or PyTorch but both not found") def set_framework_dependencies(x): if type(x) is numpy.ndarray: to_dtype = lambda a: a fw = numpy else: to_dtype = lambda a: a.to(x.dtype) fw = torch eps = fw.finfo(fw.float32).eps return fw, to_dtype, eps def support_sz(sz): def wrapper(f): f.support_sz = sz return f return wrapper @support_sz(4) def cubic(x): fw, to_dtype, eps = set_framework_dependencies(x) absx = fw.abs(x) absx2 = absx ** 2 absx3 = absx ** 3 return ((1.5 * absx3 - 2.5 * absx2 + 1.) * to_dtype(absx <= 1.) + (-0.5 * absx3 + 2.5 * absx2 - 4. * absx + 2.) * to_dtype((1. < absx) & (absx <= 2.))) @support_sz(4) def lanczos2(x): fw, to_dtype, eps = set_framework_dependencies(x) return (((fw.sin(pi * x) * fw.sin(pi * x / 2) + eps) / ((pi**2 * x**2 / 2) + eps)) * to_dtype(abs(x) < 2)) @support_sz(6) def lanczos3(x): fw, to_dtype, eps = set_framework_dependencies(x) return (((fw.sin(pi * x) * fw.sin(pi * x / 3) + eps) / ((pi**2 * x**2 / 3) + eps)) * to_dtype(abs(x) < 3)) @support_sz(2) def linear(x): fw, to_dtype, eps = set_framework_dependencies(x) return ((x + 1) * to_dtype((-1 <= x) & (x < 0)) + (1 - x) * to_dtype((0 <= x) & (x <= 1))) @support_sz(1) def box(x): fw, to_dtype, eps = set_framework_dependencies(x) return to_dtype((-1 <= x) & (x < 0)) + to_dtype((0 <= x) & (x <= 1)) ================================================ FILE: deforum_nodes/modules/better_resize/resize_right.py ================================================ from typing import Tuple import warnings from math import ceil from . import interp_methods from fractions import Fraction class NoneClass: pass try: import torch from torch import nn nnModuleWrapped = nn.Module except ImportError: warnings.warn('No PyTorch found, will work only with Numpy') torch = None nnModuleWrapped = NoneClass try: import numpy except ImportError: warnings.warn('No Numpy found, will work only with PyTorch') numpy = None if numpy is None and torch is None: raise ImportError("Must have either Numpy or PyTorch but both not found") def resize(input, scale_factors=None, out_shape=None, interp_method=interp_methods.cubic, support_sz=None, antialiasing=True, by_convs=False, scale_tolerance=None, max_numerator=10, pad_mode='constant'): # get properties of the input tensor in_shape, n_dims = input.shape, input.ndim # fw stands for framework that can be either numpy or torch, # determined by the input type fw = numpy if type(input) is numpy.ndarray else torch eps = fw.finfo(fw.float32).eps device = input.device if fw is torch else None # set missing scale factors or output shapem one according to another, # scream if both missing. this is also where all the defults policies # take place. also handling the by_convs attribute carefully. scale_factors, out_shape, by_convs = set_scale_and_out_sz(in_shape, out_shape, scale_factors, by_convs, scale_tolerance, max_numerator, eps, fw) # sort indices of dimensions according to scale of each dimension. # since we are going dim by dim this is efficient sorted_filtered_dims_and_scales = [(dim, scale_factors[dim], by_convs[dim], in_shape[dim], out_shape[dim]) for dim in sorted(range(n_dims), key=lambda ind: scale_factors[ind]) if scale_factors[dim] != 1.] # unless support size is specified by the user, it is an attribute # of the interpolation method if support_sz is None: support_sz = interp_method.support_sz # output begins identical to input and changes with each iteration output = input # iterate over dims for (dim, scale_factor, dim_by_convs, in_sz, out_sz ) in sorted_filtered_dims_and_scales: # STEP 1- PROJECTED GRID: The non-integer locations of the projection # of output pixel locations to the input tensor projected_grid = get_projected_grid(in_sz, out_sz, scale_factor, fw, dim_by_convs, device) # STEP 1.5: ANTIALIASING- If antialiasing is taking place, we modify # the window size and the interpolation method (see inside function) cur_interp_method, cur_support_sz = apply_antialiasing_if_needed( interp_method, support_sz, scale_factor, antialiasing) # STEP 2- FIELDS OF VIEW: for each output pixels, map the input pixels # that influence it. Also calculate needed padding and update grid # accoedingly field_of_view = get_field_of_view(projected_grid, cur_support_sz, fw, eps, device) # STEP 2.5- CALCULATE PAD AND UPDATE: according to the field of view, # the input should be padded to handle the boundaries, coordinates # should be updated. actual padding only occurs when weights are # aplied (step 4). if using by_convs for this dim, then we need to # calc right and left boundaries for each filter instead. pad_sz, projected_grid, field_of_view = calc_pad_sz(in_sz, out_sz, field_of_view, projected_grid, scale_factor, dim_by_convs, fw, device) # STEP 3- CALCULATE WEIGHTS: Match a set of weights to the pixels in # the field of view for each output pixel weights = get_weights(cur_interp_method, projected_grid, field_of_view) # STEP 4- APPLY WEIGHTS: Each output pixel is calculated by multiplying # its set of weights with the pixel values in its field of view. # We now multiply the fields of view with their matching weights. # We do this by tensor multiplication and broadcasting. # if by_convs is true for this dim, then we do this action by # convolutions. this is equivalent but faster. if not dim_by_convs: output = apply_weights(output, field_of_view, weights, dim, n_dims, pad_sz, pad_mode, fw) else: output = apply_convs(output, scale_factor, in_sz, out_sz, weights, dim, pad_sz, pad_mode, fw) return output def get_projected_grid(in_sz, out_sz, scale_factor, fw, by_convs, device=None): # we start by having the ouput coordinates which are just integer locations # in the special case when usin by_convs, we only need two cycles of grid # points. the first and last. grid_sz = out_sz if not by_convs else scale_factor.numerator out_coordinates = fw_arange(grid_sz, fw, device) # This is projecting the ouput pixel locations in 1d to the input tensor, # as non-integer locations. # the following fomrula is derived in the paper # "From Discrete to Continuous Convolutions" by Shocher et al. return (out_coordinates / float(scale_factor) + (in_sz - 1) / 2 - (out_sz - 1) / (2 * float(scale_factor))) def get_field_of_view(projected_grid, cur_support_sz, fw, eps, device): # for each output pixel, map which input pixels influence it, in 1d. # we start by calculating the leftmost neighbor, using half of the window # size (eps is for when boundary is exact int) left_boundaries = fw_ceil(projected_grid - cur_support_sz / 2 - eps, fw) # then we simply take all the pixel centers in the field by counting # window size pixels from the left boundary ordinal_numbers = fw_arange(ceil(cur_support_sz - eps), fw, device) return left_boundaries[:, None] + ordinal_numbers def calc_pad_sz(in_sz, out_sz, field_of_view, projected_grid, scale_factor, dim_by_convs, fw, device): if not dim_by_convs: # determine padding according to neighbor coords out of bound. # this is a generalized notion of padding, when pad<0 it means crop pad_sz = [-field_of_view[0, 0].item(), field_of_view[-1, -1].item() - in_sz + 1] # since input image will be changed by padding, coordinates of both # field_of_view and projected_grid need to be updated field_of_view += pad_sz[0] projected_grid += pad_sz[0] else: # only used for by_convs, to calc the boundaries of each filter the # number of distinct convolutions is the numerator of the scale factor num_convs, stride = scale_factor.numerator, scale_factor.denominator # calculate left and right boundaries for each conv. left can also be # negative right can be bigger than in_sz. such cases imply padding if # needed. however if# both are in-bounds, it means we need to crop, # practically apply the conv only on part of the image. left_pads = -field_of_view[:, 0] # next calc is tricky, explanation by rows: # 1) counting output pixels between the first position of each filter # to the right boundary of the input # 2) dividing it by number of filters to count how many 'jumps' # each filter does # 3) multiplying by the stride gives us the distance over the input # coords done by all these jumps for each filter # 4) to this distance we add the right boundary of the filter when # placed in its leftmost position. so now we get the right boundary # of that filter in input coord. # 5) the padding size needed is obtained by subtracting the rightmost # input coordinate. if the result is positive padding is needed. if # negative then negative padding means shaving off pixel columns. right_pads = (((out_sz - fw_arange(num_convs, fw, device) - 1) # (1) // num_convs) # (2) * stride # (3) + field_of_view[:, -1] # (4) - in_sz + 1) # (5) # in the by_convs case pad_sz is a list of left-right pairs. one per # each filter pad_sz = list(zip(left_pads, right_pads)) return pad_sz, projected_grid, field_of_view def get_weights(interp_method, projected_grid, field_of_view): # the set of weights per each output pixels is the result of the chosen # interpolation method applied to the distances between projected grid # locations and the pixel-centers in the field of view (distances are # directed, can be positive or negative) weights = interp_method(projected_grid[:, None] - field_of_view) # we now carefully normalize the weights to sum to 1 per each output pixel sum_weights = weights.sum(1, keepdims=True) sum_weights[sum_weights == 0] = 1 return weights / sum_weights def apply_weights(input, field_of_view, weights, dim, n_dims, pad_sz, pad_mode, fw): # for this operation we assume the resized dim is the first one. # so we transpose and will transpose back after multiplying tmp_input = fw_swapaxes(input, dim, 0, fw) # apply padding tmp_input = fw_pad(tmp_input, fw, pad_sz, pad_mode) # field_of_view is a tensor of order 2: for each output (1d location # along cur dim)- a list of 1d neighbors locations. # note that this whole operations is applied to each dim separately, # this is why it is all in 1d. # neighbors = tmp_input[field_of_view] is a tensor of order image_dims+1: # for each output pixel (this time indicated in all dims), these are the # values of the neighbors in the 1d field of view. note that we only # consider neighbors along the current dim, but such set exists for every # multi-dim location, hence the final tensor order is image_dims+1. neighbors = tmp_input[field_of_view] # weights is an order 2 tensor: for each output location along 1d- a list # of weights matching the field of view. we augment it with ones, for # broadcasting, so that when multiplies some tensor the weights affect # only its first dim. tmp_weights = fw.reshape(weights, (*weights.shape, * [1] * (n_dims - 1))) # now we simply multiply the weights with the neighbors, and then sum # along the field of view, to get a single value per out pixel tmp_output = (neighbors * tmp_weights).sum(1) # we transpose back the resized dim to its original position return fw_swapaxes(tmp_output, 0, dim, fw) def apply_convs(input, scale_factor, in_sz, out_sz, weights, dim, pad_sz, pad_mode, fw): # for this operations we assume the resized dim is the last one. # so we transpose and will transpose back after multiplying input = fw_swapaxes(input, dim, -1, fw) # the stride for all convs is the denominator of the scale factor stride, num_convs = scale_factor.denominator, scale_factor.numerator # prepare an empty tensor for the output tmp_out_shape = list(input.shape) tmp_out_shape[-1] = out_sz tmp_output = fw_empty(tuple(tmp_out_shape), fw, input.device) # iterate over the conv operations. we have as many as the numerator # of the scale-factor. for each we need boundaries and a filter. for conv_ind, (pad_sz, filt) in enumerate(zip(pad_sz, weights)): # apply padding (we pad last dim, padding can be negative) pad_dim = input.ndim - 1 tmp_input = fw_pad(input, fw, pad_sz, pad_mode, dim=pad_dim) # apply convolution over last dim. store in the output tensor with # positional strides so that when the loop is comlete conv results are # interwind tmp_output[..., conv_ind::num_convs] = fw_conv(tmp_input, filt, stride) return fw_swapaxes(tmp_output, -1, dim, fw) def set_scale_and_out_sz(in_shape, out_shape, scale_factors, by_convs, scale_tolerance, max_numerator, eps, fw): # eventually we must have both scale-factors and out-sizes for all in/out # dims. however, we support many possible partial arguments if scale_factors is None and out_shape is None: raise ValueError("either scale_factors or out_shape should be " "provided") if out_shape is not None: # if out_shape has less dims than in_shape, we defaultly resize the # first dims for numpy and last dims for torch out_shape = (list(out_shape) + list(in_shape[len(out_shape):]) if fw is numpy else list(in_shape[:-len(out_shape)]) + list(out_shape)) if scale_factors is None: # if no scale given, we calculate it as the out to in ratio # (not recomended) scale_factors = [out_sz / in_sz for out_sz, in_sz in zip(out_shape, in_shape)] if scale_factors is not None: # by default, if a single number is given as scale, we assume resizing # two dims (most common are images with 2 spatial dims) scale_factors = (scale_factors if isinstance(scale_factors, (list, tuple)) else [scale_factors, scale_factors]) # if less scale_factors than in_shape dims, we defaultly resize the # first dims for numpy and last dims for torch scale_factors = (list(scale_factors) + [1] * (len(in_shape) - len(scale_factors)) if fw is numpy else [1] * (len(in_shape) - len(scale_factors)) + list(scale_factors)) if out_shape is None: # when no out_shape given, it is calculated by multiplying the # scale by the in_shape (not recomended) out_shape = [ceil(scale_factor * in_sz) for scale_factor, in_sz in zip(scale_factors, in_shape)] # next part intentionally after out_shape determined for stability # we fix by_convs to be a list of truth values in case it is not if not isinstance(by_convs, (list, tuple)): by_convs = [by_convs] * len(out_shape) # next loop fixes the scale for each dim to be either frac or float. # this is determined by by_convs and by tolerance for scale accuracy. for ind, (sf, dim_by_convs) in enumerate(zip(scale_factors, by_convs)): # first we fractionaize if dim_by_convs: frac = Fraction(1/sf).limit_denominator(max_numerator) frac = Fraction(numerator=frac.denominator, denominator=frac.numerator) # if accuracy is within tolerance scale will be frac. if not, then # it will be float and the by_convs attr will be set false for # this dim if scale_tolerance is None: scale_tolerance = eps if dim_by_convs and abs(frac - sf) < scale_tolerance: scale_factors[ind] = frac else: scale_factors[ind] = float(sf) by_convs[ind] = False return scale_factors, out_shape, by_convs def apply_antialiasing_if_needed(interp_method, support_sz, scale_factor, antialiasing): # antialiasing is "stretching" the field of view according to the scale # factor (only for downscaling). this is low-pass filtering. this # requires modifying both the interpolation (stretching the 1d # function and multiplying by the scale-factor) and the window size. scale_factor = float(scale_factor) if scale_factor >= 1.0 or not antialiasing: return interp_method, support_sz cur_interp_method = (lambda arg: scale_factor * interp_method(scale_factor * arg)) cur_support_sz = support_sz / scale_factor return cur_interp_method, cur_support_sz def fw_ceil(x, fw): if fw is numpy: return fw.int_(fw.ceil(x)) else: return x.ceil().long() def fw_floor(x, fw): if fw is numpy: return fw.int_(fw.floor(x)) else: return x.floor().long() def fw_cat(x, fw): if fw is numpy: return fw.concatenate(x) else: return fw.cat(x) def fw_swapaxes(x, ax_1, ax_2, fw): if fw is numpy: return fw.swapaxes(x, ax_1, ax_2) else: return x.transpose(ax_1, ax_2) def fw_pad(x, fw, pad_sz, pad_mode, dim=0): if pad_sz == (0, 0): return x if fw is numpy: pad_vec = [(0, 0)] * x.ndim pad_vec[dim] = pad_sz return fw.pad(x, pad_width=pad_vec, mode=pad_mode) else: if x.ndim < 3: x = x[None, None, ...] pad_vec = [0] * ((x.ndim - 2) * 2) pad_vec[0:2] = pad_sz return fw.nn.functional.pad(x.transpose(dim, -1), pad=pad_vec, mode=pad_mode).transpose(dim, -1) def fw_conv(input, filter, stride): # we want to apply 1d conv to any nd array. the way to do it is to reshape # the input to a 4D tensor. first two dims are singeletons, 3rd dim stores # all the spatial dims that we are not convolving along now. then we can # apply conv2d with a 1xK filter. This convolves the same way all the other # dims stored in the 3d dim. like depthwise conv over these. # TODO: numpy support reshaped_input = input.reshape(1, 1, -1, input.shape[-1]) reshaped_output = torch.nn.functional.conv2d(reshaped_input, filter.view(1, 1, 1, -1), stride=(1, stride)) return reshaped_output.reshape(*input.shape[:-1], -1) def fw_arange(upper_bound, fw, device): if fw is numpy: return fw.arange(upper_bound) else: return fw.arange(upper_bound, device=device) def fw_empty(shape, fw, device): if fw is numpy: return fw.empty(shape) else: return fw.empty(size=(*shape,), device=device) ================================================ FILE: deforum_nodes/modules/deforum_comfy_sampler.py ================================================ import secrets import torch import numpy as np from PIL import Image from deforum import ImageRNGNoise from deforum.utils.deforum_cond_utils import blend_tensors rng = None def common_ksampler_with_custom_noise(model, seed, steps, cfg, sampler_name, scheduler, positive, negative, latent, denoise=1.0, disable_noise=False, start_step=None, last_step=None, force_full_denoise=False, noise=None): latent_image = latent["samples"] if noise is not None: noise = noise.next()#.detach().cpu() # noise = rng_noise.clone() else: if disable_noise: noise = torch.zeros(latent_image.size(), dtype=latent_image.dtype, layout=latent_image.layout, device="cpu") else: batch_inds = latent["batch_index"] if "batch_index" in latent else None from comfy.sample import prepare_noise noise = prepare_noise(latent_image, seed, batch_inds) noise_mask = None if "noise_mask" in latent: noise_mask = latent["noise_mask"] # callback = latent_preview.prepare_callback(model, steps) # disable_pbar = not comfy.utils.PROGRESS_BAR_ENABLED from comfy.sample import sample as sample_k samples = sample_k(model, noise, steps, cfg, sampler_name, scheduler, positive, negative, latent_image, denoise=denoise, disable_noise=disable_noise, start_step=start_step, last_step=last_step, force_full_denoise=force_full_denoise, noise_mask=noise_mask, callback=None, disable_pbar=False, seed=seed) out = latent.copy() out["samples"] = samples return (out,) def sample_deforum( model, clip, vae, controlnet=None, prompt="", pooled_prompts=None, next_prompt=None, prompt_blend=None, negative_prompt="", steps=25, scale=7.5, sampler_name="dpmpp_2m_sde", scheduler="karras", width=None, height=None, seed=-1, strength=1.0, init_image=None, subseed=-1, subseed_strength=0.6, cnet_image=None, cond=None, n_cond=None, return_latent=None, latent=None, last_step=None, seed_resize_from_h=1024, seed_resize_from_w=1024, reset_noise=False, enable_prompt_blend=True, use_areas= False, areas= None, *args, **kwargs): # import comfy.sd # model, clip, vae, clipvision = comfy.sd.load_checkpoint_guess_config(self.model_path, # output_vae=True, # output_clip=True, # embedding_directory="models/embeddings", # output_clipvision=False, # ) if seed == -1: seed = secrets.randbelow(18446744073709551615) # strength = 1 - strength if strength <= 0.0 or strength >= 1.0: strength = 1.0 reset_noise = True init_image = None if subseed == -1: subseed = secrets.randbelow(18446744073709551615) if cnet_image is not None: cnet_image = torch.from_numpy(np.array(cnet_image).astype(np.float32) / 255.0).unsqueeze(0) if init_image is None or reset_noise: strength = 1.0 if latent is None: if width is None: width = 1024 if height is None: height = 960 latent = generate_latent(width, height, seed, subseed, subseed_strength, seed_resize_from_h, seed_resize_from_w, reset_noise) else: if isinstance(latent, torch.Tensor): latent = {"samples":latent} elif isinstance(latent, list): latent = {"samples":torch.stack(latent, dim=0)} else: latent = latent else: latent = torch.from_numpy(np.array(init_image).astype(np.float32) / 255.0).unsqueeze(0) latent = encode_latent(vae, latent, seed, subseed, subseed_strength, seed_resize_from_h, seed_resize_from_w) assert isinstance(latent, dict), \ "Our Latents have to be in a dict format with the latent being the 'samples' value" cond = [] if pooled_prompts is None and prompt is not None: cond = get_conds(clip, prompt) elif pooled_prompts is not None: cond = pooled_prompts if use_areas and areas is not None: from nodes import ConditioningSetArea area_setter = ConditioningSetArea() for area in areas: print("[deforum] Area Prompt:", area) prompt = area.get("prompt", None) if prompt: new_cond = get_conds(clip, area["prompt"]) new_cond = area_setter.append(conditioning=new_cond, width=int(area["w"]), height=int(area["h"]), x=int(area["x"]), y=int(area["y"]), strength=area["s"])[0] cond += new_cond n_cond = get_conds(clip, negative_prompt) if next_prompt is not None and enable_prompt_blend: if next_prompt != prompt and next_prompt != "": if 0.0 < prompt_blend < 1.0: next_cond = get_conds(clip, next_prompt) cond = blend_tensors(cond[0], next_cond[0], blend_value=prompt_blend) if cnet_image is not None: cond = apply_controlnet(cond, controlnet, cnet_image, 1.0) from nodes import common_ksampler as ksampler #steps = int((strength) * steps) if (strength != 1.0 or not reset_noise) else steps last_step = steps# if last_step is None else last_step sample = common_ksampler_with_custom_noise(model=model, seed=seed, steps=steps, cfg=scale, sampler_name=sampler_name, scheduler=scheduler, positive=cond, negative=n_cond, latent=latent, denoise=strength, disable_noise=False, start_step=0, last_step=last_step, force_full_denoise=True, noise=rng) if sample[0]["samples"].shape[0] == 1: decoded = decode_sample(vae, sample[0]["samples"]) np_array = np.clip(255. * decoded.cpu().numpy(), 0, 255).astype(np.uint8)[0] image = Image.fromarray(np_array) # image = Image.fromarray(np.clip(255. * decoded.cpu().numpy(), 0, 255).astype(np.uint8)[0]) image = image.convert("RGB") image.save('test.png', "PNG") if return_latent: return sample[0]["samples"], image else: return image else: images = [] x_samples = vae.decode_tiled(sample[0]["samples"]) for sample in x_samples: np_array = np.clip(255. * sample.cpu().numpy(), 0, 255).astype(np.uint8) image = Image.fromarray(np_array) # image = Image.fromarray(np.clip(255. * decoded.cpu().numpy(), 0, 255).astype(np.uint8)[0]) image = image.convert("RGB") images.append(image) return images def encode_latent(vae, latent, seed, subseed, subseed_strength, seed_resize_from_h, seed_resize_from_w, reset_noise=False): global rng subseed_strength = 0.6 with torch.inference_mode(): latent = latent.to(torch.float32) latent = vae.encode_tiled(latent[:, :, :, :3]) latent = latent.to("cuda") if rng is None or reset_noise: rng = ImageRNGNoise(shape=latent[0].shape, seeds=[seed], subseeds=[subseed], subseed_strength=subseed_strength, seed_resize_from_h=seed_resize_from_h, seed_resize_from_w=seed_resize_from_w) # noise = self.rng.first() # noise = slerp(subseed_strength, noise, latent) # else: # #noise = self.rng.next() # #noise = slerp(subseed_strength, noise, latent) # noise = latent return {"samples": latent} def generate_latent(width, height, seed, subseed, subseed_strength, seed_resize_from_h=None, seed_resize_from_w=None, reset_noise=False): shape = [4, height // 8, width // 8] global rng if rng is None or reset_noise: rng = ImageRNGNoise(shape=shape, seeds=[seed], subseeds=[subseed], subseed_strength=subseed_strength, seed_resize_from_h=seed_resize_from_h, seed_resize_from_w=seed_resize_from_w) noise = rng.next() # noise = torch.zeros([1, 4, width // 8, height // 8]) return {"samples": noise} def get_conds(clip, prompt): with torch.inference_mode(): # clip_skip = -1 # if clip_skip != clip_skip or clip.layer_idx != clip_skip: # clip.layer_idx = clip_skip # clip.clip_layer(clip_skip) # self.clip_skip = clip_skip tokens = clip.tokenize(prompt) cond, pooled = clip.encode_from_tokens(tokens, return_pooled=True) return [[cond, {"pooled_output": pooled}]] def apply_controlnet(conditioning, control_net, image, strength): with torch.inference_mode(): if strength == 0: return (conditioning,) c = [] control_hint = image.movedim(-1, 1) for t in conditioning: n = [t[0], t[1].copy()] c_net = control_net.copy().set_cond_hint(control_hint, strength) if 'control' in t[1]: c_net.set_previous_controlnet(t[1]['control']) n[1]['control'] = c_net n[1]['control_apply_to_uncond'] = True c.append(n) return c def decode_sample(vae, sample): # with torch.inference_mode(): # sample = sample.to(torch.float32) # vae.first_stage_model.cuda() decoded = vae.decode_tiled(sample).detach() return decoded ================================================ FILE: deforum_nodes/modules/deforum_comfyui_helpers.py ================================================ import base64 import math import os import random import re from io import BytesIO import cv2 import numexpr import numpy as np import pandas as pd import torch from PIL import Image from deforum.pipelines.deforum_animation.animation_helpers import DeforumAnimKeys from deforum.pipelines.deforum_animation.pipeline_deforum_animation import interpolate_areas blend_methods = ["linear", "sigmoidal", "gaussian", "pyramid", "none"] import torch import torch.nn.functional as F import math def pad_to_match(tensor1, tensor2): """ Pad the smaller tensor with zeros to match the size of the larger tensor. """ if tensor1.size() == tensor2.size(): return tensor1, tensor2 else: # Find the size difference between the two tensors diff_dims = [abs(s1 - s2) for s1, s2 in zip(tensor1.size(), tensor2.size())] # Padding format (left, right, top, bottom) for 4D tensors, e.g., (batch, channels, height, width) padding = [] for diff in reversed(diff_dims): # Reverse to start padding from last dimensions padding.extend([diff // 2, diff - diff // 2]) # Apply padding to the smaller tensor if tensor1.numel() < tensor2.numel(): tensor1 = F.pad(tensor1, padding) else: tensor2 = F.pad(tensor2, padding) return tensor1, tensor2 def parse_widget(widget_info: dict) -> tuple: parsed_widget = None t = widget_info["type"] if t == "dropdown": parsed_widget = (widget_info["choices"],) elif t == "checkbox": parsed_widget = ("BOOLEAN", {"default": widget_info['default']}) elif t == "lineedit": parsed_widget = ("STRING", {"default": widget_info['default']}) elif t == "spinbox": parsed_widget = ("INT", {"default": widget_info['default']}) elif t == "doublespinbox": parsed_widget = ("FLOAT", {"default": widget_info['default']}) return parsed_widget def get_node_params(input_params): data_info = {"required": {}, } if input_params: for name, widget_info in input_params.items(): data_info["required"][name] = parse_widget(widget_info) data_info["optional"] = {"deforum_data": ("deforum_data",)} return data_info def get_current_keys(anim_args, seed, root, parseq_args=None, video_args=None, area_prompts=None): use_parseq = False if parseq_args == None else True anim_args.max_frames += 2 keys = DeforumAnimKeys(anim_args, seed) # if not use_parseq else ParseqAnimKeys(parseq_args, video_args) areas = None # Always enable pseudo-3d with parseq. No need for an extra toggle: # Whether it's used or not in practice is defined by the schedules if use_parseq: anim_args.flip_2d_perspective = True # expand prompts out to per-frame if use_parseq and keys.manages_prompts(): prompt_series = keys.prompts else: prompt_series = None if hasattr(root, 'animation_prompts'): if root.animation_prompts is not None: prompt_series = pd.Series([np.nan for a in range(anim_args.max_frames + 1)]) for i, prompt in root.animation_prompts.items(): if str(i).isdigit(): prompt_series[int(i)] = prompt else: prompt_series[int(numexpr.evaluate(i))] = prompt prompt_series = prompt_series.ffill().bfill() if area_prompts is not None: areas = interpolate_areas(area_prompts, anim_args.max_frames) anim_args.max_frames -= 2 return keys, prompt_series, areas def tensor2pil(image): if image is not None: #with torch.inference_mode(): return Image.fromarray(np.clip(255. * image.detach().cpu().numpy().squeeze(), 0, 255).astype(np.uint8)) else: return None # PIL to Tensor def pil2tensor(image): return torch.from_numpy(np.array(image).astype(np.float32) / 255.0).unsqueeze(0) def tensor2np(img): np_img = np.array(tensor2pil(img)) return np_img def get_latent_with_seed(seed): return torch.randn(generator=torch.manual_seed(seed)) def pil_image_to_base64(pil_image): buffer = BytesIO() pil_image.save(buffer, format="WEBP") # Or JPEG return base64.b64encode(buffer.getvalue()).decode() def tensor_to_webp_base64(tensor): # Ensure tensor is in CPU and detach it from the computation graph tensor = tensor.clone().detach().cpu() # Convert tensor to a numpy array with value range [0, 255] # Transpose the tensor to have the channel in the correct order (H, W, C) np_image = (tensor.numpy().squeeze() * 255).clip(0, 255).astype(np.uint8) if np_image.ndim == 2: # if it's a grayscale image, convert to RGB np_image = cv2.cvtColor(np_image, cv2.COLOR_GRAY2RGB) elif np_image.shape[0] == 3: # Convert CHW to HWC np_image = np_image.transpose(1, 2, 0) np_image = cv2.cvtColor(np_image, cv2.COLOR_BGR2RGB) # Encode the numpy array to WEBP using OpenCV compression_quality = 80 _, encoded_image = cv2.imencode('.webp', np_image, [cv2.IMWRITE_WEBP_QUALITY, compression_quality]) # Convert the encoded image to base64 base64_str = base64.b64encode(encoded_image).decode() return base64_str def generate_seed_list(max_frames, mode='fixed', start_seed=0, step=1): """ Generates a list of seed integers compatible with PyTorch in various manners. Parameters: - max_frames (int): The maximum number of frames/length of the seed list. - mode (str): The mode of seed generation, one of 'fixed', 'random', 'ladder', 'incrementing', or 'decrementing'. - start_seed (int): The starting seed value for modes other than 'random'. - step (int): The step size for 'incrementing', 'decrementing', and 'ladder' modes. Returns: - list: A list of seed integers. """ if mode == 'fixed': return [start_seed for _ in range(max_frames)] elif mode == 'random': return [random.randint(0, 2**32 - 1) for _ in range(max_frames)] elif mode == 'ladder': # Generate a ladder sequence where the sequence is repeated after reaching the max_frames return [(start_seed + i // 2 * step if i % 2 == 0 else start_seed + (i // 2 + 1) * step) % (2**32) for i in range(max_frames)] elif mode == 'incrementing' or 'iter': return [(start_seed + i * step) % (2**32) for i in range(max_frames)] elif mode == 'decrementing': return [(start_seed - i * step) % (2**32) for i in range(max_frames)] else: raise ValueError("Invalid mode specified. Choose among 'fixed', 'random', 'ladder', 'incrementing', 'decrementing'.") def find_next_index(output_dir, filename_prefix, format): """ Finds the next index for an MP4 file given an output directory and a filename prefix. Parameters: - output_dir: The directory where the MP4 files are saved. - filename_prefix: The prefix for the filenames. Returns: - An integer representing the next index for a new MP4 file. """ # Compile a regular expression pattern to match the filenames # This assumes the index is at the end of the filename, before the .mp4 extension pattern = re.compile(rf"^{re.escape(filename_prefix)}_(\d+)\.{format.lower()}$") max_index = -1 for filename in os.listdir(output_dir): match = pattern.match(filename) if match: # Extract the current index from the filename current_index = int(match.group(1)) # Update the max index found if current_index > max_index: max_index = current_index # The next index is one more than the highest index found next_index = max_index + 1 return next_index import torch.nn.functional as F def pyramid_blend(tensor1, tensor2, blend_value): tensor1, tensor2 = pad_to_match(tensor1, tensor2) # For simplicity, we'll use two levels of blending downsampled1 = F.avg_pool2d(tensor1, 2) downsampled2 = F.avg_pool2d(tensor2, 2) blended_low = (1 - blend_value) * downsampled1 + blend_value * downsampled2 blended_high = tensor1 + tensor2 - F.interpolate(blended_low, scale_factor=2) return blended_high def gaussian_blend(tensor2, tensor1, blend_value): tensor1, tensor2 = pad_to_match(tensor1, tensor2) sigma = 0.5 # Adjust for desired smoothness weight = math.exp(-((blend_value - 0.5) ** 2) / (2 * sigma ** 2)) return (1 - weight) * tensor1 + weight * tensor2 def sigmoidal_blend(tensor1, tensor2, blend_value): tensor1, tensor2 = pad_to_match(tensor1, tensor2) # Convert blend_value into a tensor with the same shape as tensor1 and tensor2 blend_tensor = torch.full_like(tensor1, blend_value) weight = 1 / (1 + torch.exp(-10 * (blend_tensor - 0.5))) # Sigmoid function centered at 0.5 return (1 - weight) * tensor1 + weight * tensor2 def blend_tensors(obj1, obj2, blend_value, blend_method="linear"): """ Blends tensors in two given objects based on a blend value using various blending strategies. """ if blend_method == "linear": weight = blend_value obj1[0], obj2[0] = pad_to_match(obj1[0], obj2[0]) obj1[1]['pooled_output'], obj2[1]['pooled_output'] = pad_to_match(obj1[1]['pooled_output'], obj2[1]['pooled_output']) blended_cond = (1 - weight) * obj1[0] + weight * obj2[0] blended_pooled = (1 - weight) * obj1[1]['pooled_output'] + weight * obj2[1]['pooled_output'] elif blend_method == "sigmoidal": blended_cond = sigmoidal_blend(obj1[0], obj2[0], blend_value) blended_pooled = sigmoidal_blend(obj1[1]['pooled_output'], obj2[1]['pooled_output'], blend_value) elif blend_method == "gaussian": blended_cond = gaussian_blend(obj1[0], obj2[0], blend_value) blended_pooled = gaussian_blend(obj1[1]['pooled_output'], obj2[1]['pooled_output'], blend_value) elif blend_method == "pyramid": blended_cond = pyramid_blend(obj1[0], obj2[0], blend_value) blended_pooled = pyramid_blend(obj1[1]['pooled_output'], obj2[1]['pooled_output'], blend_value) return [[blended_cond, {"pooled_output": blended_pooled}]] ================================================ FILE: deforum_nodes/modules/deforum_constants.py ================================================ class DeforumStorage: _instance = None # Class attribute that holds the singleton instance def __new__(cls, *args, **kwargs): if cls._instance is None: cls._instance = super(DeforumStorage, cls).__new__(cls, *args, **kwargs) # Initialize your object here if needed cls._instance.deforum_models = {} cls._instance.deforum_cache = {} cls._instance.deforum_depth_algo = "" cls._instance.reset = None return cls._instance def __init__(self): # Initialization code here. If there are attributes that should not be reinitialized, # you can check if they already exist before setting them. pass ================================================ FILE: deforum_nodes/modules/deforum_node_base.py ================================================ class DeforumDataBase: # @classmethod # def INPUT_TYPES(s): # return s.params RETURN_TYPES = (("deforum_data",)) FUNCTION = "get" OUTPUT_NODE = True CATEGORY = f"deforum/parameters" @classmethod def IS_CHANGED(cls, text, autorefresh): # Force re-evaluation of the node if autorefresh == "Yes": return float("NaN") def get(self, deforum_data=None, *args, **kwargs): if deforum_data: deforum_data.update(**kwargs) else: deforum_data = kwargs return (deforum_data,) ================================================ FILE: deforum_nodes/modules/deforum_ui_data.py ================================================ import comfy deforum_base_params = { "width": { "type": "spinbox", "default": 512, "min": 64, "max": 4096, "step": 64 }, "height": { "type": "spinbox", "default": 512, "min": 64, "max": 4096, "step": 64 }, "seed_schedule": { "type": "lineedit", "default": "0: (-1)", }, "seed_behavior": { "type": "dropdown", "choices": ["fixed", "random"] }, "sampler_name": { "type": "dropdown", "choices": comfy.samplers.KSampler.SAMPLERS }, "scheduler": { "type": "dropdown", "choices": comfy.samplers.KSampler.SCHEDULERS }, # "steps": { # "type": "spinbox", # "default": 25, # "min": 0, # "max": 10000, # "step": 1 # }, # "scale": { # "type": "spinbox", # "default": 7, # "min": 0, # "max": 10000, # "step": 1 # }, # "n_batch": { # "type": "spinbox", # "default": 1, # "min": 0, # "max": 10000, # "step": 1 # }, # "batch_name": { # "type": "lineedit", # "default": "Deforum_{timestring}" # }, # # "seed_iter_N": { # "type": "spinbox", # "default": 1, # "min": 0, # "max": 10000, # "step": 1 # }, # "outdir": { # "type": "lineedit", # "default": "output/deforum" # }, # "strength": { # "type": "doublespinbox", # "default": 0.8, # "min": 0, # "max": 1, # "step": 0.01 # }, # "save_settings": { # "type": "checkbox", # "default": True # }, # "save_sample_per_step": { # "type": "checkbox", # "default": False # }, "prompt_weighting": { "type": "checkbox", "default": True }, "normalize_prompt_weights": { "type": "checkbox", "default": True }, "log_weighted_subprompts": { "type": "checkbox", "default": False }, # "prompt": { # "type": "lineedit", # "default": "" # }, # "timestring": { # "type": "lineedit", # "default": "" # }, # "seed_internal": { # "type": "lineedit", # "default": "-1", # }, } deforum_anim_params = { "animation_mode": { "type": "dropdown", "choices": ["None", "2D", "3D", "Video Input", "Interpolation"] }, "max_frames": { "type": "spinbox", "min": 1, "max": 100000, "default": 120, "step": 1 }, "border": { "type": "dropdown", "choices": ["wrap", "replicate", "zeros"] }, # "resume_from_timestring": { # "type": "checkbox", # "default": False # }, # "resume_timestring": { # "type": "lineedit", # "default": "20230129210106" # }, # "use_looper": { # "type": "checkbox", # "default": False # }, } deforum_translation_params = { "angle": { "type": "lineedit", "default": "0:(0)" }, "zoom": { "type": "lineedit", "default": "0:(1.0025+0.002*sin(1.25*3.14*t/30))" }, "translation_x": { "type": "lineedit", "default": "0:(0)" }, "translation_y": { "type": "lineedit", "default": "0:(0)" }, "translation_z": { "type": "lineedit", "default": "0:(1.75)" }, "transform_center_x": { "type": "lineedit", "default": "0:(0.5)" }, "transform_center_y": { "type": "lineedit", "default": "0:(0.5)" }, "rotation_3d_x": { "type": "lineedit", "default": "0:(0)" }, "rotation_3d_y": { "type": "lineedit", "default": "0:(0)" }, "rotation_3d_z": { "type": "lineedit", "default": "0:(0)" }, } deforum_hybrid_video_params = { # "hybrid_generate_inputframes": { # "type": "checkbox", # "default": False # }, # "hybrid_generate_human_masks": { # "type": "dropdown", # "choices": ["None", "PNGs", "Video", "Both"], # "default": "None" # }, "hybrid_use_first_frame_as_init_image": { "type": "checkbox", "default": True }, "hybrid_motion": { "type": "dropdown", "choices": ["None", "Optical Flow", "Perspective", "Affine"], "default": "None" }, "hybrid_motion_use_prev_img": { "type": "checkbox", "default": False }, "hybrid_flow_consistency": { "type": "checkbox", "default": False }, "hybrid_consistency_blur": { "type": "spinbox", "min": 0, "max": 10, "default": 2, "step": 1 }, "hybrid_flow_method": { "type": "dropdown", "choices": ["RAFT", "DIS Medium", "DIS Fine", "Farneback"], "default": "RAFT" }, "hybrid_composite": { "type": "dropdown", "choices": ["None", "Normal", "Before Motion", "After Generation"], "default": "None" }, # "hybrid_use_init_image": { # "type": "checkbox", # "default": False # }, # "hybrid_comp_mask_type": { # "type": "dropdown", # "choices": ["None", "Depth", "Video Depth", "Blend", "Difference"], # "default": "None" # }, # "hybrid_comp_mask_inverse": { # "type": "checkbox", # "default": False # }, # "hybrid_comp_mask_equalize": { # "type": "dropdown", # "choices": ["None", "Before", "After", "Both"], # "default": "None" # }, # "hybrid_comp_mask_auto_contrast": { # "type": "checkbox", # "default": False # }, # "hybrid_comp_save_extra_frames": { # "type": "checkbox", # "default": False # }, } deforum_hybrid_video_schedules = { # "hybrid_comp_alpha_schedule": { # "type": "lineedit", # "default": "0:(0.5)" # }, # "hybrid_comp_mask_blend_alpha_schedule": { # "type": "lineedit", # "default": "0:(0.5)" # }, # "hybrid_comp_mask_contrast_schedule": { # "type": "lineedit", # "default": "0:(1)" # }, # "hybrid_comp_mask_auto_contrast_cutoff_high_schedule": { # "type": "lineedit", # "default": "0:(100)" # }, # "hybrid_comp_mask_auto_contrast_cutoff_low_schedule": { # "type": "lineedit", # "default": "0:(0)" # }, "hybrid_flow_factor_schedule": { "type": "lineedit", "default": "0:(1)" }, } deforum_color_coherence_params = { "color_coherence": { "type": "dropdown", "choices": ["None", "HSV", "LAB", "RGB", "Video Input", "Image"] }, "color_coherence_image_path": { "type": "lineedit", "default": "" }, "color_coherence_video_every_N_frames": { "type": "spinbox", "min": 1, "max": 100, "default": 1, "step": 1 }, "color_force_grayscale": { "type": "checkbox", "default": False }, "legacy_colormatch": { "type": "checkbox", "default": False }, } deforum_depth_params = { "use_depth_warping": { "type": "checkbox", "default": True }, "depth_algorithm": { "type": "dropdown", "choices": [ "Midas-3-Hybrid", "Midas+AdaBins (old)", "Midas-3.1-BeitLarge", "AdaBins", "Leres", "Zoe", "Zoe+AdaBins (old)", ], "default": "Zoe" }, "midas_weight": { "type": "doublespinbox", "min": 0, "max": 1, "default": 0.2, "step": 0.01 }, "padding_mode": { "type": "dropdown", "choices": ["border", "reflection", "zeros"], "default": "border" }, "sampling_mode": { "type": "dropdown", "choices": ["bicubic", "bilinear", "nearest"], "default": "bicubic" }, "save_depth_maps": { "type": "checkbox", "default": False }, } deforum_cadence_params = { "diffusion_cadence": { "type": "spinbox", "min": 0, "max": 100, "default": 4, "step": 1 }, "optical_flow_cadence": { "type": "dropdown", "choices": ["None", "RAFT", "DIS Medium", "DIS Fine", "Farneback"] }, "cadence_flow_factor_schedule": { "type": "lineedit", "default": "0: (1)" }, "optical_flow_redo_generation": { "type": "dropdown", "choices": ["None", "RAFT", "DIS Medium", "DIS Fine", "Farneback"] }, "redo_flow_factor_schedule": { "type": "lineedit", "default": "0: (1)" }, "diffusion_redo": { "type": "lineedit", "default": "0" }, } deforum_video_init_params = { "video_init_path": { "type": "lineedit", "default": "https://deforum.github.io/a1/V1.mp4" }, "extract_nth_frame": { "type": "spinbox", "min": 1, "max": 100, "default": 1, "step": 1 }, "extract_from_frame": { "type": "spinbox", "min": 0, "max": 1000, "default": 0, "step": 1 }, "extract_to_frame": { "type": "spinbox", "min": -1, "max": 1000, "default": -1, "step": 1 }, "overwrite_extracted_frames": { "type": "checkbox", "default": True }, "use_mask_video": { "type": "checkbox", "default": False }, "video_mask_path": { "type": "lineedit", "default": "https://deforum.github.io/a1/VM1.mp4" }, } deforum_masking_params = { "use_mask": { "type": "checkbox", "default": False }, "use_alpha_as_mask": { "type": "checkbox", "default": False }, "mask_file": { "type": "lineedit", "default": "https://deforum.github.io/a1/M1.jpg" }, "mask_image": { "type": "lineedit", "default": None }, "noise_mask": { "type": "lineedit", "default": None }, "invert_mask": { "type": "checkbox", "default": False }, "mask_contrast_adjust": { "type": "doublespinbox", "default": 1.0, "min": 0, "max": 10, "step": 0.1 }, "mask_brightness_adjust": { "type": "doublespinbox", "default": 1.0, "min": 0, "max": 10, "step": 0.1 }, "overlay_mask": { "type": "checkbox", "default": True }, "mask_overlay_blur": { "type": "spinbox", "default": 4, "min": 0, "max": 100, "step": 1 }, "fill": { "type": "spinbox", "default": 1, "min": 0, "max": 10000, "step": 1 }, "full_res_mask": { "type": "checkbox", "default": True }, "full_res_mask_padding": { "type": "spinbox", "default": 4, "min": 0, "max": 10000, "step": 1 }, } deforum_diffusion_schedule_params = { "noise_schedule": { "type": "lineedit", "default": "0: (0.065)" }, "strength_schedule": { "type": "lineedit", "default": "0: (0.65)" }, "contrast_schedule": { "type": "lineedit", "default": "0: (1.0)" }, "cfg_scale_schedule": { "type": "lineedit", "default": "0: (7)" }, "enable_steps_scheduling": { "type": "checkbox", "default": False }, "steps_schedule": { "type": "lineedit", "default": "0: (25)" }, "enable_ddim_eta_scheduling": { "type": "checkbox", "default": False }, "ddim_eta_schedule": { "type": "lineedit", "default": "0:(0)" }, "enable_ancestral_eta_scheduling": { "type": "checkbox", "default": False }, "ancestral_eta_schedule": { "type": "lineedit", "default": "0:(1)" } } deforum_noise_params = { "enable_noise_multiplier_scheduling": { "type": "checkbox", "default": True }, "noise_multiplier_schedule": { "type": "lineedit", "default": "0: (1.05)" }, "amount_schedule": { "type": "lineedit", "default": "0: (0.1)" }, "kernel_schedule": { "type": "lineedit", "default": "0: (5)" }, "sigma_schedule": { "type": "lineedit", "default": "0: (1.0)" }, "threshold_schedule": { "type": "lineedit", "default": "0: (0.0)" }, "noise_type": { "type": "dropdown", "choices": ["uniform", "perlin"] }, "perlin_w": { "type": "spinbox", "min": 1, "max": 100, "default": 8, "step": 1 }, "perlin_h": { "type": "spinbox", "min": 1, "max": 100, "default": 8, "step": 1 }, "perlin_octaves": { "type": "spinbox", "min": 1, "max": 10, "default": 4, "step": 1 }, "perlin_persistence": { "type": "doublespinbox", "min": 0.1, "max": 1.0, "default": 0.5, "step": 0.1 }, } deforum_args_layout = { "enable_perspective_flip": { "type": "checkbox", "default": False }, "perspective_flip_theta": { "type": "lineedit", "default": "0:(0)" }, "perspective_flip_phi": { "type": "lineedit", "default": "0:(0)" }, "perspective_flip_gamma": { "type": "lineedit", "default": "0:(0)" }, "perspective_flip_fv": { "type": "lineedit", "default": "0:(53)" }, "fov_schedule": { "type": "lineedit", "default": "0: (70)" }, "aspect_ratio_schedule": { "type": "lineedit", "default": "0: (1)" }, "aspect_ratio_use_old_formula": { "type": "checkbox", "default": False }, "near_schedule": { "type": "lineedit", "default": "0: (200)" }, "far_schedule": { "type": "lineedit", "default": "0: (10000)" }, "seed_schedule": { "type": "lineedit", "default": "0:(s), 1:(-1), \"max_f-2\":(-1), \"max_f-1\":(s)" }, "pix2pix_img_cfg_scale": { "type": "doublespinbox", "min": 0.0, "max": 10.0, "default": 1.5, "step": 0.1 }, "pix2pix_img_cfg_scale_schedule": { "type": "lineedit", "default": "0:(1.5)" }, "enable_subseed_scheduling": { "type": "checkbox", "default": False }, "subseed_schedule": { "type": "lineedit", "default": "0:(1)" }, "subseed_strength_schedule": { "type": "lineedit", "default": "0:(0)" }, "enable_sampler_scheduling": { "type": "checkbox", "default": False }, "sampler_schedule": { "type": "lineedit", "default": "0: (\"Euler a\")" }, "use_noise_mask": { "type": "checkbox", "default": False }, "mask_schedule": { "type": "lineedit", "default": "0: (\"{video_mask}\")" }, "noise_mask_schedule": { "type": "lineedit", "default": "0: (\"{video_mask}\")" }, "enable_checkpoint_scheduling": { "type": "checkbox", "default": False }, "checkpoint_schedule": { "type": "lineedit", "default": "0: (\"model1.ckpt\"), 100: (\"model2.safetensors\")" }, "enable_clipskip_scheduling": { "type": "checkbox", "default": False }, "clipskip_schedule": { "type": "lineedit", "default": "0: (2)" }, } deforum_image_init_params = { "strength_0_no_init": { "type": "checkbox", "default": True }, "init_image": { "type": "lineedit", "default": "https://deforum.github.io/a1/I1.png" }, "init_sample": { "type": "lineedit", "default": None }, "use_init": { "type": "checkbox", "default": False }, } ================================================ FILE: deforum_nodes/modules/interp.py ================================================ import os import random import cv2 import numpy as np # def optical_flow_cadence(i1, i2, cadence, method="DIS Medium"): # imgs = [] # if i1 is not None and i2 is not None: # imgs.append(i1) # flow = get_flow_from_images(i1, i2, method) # i2 = image_transform_optical_flow(i2, -flow) # for i in range(1, cadence): # weight = i / cadence # flow_inc = flow * weight # img = cv2.addWeighted(i1, 1 - weight, i2, weight, 0) # img = image_transform_optical_flow(img, flow_inc, cv2.BORDER_REPLICATE) # imgs.append(img) # imgs.append(i2) # return imgs def optical_flow_cadence(i1, i2, cadence, method="DIS Medium"): imgs = [] if i1 is not None and i2 is not None: flow = get_flow_from_images(i1, i2, method) # Warp i2 using the negative of the calculated flow to align it with i1 i2_warped = image_transform_optical_flow(i2, -flow) for i in range(cadence): weight = i / (cadence - 1) # Adjust weight calculation to include both endpoints # Directly interpolate between i1 and warped i2 for each frame img = cv2.addWeighted(i1, 1 - weight, i2_warped, weight, 0) flow_inc = flow * weight img = image_transform_optical_flow(img, flow_inc, cv2.BORDER_REPLICATE) imgs.append(img) # Ensure the last frame is exactly i2, to avoid any discrepancies #imgs[-1] = i2 return imgs def get_matrix_for_hybrid_motion(frame_idx, dimensions, inputfiles, hybrid_motion): img1 = cv2.cvtColor(get_resized_image_from_filename(str(inputfiles[frame_idx - 1]), dimensions), cv2.COLOR_BGR2GRAY) img2 = cv2.cvtColor(get_resized_image_from_filename(str(inputfiles[frame_idx]), dimensions), cv2.COLOR_BGR2GRAY) matrix = get_transformation_matrix_from_images(img1, img2, hybrid_motion) print(f"[deforum] Calculating {hybrid_motion} RANSAC matrix for frames {frame_idx} to {frame_idx + 1}") return matrix def get_matrix_for_hybrid_motion_prev(frame_idx, dimensions, inputfiles, prev_img, hybrid_motion): # first handle invalid images from cadence by returning default matrix height, width = prev_img.shape[:2] if height == 0 or width == 0 or prev_img != np.uint8: return get_hybrid_motion_default_matrix(hybrid_motion) else: prev_img_gray = cv2.cvtColor(prev_img, cv2.COLOR_BGR2GRAY) img = cv2.cvtColor(get_resized_image_from_filename(str(inputfiles[frame_idx]), dimensions), cv2.COLOR_BGR2GRAY) matrix = get_transformation_matrix_from_images(prev_img_gray, img, hybrid_motion) print(f"[deforum] Calculating {hybrid_motion} RANSAC matrix for frames {frame_idx} to {frame_idx + 1}") return matrix def get_flow_for_hybrid_motion(frame_idx, dimensions, inputfiles, hybrid_frame_path, method, do_flow_visualization=False): print(f"[deforum] Calculating {method} optical flow for frames {frame_idx} to {frame_idx + 1}") i1 = get_resized_image_from_filename(str(inputfiles[frame_idx]), dimensions) i2 = get_resized_image_from_filename(str(inputfiles[frame_idx + 1]), dimensions) flow = get_flow_from_images(i1, i2, method) if do_flow_visualization: save_flow_visualization(frame_idx, dimensions, flow, inputfiles, hybrid_frame_path) return flow def get_flow_for_hybrid_motion_prev(frame_idx, dimensions, inputfiles, hybrid_frame_path, prev_img, method, do_flow_visualization=False): print(f"[deforum] Calculating {method} optical flow for frames {frame_idx} to {frame_idx + 1}") # first handle invalid images from cadence by returning default matrix height, width = prev_img.shape[:2] if height == 0 or width == 0: flow = get_hybrid_motion_default_flow(dimensions) else: i1 = prev_img.astype(np.uint8) i2 = get_resized_image_from_filename(str(inputfiles[frame_idx]), dimensions) flow = get_flow_from_images(i1, i2, method) if do_flow_visualization: save_flow_visualization(frame_idx, dimensions, flow, inputfiles, hybrid_frame_path) return flow def get_flow_for_hybrid_motion_prev_imgs(frame_idx, dimensions, current_img, prev_img, method, do_flow_visualization=False): print(f"[deforum] Calculating {method} optical flow for frames {frame_idx} to {frame_idx + 1}") # first handle invalid images from cadence by returning default matrix height, width = prev_img.shape[:2] if height == 0 or width == 0: flow = get_hybrid_motion_default_flow(dimensions) else: i1 = prev_img.astype(np.uint8) i2 = current_img.astype(np.uint8) flow = get_flow_from_images(i1, i2, method) # if do_flow_visualization: # save_flow_visualization(frame_idx, dimensions, flow, inputfiles, hybrid_frame_path) return flow def image_transform_ransac(image_cv2, xform, hybrid_motion, border_mode=cv2.BORDER_REPLICATE): if hybrid_motion == "Perspective": return image_transform_perspective(image_cv2, xform, border_mode=border_mode) else: # Affine return image_transform_affine(image_cv2, xform, border_mode=border_mode) def image_transform_optical_flow(img, flow, border_mode=cv2.BORDER_REPLICATE, flow_reverse=False): if not flow_reverse: flow = -flow h, w = img.shape[:2] flow[:, :, 0] += np.arange(w) flow[:, :, 1] += np.arange(h)[:, np.newaxis] return remap(img, flow, border_mode) def image_transform_affine(image_cv2, xform, border_mode=cv2.BORDER_REPLICATE): return cv2.warpAffine( image_cv2, xform, (image_cv2.shape[1], image_cv2.shape[0]), borderMode=border_mode ) def image_transform_perspective(image_cv2, xform, border_mode=cv2.BORDER_REPLICATE): return cv2.warpPerspective( image_cv2, xform, (image_cv2.shape[1], image_cv2.shape[0]), borderMode=border_mode ) def get_hybrid_motion_default_matrix(hybrid_motion): if hybrid_motion == "Perspective": arr = np.array([[1., 0., 0.], [0., 1., 0.], [0., 0., 1.]]) else: arr = np.array([[1., 0., 0.], [0., 1., 0.]]) return arr def get_hybrid_motion_default_flow(dimensions): cols, rows = dimensions flow = np.zeros((rows, cols, 2), np.float32) return flow def get_transformation_matrix_from_images(img1, img2, hybrid_motion, max_corners=200, quality_level=0.01, min_distance=30, block_size=3): # Detect feature points in previous frame prev_pts = cv2.goodFeaturesToTrack(img1, maxCorners=max_corners, qualityLevel=quality_level, minDistance=min_distance, blockSize=block_size) if prev_pts is None or len(prev_pts) < 8 or img1 is None or img2 is None: return get_hybrid_motion_default_matrix(hybrid_motion) # Get optical flow curr_pts, status, err = cv2.calcOpticalFlowPyrLK(img1, img2, prev_pts, None) # Filter only valid points idx = np.where(status == 1)[0] prev_pts = prev_pts[idx] curr_pts = curr_pts[idx] if len(prev_pts) < 8 or len(curr_pts) < 8: return get_hybrid_motion_default_matrix(hybrid_motion) if hybrid_motion == "Perspective": # Perspective - Find the transformation between points transformation_matrix, mask = cv2.findHomography(prev_pts, curr_pts, cv2.RANSAC, 5.0) return transformation_matrix else: # Affine - Compute a rigid transformation (without depth, only scale + rotation + translation) transformation_rigid_matrix, rigid_mask = cv2.estimateAffinePartial2D(prev_pts, curr_pts) return transformation_rigid_matrix def get_flow_from_images(i1, i2, method): if method == "DIS Medium": r = get_flow_from_images_DIS(i1, i2, cv2.DISOPTICAL_FLOW_PRESET_MEDIUM) elif method == "DIS Fast": r = get_flow_from_images_DIS(i1, i2, cv2.DISOPTICAL_FLOW_PRESET_FAST) elif method == "DIS UltraFast": r = get_flow_from_images_DIS(i1, i2, cv2.DISOPTICAL_FLOW_PRESET_ULTRAFAST) elif method == "DenseRLOF": # requires running opencv-contrib-python (full opencv) INSTEAD of opencv-python r = get_flow_from_images_Dense_RLOF(i1, i2) elif method == "SF": # requires running opencv-contrib-python (full opencv) INSTEAD of opencv-python r = get_flow_from_images_SF(i1, i2) elif method == "Farneback Fine": r = get_flow_from_images_Farneback(i1, i2, 'fine') else: # Farneback Normal: r = get_flow_from_images_Farneback(i1, i2) return r def get_flow_from_images_DIS(i1, i2, preset): i1 = cv2.cvtColor(i1, cv2.COLOR_BGR2GRAY) i2 = cv2.cvtColor(i2, cv2.COLOR_BGR2GRAY) dis = cv2.DISOpticalFlow_create(preset) return dis.calc(i1, i2, None) def get_flow_from_images_Dense_RLOF(i1, i2, last_flow=None): return cv2.optflow.calcOpticalFlowDenseRLOF(i1, i2, flow=last_flow) def get_flow_from_images_SF(i1, i2, last_flow=None, layers=3, averaging_block_size=2, max_flow=4): return cv2.optflow.calcOpticalFlowSF(i1, i2, layers, averaging_block_size, max_flow) def get_flow_from_images_Farneback(i1, i2, preset="normal", last_flow=None, pyr_scale=0.5, levels=3, winsize=15, iterations=3, poly_n=5, poly_sigma=1.2, flags=0): flags = cv2.OPTFLOW_FARNEBACK_GAUSSIAN # Specify the operation flags pyr_scale = 0.5 # The image scale (<1) to build pyramids for each image if preset == "fine": levels = 13 # The number of pyramid layers, including the initial image winsize = 77 # The averaging window size iterations = 13 # The number of iterations at each pyramid level poly_n = 15 # The size of the pixel neighborhood used to find polynomial expansion in each pixel poly_sigma = 0.8 # The standard deviation of the Gaussian used to smooth derivatives used as a basis for the polynomial expansion else: # "normal" levels = 5 # The number of pyramid layers, including the initial image winsize = 21 # The averaging window size iterations = 5 # The number of iterations at each pyramid level poly_n = 7 # The size of the pixel neighborhood used to find polynomial expansion in each pixel poly_sigma = 1.2 # The standard deviation of the Gaussian used to smooth derivatives used as a basis for the polynomial expansion i1 = cv2.cvtColor(i1, cv2.COLOR_BGR2GRAY) i2 = cv2.cvtColor(i2, cv2.COLOR_BGR2GRAY) flags = 0 # flags = cv2.OPTFLOW_USE_INITIAL_FLOW flow = cv2.calcOpticalFlowFarneback(i1, i2, last_flow, pyr_scale, levels, winsize, iterations, poly_n, poly_sigma, flags) return flow def save_flow_visualization(frame_idx, dimensions, flow, inputfiles, hybrid_frame_path): flow_img_file = os.path.join(hybrid_frame_path, f"flow{frame_idx:05}.jpg") flow_img = cv2.imread(str(inputfiles[frame_idx])) flow_img = cv2.resize(flow_img, (dimensions[0], dimensions[1]), cv2.INTER_AREA) flow_img = cv2.cvtColor(flow_img, cv2.COLOR_RGB2GRAY) flow_img = cv2.cvtColor(flow_img, cv2.COLOR_GRAY2BGR) flow_img = draw_flow_lines_in_grid_in_color(flow_img, flow) flow_img = cv2.cvtColor(flow_img, cv2.COLOR_BGR2RGB) cv2.imwrite(flow_img_file, flow_img) print(f"[deforum] Saved optical flow visualization: {flow_img_file}") def draw_flow_lines_in_grid_in_color(img, flow, step=8, magnitude_multiplier=1, min_magnitude=1, max_magnitude=10000): flow = flow * magnitude_multiplier h, w = img.shape[:2] y, x = np.mgrid[step / 2:h:step, step / 2:w:step].reshape(2, -1).astype(int) fx, fy = flow[y, x].T lines = np.vstack([x, y, x + fx, y + fy]).T.reshape(-1, 2, 2) lines = np.int32(lines + 0.5) vis = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) vis = cv2.cvtColor(vis, cv2.COLOR_GRAY2BGR) mag, ang = cv2.cartToPolar(flow[..., 0], flow[..., 1]) hsv = np.zeros((flow.shape[0], flow.shape[1], 3), dtype=np.uint8) hsv[..., 0] = ang * 180 / np.pi / 2 hsv[..., 1] = 255 hsv[..., 2] = cv2.normalize(mag, None, 0, 255, cv2.NORM_MINMAX) bgr = cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR) vis = cv2.add(vis, bgr) # Iterate through the lines for (x1, y1), (x2, y2) in lines: # Calculate the magnitude of the line magnitude = np.sqrt((x2 - x1) ** 2 + (y2 - y1) ** 2) # Only draw the line if it falls within the magnitude range if min_magnitude <= magnitude <= max_magnitude: b = int(bgr[y1, x1, 0]) g = int(bgr[y1, x1, 1]) r = int(bgr[y1, x1, 2]) color = (b, g, r) cv2.arrowedLine(vis, (x1, y1), (x2, y2), color, thickness=1, tipLength=0.1) return vis def draw_flow_lines_in_color(img, flow, threshold=3, magnitude_multiplier=1, min_magnitude=0, max_magnitude=10000): # h, w = img.shape[:2] vis = img.copy() # Create a copy of the input image # Find the locations in the flow field where the magnitude of the flow is greater than the threshold mag, ang = cv2.cartToPolar(flow[..., 0], flow[..., 1]) idx = np.where(mag > threshold) # Create HSV image hsv = np.zeros((flow.shape[0], flow.shape[1], 3), dtype=np.uint8) hsv[..., 0] = ang * 180 / np.pi / 2 hsv[..., 1] = 255 hsv[..., 2] = cv2.normalize(mag, None, 0, 255, cv2.NORM_MINMAX) # Convert HSV image to BGR bgr = cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR) # Add color from bgr vis = cv2.add(vis, bgr) # Draw an arrow at each of these locations to indicate the direction of the flow for i, (y, x) in enumerate(zip(idx[0], idx[1])): # Calculate the magnitude of the line x2 = x + magnitude_multiplier * int(flow[y, x, 0]) y2 = y + magnitude_multiplier * int(flow[y, x, 1]) magnitude = np.sqrt((x2 - x) ** 2 + (y2 - y) ** 2) # Only draw the line if it falls within the magnitude range if min_magnitude <= magnitude <= max_magnitude: if i % random.randint(100, 200) == 0: b = int(bgr[y, x, 0]) g = int(bgr[y, x, 1]) r = int(bgr[y, x, 2]) color = (b, g, r) cv2.arrowedLine(vis, (x, y), (x2, y2), color, thickness=1, tipLength=0.25) return vis def autocontrast_grayscale(image, low_cutoff=0, high_cutoff=100): # Perform autocontrast on a grayscale np array image. # Find the minimum and maximum values in the image min_val = np.percentile(image, low_cutoff) max_val = np.percentile(image, high_cutoff) # Scale the image so that the minimum value is 0 and the maximum value is 255 image = 255 * (image - min_val) / (max_val - min_val) # Clip values that fall outside the range [0, 255] image = np.clip(image, 0, 255) return image def get_resized_image_from_filename(im, dimensions): img = cv2.imread(im) return cv2.resize(img, (dimensions[0], dimensions[1]), cv2.INTER_AREA) def remap(img, flow, border_mode=cv2.BORDER_REFLECT_101): # copyMakeBorder doesn't support wrap, but supports replicate. Replaces wrap with reflect101. if border_mode == cv2.BORDER_WRAP: border_mode = cv2.BORDER_REFLECT_101 h, w = img.shape[:2] displacement = int(h * 0.25), int(w * 0.25) larger_img = cv2.copyMakeBorder(img, displacement[0], displacement[0], displacement[1], displacement[1], border_mode) lh, lw = larger_img.shape[:2] larger_flow = extend_flow(flow, lw, lh) remapped_img = cv2.remap(larger_img, larger_flow, None, cv2.INTER_LINEAR, border_mode) output_img = center_crop_image(remapped_img, w, h) return output_img def center_crop_image(img, w, h): y, x, _ = img.shape width_indent = int((x - w) / 2) height_indent = int((y - h) / 2) cropped_img = img[height_indent:y - height_indent, width_indent:x - width_indent] return cropped_img def extend_flow(flow, w, h): # Get the shape of the original flow image flow_h, flow_w = flow.shape[:2] # Calculate the position of the image in the new image x_offset = int((w - flow_w) / 2) y_offset = int((h - flow_h) / 2) # Generate the X and Y grids x_grid, y_grid = np.meshgrid(np.arange(w), np.arange(h)) # Create the new flow image and set it to the X and Y grids new_flow = np.dstack((x_grid, y_grid)).astype(np.float32) # Shift the values of the original flow by the size of the border flow[:, :, 0] += x_offset flow[:, :, 1] += y_offset # Overwrite the middle of the grid with the original flow new_flow[y_offset:y_offset + flow_h, x_offset:x_offset + flow_w, :] = flow # Return the extended image return new_flow ================================================ FILE: deforum_nodes/modules/standalone_cadence.py ================================================ import os import cv2 import numpy as np import torch from deforum.generators.deforum_flow_generator import rel_flow_to_abs_flow, abs_flow_to_rel_flow, get_flow_from_images, \ get_flow_for_hybrid_motion_prev_imgs from deforum.utils.deforum_framewarp_utils import anim_frame_warp from deforum.utils.deforum_hybrid_animation import get_matrix_for_hybrid_motion_prev_imgs from deforum.utils.image_utils import image_transform_optical_flow, image_transform_ransac import cv2 import numpy as np import PIL.Image class CadenceInterpolator: def __init__(self): self.turbo_prev_image, self.turbo_prev_frame_idx = None, 0 self.turbo_next_image, self.turbo_next_frame_idx = None, 0 self.start_frame = 0 self.depth = None self.prev_img = None self.prev_flow = None def new_standalone_cadence(self, args, anim_args, root, keys, frame_idx, depth_model, raft_model, depth_strength=1.0, logger=None, hybrid_provider=None): # global self.turbo_prev_image, self.turbo_next_image, self.turbo_prev_frame_idx, self.turbo_next_frame_idx, self.start_frame, self.depth if not hasattr(self, 'prev_flow'): self.prev_flow = None self.prev_img = self.turbo_prev_image goal_img = self.turbo_next_image turbo_steps = int(anim_args.diffusion_cadence) cadence_flow_factor = keys.cadence_flow_factor_schedule_series[frame_idx] hybrid_flow_factor = keys.hybrid_flow_factor_schedule_series[frame_idx] hybrid_comp_schedules = keys.hybrid_comp_alpha_schedule_series[frame_idx] imgs = [] x = 0 # emit in-between frames if turbo_steps > 1: tween_frame_start_idx = max(self.start_frame, frame_idx - turbo_steps) cadence_flow = None for tween_frame_idx in range(tween_frame_start_idx, frame_idx): # update progress during cadence # state.job = f"frame {tween_frame_idx + 1}/{anim_args.max_frames}" # state.job_no = tween_frame_idx + 1 # cadence vars tween = float(tween_frame_idx - tween_frame_start_idx + 1) / float(frame_idx - tween_frame_start_idx) advance_prev = self.turbo_prev_image is not None and tween_frame_idx > self.turbo_prev_frame_idx advance_next = tween_frame_idx > self.turbo_next_frame_idx # optical flow cadence setup before animation warping if anim_args.animation_mode in ['2D', '3D'] and anim_args.optical_flow_cadence != 'None': if keys.strength_schedule_series[tween_frame_start_idx] > 0: if cadence_flow is None and self.turbo_prev_image is not None and self.turbo_next_image is not None: cadence_flow = get_flow_from_images(self.turbo_prev_image, self.turbo_next_image, anim_args.optical_flow_cadence, raft_model) / 2 self.turbo_next_image = image_transform_optical_flow(self.turbo_next_image, -cadence_flow, 1) # if opts.data.get("deforum_save_gen_info_as_srt"): # params_to_print = opts.data.get("deforum_save_gen_info_as_srt_params", ['Seed']) # params_string = format_animation_params(keys, prompt_series, tween_frame_idx, params_to_print) # write_frame_subtitle(srt_filename, tween_frame_idx, srt_frame_duration, # f"F#: {tween_frame_idx}; Cadence: {tween < 1.0}; Seed: {args.seed}; {params_string}") # params_string = None # if logger: # logger.update_row("Status", ["Cadence", str(tween_frame_idx), f"tween:{tween:0.2f}"]) # print( # f"[deforum] Creating in-between {'' if cadence_flow is None else anim_args.optical_flow_cadence + ' optical flow '}cadence frame: {tween_frame_idx}; tween:{tween:0.2f};") if depth_model is not None: assert (self.turbo_next_image is not None) self.depth = depth_model.predict(self.turbo_next_image, anim_args.midas_weight, root.half_precision) * depth_strength if advance_prev: self.turbo_prev_image, _, _ = anim_frame_warp(self.turbo_prev_image, args, anim_args, keys, tween_frame_idx, depth_model, depth=self.depth, device=root.device, half_precision=root.half_precision) if advance_next: self.turbo_next_image, _, _ = anim_frame_warp(self.turbo_next_image, args, anim_args, keys, tween_frame_idx, depth_model, depth=self.depth, device=root.device, half_precision=root.half_precision) # hybrid video motion - warps self.turbo_prev_image or self.turbo_next_image to match motion if tween_frame_idx > 0: if anim_args.hybrid_motion in ['Affine', 'Perspective']: if anim_args.hybrid_motion_use_prev_img: matrix = get_matrix_for_hybrid_motion_prev_imgs(tween_frame_idx - 1, (args.width, args.height), self.turbo_next_image, self.turbo_prev_image, anim_args.hybrid_motion) if advance_prev: self.turbo_prev_image = image_transform_ransac(self.turbo_prev_image, matrix, anim_args.hybrid_motion) if advance_next: self.turbo_next_image = image_transform_ransac(self.turbo_next_image, matrix, anim_args.hybrid_motion) else: if hybrid_provider is not None: # matrix = get_matrix_for_hybrid_motion(tween_frame_idx - 1, (args.W, args.H), inputfiles, # anim_args.hybrid_motion) matrix = get_matrix_for_hybrid_motion_prev_imgs(tween_frame_idx - 1, (args.width, args.height), hybrid_provider[x + 1], hybrid_provider[x + 1], anim_args.hybrid_motion) if advance_prev: self.turbo_prev_image = image_transform_ransac(self.turbo_prev_image, matrix, anim_args.hybrid_motion) if advance_next: self.turbo_next_image = image_transform_ransac(self.turbo_next_image, matrix, anim_args.hybrid_motion) if anim_args.hybrid_motion in ['Optical Flow']: if anim_args.hybrid_motion_use_prev_img: flow = get_flow_for_hybrid_motion_prev_imgs(self.turbo_next_image, self.prev_flow, self.turbo_prev_image, anim_args.hybrid_flow_method, raft_model) if advance_prev: self.turbo_prev_image = image_transform_optical_flow(self.turbo_prev_image, flow, #hybrid_comp_schedules['flow_factor']) hybrid_flow_factor) if advance_next: self.turbo_next_image = image_transform_optical_flow(self.turbo_next_image, flow, #hybrid_comp_schedules['flow_factor']) hybrid_flow_factor) self.prev_flow = flow else: if hybrid_provider is not None: flow = get_flow_for_hybrid_motion_prev_imgs(hybrid_provider[x + 1], self.prev_flow, hybrid_provider[x], anim_args.hybrid_flow_method, raft_model) if advance_prev: self.turbo_prev_image = image_transform_optical_flow(self.turbo_prev_image, flow, hybrid_flow_factor) if advance_next: self.turbo_next_image = image_transform_optical_flow(self.turbo_next_image, flow, hybrid_flow_factor) self.prev_flow = flow # do optical flow cadence after animation warping if cadence_flow is not None: cadence_flow = abs_flow_to_rel_flow(cadence_flow, args.width, args.height) cadence_flow, _, _ = anim_frame_warp(cadence_flow, args, anim_args, keys, tween_frame_idx, depth_model, depth=self.depth, device=root.device, half_precision=root.half_precision) cadence_flow_inc = rel_flow_to_abs_flow(cadence_flow, args.width, args.height) * tween if advance_prev: self.turbo_prev_image = image_transform_optical_flow(self.turbo_prev_image, cadence_flow_inc, cadence_flow_factor) if advance_next: self.turbo_next_image = image_transform_optical_flow(self.turbo_next_image, cadence_flow_inc, cadence_flow_factor) self.turbo_prev_frame_idx = self.turbo_next_frame_idx = tween_frame_idx if self.turbo_prev_image is not None and tween < 1.0: img = self.turbo_prev_image * (1.0 - tween) + self.turbo_next_image * tween else: img = self.turbo_next_image # intercept and override to grayscale if anim_args.color_force_grayscale: img = cv2.cvtColor(img.astype(np.uint8), cv2.COLOR_BGR2GRAY) img = cv2.cvtColor(img, cv2.COLOR_GRAY2BGR) # overlay mask # if args.overlay_mask and (anim_args.use_mask_video or args.use_mask): # img = do_overlay_mask(args, anim_args, img, tween_frame_idx, True) if logger: # preview = preview pil_img = PIL.Image.fromarray(img.astype(np.uint8)) logger.update_absolute(turbo_steps - (frame_idx - tween_frame_idx) + 1, preview=("JPEG", pil_img, 512)) imgs.append(img) x += 1 return imgs # # def standalone_cadence(img, prev_img, frame_idx, cadence_frames, args, anim_args, keys, raft_model=None, depth_model=None): # if img is not None: # if cadence_frames > 1: # if isinstance(img, PIL.Image.Image): # img = cv2.cvtColor(np.array(img), cv2.COLOR_RGB2BGR) # if isinstance(prev_img, PIL.Image.Image): # prev_img = cv2.cvtColor(np.array(prev_img), cv2.COLOR_RGB2BGR) # # tween_frame_start_idx = max(0, frame_idx - cadence_frames) # cadence_flow = None # return_frames = [] # for tween_frame_idx in range(tween_frame_start_idx, frame_idx): # tween = float(tween_frame_idx - tween_frame_start_idx + 1) / float(frame_idx - tween_frame_start_idx) # advance_prev = prev_img is not None and tween_frame_idx > 0 # advance_next = tween_frame_idx > 0 # # if anim_args.animation_mode in ['2D', '3D'] and anim_args.optical_flow_cadence != 'None': # if keys.strength_schedule_series[tween_frame_start_idx] > 0: # if cadence_flow is None and prev_img is not None and img is not None: # cadence_flow = get_flow_from_images(prev_img, img, anim_args.optical_flow_cadence, raft_model) / 2 # img = image_transform_optical_flow(img, -cadence_flow, 1) # # if depth_model is not None: # depth = depth_model.predict(img, anim_args.midas_weight, True) # if advance_prev: # prev_img, _, _ = anim_frame_warp(prev_img, args, anim_args, keys, tween_frame_idx, depth_model=depth_model, depth=self.depth, device='cuda', half_precision=True) # if advance_next: # img, _, _ = anim_frame_warp(img, args, anim_args, keys, tween_frame_idx, depth_model=depth_model, depth=self.depth, device='cuda', half_precision=True) # # if cadence_flow is not None: # cadence_flow = abs_flow_to_rel_flow(cadence_flow, args.width, args.height) # cadence_flow, _, _ = anim_frame_warp(cadence_flow, args, anim_args, keys, tween_frame_idx, depth_model=depth_model, depth=self.depth, device='cuda', half_precision=True) # cadence_flow_inc = rel_flow_to_abs_flow(cadence_flow, args.width, args.height) * tween # if advance_prev: # prev_img = image_transform_optical_flow(prev_img, cadence_flow_inc, 1) # if advance_next: # img = image_transform_optical_flow(img, cadence_flow_inc, 1) # # if prev_img is not None and tween < 1.0: # combined_img = prev_img * (1.0 - tween) + img * tween # else: # combined_img = img # # if anim_args.color_force_grayscale: # combined_img = cv2.cvtColor(combined_img.astype(np.uint8), cv2.COLOR_BGR2GRAY) # combined_img = cv2.cvtColor(combined_img, cv2.COLOR_GRAY2BGR) # res = combined_img.astype(np.uint8) # return_frames.append(res) # return return_frames, prev_img, img ================================================ FILE: deforum_nodes/nodes/__init__.py ================================================ ================================================ FILE: deforum_nodes/nodes/deforum_advnoise_node.py ================================================ import secrets import torch import numpy as np import math import pywt import torch.nn.functional as F from opensimplex import OpenSimplex """ Noise types to implement: Dithering Noise: Introduces a monochrome or color dithering effect that simulates more colors in images with limited palettes. This can create a retro, pixelated look or be used to prevent banding in gradients. JPEG Compression Noise: Simulates the artifacts introduced by JPEG compression, such as blockiness and loss of detail, which can be relevant for training models to be robust against compression artifacts. Moire Pattern Noise: Simulates the moiré pattern, an interference pattern that can occur when an image with repetitive details is digitized. This is particularly relevant for document scanning and OCR applications. Defocus Blur Noise: Simulates the effect of an out-of-focus lens, which can be used to generate data for depth estimation or autofocus algorithms. Lens Distortion Noise: Introduces barrel or pincushion distortion to simulate lens imperfections, useful for calibration and correction algorithms. Chromatic Aberration: Simulates the color fringing around high-contrast edges caused by lens dispersion, adding realism to synthetic images or for artistic effects. Color Shift Noise: Randomly shifts the color balance of the image, simulating white balance inaccuracies or artistic color grading. Speckle Noise in Color: While speckle noise is typically applied in a monochromatic context, it can be adapted to color images to simulate sensor noise in color imaging devices. Atmospheric Turbulence Noise: Simulates the effect of atmospheric distortion, relevant for images taken over long distances in outdoor environments, like satellite or surveillance imagery. Halftone Pattern Noise: Simulates the halftone dots used in printed media, which can be useful for preparing models that need to understand scanned printed documents or for creating retro effects. Holographic Interference Noise: Simulates the pattern seen in holograms, which could be interesting for security feature recognition or artistic effects. Color Quantization Noise: While quantization was mentioned, specifically focusing on reducing the color depth of an image can simulate older digital or printed media. Film Grain: Simulates the random texture characteristic of film photography, adding a nostalgic or artistic quality to digital images. Pattern Overlay Noise: Overlaying a fixed pattern, such as a grid or text watermark, to simulate watermarked paper or screen overlays. """ class AddAdvancedNoiseNode: """ A node to add various types of advanced noise to an image using PyTorch. """ @classmethod def IS_CHANGED(cls, text, autorefresh): # Force re-evaluation of the node if autorefresh == "Yes": return float("NaN") @classmethod def INPUT_TYPES(cls): return { "required": { "images": ("IMAGE",), "noise_type": (["wavelet", "value", "flow", "turbulence", "ridged_multifractal", "reaction_diffusion", "voronoi", "simplex"],), # "gabor" "amount": ("FLOAT", { "default": 0.1, "min": 0.0, "max": 100.0, "step": 0.01, "display": "number" }), }, "optional": { "seed": ("INT", { "default": None, "min": 0, "max": 2 ** 32 - 1, "step": 1, "display": "number" }), # Voronoi specific parameters "num_points": ("INT", { "default": 100, "min": 10, "max": 1000, "step": 10, "display": "number" }), # Simplex Noise specific parameters "scale": ("FLOAT", { "default": 0.1, "min": 0.01, "max": 1.0, "step": 0.01, "display": "number" }), "octaves": ("INT", { "default": 1, "min": 1, "max": 10, "step": 1, "display": "number" }), "persistence": ("FLOAT", { "default": 0.5, "min": 0.1, "max": 1.0, "step": 0.01, "display": "number" }), "lacunarity": ("FLOAT", { "default": 2.0, "min": 1.0, "max": 3.0, "step": 0.01, "display": "number" }), # Wavelet Noise specific parameters "wavelet": (["haar", "db1", "sym2", "coif1", "bior1.3", "rbio1.3"],), "mode": (["symmetric", "periodic", "reflect", "zero-padding"],), # Additional parameters specific to Gabor noise "frequency": ("FLOAT", { "default": 0.1, "min": 0.01, "max": 1.0, "step": 0.01, "display": "number" }), "theta": ("FLOAT", { "default": 0.0, "min": 0.0, "max": 2 * math.pi, "step": 0.01, "display": "slider" }), "sigma_x": ("FLOAT", { "default": 10.0, "min": 1.0, "max": 50.0, "step": 1.0, "display": "number" }), "sigma_y": ("FLOAT", { "default": 10.0, "min": 1.0, "max": 50.0, "step": 1.0, "display": "number" }), # Value Noise specific parameters "res": ("INT", { "default": 16, "min": 4, "max": 64, "step": 1, "display": "number" }), # Additional parameters for Flow Noise "flow_scale": ("FLOAT", { "default": 0.1, "min": 0.01, "max": 1.0, "step": 0.01, "display": "number" }), "flow_angle": ("FLOAT", { "default": 0.0, "min": 0.0, "max": 2 * np.pi, "step": 0.01, "display": "number" }), # Reaction Diffusion Noise "steps": ("INT", { "default": 10, "min": 1, "max": 100, "step": 1, "display": "number" }), "Du": ("FLOAT", { "default": 0.16, "min": 0.01, "max": 0.5, "step": 0.01, "display": "number" }), "Dv": ("FLOAT", { "default": 0.08, "min": 0.01, "max": 0.5, "step": 0.01, "display": "number" }), "feed_rate": ("FLOAT", { "default": 0.035, "min": 0.01, "max": 0.1, "step": 0.001, "display": "slider" }), "kill_rate": ("FLOAT", { "default": 0.06, "min": 0.01, "max": 0.1, "step": 0.001, "display": "slider" }), # Additional parameters specific to each noise type can be defined here } } RETURN_TYPES = ("IMAGE","IMAGE",) RETURN_NAMES = ("NOISED_IMAGE","NOISE",) FUNCTION = "add_advanced_noise" CATEGORY = "deforum/noise" display_name = "Add Advanced Noise" def add_advanced_noise(self, images, noise_type, amount, seed=None, **kwargs): B, H, W, C = images.shape if seed is not None: np.random.seed(seed) torch.manual_seed(seed) # Adjusted noise function dictionary noise_function = { "voronoi": generate_voronoi_noise, "simplex": lambda H, W, **kw: generate_simplex_noise(H, W, **kw).unsqueeze(-1).repeat(1, 1, 1, C), "wavelet": lambda H, W, **kw: generate_wavelet_noise(H, W, **kw).unsqueeze(-1).repeat(1, 1, 1, C), "gabor": lambda H, W, **kw: generate_gabor_noise(H, W, **kw), "value": lambda H, W, **kw: generate_value_noise(H, W, **kw).unsqueeze(-1).repeat(1, 1, 1, C), "flow": lambda H, W, **kw: generate_flow_noise(H, W, **kw), "turbulence": lambda H, W, **kw: generate_turbulence_noise(H, W, **kw), "ridged_multifractal": lambda H, W, **kw: generate_ridged_multifractal_noise(H, W, **kw), "reaction_diffusion": lambda H, W, **kw: generate_reaction_diffusion_noise(H, W, **kw), # Additional noise functions adjusted similarly }.get(noise_type, lambda H, W, **kw: torch.zeros((B, H, W, C))) # Generate the noise pattern noise = noise_function(H, W, seed=seed, **kwargs) # Adjust for the case where noise is not returned with a batch dimension or channel dimension if noise.dim() == 2: # Assuming noise is of shape (H, W) noise = noise.unsqueeze(0).unsqueeze(-1) # Add batch and channel dimension noise = noise.expand(B, -1, -1, C) # Expand to match (B, H, W, C) elif noise.dim() == 3: # Assuming noise is of shape (B, H, W) noise = noise.unsqueeze(-1) # Add channel dimension noise = noise.expand(-1, -1, -1, C) # Expand to match (B, H, W, C) # Ensure noise tensor device matches the images tensor device noise = noise.to(images.device) # from ..modules.better_resize.resize_right import resize # belnd_img = resize(images, noise.shape, antialiasing=False) # Apply the noise to the images noisy_images = images.clone() + amount * noise.clone() return (torch.clamp(noisy_images, 0, 1),noise,) def generate_voronoi_noise(H, W, num_points=100, seed=None, *args, **kwargs): """ Generates a 2D Voronoi (Worley) Noise pattern. Args: H, W (int): The height and width of the output noise pattern. num_points (int): Number of seed points to generate. seed (int, optional): Random seed for reproducibility. Returns: torch.Tensor: A tensor containing the Voronoi noise pattern. """ if seed is not None: torch.manual_seed(seed) np.random.seed(seed) # Generate random seed points seeds_x = np.random.uniform(0, W, num_points) seeds_y = np.random.uniform(0, H, num_points) seeds = np.stack((seeds_x, seeds_y), axis=-1) # Initialize the noise tensor noise = torch.full((H, W), float('inf')) # Compute distance from each pixel to the closest seed point for y in range(H): for x in range(W): distances = torch.tensor([((x - sx) ** 2 + (y - sy) ** 2) for sx, sy in seeds]) noise[y, x] = torch.min(distances).sqrt() # Normalize the noise pattern noise = (noise - noise.min()) / (noise.max() - noise.min()) return noise def generate_simplex_noise(H, W, scale=0.05, octaves=4, persistence=0.5, lacunarity=2.0, seed=None, **kwargs): if seed is not None: gen = OpenSimplex(seed) else: gen = OpenSimplex() noise_pattern = np.zeros((H, W), dtype=np.float32) for y in range(H): for x in range(W): nx = x / W - 0.5 # Normalize x coordinate to [-0.5, 0.5] ny = y / H - 0.5 # Normalize y coordinate to [-0.5, 0.5] noise_val = 0 amplitude = 1.0 frequency = 1.0 max_value = 0.0 for o in range(octaves): noise_val += gen.noise2(x=nx * scale * frequency, y=ny * scale * frequency) * amplitude max_value += amplitude amplitude *= persistence frequency *= lacunarity # Normalize each point individually noise_pattern[y, x] = (noise_val / max_value + 1) / 2 # Normalize to [0, 1] return torch.from_numpy(noise_pattern) def generate_wavelet_noise(H, W, octaves=3, wavelet='haar', mode='symmetric', **kwargs): seed = kwargs.get('seed', secrets.randbelow(2 ** 32)) if seed is not None: np.random.seed(seed) # Generate initial random noise noise = np.random.randn(H, W) # Perform wavelet decomposition and reconstruction coeffs = pywt.wavedec2(noise, wavelet=wavelet, mode=mode, level=octaves) wavelet_noise = pywt.waverec2(coeffs, wavelet=wavelet, mode=mode) # Normalize the wavelet noise to have values between 0 and 1 wavelet_noise = (wavelet_noise - np.min(wavelet_noise)) / (np.max(wavelet_noise) - np.min(wavelet_noise)) # Convert to a PyTorch tensor and add a channel dimension since input is assumed to be BxHxWxC wavelet_noise_tensor = torch.from_numpy(wavelet_noise).float().unsqueeze(0).unsqueeze(0) # Interpolate to match input size (H, W) # Note: 'align_corners=False' is generally used for non-exact sizes but you might want to experiment # with 'align_corners=True' or 'recompute_scale_factor=True' depending on your specific use case wavelet_noise_tensor = F.interpolate(wavelet_noise_tensor, size=(H, W), mode='bilinear', align_corners=False) # Remove the extra dimensions added for interpolation if necessary wavelet_noise_tensor = wavelet_noise_tensor.squeeze() return wavelet_noise_tensor def generate_gabor_kernel(frequency, theta, sigma_x, sigma_y, n_stds=3, grid_size=28): """ Generates a Gabor kernel. Args: frequency (float): The frequency of the sinusoidal component. theta (float): Orientation of the Gabor filter in radians. sigma_x, sigma_y (float): Standard deviation of the Gaussian envelope along x and y axes. n_stds (int): Number of standard deviations to consider for the kernel size. grid_size (int): The size of the output tensor. Returns: torch.Tensor: A 2D tensor with the Gabor kernel. """ xmax = ymax = grid_size // 2 xmin = ymin = -xmax (y, x) = torch.meshgrid(torch.arange(ymin, ymax + 1), torch.arange(xmin, xmax + 1)) x = x.float() y = y.float() # Convert theta to a tensor to use with torch.cos and torch.sin theta_tensor = torch.tensor(theta) rotx = x * torch.cos(theta_tensor) + y * torch.sin(theta_tensor) roty = -x * torch.sin(theta_tensor) + y * torch.cos(theta_tensor) g = torch.zeros(y.shape) g = torch.exp(-0.5 * ((rotx ** 2 / sigma_x ** 2) + (roty ** 2 / sigma_y ** 2))) * torch.cos( 2 * np.pi * frequency * rotx) return g def generate_gabor_noise(H, W, frequency=0.1, theta=0.0, sigma_x=10.0, sigma_y=10.0, batch_size=1, channels=3, **kwargs): """ Generates a 2D Gabor Noise pattern and scales it to the size of the input images. Args: H, W (int): Height and width of the output noise pattern. frequency (float): Frequency of the sinusoidal component. theta (float): Orientation of the Gabor filter in radians. sigma_x, sigma_y (float): Standard deviation of the Gaussian envelope. batch_size (int): Number of images in the batch. channels (int): Number of channels in the images. Returns: torch.Tensor: A tensor containing the Gabor noise pattern, resized to match input images dimensions. """ gabor_kernel = generate_gabor_kernel(frequency, theta, sigma_x, sigma_y, grid_size=min(H, W)) # Center the kernel in a larger tensor matching the image dimensions pad_height = (H - gabor_kernel.size(0)) // 2 pad_width = (W - gabor_kernel.size(1)) // 2 pad_height_extra = H - gabor_kernel.size(0) - pad_height * 2 pad_width_extra = W - gabor_kernel.size(1) - pad_width * 2 # Pad the kernel to match the target size gabor_padded = F.pad(gabor_kernel, [pad_width, pad_width + pad_width_extra, pad_height, pad_height_extra], "constant", 0) # Ensure the pattern has the correct dimensions: batch_size x H x W x channels gabor_padded = gabor_padded.unsqueeze(0) # Add a batch dimension gabor_padded = gabor_padded.unsqueeze(-1) # Add a channel dimension gabor_padded = gabor_padded.expand(batch_size, -1, -1, channels) # Expand to match the full dimensions return gabor_padded def generate_value_noise(H, W, res=16, seed=None, **kwargs): """ Generates a 2D Value Noise pattern with edge handling. Args: H, W (int): Height and width of the output noise pattern. res (int): Resolution of the noise grid (smaller values mean smoother noise). seed (int, optional): Seed for random number generator. Returns: torch.Tensor: A tensor containing the Value noise pattern. """ if seed is not None: torch.manual_seed(seed) # Create a grid of random values grid_res_x, grid_res_y = W // res + 1, H // res + 1 random_values = torch.rand((grid_res_y, grid_res_x)) # Initialize output noise pattern noise_pattern = torch.zeros((H, W)) # Generate the noise for y in range(H): for x in range(W): # Grid cell coordinates in the random grid cell_x, cell_y = x // res, y // res # Ensure indices stay within bounds next_cell_x = min(cell_x + 1, grid_res_x - 1) next_cell_y = min(cell_y + 1, grid_res_y - 1) # Local x and y in the grid cell local_x, local_y = (x % res) / res, (y % res) / res # Corners of the cell in the grid c00 = random_values[cell_y, cell_x] c10 = random_values[cell_y, next_cell_x] c01 = random_values[next_cell_y, cell_x] c11 = random_values[next_cell_y, next_cell_x] # Interpolate between grid corner values nx0 = lerp(c00, c10, fade(local_x)) nx1 = lerp(c01, c11, fade(local_x)) nxy = lerp(nx0, nx1, fade(local_y)) noise_pattern[y, x] = nxy # Normalize the noise pattern noise_pattern = (noise_pattern - noise_pattern.min()) / (noise_pattern.max() - noise_pattern.min()) return noise_pattern def lerp(a, b, t): """Linear interpolation between a and b with t in [0, 1].""" return a + t * (b - a) def fade(t): """Fade function as defined by Ken Perlin; eases coordinate values.""" return t * t * t * (t * (t * 6 - 15) + 10) def generate_flow_noise_pattern(H, W, scale=0.1, angle=0.0, seed=None): """ Generates a 2D Flow Noise pattern using a base noise to perturb the sampling space of another noise layer. Args: H, W (int): Height and width of the output noise pattern. scale (float): Scale of the underlying noise. angle (float): Base flow direction in radians. seed (int, optional): Random seed for reproducibility. Returns: torch.Tensor: A tensor containing the Flow noise pattern. """ # Base noise for perturbation base_noise = generate_simplex_noise(H, W, scale=scale, seed=seed) # Generate gradient from angle for flow direction flow_x = torch.cos(torch.tensor(angle)) flow_y = torch.sin(torch.tensor(angle)) # Apply perturbation based on the base noise and flow direction perturb_x = base_noise * flow_x perturb_y = base_noise * flow_y # Another layer of noise for actual flow appearance flow_noise = generate_simplex_noise(H, W, scale=scale / 2, seed=seed if seed is None else seed + 1) # Final flow noise is a combination of perturbed coordinates and the flow noise layer final_noise = flow_noise + perturb_x + perturb_y # Normalize the noise pattern final_noise = (final_noise - final_noise.min()) / (final_noise.max() - final_noise.min()) return final_noise def generate_flow_noise(H, W, flow_scale=0.1, flow_angle=0.0, seed=None, batch_size=1, channels=3, **kwargs): """ Generates a 2D Flow Noise pattern and resizes it to match the input image size. Args: H, W (int): Height and width of the input images. scale, angle: Parameters for the flow noise generation. seed (int, optional): Seed for random number generator. batch_size (int): The batch size of the input images. channels (int): The channel count of the input images. Returns: torch.Tensor: A tensor of the Flow noise pattern, matched to input dimensions. """ # Generate the base noise pattern base_noise = generate_flow_noise_pattern(H, W, flow_scale, flow_angle, seed) # Ensure the noise tensor has the same dimensions as the input: (batch_size, H, W, channels) # Interpolate noise to match the input size if needed noise_resized = F.interpolate(base_noise.unsqueeze(0).unsqueeze(0), size=(H, W), mode='bilinear', align_corners=False) # Expand to match the batch and channel dimensions noise_resized = noise_resized.repeat(batch_size, channels, 1, 1) noise_resized = noise_resized.permute(0, 2, 3, 1) # Rearrange dimensions to match (B, H, W, C) return noise_resized def generate_turbulence_noise(H, W, scale=0.05, octaves=4, persistence=0.5, lacunarity=2.0, seed=None, **kwargs): if seed is not None: gen = OpenSimplex(seed) else: gen = OpenSimplex() noise_pattern = np.zeros((H, W), dtype=np.float32) for y in range(H): for x in range(W): nx = x / W - 0.5 # Normalize x coordinate to [-0.5, 0.5] ny = y / H - 0.5 # Normalize y coordinate to [-0.5, 0.5] noise_val = 0 amplitude = 1.0 frequency = 1.0 max_value = 0.0 for o in range(octaves): # The key change for turbulence: using the absolute value of the noise noise_val += abs(gen.noise2(x=nx * scale * frequency, y=ny * scale * frequency)) * amplitude max_value += amplitude amplitude *= persistence frequency *= lacunarity # Normalize each point individually noise_pattern[y, x] = (noise_val / max_value) # Normalize the whole pattern to [0, 1] range if desired noise_pattern = (noise_pattern - noise_pattern.min()) / (noise_pattern.max() - noise_pattern.min()) return torch.from_numpy(noise_pattern) def generate_ridged_multifractal_noise(H, W, scale=0.05, octaves=4, persistence=0.5, lacunarity=2.0, seed=None, **kwargs): if seed is not None: gen = OpenSimplex(seed) else: gen = OpenSimplex() noise_pattern = np.zeros((H, W), dtype=np.float32) for y in range(H): for x in range(W): nx = x / W - 0.5 # Normalize x coordinate to [-0.5, 0.5] ny = y / H - 0.5 # Normalize y coordinate to [-0.5, 0.5] noise_val = 0 amplitude = 1.0 frequency = 1.0 max_value = 0.0 for o in range(octaves): signal = gen.noise2(x=nx * scale * frequency, y=ny * scale * frequency) # Transformation for ridged noise: invert and square the signal signal = 1.0 - abs(signal) signal *= signal noise_val += signal * amplitude max_value += amplitude amplitude *= persistence frequency *= lacunarity # Normalize each point individually noise_pattern[y, x] = (noise_val / max_value) # Normalize the whole pattern to [0, 1] range if desired noise_pattern = (noise_pattern - noise_pattern.min()) / (noise_pattern.max() - noise_pattern.min()) return torch.from_numpy(noise_pattern) def generate_reaction_diffusion_noise(H, W, steps=100, Du=0.16, Dv=0.08, feed_rate=0.035, kill_rate=0.06, seed=None, **kwargs): """ Generates a 2D Reaction-Diffusion pattern using a simplified Gray-Scott model. Args: H, W (int): Height and width of the output noise pattern. steps (int): Number of simulation steps. Du, Dv (float): Diffusion rates for substances A and B. feed_rate, kill_rate (float): Rates for feed and kill reactions. seed (int, optional): Seed for initializing the pattern. Returns: torch.Tensor: A tensor containing the Reaction-Diffusion noise pattern. """ if seed is not None: torch.manual_seed(seed) np.random.seed(seed) # Initialize A and B concentrations with A being fully saturated and B having a noise pattern A = torch.ones((H, W), dtype=torch.float32) B = torch.rand((H, W), dtype=torch.float32) * 0.25 # Starting with a low concentration of B # Laplacian kernel for diffusion laplacian_kernel = torch.tensor([[0.05, 0.2, 0.05], [0.2, -1.0, 0.2], [0.05, 0.2, 0.05]], dtype=torch.float32) for _ in range(steps): LA = torch.nn.functional.conv2d(A.unsqueeze(0).unsqueeze(0), laplacian_kernel.unsqueeze(0).unsqueeze(0), padding='same').squeeze() LB = torch.nn.functional.conv2d(B.unsqueeze(0).unsqueeze(0), laplacian_kernel.unsqueeze(0).unsqueeze(0), padding='same').squeeze() # Reaction-diffusion equations AB2 = A * B.pow(2) dA = Du * LA - AB2 + feed_rate * (1 - A) dB = Dv * LB + AB2 - (feed_rate + kill_rate) * B A += dA B += dB # Normalize the B concentration pattern to be between 0 and 1 pattern = (B - B.min()) / (B.max() - B.min()) return pattern.unsqueeze(0) # Add a channel dimension ================================================ FILE: deforum_nodes/nodes/deforum_audiosync_nodes.py ================================================ import math import pandas as pd import scipy.signal import scipy.ndimage import numpy as np from pydub import AudioSegment from scipy.signal import butter, filtfilt, find_peaks import librosa schedule_types = [ "angle", "transform_center_x", "transform_center_y", "zoom", "translation_x", "translation_y", "translation_z", "rotation_3d_x", "rotation_3d_y", "rotation_3d_z", "perspective_flip_theta", "perspective_flip_phi", "perspective_flip_gamma", "perspective_flip_fv", "noise_schedule", "strength_schedule", "contrast_schedule", "cfg_scale_schedule", "ddim_eta_schedule", "ancestral_eta_schedule", "pix2pix_img_cfg_scale", "subseed_schedule", "subseed_strength_schedule", "checkpoint_schedule", "steps_schedule", "seed_schedule", "sampler_schedule", "clipskip_schedule", "noise_multiplier_schedule", "mask_schedule", "noise_mask_schedule", "kernel_schedule", "sigma_schedule", "amount_schedule", "threshold_schedule", "aspect_ratio", "fov", "near", "cadence_flow_factor_schedule", "redo_flow_factor_schedule", "far", "hybrid_comp_alpha_schedule", "hybrid_comp_mask_blend_alpha_schedule", "hybrid_comp_mask_contrast_schedule", "hybrid_comp_mask_auto_contrast_cutoff_high_schedule", "hybrid_comp_mask_auto_contrast_cutoff_low_schedule", "hybrid_flow_factor_schedule" ] import numpy as np from scipy.signal import savgol_filter class ExtractDominantNoteAmplitude: @classmethod def INPUT_TYPES(s): return {"required": { "audio_fft": ("AUDIO_FFT",), }, "optional": { "min_frequency": ("FLOAT", {"default": 20.0, "min": 0.0, "max": 20000.0, "step": 1.0}), "max_frequency": ("FLOAT", {"default": 8000.0, "min": 0.0, "max": 20000.0, "step": 1.0}), "magnitude_threshold": ("FLOAT", {"default": 0.01, "min": 0.0, "max": 1.0, "step": 0.01}), "smoothing_window_size": ("INT", {"default": 5, "min": 1, "max": 101, "step": 1}), # Odd number, savgol_filter constraint }, } CATEGORY = "deforum/audio" RETURN_TYPES = ("AMPLITUDE","AMPLITUDE",) RETURN_NAMES = ("dominant_note_amplitude","phase_amplitude",) FUNCTION = "extract" def extract(self, audio_fft, min_frequency, max_frequency, magnitude_threshold, smoothing_window_size): dominant_frequencies_amplitude = [] dominant_frequencies_phase = [] for fft_data in audio_fft: indices = fft_data.get_indices_for_frequency_bands(min_frequency, max_frequency) magnitudes = np.abs(fft_data.fft)[indices] phases = np.angle(fft_data.fft)[indices] # Optional: Apply smoothing to magnitudes if smoothing_window_size > 1 and len(magnitudes) > smoothing_window_size: try: magnitudes = savgol_filter(magnitudes, smoothing_window_size, 3) # window size, polynomial order except ValueError: # In case the smoothing_window_size is inappropriate for the data length pass # Apply magnitude threshold magnitudes[magnitudes < magnitude_threshold] = 0 # Identify dominant frequency index, its amplitude, and phase if magnitudes.size > 0: # Ensure there's data after filtering dominant_index = np.argmax(magnitudes) dominant_frequencies_amplitude.append(magnitudes[dominant_index]) dominant_frequencies_phase.append(phases[dominant_index]) else: dominant_frequencies_amplitude.append(0) dominant_frequencies_phase.append(0) # Use a default phase (e.g., 0) when no dominant frequency is found # Convert to numpy array for consistency return (np.array(dominant_frequencies_amplitude), np.array(dominant_frequencies_phase),) class InverseFFTNode: @classmethod def INPUT_TYPES(cls): return { "required": { "magnitude": ("AMPLITUDE",), "phase": ("AMPLITUDE",), }, } CATEGORY = "signal_processing" RETURN_TYPES = ("AMPLITUDE",) RETURN_NAMES = ("time_domain_signal",) FUNCTION = "synthesize" def synthesize(self, magnitude, phase): # Ensure magnitude and phase arrays have the same length if magnitude.shape != phase.shape: raise ValueError("Magnitude and phase arrays must have the same length.") # Combine magnitude and phase to form the complex spectrum complex_spectrum = magnitude * np.exp(1j * phase) # Perform the inverse FFT to convert the spectrum back to the time domain time_domain_signal = np.fft.ifft(complex_spectrum) # Since the input is real, the output should also be real. We take the real part to avoid numerical errors that might introduce imaginary parts. return (np.real(time_domain_signal),) class AmplitudeToAudio: @classmethod def INPUT_TYPES(cls): return { "required": { "amplitude": ("AMPLITUDE",), "phase": ("AMPLITUDE",), }, "optional": { # You can define additional parameters such as sample rate if needed "sample_rate": ("INT", {"default": 44100, "min": 1, "max": 192000, "step": 1}), }, } CATEGORY = "AudioProcessing" RETURN_TYPES = ("AUDIO",) RETURN_NAMES = ("audio_samples",) FUNCTION = "convert" def convert(self, amplitude, phase, sample_rate=44100): if len(amplitude) != len(phase): raise ValueError("Amplitude and phase sequences must have the same length.") # Reconstruct the complex spectrum from amplitude and phase complex_spectrum = amplitude * np.exp(1j * phase) # Perform the inverse FFT audio_samples = np.fft.ifft(complex_spectrum).real # Taking the real part as the output # Convert the normalized audio samples to 16-bit integers audio_samples_int16 = np.int16(audio_samples / np.max(np.abs(audio_samples)) * 32767) # Creating an AudioSegment object from the NumPy array audio_segment = AudioSegment( audio_samples_int16.tobytes(), frame_rate=sample_rate, sample_width=audio_samples_int16.dtype.itemsize, channels=1 ) # Create the AudioData object audio_data = AudioData(audio_segment) return (audio_data,) class AudioData: def __init__(self, audio_segment) -> None: # Extract the sample rate self.sample_rate = audio_segment.frame_rate # Get the number of audio channels self.num_channels = audio_segment.channels # Extract the audio data as a NumPy array samples = np.array(audio_segment.get_array_of_samples()) if audio_segment.channels == 2: self.audio_data = np.reshape(samples, (len(samples)//2, 2)) else: self.audio_data = samples def get_channel_audio_data(self, channel: int): if channel < 0 or channel >= self.num_channels: raise IndexError(f"Channel '{channel}' out of range. total channels is '{self.num_channels}'.") return self.audio_data[channel::self.num_channels] def get_channel_fft(self, channel: int): audio_data = self.get_channel_audio_data(channel) return np.fft.fft(audio_data) def xor(a, b): return bool(a) ^ bool(b) # class DeforumAmplitudeToKeyframeSeriesNode: # @classmethod # def INPUT_TYPES(s): # return {"required": # {"type_name": (schedule_types,), # "amplitude": ("AMPLITUDE",), # }, # "optional": # { # "max_frames": ("INT", {"default": 500, "min": 1, "max": 16500, "step": 1}), # "math": ("STRING", {"default": "x/100"}), # "filter_window": ("INT", {"default": 0, "min": 0, "max": 100, "step": 1}), # "deforum_frame_data": ("DEFORUM_FRAME_DATA",) # } # } # # RETURN_TYPES = ("DEFORUM_FRAME_DATA", "AMPLITUDE", "STRING") # FUNCTION = "convert" # display_name = "Amplitude to Schedule" # CATEGORY = "deforum/audio" # # def safe_eval(self, expr, t, x, max_f): # allowed_locals = { # "sin": math.sin, "cos": math.cos, "tan": math.tan, # "asin": math.asin, "acos": math.acos, "atan": math.atan, # "sinh": math.sinh, "cosh": math.cosh, "tanh": math.tanh, # "asinh": math.asinh, "acosh": math.acosh, "atanh": math.atanh, # "sqrt": math.sqrt, "exp": math.exp, "log": math.log, # "abs": math.fabs, "pow": math.pow, "floor": math.floor, "ceil": math.ceil, # "round": round, "min": min, "max": max, "pi": math.pi, "e": math.e, # "factorial": math.factorial, # "xor": xor, # Add custom xor function # "t": t, "x": x, "max_f": max_f, # # Adding boolean operations as lambda functions to simulate 'and', 'or', 'not' # "and": lambda a, b: a and b, # "or": lambda a, b: a or b, # "not": lambda a: not a, # } # try: # # Directly using 'if else' in expr is supported by Python syntax # # Logical operations 'and', 'or', 'not', and 'xor' are supported via the above definitions # return eval(expr, {"__builtins__": {}}, allowed_locals) # except NameError as e: # raise ValueError(f"Invalid expression: {e}") # except Exception as e: # raise ValueError(f"Error evaluating expression: {repr(e)}") # # @classmethod # def IS_CHANGED(self, *args, **kwargs): # return float("NaN") # # def convert(self, type_name, amplitude, max_frames=1500, math="x/100", filter_window=0, deforum_frame_data={}): # max_f = int(max_frames) # # # Optionally apply a smoothing filter to the amplitude data # if filter_window > 0: # amplitude_smoothed = np.convolve(amplitude, np.ones(filter_window) / filter_window, mode='same') # else: # amplitude_smoothed = amplitude # # frame_index = 0 # modified_amplitude_list = [] # for x in amplitude_smoothed: # modified_value = self.safe_eval(math, frame_index, x, max_f) # modified_amplitude_list.append(modified_value) # frame_index += 1 # # # Ensure the modified amplitude list can still be used as an amplitude input # modified_amplitude_series = pd.Series(modified_amplitude_list) # # # Format the series for visualization or further processing # formatted_strings = [f"{idx}:({val})" for idx, val in modified_amplitude_series.items()] # formatted_string = ", ".join(formatted_strings[:-1]) + " and " + formatted_strings[-1] if len(formatted_strings) > 1 else formatted_strings[0] # # # Update deforum_frame_data if necessary # if "keys" in deforum_frame_data: # deforum_frame_data["keys"][f"{type_name}_series"] = modified_amplitude_series.to_dict() # # return (deforum_frame_data, modified_amplitude_series.to_numpy(), formatted_string,) import numpy as np import pandas as pd import math class DeforumAmplitudeToKeyframeSeriesNode: @classmethod def INPUT_TYPES(cls): return {"required": {"type_name": (schedule_types,), "amplitude": ("AMPLITUDE",), }, "optional": { "max_frames": ("INT", {"default": 500, "min": 1, "max": 16500, "step": 1}), "math": ("STRING", {"default": "x/100"}), "filter_window": ("INT", {"default": 0, "min": 0, "max": 100, "step": 1}), "deforum_frame_data": ("DEFORUM_FRAME_DATA",), "y": ("AMPLITUDE",), "z": ("AMPLITUDE",), } } RETURN_TYPES = ("DEFORUM_FRAME_DATA", "AMPLITUDE", "STRING") FUNCTION = "convert" display_name = "Amplitude to Schedule" CATEGORY = "deforum/audio" def safe_eval(self, expr, t, x, max_f, y=None, z=None): allowed_locals = { "sin": math.sin, "cos": math.cos, "tan": math.tan, "asin": math.asin, "acos": math.acos, "atan": math.atan, "sinh": math.sinh, "cosh": math.cosh, "tanh": math.tanh, "asinh": math.asinh, "acosh": math.acosh, "atanh": math.atanh, "sqrt": math.sqrt, "exp": math.exp, "log": math.log, "abs": math.fabs, "pow": math.pow, "floor": math.floor, "ceil": math.ceil, "round": round, "min": min, "max": max, "pi": math.pi, "e": math.e, "factorial": math.factorial, "xor": lambda a, b: a ^ b, # Python's `^` operator for XOR "t": t, "x": x, "max_f": max_f, "y": y, "z": z, # Boolean operations via lambda functions "and": lambda a, b: a and b, "or": lambda a, b: a or b, "not": lambda a: not a, } try: return eval(expr, {"__builtins__": {}}, allowed_locals) except NameError as e: raise ValueError(f"Invalid expression: {e}") except Exception as e: raise ValueError(f"Error evaluating expression: {repr(e)}") def convert(self, type_name, amplitude, max_frames=500, math="x/100", filter_window=0, deforum_frame_data={}, y=None, z=None): max_f = int(max_frames) # Apply smoothing filter to the amplitude data if needed amplitude_smoothed = np.convolve(amplitude, np.ones(filter_window) / filter_window, mode='same') if filter_window > 0 else amplitude y_smoothed = np.convolve(y, np.ones(filter_window) / filter_window, mode='same') if y is not None and filter_window > 0 else y z_smoothed = np.convolve(z, np.ones(filter_window) / filter_window, mode='same') if z is not None and filter_window > 0 else z frame_index = 0 modified_amplitude_list = [] for idx, x in enumerate(amplitude_smoothed): y_val = y_smoothed[idx] if y is not None else None z_val = z_smoothed[idx] if z is not None else None modified_value = self.safe_eval(math, frame_index, x, max_f, y=y_val, z=z_val) modified_amplitude_list.append(modified_value) frame_index += 1 modified_amplitude_series = pd.Series(modified_amplitude_list) formatted_strings = [f"{idx}:({val})" for idx, val in modified_amplitude_series.items()] formatted_string = ", ".join(formatted_strings[:-1]) + " and " + formatted_strings[-1] if len(formatted_strings) > 1 else formatted_strings[0] if "keys" in deforum_frame_data: deforum_frame_data["keys"][f"{type_name}_series"] = modified_amplitude_series.to_dict() return (deforum_frame_data, modified_amplitude_series.to_numpy(), formatted_string,) class DeforumAmplitudeToString: @classmethod def INPUT_TYPES(s): return {"required": {"amplitude": ("AMPLITUDE",), }, } RETURN_TYPES = ("STRING",) # RETURN_NAMES = ("POSITIVE", "NEGATIVE") FUNCTION = "convert" display_name = "Amplitude to String" CATEGORY = "deforum/audio" @classmethod def IS_CHANGED(self, *args, **kwargs): # Force re-evaluation of the node return float("NaN") def convert(self, amplitude): return (str(amplitude),) class DerivativeOfAmplitude: @classmethod def INPUT_TYPES(s): return {"required": { "amplitude": ("AMPLITUDE",), },} CATEGORY = "deforum/audio" RETURN_TYPES = ("AMPLITUDE",) RETURN_NAMES = ("amplitude_derivative",) FUNCTION = "derive" display_name = "Derive Amplitude" def derive(self, amplitude,): derivative = np.diff(amplitude, prepend=amplitude[0]) # Use np.diff with prepend to maintain array length return (derivative,) class SpectralCentroid: @classmethod def INPUT_TYPES(s): return {"required": { "audio_fft": ("AUDIO_FFT",), },} CATEGORY = "deforum/audio" RETURN_TYPES = ("FLOAT",) RETURN_NAMES = ("spectral_centroid",) FUNCTION = "calculate" display_name = "Amplitude Spectral Centoid" def calculate(self, audio_fft,): magnitudes = np.abs(audio_fft) / len(audio_fft) frequencies = np.linspace(0, audio_fft.sample_rate / 2, len(magnitudes)) centroid = np.sum(frequencies * magnitudes) / np.sum(magnitudes) return (centroid,) class TimeSmoothing: @classmethod def INPUT_TYPES(s): return {"required": { "amplitude": ("AMPLITUDE",), }, "optional": { "window_size": ("INT", {"default": 5, "min": 1}), } } CATEGORY = "deforum/audio" RETURN_TYPES = ("AMPLITUDE",) RETURN_NAMES = ("smoothed_amplitude",) FUNCTION = "smooth" display_name = "Amplitude Time Smoothing" def smooth(self, amplitude, window_size,): smoothed_amplitude = np.convolve(amplitude, np.ones(window_size)/window_size, mode='same') return (smoothed_amplitude,) class BeatDetection: @classmethod def INPUT_TYPES(s): return {"required": { "audio": ("AUDIO",), }, } CATEGORY = "deforum/audio" RETURN_TYPES = ("AMPLITUDE",) RETURN_NAMES = ("beat_times",) FUNCTION = "detect" display_name = "Audio to Beat Amplitude" def detect(self, audio): beat_times = self.find_beat_times(audio) # Assuming beat_times are indices, we convert these to time values beat_times_in_seconds = beat_times / audio.sample_rate beat_times_series = pd.Series(beat_times_in_seconds) return (beat_times_series,) def find_beat_times(self, audio): if audio.num_channels > 1: audio_data = audio.get_channel_audio_data(0) else: audio_data = audio.audio_data envelope = self.extract_envelope(audio_data, audio.sample_rate) smoothed_envelope = scipy.ndimage.gaussian_filter1d(envelope, sigma=5) normalized_envelope = (smoothed_envelope - np.min(smoothed_envelope)) / (np.max(smoothed_envelope) - np.min(smoothed_envelope)) peaks, _ = scipy.signal.find_peaks(normalized_envelope, height=0.3) # The height threshold may need adjustment return peaks def extract_envelope(self, audio_data, sample_rate): analytic_signal = scipy.signal.hilbert(audio_data) envelope = np.abs(analytic_signal) return envelope class FrequencyRangeAmplitude: @classmethod def INPUT_TYPES(s): return { "required": { "audio": ("AUDIO",), }, "optional": { "frequency_range": ("TUPLE", {"default": (20, 20000)}), "window_size": ("INT", {"default": 1}), "inverted": ("BOOLEAN", {"default": False}), }, } CATEGORY = "deforum/audio" RETURN_TYPES = ("AMPLITUDE",) RETURN_NAMES = ("frequency_range_amplitude",) FUNCTION = "analyze_frequency_range" display_name = "Frequency Range Amplitude" def analyze_frequency_range(self, audio, frequency_range=(20, 20000), window_size=1, inverted=False): if audio.num_channels > 1: audio_data = audio.get_channel_audio_data(0) else: audio_data = audio.audio_data sample_rate = audio.sample_rate # Apply bandpass filter filtered_audio = self.bandpass_filter(audio_data, frequency_range[0], frequency_range[1], sample_rate) # Calculate FFT fft_result = np.fft.rfft(filtered_audio) fft_freqs = np.fft.rfftfreq(len(filtered_audio), 1 / sample_rate) # Extract amplitude amplitudes = np.abs(fft_result) # Normalize and smooth if necessary normalized_amplitudes = self.normalize(amplitudes, window_size=window_size, inverted=inverted) return (normalized_amplitudes,) def bandpass_filter(self, data, lowcut, highcut, sample_rate, order=5): nyquist = 0.5 * sample_rate low = lowcut / nyquist high = highcut / nyquist b, a = butter(order, [low, high], btype='band') y = filtfilt(b, a, data) return y def normalize(self, data, min_value=0, max_value=1, window_size=1, inverted=False): # Normalization and smoothing logic as described in the notebook # Return normalized data pass # Implement normalization and optional smoothing as described class BeatDetectionNode: @classmethod def INPUT_TYPES(s): return { "required": { "audio": ("AUDIO",), "sample_rate": ("INT",{"default": 44100, "min": 8000, "max": 200000, "step":1}), } } CATEGORY = "deforum/audio" RETURN_TYPES = ("AMPLITUDE",) RETURN_NAMES = ("beat_times",) FUNCTION = "detect_beats" display_name = "Beat Detection v2" def detect_beats(self, audio, sample_rate): if audio.num_channels > 1: audio_data = audio.get_channel_audio_data(0) else: audio_data = audio.audio_data # Assuming audio_data is already pre-processed to be in the correct format tempo, beats = librosa.beat.beat_track(y=audio_data, sr=sample_rate) beat_times = librosa.frames_to_time(beats, sr=sample_rate) return (beat_times,) class TempoChangeDetectionNode: @classmethod def INPUT_TYPES(cls): return { "required": {"audio": ("AUDIO",)}, "optional": {"threshold": ("FLOAT", {"default": 0.5, "min":0.0, "max":1250.0, "step":0.1})}, } CATEGORY = "deforum/audio" RETURN_TYPES = ("AMPLITUDE",) RETURN_NAMES = ("tempo_change_times",) FUNCTION = "detect_tempo_changes" display_name = "Tempo Change Detection" def detect_tempo_changes(self, audio, threshold=0.5, fps=24.0): audio_data = audio.get_channel_audio_data(0) if audio.num_channels > 1 else audio.audio_data sample_rate = audio.sample_rate # Calculate onset strength onset_env = librosa.onset.onset_strength(y=audio_data.astype(np.float32), sr=sample_rate) tempogram = librosa.feature.tempogram(onset_envelope=onset_env, sr=sample_rate) dynamic_tempo = np.mean(tempogram, axis=1) times = librosa.frames_to_time(np.arange(len(dynamic_tempo)), sr=sample_rate) # Detect change points change_points = np.abs(np.diff(dynamic_tempo)) > threshold change_times = times[:-1][change_points] # Convert change times to a sequence of amplitudes with the desired fps # First, create a boolean array indicating change points full_length = int(np.ceil(times[-1] * fps)) change_points_sequence = np.zeros(full_length, dtype=bool) interpolated_times = np.arange(0, times[-1], step=1.0 / fps) for time in interpolated_times: index = int(time * fps) if index < full_length: change_points_sequence[index] = True # Optionally, fill in missing values if needed, but here we map detected changes directly # Return the sequence as desired print(len(change_points_sequence)) # Debug print to check the length of the output sequence return (change_points_sequence,) class ConvertNormalizedAmplitude: @classmethod def INPUT_TYPES(cls): return { "required": {"normalized_amp": ("NORMALIZED_AMPLITUDE",)}, } CATEGORY = "deforum/audio" RETURN_TYPES = ("AMPLITUDE",) RETURN_NAMES = ("amplitude",) FUNCTION = "convert_normalized_amplitude" display_name = "Convert Normalized Amplitude" def convert_normalized_amplitude(self, normalized_amp): return (normalized_amp,) ================================================ FILE: deforum_nodes/nodes/deforum_cache_nodes.py ================================================ class DeforumCacheLatentNode: @classmethod def IS_CHANGED(cls, text, autorefresh): # Force re-evaluation of the node if autorefresh == "Yes": return float("NaN") @classmethod def INPUT_TYPES(cls): return { "required": { "latent": ("LATENT",), "cache_index": ("INT", {"default":0, "min": 0, "max": 16, "step": 1}) } } RETURN_TYPES = (("LATENT",)) FUNCTION = "cache_it" CATEGORY = f"deforum/cache" display_name = "Cache Latent" OUTPUT_NODE = True def cache_it(self, latent=None, cache_index=0): from ..mapping import gs from ..mapping import gs if "latent" not in gs.deforum_cache: gs.deforum_cache["latent"] = {} gs.deforum_cache["latent"][cache_index] = latent return (latent,) class DeforumGetCachedLatentNode: @classmethod def IS_CHANGED(cls, text, autorefresh): # Force re-evaluation of the node if autorefresh == "Yes": return float("NaN") @classmethod def INPUT_TYPES(cls): return {"required": { "cache_index": ("INT", {"default": 0, "min": 0, "max": 16, "step": 1}) }} RETURN_TYPES = (("LATENT",)) FUNCTION = "get_cached_latent" CATEGORY = f"deforum/cache" OUTPUT_NODE = True display_name = "Load Cached Latent" def get_cached_latent(self, cache_index=0): from ..mapping import gs if gs.reset: return (None,) latent_dict = gs.deforum_cache.get("latent", {}) latent = latent_dict.get(cache_index) return (latent,) class DeforumCacheImageNode: @classmethod def IS_CHANGED(cls, text, autorefresh): # Force re-evaluation of the node if autorefresh == "Yes": return float("NaN") @classmethod def INPUT_TYPES(cls): return { "required": { "image": ("IMAGE",), "cache_index": ("INT", {"default":0, "min": 0, "max": 16, "step": 1}) } } RETURN_TYPES = (("IMAGE",)) FUNCTION = "cache_it" CATEGORY = f"deforum/cache" display_name = "Cache Image" OUTPUT_NODE = True def cache_it(self, image=None, cache_index=0): from ..mapping import gs if "image" not in gs.deforum_cache: gs.deforum_cache["image"] = {} if image is not None: gs.deforum_cache["image"][cache_index] = image.detach().clone() return (image,) class DeforumGetCachedImageNode: @classmethod def IS_CHANGED(cls, text, autorefresh): # Force re-evaluation of the node if autorefresh == "Yes": return float("NaN") @classmethod def INPUT_TYPES(cls): return {"required": { "cache_index": ("INT", {"default": 0, "min": 0, "max": 16, "step": 1}) }} RETURN_TYPES = (("IMAGE","MASK")) RETURN_NAMES = ("IMAGE","MASK") FUNCTION = "get_cached_latent" CATEGORY = f"deforum/cache" OUTPUT_NODE = True display_name = "Load Cached Image" def get_cached_latent(self, cache_index=0): from ..mapping import gs if gs.reset: return (None, None) img_dict = gs.deforum_cache.get("image", {}) image = img_dict.get(cache_index) mask = None if image is not None: mask = image[:, :, :, 0] image = image.detach().clone() return (image ,mask,) class DeforumCacheStringNode: @classmethod def IS_CHANGED(cls, text, autorefresh): # Force re-evaluation of the node if autorefresh == "Yes": return float("NaN") @classmethod def INPUT_TYPES(cls): return { "required": { "input_string": ("STRING",{"default":""}), "cache_index": ("INT", {"default":0, "min": 0, "max": 16, "step": 1}) } } RETURN_TYPES = (("STRING",)) FUNCTION = "cache_it" CATEGORY = f"deforum/cache" display_name = "Cache String" OUTPUT_NODE = True def cache_it(self, input_string=None, cache_index=0): from ..mapping import gs if "string" not in gs.deforum_cache: gs.deforum_cache["string"] = {} gs.deforum_cache["string"][cache_index] = input_string return (input_string,) class DeforumGetCachedStringNode: @classmethod def IS_CHANGED(cls, text, autorefresh): # Force re-evaluation of the node if autorefresh == "Yes": return float("NaN") @classmethod def INPUT_TYPES(cls): return {"required": { "cache_index": ("INT", {"default": 0, "min": 0, "max": 16, "step": 1}) }} RETURN_TYPES = (("STRING",)) FUNCTION = "get_cached_string" CATEGORY = f"deforum/cache" OUTPUT_NODE = True display_name = "Load Cached String" def get_cached_string(self, cache_index=0): from ..mapping import gs img_dict = gs.deforum_cache.get("string", {}) string = img_dict.get(cache_index) return (str(string),) ================================================ FILE: deforum_nodes/nodes/deforum_cnet_nodes.py ================================================ class DeforumControlNetApply: @classmethod def INPUT_TYPES(s): return {"required": {"conditioning": ("CONDITIONING", ), "control_net": ("CONTROL_NET", ), "strength": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.01}) }, "optional": {"image": ("IMAGE",)} } RETURN_TYPES = ("CONDITIONING",) FUNCTION = "apply_controlnet" display_name = "Apply ControlNet [safe]" CATEGORY = "deforum/controlnet" def apply_controlnet(self, conditioning, control_net, strength, image=None): if strength == 0 or image is None: return (conditioning, ) c = [] control_hint = image.movedim(-1,1) for t in conditioning: n = [t[0], t[1].copy()] c_net = control_net.copy().set_cond_hint(control_hint, strength) if 'control' in t[1]: c_net.set_previous_controlnet(t[1]['control']) n[1]['control'] = c_net n[1]['control_apply_to_uncond'] = True c.append(n) return (c, ) ================================================ FILE: deforum_nodes/nodes/deforum_cond_nodes.py ================================================ import random import torch from ..modules.deforum_comfyui_helpers import blend_tensors, blend_methods class DeforumConditioningBlendNode: def __init__(self): self.prompt = None self.n_prompt = None self.cond = None self.n_cond = None @classmethod def INPUT_TYPES(s): return {"required": {"clip": ("CLIP",), "deforum_frame_data": ("DEFORUM_FRAME_DATA",), "blend_method": ([blend_methods]), } } RETURN_TYPES = ("CONDITIONING", "CONDITIONING") RETURN_NAMES = ("POSITIVE", "NEGATIVE") FUNCTION = "fn" display_name = "Blend Conditionings" CATEGORY = "deforum/conditioning" def fn(self, clip, deforum_frame_data, blend_method): areas = deforum_frame_data.get("areas") negative_prompt = deforum_frame_data.get("negative_prompt", "") n_cond = self.get_conditioning(prompt=negative_prompt, clip=clip) if not areas: prompt = deforum_frame_data.get("prompt", "") next_prompt = deforum_frame_data.get("next_prompt", None) cond = self.get_conditioning(prompt=prompt, clip=clip) # image = self.getInputData(2) # controlnet = self.getInputData(3) prompt_blend = deforum_frame_data.get("prompt_blend", 0.0) #method = self.content.blend_method.currentText() if blend_method != 'none': if next_prompt != prompt and prompt_blend != 0.0 and next_prompt is not None: next_cond = self.get_conditioning(prompt=next_prompt, clip=clip) cond = blend_tensors(cond[0], next_cond[0], prompt_blend, blend_method) print(f"[deforum] Blending next prompt: {next_prompt}, with alpha: {prompt_blend} ]") else: from nodes import ConditioningSetArea area_setter = ConditioningSetArea() cond = [] for area in areas: prompt = area.get("prompt", None) if prompt: new_cond = self.get_conditioning(clip=clip, prompt=area["prompt"]) new_cond = area_setter.append(conditioning=new_cond, width=int(area["w"]), height=int(area["h"]), x=int(area["x"]), y=int(area["y"]), strength=area["s"])[0] cond += new_cond return (cond, n_cond,) def get_conditioning(self, prompt="", clip=None, progress_callback=None): tokens = clip.tokenize(prompt) cond, pooled = clip.encode_from_tokens(tokens, return_pooled=True) return [[cond, {"pooled_output": pooled}]] class DeforumInpaintModelConditioning: @classmethod def INPUT_TYPES(s): return {"required": { "positive": ("CONDITIONING", ), "negative": ("CONDITIONING", ), "vae": ("VAE", ),}, "optional": { "pixels": ("IMAGE",), "mask": ("MASK",), "latent": ("LATENT",), "deforum_frame_data": ("DEFORUM_FRAME_DATA",), } } RETURN_TYPES = ("CONDITIONING","CONDITIONING","LATENT") RETURN_NAMES = ("positive", "negative", "latent") FUNCTION = "encode" display_name = "InpaintModelConditioning [safe]" CATEGORY = "deforum/conditioning" def encode(self, positive, negative, vae, pixels, mask, latent, deforum_frame_data={}): reset = deforum_frame_data.get("reset", False) if (pixels is not None and mask is not None) and not reset: x = (pixels.shape[1] // 8) * 8 y = (pixels.shape[2] // 8) * 8 mask = torch.nn.functional.interpolate(mask.reshape((-1, 1, mask.shape[-2], mask.shape[-1])), size=(pixels.shape[1], pixels.shape[2]), mode="bilinear") orig_pixels = pixels pixels = orig_pixels.clone() if pixels.shape[1] != x or pixels.shape[2] != y: x_offset = (pixels.shape[1] % 8) // 2 y_offset = (pixels.shape[2] % 8) // 2 pixels = pixels[:,x_offset:x + x_offset, y_offset:y + y_offset,:] mask = mask[:,:,x_offset:x + x_offset, y_offset:y + y_offset] m = (1.0 - mask.round()).squeeze(1) for i in range(3): pixels[:,:,:,i] -= 0.5 pixels[:,:,:,i] *= m pixels[:,:,:,i] += 0.5 concat_latent = vae.encode(pixels) orig_latent = vae.encode(orig_pixels) out_latent = {} out_latent["samples"] = orig_latent out_latent["noise_mask"] = mask out = [] for conditioning in [positive, negative]: c = [] for t in conditioning: d = t[1].copy() d["concat_latent_image"] = concat_latent d["concat_mask"] = mask n = [t[0], d] c.append(n) out.append(c) return (out[0], out[1], out_latent) else: return (positive, negative, latent,) class DeforumShuffleTokenizer: @classmethod def INPUT_TYPES(s): return {"required": {"clip": ("CLIP",), "seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff}), } } RETURN_TYPES = ("CLIP",) FUNCTION = "fn" display_name = "Shuffle Tokenizer" CATEGORY = "deforum/conditioning" def fn(self, clip, seed=42): # Access the tokenizer from the clip object tokenizer = clip.tokenizer # Copy the original vocabulary for restoration if needed original_vocab = tokenizer.vocab.copy() # Seed the random number generator for reproducibility seeded_random = random.Random(seed) # Create a list of (key, value) pairs, shuffle it, then convert it back to a dictionary items = list(original_vocab.items()) seeded_random.shuffle(items) shuffled_vocab = dict(items) # Update the tokenizer's vocabulary with the shuffled version # This step is highly dependent on the tokenizer's implementation. # If the tokenizer has a method to set its vocab, use it. # Otherwise, you might need to directly set the attribute, if possible. # tokenizer.set_vocab(shuffled_vocab) # Hypothetical method tokenizer.vocab = shuffled_vocab # Direct attribute setting, if no method available return (clip,) ================================================ FILE: deforum_nodes/nodes/deforum_data_nodes.py ================================================ from ..modules.deforum_ui_data import (deforum_base_params, deforum_anim_params, deforum_translation_params, deforum_cadence_params, deforum_depth_params, deforum_noise_params, deforum_color_coherence_params, deforum_diffusion_schedule_params, deforum_hybrid_video_params, deforum_hybrid_video_schedules) from ..modules.deforum_node_base import DeforumDataBase from ..modules.deforum_comfyui_helpers import get_node_params class DeforumBaseParamsNode(DeforumDataBase): params = get_node_params(deforum_base_params) display_name = "Base Parameters" def __init__(self): super().__init__() @classmethod def INPUT_TYPES(s): return s.params class DeforumAnimParamsNode(DeforumDataBase): params = get_node_params(deforum_anim_params) display_name = "Animation Parameters" def __init__(self): super().__init__() @classmethod def INPUT_TYPES(s): return s.params class DeforumTranslationParamsNode(DeforumDataBase): params = get_node_params(deforum_translation_params) display_name = "Translate Parameters" def __init__(self): super().__init__() @classmethod def INPUT_TYPES(s): return s.params class DeforumDepthParamsNode(DeforumDataBase): params = get_node_params(deforum_depth_params) display_name = "Depth Parameters" def __init__(self): super().__init__() @classmethod def INPUT_TYPES(s): return s.params class DeforumNoiseParamsNode(DeforumDataBase): params = get_node_params(deforum_noise_params) display_name = "Noise Parameters" def __init__(self): super().__init__() @classmethod def INPUT_TYPES(s): return s.params class DeforumColorParamsNode(DeforumDataBase): params = get_node_params(deforum_color_coherence_params) display_name = "ColorMatch Parameters" def __init__(self): super().__init__() @classmethod def INPUT_TYPES(s): return s.params class DeforumDiffusionParamsNode(DeforumDataBase): params = get_node_params(deforum_diffusion_schedule_params) display_name = "Diffusion Parameters" def __init__(self): super().__init__() @classmethod def INPUT_TYPES(s): return s.params class DeforumCadenceParamsNode(DeforumDataBase): params = get_node_params(deforum_cadence_params) display_name = "Cadence Parameters" def __init__(self): super().__init__() @classmethod def INPUT_TYPES(s): return s.params class DeforumHybridParamsNode(DeforumDataBase): params = get_node_params(deforum_hybrid_video_params) display_name = "Hybrid Parameters" def __init__(self): super().__init__() @classmethod def INPUT_TYPES(s): return s.params class DeforumHybridScheduleNode(DeforumDataBase): params = get_node_params(deforum_hybrid_video_schedules) display_name = "Hybrid Schedule" def __init__(self): super().__init__() @classmethod def INPUT_TYPES(s): return s.params class DeforumFrameDataExtract: @classmethod def INPUT_TYPES(s): return {"required": {"deforum_frame_data": ("DEFORUM_FRAME_DATA",), } } RETURN_TYPES = ("INT","INT", "INT", "FLOAT", "STRING", "STRING", "FLOAT", "FLOAT", "BOOLEAN") RETURN_NAMES = ("frame_idx", "seed", "steps", "cfg_scale", "sampler_name", "scheduler_name", "denoise", "subseed_strength", "first_run") FUNCTION = "get_data" display_name = "Frame Data Extract" CATEGORY = "deforum/parameters" def get_data(self, deforum_frame_data): seed = deforum_frame_data.get("seed", 0) steps = deforum_frame_data.get("steps", 10) cfg = deforum_frame_data.get("cfg", 7.5) sampler_name = deforum_frame_data.get("sampler_name", "euler_a") scheduler = deforum_frame_data.get("scheduler", "normal") denoise = deforum_frame_data.get("denoise", 1.0) keys = deforum_frame_data.get("keys") frame_idx = deforum_frame_data.get("frame_idx") subseed_str = keys.subseed_strength_schedule_series[frame_idx] first_run = deforum_frame_data.get("second_run", False) return (frame_idx, seed, steps, cfg, sampler_name, scheduler, denoise, subseed_str, first_run, ) ================================================ FILE: deforum_nodes/nodes/deforum_framewarp_node.py ================================================ import cv2 import numpy as np import torch from PIL import Image from einops import rearrange, repeat import comfy from ..modules.deforum_comfyui_helpers import tensor2np, pil2tensor class DeforumFrameWarpNode: def __init__(self): self.depth_model = None self.depth = None self.algo = "" self.depth_min, self.depth_max = 1000, -1000 @classmethod def INPUT_TYPES(s): return {"required": {"image": ("IMAGE",), "deforum_frame_data": ("DEFORUM_FRAME_DATA",), "warp_depth_image": ("BOOLEAN",{"default":False}), }, "optional": { "depth_image":("IMAGE",), } } RETURN_TYPES = ("IMAGE","IMAGE","IMAGE") RETURN_NAMES = ("IMAGE","DEPTH", "WARPED_DEPTH") FUNCTION = "fn" display_name = "Frame Warp" CATEGORY = "deforum/image" def fn(self, image, deforum_frame_data, warp_depth_image, depth_image=None): from deforum.models import DepthModel from deforum.utils.deforum_framewarp_utils import anim_frame_warp np_image = None data = deforum_frame_data if image is not None: if image.shape[0] > 1: for img in image: np_image = tensor2np(img) else: np_image = tensor2np(image) np_image = cv2.cvtColor(np_image, cv2.COLOR_RGB2BGR) args = data.get("args") anim_args = data.get("anim_args") keys = data.get("keys") frame_idx = data.get("frame_idx") if frame_idx == 0: self.depth = None predict_depths = ( anim_args.animation_mode == '3D' and anim_args.use_depth_warping) or anim_args.save_depth_maps predict_depths = predict_depths or ( anim_args.hybrid_composite and anim_args.hybrid_comp_mask_type in ['Depth', 'Video Depth']) if depth_image is not None: predict_depths = False if self.depth_model == None or self.algo != anim_args.depth_algorithm: self.vram_state = "high" if self.depth_model is not None: self.depth_model.to("cpu") del self.depth_model # torch_gc() self.algo = anim_args.depth_algorithm if predict_depths: keep_in_vram = True if self.vram_state == 'high' else False # device = ('cpu' if cmd_opts.lowvram or cmd_opts.medvram else self.root.device) # TODO Set device in root in webui device = 'cuda' self.depth_model = DepthModel("models/other", device, keep_in_vram=keep_in_vram, depth_algorithm=anim_args.depth_algorithm, Width=args.width, Height=args.height, midas_weight=anim_args.midas_weight) # depth-based hybrid composite mask requires saved depth maps if anim_args.hybrid_composite != 'None' and anim_args.hybrid_comp_mask_type == 'Depth': anim_args.save_depth_maps = True else: self.depth_model = None anim_args.save_depth_maps = False if self.depth_model != None and not predict_depths: self.depth_model = None if self.depth_model is not None: self.depth_model.to('cuda') if depth_image is not None: depth_image = comfy.utils.common_upscale(depth_image.permute(0,3,1,2), args.width, args.height, upscale_method="bislerp", crop="disabled") depth_image = depth_image.permute(0,2,3,1) * 255.0 if depth_image.dim() > 2: depth_image = depth_image[0].mean(dim=-1) # Take the mean across the color channels warped_np_img, depth, mask = anim_frame_warp(np_image, args, anim_args, keys, frame_idx, depth_model=self.depth_model, depth=depth_image, device='cuda', half_precision=True) image = Image.fromarray(cv2.cvtColor(warped_np_img, cv2.COLOR_BGR2RGB)) tensor = pil2tensor(image) if depth is not None: num_channels = len(depth.shape) if num_channels <= 3: depth_image_pil = self.to_image(depth.detach().cpu()) else: depth_image_pil = self.to_image(depth[0].detach().cpu()) ret_depth = pil2tensor(depth_image_pil).detach().cpu() if warp_depth_image: warped_depth, _, _ = anim_frame_warp(np.array(depth_image_pil), args, anim_args, keys, frame_idx, depth_model=self.depth_model, depth=depth_image, device='cuda', half_precision=True) warped_depth_image = Image.fromarray(warped_depth) warped_ret = pil2tensor(warped_depth_image).detach().cpu() else: warped_ret = ret_depth else: ret_depth = tensor warped_ret = tensor self.depth = depth # if gs.vram_state in ["low", "medium"] and self.depth_model is not None: # self.depth_model.to('cpu') # if mask is not None: # mask = mask.detach().cpu() # # mask = mask.reshape((-1, 1, mask.shape[-2], mask.shape[-1])).movedim(1, -1).expand(-1, -1, -1, 3) # mask = mask.mean(dim=0, keepdim=False) # mask[mask > 1e-05] = 1 # mask[mask < 1e-05] = 0 # mask = mask[0].unsqueeze(0) # from ai_nodes.ainodes_engine_base_nodes.ainodes_backend.resizeRight import resizeright # from ai_nodes.ainodes_engine_base_nodes.ainodes_backend.resizeRight import interp_methods # mask = resizeright.resize(mask, scale_factors=None, # out_shape=[mask.shape[0], int(mask.shape[1] // 8), int(mask.shape[2] // 8) # ], # interp_method=interp_methods.lanczos3, support_sz=None, # antialiasing=True, by_convs=True, scale_tolerance=None, # max_numerator=10, pad_mode='reflect') return (tensor, ret_depth,warped_ret,) # return [data, tensor, mask, ret_depth, self.depth_model] else: return (image, image,image,) def to_image(self, depth: torch.Tensor): depth = depth.cpu().numpy() depth = np.expand_dims(depth, axis=0) if len(depth.shape) == 2 else depth self.depth_min, self.depth_max = min(self.depth_min, depth.min()), max(self.depth_max, depth.max()) denom = max(1e-8, self.depth_max - self.depth_min) temp = rearrange((depth - self.depth_min) / denom * 255, 'c h w -> h w c') return Image.fromarray(repeat(temp, 'h w 1 -> h w c', c=3).astype(np.uint8)) ================================================ FILE: deforum_nodes/nodes/deforum_hybrid_nodes.py ================================================ import copy import cv2 import numpy as np from PIL import Image from deforum.generators.deforum_flow_generator import get_flow_from_images from deforum.models import RAFT from deforum.utils.image_utils import image_transform_optical_flow from ..modules.deforum_comfyui_helpers import tensor2np, tensor2pil, pil2tensor # from ..modules.deforum_constants import deforum_models, deforum_depth_algo from ..mapping import gs class DeforumApplyFlowNode: methods = ['RAFT', 'DIS Medium', 'DIS Fine', 'Farneback'] @classmethod def INPUT_TYPES(cls): return { "required": { "image": ("IMAGE",), "flow_image": ("IMAGE",), "flow_method": ([cls.methods]), "flow_factor": ("FLOAT", {"default": 0.8, "min": 0, "max": 1.0}), }, "optional":{ "deforum_frame_data": ("DEFORUM_FRAME_DATA",), } } RETURN_TYPES = (("IMAGE",)) FUNCTION = "apply_flow" CATEGORY = f"deforum/image" OUTPUT_NODE = True display_name = "Apply Flow" def __init__(self): self.image_cache = [] def apply_flow(self, image, flow_image, flow_method, flow_factor, deforum_frame_data={}): # global deforum_models if "raft_model" not in gs.deforum_models: gs.deforum_models["raft_model"] = RAFT() if deforum_frame_data.get("reset", None): self.image_cache.clear() if flow_image is not None: temp_np = tensor2np(flow_image) else: temp_np = tensor2np(image) self.image_cache.append(temp_np) if len(self.image_cache) >= 2: flow = get_flow_from_images(self.image_cache[0], self.image_cache[1], flow_method, gs.deforum_models["raft_model"]) img = image_transform_optical_flow(tensor2np(image), flow, flow_factor) ret = pil2tensor(img) self.image_cache = [self.image_cache[1]] return (ret,) else: return (image,) class DeforumHybridMotionNode: raft_model = None methods = ['RAFT', 'DIS Medium', 'DIS Fine', 'Farneback'] def __init__(self): self.prev_image = None self.flow = None self.image_size = None @classmethod def INPUT_TYPES(s): return {"required": {"image": ("IMAGE",), "hybrid_image": ("IMAGE",), "deforum_frame_data": ("DEFORUM_FRAME_DATA",), "hybrid_method": ([s.methods]), } } RETURN_TYPES = ("IMAGE",) FUNCTION = "fn" display_name = "Hybrid Motion" CATEGORY = "deforum/image" def fn(self, image, hybrid_image, deforum_frame_data, hybrid_method): if self.raft_model is None: self.raft_model = RAFT() flow_factor = deforum_frame_data["keys"].hybrid_flow_factor_schedule_series[deforum_frame_data["frame_index"]] p_img = tensor2pil(image) size = p_img.size pil_image = np.array(p_img).astype(np.uint8) if self.image_size != size: self.prev_image = None self.flow = None self.image_size = size bgr_image = cv2.cvtColor(pil_image, cv2.COLOR_RGB2BGR) if hybrid_image is None: if self.prev_image is None: self.prev_image = bgr_image return (image,) else: self.flow = get_flow_from_images(self.prev_image, bgr_image, hybrid_method, self.raft_model, self.flow) self.prev_image = copy.deepcopy(bgr_image) bgr_image = image_transform_optical_flow(bgr_image, self.flow, flow_factor) rgb_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2RGB) return (pil2tensor(rgb_image),) else: pil_image_ref = np.array(tensor2pil(hybrid_image).resize((self.image_size), Image.Resampling.LANCZOS)).astype(np.uint8) bgr_image_ref = cv2.cvtColor(pil_image_ref, cv2.COLOR_RGB2BGR) bgr_image_ref = cv2.resize(bgr_image_ref, (bgr_image.shape[1], bgr_image.shape[0])) if self.prev_image is None: self.prev_image = bgr_image_ref return (image,) else: self.flow = get_flow_from_images(self.prev_image, bgr_image_ref, hybrid_method, self.raft_model, self.flow) bgr_image = image_transform_optical_flow(bgr_image, self.flow, flow_factor) rgb_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2RGB) return (pil2tensor(rgb_image),) ================================================ FILE: deforum_nodes/nodes/deforum_image_nodes.py ================================================ import cv2 from PIL import Image import numpy as np from deforum.generators.deforum_noise_generator import add_noise from deforum.utils.image_utils import maintain_colors, unsharp_mask, compose_mask_with_check from ..modules.deforum_comfyui_helpers import tensor2pil, tensor2np, pil2tensor class DeforumColorMatchNode: def __init__(self): self.depth_model = None self.algo = "" self.color_match_sample = None @classmethod def INPUT_TYPES(s): return {"required": {"image": ("IMAGE",), "deforum_frame_data": ("DEFORUM_FRAME_DATA",), "force_use_sample": ("BOOLEAN", {"default":False},) }, "optional": {"force_sample_image":("IMAGE",)} } RETURN_TYPES = ("IMAGE",) FUNCTION = "fn" display_name = "Color Match" CATEGORY = "deforum/image" def fn(self, image, deforum_frame_data, force_use_sample, force_sample_image=None): if image is not None: anim_args = deforum_frame_data.get("anim_args") image = np.array(tensor2pil(image)) # image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR) frame_idx = deforum_frame_data.get("frame_idx", 0) if frame_idx == 0 and not force_use_sample: self.color_match_sample = None return (pil2tensor(image),) if force_use_sample: if force_sample_image is not None: self.color_match_sample = np.array(tensor2pil(force_sample_image)).copy() if anim_args.color_coherence != 'None' and self.color_match_sample is not None: image = maintain_colors(image, self.color_match_sample, anim_args.color_coherence) print(f"[deforum] ColorMatch: {anim_args.color_coherence}") if self.color_match_sample is None: self.color_match_sample = image.copy() if anim_args.color_force_grayscale: image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) image = cv2.cvtColor(image, cv2.COLOR_GRAY2BGR) # image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) image = pil2tensor(image) return (image,) class DeforumAddNoiseNode: @classmethod def INPUT_TYPES(s): return {"required": {"image": ("IMAGE",), "deforum_frame_data": ("DEFORUM_FRAME_DATA",), } } RETURN_TYPES = ("IMAGE",) FUNCTION = "fn" display_name = "Add Noise" CATEGORY = "deforum/noise" def fn(self, image, deforum_frame_data): if image is not None: keys = deforum_frame_data.get("keys") args = deforum_frame_data.get("args") anim_args = deforum_frame_data.get("anim_args") root = deforum_frame_data.get("root") frame_idx = deforum_frame_data.get("frame_idx") noise = keys.noise_schedule_series[frame_idx] kernel = int(keys.kernel_schedule_series[frame_idx]) sigma = keys.sigma_schedule_series[frame_idx] amount = keys.amount_schedule_series[frame_idx] threshold = keys.threshold_schedule_series[frame_idx] contrast = keys.contrast_schedule_series[frame_idx] if anim_args.use_noise_mask and keys.noise_mask_schedule_series[frame_idx] is not None: noise_mask_seq = keys.noise_mask_schedule_series[frame_idx] else: noise_mask_seq = None mask_vals = {} noise_mask_vals = {} mask_vals['everywhere'] = Image.new('1', (args.width, args.height), 1) noise_mask_vals['everywhere'] = Image.new('1', (args.width, args.height), 1) # from ainodes_frontend.nodes.deforum_nodes.deforum_framewarp_node import tensor2np prev_img = tensor2np(image) mask_image = None # apply scaling contrast_image = (prev_img * contrast).round().astype(np.uint8) # anti-blur if amount > 0: contrast_image = unsharp_mask(contrast_image, (kernel, kernel), sigma, amount, threshold, mask_image if args.use_mask else None) # apply frame noising if args.use_mask or anim_args.use_noise_mask: root.noise_mask = compose_mask_with_check(root, args, noise_mask_seq, noise_mask_vals, Image.fromarray( cv2.cvtColor(contrast_image, cv2.COLOR_BGR2RGB))) noised_image = add_noise(contrast_image, noise, int(args.seed), anim_args.noise_type, (anim_args.perlin_w, anim_args.perlin_h, anim_args.perlin_octaves, anim_args.perlin_persistence), root.noise_mask, args.invert_mask) # image = Image.fromarray(noised_image) print(f"[deforum] Adding Noise {noise} {anim_args.noise_type}") image = pil2tensor(noised_image).detach().cpu() return (image,) ================================================ FILE: deforum_nodes/nodes/deforum_interpolation_nodes.py ================================================ import cv2 import torch import numpy as np from deforum import FilmModel from deforum.models import DepthModel, RAFT from comfy import model_management from ..modules.standalone_cadence import CadenceInterpolator from ..modules.deforum_comfyui_helpers import tensor2pil, pil2tensor # from ..modules.deforum_constants import deforum_models, deforum_depth_algo # from .deforum_cache_nodes import deforum_cache # deforum_models = {} # deforum_depth_algo = "" from ..mapping import gs class DeforumFILMInterpolationNode: def __init__(self): self.FILM_temp = [] self.model = None @classmethod def INPUT_TYPES(s): return {"required": {"image": ("IMAGE",), "inter_amount": ("INT", {"default": 2, "min": 1, "max": 10000},), "skip_first": ("BOOLEAN", {"default":True}), "skip_last": ("BOOLEAN", {"default":False}), } } RETURN_TYPES = ("IMAGE",) # RETURN_NAMES = ("POSITIVE", "NEGATIVE") FUNCTION = "fn" display_name = "FILM Interpolation" CATEGORY = "deforum/interpolation" @classmethod def IS_CHANGED(self, *args, **kwargs): # Force re-evaluation of the node return float("NaN") def interpolate(self, image, inter_frames, skip_first, skip_last): if self.model is None: self.model = FilmModel() self.model.model.cuda() return_frames = [] pil_image = tensor2pil(image.clone().detach()) np_image = np.array(pil_image.convert("RGB")) self.FILM_temp.append(np_image) if len(self.FILM_temp) == 2: # with torch.inference_mode(): with torch.no_grad(): frames = self.model.inference(self.FILM_temp[0], self.FILM_temp[1], inter_frames=inter_frames) # skip_first, skip_last = True, False if skip_first: frames.pop(0) if skip_last: frames.pop(-1) for frame in frames: tensor = pil2tensor(frame)[0] return_frames.append(tensor.detach().cpu()) self.FILM_temp = [self.FILM_temp[1]] print(f"[deforum] FILM: {len(return_frames)} frames") if len(return_frames) > 0: return_frames = torch.stack(return_frames, dim=0) return return_frames else: return image.unsqueeze(0) def fn(self, image, inter_amount, skip_first, skip_last): result = [] if image.shape[0] > 1: for img in image: interpolated_frames = self.interpolate(img, inter_amount, skip_first, skip_last) for f in interpolated_frames: result.append(f) ret = torch.stack(result, dim=0) else: ret = self.interpolate(image[0], inter_amount, skip_first, skip_last) return (ret,) class DeforumSimpleInterpolationNode: def __init__(self): self.FILM_temp = [] self.model = None @classmethod def INPUT_TYPES(s): return {"required": {"image": ("IMAGE",), "method": (["DIS Medium", "DIS Fast", "DIS UltraFast", "Farneback Fine", "Normal"],), # "DenseRLOF", "SF", "inter_amount": ("INT", {"default": 2, "min": 1, "max": 10000},), "skip_first": ("BOOLEAN", {"default":False}), "skip_last": ("BOOLEAN", {"default":False}), }, "optional":{ "first_image": ("IMAGE",), "deforum_frame_data": ("DEFORUM_FRAME_DATA",), } } RETURN_TYPES = ("IMAGE", "IMAGE") RETURN_NAMES = ("IMAGES", "LAST_IMAGE") FUNCTION = "fn" display_name = "Simple Interpolation" CATEGORY = "deforum/interpolation" @classmethod def IS_CHANGED(self, *args, **kwargs): # Force re-evaluation of the node return float("NaN") def interpolate(self, image, method, inter_frames, skip_first, skip_last): return_frames = [] pil_image = tensor2pil(image.clone().detach()) np_image = np.array(pil_image.convert("RGB")) self.FILM_temp.append(np_image) print(len(self.FILM_temp)) if len(self.FILM_temp) == 2: if inter_frames > 1: if method != "Dyna": from ..modules.interp import optical_flow_cadence frames = optical_flow_cadence(self.FILM_temp[0], self.FILM_temp[1], inter_frames + 1, method) # skip_first, skip_last = True, False if skip_first: frames.pop(0) if skip_last: frames.pop(-1) for frame in frames: tensor = pil2tensor(frame)[0] return_frames.append(tensor) else: if not self.model: from ..modules.lvdm.i2v_pipeline import Image2Video self.model = Image2Video() frames = self.model.get_image(self.FILM_temp[0], "cat sushi", steps=50, cfg_scale=7.5, eta=1.0, fs=5, seed=123, image2=self.FILM_temp[1], frames=inter_frames) for frame in frames: tensor = pil2tensor(frame)[0] return_frames.append(tensor) else: return_frames = [i for i in pil2tensor(self.FILM_temp)[0]] self.FILM_temp = [self.FILM_temp[1]] print(f"[deforum] Simple Interpolation {len(return_frames)} frames" ) if len(return_frames) > 0: return_frames = torch.stack(return_frames, dim=0) return return_frames else: return None def fn(self, image, method, inter_amount, skip_first, skip_last, first_image=None, deforum_frame_data={}): last = None if deforum_frame_data.get("reset"): print("RESETTING DYNA") self.FILM_temp = [] print(image) if image is not None: result = [] if image.shape[0] > 1: for img in image: interpolated_frames = self.interpolate(img, method, inter_amount, skip_first, skip_last) for f in interpolated_frames: result.append(f) ret = torch.stack(result, dim=0) else: ret = self.interpolate(image[0], method, inter_amount, skip_first, skip_last) if ret is not None: last = ret[-1].unsqueeze(0) return (ret, last,) else: return (None, None) class DeforumCadenceNode: def __init__(self): self.FILM_temp = [] self.model = None self.logger = None @classmethod def INPUT_TYPES(s): return {"required": {"image": ("IMAGE",), "first_image": ("IMAGE",), "deforum_frame_data": ("DEFORUM_FRAME_DATA",), "depth_strength": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.01}), "preview": ("BOOLEAN", {"default":False}), }, "optional": {"hybrid_images": ("IMAGE",),} } RETURN_TYPES = ("IMAGE", "IMAGE") FUNCTION = "fn" display_name = "Cadence Interpolation" CATEGORY = "deforum/interpolation" @classmethod def IS_CHANGED(self, *args, **kwargs): # Force re-evaluation of the node return float("NaN") def interpolate(self, image, first_image, deforum_frame_data, depth_strength, preview=False, dry_run=False, hybrid_images=None): self.skip_return = False hybrid_provider = None #global deforum_depth_algo, deforum_models # import turbo_prev_image, turbo_next_image, turbo_next_frame_idx, turbo_prev_frame_idx return_frames = [] if not dry_run: pil_image = tensor2pil(image.clone().detach()) # Convert PIL image to RGB NumPy array and cast to np.float32 np_image = np.array(pil_image.convert("RGB")).astype(np.uint8) # Convert from RGB to BGR for OpenCV compatibility #np_image = cv2.cvtColor(np_image, cv2.COLOR_RGB2BGR) np_image = cv2.normalize(np_image, None, alpha=0, beta=255, norm_type=cv2.NORM_MINMAX) np_image = np_image.astype(np.uint8) else: np_image = None args = deforum_frame_data["args"] anim_args = deforum_frame_data["anim_args"] predict_depths = ( anim_args.animation_mode == '3D' and anim_args.use_depth_warping) or anim_args.save_depth_maps predict_depths = predict_depths or ( anim_args.hybrid_composite and anim_args.hybrid_comp_mask_type in ['Depth', 'Video Depth']) if "depth_model" not in gs.deforum_models or gs.deforum_depth_algo != anim_args.depth_algorithm: self.vram_state = "high" if "depth_model" in gs.deforum_models: try: gs.deforum_models["depth_model"].to("cpu") except: pass del gs.deforum_models["depth_model"] deforum_depth_algo = anim_args.depth_algorithm if predict_depths: keep_in_vram = True if self.vram_state == 'high' else False # device = ('cpu' if cmd_opts.lowvram or cmd_opts.medvram else self.root.device) # TODO Set device in root in webui device = 'cuda' gs.deforum_models["depth_model"] = DepthModel("models/other", device, keep_in_vram=keep_in_vram, depth_algorithm=anim_args.depth_algorithm, Width=args.width, Height=args.height, midas_weight=anim_args.midas_weight) # depth-based hybrid composite mask requires saved depth maps if anim_args.hybrid_composite != 'None' and anim_args.hybrid_comp_mask_type == 'Depth': anim_args.save_depth_maps = True else: gs.deforum_models["depth_model"] = None anim_args.save_depth_maps = False if gs.deforum_models["depth_model"] != None and not predict_depths: gs.deforum_models["depth_model"] = None if gs.deforum_models["depth_model"] is not None: gs.deforum_models["depth_model"].to('cuda') if "raft_model" not in gs.deforum_models: gs.deforum_models["raft_model"] = RAFT() first_gen = False if deforum_frame_data.get("reset") or not hasattr(self, "interpolator"): self.interpolator = CadenceInterpolator() #deforum_frame_data["frame_idx"] += anim_args.diffusion_cadence first_gen = True # if self.interpolator.turbo_next_image is not None: self.interpolator.turbo_prev_image, self.interpolator.turbo_prev_frame_idx = self.interpolator.turbo_next_image, self.interpolator.turbo_next_frame_idx self.interpolator.turbo_next_image, self.interpolator.turbo_next_frame_idx = np_image, deforum_frame_data["frame_idx"] if self.interpolator.turbo_next_frame_idx == 0 and first_image is not None: pil_image = tensor2pil(first_image.clone().detach()) # Convert PIL image to RGB NumPy array and cast to np.float32 np_image = np.array(pil_image.convert("RGB")).astype(np.uint8) # Convert from RGB to BGR for OpenCV compatibility #np_image = cv2.cvtColor(np_image, cv2.COLOR_RGB2BGR) np_image = cv2.normalize(np_image, None, alpha=0, beta=255, norm_type=cv2.NORM_MINMAX) self.interpolator.turbo_prev_image = np_image.astype(np.uint8) self.interpolator.turbo_prev_frame_idx = 0 self.interpolator.turbo_next_frame_idx = anim_args.diffusion_cadence deforum_frame_data["frame_idx"] = anim_args.diffusion_cadence self.skip_return = True if hybrid_images is not None: # try: hybrid_provider = [] for i in hybrid_images: pil_image = tensor2pil(i.clone().detach()) np_image = np.array(pil_image.convert("RGB")).astype(np.uint8) np_image = cv2.normalize(np_image, None, alpha=0, beta=255, norm_type=cv2.NORM_MINMAX) hybrid_provider.append(np_image) if len(hybrid_provider) > 0: if len(hybrid_provider) - 1 > anim_args.diffusion_cadence: anim_args.diffusion_cadence = len(hybrid_provider) - 1 # if first_gen: # self.interpolator.turbo_prev_image = np_image # return image # with torch.inference_mode(): if not dry_run: with torch.no_grad(): if not self.logger: import comfy self.logger = comfy.utils.ProgressBar(anim_args.diffusion_cadence) # from ..modules.standalone_cadence import new_standalone_cadence frames = self.interpolator.new_standalone_cadence(deforum_frame_data["args"], deforum_frame_data["anim_args"], deforum_frame_data["root"], deforum_frame_data["keys"], deforum_frame_data["frame_idx"], gs.deforum_models["depth_model"], gs.deforum_models["raft_model"], depth_strength, self.logger if preview else None, hybrid_provider=hybrid_provider) # for frame in frames: # tensor = pil2tensor(frame) # return_frames.append(tensor.squeeze(0)) # print(f"[deforum] [rbn] Cadence: {len(return_frames)} frames") # if len(return_frames) > 0: # return_frames = torch.stack(return_frames, dim=0) return frames # else: # return None else: return None def fn(self, image, first_image, deforum_frame_data, depth_strength, preview, hybrid_images=None): result = [] ret = None if image is not None and not deforum_frame_data.get("reset"): device = model_management.get_torch_device() free_memory = model_management.get_free_memory(device) if free_memory < 4000000000: print("Low current VRam, offloading all models before executing Cadence") model_management.unload_all_models() # Check if there are multiple images in the batch if image.shape[0] > 1: for img in image: # Ensure img has batch dimension of 1 for interpolation interpolated_frames = self.interpolate(img.unsqueeze(0), first_image, deforum_frame_data, depth_strength, preview=preview, hybrid_images=hybrid_images) # Collect all interpolated frames for f in interpolated_frames: result.append(f) # Stack all results into a single tensor, preserving color channels ret = torch.stack(result, dim=0) else: # Directly interpolate if only one image is present ret = self.interpolate(image, first_image, deforum_frame_data, depth_strength, preview=preview, hybrid_images=hybrid_images) #ret = torch.stack([torch.from_numpy(i) / 255.0 for i in ret], dim=0) if ret is not None: #last = ret[-1].unsqueeze(0) # Preserve the last frame separately with batch dimension last = torch.from_numpy(ret[-1]).unsqueeze(0) / 255.0 # Preserve the last frame separately with batch dimension if self.skip_return: return (None, last) else: return (ret, last,) else: _ = self.interpolate(None, None, deforum_frame_data, depth_strength, dry_run=True) return (None, None,) ================================================ FILE: deforum_nodes/nodes/deforum_iteration_nodes.py ================================================ import math import secrets from types import SimpleNamespace import torch from deforum import ImageRNGNoise from deforum.generators.rng_noise_generator import slerp from deforum.pipeline_utils import next_seed from deforum.pipelines.deforum_animation.animation_params import RootArgs, DeforumArgs, DeforumAnimArgs, \ DeforumOutputArgs, LoopArgs from deforum.utils.string_utils import split_weighted_subprompts import comfy from ..modules.deforum_comfyui_helpers import get_current_keys, generate_seed_list class DeforumIteratorNode: def __init__(self): self.first_run = True self.frame_index = 0 self.seed = "" self.seeds = [] self.second_run = True self.logger = None @classmethod def IS_CHANGED(cls, *args, **kwargs): # Force re-evaluation of the node # if autorefresh == "Yes": return float("NaN") @classmethod def INPUT_TYPES(cls): return { "required": { "deforum_data": ("deforum_data",), "latent_type": (["stable_diffusion", "stable_cascade", "sd3"],) }, "optional": { "latent": ("LATENT",), "init_latent": ("LATENT",), "seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff}), "subseed": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff}), "subseed_strength": ("FLOAT", {"default": 0.8, "min": 0, "max": 1.0}), "slerp_strength": ("FLOAT", {"default": 0.1, "min": 0, "max": 1.0}), "reset_counter":("BOOLEAN", {"default": False},), "reset_latent":("BOOLEAN", {"default": False},), "enable_autoqueue":("BOOLEAN", {"default": False},), } } RETURN_TYPES = (("DEFORUM_FRAME_DATA", "LATENT", "STRING", "STRING")) RETURN_NAMES = (("deforum_frame_data", "latent", "positive_prompt", "negative_prompt")) FUNCTION = "get" OUTPUT_NODE = True CATEGORY = f"deforum/logic" display_name = "Iterator Node" @torch.inference_mode() def get(self, deforum_data, latent_type, latent=None, init_latent=None, seed=None, subseed=None, subseed_strength=None, slerp_strength=None, reset_counter=False, reset_latent=False, enable_autoqueue=False, *args, **kwargs): from ..mapping import gs if gs.reset: reset_counter = True reset_latent = True # global deforum_cache root_dict = RootArgs() args_dict = {key: value["value"] for key, value in DeforumArgs().items()} anim_args_dict = {key: value["value"] for key, value in DeforumAnimArgs().items()} output_args_dict = {key: value["value"] for key, value in DeforumOutputArgs().items()} loop_args_dict = {key: value["value"] for key, value in LoopArgs().items()} root = SimpleNamespace(**root_dict) args = SimpleNamespace(**args_dict) anim_args = SimpleNamespace(**anim_args_dict) anim_args.diffusion_cadence = 1 video_args = SimpleNamespace(**output_args_dict) parseq_args = None loop_args = SimpleNamespace(**loop_args_dict) controlnet_args = SimpleNamespace(**{"controlnet_args": "None"}) for key, value in args.__dict__.items(): if key in deforum_data: if deforum_data[key] == "": val = None else: val = deforum_data[key] setattr(args, key, val) for key, value in anim_args.__dict__.items(): if key in deforum_data: if deforum_data[key] == "" and "schedule" not in key: val = None else: val = deforum_data[key] setattr(anim_args, key, val) for key, value in video_args.__dict__.items(): if key in deforum_data: if deforum_data[key] == "" and "schedule" not in key: val = None else: val = deforum_data[key] setattr(anim_args, key, val) for key, value in root.__dict__.items(): if key in deforum_data: if deforum_data[key] == "": val = None else: val = deforum_data[key] setattr(root, key, val) for key, value in loop_args.__dict__.items(): if key in deforum_data: if deforum_data[key] == "": val = None else: val = deforum_data[key] setattr(loop_args, key, val) root.animation_prompts = deforum_data.get("prompts") anim_args.fps = 24 keys, prompt_series, areas = get_current_keys(anim_args, args.seed, root, area_prompts=deforum_data.get("area_prompts")) if self.frame_index >= anim_args.max_frames or reset_counter: # if self.logger: # self.logger.stop_live_display() # config = { # "Header": {"type": "full", "columns": ["DEFORUM COMFY ANIMATOR - Logging"]}, # "Status": {"type": "columns", "columns": ["Frame", "Progress", "Errors"]} # } # self.logger = TerminalTableLogger(config) # self.logger.start_live_display() self.reset_counter = False self.frame_index = 0 self.first_run = True self.second_run = True if not self.logger: self.logger = comfy.utils.ProgressBar(anim_args.max_frames) self.logger.update_absolute(self.frame_index) # else: args.scale = keys.cfg_scale_schedule_series[self.frame_index] if prompt_series is not None: args.prompt = prompt_series[self.frame_index] seed = keys.seed_schedule_series[self.frame_index] if args.seed_behavior == 'random': seed = secrets.randbelow(2 ** 32 - 1) self.seed = seed args.seed = seed self.seeds.append(seed) # args.seed = int(args.seed) # root.seed_internal = int(root.seed_internal) # args.seed_iter_N = int(args.seed_iter_N) # # if self.seed == "": # self.seed = args.seed # self.seed_internal = root.seed_internal # self.seed_iter_N = args.seed_iter_N # # self.seed = next_seed(args, root) # args.seed = self.seed # self.seeds.append(self.seed) blend_value = 0.0 next_frame = self.frame_index + anim_args.diffusion_cadence next_prompt = None def generate_blend_values(distance_to_next_prompt, blend_type="linear"): if blend_type == "linear": return [i / distance_to_next_prompt for i in range(distance_to_next_prompt + 1)] elif blend_type == "exponential": base = 2 return [1 / (1 + math.exp(-8 * (i / distance_to_next_prompt - 0.5))) for i in range(distance_to_next_prompt + 1)] else: raise ValueError(f"Unknown blend type: {blend_type}") def find_last_prompt_change(current_index, prompt_series): # Step backward from the current position for i in range(current_index - 1, -1, -1): if prompt_series[i] != prompt_series[current_index]: return i return 0 # default to the start if no change found def find_next_prompt_change(current_index, prompt_series): # Step forward from the current position for i in range(current_index + 1, len(prompt_series) - 1): if i < anim_args.max_frames: if prompt_series[i] != prompt_series[current_index]: return i return len(prompt_series) - 1 # default to the end if no change found if prompt_series is not None: last_prompt_change = find_last_prompt_change(self.frame_index, prompt_series) next_prompt_change = find_next_prompt_change(self.frame_index, prompt_series) distance_between_changes = next_prompt_change - last_prompt_change current_distance_from_last = self.frame_index - last_prompt_change # Generate blend values for the distance between prompt changes blend_values = generate_blend_values(distance_between_changes, blend_type="exponential") # Fetch the blend value based on the current frame's distance from the last prompt change blend_value = blend_values[current_distance_from_last] if len(prompt_series) - 1 > next_prompt_change: next_prompt = prompt_series[next_prompt_change] gen_args = self.get_current_frame(args, anim_args, root, keys, self.frame_index, areas) self.args = args self.root = root if next_prompt is not None: gen_args["next_prompt"], _ = split_weighted_subprompts(str(next_prompt)) gen_args["prompt_blend"] = blend_value gen_args["frame_index"] = self.frame_index gen_args["max_frames"] = anim_args.max_frames seeds = generate_seed_list(anim_args.max_frames + 1, args.seed_behavior, seed, args.seed_iter_N) subseeds = generate_seed_list(anim_args.max_frames + 1, args.seed_behavior, subseed, args.seed_iter_N) if reset_counter: print("[deforum] RESET COUNTER") if reset_latent or not hasattr(self, "rng"): print("[deforum] RESET LATENT" ) if "image" in gs.deforum_cache: gs.deforum_cache["image"].clear() if "latent" in gs.deforum_cache: gs.deforum_cache["latent"].clear() gs.reset = True if latent_type == "stable_diffusion": channels = 4 compression = 8 else: channels = 16 compression = 42 if init_latent is not None: args.height, args.width = init_latent["samples"].shape[2] * 8, init_latent["samples"].shape[3] * 8 self.rng = ImageRNGNoise((channels, args.height // compression, args.width // compression), [seeds[self.frame_index]], [subseeds[self.frame_index]], 0.6, 1024, 1024) if latent_type == "stable_diffusion": l = self.rng.first().half().to(comfy.model_management.intermediate_device()) elif latent_type == "stable_cascade": l = torch.zeros([1, 16, args.height // 42, args.width // 42]).to(comfy.model_management.intermediate_device()) elif latent_type == "sd3": l = torch.ones([1, 16, args.height // 8, args.width // 8], device=comfy.model_management.intermediate_device()) * 0.0609 latent = {"samples": l} gen_args["denoise"] = 1.0 else: if latent_type == "stable_diffusion" and slerp_strength > 0: args.height, args.width = latent["samples"].shape[2] * 8, latent["samples"].shape[3] * 8 l = self.rng.next().clone().to(comfy.model_management.intermediate_device()) s = latent["samples"].clone().to(comfy.model_management.intermediate_device()) latent = {"samples":slerp(slerp_strength, s, l)} print(f"[deforum] Frame: {self.frame_index} of {anim_args.max_frames}") gen_args["noise"] = self.rng gen_args["seed"] = int(seed) if self.frame_index == 0 and init_latent is not None: latent = init_latent gen_args["denoise"] = keys.strength_schedule_series[0] #if anim_args.diffusion_cadence > 1: # global turbo_prev_img, turbo_prev_frame_idx, turbo_next_image, turbo_next_frame_idx, opencv_image if anim_args.diffusion_cadence > 1: self.frame_index += anim_args.diffusion_cadence if not self.first_run else 0# if anim_args.diffusion_cadence == 1 if not self.first_run: if self.second_run: self.frame_index = 0 self.second_run = False # if turbo_steps > 1: # turbo_prev_image, turbo_prev_frame_idx = turbo_next_image, turbo_next_frame_idx # turbo_next_image, turbo_next_frame_idx = opencv_image, self.frame_index # frame_idx += turbo_steps self.first_run = False else: self.frame_index += 1 if not self.first_run else 0 self.first_run = False self.second_run = False if self.frame_index > anim_args.max_frames: self.frame_index = anim_args.max_frames if latent is not None: latent["samples"] = latent["samples"].float() from ..mapping import gs gs.reset = False if not self.first_run else True enable_autoqueue = enable_autoqueue if self.frame_index == 0 else False gen_args["sampler_name"] = deforum_data.get("sampler_name", "euler_a") gen_args["scheduler"] = deforum_data.get("scheduler", "normal") gen_args["reset"] = reset_latent or reset_counter gen_args["frame_idx"] = self.frame_index gen_args["first_run"] = self.first_run gen_args["second_run"] = self.second_run gen_args["logger"] = self.logger torch.cuda.synchronize() return {"ui": {"counter":(self.frame_index,), "max_frames":(anim_args.max_frames,), "enable_autoqueue":(enable_autoqueue,)}, "result": (gen_args, latent, gen_args["prompt"], gen_args["negative_prompt"],),} # return (gen_args, latent, gen_args["prompt"], gen_args["negative_prompt"],) def get_current_frame(self, args, anim_args, root, keys, frame_idx, areas=None): if hasattr(args, 'prompt'): prompt, negative_prompt = split_weighted_subprompts(args.prompt, frame_idx, anim_args.max_frames) else: prompt = "" negative_prompt = "" strength = keys.strength_schedule_series[frame_idx] return {"prompt": prompt, "negative_prompt": negative_prompt, "denoise": strength, "cfg": args.scale, "steps": int(keys.steps_schedule_series[self.frame_index]), "root": root, "keys": keys, "frame_idx": frame_idx, "anim_args": anim_args, "args": args, "areas":areas[frame_idx] if areas is not None else None, "logger":self.logger} class DeforumSeedNode: @classmethod def IS_CHANGED(cls, *args, **kwargs): return float("NaN") @classmethod def INPUT_TYPES(cls): return { "required": { "seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff}), }, } FUNCTION = "get" OUTPUT_NODE = True CATEGORY = f"deforum/logic" RETURN_TYPES = (("INT",)) display_name = "Seed Node" @torch.inference_mode() def get(self, seed, *args, **kwargs): return (seed,) class DeforumBigBoneResetNode: @classmethod def IS_CHANGED(cls, text, autorefresh): # Force re-evaluation of the node # if autorefresh == "Yes": return float("NaN") @classmethod def INPUT_TYPES(cls): return { "required": { "reset_deforum":("BOOLEAN", {"default": False},),}, } FUNCTION = "get" CATEGORY = f"deforum/logic" RETURN_TYPES = (("BOOLEAN",)) display_name = "Big Bone Reset Node" def get(self, reset_deforum, *args, **kwargs): from ..mapping import gs gs.reset = reset_deforum # deforum_frame_data["reset"] = reset_deforum # deforum_frame_data["reset_latent"] = reset_deforum # deforum_frame_data["reset_counter"] = reset_deforum # deforum_frame_data["first_run"] = reset_deforum return {"ui":{"reset":(reset_deforum,)}, "result":(reset_deforum,)} ================================================ FILE: deforum_nodes/nodes/deforum_legacy_nodes.py ================================================ import os import time from types import SimpleNamespace import torch import numpy as np from deforum import DeforumAnimationPipeline from deforum.pipelines.deforum_animation.animation_helpers import DeforumAnimKeys from deforum.pipelines.deforum_animation.animation_params import RootArgs, DeforumArgs, DeforumAnimArgs, \ DeforumOutputArgs, LoopArgs, ParseqArgs from deforum.utils.string_utils import substitute_placeholders class DeforumSingleSampleNode: @classmethod def INPUT_TYPES(cls): return { "required": { "deforum_data": ("deforum_data",), "model": ("MODEL",), "clip": ("CLIP",), "vae": ("VAE",) }, } RETURN_TYPES = (("IMAGE",)) FUNCTION = "get" OUTPUT_NODE = True CATEGORY = f"deforum/sampling" display_name = "Integrated Pipeline" @torch.inference_mode() def get(self, deforum_data, model, clip, vae, *args, **kwargs): root_dict = RootArgs() args_dict = {key: value["value"] for key, value in DeforumArgs().items()} anim_args_dict = {key: value["value"] for key, value in DeforumAnimArgs().items()} output_args_dict = {key: value["value"] for key, value in DeforumOutputArgs().items()} loop_args_dict = {key: value["value"] for key, value in LoopArgs().items()} parseq_args_dict = {key: value["value"] for key, value in ParseqArgs().items()} root = SimpleNamespace(**root_dict) args = SimpleNamespace(**args_dict) anim_args = SimpleNamespace(**anim_args_dict) video_args = SimpleNamespace(**output_args_dict) parseq_args = SimpleNamespace(**parseq_args_dict) parseq_args.parseq_manifest = "" # #parseq_args = None loop_args = SimpleNamespace(**loop_args_dict) controlnet_args = SimpleNamespace(**{"controlnet_args": "None"}) for key, value in args.__dict__.items(): if key in deforum_data: if deforum_data[key] == "": val = None else: val = deforum_data[key] setattr(args, key, val) for key, value in anim_args.__dict__.items(): if key in deforum_data: if deforum_data[key] == "" and "schedule" not in key: val = None else: val = deforum_data[key] setattr(anim_args, key, val) for key, value in video_args.__dict__.items(): if key in deforum_data: if deforum_data[key] == "" and "schedule" not in key: val = None else: val = deforum_data[key] setattr(anim_args, key, val) for key, value in root.__dict__.items(): if key in deforum_data: if deforum_data[key] == "": val = None else: val = deforum_data[key] setattr(root, key, val) for key, value in loop_args.__dict__.items(): if key in deforum_data: if deforum_data[key] == "": val = None else: val = deforum_data[key] setattr(loop_args, key, val) success = None root.timestring = time.strftime('%Y%m%d%H%M%S') args.timestring = root.timestring args.strength = max(0.0, min(1.0, args.strength)) root.animation_prompts = deforum_data.get("prompts", {}) if not args.use_init and not anim_args.hybrid_use_init_image: args.init_image = None elif anim_args.animation_mode == 'Video Input': args.use_init = True current_arg_list = [args, anim_args, video_args, parseq_args, root] full_base_folder_path = os.path.join(os.getcwd(), "output/deforum") args.batch_name = f"aiNodes_Deforum_{args.timestring}" args.outdir = os.path.join(full_base_folder_path, args.batch_name) root.raw_batch_name = args.batch_name args.batch_name = substitute_placeholders(args.batch_name, current_arg_list, full_base_folder_path) # os.makedirs(args.outdir, exist_ok=True) def generate(*args, **kwargs): from ..modules.deforum_comfy_sampler import sample_deforum image = sample_deforum(model, clip, vae, **kwargs) return image self.deforum = DeforumAnimationPipeline(generate) self.deforum.config_dir = os.path.join(os.getcwd(), "output/_deforum_configs") os.makedirs(self.deforum.config_dir, exist_ok=True) # self.deforum.generate_inpaint = self.generate_inpaint import comfy pbar = comfy.utils.ProgressBar(deforum_data["max_frames"]) def datacallback(data=None): if data: if "image" in data: pbar.update_absolute(data["frame_idx"], deforum_data["max_frames"], ("JPEG", data["image"], 512)) self.deforum.datacallback = datacallback deforum_data["turbo_steps"] = deforum_data.get("diffusion_cadence", 0) deforum_data["store_frames_in_ram"] = True deforum_data["skip_video_creation"] = True animation = self.deforum(**deforum_data) results = [] for i in self.deforum.images: tensor = torch.from_numpy(np.array(i).astype(np.float32) / 255.0) results.append(tensor) result = torch.stack(results, dim=0) return (result,) class DeforumSetVAEDownscaleRatioNode: @classmethod def INPUT_TYPES(s): return {"required": {"vae": ("VAE",), "downscale_ratio": ("INT", {"default": 42, "min": 32, "max": 64, "step": 1}), }, } RETURN_TYPES = ("VAE",) FUNCTION = "fn" display_name = "Set VAE Downscale Ratio" CATEGORY = "deforum/_for_testing" def fn(self, vae, downscale_ratio): vae.downscale_ratio = downscale_ratio return (vae,) ================================================ FILE: deforum_nodes/nodes/deforum_logic_nodes.py ================================================ class DeforumImageSwitcherNode: @classmethod def INPUT_TYPES(cls): return { "required": { "option": ("BOOLEAN", {"default": False}), }, "optional": { "image_true": ("IMAGE",), "image_false": ("IMAGE",), } } @classmethod def IS_CHANGED(cls, text, autorefresh): # Force re-evaluation of the node if autorefresh == "Yes": return float("NaN") RETURN_TYPES = ("IMAGE",) FUNCTION = "compare" display_name = "Image Switcher" CATEGORY = "deforum/logic" OUTPUT_NODE = True def compare(self, option=True, image_true=None, image_false=None): if option: return (image_true,) else: return (image_false,) class DeforumComparatorNode: @classmethod def INPUT_TYPES(cls): return { "required": { "int_1": ("INT",{"default":0, "min":-999999, "max":2 ** 32, "step":1}), "int_2": ("INT",{"default":0, "min":-999999, "max":2 ** 32, "step":1}), "condition": (["<", "<=", ">", ">=", "==" ],), } } @classmethod def IS_CHANGED(cls, text, autorefresh): # Force re-evaluation of the node if autorefresh == "Yes": return float("NaN") RETURN_TYPES = ("BOOLEAN",) FUNCTION = "compare" display_name = "INT Comparator" CATEGORY = "deforum/logic" OUTPUT_NODE = True def compare(self, int_1, int_2, condition): if condition == "<": return (int_1 < int_2,) elif condition == "<=": return (int_1 <= int_2,) elif condition == ">": return (int_1 > int_2,) elif condition == ">=": return (int_1 >= int_2,) elif condition == "==": return (int_1 == int_2,) else: raise ValueError("Invalid condition") class DeforumFloatComparatorNode: @classmethod def INPUT_TYPES(cls): return { "required": { "float_1": ("FLOAT",{"default":0.00, "min":-999999.00, "max":2 ** 32, "step":0.01}), "float_2": ("FLOAT",{"default":0.00, "min":-999999.00, "max":2 ** 32, "step":0.01}), "condition": (["<", "<=", ">", ">=", "==" ],), } } @classmethod def IS_CHANGED(cls, text, autorefresh): # Force re-evaluation of the node if autorefresh == "Yes": return float("NaN") RETURN_TYPES = ("BOOLEAN",) FUNCTION = "compare" display_name = "FLOAT Comparator" CATEGORY = "deforum/logic" OUTPUT_NODE = True def compare(self, float_1, float_2, condition): if condition == "<": return (float_1 < float_2,) elif condition == "<=": return (float_1 <= float_2,) elif condition == ">": return (float_1 > float_2,) elif condition == ">=": return (float_1 >= float_2,) elif condition == "==": return (float_1 == float_2,) else: raise ValueError("Invalid condition") class DeforumAndNode: @classmethod def INPUT_TYPES(cls): return { "required": { "condition_1": ("BOOLEAN",), "condition_2": ("BOOLEAN",), # Add more conditions if needed } } RETURN_TYPES = ("BOOLEAN",) FUNCTION = "logical_and" display_name = "Logical AND" CATEGORY = f"deforum/logic" def logical_and(self, condition_1, condition_2, *additional_conditions): return (all([condition_1, condition_2] + list(additional_conditions)),) class DeforumOrNode: @classmethod def INPUT_TYPES(cls): return { "required": { "condition_1": ("BOOLEAN",), "condition_2": ("BOOLEAN",), # Add more conditions if needed } } RETURN_TYPES = ("BOOLEAN",) FUNCTION = "logical_or" display_name = "Logical OR" CATEGORY = f"deforum/logic" def logical_or(self, condition_1, condition_2, *additional_conditions): return (any([condition_1, condition_2] + list(additional_conditions)),) class DeforumNotNode: @classmethod def INPUT_TYPES(cls): return { "required": { "condition": ("BOOLEAN",), } } RETURN_TYPES = ("BOOLEAN",) FUNCTION = "logical_not" display_name = "Logical NOT" CATEGORY = f"deforum/logic" def logical_not(self, condition): return (not condition,) ================================================ FILE: deforum_nodes/nodes/deforum_noise_nodes.py ================================================ import secrets import torch import numpy as np class AddCustomNoiseNode: """ A node to add various types of noise to an image using PyTorch. """ @classmethod def IS_CHANGED(cls, text, autorefresh): # Force re-evaluation of the node if autorefresh == "Yes": return float("NaN") @classmethod def INPUT_TYPES(cls): return { "required": { "images": ("IMAGE",), "noise_type": ( ["speckle", "uniform", "rayleigh", "exponential", "gamma", "random_valued_impulse", "laplace", "perlin", "brownian", "quantization", "shot", "multiplicative", "flicker", "fractal", "cellular", "gaussian", "thermal", "salt_pepper", "poisson",],), #TODO ["blue", "anisotropic"] "amount": ("FLOAT", { "default": 0.1, "min": 0.0, "max": 100.0, "step": 0.01, "display": "number" }), }, "optional": { "seed": ("INT", { "default": None, "min": 0, "max": 2 ** 32 - 1, "step": 1, "display": "number" }), "temperature_map": ("IMAGE",), "mean": ("FLOAT", { "default": 0.0, "min": -1.0, "max": 1.0, "step": 0.01, "display": "number" }), "std": ("FLOAT", { "default": 0.1, "min": 0.01, "max": 1.0, "step": 0.01, "display": "number" }), "prob": ("FLOAT", { "default": 0.05, "min": 0.0, "max": 1.0, "step": 0.01, "display": "slider" }), "scale": ("FLOAT", { "default": 0.1, "min": 0.01, "max": 10.0, "step": 0.01, "display": "number" }), "temp_scale": ("FLOAT", { "default": 1.0, "min": 0.1, "max": 10.0, "step": 0.01, "display": "number" }), "scale_factor": ("FLOAT", { "default": 1.0, "min": 0.1, "max": 10.0, "step": 0.01, "display": "number" }), "location": ("FLOAT", { "default": 0.0, "min": -10.0, "max": 10.0, "step": 0.01, "display": "number" }), "res_x": ("INT", { "default": 10, "min": 1, "max": 100, "step": 1, "display": "number" }), "res_y": ("INT", { "default": 10, "min": 1, "max": 100, "step": 1, "display": "number" }), "octaves": ("INT", { "default": 5, "min": 1, "max": 10, "step": 1, "display": "number" }), "persistence": ("FLOAT", { "default": 0.5, "min": 0.01, "max": 1.0, "step": 0.01, "display": "slider" }), "num_points": ("INT", { "default": 100, "min": 10, "max": 1000, "step": 10, "display": "number" }), "direction": ("FLOAT", { "default": 1.0, "min": 0.00, "max": 1.0, "step": 0.01, "display": "slider" }), } } RETURN_TYPES = ("IMAGE","IMAGE",) RETURN_NAMES = ("NOISED_IMAGE","NOISE",) FUNCTION = "add_noise" display_name = "Add Custom Noise" CATEGORY = "deforum/noise" def add_noise(self, images, noise_type, amount, seed=None, temperature_map=None, **kwargs): image, noise = add_noise_torch(images.clone(), noise_type, seed, amount, temperature_map, **kwargs) return (image,noise,) def add_noise_torch(images, noise_type='gaussian', seed=None, amount=0.1, temperature_map=None, **kwargs): """ Add various types of noise to an image tensor using PyTorch. Parameters: images (torch.Tensor): The input images tensor of shape (B, C, H, W). noise_type (str): Type of noise to add. Supports 'gaussian', 'salt_pepper', 'poisson', 'speckle', 'uniform', 'rayleigh', 'exponential', 'gamma', 'random_valued_impulse'. seed (int): Seed value for randomness. Default is None. amount (float): General parameter to control noise amount, interpretation depends on noise_type. **kwargs: Additional noise-specific parameters. Returns: torch.Tensor: The noisy images tensor of the same shape as input. """ B, C, H, W = images.shape noise = images if seed is not None: torch.manual_seed(seed) else: seed = secrets.randbelow(2 ** 32) if noise_type == 'gaussian': mean = kwargs.get('mean', 0.0) std = kwargs.get('std', 0.1) noisy_images = images + amount * torch.randn_like(images) * std + mean elif noise_type == 'thermal': if temperature_map is None: raise ValueError("Temperature map must be provided for thermal noise type.") # Normalize temperature map to have a meaningful scale for noise temp_min = temperature_map.min() temp_max = temperature_map.max() normalized_temp_map = (temperature_map - temp_min) / (temp_max - temp_min) # Scale temperature map to control the amount of thermal noise temp_scale = kwargs.get('temp_scale', 1.0) # Scale factor for temperature effect std_map = amount * normalized_temp_map * temp_scale # Generate thermal noise based on temperature map noise = torch.randn_like(images) * std_map.unsqueeze( 1) # Ensure std_map matches the channel dimension of images noisy_images = images + noise elif noise_type == 'salt_pepper': prob = kwargs.get('prob', 0.05) # Probability for both salt and pepper mask = torch.rand_like(images) < prob images[mask] = torch.rand(1).item() * mask[mask] # Salt mask = torch.rand_like(images) < prob images[mask] = 0 # Pepper noisy_images = images elif noise_type == 'poisson': noisy_images = torch.poisson(images * amount) / amount elif noise_type == 'speckle': noise = torch.randn_like(images) noisy_images = images + images * noise * amount elif noise_type == 'uniform': noise = torch.rand_like(images) noisy_images = images + amount * (noise - 0.5) elif noise_type == 'rayleigh': scale = kwargs.get('scale', 0.1) noise = torch.sqrt(-2 * scale * torch.log(1 - torch.rand_like(images))) noisy_images = images + amount * noise elif noise_type == 'exponential': scale = kwargs.get('scale', 0.1) noise = torch.distributions.Exponential(scale).sample(images.shape) noisy_images = images + amount * noise noise = torch.clamp(1 / noise, 0, 1.0) * amount elif noise_type == 'gamma': shape = kwargs.get('shape', 2.0) scale = kwargs.get('scale', 0.1) noise = torch.distributions.Gamma(shape, scale).sample(images.shape) noisy_images = images + amount * noise noise = torch.clamp(1 / noise, 0, 1.0) * amount elif noise_type == 'random_valued_impulse': prob = kwargs.get('prob', 0.05) mask = torch.rand_like(images) < prob # Fix: Use logical NOT operator on the mask inverted_mask = ~mask noise = torch.rand_like(images) * mask noisy_images = images * inverted_mask.float() + noise noise = (torch.clamp(noise * 255.0, 0, 1.0) * mask) * amount elif noise_type == 'laplace': location = kwargs.get('location', 0.0) scale = kwargs.get('scale', 0.1) noise = torch.distributions.Laplace(location, scale).sample(images.shape) noisy_images = images + amount * noise noise = torch.clamp(noise * 255.0, 0, 1.0) * amount elif noise_type == 'perlin': # Hypothetical function call - you need to implement or integrate a Perlin noise generator. noise = generate_perlin_noise(B, C, H, W, **kwargs) noisy_images = images + amount * noise elif noise_type == 'brownian': scale = kwargs.get('scale', 0.1) noise = generate_brownian_noise(images.shape, scale, seed) noisy_images = images + amount * noise / torch.max(noise) elif noise_type == 'quantization': # Quantization levels levels = kwargs.get('levels', 256) # Default is 256 levels for an 8-bit image # Quantize the image max_val = images.max() quantized_images = torch.round(images * (levels - 1) / max_val) * max_val / (levels - 1) # Calculate quantization noise as the difference between original and quantized images noise = images - quantized_images # Optionally scale the noise noisy_images = images + amount * noise noise = torch.clamp(noise, 0, 1.0) elif noise_type == 'shot': # Scale images to simulate intensity as photon counts scale_factor = kwargs.get('scale_factor', 1.0) # Scale factor to adjust the intensity scaled_images = images * scale_factor # Apply Poisson distribution to simulate shot noise # Since Poisson function expects lambda > 0, ensure scaled_images is positive noisy_images = torch.poisson(scaled_images) / scale_factor elif noise_type == 'multiplicative': # Amount parameter controls the variance of the multiplicative noise mean = kwargs.get('mean', 1.0) # Multiplicative noise mean. Typically, it should be around 1. std = kwargs.get('std', 0.1) # Standard deviation for the multiplicative noise # Generate multiplicative noise noise = torch.randn_like(images) * std + mean # Apply the multiplicative noise noisy_images = images * noise elif noise_type == 'blue': # Generate approximate Blue Noise noise = generate_approx_blue_noise((B, C, H, W), seed=seed, **kwargs) # Apply the Blue Noise pattern to the images # The amount parameter controls the visibility of the blue noise noisy_images = images + amount * noise elif noise_type == 'flicker': # Create frequency domain noise # Create frequency domain noise real_part = torch.randn((B, C, H, W // 2 + 1), device=images.device, generator=torch.manual_seed(seed)) imag_part = torch.randn((B, C, H, W // 2 + 1), device=images.device, generator=torch.manual_seed(seed)) freq_domain_noise = real_part + 1j * imag_part # Correctly compute frequencies for 1/f scaling frequencies_x = torch.fft.rfftfreq(W, d=1 / W).to(images.device) frequencies_y = torch.fft.fftfreq(H, d=1 / H).to(images.device) # Use broadcasting to create a meshgrid of frequencies freq_mesh_x = frequencies_x[None, None, None, :] # Add dimensions for B, C, and H for broadcasting freq_mesh_y = frequencies_y[None, None, :, None] # Add dimensions for B, C, and W for broadcasting # Calculate the hypotenuse of the frequency meshgrid to apply 1/f scaling freq_mesh = torch.sqrt(freq_mesh_x ** 2 + freq_mesh_y ** 2) # Avoid division by zero by adding a small value, no need to expand as broadcasting handles it freq_domain_noise /= (freq_mesh + 1e-5) # Convert back to the spatial domain noise = torch.fft.irfft(freq_domain_noise, n=W, dim=-1) # Apply noise to images noisy_images = images + amount * noise elif noise_type == 'anisotropic': # Generate base noise base_noise = torch.randn_like(images) # Apply a directional gradient dir_min = kwargs.get('dir_min', 0.0) dir_max = kwargs.get('dir_min', 1.0) direction = torch.tensor([dir_max, dir_min]) gradient = torch.outer(torch.arange(H), direction[0]) + torch.outer(torch.arange(W), direction[1]) gradient = gradient.unsqueeze(0).unsqueeze(0).to(images.device) # Apply directional bias to the noise noise = base_noise * gradient noisy_images = images + amount * noise elif noise_type == 'fractal': res_x = kwargs.get('res_x', 10) # Base resolution for Perlin noise res_y = kwargs.get('res_y', 10) octaves = kwargs.get('octaves', 5) # Number of noise layers persistence = kwargs.get('persistence', 0.5) # Amplitude decay per layer noise = generate_fractal_noise(B, C, H, W, (res_x, res_y), octaves, persistence) noisy_images = images + amount * noise elif noise_type == 'cellular': num_points = kwargs.get('num_points', 100) # Default number of points cellular_noise = generate_cellular_noise(H, W, num_points=num_points) # Expand Cellular noise to match input dimensions noise = cellular_noise.unsqueeze(0).unsqueeze(0).repeat(B, C, 1, 1) noisy_images = images + amount * noise else: raise ValueError(f"Unsupported noise type: {noise_type}") return torch.clamp(noisy_images, 0, 1), noise def generate_perlin_noise_2d(shape, res, seed=None): """ Generate a 2D numpy array of Perlin noise. Args: shape: The shape of the generated array (height, width). res: The resolution of the noise in terms of number of cells in each dimension. Returns: A 2D numpy array of Perlin noise. """ if seed is not None: np.random.seed(seed) # Set the seed if provided def interpolate(t): return t * t * t * (t * (t * 6 - 15) + 10) def gradient(h, x, y): vectors = np.array([[0,1],[0,-1],[1,0],[-1,0]]) g = vectors[h % 4] return g[:, :, 0] * x + g[:, :, 1] * y grid_x, grid_y = np.meshgrid(np.arange(shape[1]), np.arange(shape[0])) # Rescale grid to fit res grid_x = grid_x * res[0] / shape[1] grid_y = grid_y * res[1] / shape[0] # Determine grid cell coordinates x0 = grid_x.astype(int) y0 = grid_y.astype(int) # Relative x and y coordinates within each cell x_rel = grid_x - x0 y_rel = grid_y - y0 # Random gradients np.random.seed(0) # Optional: for reproducibility gradients = np.random.randint(0, 4, (res[0] + 1, res[1] + 1)) # Compute the dot-product n0 = gradient(gradients[y0, x0], x_rel, y_rel) n1 = gradient(gradients[y0, x0 + 1], x_rel - 1, y_rel) ix0 = interpolate(n0) + (interpolate(n1) - interpolate(n0)) * interpolate(x_rel) n0 = gradient(gradients[y0 + 1, x0], x_rel, y_rel - 1) n1 = gradient(gradients[y0 + 1, x0 + 1], x_rel - 1, y_rel - 1) ix1 = interpolate(n0) + (interpolate(n1) - interpolate(n0)) * interpolate(x_rel) value = ix0 + (ix1 - ix0) * interpolate(y_rel) return (value - np.min(value)) / (np.max(value) - np.min(value)) def generate_perlin_noise(B, C, H, W, res_x, res_y, seed=None, *args, **kwargs): """ Generates Perlin noise for a batch of images in PyTorch. Args: B, C, H, W: Dimensions of the output tensor. res_x, res_y: Resolution of the Perlin noise grid. Returns: A PyTorch tensor of shape (B, C, H, W) containing Perlin noise. """ noise = np.zeros((B, C, H, W)) for b in range(B): for c in range(C): noise[b, c] = generate_perlin_noise_2d((H, W), (res_x, res_y), seed=seed) return torch.FloatTensor(noise) def generate_brownian_noise(shape, scale=1.0, seed=None): """ Generate Brownian (Red) noise for a given tensor shape. Parameters: - shape (tuple): The shape of the output tensor (e.g., (B, C, H, W)). - scale (float): Scaling factor for noise intensity. Default is 1.0. - seed (int): Optional seed for reproducibility. Default is None. Returns: - torch.Tensor: Tensor containing Brownian noise of the specified shape. """ if seed is not None: torch.manual_seed(seed) # Initialize a tensor of random noise random_noise = torch.randn(shape) # Cumulatively sum the noise along the last dimension to simulate Brownian motion brownian_noise = torch.cumsum(random_noise, dim=-1) * scale # Normalize the noise to have values between 0 and 1 brownian_noise -= brownian_noise.min() brownian_noise /= brownian_noise.max() return brownian_noise def generate_approx_blue_noise(shape, seed=None, min_dist=1.0, sample_fraction=0.1, *args, **kwargs): """ Generates an approximate Blue Noise pattern using a simplified, naive approach. Note: This is computationally expensive and not optimized for performance or quality. Parameters: shape (tuple): Shape of the output tensor (B, C, H, W). seed (int): Random seed for reproducibility. min_dist (float): Minimum distance between points. Controls the density. sample_fraction (float): Fraction of points to sample, lower means sparser. Returns: torch.Tensor: Tensor containing an approximate Blue Noise pattern. """ if seed is not None: torch.manual_seed(seed) _, _, H, W = shape pattern = torch.zeros((H, W)) # Number of points to sample num_points = int(H * W * sample_fraction) # Randomly sample points with minimum distance for _ in range(num_points): x, y = torch.randint(0, H, (1,)), torch.randint(0, W, (1,)) # Check if any existing point is within min_dist y_grid, x_grid = torch.meshgrid(torch.arange(H), torch.arange(W), indexing='ij') distances = torch.sqrt((x_grid - x) ** 2 + (y_grid - y) ** 2) if torch.min(distances * (pattern > 0).float()) >= min_dist or torch.sum(pattern) == 0: pattern[x, y] = 1 # Convert the pattern to match the input shape, assuming binary pattern applied across all channels blue_noise_pattern = pattern.repeat(shape[1], 1, 1).unsqueeze(0).repeat(shape[0], 1, 1, 1) return blue_noise_pattern.float() def generate_fractal_noise(B, C, H, W, res, octaves=5, persistence=0.5, *args, **kwargs): """ Generates fractal noise for a batch of images in PyTorch. Args: B (int): Batch size. C (int): Number of channels. H, W (int): Height and Width of the images. res (tuple): Resolution of the base Perlin noise (res_x, res_y). octaves (int): Number of layers of noise to generate. persistence (float): Amplitude decay per octave (determines "roughness"). Returns: A PyTorch tensor of shape (B, C, H, W) containing fractal noise. """ noise = torch.zeros((B, C, H, W)) amplitude = 1.0 frequency = 1.0 max_amplitude = 0.0 for _ in range(octaves): perlin_noise = generate_perlin_noise(B, C, H, W, res_x=(int(res[0] * frequency)), res_y=int(res[1] * frequency)) noise += perlin_noise * amplitude max_amplitude += amplitude amplitude *= persistence frequency *= 2 # Normalize the noise noise /= max_amplitude return noise def generate_cellular_noise(H, W, num_points=100, *args, **kwargs): """ Generates a 2D Cellular (Worley) Noise pattern using PyTorch. Args: H, W (int): The height and width of the output noise pattern. num_points (int): Number of seed points to generate. Returns: torch.Tensor: A tensor containing the Cellular noise pattern. """ # Ensure PyTorch is using the same device as the input images device = kwargs.get('device', 'cpu') # Generate random points (the seeds of the Cellular noise) points = torch.rand(num_points, 2, device=device) * torch.tensor([W, H], device=device) # Initialize the noise pattern noise = torch.full((H, W), float('inf'), device=device) # Compute distance from each pixel to the nearest point using broadcasting y_coords, x_coords = torch.meshgrid(torch.arange(H, device=device), torch.arange(W, device=device), indexing='ij') for point in points: dist = torch.sqrt((x_coords - point[0]) ** 2 + (y_coords - point[1]) ** 2) noise = torch.min(noise, dist) # Normalize the noise pattern noise = (noise - torch.min(noise)) / (torch.max(noise) - torch.min(noise)) return noise ================================================ FILE: deforum_nodes/nodes/deforum_prompt_nodes.py ================================================ import torch from nodes import MAX_RESOLUTION from ..modules.deforum_node_base import DeforumDataBase class DeforumPromptNode(DeforumDataBase): def __init__(self): super().__init__() @classmethod def INPUT_TYPES(cls): return { "required": { "prompts": ("STRING", {"forceInput": False, "multiline": True, "default": "0:'Cat Sushi'"}), }, "optional": { "deforum_data": ("deforum_data",), }, } RETURN_TYPES = (("deforum_data",)) FUNCTION = "get" OUTPUT_NODE = True CATEGORY = f"deforum/prompt" display_name = "Prompt" @torch.inference_mode() def get(self, prompts, deforum_data=None): if deforum_data and "prompts" in deforum_data: # Convert the formatted prompts back to a string for editing formatted_prompts = deforum_data["prompts"] prompts = "\n".join([f"{key}:'{value}'" for key, value in formatted_prompts.items()]) else: # Splitting the data into rows rows = prompts.split('\n') # Creating an empty dictionary prompts_dict = {} # Parsing each row for row in rows: key, value = row.split(':', 1) key = int(key) value = value.strip('"') prompts_dict[key] = value if deforum_data: deforum_data["prompts"] = prompts_dict else: deforum_data = {"prompts": prompts_dict} return (deforum_data,) class DeforumAreaPromptNode(DeforumDataBase): default_area_prompt = '[{"0": [{"prompt": "a vast starscape with distant nebulae and galaxies", "x": 0, "y": 0, "w": 1024, "h": 1024, "s": 0.7}, {"prompt": "detailed sci-fi spaceship", "x": 512, "y": 512, "w": 50, "h": 50, "s": 0.7}]}, {"50": [{"prompt": "a vast starscape with distant nebulae and galaxies", "x": 0, "y": 0, "w": 1024, "h": 1024, "s": 0.7}, {"prompt": "detailed sci-fi spaceship", "x": 412, "y": 412, "w": 200, "h": 200, "s": 0.7}]}, {"100": [{"prompt": "a vast starscape with distant nebulae and galaxies", "x": 0, "y": 0, "w": 1024, "h": 1024, "s": 0.7}, {"prompt": "detailed sci-fi spaceship", "x": 112, "y": 112, "w": 800, "h": 800, "s": 0.7}]}]' default_prompt = "Alien landscape" def __init__(self): super().__init__() @classmethod def INPUT_TYPES(cls): return { "required": { "keyframe": ("INT", {"default": 0, "min": 0, "max": 8192, "step": 1}), "mode":(["default", "percentage", "strength"],), "prompt": ("STRING", {"forceInput": False, "multiline": True, 'default': cls.default_prompt,}), "width": ("INT", {"default": 64, "min": 64, "max": MAX_RESOLUTION, "step": 8}), "height": ("INT", {"default": 64, "min": 64, "max": MAX_RESOLUTION, "step": 8}), "x": ("INT", {"default": 0, "min": 0, "max": MAX_RESOLUTION, "step": 8}), "y": ("INT", {"default": 0, "min": 0, "max": MAX_RESOLUTION, "step": 8}), "strength": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.01}), }, "optional": { "deforum_data": ("deforum_data",), }, } RETURN_TYPES = (("deforum_data",)) FUNCTION = "get" OUTPUT_NODE = True CATEGORY = f"deforum/prompt" display_name = "Area Prompt" @torch.inference_mode() def get(self, keyframe, mode, prompt, width, height, x, y, strength, deforum_data=None): area_prompt = {"prompt": prompt, "x": x, "y": y, "w": width, "h": height, "s": strength, "mode":mode} area_prompt_dict = {f"{keyframe}": [area_prompt]} if not deforum_data: deforum_data = {"area_prompts":[area_prompt_dict]} if "area_prompts" not in deforum_data: deforum_data["area_prompts"] = [area_prompt_dict] else: added = None for item in deforum_data["area_prompts"]: for k, v in item.items(): if int(k) == keyframe: if area_prompt not in v: v.append(area_prompt) added = True else: added = True if not added: deforum_data["area_prompts"].append(area_prompt_dict) deforum_data["prompts"] = None return (deforum_data,) class DeforumUnformattedPromptNode(DeforumDataBase): def __init__(self): super().__init__() @classmethod def INPUT_TYPES(cls): return { "required": { "unformatted_prompts": ("STRING", {"forceInput": False, "multiline": True}), "keyframe_interval": ("INT", {"default": 50, "min": 1, "max": 8192, "step": 1}), }, "optional": { "deforum_data": ("deforum_data",), }, } RETURN_TYPES = (("deforum_data",)) FUNCTION = "get" OUTPUT_NODE = True CATEGORY = f"deforum/prompt" display_name = "Unformatted Prompt" @torch.inference_mode() def get(self, unformatted_prompts, keyframe_interval, deforum_data=None): # Splitting the unformatted prompts into lines lines = unformatted_prompts.split('\n') # Creating an empty dictionary for formatted prompts formatted_prompts = {} # Parsing each line and formatting the prompts for i, line in enumerate(lines): keyframe = i * keyframe_interval formatted_prompts[keyframe] = line.strip() if deforum_data: deforum_data["prompts"] = formatted_prompts else: deforum_data = {"prompts": formatted_prompts} return (deforum_data,) class DeforumAppendNode(DeforumDataBase): def __init__(self): super().__init__() @classmethod def INPUT_TYPES(cls): return { "required": { "append_text": ("STRING", {"multiline": True, "default": ""}), "keyframe_interval": ("INT", {"default": 50, "min": 1, "max": 8192, "step": 1}), }, "optional": { "deforum_data": ("deforum_data",), "append_to_all": (["No", "Yes"], {"default": "No"}), "use_neg": (["No", "Yes"], {"default": "No"}), }, } RETURN_TYPES = (("deforum_data",)) FUNCTION = "get" OUTPUT_NODE = True CATEGORY = f"deforum/prompt" display_name = "Append" @torch.inference_mode() def get(self, append_text, keyframe_interval, deforum_data=None, append_to_all="No", use_neg="No"): print("Append Text:", append_text) print("Keyframe Interval:", keyframe_interval) print("Deforum Data:", deforum_data) print("Append to All:", append_to_all) print("Use --neg:", use_neg) if deforum_data and "prompts" in deforum_data: formatted_prompts = deforum_data["prompts"] print("Formatted Prompts (Before Append):", formatted_prompts) neg_prefix = "--neg " if use_neg == "Yes" else "" if append_to_all == "Yes": # Append the first line of append_text to every prompt first_line = append_text.split('\n')[0] for key in formatted_prompts: formatted_prompts[key] = f"{formatted_prompts[key]} {neg_prefix}{first_line}" else: # Append the append_text to prompts based on keyframe interval lines = append_text.split('\n') for i, line in enumerate(lines): keyframe = i * keyframe_interval if keyframe in formatted_prompts: formatted_prompts[keyframe] = f"{formatted_prompts[keyframe]} {neg_prefix}{line}" print("Formatted Prompts (After Append):", formatted_prompts) deforum_data["prompts"] = formatted_prompts else: deforum_data = {"prompts": {}} return (deforum_data,) ================================================ FILE: deforum_nodes/nodes/deforum_sampler_nodes.py ================================================ class DeforumKSampler: @classmethod def INPUT_TYPES(s): return {"required": {"model": ("MODEL",), "latent": ("LATENT",), "positive": ("CONDITIONING",), "negative": ("CONDITIONING",), "deforum_frame_data": ("DEFORUM_FRAME_DATA",), } } RETURN_TYPES = ("LATENT",) FUNCTION = "sample" display_name = "KSampler" CATEGORY = "deforum/sampling" def sample(self, model, latent, positive, negative, deforum_frame_data): from nodes import common_ksampler seed = deforum_frame_data.get("seed", 0) steps = deforum_frame_data.get("steps", 10) cfg = deforum_frame_data.get("cfg", 7.5) sampler_name = deforum_frame_data.get("sampler_name", "euler_a") scheduler = deforum_frame_data.get("scheduler", "normal") denoise = deforum_frame_data.get("denoise", 1.0) latent["samples"] = latent["samples"].float() return common_ksampler(model, seed, steps, cfg, sampler_name, scheduler, positive, negative, latent, denoise=denoise) class DeforumVAEEncode: @classmethod def INPUT_TYPES(s): return {"required": { "vae": ("VAE", )}, "optional": { "pixels": ("IMAGE",), "latent": ("LATENT",) } } RETURN_TYPES = ("LATENT",) FUNCTION = "encode" display_name = "VAEEncode [safe]" CATEGORY = "deforum/latent" def encode(self, vae, pixels, latent): if pixels is not None: t = vae.encode(pixels[:,:,:,:3]) return ({"samples":t}, ) else: return (latent,) ================================================ FILE: deforum_nodes/nodes/deforum_schedule_visualizer.py ================================================ import random from matplotlib.figure import Figure from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas from PIL import Image import io from deforum.pipelines.deforum_animation.animation_helpers import FrameInterpolator from ..modules.deforum_comfyui_helpers import tensor2pil, tensor2np, pil2tensor templates = [ "0:(0), max_f:(100)", # Linearly interpolates from 0 to 100. "0:(100), max_f:(0)", # Linearly interpolates from 100 to 0. "0:(sin(2*3.14*t/max_f))", # Sinusoidal wave across the frame range. "0:(exp(-t/max_f)*sin(4*3.14*t/max_f))", # Damped sinusoidal wave. "0:(s%100)", # Outputs a constant value based on the seed modulo 100. "0:(sin(2*3.14*(t+s)/max_f))", # Sinusoidal wave with phase shift based on seed. "0:(t**2/max_f**2)", # Quadratic growth over frames. "0:(-t**2/max_f**2 + 1)", # Inverted quadratic growth. "0:(0.5*sin(2*3.14*t/max_f) + 0.5*sin(4*3.14*t/max_f))", # Superposition of two sinusoidal waves. "0:(0.5*cos(2*3.14*t/max_f) + t/max_f)", # Combination of cosine wave and linear growth. "0:(sin(2*3.14*t/max_f + s))", # Sinusoidal wave with phase offset by seed. "0:(sin(2*3.14*t/max_f) * (s % 10))", # Sinusoidal wave amplitude modulated by seed. "0:(cos(2*3.14*t/max_f))", # Cosine wave over the frame range. "0:(1 - abs(2*t/max_f - 1))", # Triangle wave that peaks at mid-frame. "0:(sin(2*3.14*t/max_f) * cos(2*3.14*t/max_f))", # Product of sine and cosine waves. "0:(t % 20)", # Modulo operation creates a sawtooth wave. "0:(50 + 50*sin(2*3.14*t/max_f))", # Sinusoidal oscillation around 50. "0:(s % 10 + sin(2*3.14*t/max_f))", # Sinusoidal wave with base level determined by seed. "0:(log(t+1))", # Logarithmic growth. "0:(-cos(2*3.14*t/max_f) + 2)", # Inverted cosine wave shifted upwards. "0:(sin(2*3.14*t/max_f) if t < max_f/2 else cos(2*3.14*t/max_f))", # Sinusoidal that switches to cosine at halfway. "0:(sin(4*3.14*t/max_f) + cos(2*3.14*t/max_f))", # Sum of two waves with different frequencies. "0:(tan(3.14*t/max_f))", # Tangent wave, potentially extreme values. "0:(1 - t/max_f)", # Linear decrease from 1 to 0. "0:(sqrt(t/max_f))", # Square root growth. "0:(-sqrt(t/max_f) + 1)", # Inverted square root curve. "0:(abs(sin(2*3.14*t/max_f)))", # Absolute value of a sinusoidal wave, creating peaks. "0:(abs(cos(2*3.14*t/max_f)))", # Absolute value of a cosine wave, creating peaks. "0:(sin(2*3.14*t/max_f)**2)", # Square of a sinusoidal wave, smoothing negative values. "0:(cos(2*3.14*t/max_f)**2)", # Square of a cosine wave, smoothing negative values. "0:(sin(2*3.14*t/(max_f+s)))", # Sinusoidal wave with frequency modulated by seed. "0:(exp(t/max_f) - 1)", # Exponential growth. "0:(2**(t/max_f) - 1)", # Exponential growth with base 2. "0:(-2**(t/max_f) + 2)", # Inverted exponential curve with base 2. "0:(s % 10 * t/max_f)", # Linear growth modulated by seed. "0:(sin(2*3.14*(t+s)%max_f/max_f))", # Sinusoidal wave with frequency and phase modulated by seed. "0:(cos(2*3.14*(t+s)%max_f/max_f))", # Cosine wave with frequency and phase modulated by seed. "0:(s * sin(2*3.14*t/max_f))", # Sinusoidal wave with amplitude modulated by seed. "0:(cos(t/max_f * 3.14 * (s%5)))", # Cosine wave with frequency modulated by seed. "0:(sin(t**2/max_f**2))", # Sinusoidal wave applied to quadratic growth. "0:(cos(s + t/max_f))", # Cosine wave with phase shift linearly increasing and offset by seed. "0:(tan(s * t/max_f))", # Tangent wave with slope modulated by seed. "0:(-1*sin(3.14*t/max_f))", # Inverted sinusoidal wave. "0:(exp(-t/max_f)*cos(2*3.14*t/max_f))", # Damped cosine wave. "0:(sin(2*3.14*t/max_f)**3)", # Cubed sinusoidal wave for enhanced contrast. "0:(cos(2*3.14*t/max_f)**3)", # Cubed cosine wave for enhanced contrast. "0:(log(t+1)*sin(2*3.14*t/max_f))", # Sinusoidal wave amplitude modulated by a logarithmic function. "0:(sin(t/max_f)*cos(t/max_f))", # Product of sine and cosine for varying wave patterns. "0:(t**3/max_f**3)", # Cubic growth for accelerated change. "0:(-t**3/max_f**3 + 1)", # Inverted cubic curve for decelerated change towards the end. "0:(abs(sin(4*3.14*t/max_f)))", # Absolute sinusoidal wave with higher frequency. "0:(abs(cos(4*3.14*t/max_f)))", # Absolute cosine wave with higher frequency. "0:(sin(2*3.14*t/max_f)*s)", # Sinusoidal wave with amplitude directly proportional to seed. "0:(cos(2*3.14*t/max_f)*s)", # Cosine wave with amplitude directly proportional to seed. "0:(tan(2*3.14*t/max_f + s))", # Tangent wave with phase shift based on seed. "0:(exp(t/max_f)*sin(2*3.14*t/max_f))", # Exponentially growing sinusoidal wave. "0:(2**(t/max_f)*sin(2*3.14*t/max_f))", # Sinusoidal wave with exponentially growing amplitude. "0:(-2**(t/max_f)*cos(2*3.14*t/max_f) + 2)", # Exponentially damped cosine wave, shifted up. "0:(s%20 * sin(2*3.14*t/max_f))", # Sinusoidal wave with amplitude modulated by seed modulo 20. "0:(cos(2*3.14*(t+s)%max_f/max_f)*s)", # Cosine wave with frequency, phase modulated by seed, and amplitude scaling. "0:(sin(2*3.14*(t+s)%max_f/max_f)*s)", # Sinusoidal wave with frequency, phase modulated by seed, and amplitude scaling. "0:(sin(2*3.14*t/max_f)/(t+1))", # Sinusoidal wave with amplitude inversely proportional to frame. "0:(cos(2*3.14*t/max_f)/(t+1))", # Cosine wave with amplitude inversely proportional to frame. "0:(sin(2*3.14*t/max_f)*t/max_f)", # Sinusoidal wave with linearly increasing amplitude. "0:(cos(2*3.14*t/max_f)*t/max_f)", # Cosine wave with linearly increasing amplitude. "0:(sin(2*3.14*t/max_f)*sqrt(t/max_f))", # Sinusoidal wave with amplitude modulated by square root of frame. "0:(cos(2*3.14*t/max_f)*sqrt(t/max_f))", # Cosine wave with amplitude modulated by square root of frame. "0:(tan(2*3.14*t/max_f)*sqrt(t/max_f))", # Tangent wave with amplitude modulated by square root of frame. "0:(sin(2*3.14*t/max_f)*log(t+1))", # Sinusoidal wave with logarithmically increasing amplitude. "0:(cos(2*3.14*t/max_f)*log(t+1))", # Cosine wave with logarithmically increasing amplitude. "0:(tan(2*3.14*t/max_f)*log(t+1))" # Tangent wave with logarithmically increasing amplitude. "0:((exp(t/max_f)) * (log(1+abs(t))) + (exp(1000/max_f)) + 1)", # Something Random "0:((exp(t/max_f)) / (cos(2*3.14*t/max_f)) / (log(1+abs(t))) / (tan(2*3.14*t/max_f)) + 1)", "0:((log(1+abs(t))) + (log(1+abs(1000))) - (cos(2*3.14*1000/max_f)) + (cos(2*3.14*t/max_f)) + (sin(2*3.14*1000/max_f)) + 1)" ] # audio_templates = [ # "x * t / max_f", # Linear scaling of x with respect to time t. # "x * sin(2 * pi * t / max_f)", # Sinusoidal modulation of x based on frame time. # "x * (1 - exp(-t / max_f))", # Exponential approach of x to its maximum over time. # "x * exp(-t / max_f)", # Exponential decay of x over time. # "x * sin(2 * pi * e * t / max_f)", # Sinusoidal modulation with base e to vary frequency. # "x * (1 if t < max_f / 2 else 0)", # Binary step function in time, for a sudden change. # "x * log(e + t / max_f)", # Logarithmic scaling with base e, gentle growth over time. # "x * cos(2 * pi * t / max_f) + e ** (t / max_f)", # Cosine wave plus exponential growth factor e. # "x * pow(t / max_f, 2)", # Quadratic growth of x over time. # "x * sqrt(t / max_f)", # Square root scaling of x, slower increase over time. # "x * tan(pi * t / max_f)", # Tangent modulation, note potential for extreme values. # "x * asin(sin(2 * pi * t / max_f))", # Arcsine of a sinusoidal wave, for harmonic effects. # "x * (2 ** (t / max_f) - 1)", # Exponential growth based on power of 2. # "x * factorial(int(t) % 5)", # Modulating x by the factorial of t modulo 5, for periodic jumps. # "x * (1 - abs(2 * t / max_f - 1))", # Triangle wave shaping of x over time. # "x * abs(sin(2 * pi * t / max_f))", # Absolute value of a sinusoidal wave, ensuring positive values. # "x * sin(2 * pi * t / max_f) * cos(2 * pi * t / max_f)", # Product of sine and cosine for complex modulation. # "x * (e ** (cos(2 * pi * t / max_f)) - 1)", # Exponential function modulated by a cosine wave. # "x * if(t < max_f / 2, sin(2 * pi * t / max_f), cos(2 * pi * t / max_f))", # Conditional modulation with half period sine, half period cosine. # "x * sin(4 * pi * t / max_f) + x * cos(2 * pi * t / max_f)", # Combination of sine and cosine waves with different frequencies. # "x * (sin(2 * pi * t / max_f) if t < max_f / 3 else sin(4 * pi * t / max_f))", # Conditional frequency change in sine wave. # "x * (exp(t / max_f) - 1) / log(e + t)", # Exponential increase tempered by a logarithmic factor. # "(x * cos(2 * pi * t / max_f)) / (1 + log(t + 1))", # Cosine wave scaled by x with a logarithmic denominator to moderate growth. # "x * sin(2 * pi * t / max_f) ** 2", # Square of a sinusoidal wave, creating a smoothed, non-negative waveform. # "x * cos(2 * pi * t / max_f) ** 2", # Square of a cosine wave, similar effect as above but phase-shifted. # ] audio_templates = [ # Simple Amplitude Modulations "x * 2", # Doubling the amplitude. "x / 2", # Halving the amplitude. "abs(x)", # Absolute value of the amplitude. "x * x", # Squaring the amplitude. "sqrt(abs(x))", # Square root of the absolute amplitude. "log(abs(x) + 1)", # Logarithmic scaling of the amplitude. "x * sin(x)", # Sine modulation based on amplitude itself. "x * cos(x)", # Cosine modulation based on amplitude itself. "-x", # Inverting the amplitude. "x * (exp(x) - 1)", # Exponential scaling based on the amplitude. "x * pow(e, x - 1)", # Exponential growth using base e, adjusted for amplitude. "x * factorial(int(abs(x)) % 5)", # Factorial modulation based on the absolute amplitude modulo 5. "x if x > 0.5 else x * 2", # Conditional scaling for amplitudes greater than 0.5. "min(x, 0.5)", # Clipping amplitude to a maximum of 0.5. "max(x, -0.5)", # Ensuring amplitude is not less than -0.5. "x % 0.5", # Amplitude modulo 0.5, creating a repeating pattern. "tan(x)", # Tangent modulation of amplitude. "asin(min(1, max(-1, x))) / pi", # Arcsine of amplitude normalized to [-1, 1], scaled by π. "1 / (abs(x) + 1)", # Inverse scaling of amplitude, avoiding division by zero. "pow(e, -abs(x))", # Exponential decay based on the absolute amplitude. # Temporal Expressions Involving Time and Maximum Frame "x * t / max_f", # Linear scaling of amplitude with respect to time. "x * sin(2 * pi * t / max_f)", # Sinusoidal modulation over time. "x * exp(-t / max_f)", # Exponential decay over time. "x * (1 - exp(-t / max_f))", # Inverse exponential growth. "x * cos(2 * pi * e * t / max_f)", # Cosine wave modulation with base e to alter frequency. "x * if(t < max_f / 2, 1, 0)", # Binary switch based on time, for sudden change. "x * pow(t / max_f, 2)", # Quadratic growth of amplitude over time. "x * sqrt(t / max_f)", # Square root scaling over time. "x * tan(pi * t / max_f)", # Tangent modulation over time. "x * sin(2 * pi * t / max_f) * cos(2 * pi * t / max_f)", # Product of sine and cosine for complex temporal modulation. "x * sin(2 * pi * t / max_f) if t < max_f / 2 else x * cos(2 * pi * t / max_f)", # Conditional wave modulation. "x * sin(4 * pi * t / max_f) + x * cos(2 * pi * t / max_f)", # Sum of waves with different frequencies. "x * (exp(t / max_f) - 1) / log(e + t)", # Exponential growth tempered by logarithm of time. "(x * cos(2 * pi * t / max_f)) / (1 + log(t + 1))", # Cosine wave moderated by logarithmic factor. # Complex and Combined Expressions "x * (sin(2 * pi * t / max_f) ** 2 + cos(2 * pi * t / max_f) ** 2)", # Sum of squared sine and cosine waves. "x * (1 - abs(2 * t / max_f - 1))", # Triangle wave shaping over time. "x * abs(sin(2 * pi * t / max_f))", # Absolute sine wave for positive values. "x * pow(e, cos(2 * pi * t / max_f))", # Exponential function modulated by cosine. "x * asin(sin(2 * pi * t / max_f)) / pi", # Arcsine of sine wave normalized and scaled. "x * pow(2, sin(2 * pi * t / max_f))", # Exponential growth with base 2, modulated by sine wave. "x * log(1 + abs(sin(2 * pi * t / max_f)))", # Logarithmic scaling modulated by absolute sine wave. "x * (sin(2 * pi * t / (max_f + t)))", # Sinusoidal frequency modulation by time. "(x * cos(2 * pi * t / max_f)) / (1 + log(t + 1)) * sin(2 * pi * t / max_f)", # Combined cosine and sine waves with logarithmic softening. "x * (sin(2 * pi * t / max_f) * (t % 10))", # Sinusoidal wave amplitude modulated by frame modulo. "x * exp(t / max_f) * sin(2 * pi * t / max_f)", # Exponential growth combined with sinusoidal modulation. "x * (if(t < max_f / 3, sin(2 * pi * t / max_f), cos(2 * pi * t / max_f)))", # Conditional frequency modulation. "x * (1 / (1 + exp(-t / max_f)))", # Logistic sigmoid function for smooth transitions. "x * pow(e, -t / max_f) * cos(2 * pi * t / max_f)", # Exponential decay combined with cosine modulation. "x * sin(2 * pi * t / max_f) / (1 + exp(-t / max_f))", # Sinusoidal wave with logistic growth control. "x * (e ** (t / max_f) - e ** (-t / max_f)) / 2", # Hyperbolic sine function for symmetric exponential growth and decay. "x * (1 - (t / max_f) ** 2)", # Parabolic decrease towards the end of the timeline. "x * tanh(2 * pi * t / max_f)", # Hyperbolic tangent for smooth transitions between -1 and 1. "x * factorial((int(t) % 5) + 1)", # Factorial modulation based on time modulo 5. ] def generate_complex_random_expression(max_frames, seed=None, max_parts=3): """ Generates a complex random mathematical expression using a variety of functions, operators, and globals. Parameters: - max_frames: The maximum number of frames, for normalizing 't' in expressions. - seed: Optional seed for random number generator for reproducibility. - max_parts: Maximum number of parts (expressions) to combine. Returns: A string formatted like "0:(expression), max_f:(expression)". """ if seed is not None: random.seed(seed) funcs = ['sin', 'cos', 'tan', 'exp', 'log', 'abs'] operators = ['+', '-', '*', '/'] globals = ['t', 'max_f', 's'] parts = [] for _ in range(random.randint(1, max_parts)): func = random.choice(funcs) operator = random.choice(operators) if parts else '' # No leading operator for the first part global_var = random.choice(globals) if global_var == 'max_f': global_value = str(max_frames) else: global_value = global_var if func in ['sin', 'cos', 'tan']: part = f"{func}(2*3.14*{global_value}/max_f)" elif func == 'log': part = f"{func}(1+abs({global_value}))" elif func == 'exp': part = f"{func}({global_value}/max_f)" else: # abs or other functions without specific handling part = f"{func}({global_value})" parts.append(f"{operator} ({part})") # Combine parts with randomly chosen operators expression = ' '.join(parts).strip() # Final check to make the expression safer for division and ensure it's not empty if '/' in expression: expression += " + 1" # Avoid division by zero if not expression: expression = "0" # Fallback to a simple zero expression if somehow it ends up empty return f"0:({expression})" class DeforumScheduleTemplate: @classmethod def INPUT_TYPES(cls): return { "required": { "expression": (templates,), } } # @classmethod # def IS_CHANGED(cls, text, autorefresh): # # Force re-evaluation of the node # if autorefresh == "Yes": # return float("NaN") RETURN_TYPES = ("STRING",) FUNCTION = "show" display_name = "Schedule Templates" CATEGORY = "deforum/help" OUTPUT_NODE = True def show(self, expression): return(str(expression),) class DeforumAudioScheduleTemplate: @classmethod def INPUT_TYPES(cls): return { "required": { "expression": (audio_templates,), } } # @classmethod # def IS_CHANGED(cls, text, autorefresh): # # Force re-evaluation of the node # if autorefresh == "Yes": # return float("NaN") RETURN_TYPES = ("STRING",) FUNCTION = "show" display_name = "Audio Schedule Expression Templates" CATEGORY = "deforum/help" OUTPUT_NODE = True def show(self, expression): return(str(expression),) class DeforumScheduleTemplateRandomizer: @classmethod def INPUT_TYPES(cls): return { "required": { "seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff}), "max_frames": ("INT", {"default": 1, "min": 1, "max": 4096, "step": 1}), "max_parts": ("INT", {"default": 1, "min": 1, "max": 4096, "step": 1}), } } @classmethod def IS_CHANGED(cls, text, autorefresh): # Force re-evaluation of the node if autorefresh == "Yes": return float("NaN") RETURN_TYPES = ("STRING",) FUNCTION = "show" display_name = "Schedule Randomizer" CATEGORY = "deforum/utils" OUTPUT_NODE = True def show(self, seed, max_frames, max_parts): return(str(generate_complex_random_expression(max_frames, seed, max_parts)),) class DeforumScheduleVisualizer: @classmethod def INPUT_TYPES(cls): return { "required": { "schedule": ("STRING", {"default": "0: (1.0)"}), "max_frames": ("INT", {"default": 0, "min": 0, "max": 128000, "step": 1}), "grid": ("BOOLEAN", {"default": False}), } } # @classmethod # def IS_CHANGED(cls, text, autorefresh): # # Force re-evaluation of the node # if autorefresh == "Yes": # return float("NaN") RETURN_TYPES = ("IMAGE",) FUNCTION = "show" display_name = "Schedule Visualizer" CATEGORY = "deforum/utils" OUTPUT_NODE = True def show(self, schedule, max_frames, grid): if max_frames == 0: max_frames = len(schedule.split(',')) fi = FrameInterpolator(max_frames, -1) series = fi.get_inbetweens(fi.parse_key_frames(schedule)) # Create a figure for plotting fig = Figure() canvas = FigureCanvas(fig) ax = fig.add_subplot(111) if grid: ax.grid(True, which='both', linestyle='--', linewidth=0.5, color='gray') # Plot the series ax.plot(series) ax.set_title("Schedule Visualization") ax.set_xlabel("Frame") ax.set_ylabel("Value") # Convert the Matplotlib figure to a PIL Image buf = io.BytesIO() fig.savefig(buf, format='png') buf.seek(0) pil_img = Image.open(buf) return(pil2tensor(pil_img),) ================================================ FILE: deforum_nodes/nodes/deforum_video_nodes.py ================================================ import base64 import gc import os import shutil from io import BytesIO from aiohttp import web import hashlib import server import cv2 import imageio import numpy as np import torch from PIL import Image from tqdm import tqdm import folder_paths from ..modules.deforum_comfyui_helpers import tensor2pil, pil2tensor, find_next_index, pil_image_to_base64, tensor_to_webp_base64 video_extensions = ['webm', 'mp4', 'mkv', 'gif'] import moviepy.editor as mp from scipy.io.wavfile import write import tempfile def save_to_file(data, filepath: str): # Ensure the audio data is reshaped properly for mono/stereo if data.num_channels > 1: audio_data_reshaped = data.audio_data.reshape((-1, data.num_channels)) else: audio_data_reshaped = data.audio_data write(filepath, data.sample_rate, audio_data_reshaped.astype(np.int16)) return True class DeforumLoadVideo: def __init__(self): self.video_path = None @classmethod def INPUT_TYPES(s): input_dir = folder_paths.get_input_directory() files = [] for f in os.listdir(input_dir): if os.path.isfile(os.path.join(input_dir, f)): file_parts = f.split('.') if len(file_parts) > 1 and (file_parts[-1] in video_extensions): files.append(f) return {"required": { "video": (sorted(files),), "reset": ("BOOLEAN", {"default": False},), "iterative": ("BOOLEAN", {"default": True},), "start_frame": ("INT", {"default": 0, "min": 0, "max": 1000000},), "return_frames": ("INT", {"default": 1, "min": 1, "max": 1000000},), },} CATEGORY = "deforum/video" display_name = "Load Video" RETURN_TYPES = ("IMAGE","INT","INT") RETURN_NAMES = ("IMAGE","FRAME_IDX","MAX_FRAMES") FUNCTION = "load_video_frame" def __init__(self): self.cap = None self.current_frame = None # def load_video_frame(self, video, reset, iterative, start_frame, return_frames): # video_path = folder_paths.get_annotated_filepath(video) # # # Initialize or reset video capture # if self.cap is None or self.cap.get(cv2.CAP_PROP_POS_FRAMES) >= self.cap.get(cv2.CAP_PROP_FRAME_COUNT) or self.video_path != video_path or reset: # try: # self.cap.release() # except: # pass # self.cap = cv2.VideoCapture(video_path) # # self.cap = cv2.VideoCapture(video_path) # self.current_frame = -1 # self.video_path = video_path # success, frame = self.cap.read() # if success: # self.current_frame += 1 # frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) # frame = np.array(frame).astype(np.float32) # frame = pil2tensor(frame) # Convert to torch tensor # else: # # Reset if reached the end of the video # self.cap.set(cv2.CAP_PROP_POS_FRAMES, 0) # success, frame = self.cap.read() # self.current_frame = 0 # frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) # frame = np.array(frame).astype(np.float32) # frame = pil2tensor(frame) # Convert to torch tensor # # return (frame,self.current_frame,self.cap.get(cv2.CAP_PROP_POS_FRAMES),) def load_video_frame(self, video, reset, iterative, start_frame, return_frames): video_path = folder_paths.get_annotated_filepath(video) max_frames = 0 frames = [] # Initialize or reset video capture if self.cap is None or self.cap.get(cv2.CAP_PROP_POS_FRAMES) >= self.cap.get(cv2.CAP_PROP_FRAME_COUNT) or self.video_path != video_path or reset: try: self.cap.release() except: pass self.cap = cv2.VideoCapture(video_path) self.current_frame = -1 self.video_path = video_path if not iterative: self.cap.set(cv2.CAP_PROP_POS_FRAMES, start_frame) self.current_frame = start_frame - 1 max_frames = int(self.cap.get(cv2.CAP_PROP_FRAME_COUNT)) for _ in range(return_frames): success, frame = self.cap.read() if not success: self.cap.set(cv2.CAP_PROP_POS_FRAMES, 0) success, frame = self.cap.read() self.current_frame = -1 if success: self.current_frame += 1 frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) frame = np.array(frame).astype(np.float32) frame = pil2tensor(frame) # Convert to torch tensor frames.append(frame[0]) else: break if len(frames) <= 1: frame = torch.stack(frames) if frames else None return (frame, self.current_frame, max_frames) else: # Stack frames along a new dimension (simulate batch dimension) frame = torch.stack(frames) return (frame, self.current_frame, max_frames) @classmethod def IS_CHANGED(cls, text, autorefresh): # Force re-evaluation of the node if autorefresh == "Yes": return float("NaN") @classmethod def VALIDATE_INPUTS(cls, video): if not folder_paths.exists_annotated_filepath(video): return "Invalid video file: {}".format(video) return True temp_dir = tempfile.mkdtemp() hash_object = hashlib.md5(temp_dir.encode()) hex_dig = hash_object.hexdigest() endpoint = f"/tmp/{hex_dig}/{{filename}}" @server.PromptServer.instance.routes.get(endpoint) async def serve_temp_file(request): filename = request.match_info['filename'] if '..' in filename or filename.startswith('/'): return web.Response(status=400, text="Invalid file path.") file_path = os.path.join(temp_dir, filename) if os.path.isfile(file_path): return web.FileResponse(file_path) else: return web.Response(status=404, text="File not found.") # Dynamically add the route to the server's router # server.PromptServer.instance.routes.get(endpoint)(serve_temp_file) print(f"Endpoint {endpoint} registered for serving files from {temp_dir}") class DeforumVideoSaveNode: def __init__(self): self.output_dir = folder_paths.get_output_directory() self.images = [] self.size = None self.temp_dir = temp_dir self.hex_dig = hex_dig self.audio_path = None self.filepath = "" def clear_cache_directory(self): for filename in os.listdir(self.temp_dir): file_path = os.path.join(self.temp_dir, filename) try: if os.path.isfile(file_path) or os.path.islink(file_path): os.unlink(file_path) elif os.path.isdir(file_path): shutil.rmtree(file_path) except Exception as e: print(f'Failed to delete {file_path}. Reason: {e}') @classmethod def INPUT_TYPES(s): return {"required": {"image": ("IMAGE",), "filename_prefix": ("STRING",{"default":"Deforum"}), "fps": ("INT", {"default": 24, "min": 1, "max": 10000},), "codec": (["libx265", "libx264", "libvpx-vp9", "libaom-av1", "mpeg4", "libvpx"],), "pixel_format": (["yuv420p", "yuv422p", "yuv444p", "yuvj420p", "yuvj422p", "yuvj444p", "rgb24", "rgba", "nv12", "nv21"],), "format": (["mp4", "mov", "gif", "avi"],), "quality": ("INT", {"default": 10, "min": 1, "max": 10},), "dump_by": (["max_frames", "per_N_frames"],), "dump_every": ("INT", {"default": 0, "min": 0, "max": 4096},), "dump_now": ("BOOLEAN", {"default": False},), "skip_save": ("BOOLEAN", {"default": False},), "skip_return": ("BOOLEAN", {"default": True},), "enable_preview": ("BOOLEAN", {"default": True},), "restore": ("BOOLEAN", {"default": False},), "clear_cache": ("BOOLEAN", {"default": False},), }, "optional": { "deforum_frame_data": ("DEFORUM_FRAME_DATA",), "audio": ("AUDIO",), "waveform_image": ("IMAGE",), }, "hidden": { "js_frames": ("INT", {"default": 0, "min": 0, "max": 9999999999},), } } RETURN_TYPES = ("IMAGE","STRING",) RETURN_NAMES = ("IMAGES","VIDEOPATH",) OUTPUT_NODE = True FUNCTION = "fn" display_name = "Save Video" CATEGORY = "deforum/video" def add_image(self, image): frame_path = os.path.join(self.temp_dir, f"frame_{len(self.images):05d}.png") if isinstance(image, torch.Tensor): tensor2pil(image).save(frame_path) else: im = cv2.cvtColor(image.astype(np.uint8), cv2.COLOR_BGR2RGB) cv2.imwrite(frame_path, im) self.images.append(frame_path) # del image return frame_path # self.images.append(image) def fn(self, image, filename_prefix, fps, codec, pixel_format, format, quality, dump_by, dump_every, dump_now, skip_save, skip_return, enable_preview, deforum_frame_data={}, audio=None, waveform_image=None, restore=False, clear_cache=False): new_images = [] base64_audio = "" dump = False ret = None full_output_folder, filename, counter, subfolder, filename_prefix = folder_paths.get_save_image_path( filename_prefix, self.output_dir) counter = find_next_index(full_output_folder, filename_prefix, format) anim_args = deforum_frame_data.get("anim_args") if image is not None: if anim_args is not None: max_frames = anim_args.max_frames else: max_frames = image.shape[0] + len(self.images) + 2 if not deforum_frame_data.get("reset", None): if len(image) > 1: for img in image: new_images.append(self.add_image(img)) else: new_images.append(self.add_image(image[0])) print(f"[deforum] Video Save node cached {len(self.images)} frames") # print("THE CAT IS FINE. SOMETHING WAS False THOUGH THE CAT IS NOT") # When the current frame index reaches the last frame, save the video if dump_by == "max_frames": dump = len(self.images) >= max_frames else: dump = len(self.images) >= dump_every if deforum_frame_data.get("reset", None): dump = True clear_cache = True ret = None if dump or dump_now: # frame_idx is 0-based if len(self.images) >= 2: if not skip_save: self.filepath = self.save_video(full_output_folder, filename, counter, fps, audio, codec, format) if not skip_return: ret = torch.stack([pil2tensor(i)[0] for i in self.images], dim=0) if clear_cache: self.images = [] # self.clear_cache_directory() enable_preview = True if deforum_frame_data.get("reset", None): if image.shape[0] > 1: for img in image: self.add_image(img) else: self.add_image(image[0]) if enable_preview and image is not None: # if audio is not None: # Delete the previous temporary audio file if it exists if self.audio_path and os.path.exists(os.path.join(self.temp_dir, self.audio_path)): os.remove(os.path.join(self.temp_dir, self.audio_path)) self.audio_path = self.encode_audio_base64(audio, len(self.images), fps, 0) ui_ret = {"counter":(len(self.images),), "should_dump":(clear_cache,), "frames":([f"/tmp/{self.hex_dig}/{os.path.basename(frame_path)}" for frame_path in self.images] if restore else [f"/tmp/{self.hex_dig}/{os.path.basename(frame_path)}" for frame_path in new_images]), "fps":(fps,), "audio":(f"/tmp/{self.hex_dig}/{self.audio_path}",)} if waveform_image is not None: ui_ret["waveform"] = (tensor_to_webp_base64(waveform_image),) else: if anim_args is not None: max_frames = anim_args.max_frames else: max_frames = len(self.images) + 5 if dump_by == "max_frames": dump = len(self.images) >= max_frames else: dump = len(self.images) >= dump_every if deforum_frame_data.get("reset", None): dump = True dump_now = True clear_cache = True if dump or dump_now: # frame_idx is 0-based if len(self.images) >= 2: if not skip_save: self.filepath = self.save_video(full_output_folder, filename, counter, fps, audio, codec, format) if not skip_return: ret = torch.stack([pil2tensor(Image.open(i))[0] for i in self.images], dim=0) if clear_cache: self.images = self.images = [] # self.clear_cache_directory() ui_ret = {"counter":(len(self.images),), "should_dump":(clear_cache,), "frames":([]), "fps":(fps,)} del base64_audio, image return {"ui": ui_ret, "result": (ret,self.filepath,)} def encode_audio_base64(self, audio_data, frame_count, fps, start_frame): sample_rate = 44100 # Assuming a default sample rate if audio_data is None: # Generate silent audio data for the specified duration duration_in_seconds = frame_count / float(fps) silence = np.zeros(int(duration_in_seconds * sample_rate), dtype=np.int16) audio_data_reshaped = silence else: # Calculate start and end samples start_sample = 0 end_sample = start_sample + int((frame_count / fps) * audio_data.sample_rate) # Handle actual audio data if audio_data.num_channels > 1: # Ensure the audio data is properly reshaped for multi-channel data audio_data_reshaped = audio_data.audio_data.reshape((-1, audio_data.num_channels)) else: audio_data_reshaped = audio_data.audio_data # Loop the audio if the end_sample exceeds the length of the audio data total_samples = audio_data_reshaped.shape[0] if end_sample > total_samples: looped_audio_data = [] while end_sample > len(looped_audio_data): remaining_samples = end_sample - len(looped_audio_data) looped_audio_data.extend(audio_data_reshaped[:min(remaining_samples, total_samples)]) audio_data_reshaped = np.array(looped_audio_data)[:end_sample - start_sample] else: # Slice the audio data from start_sample to end_sample audio_data_reshaped = audio_data_reshaped[start_sample:end_sample] with tempfile.NamedTemporaryFile(delete=False, dir=self.temp_dir, suffix='.wav') as tmp_file: temp_audio_path = tmp_file.name from scipy.io.wavfile import write as wav_write wav_write(temp_audio_path, 48000, audio_data_reshaped) # Return the path relative to the temp directory for URL construction return os.path.basename(temp_audio_path) def save_video(self, full_output_folder, filename, counter, fps, audio, codec, ext): output_path = os.path.join(full_output_folder, f"{filename}_{counter}.{ext}") print("[deforum] Saving video:", output_path) # writer = imageio.get_writer(output_path, fps=fps, codec=codec, quality=quality, pixelformat=pixel_format, format=format) # for frame in tqdm(self.images, desc=f"Saving {format} (imageio)"): # writer.append_data(np.clip(255. * frame.detach().cpu().numpy().squeeze(), 0, 255).astype(np.uint8)) # writer.close() frames = [Image.open(frame_path) for frame_path in self.images] video_clip = mp.ImageSequenceClip([np.array(frame) for frame in frames], fps=fps) if audio is not None: # Generate a temporary file for the audio with tempfile.NamedTemporaryFile(delete=False, suffix='.wav') as tmp_audio_file: save_to_file(audio, tmp_audio_file.name) # Load the audio clip audio_clip = mp.AudioFileClip(tmp_audio_file.name) # Calculate video duration video_duration = len(self.images) / fps # Trim or loop the audio clip to match the video length if audio_clip.duration > video_duration: audio_clip = audio_clip.subclip(0, video_duration) # Trim the audio to match video length elif audio_clip.duration < video_duration: # If you want to loop the audio, uncomment the following line # audio_clip = audio_clip.loop(duration=video_duration) pass # If you prefer silence after the audio ends, do nothing # Set the audio on the video clip video_clip = video_clip.set_audio(audio_clip) del tmp_audio_file video_clip.write_videofile(output_path, codec=codec, audio_codec='aac') return output_path @classmethod def IS_CHANGED(s, text, autorefresh): # Force re-evaluation of the node if autorefresh == "Yes": return float("NaN") def encode_audio_base64(audio_data, frame_count, fps): # Calculate the target duration of the audio in seconds based on the video duration target_audio_duration = frame_count / fps # Calculate the number of samples to keep based on the target duration and the sample rate num_samples_to_keep = int(target_audio_duration * audio_data.sample_rate) # Reshape the audio data based on the number of channels if audio_data.num_channels > 1: audio_data_reshaped = audio_data.audio_data.reshape((-1, audio_data.num_channels)) else: audio_data_reshaped = audio_data.audio_data # Trim or pad the audio data to match the target duration actual_samples = audio_data_reshaped.shape[0] if actual_samples > num_samples_to_keep: # Trim the audio data if it's longer than the target duration audio_data_reshaped = audio_data_reshaped[:num_samples_to_keep, ...] elif actual_samples < num_samples_to_keep: # Pad the audio data with zeros if it's shorter than the target duration padding_length = num_samples_to_keep - actual_samples if audio_data.num_channels > 1: padding = np.zeros((padding_length, audio_data.num_channels), dtype=audio_data_reshaped.dtype) else: padding = np.zeros(padding_length, dtype=audio_data_reshaped.dtype) audio_data_reshaped = np.vstack((audio_data_reshaped, padding)) # Convert the adjusted numpy array to bytes output = BytesIO() write(output, audio_data.sample_rate, audio_data_reshaped.astype(np.int16)) # Encode bytes to base64 base64_audio = base64.b64encode(output.getvalue()).decode('utf-8') return base64_audio def save_to_file(data, filepath: str): # Ensure the audio data is reshaped properly for mono/stereo if data.num_channels > 1: audio_data_reshaped = data.audio_data.reshape((-1, data.num_channels)) else: audio_data_reshaped = data.audio_data write(filepath, data.sample_rate, audio_data_reshaped.astype(np.int16)) return True ================================================ FILE: deforum_nodes/nodes/redirect_console_node.py ================================================ import sys import asyncio console_redirected = None stdout_backup = sys.stdout stderr_backup = sys.stderr class StreamToWebSocket: def __init__(self, original_stream, server, stream_type='stdout'): self.original_stream = original_stream self.server = server self.stream_type = stream_type def write(self, message): # Write to the original stdout or stderr self.original_stream.write(message) # Asynchronously send to the frontend via WebSocket if message.strip(): # Avoid sending empty messages asyncio.run_coroutine_threadsafe( self.server.send('console_output', {'message': message, 'stream': self.stream_type}), self.server.loop ) def flush(self): self.original_stream.flush() def __getattr__(self, attr): # Delegate attribute access to the original stream return getattr(self.original_stream, attr) class DeforumRedirectConsole: @classmethod def INPUT_TYPES(s): return {"required": {"redirect_console": ("BOOLEAN", {"default": False},),} } RETURN_TYPES = ("BOOLEAN",) OUTPUT_NODE = True FUNCTION = "fn" display_name = "Redirect Console" CATEGORY = "deforum/utils" def fn(self, redirect_console): global console_redirected if redirect_console: if not console_redirected: try: import server server_instance = server.PromptServer.instance sys.stdout = StreamToWebSocket(sys.stdout, server_instance, 'stdout') sys.stderr = StreamToWebSocket(sys.stderr, server_instance, 'stderr') console_redirected = True except: pass else: sys.stdout = stdout_backup sys.stderr = stderr_backup else: if console_redirected: sys.stdout = stdout_backup sys.stderr = stderr_backup console_redirected = False return (console_redirected,) ================================================ FILE: examples/deforum_base.json ================================================ { "last_node_id": 186, "last_link_id": 579, "nodes": [ { "id": 141, "type": "Reroute", "pos": { "0": 532, "1": 972 }, "size": [ 75, 26 ], "flags": {}, "order": 17, "mode": 0, "inputs": [ { "name": "", "type": "*", "link": 577 } ], "outputs": [ { "name": "", "type": "DEFORUM_FRAME_DATA", "links": [ 441, 443, 447, 459 ] } ], "properties": { "showOutputText": false, "horizontal": false } }, { "id": 130, "type": "DeforumKSampler", "pos": { "0": 650, "1": 120 }, "size": [ 325.93902587890625, 326 ], "flags": {}, "order": 18, "mode": 0, "inputs": [ { "name": "model", "type": "MODEL", "link": 533 }, { "name": "latent", "type": "LATENT", "link": 579, "slot_index": 1 }, { "name": "positive", "type": "CONDITIONING", "link": 430 }, { "name": "negative", "type": "CONDITIONING", "link": 431, "slot_index": 3 }, { "name": "deforum_frame_data", "type": "DEFORUM_FRAME_DATA", "link": 459 } ], "outputs": [ { "name": "LATENT", "type": "LATENT", "links": [ 416 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumKSampler" }, "widgets_values": [] }, { "id": 88, "type": "DeforumLoadVideo", "pos": { "0": 83, "1": 12 }, "size": { "0": 317.5899658203125, "1": 294 }, "flags": {}, "order": 0, "mode": 4, "inputs": [], "outputs": [ { "name": "IMAGE", "type": "IMAGE", "links": [ 224, 453 ], "shape": 3, "slot_index": 0 }, { "name": "FRAME_IDX", "type": "INT", "links": null, "shape": 3 }, { "name": "MAX_FRAMES", "type": "INT", "links": null, "shape": 3 } ], "properties": { "Node name for S&R": "DeforumLoadVideo" }, "widgets_values": [ "Recording 2024-02-08 214819.mp4", "image", "image", 0, 1, "image" ] }, { "id": 62, "type": "PreviewImage", "pos": { "0": 98, "1": 388 }, "size": { "0": 309.7149658203125, "1": 251.35919189453125 }, "flags": {}, "order": 5, "mode": 4, "inputs": [ { "name": "images", "type": "IMAGE", "link": 224, "slot_index": 0 } ], "outputs": [], "properties": { "Node name for S&R": "PreviewImage" }, "widgets_values": [] }, { "id": 170, "type": "Note", "pos": { "0": 484, "1": -347 }, "size": { "0": 633.6494140625, "1": 221.03656005859375 }, "flags": {}, "order": 1, "mode": 0, "inputs": [], "outputs": [], "properties": { "text": "" }, "widgets_values": [ "Welcome to Deforum!\n\nThe below graph provides a pluggable Deforum Animation pipeline in ComfyUI. It works frame-by-frame, so the best practice is to enable Extra options, and Auto Queue.\n\nDeforum Parameter and Schedule nodes represent all settings available to consume after chaining up by the Deforum Iterator node, which keeps track of the current frame, generates/gets the cached latent to be denoised in the next pass. The parameters and schedule's are the same as in the auto1111 extension and in the Colab version.\n\nHybrid nodes are currently bypassed (purple), enabling them, and selecting a video transforms the pipeline into Deforum Hybrid. Iteration can be reset to frame 0 with the reset value being set to 1 on the iterator node. Don't forget to switch it back to 0 to use the generated image/latent.\"\n\nDeforum Video Save node dumps its collected frames when the current frame's id reaches/has passed max_frames." ], "color": "#432", "bgcolor": "#653" }, { "id": 133, "type": "DeforumConditioningBlendNode", "pos": { "0": 640, "1": -40 }, "size": { "0": 342.5999755859375, "1": 78 }, "flags": {}, "order": 16, "mode": 0, "inputs": [ { "name": "clip", "type": "CLIP", "link": 428 }, { "name": "deforum_frame_data", "type": "DEFORUM_FRAME_DATA", "link": 576 } ], "outputs": [ { "name": "POSITIVE", "type": "CONDITIONING", "links": [ 430 ], "shape": 3 }, { "name": "NEGATIVE", "type": "CONDITIONING", "links": [ 431 ], "shape": 3 } ], "properties": { "Node name for S&R": "DeforumConditioningBlendNode" }, "widgets_values": [ "linear" ] }, { "id": 175, "type": "DeforumDepthParamsNode", "pos": { "0": -740, "1": 460 }, "size": { "0": 317.4000244140625, "1": 178 }, "flags": {}, "order": 7, "mode": 0, "inputs": [ { "name": "deforum_data", "type": "deforum_data", "link": 557, "shape": 7 } ], "outputs": [ { "name": "deforum_data", "type": "deforum_data", "links": [ 558 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumDepthParamsNode" }, "widgets_values": [ true, "Midas-3-Hybrid", 0.2, "border", "bicubic", false ] }, { "id": 176, "type": "DeforumTranslationParamsNode", "pos": { "0": -740, "1": 710 }, "size": { "0": 317.4000244140625, "1": 274 }, "flags": {}, "order": 8, "mode": 0, "inputs": [ { "name": "deforum_data", "type": "deforum_data", "link": 558, "shape": 7 } ], "outputs": [ { "name": "deforum_data", "type": "deforum_data", "links": [ 561 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumTranslationParamsNode" }, "widgets_values": [ "0:(0)", "0: (1.0025+0.002*sin(1.25*3.14*t/30))", "0:(0)", "0:(0)", "0:(0)", "0:(0.5)", "0:(0.5)", "0:(0)", "0:(0)", "0:(0)" ] }, { "id": 178, "type": "DeforumDiffusionParamsNode", "pos": { "0": -290, "1": 530 }, "size": { "0": 278.891845703125, "1": 274 }, "flags": {}, "order": 12, "mode": 0, "inputs": [ { "name": "deforum_data", "type": "deforum_data", "link": 573, "shape": 7 } ], "outputs": [ { "name": "deforum_data", "type": "deforum_data", "links": [ 562 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumDiffusionParamsNode" }, "widgets_values": [ "0: (0.065)", "0: (0.48)", "0: (1.0)", "0: (5)", false, "0: (18)", false, "0:(0)", false, "0:(1)" ] }, { "id": 179, "type": "DeforumColorParamsNode", "pos": { "0": -750, "1": 1040 }, "size": { "0": 317.4000244140625, "1": 154 }, "flags": {}, "order": 9, "mode": 0, "inputs": [ { "name": "deforum_data", "type": "deforum_data", "link": 561, "shape": 7 } ], "outputs": [ { "name": "deforum_data", "type": "deforum_data", "links": [ 564 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumColorParamsNode" }, "widgets_values": [ "None", "", 1, false, false ] }, { "id": 180, "type": "DeforumHybridScheduleNode", "pos": { "0": -290, "1": 850 }, "size": { "0": 274.891845703125, "1": 178 }, "flags": {}, "order": 13, "mode": 0, "inputs": [ { "name": "deforum_data", "type": "deforum_data", "link": 562, "shape": 7 } ], "outputs": [ { "name": "deforum_data", "type": "deforum_data", "links": [ 559 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumHybridScheduleNode" }, "widgets_values": [ "0:(0.5)" ] }, { "id": 182, "type": "DeforumCadenceParamsNode", "pos": { "0": -740, "1": 1260 }, "size": { "0": 317.4000244140625, "1": 178 }, "flags": {}, "order": 10, "mode": 0, "inputs": [ { "name": "deforum_data", "type": "deforum_data", "link": 564, "shape": 7 } ], "outputs": [ { "name": "deforum_data", "type": "deforum_data", "links": [ 572 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumCadenceParamsNode" }, "widgets_values": [ 0, "None", "0: (1)", "None", "0: (1)", "0" ] }, { "id": 177, "type": "DeforumNoiseParamsNode", "pos": { "0": -290, "1": 1080 }, "size": { "0": 272.06170654296875, "1": 298 }, "flags": {}, "order": 14, "mode": 0, "inputs": [ { "name": "deforum_data", "type": "deforum_data", "link": 559, "shape": 7 } ], "outputs": [ { "name": "deforum_data", "type": "deforum_data", "links": [ 574 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumNoiseParamsNode" }, "widgets_values": [ true, "0: (0.8)", "0: (0.1)", "0: (5)", "0: (1.0)", "0: (0.0)", "perlin", 8, 8, 4, 0.5 ] }, { "id": 16, "type": "PreviewImage", "pos": { "0": 1120, "1": 821 }, "size": { "0": 587.271728515625, "1": 617.682373046875 }, "flags": {}, "order": 20, "mode": 0, "inputs": [ { "name": "images", "type": "IMAGE", "link": 462, "slot_index": 0 } ], "outputs": [], "properties": { "Node name for S&R": "PreviewImage" }, "widgets_values": [] }, { "id": 183, "type": "DeforumPromptNode", "pos": { "0": -754, "1": -282 }, "size": { "0": 335.66650390625, "1": 417 }, "flags": {}, "order": 2, "mode": 0, "inputs": [ { "name": "deforum_data", "type": "deforum_data", "link": null, "shape": 7 } ], "outputs": [ { "name": "deforum_data", "type": "deforum_data", "links": [ 556 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumPromptNode" }, "widgets_values": [ "0:\" tiny cute swamp bunny, highly detailed, intricate, ultra hd, sharp photo, crepuscular rays, in focus, 4k, landscape --neg nsfw, nude\",\n30:\" anthropomorphic clean cat, surrounded by mandelbulb fractals, epic angle and pose, symmetrical, 3d, depth of field --neg nsfw, nude\",\n60:\" a beautiful coconut --neg photo, realistic nsfw, nude\",\n90:\" a beautiful durian, amazing award winning photography --neg nsfw, nude\"" ] }, { "id": 174, "type": "DeforumAnimParamsNode", "pos": { "0": -740, "1": 210 }, "size": { "0": 317.4000244140625, "1": 178 }, "flags": {}, "order": 6, "mode": 0, "inputs": [ { "name": "deforum_data", "type": "deforum_data", "link": 556, "shape": 7 } ], "outputs": [ { "name": "deforum_data", "type": "deforum_data", "links": [ 557 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumAnimParamsNode" }, "widgets_values": [ "3D", 120, "wrap" ] }, { "id": 127, "type": "CheckpointLoaderSimple", "pos": { "0": 86, "1": 743 }, "size": { "0": 315, "1": 98 }, "flags": {}, "order": 3, "mode": 0, "inputs": [], "outputs": [ { "name": "MODEL", "type": "MODEL", "links": [ 533 ], "shape": 3, "slot_index": 0 }, { "name": "CLIP", "type": "CLIP", "links": [ 428 ], "shape": 3, "slot_index": 1 }, { "name": "VAE", "type": "VAE", "links": [ 417, 450 ], "shape": 3 } ], "properties": { "Node name for S&R": "CheckpointLoaderSimple" }, "widgets_values": [ "realismEngineSDXL_v30VAE.safetensors" ] }, { "id": 137, "type": "VAEEncode", "pos": { "0": 696, "1": 1241 }, "size": { "0": 210, "1": 46 }, "flags": {}, "order": 25, "mode": 0, "inputs": [ { "name": "pixels", "type": "IMAGE", "link": 489 }, { "name": "vae", "type": "VAE", "link": 450, "slot_index": 1 } ], "outputs": [ { "name": "LATENT", "type": "LATENT", "links": [ 438 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "VAEEncode" }, "widgets_values": [] }, { "id": 139, "type": "DeforumAddNoiseNode", "pos": { "0": 656, "1": 1063 }, "size": { "0": 324.6453552246094, "1": 46 }, "flags": {}, "order": 24, "mode": 0, "inputs": [ { "name": "image", "type": "IMAGE", "link": 488, "slot_index": 0 }, { "name": "deforum_frame_data", "type": "DEFORUM_FRAME_DATA", "link": 443, "slot_index": 1 } ], "outputs": [ { "name": "IMAGE", "type": "IMAGE", "links": [ 489 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumAddNoiseNode" }, "widgets_values": [] }, { "id": 138, "type": "DeforumFrameWarpNode", "pos": { "0": 665, "1": 888 }, "size": { "0": 304.79998779296875, "1": 98 }, "flags": {}, "order": 23, "mode": 0, "inputs": [ { "name": "image", "type": "IMAGE", "link": 440 }, { "name": "deforum_frame_data", "type": "DEFORUM_FRAME_DATA", "link": 441 }, { "name": "depth_image", "type": "IMAGE", "link": null, "shape": 7 } ], "outputs": [ { "name": "IMAGE", "type": "IMAGE", "links": [ 488 ], "shape": 3, "slot_index": 0 }, { "name": "DEPTH", "type": "IMAGE", "links": null, "shape": 3 }, { "name": "WARPED_DEPTH", "type": "IMAGE", "links": null, "shape": 3 } ], "properties": { "Node name for S&R": "DeforumFrameWarpNode" }, "widgets_values": [ false ] }, { "id": 142, "type": "DeforumHybridMotionNode", "pos": { "0": 657, "1": 676 }, "size": { "0": 315, "1": 98 }, "flags": {}, "order": 21, "mode": 4, "inputs": [ { "name": "image", "type": "IMAGE", "link": 469 }, { "name": "hybrid_image", "type": "IMAGE", "link": 453 }, { "name": "deforum_frame_data", "type": "DEFORUM_FRAME_DATA", "link": 447 } ], "outputs": [ { "name": "IMAGE", "type": "IMAGE", "links": [ 440 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumHybridMotionNode" }, "widgets_values": [ "RAFT" ] }, { "id": 131, "type": "VAEDecode", "pos": { "0": 713, "1": 545 }, "size": { "0": 210, "1": 46 }, "flags": {}, "order": 19, "mode": 0, "inputs": [ { "name": "samples", "type": "LATENT", "link": 416 }, { "name": "vae", "type": "VAE", "link": 417, "slot_index": 1 } ], "outputs": [ { "name": "IMAGE", "type": "IMAGE", "links": [ 462, 469, 551 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "VAEDecode" }, "widgets_values": [] }, { "id": 185, "type": "DeforumBaseParamsNode", "pos": { "0": -309, "1": 7 }, "size": { "0": 317.4000244140625, "1": 442 }, "flags": {}, "order": 11, "mode": 0, "inputs": [ { "name": "deforum_data", "type": "deforum_data", "link": 572, "shape": 7 } ], "outputs": [ { "name": "deforum_data", "type": "deforum_data", "links": [ 573 ] } ], "properties": { "Node name for S&R": "DeforumBaseParamsNode" }, "widgets_values": [ 512, 512, "0: (-1)", "random", "euler", "normal", true, true, false ] }, { "id": 173, "type": "DeforumVideoSaveNode", "pos": { "0": 1103, "1": -54 }, "size": { "0": 573, "1": 686.5 }, "flags": {}, "order": 22, "mode": 0, "inputs": [ { "name": "image", "type": "IMAGE", "link": 551 }, { "name": "deforum_frame_data", "type": "DEFORUM_FRAME_DATA", "link": 578, "shape": 7 }, { "name": "audio", "type": "AUDIO", "link": null, "shape": 7 }, { "name": "waveform_image", "type": "IMAGE", "link": null, "shape": 7 } ], "outputs": [ { "name": "IMAGES", "type": "IMAGE", "links": null, "shape": 3 }, { "name": "VIDEOPATH", "type": "STRING", "links": null, "shape": 3 } ], "properties": { "Node name for S&R": "DeforumVideoSaveNode" }, "widgets_values": [ "Deforum", 24, "libx265", "yuv420p", "mp4", 10, "max_frames", 0, false, true, true, true, false, false, { "hidden": false, "paused": false, "params": {} } ] }, { "id": 135, "type": "DeforumGetCachedLatentNode", "pos": { "0": -259, "1": 1437 }, "size": { "0": 235.1999969482422, "1": 58 }, "flags": {}, "order": 4, "mode": 0, "inputs": [], "outputs": [ { "name": "LATENT", "type": "LATENT", "links": [ 575 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumGetCachedLatentNode" }, "widgets_values": [ 0 ] }, { "id": 186, "type": "DeforumIteratorNode", "pos": { "0": 67, "1": 1041 }, "size": { "0": 393, "1": 310 }, "flags": {}, "order": 15, "mode": 0, "inputs": [ { "name": "deforum_data", "type": "deforum_data", "link": 574 }, { "name": "latent", "type": "LATENT", "link": 575, "shape": 7 }, { "name": "init_latent", "type": "LATENT", "link": null, "shape": 7 } ], "outputs": [ { "name": "deforum_frame_data", "type": "DEFORUM_FRAME_DATA", "links": [ 576, 577, 578 ] }, { "name": "latent", "type": "LATENT", "links": [ 579 ] }, { "name": "positive_prompt", "type": "STRING", "links": null }, { "name": "negative_prompt", "type": "STRING", "links": null } ], "properties": { "Node name for S&R": "DeforumIteratorNode" }, "widgets_values": [ "stable_diffusion", 287373190076054, "randomize", 0, 0.8, 0.1, true, true, true ] }, { "id": 136, "type": "DeforumCacheLatentNode", "pos": { "0": 714, "1": 1393 }, "size": { "0": 210, "1": 58 }, "flags": {}, "order": 26, "mode": 0, "inputs": [ { "name": "latent", "type": "LATENT", "link": 438 } ], "outputs": [ { "name": "LATENT", "type": "LATENT", "links": null, "shape": 3 } ], "properties": { "Node name for S&R": "DeforumCacheLatentNode" }, "widgets_values": [ 0 ] } ], "links": [ [ 224, 88, 0, 62, 0, "IMAGE" ], [ 416, 130, 0, 131, 0, "LATENT" ], [ 417, 127, 2, 131, 1, "VAE" ], [ 428, 127, 1, 133, 0, "CLIP" ], [ 430, 133, 0, 130, 2, "CONDITIONING" ], [ 431, 133, 1, 130, 3, "CONDITIONING" ], [ 438, 137, 0, 136, 0, "LATENT" ], [ 440, 142, 0, 138, 0, "IMAGE" ], [ 441, 141, 0, 138, 1, "DEFORUM_FRAME_DATA" ], [ 443, 141, 0, 139, 1, "DEFORUM_FRAME_DATA" ], [ 447, 141, 0, 142, 2, "DEFORUM_FRAME_DATA" ], [ 450, 127, 2, 137, 1, "VAE" ], [ 453, 88, 0, 142, 1, "IMAGE" ], [ 459, 141, 0, 130, 4, "DEFORUM_FRAME_DATA" ], [ 462, 131, 0, 16, 0, "IMAGE" ], [ 469, 131, 0, 142, 0, "IMAGE" ], [ 488, 138, 0, 139, 0, "IMAGE" ], [ 489, 139, 0, 137, 0, "IMAGE" ], [ 533, 127, 0, 130, 0, "MODEL" ], [ 551, 131, 0, 173, 0, "IMAGE" ], [ 556, 183, 0, 174, 0, "deforum_data" ], [ 557, 174, 0, 175, 0, "deforum_data" ], [ 558, 175, 0, 176, 0, "deforum_data" ], [ 559, 180, 0, 177, 0, "deforum_data" ], [ 561, 176, 0, 179, 0, "deforum_data" ], [ 562, 178, 0, 180, 0, "deforum_data" ], [ 564, 179, 0, 182, 0, "deforum_data" ], [ 572, 182, 0, 185, 0, "deforum_data" ], [ 573, 185, 0, 178, 0, "deforum_data" ], [ 574, 177, 0, 186, 0, "deforum_data" ], [ 575, 135, 0, 186, 1, "LATENT" ], [ 576, 186, 0, 133, 1, "DEFORUM_FRAME_DATA" ], [ 577, 186, 0, 141, 0, "*" ], [ 578, 186, 0, 173, 1, "DEFORUM_FRAME_DATA" ], [ 579, 186, 1, 130, 1, "LATENT" ] ], "groups": [], "config": {}, "extra": { "groupNodes": {}, "workspace_info": { "id": "FNCJfZPX52ugkXlmp-lm_" }, "ds": { "scale": 0.5445000000000005, "offset": [ 1153.9036380571622, 530.2484150930318 ] } }, "version": 0.4 } ================================================ FILE: examples/deforum_cadence.json ================================================ { "last_node_id": 191, "last_link_id": 595, "nodes": [ { "id": 141, "type": "Reroute", "pos": { "0": 532, "1": 972 }, "size": [ 75, 26 ], "flags": {}, "order": 17, "mode": 0, "inputs": [ { "name": "", "type": "*", "link": 589 } ], "outputs": [ { "name": "", "type": "DEFORUM_FRAME_DATA", "links": [ 459 ] } ], "properties": { "showOutputText": false, "horizontal": false } }, { "id": 157, "type": "DeforumHybridScheduleNode", "pos": { "0": -290, "1": 950 }, "size": { "0": 274.891845703125, "1": 178 }, "flags": {}, "order": 13, "mode": 0, "inputs": [ { "name": "deforum_data", "type": "deforum_data", "link": 540, "shape": 7 } ], "outputs": [ { "name": "deforum_data", "type": "deforum_data", "links": [ 541 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumHybridScheduleNode" }, "widgets_values": [ "0:(0.5)" ] }, { "id": 170, "type": "Note", "pos": { "0": 484, "1": -347 }, "size": { "0": 633.6494140625, "1": 221.03656005859375 }, "flags": {}, "order": 0, "mode": 0, "inputs": [], "outputs": [], "properties": { "text": "" }, "widgets_values": [ "Welcome to Deforum!\n\nThe below graph provides a pluggable Deforum Animation pipeline in ComfyUI. It works frame-by-frame, so the best practice is to enable Extra options, and Auto Queue.\n\nDeforum Parameter and Schedule nodes represent all settings available to consume after chaining up by the Deforum Iterator node, which keeps track of the current frame, generates/gets the cached latent to be denoised in the next pass. The parameters and schedule's are the same as in the auto1111 extension and in the Colab version.\n\nHybrid nodes are currently bypassed (purple), enabling them, and selecting a video transforms the pipeline into Deforum Hybrid. Iteration can be reset to frame 0 with the reset value being set to 1 on the iterator node. Don't forget to switch it back to 0 to use the generated image/latent.\"\n\nDeforum Video Save node dumps its collected frames when the current frame's id reaches/has passed max_frames." ], "color": "#432", "bgcolor": "#653" }, { "id": 153, "type": "DeforumNoiseParamsNode", "pos": { "0": -290, "1": 1180 }, "size": { "0": 272.06170654296875, "1": 298 }, "flags": {}, "order": 14, "mode": 0, "inputs": [ { "name": "deforum_data", "type": "deforum_data", "link": 541, "shape": 7 } ], "outputs": [ { "name": "deforum_data", "type": "deforum_data", "links": [ 586 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumNoiseParamsNode" }, "widgets_values": [ true, "0: (0.8)", "0: (0.1)", "0: (5)", "0: (1.0)", "0: (0.0)", "perlin", 8, 8, 4, 0.5 ] }, { "id": 131, "type": "VAEDecode", "pos": { "0": 760, "1": 1210 }, "size": { "0": 210, "1": 46 }, "flags": {}, "order": 25, "mode": 0, "inputs": [ { "name": "samples", "type": "LATENT", "link": 416 }, { "name": "vae", "type": "VAE", "link": 417, "slot_index": 1 } ], "outputs": [ { "name": "IMAGE", "type": "IMAGE", "links": [ 556, 558, 566 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "VAEDecode" }, "widgets_values": [] }, { "id": 155, "type": "DeforumColorParamsNode", "pos": { "0": -750, "1": 1140 }, "size": { "0": 317.4000244140625, "1": 154 }, "flags": {}, "order": 9, "mode": 0, "inputs": [ { "name": "deforum_data", "type": "deforum_data", "link": 535, "shape": 7 } ], "outputs": [ { "name": "deforum_data", "type": "deforum_data", "links": [ 567 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumColorParamsNode" }, "widgets_values": [ "None", "", 1, false, false ] }, { "id": 176, "type": "DeforumFrameDataExtract", "pos": { "0": 70, "1": 1070 }, "size": { "0": 443.4000244140625, "1": 186 }, "flags": {}, "order": 19, "mode": 0, "inputs": [ { "name": "deforum_frame_data", "type": "DEFORUM_FRAME_DATA", "link": 592, "slot_index": 0 } ], "outputs": [ { "name": "frame_idx", "type": "INT", "links": null, "shape": 3 }, { "name": "seed", "type": "INT", "links": null, "shape": 3 }, { "name": "steps", "type": "INT", "links": null, "shape": 3 }, { "name": "cfg_scale", "type": "FLOAT", "links": null, "shape": 3 }, { "name": "sampler_name", "type": "STRING", "links": null, "shape": 3 }, { "name": "scheduler_name", "type": "STRING", "links": null, "shape": 3 }, { "name": "denoise", "type": "FLOAT", "links": null, "shape": 3 }, { "name": "subseed_strength", "type": "FLOAT", "links": null, "shape": 3 }, { "name": "first_run", "type": "BOOLEAN", "links": [ 555 ], "shape": 3, "slot_index": 8 } ], "properties": { "Node name for S&R": "DeforumFrameDataExtract" }, "widgets_values": [] }, { "id": 135, "type": "DeforumGetCachedLatentNode", "pos": { "0": 162, "1": 950 }, "size": { "0": 235.1999969482422, "1": 58 }, "flags": {}, "order": 1, "mode": 0, "inputs": [], "outputs": [ { "name": "LATENT", "type": "LATENT", "links": [ 587 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumGetCachedLatentNode" }, "widgets_values": [ 0 ] }, { "id": 173, "type": "DeforumVideoSaveNode", "pos": { "0": 1309, "1": 10 }, "size": [ 620, 1064 ], "flags": {}, "order": 20, "mode": 0, "inputs": [ { "name": "image", "type": "IMAGE", "link": 565 }, { "name": "deforum_frame_data", "type": "DEFORUM_FRAME_DATA", "link": 590, "shape": 7 }, { "name": "audio", "type": "AUDIO", "link": null, "shape": 7 }, { "name": "waveform_image", "type": "IMAGE", "link": null, "shape": 7 } ], "outputs": [ { "name": "IMAGES", "type": "IMAGE", "links": null, "shape": 3 }, { "name": "VIDEOPATH", "type": "STRING", "links": null, "shape": 3 } ], "properties": { "Node name for S&R": "DeforumVideoSaveNode" }, "widgets_values": [ "Deforum", 24, "libx265", "yuv420p", "mp4", 10, "max_frames", 0, false, false, true, true, false, false, { "hidden": false, "paused": false, "params": {} } ] }, { "id": 186, "type": "DeforumPromptNode", "pos": { "0": -738, "1": -31 }, "size": { "0": 313.04736328125, "1": 276.55352783203125 }, "flags": {}, "order": 2, "mode": 0, "inputs": [ { "name": "deforum_data", "type": "deforum_data", "link": null, "shape": 7 } ], "outputs": [ { "name": "deforum_data", "type": "deforum_data", "links": [ 571 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumPromptNode" }, "widgets_values": [ "0:\" tiny cute swamp bunny, highly detailed, intricate, ultra hd, sharp photo, crepuscular rays, in focus, 4k, landscape --neg nsfw, nude\",\n30:\" anthropomorphic clean cat, surrounded by mandelbulb fractals, epic angle and pose, symmetrical, 3d, depth of field --neg nsfw, nude\",\n60:\" a beautiful coconut --neg photo, realistic nsfw, nude\",\n90:\" a beautiful durian, amazing award winning photography --neg nsfw, nude\"" ] }, { "id": 185, "type": "DeforumCadenceParamsNode", "pos": { "0": -740, "1": 1360 }, "size": { "0": 317.4000244140625, "1": 178 }, "flags": {}, "order": 10, "mode": 0, "inputs": [ { "name": "deforum_data", "type": "deforum_data", "link": 567, "shape": 7 } ], "outputs": [ { "name": "deforum_data", "type": "deforum_data", "links": [ 575 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumCadenceParamsNode" }, "widgets_values": [ 12, "RAFT", "0: (1)", "None", "0: (1)", "0" ] }, { "id": 152, "type": "DeforumTranslationParamsNode", "pos": { "0": -740, "1": 810 }, "size": { "0": 317.4000244140625, "1": 274 }, "flags": {}, "order": 8, "mode": 0, "inputs": [ { "name": "deforum_data", "type": "deforum_data", "link": 498, "shape": 7 } ], "outputs": [ { "name": "deforum_data", "type": "deforum_data", "links": [ 535 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumTranslationParamsNode" }, "widgets_values": [ "0:(0)", "0: (1.0025+0.002*sin(1.25*3.14*t/30))", "0:(0)", "0:(0)", "0:(0)", "0:(0.5)", "0:(0.5)", "0:(0)", "0:(0)", "0:(0)" ] }, { "id": 187, "type": "DeforumAddNoiseNode", "pos": { "0": 679, "1": 547 }, "size": { "0": 304.79998779296875, "1": 46 }, "flags": {}, "order": 21, "mode": 0, "inputs": [ { "name": "image", "type": "IMAGE", "link": 572 }, { "name": "deforum_frame_data", "type": "DEFORUM_FRAME_DATA", "link": 593, "slot_index": 1 } ], "outputs": [ { "name": "IMAGE", "type": "IMAGE", "links": [ 573 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumAddNoiseNode" }, "widgets_values": [] }, { "id": 151, "type": "DeforumDepthParamsNode", "pos": { "0": -740, "1": 560 }, "size": { "0": 317.4000244140625, "1": 178 }, "flags": {}, "order": 7, "mode": 0, "inputs": [ { "name": "deforum_data", "type": "deforum_data", "link": 497, "shape": 7 } ], "outputs": [ { "name": "deforum_data", "type": "deforum_data", "links": [ 498 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumDepthParamsNode" }, "widgets_values": [ true, "Midas-3-Hybrid", 0.2, "border", "bicubic", false ] }, { "id": 154, "type": "DeforumDiffusionParamsNode", "pos": { "0": -290, "1": 630 }, "size": { "0": 278.891845703125, "1": 274 }, "flags": {}, "order": 12, "mode": 0, "inputs": [ { "name": "deforum_data", "type": "deforum_data", "link": 576, "shape": 7 } ], "outputs": [ { "name": "deforum_data", "type": "deforum_data", "links": [ 540 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumDiffusionParamsNode" }, "widgets_values": [ "0: (0.065)", "0: (0.48)", "0: (1.0)", "0: (5)", false, "0: (18)", false, "0:(0)", false, "0:(1)" ] }, { "id": 150, "type": "DeforumAnimParamsNode", "pos": { "0": -740, "1": 310 }, "size": { "0": 317.4000244140625, "1": 178 }, "flags": {}, "order": 6, "mode": 0, "inputs": [ { "name": "deforum_data", "type": "deforum_data", "link": 571, "shape": 7 } ], "outputs": [ { "name": "deforum_data", "type": "deforum_data", "links": [ 497 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumAnimParamsNode" }, "widgets_values": [ "3D", 120, "wrap" ] }, { "id": 188, "type": "DeforumBaseParamsNode", "pos": { "0": -300, "1": 89 }, "size": { "0": 317.4000244140625, "1": 442 }, "flags": {}, "order": 11, "mode": 0, "inputs": [ { "name": "deforum_data", "type": "deforum_data", "link": 575, "shape": 7 } ], "outputs": [ { "name": "deforum_data", "type": "deforum_data", "links": [ 576 ] } ], "properties": { "Node name for S&R": "DeforumBaseParamsNode" }, "widgets_values": [ 512, 512, "0: (-1)", "random", "euler", "normal", true, true, false ] }, { "id": 127, "type": "CheckpointLoaderSimple", "pos": { "0": 120, "1": 380 }, "size": { "0": 315, "1": 98 }, "flags": {}, "order": 3, "mode": 0, "inputs": [], "outputs": [ { "name": "MODEL", "type": "MODEL", "links": [ 533 ], "shape": 3, "slot_index": 0 }, { "name": "CLIP", "type": "CLIP", "links": [ 428 ], "shape": 3, "slot_index": 1 }, { "name": "VAE", "type": "VAE", "links": [ 417, 562 ], "shape": 3, "slot_index": 2 } ], "properties": { "Node name for S&R": "CheckpointLoaderSimple" }, "widgets_values": [ "realismEngineSDXL_v30VAE.safetensors" ] }, { "id": 178, "type": "DeforumCacheImageNode", "pos": { "0": 968, "1": 1801 }, "size": { "0": 315, "1": 58 }, "flags": {}, "order": 27, "mode": 0, "inputs": [ { "name": "image", "type": "IMAGE", "link": 558 } ], "outputs": [ { "name": "IMAGE", "type": "IMAGE", "links": null, "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumCacheImageNode" }, "widgets_values": [ 1 ] }, { "id": 175, "type": "DeforumImageSwitcherNode", "pos": { "0": 573, "1": 1550 }, "size": { "0": 315, "1": 78 }, "flags": {}, "order": 26, "mode": 0, "inputs": [ { "name": "image_true", "type": "IMAGE", "link": 556, "shape": 7 }, { "name": "image_false", "type": "IMAGE", "link": null, "shape": 7 }, { "name": "option", "type": "BOOLEAN", "link": 555, "widget": { "name": "option" }, "slot_index": 2 } ], "outputs": [ { "name": "IMAGE", "type": "IMAGE", "links": [ 557 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumImageSwitcherNode" }, "widgets_values": [ false ] }, { "id": 182, "type": "DeforumGetCachedImageNode", "pos": { "0": 813, "1": 162 }, "size": { "0": 315, "1": 78 }, "flags": {}, "order": 4, "mode": 0, "inputs": [], "outputs": [ { "name": "IMAGE", "type": "IMAGE", "links": [ 564 ], "shape": 3, "slot_index": 0 }, { "name": "MASK", "type": "MASK", "links": null, "shape": 3 } ], "properties": { "Node name for S&R": "DeforumGetCachedImageNode" }, "widgets_values": [ 1 ] }, { "id": 181, "type": "DeforumGetCachedImageNode", "pos": { "0": 386, "1": 148 }, "size": { "0": 315, "1": 78 }, "flags": {}, "order": 5, "mode": 0, "inputs": [], "outputs": [ { "name": "IMAGE", "type": "IMAGE", "links": [ 563 ], "shape": 3, "slot_index": 0 }, { "name": "MASK", "type": "MASK", "links": null, "shape": 3 } ], "properties": { "Node name for S&R": "DeforumGetCachedImageNode" }, "widgets_values": [ 0 ] }, { "id": 179, "type": "DeforumCadenceNode", "pos": { "0": 705, "1": 349 }, "size": { "0": 315, "1": 142 }, "flags": {}, "order": 18, "mode": 0, "inputs": [ { "name": "image", "type": "IMAGE", "link": 564 }, { "name": "first_image", "type": "IMAGE", "link": 563 }, { "name": "deforum_frame_data", "type": "DEFORUM_FRAME_DATA", "link": 591 }, { "name": "hybrid_images", "type": "IMAGE", "link": null, "shape": 7 } ], "outputs": [ { "name": "IMAGE", "type": "IMAGE", "links": [ 565 ], "shape": 3, "slot_index": 0 }, { "name": "IMAGE", "type": "IMAGE", "links": [ 572 ], "shape": 3, "slot_index": 1 } ], "properties": { "Node name for S&R": "DeforumCadenceNode" }, "widgets_values": [ 1, false ] }, { "id": 177, "type": "DeforumCacheImageNode", "pos": { "0": 409, "1": 1802 }, "size": { "0": 315, "1": 58 }, "flags": {}, "order": 29, "mode": 0, "inputs": [ { "name": "image", "type": "IMAGE", "link": 557 } ], "outputs": [ { "name": "IMAGE", "type": "IMAGE", "links": null, "shape": 3 } ], "properties": { "Node name for S&R": "DeforumCacheImageNode" }, "widgets_values": [ 0 ] }, { "id": 130, "type": "DeforumKSampler", "pos": { "0": 709, "1": 1118 }, "size": [ 325.93902587890625, 326 ], "flags": {}, "order": 23, "mode": 0, "inputs": [ { "name": "model", "type": "MODEL", "link": 533 }, { "name": "latent", "type": "LATENT", "link": 561, "slot_index": 1 }, { "name": "positive", "type": "CONDITIONING", "link": 430 }, { "name": "negative", "type": "CONDITIONING", "link": 431, "slot_index": 3 }, { "name": "deforum_frame_data", "type": "DEFORUM_FRAME_DATA", "link": 459 } ], "outputs": [ { "name": "LATENT", "type": "LATENT", "links": [ 416 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumKSampler" }, "widgets_values": [] }, { "id": 133, "type": "DeforumConditioningBlendNode", "pos": { "0": 700, "1": 976 }, "size": { "0": 342.5999755859375, "1": 78 }, "flags": {}, "order": 16, "mode": 0, "inputs": [ { "name": "clip", "type": "CLIP", "link": 428 }, { "name": "deforum_frame_data", "type": "DEFORUM_FRAME_DATA", "link": 588 } ], "outputs": [ { "name": "POSITIVE", "type": "CONDITIONING", "links": [ 430 ], "shape": 3 }, { "name": "NEGATIVE", "type": "CONDITIONING", "links": [ 431 ], "shape": 3 } ], "properties": { "Node name for S&R": "DeforumConditioningBlendNode" }, "widgets_values": [ "linear" ] }, { "id": 191, "type": "DeforumCacheLatentNode", "pos": { "0": 698, "1": 826 }, "size": { "0": 315, "1": 58 }, "flags": {}, "order": 24, "mode": 0, "inputs": [ { "name": "latent", "type": "LATENT", "link": 595 } ], "outputs": [ { "name": "LATENT", "type": "LATENT", "links": null } ], "properties": { "Node name for S&R": "DeforumCacheLatentNode" }, "widgets_values": [ 0 ] }, { "id": 180, "type": "DeforumVAEEncode", "pos": { "0": 739, "1": 675 }, "size": { "0": 218.39999389648438, "1": 66 }, "flags": {}, "order": 22, "mode": 0, "inputs": [ { "name": "vae", "type": "VAE", "link": 562 }, { "name": "pixels", "type": "IMAGE", "link": 573, "shape": 7 }, { "name": "latent", "type": "LATENT", "link": 594, "shape": 7 } ], "outputs": [ { "name": "LATENT", "type": "LATENT", "links": [ 561, 595 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumVAEEncode" }, "widgets_values": [] }, { "id": 183, "type": "PreviewImage", "pos": { "0": 1309, "1": 1169 }, "size": { "0": 614.161865234375, "1": 543.7197265625 }, "flags": {}, "order": 28, "mode": 0, "inputs": [ { "name": "images", "type": "IMAGE", "link": 566 } ], "outputs": [], "properties": { "Node name for S&R": "PreviewImage" }, "widgets_values": [] }, { "id": 190, "type": "DeforumIteratorNode", "pos": { "0": 80, "1": 560 }, "size": { "0": 393, "1": 310 }, "flags": {}, "order": 15, "mode": 0, "inputs": [ { "name": "deforum_data", "type": "deforum_data", "link": 586 }, { "name": "latent", "type": "LATENT", "link": 587, "shape": 7 }, { "name": "init_latent", "type": "LATENT", "link": null, "shape": 7 } ], "outputs": [ { "name": "deforum_frame_data", "type": "DEFORUM_FRAME_DATA", "links": [ 588, 589, 590, 591, 592, 593 ] }, { "name": "latent", "type": "LATENT", "links": [ 594 ] }, { "name": "positive_prompt", "type": "STRING", "links": null }, { "name": "negative_prompt", "type": "STRING", "links": null } ], "properties": { "Node name for S&R": "DeforumIteratorNode" }, "widgets_values": [ "stable_diffusion", 478403467058392, "randomize", 0, 0.8, 0.1, true, true, true ] } ], "links": [ [ 416, 130, 0, 131, 0, "LATENT" ], [ 417, 127, 2, 131, 1, "VAE" ], [ 428, 127, 1, 133, 0, "CLIP" ], [ 430, 133, 0, 130, 2, "CONDITIONING" ], [ 431, 133, 1, 130, 3, "CONDITIONING" ], [ 459, 141, 0, 130, 4, "DEFORUM_FRAME_DATA" ], [ 497, 150, 0, 151, 0, "deforum_data" ], [ 498, 151, 0, 152, 0, "deforum_data" ], [ 533, 127, 0, 130, 0, "MODEL" ], [ 535, 152, 0, 155, 0, "deforum_data" ], [ 540, 154, 0, 157, 0, "deforum_data" ], [ 541, 157, 0, 153, 0, "deforum_data" ], [ 555, 176, 8, 175, 2, "BOOLEAN" ], [ 556, 131, 0, 175, 0, "IMAGE" ], [ 557, 175, 0, 177, 0, "IMAGE" ], [ 558, 131, 0, 178, 0, "IMAGE" ], [ 561, 180, 0, 130, 1, "LATENT" ], [ 562, 127, 2, 180, 0, "VAE" ], [ 563, 181, 0, 179, 1, "IMAGE" ], [ 564, 182, 0, 179, 0, "IMAGE" ], [ 565, 179, 0, 173, 0, "IMAGE" ], [ 566, 131, 0, 183, 0, "IMAGE" ], [ 567, 155, 0, 185, 0, "deforum_data" ], [ 571, 186, 0, 150, 0, "deforum_data" ], [ 572, 179, 1, 187, 0, "IMAGE" ], [ 573, 187, 0, 180, 1, "IMAGE" ], [ 575, 185, 0, 188, 0, "deforum_data" ], [ 576, 188, 0, 154, 0, "deforum_data" ], [ 586, 153, 0, 190, 0, "deforum_data" ], [ 587, 135, 0, 190, 1, "LATENT" ], [ 588, 190, 0, 133, 1, "DEFORUM_FRAME_DATA" ], [ 589, 190, 0, 141, 0, "*" ], [ 590, 190, 0, 173, 1, "DEFORUM_FRAME_DATA" ], [ 591, 190, 0, 179, 2, "DEFORUM_FRAME_DATA" ], [ 592, 190, 0, 176, 0, "DEFORUM_FRAME_DATA" ], [ 593, 190, 0, 187, 1, "DEFORUM_FRAME_DATA" ], [ 594, 190, 1, 180, 2, "LATENT" ], [ 595, 180, 0, 191, 0, "LATENT" ] ], "groups": [], "config": {}, "extra": { "groupNodes": {}, "workspace_info": { "id": "6fH6Rsqch6zpTq8oT6FAF" }, "ds": { "scale": 0.5989500000000006, "offset": [ 868.0037162140221, 159.90042862989674 ] } }, "version": 0.4 } ================================================ FILE: examples/deforum_integrated.json ================================================ { "last_node_id": 6, "last_link_id": 7, "nodes": [ { "id": 5, "type": "DeforumPromptNode", "pos": [ 1277, 213 ], "size": { "0": 400, "1": 200 }, "flags": {}, "order": 3, "mode": 0, "inputs": [ { "name": "deforum_data", "type": "deforum_data", "link": 5 } ], "outputs": [ { "name": "deforum_data", "type": "deforum_data", "links": [ 6 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumPromptNode" }, "widgets_values": [ "0:'Cat Sushi'" ] }, { "id": 1, "type": "DeforumSingleSampleNode", "pos": [ 1841, 253 ], "size": [ 229.20001220703125, 306 ], "flags": {}, "order": 4, "mode": 0, "inputs": [ { "name": "deforum_data", "type": "deforum_data", "link": 6 }, { "name": "model", "type": "MODEL", "link": 1 }, { "name": "clip", "type": "CLIP", "link": 2 }, { "name": "vae", "type": "VAE", "link": 3 } ], "outputs": [ { "name": "IMAGE", "type": "IMAGE", "links": [ 7 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumSingleSampleNode" } }, { "id": 2, "type": "CheckpointLoaderSimple", "pos": [ 1348, 482 ], "size": { "0": 315, "1": 98 }, "flags": {}, "order": 0, "mode": 0, "outputs": [ { "name": "MODEL", "type": "MODEL", "links": [ 1 ], "shape": 3, "slot_index": 0 }, { "name": "CLIP", "type": "CLIP", "links": [ 2 ], "shape": 3, "slot_index": 1 }, { "name": "VAE", "type": "VAE", "links": [ 3 ], "shape": 3, "slot_index": 2 } ], "properties": { "Node name for S&R": "CheckpointLoaderSimple" }, "widgets_values": [ "protovisionXLHighFidelity3D_releaseV660Bakedvae.safetensors" ] }, { "id": 4, "type": "DeforumAnimParamsNode", "pos": [ 898, 214 ], "size": { "0": 317.4000244140625, "1": 106 }, "flags": {}, "order": 2, "mode": 0, "inputs": [ { "name": "deforum_data", "type": "deforum_data", "link": 4 } ], "outputs": [ { "name": "deforum_data", "type": "deforum_data", "links": [ 5 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumAnimParamsNode" }, "widgets_values": [ "None", 16, "wrap" ] }, { "id": 3, "type": "DeforumBaseParamsNode", "pos": [ 526, 214 ], "size": { "0": 317.4000244140625, "1": 250 }, "flags": {}, "order": 1, "mode": 0, "inputs": [ { "name": "deforum_data", "type": "deforum_data", "link": null } ], "outputs": [ { "name": "deforum_data", "type": "deforum_data", "links": [ 4 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumBaseParamsNode" }, "widgets_values": [ 512, 512, "0: (-1)", "random", "euler", "normal", true, true, false ] }, { "id": 6, "type": "PreviewImage", "pos": [ 2158, 223 ], "size": [ 210, 246 ], "flags": {}, "order": 5, "mode": 0, "inputs": [ { "name": "images", "type": "IMAGE", "link": 7 } ], "properties": { "Node name for S&R": "PreviewImage" } } ], "links": [ [ 1, 2, 0, 1, 1, "MODEL" ], [ 2, 2, 1, 1, 2, "CLIP" ], [ 3, 2, 2, 1, 3, "VAE" ], [ 4, 3, 0, 4, 0, "deforum_data" ], [ 5, 4, 0, 5, 0, "deforum_data" ], [ 6, 5, 0, 1, 0, "deforum_data" ], [ 7, 1, 0, 6, 0, "IMAGE" ] ], "groups": [], "config": {}, "extra": { "ds": { "scale": 0.9090909090909091, "offset": [ -231.98330528838665, 480.9216297942885 ] } }, "version": 0.4 } ================================================ FILE: examples/deforum_ip_adapter.json ================================================ { "last_node_id": 176, "last_link_id": 558, "nodes": [ { "id": 141, "type": "Reroute", "pos": [ 532, 972 ], "size": [ 75, 26 ], "flags": {}, "order": 19, "mode": 0, "inputs": [ { "name": "", "type": "*", "link": 554 } ], "outputs": [ { "name": "", "type": "DEFORUM_FRAME_DATA", "links": [ 441, 443, 447, 459 ] } ], "properties": { "showOutputText": false, "horizontal": false } }, { "id": 157, "type": "DeforumHybridScheduleNode", "pos": [ -300, 550 ], "size": { "0": 274.891845703125, "1": 178 }, "flags": {}, "order": 15, "mode": 0, "inputs": [ { "name": "deforum_data", "type": "deforum_data", "link": 540 } ], "outputs": [ { "name": "deforum_data", "type": "deforum_data", "links": [ 541 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumHybridScheduleNode" }, "widgets_values": [ "0:(0.5)", "0:(0.5)", "0:(1)", "0:(100)", "0:(0)", "0:(0.8)" ] }, { "id": 155, "type": "DeforumColorParamsNode", "pos": [ -660, 910 ], "size": { "0": 317.4000244140625, "1": 154 }, "flags": {}, "order": 12, "mode": 0, "inputs": [ { "name": "deforum_data", "type": "deforum_data", "link": 535 } ], "outputs": [ { "name": "deforum_data", "type": "deforum_data", "links": [ 548 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumColorParamsNode" }, "widgets_values": [ "Image", "", 1, false, false ] }, { "id": 126, "type": "DeforumPromptNode", "pos": [ -650, -270 ], "size": { "0": 313.04736328125, "1": 276.55352783203125 }, "flags": {}, "order": 0, "mode": 0, "inputs": [ { "name": "deforum_data", "type": "deforum_data", "link": null } ], "outputs": [ { "name": "deforum_data", "type": "deforum_data", "links": [ 537 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumPromptNode" }, "widgets_values": [ "0:'the deathstar colliding with a planet in space, explosion'\n100:'dart vader in the desert, DUNE GIANT WORM, by moebius'" ] }, { "id": 150, "type": "DeforumAnimParamsNode", "pos": [ -660, 70 ], "size": { "0": 317.4000244140625, "1": 178 }, "flags": {}, "order": 7, "mode": 0, "inputs": [ { "name": "deforum_data", "type": "deforum_data", "link": 537 } ], "outputs": [ { "name": "deforum_data", "type": "deforum_data", "links": [ 497 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumAnimParamsNode" }, "widgets_values": [ "3D", 100, "wrap", false, "20230129210106", false ] }, { "id": 151, "type": "DeforumDepthParamsNode", "pos": [ -660, 320 ], "size": { "0": 317.4000244140625, "1": 178 }, "flags": {}, "order": 10, "mode": 0, "inputs": [ { "name": "deforum_data", "type": "deforum_data", "link": 497 } ], "outputs": [ { "name": "deforum_data", "type": "deforum_data", "links": [ 498 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumDepthParamsNode" }, "widgets_values": [ true, "Zoe", 0.2, "border", "bicubic", false ] }, { "id": 152, "type": "DeforumTranslationParamsNode", "pos": [ -660, 570 ], "size": { "0": 317.4000244140625, "1": 274 }, "flags": {}, "order": 11, "mode": 0, "inputs": [ { "name": "deforum_data", "type": "deforum_data", "link": 498 } ], "outputs": [ { "name": "deforum_data", "type": "deforum_data", "links": [ 535 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumTranslationParamsNode" }, "widgets_values": [ "0:(0)", "0:(1.0)", "0:(0)", "0:(0)", "0:(0)", "0:(0.5)", "0:(0.5)", "0:(0)", "0:(0)", "0:(0)" ] }, { "id": 130, "type": "DeforumKSampler", "pos": [ 650, 120 ], "size": { "0": 325.93902587890625, "1": 106 }, "flags": {}, "order": 20, "mode": 0, "inputs": [ { "name": "model", "type": "MODEL", "link": 547 }, { "name": "latent", "type": "LATENT", "link": 555, "slot_index": 1 }, { "name": "positive", "type": "CONDITIONING", "link": 430 }, { "name": "negative", "type": "CONDITIONING", "link": 431, "slot_index": 3 }, { "name": "deforum_frame_data", "type": "DEFORUM_FRAME_DATA", "link": 459 } ], "outputs": [ { "name": "LATENT", "type": "LATENT", "links": [ 416 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumKSampler" } }, { "id": 131, "type": "VAEDecode", "pos": [ 690, 310 ], "size": { "0": 210, "1": 46 }, "flags": {}, "order": 21, "mode": 0, "inputs": [ { "name": "samples", "type": "LATENT", "link": 416 }, { "name": "vae", "type": "VAE", "link": 417, "slot_index": 1 } ], "outputs": [ { "name": "IMAGE", "type": "IMAGE", "links": [ 462, 469, 475 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "VAEDecode" } }, { "id": 62, "type": "PreviewImage", "pos": [ 98, 388 ], "size": { "0": 309.7149658203125, "1": 251.35919189453125 }, "flags": {}, "order": 8, "mode": 0, "inputs": [ { "name": "images", "type": "IMAGE", "link": 556, "slot_index": 0 } ], "properties": { "Node name for S&R": "PreviewImage" } }, { "id": 142, "type": "DeforumHybridMotionNode", "pos": [ 639, 428 ], "size": { "0": 315, "1": 98 }, "flags": {}, "order": 23, "mode": 4, "inputs": [ { "name": "image", "type": "IMAGE", "link": 469 }, { "name": "hybrid_image", "type": "IMAGE", "link": 557 }, { "name": "deforum_frame_data", "type": "DEFORUM_FRAME_DATA", "link": 447 } ], "outputs": [ { "name": "IMAGE", "type": "IMAGE", "links": [ 440, 460 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumHybridMotionNode" }, "widgets_values": [ "RAFT" ] }, { "id": 47, "type": "PreviewImage", "pos": [ 989, 391 ], "size": { "0": 192.6160888671875, "1": 169.56472778320312 }, "flags": {}, "order": 26, "mode": 4, "inputs": [ { "name": "images", "type": "IMAGE", "link": 460 } ], "properties": { "Node name for S&R": "PreviewImage" } }, { "id": 48, "type": "PreviewImage", "pos": [ 992, 607 ], "size": [ 188.46986389160156, 246 ], "flags": {}, "order": 27, "mode": 0, "inputs": [ { "name": "images", "type": "IMAGE", "link": 455 } ], "properties": { "Node name for S&R": "PreviewImage" } }, { "id": 50, "type": "PreviewImage", "pos": [ 995, 817 ], "size": [ 189.73870849609375, 246 ], "flags": {}, "order": 29, "mode": 0, "inputs": [ { "name": "images", "type": "IMAGE", "link": 456, "slot_index": 0 } ], "properties": { "Node name for S&R": "PreviewImage" } }, { "id": 138, "type": "DeforumFrameWarpNode", "pos": [ 660, 668 ], "size": { "0": 304.79998779296875, "1": 46 }, "flags": {}, "order": 25, "mode": 0, "inputs": [ { "name": "image", "type": "IMAGE", "link": 440 }, { "name": "deforum_frame_data", "type": "DEFORUM_FRAME_DATA", "link": 441 } ], "outputs": [ { "name": "IMAGE", "type": "IMAGE", "links": [ 455, 488 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumFrameWarpNode" } }, { "id": 139, "type": "DeforumAddNoiseNode", "pos": [ 642, 875 ], "size": { "0": 324.6453552246094, "1": 46 }, "flags": {}, "order": 28, "mode": 0, "inputs": [ { "name": "image", "type": "IMAGE", "link": 488, "slot_index": 0 }, { "name": "deforum_frame_data", "type": "DEFORUM_FRAME_DATA", "link": 443, "slot_index": 1 } ], "outputs": [ { "name": "IMAGE", "type": "IMAGE", "links": [ 456, 489 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumAddNoiseNode" } }, { "id": 137, "type": "VAEEncode", "pos": [ 691, 1001 ], "size": { "0": 210, "1": 46 }, "flags": {}, "order": 30, "mode": 0, "inputs": [ { "name": "pixels", "type": "IMAGE", "link": 489 }, { "name": "vae", "type": "VAE", "link": 450, "slot_index": 1 } ], "outputs": [ { "name": "LATENT", "type": "LATENT", "links": [ 438 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "VAEEncode" } }, { "id": 136, "type": "DeforumCacheLatentNode", "pos": [ 681, 1134 ], "size": { "0": 210, "1": 26 }, "flags": {}, "order": 31, "mode": 0, "inputs": [ { "name": "latent", "type": "LATENT", "link": 438 } ], "outputs": [ { "name": "LATENT", "type": "LATENT", "links": null, "shape": 3 } ], "properties": { "Node name for S&R": "DeforumCacheLatentNode" } }, { "id": 170, "type": "Note", "pos": [ 474, -603 ], "size": { "0": 633.6494140625, "1": 221.03656005859375 }, "flags": {}, "order": 1, "mode": 0, "properties": { "text": "" }, "widgets_values": [ "Welcome to Deforum!\n\nThe below graph provides a pluggable Deforum Animation pipeline in ComfyUI. It works frame-by-frame, so the best practice is to enable Extra options, and Auto Queue.\n\nDeforum Parameter and Schedule nodes represent all settings available to consume after chaining up by the Deforum Iterator node, which keeps track of the current frame, generates/gets the cached latent to be denoised in the next pass. The parameters and schedule's are the same as in the auto1111 extension and in the Colab version.\n\nHybrid nodes are currently bypassed (purple), enabling them, and selecting a video transforms the pipeline into Deforum Hybrid. Iteration can be reset to frame 0 with the reset value being set to 1 on the iterator node. Don't forget to switch it back to 0 to use the generated image/latent.\"\n\nDeforum Video Save node dumps its collected frames when the current frame's id reaches/has passed max_frames." ], "color": "#432", "bgcolor": "#653" }, { "id": 133, "type": "DeforumConditioningBlendNode", "pos": [ 637, -334 ], "size": { "0": 342.5999755859375, "1": 78 }, "flags": {}, "order": 18, "mode": 0, "inputs": [ { "name": "clip", "type": "CLIP", "link": 428 }, { "name": "deforum_frame_data", "type": "DEFORUM_FRAME_DATA", "link": 553 } ], "outputs": [ { "name": "POSITIVE", "type": "CONDITIONING", "links": [ 430 ], "shape": 3 }, { "name": "NEGATIVE", "type": "CONDITIONING", "links": [ 431 ], "shape": 3 } ], "properties": { "Node name for S&R": "DeforumConditioningBlendNode" }, "widgets_values": [ "gaussian" ] }, { "id": 127, "type": "CheckpointLoaderSimple", "pos": [ 91, 743 ], "size": { "0": 315, "1": 98 }, "flags": {}, "order": 2, "mode": 0, "outputs": [ { "name": "MODEL", "type": "MODEL", "links": [ 546 ], "shape": 3, "slot_index": 0 }, { "name": "CLIP", "type": "CLIP", "links": [ 428 ], "shape": 3, "slot_index": 1 }, { "name": "VAE", "type": "VAE", "links": [ 417, 450 ], "shape": 3 } ], "properties": { "Node name for S&R": "CheckpointLoaderSimple" }, "widgets_values": [ "protovisionXLHighFidelity3D_beta0520Bakedvae.safetensors" ] }, { "id": 173, "type": "IPAdapterApply", "pos": [ 695, -199 ], "size": { "0": 210, "1": 258 }, "flags": {}, "order": 9, "mode": 0, "inputs": [ { "name": "ipadapter", "type": "IPADAPTER", "link": 543 }, { "name": "clip_vision", "type": "CLIP_VISION", "link": 544 }, { "name": "image", "type": "IMAGE", "link": 558 }, { "name": "model", "type": "MODEL", "link": 546 }, { "name": "attn_mask", "type": "MASK", "link": null } ], "outputs": [ { "name": "MODEL", "type": "MODEL", "links": [ 547 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "IPAdapterApply" }, "widgets_values": [ 0.8, 0, "original", 0, 1, false ] }, { "id": 172, "type": "CLIPVisionLoader", "pos": [ 103, -115 ], "size": { "0": 300, "1": 60 }, "flags": {}, "order": 3, "mode": 0, "outputs": [ { "name": "CLIP_VISION", "type": "CLIP_VISION", "links": [ 544 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "CLIPVisionLoader" }, "widgets_values": [ "ipadapter_image_encoder_15.safetensors" ] }, { "id": 153, "type": "DeforumNoiseParamsNode", "pos": [ -300, 810 ], "size": { "0": 272.06170654296875, "1": 298 }, "flags": {}, "order": 16, "mode": 0, "inputs": [ { "name": "deforum_data", "type": "deforum_data", "link": 541 } ], "outputs": [ { "name": "deforum_data", "type": "deforum_data", "links": [ 550 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumNoiseParamsNode" }, "widgets_values": [ true, "0: (0.8)", "0: (0.1)", "0: (5)", "0: (1.0)", "0: (0.0)", "perlin", 8, 8, 4, 0.5 ] }, { "id": 16, "type": "PreviewImage", "pos": [ 1270, -263 ], "size": { "0": 894.2135009765625, "1": 941.8845825195312 }, "flags": {}, "order": 22, "mode": 0, "inputs": [ { "name": "images", "type": "IMAGE", "link": 462, "slot_index": 0 } ], "properties": { "Node name for S&R": "PreviewImage" } }, { "id": 102, "type": "DeforumVideoSaveNode", "pos": [ 1524, 804 ], "size": { "0": 315, "1": 150 }, "flags": {}, "order": 24, "mode": 0, "inputs": [ { "name": "image", "type": "IMAGE", "link": 475 }, { "name": "deforum_frame_data", "type": "DEFORUM_FRAME_DATA", "link": 552 } ], "properties": { "Node name for S&R": "DeforumVideoSaveNode" }, "widgets_values": [ "deforum_", 12, "max_frames", 0 ] }, { "id": 174, "type": "DeforumBaseParamsNode", "pos": [ -318, -265 ], "size": { "0": 317.4000244140625, "1": 442 }, "flags": {}, "order": 13, "mode": 0, "inputs": [ { "name": "deforum_data", "type": "deforum_data", "link": 548 } ], "outputs": [ { "name": "deforum_data", "type": "deforum_data", "links": [ 549 ], "shape": 3 } ], "properties": { "Node name for S&R": "DeforumBaseParamsNode" }, "widgets_values": [ 1024, 1024, "euler", "normal", 25, 7, 1, "Deforum_{timestring}", "fixed", 1, "output/deforum", 0.8, true, false, false, true, false ] }, { "id": 175, "type": "DeforumIteratorNode", "pos": [ 55, 914 ], "size": { "0": 393, "1": 286 }, "flags": {}, "order": 17, "mode": 0, "inputs": [ { "name": "deforum_data", "type": "deforum_data", "link": 550 }, { "name": "latent", "type": "LATENT", "link": 551 } ], "outputs": [ { "name": "deforum_frame_data", "type": "DEFORUM_FRAME_DATA", "links": [ 552, 553, 554 ], "shape": 3 }, { "name": "latent", "type": "LATENT", "links": [ 555 ], "shape": 3 }, { "name": "positive_prompt", "type": "STRING", "links": null, "shape": 3 }, { "name": "negative_prompt", "type": "STRING", "links": null, "shape": 3 } ], "properties": { "Node name for S&R": "DeforumIteratorNode" }, "widgets_values": [ "stable_diffusion", 791126192911641, "randomize", 0, 0.8, 0.1, false, false ] }, { "id": 135, "type": "DeforumGetCachedLatentNode", "pos": [ 132, 1262 ], "size": { "0": 218.39999389648438, "1": 26 }, "flags": {}, "order": 4, "mode": 0, "outputs": [ { "name": "LATENT", "type": "LATENT", "links": [ 551 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumGetCachedLatentNode" } }, { "id": 176, "type": "DeforumLoadVideo", "pos": [ 94, 17 ], "size": [ 315, 294 ], "flags": {}, "order": 5, "mode": 0, "outputs": [ { "name": "IMAGE", "type": "IMAGE", "links": [ 556, 557, 558 ], "shape": 3 } ], "properties": { "Node name for S&R": "DeforumLoadVideo" }, "widgets_values": [ "2000_FILM (1).mp4", "image" ] }, { "id": 154, "type": "DeforumDiffusionParamsNode", "pos": [ -300, 230 ], "size": { "0": 278.891845703125, "1": 274 }, "flags": {}, "order": 14, "mode": 0, "inputs": [ { "name": "deforum_data", "type": "deforum_data", "link": 549 } ], "outputs": [ { "name": "deforum_data", "type": "deforum_data", "links": [ 540 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumDiffusionParamsNode" }, "widgets_values": [ "0: (0.03)", "0: (0.3)", "0: (1.0)", "0: (5)", false, "0: (25)", false, "0:(0)", false, "0:(1)" ] }, { "id": 171, "type": "IPAdapterModelLoader", "pos": [ 105, -243 ], "size": { "0": 300, "1": 60 }, "flags": {}, "order": 6, "mode": 0, "outputs": [ { "name": "IPADAPTER", "type": "IPADAPTER", "links": [ 543 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "IPAdapterModelLoader" }, "widgets_values": [ "ip-adapter_sdxl_vit-h.safetensors" ] } ], "links": [ [ 416, 130, 0, 131, 0, "LATENT" ], [ 417, 127, 2, 131, 1, "VAE" ], [ 428, 127, 1, 133, 0, "CLIP" ], [ 430, 133, 0, 130, 2, "CONDITIONING" ], [ 431, 133, 1, 130, 3, "CONDITIONING" ], [ 438, 137, 0, 136, 0, "LATENT" ], [ 440, 142, 0, 138, 0, "IMAGE" ], [ 441, 141, 0, 138, 1, "DEFORUM_FRAME_DATA" ], [ 443, 141, 0, 139, 1, "DEFORUM_FRAME_DATA" ], [ 447, 141, 0, 142, 2, "DEFORUM_FRAME_DATA" ], [ 450, 127, 2, 137, 1, "VAE" ], [ 455, 138, 0, 48, 0, "IMAGE" ], [ 456, 139, 0, 50, 0, "IMAGE" ], [ 459, 141, 0, 130, 4, "DEFORUM_FRAME_DATA" ], [ 460, 142, 0, 47, 0, "IMAGE" ], [ 462, 131, 0, 16, 0, "IMAGE" ], [ 469, 131, 0, 142, 0, "IMAGE" ], [ 475, 131, 0, 102, 0, "IMAGE" ], [ 488, 138, 0, 139, 0, "IMAGE" ], [ 489, 139, 0, 137, 0, "IMAGE" ], [ 497, 150, 0, 151, 0, "deforum_data" ], [ 498, 151, 0, 152, 0, "deforum_data" ], [ 535, 152, 0, 155, 0, "deforum_data" ], [ 537, 126, 0, 150, 0, "deforum_data" ], [ 540, 154, 0, 157, 0, "deforum_data" ], [ 541, 157, 0, 153, 0, "deforum_data" ], [ 543, 171, 0, 173, 0, "IPADAPTER" ], [ 544, 172, 0, 173, 1, "CLIP_VISION" ], [ 546, 127, 0, 173, 3, "MODEL" ], [ 547, 173, 0, 130, 0, "MODEL" ], [ 548, 155, 0, 174, 0, "deforum_data" ], [ 549, 174, 0, 154, 0, "deforum_data" ], [ 550, 153, 0, 175, 0, "deforum_data" ], [ 551, 135, 0, 175, 1, "LATENT" ], [ 552, 175, 0, 102, 1, "DEFORUM_FRAME_DATA" ], [ 553, 175, 0, 133, 1, "DEFORUM_FRAME_DATA" ], [ 554, 175, 0, 141, 0, "*" ], [ 555, 175, 1, 130, 1, "LATENT" ], [ 556, 176, 0, 62, 0, "IMAGE" ], [ 557, 176, 0, 142, 1, "IMAGE" ], [ 558, 176, 0, 173, 2, "IMAGE" ] ], "groups": [], "config": {}, "extra": { "groupNodes": { "Deforum Parameters": { "nodes": [ { "type": "DeforumBaseParamsNode", "pos": [ -325, 78 ], "size": { "0": 317.4000244140625, "1": 490 }, "flags": {}, "order": 4, "mode": 0, "inputs": [ { "name": "deforum_data", "type": "deforum_data", "link": null } ], "outputs": [ { "name": "deforum_data", "type": "deforum_data", "links": [], "shape": 3 } ], "properties": { "Node name for S&R": "DeforumBaseParamsNode" }, "widgets_values": [ 1366, 1024, "dpmpp_2m_sde", "karras", 14, 6, 1, "Deforum_{timestring}", "iter", 1, "output/deforum", 0.4, true, false, false, true, false ], "index": 0 }, { "type": "DeforumAnimParamsNode", "pos": [ -315, 598 ], "size": { "0": 317.4000244140625, "1": 178 }, "flags": {}, "order": 5, "mode": 0, "inputs": [ { "name": "deforum_data", "type": "deforum_data", "link": null } ], "outputs": [ { "name": "deforum_data", "type": "deforum_data", "links": [], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumAnimParamsNode" }, "widgets_values": [ "3D", 150, "wrap", false, "20230129210106", false ], "index": 1 }, { "type": "DeforumDepthParamsNode", "pos": [ 45, 68 ], "size": { "0": 317.4000244140625, "1": 178 }, "flags": {}, "order": 9, "mode": 0, "inputs": [ { "name": "deforum_data", "type": "deforum_data", "link": null } ], "outputs": [ { "name": "deforum_data", "type": "deforum_data", "links": [], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumDepthParamsNode" }, "widgets_values": [ true, "Zoe", 0.2, "border", "bicubic", false ], "index": 2 }, { "type": "DeforumTranslationParamsNode", "pos": [ 45, 308 ], "size": { "0": 317.4000244140625, "1": 274 }, "flags": {}, "order": 10, "mode": 0, "inputs": [ { "name": "deforum_data", "type": "deforum_data", "link": null } ], "outputs": [ { "name": "deforum_data", "type": "deforum_data", "links": [], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumTranslationParamsNode" }, "widgets_values": [ "0:(0)", "0:(1.0)", "0:(0)", "0:(0)", "0:(0)", "0:(0.5)", "0:(0.5)", "0:(0)", "0:(0)", "0:(0)" ], "index": 3 }, { "type": "DeforumNoiseParamsNode", "pos": [ 45, 638 ], "size": { "0": 317.4000244140625, "1": 298 }, "flags": {}, "order": 15, "mode": 0, "inputs": [ { "name": "deforum_data", "type": "deforum_data", "link": null } ], "outputs": [ { "name": "deforum_data", "type": "deforum_data", "links": [], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumNoiseParamsNode" }, "widgets_values": [ true, "0: (0.4)", "0: (0.1)", "0: (5)", "0: (1.0)", "0: (0.0)", "perlin", 8, 8, 4, 0.5 ], "index": 4 }, { "type": "DeforumDiffusionParamsNode", "pos": [ -315, 858 ], "size": { "0": 317.4000244140625, "1": 274 }, "flags": {}, "order": 16, "mode": 0, "inputs": [ { "name": "deforum_data", "type": "deforum_data", "link": null } ], "outputs": [ { "name": "deforum_data", "type": "deforum_data", "links": [], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumDiffusionParamsNode" }, "widgets_values": [ "0: (0.03)", "0: (0.45)", "0: (1.0)", "0: (5)", false, "0: (25)", false, "0:(0)", false, "0:(1)" ], "index": 5 }, { "type": "DeforumColorParamsNode", "pos": [ 40, 988 ], "size": { "0": 317.4000244140625, "1": 154 }, "flags": {}, "order": 17, "mode": 0, "inputs": [ { "name": "deforum_data", "type": "deforum_data", "link": null } ], "outputs": [ { "name": "deforum_data", "type": "deforum_data", "links": [], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumColorParamsNode" }, "widgets_values": [ "Image", "", 1, false, false ], "index": 6 } ], "links": [ [ null, 1, 0, 0, 112, "deforum_data" ], [ 0, 0, 1, 0, 20, "deforum_data" ], [ 1, 0, 2, 0, 21, "deforum_data" ], [ 2, 0, 3, 0, 42, "deforum_data" ], [ 5, 0, 4, 0, 52, "deforum_data" ], [ 3, 0, 5, 0, 44, "deforum_data" ], [ 4, 0, 6, 0, 51, "deforum_data" ] ], "external": [ [ 6, 0, "deforum_data" ] ] }, "Deforum Sampler": { "nodes": [ { "type": "DeforumPromptNode", "pos": [ 845, 445 ], "size": { "0": 304.5645751953125, "1": 211.12216186523438 }, "flags": {}, "order": 1, "mode": 0, "inputs": [ { "name": "deforum_data", "type": "deforum_data", "link": null } ], "outputs": [ { "name": "deforum_data", "type": "deforum_data", "links": [], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumPromptNode" }, "widgets_values": [ "0:'whirling chaotic tornado, lighthouse, polygons in space, abstract artwork, light illusion'\n25:'chaotic abstract painting'" ], "index": 0 }, { "type": "CheckpointLoaderSimple", "pos": [ 636, -55 ], "size": { "0": 315, "1": 98 }, "flags": {}, "order": 2, "mode": 0, "outputs": [ { "name": "MODEL", "type": "MODEL", "links": [], "shape": 3, "slot_index": 0 }, { "name": "CLIP", "type": "CLIP", "links": [], "shape": 3, "slot_index": 1 }, { "name": "VAE", "type": "VAE", "links": [], "shape": 3 } ], "properties": { "Node name for S&R": "CheckpointLoaderSimple" }, "widgets_values": [ "protovisionXLHighFidelity3D_beta0520Bakedvae.safetensors" ], "index": 1 }, { "type": "DeepCache", "pos": [ 420, 100 ], "size": { "0": 315, "1": 130 }, "flags": {}, "order": 5, "mode": 0, "inputs": [ { "name": "model", "type": "MODEL", "link": null } ], "outputs": [ { "name": "MODEL", "type": "MODEL", "links": [], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeepCache" }, "widgets_values": [ 4, 2, 0, 1000 ], "index": 2 }, { "type": "DeforumIteratorNode", "pos": [ 420, 490 ], "size": { "0": 393, "1": 166 }, "flags": {}, "order": 13, "mode": 0, "inputs": [ { "name": "deforum_data", "type": "deforum_data", "link": null }, { "name": "latent", "type": "LATENT", "link": null } ], "outputs": [ { "name": "deforum_frame_data", "type": "DEFORUM_FRAME_DATA", "links": [], "shape": 3, "slot_index": 0 }, { "name": "latent", "type": "LATENT", "links": [], "shape": 3 }, { "name": "positive_prompt", "type": "STRING", "links": [], "shape": 3 }, { "name": "negative_prompt", "type": "STRING", "links": [], "shape": 3 } ], "properties": { "Node name for S&R": "DeforumIteratorNode" }, "widgets_values": [ 282766464392687, "increment", 0 ], "index": 3 }, { "type": "DeforumKSampler", "pos": [ 460, 310 ], "size": { "0": 317.4000244140625, "1": 106 }, "flags": {}, "order": 14, "mode": 0, "inputs": [ { "name": "model", "type": "MODEL", "link": null }, { "name": "latent", "type": "LATENT", "link": null, "slot_index": 1 }, { "name": "positive", "type": "CONDITIONING", "link": null }, { "name": "negative", "type": "CONDITIONING", "link": null, "slot_index": 3 }, { "name": "deforum_frame_data", "type": "DEFORUM_FRAME_DATA", "link": null } ], "outputs": [ { "name": "LATENT", "type": "LATENT", "links": [], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumKSampler" }, "index": 4 }, { "type": "VAEDecode", "pos": [ 920, 319 ], "size": { "0": 210, "1": 46 }, "flags": {}, "order": 15, "mode": 0, "inputs": [ { "name": "samples", "type": "LATENT", "link": null }, { "name": "vae", "type": "VAE", "link": null, "slot_index": 1 } ], "outputs": [ { "name": "IMAGE", "type": "IMAGE", "links": [], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "VAEDecode" }, "index": 5 }, { "type": "DeforumConditioningBlendNode", "pos": [ 841, 148 ], "size": { "0": 342.5999755859375, "1": 78 }, "flags": {}, "order": 16, "mode": 0, "inputs": [ { "name": "clip", "type": "CLIP", "link": null }, { "name": "deforum_frame_data", "type": "DEFORUM_FRAME_DATA", "link": null, "slot_index": 1 } ], "outputs": [ { "name": "POSITIVE", "type": "CONDITIONING", "links": [], "shape": 3, "slot_index": 0 }, { "name": "NEGATIVE", "type": "CONDITIONING", "links": [], "shape": 3 } ], "properties": { "Node name for S&R": "DeforumConditioningBlendNode" }, "widgets_values": [ "sigmoidal" ], "index": 6 } ], "links": [ [ 1, 0, 2, 0, 115, "MODEL" ], [ null, 0, 3, 0, 113, "deforum_data" ], [ null, 0, 3, 1, 114, "LATENT" ], [ 2, 0, 4, 0, 117, "MODEL" ], [ 3, 1, 4, 1, 118, "LATENT" ], [ 6, 0, 4, 2, 124, "CONDITIONING" ], [ 6, 1, 4, 3, 124, "CONDITIONING" ], [ null, 5, 4, 4, 114, "DEFORUM_FRAME_DATA" ], [ 4, 0, 5, 0, 121, "LATENT" ], [ 1, 2, 5, 1, 115, "VAE" ], [ 1, 1, 6, 0, 115, "CLIP" ], [ 3, 0, 6, 1, 118, "DEFORUM_FRAME_DATA" ] ], "external": [ [ 0, 0, "deforum_data" ], [ 1, 2, "VAE" ], [ 3, 0, "DEFORUM_FRAME_DATA" ], [ 5, 0, "IMAGE" ] ] }, "Deforum Operators": { "nodes": [ { "type": "DeforumGetCachedLatentNode", "pos": [ 1356, 1073 ], "size": { "0": 218.39999389648438, "1": 26 }, "flags": {}, "order": 1, "mode": 0, "outputs": [ { "name": "LATENT", "type": "LATENT", "links": [], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumGetCachedLatentNode" }, "index": 0 }, { "type": "DeforumCacheLatentNode", "pos": [ 1337, 1803 ], "size": { "0": 210, "1": 26 }, "flags": {}, "order": 4, "mode": 0, "inputs": [ { "name": "latent", "type": "LATENT", "link": null } ], "outputs": [ { "name": "LATENT", "type": "LATENT", "links": null, "shape": 3 } ], "properties": { "Node name for S&R": "DeforumCacheLatentNode" }, "index": 1 }, { "type": "VAEEncode", "pos": [ 1344, 1696 ], "size": { "0": 210, "1": 46 }, "flags": {}, "order": 5, "mode": 0, "inputs": [ { "name": "pixels", "type": "IMAGE", "link": null }, { "name": "vae", "type": "VAE", "link": null, "slot_index": 1 } ], "outputs": [ { "name": "LATENT", "type": "LATENT", "links": [], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "VAEEncode" }, "index": 2 }, { "type": "DeforumFrameWarpNode", "pos": [ 1301, 1392 ], "size": { "0": 304.79998779296875, "1": 46 }, "flags": {}, "order": 6, "mode": 0, "inputs": [ { "name": "image", "type": "IMAGE", "link": null }, { "name": "deforum_frame_data", "type": "DEFORUM_FRAME_DATA", "link": null } ], "outputs": [ { "name": "IMAGE", "type": "IMAGE", "links": [], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumFrameWarpNode" }, "index": 3 }, { "type": "DeforumAddNoiseNode", "pos": [ 1301, 1590 ], "size": { "0": 304.79998779296875, "1": 46 }, "flags": {}, "order": 9, "mode": 0, "inputs": [ { "name": "image", "type": "IMAGE", "link": null }, { "name": "deforum_frame_data", "type": "DEFORUM_FRAME_DATA", "link": null, "slot_index": 1 } ], "outputs": [ { "name": "IMAGE", "type": "IMAGE", "links": [], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumAddNoiseNode" }, "index": 4 }, { "type": "DeforumColorMatchNode", "pos": [ 1303, 1157 ], "size": { "0": 304.79998779296875, "1": 46 }, "flags": {}, "order": 11, "mode": 0, "inputs": [ { "name": "image", "type": "IMAGE", "link": null }, { "name": "deforum_frame_data", "type": "DEFORUM_FRAME_DATA", "link": null } ], "outputs": [ { "name": "IMAGE", "type": "IMAGE", "links": [], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumColorMatchNode" }, "index": 5 }, { "type": "Reroute", "pos": [ 1079, 1292 ], "size": [ 75, 26 ], "flags": {}, "order": 12, "mode": 0, "inputs": [ { "name": "", "type": "*", "link": null } ], "outputs": [ { "name": "", "type": "*", "links": null } ], "properties": { "showOutputText": false, "horizontal": false }, "index": 6 }, { "type": "DeforumHybridMotionNode", "pos": [ 1298, 1249 ], "size": { "0": 315, "1": 98 }, "flags": {}, "order": 14, "mode": 0, "inputs": [ { "name": "image", "type": "IMAGE", "link": null }, { "name": "hybrid_image", "type": "IMAGE", "link": null }, { "name": "deforum_frame_data", "type": "DEFORUM_FRAME_DATA", "link": null } ], "outputs": [ { "name": "IMAGE", "type": "IMAGE", "links": [], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumHybridMotionNode" }, "widgets_values": [ "RAFT" ], "index": 7 }, { "type": "DeforumColorMatchNode", "pos": [ 1297, 1492 ], "size": { "0": 304.79998779296875, "1": 46 }, "flags": {}, "order": 15, "mode": 0, "inputs": [ { "name": "image", "type": "IMAGE", "link": null, "slot_index": 0 }, { "name": "deforum_frame_data", "type": "DEFORUM_FRAME_DATA", "link": null } ], "outputs": [ { "name": "IMAGE", "type": "IMAGE", "links": [], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumColorMatchNode" }, "index": 8 } ], "links": [ [ 2, 0, 1, 0, 36, "LATENT" ], [ 8, 0, 2, 0, 109, "IMAGE" ], [ null, 0, 2, 1, 112, "VAE" ], [ 7, 0, 3, 0, 108, "IMAGE" ], [ 6, 0, 3, 1, 98, "DEFORUM_FRAME_DATA" ], [ 8, 0, 4, 0, 109, "IMAGE" ], [ 6, 0, 4, 1, 98, "DEFORUM_FRAME_DATA" ], [ null, 3, 5, 0, 112, "IMAGE" ], [ 6, 0, 5, 1, 98, "DEFORUM_FRAME_DATA" ], [ null, 2, 6, 0, 112, "DEFORUM_FRAME_DATA" ], [ 5, 0, 7, 0, 95, "IMAGE" ], [ null, 0, 7, 1, 88, "IMAGE" ], [ 6, 0, 7, 2, 98, "DEFORUM_FRAME_DATA" ], [ 3, 0, 8, 0, 41, "IMAGE" ], [ 6, 0, 8, 1, 98, "DEFORUM_FRAME_DATA" ] ], "external": [ [ 0, 0, "LATENT" ], [ 3, 0, "IMAGE" ], [ 4, 0, "IMAGE" ], [ 5, 0, "IMAGE" ], [ 6, 0, "DEFORUM_FRAME_DATA" ], [ 7, 0, "IMAGE" ], [ 8, 0, "IMAGE" ] ] } } }, "version": 0.4 } ================================================ FILE: examples/deforum_simple.json ================================================ { "last_node_id": 26, "last_link_id": 56, "nodes": [ { "id": 7, "type": "PreviewImage", "pos": [ 1370, 230 ], "size": { "0": 733.25, "1": 732.75 }, "flags": {}, "order": 4, "mode": 0, "inputs": [ { "name": "images", "type": "IMAGE", "link": 56 } ], "properties": { "Node name for S&R": "PreviewImage" } }, { "id": 14, "type": "Note", "pos": [ 300, 202 ], "size": { "0": 490.25, "1": 267.75 }, "flags": {}, "order": 0, "mode": 0, "properties": { "text": "" }, "widgets_values": [ "Welcome to Deforum Comfy.\n\nThis is the simplest example to achieve Deforum-esque animations, right click on Deforum Sampling -> Convert to Nodes to expose the inner logic, and add optional Deforum image processors, or anything you'd like.\n\nThe way Deforum works requires a full run of a graph for each frame currently (with the exception of image interpolations [WIP]), thus hitting Queue Prompt generates one frame of your animation. Use Extra Options - Auto Queue for continous generation.\n\nThe Deforum Video Save node saves it's cached frames when the current_frame reaches max_frames set in Deforum. The current frame advances with each generation, and resets to 0 when reaching/passing max_frames (i.e. max frames set lower then current frame while running), or by setting the reset parameter of the Iterator node to 1. Don't forget to flip it back to 0, as it is resetting the latents as well." ], "color": "#432", "bgcolor": "#653" }, { "id": 1, "type": "CheckpointLoaderSimple", "pos": [ 800, 310 ], "size": { "0": 501.75, "1": 108 }, "flags": {}, "order": 1, "mode": 0, "outputs": [ { "name": "MODEL", "type": "MODEL", "links": [ 54 ], "shape": 3, "slot_index": 0 }, { "name": "CLIP", "type": "CLIP", "links": [ 51, 52 ], "shape": 3, "slot_index": 1 }, { "name": "VAE", "type": "VAE", "links": [ 55 ], "shape": 3, "slot_index": 2 } ], "properties": { "Node name for S&R": "CheckpointLoaderSimple" }, "widgets_values": [ "protovisionXLHighFidelity3D_beta0520Bakedvae.safetensors" ] }, { "id": 15, "type": "DeforumBaseParamsNode", "pos": [ 473, 530 ], "size": { "0": 317.4000244140625, "1": 442 }, "flags": {}, "order": 2, "mode": 0, "inputs": [ { "name": "deforum_data", "type": "deforum_data", "link": null } ], "outputs": [ { "name": "deforum_data", "type": "deforum_data", "links": [ 53 ], "shape": 3 } ], "properties": { "Node name for S&R": "DeforumBaseParamsNode" }, "widgets_values": [ 1024, 1024, "euler", "normal", 25, 7, 1, "Deforum_{timestring}", "fixed", 1, "output/deforum", 0.4, true, false, false, true, false ] }, { "id": 26, "type": "workflow/Deforum Sampler", "pos": [ 839, 540 ], "size": { "0": 443.4000244140625, "1": 418 }, "flags": {}, "order": 3, "mode": 0, "inputs": [ { "name": "clip", "type": "CLIP", "link": 51 }, { "name": "CLIPTextEncode clip", "type": "CLIP", "link": 52 }, { "name": "deforum_data", "type": "deforum_data", "link": 53 }, { "name": "init_latent", "type": "LATENT", "link": null }, { "name": "model", "type": "MODEL", "link": 54 }, { "name": "vae", "type": "VAE", "link": 55 } ], "outputs": [ { "name": "positive_prompt", "type": "STRING", "links": null, "shape": 3 }, { "name": "negative_prompt", "type": "STRING", "links": null, "shape": 3 }, { "name": "IMAGE", "type": "IMAGE", "links": [ 56 ], "shape": 3, "slot_index": 2 }, { "name": "LATENT", "type": "LATENT", "links": null, "shape": 3 } ], "properties": { "Node name for S&R": "workflow/Deforum Sampler" }, "widgets_values": [ "", "", "stable_diffusion", 785978144431937, "randomize", 0, 0.8, 0.1, false, false ] } ], "links": [ [ 51, 1, 1, 26, 0, "CLIP" ], [ 52, 1, 1, 26, 1, "CLIP" ], [ 53, 15, 0, 26, 2, "deforum_data" ], [ 54, 1, 0, 26, 4, "MODEL" ], [ 55, 1, 2, 26, 5, "VAE" ], [ 56, 26, 2, 7, 0, "IMAGE" ] ], "groups": [], "config": {}, "extra": { "groupNodes": { "Deforum Sampler": { "nodes": [ { "type": "DeforumGetCachedLatentNode", "pos": [ 926, 1437 ], "size": { "0": 239.2784423828125, "1": 26 }, "flags": {}, "order": 2, "mode": 0, "outputs": [ { "name": "LATENT", "type": "LATENT", "links": [], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumGetCachedLatentNode" }, "index": 0 }, { "type": "CLIPTextEncode", "pos": [ 875, 558 ], "size": { "0": 400, "1": 200 }, "flags": {}, "order": 4, "mode": 0, "inputs": [ { "name": "clip", "type": "CLIP", "link": null } ], "outputs": [ { "name": "CONDITIONING", "type": "CONDITIONING", "links": [], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "CLIPTextEncode" }, "widgets_values": [ "" ], "index": 1 }, { "type": "CLIPTextEncode", "pos": [ 865, 818 ], "size": { "0": 400, "1": 200 }, "flags": {}, "order": 5, "mode": 0, "inputs": [ { "name": "clip", "type": "CLIP", "link": null } ], "outputs": [ { "name": "CONDITIONING", "type": "CONDITIONING", "links": [], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "CLIPTextEncode" }, "widgets_values": [ "" ], "index": 2 }, { "type": "DeforumIteratorNode", "pos": [ 848, 1073 ], "size": { "0": 393, "1": 286 }, "flags": {}, "order": 6, "mode": 0, "inputs": [ { "name": "deforum_data", "type": "deforum_data", "link": null }, { "name": "latent", "type": "LATENT", "link": null } ], "outputs": [ { "name": "deforum_frame_data", "type": "DEFORUM_FRAME_DATA", "links": [], "shape": 3 }, { "name": "latent", "type": "LATENT", "links": [], "shape": 3 }, { "name": "positive_prompt", "type": "STRING", "links": null, "shape": 3 }, { "name": "negative_prompt", "type": "STRING", "links": null, "shape": 3 } ], "properties": { "Node name for S&R": "DeforumIteratorNode" }, "widgets_values": [ "stable_diffusion", 923224016066246, "randomize", 0, 0.8, 0.1, false, false ], "index": 3 }, { "type": "DeforumKSampler", "pos": [ 903, 1724 ], "size": { "0": 317.4000244140625, "1": 106 }, "flags": {}, "order": 7, "mode": 0, "inputs": [ { "name": "model", "type": "MODEL", "link": null }, { "name": "latent", "type": "LATENT", "link": null }, { "name": "positive", "type": "CONDITIONING", "link": null }, { "name": "negative", "type": "CONDITIONING", "link": null }, { "name": "deforum_frame_data", "type": "DEFORUM_FRAME_DATA", "link": null } ], "outputs": [ { "name": "LATENT", "type": "LATENT", "links": [], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumKSampler" }, "index": 4 }, { "type": "VAEDecode", "pos": [ 949, 1613 ], "size": { "0": 210, "1": 46 }, "flags": {}, "order": 8, "mode": 0, "inputs": [ { "name": "samples", "type": "LATENT", "link": null }, { "name": "vae", "type": "VAE", "link": null } ], "outputs": [ { "name": "IMAGE", "type": "IMAGE", "links": [], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "VAEDecode" }, "index": 5 }, { "type": "DeforumCacheLatentNode", "pos": [ 935, 1521 ], "size": { "0": 210, "1": 26 }, "flags": {}, "order": 9, "mode": 0, "inputs": [ { "name": "latent", "type": "LATENT", "link": null } ], "outputs": [ { "name": "LATENT", "type": "LATENT", "links": null, "shape": 3 } ], "properties": { "Node name for S&R": "DeforumCacheLatentNode" }, "index": 6 } ], "links": [ [ null, 1, 1, 0, 1, "CLIP" ], [ null, 1, 2, 0, 1, "CLIP" ], [ null, 0, 3, 0, 15, "deforum_data" ], [ 0, 0, 3, 1, 17, "LATENT" ], [ null, 0, 4, 0, 1, "MODEL" ], [ 3, 1, 4, 1, 24, "LATENT" ], [ 1, 0, 4, 2, 18, "CONDITIONING" ], [ 2, 0, 4, 3, 19, "CONDITIONING" ], [ 3, 0, 4, 4, 24, "DEFORUM_FRAME_DATA" ], [ 4, 0, 5, 0, 21, "LATENT" ], [ null, 2, 5, 1, 1, "VAE" ], [ 4, 0, 6, 0, 21, "LATENT" ] ], "external": [ [ 5, 0, "IMAGE" ] ] } } }, "version": 0.4 } ================================================ FILE: examples/deforum_stablecascade.json ================================================ { "last_node_id": 198, "last_link_id": 598, "nodes": [ { "id": 141, "type": "Reroute", "pos": [ 532, 972 ], "size": [ 75, 26 ], "flags": {}, "order": 18, "mode": 0, "inputs": [ { "name": "", "type": "*", "link": 589 } ], "outputs": [ { "name": "", "type": "DEFORUM_FRAME_DATA", "links": [ 441, 443, 447, 459 ] } ], "properties": { "showOutputText": false, "horizontal": false } }, { "id": 157, "type": "DeforumHybridScheduleNode", "pos": [ -310, 760 ], "size": { "0": 274.891845703125, "1": 178 }, "flags": {}, "order": 14, "mode": 0, "inputs": [ { "name": "deforum_data", "type": "deforum_data", "link": 540 } ], "outputs": [ { "name": "deforum_data", "type": "deforum_data", "links": [ 541 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumHybridScheduleNode" }, "widgets_values": [ "0:(0.5)", "0:(0.5)", "0:(1)", "0:(100)", "0:(0)", "0:(0.8)" ] }, { "id": 155, "type": "DeforumColorParamsNode", "pos": [ -670, 1120 ], "size": { "0": 317.4000244140625, "1": 154 }, "flags": {}, "order": 11, "mode": 0, "inputs": [ { "name": "deforum_data", "type": "deforum_data", "link": 535 } ], "outputs": [ { "name": "deforum_data", "type": "deforum_data", "links": [ 538 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumColorParamsNode" }, "widgets_values": [ "Image", "", 1, false, false ] }, { "id": 150, "type": "DeforumAnimParamsNode", "pos": [ -670, 280 ], "size": { "0": 317.4000244140625, "1": 178 }, "flags": {}, "order": 7, "mode": 0, "inputs": [ { "name": "deforum_data", "type": "deforum_data", "link": 537 } ], "outputs": [ { "name": "deforum_data", "type": "deforum_data", "links": [ 497 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumAnimParamsNode" }, "widgets_values": [ "3D", 100, "wrap", false, "20230129210106", false ] }, { "id": 151, "type": "DeforumDepthParamsNode", "pos": [ -670, 530 ], "size": { "0": 317.4000244140625, "1": 178 }, "flags": {}, "order": 9, "mode": 0, "inputs": [ { "name": "deforum_data", "type": "deforum_data", "link": 497 } ], "outputs": [ { "name": "deforum_data", "type": "deforum_data", "links": [ 498 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumDepthParamsNode" }, "widgets_values": [ true, "Zoe", 0.2, "border", "bicubic", false ] }, { "id": 130, "type": "DeforumKSampler", "pos": [ 650, 152 ], "size": { "0": 325.93902587890625, "1": 106 }, "flags": {}, "order": 20, "mode": 0, "inputs": [ { "name": "model", "type": "MODEL", "link": 595 }, { "name": "latent", "type": "LATENT", "link": 590, "slot_index": 1 }, { "name": "positive", "type": "CONDITIONING", "link": 430 }, { "name": "negative", "type": "CONDITIONING", "link": 431, "slot_index": 3 }, { "name": "deforum_frame_data", "type": "DEFORUM_FRAME_DATA", "link": 459 } ], "outputs": [ { "name": "LATENT", "type": "LATENT", "links": [ 568 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumKSampler" } }, { "id": 191, "type": "ConditioningZeroOut", "pos": [ 692, 320 ], "size": { "0": 211.60000610351562, "1": 26 }, "flags": {}, "order": 19, "mode": 0, "inputs": [ { "name": "conditioning", "type": "CONDITIONING", "link": 569 } ], "outputs": [ { "name": "CONDITIONING", "type": "CONDITIONING", "links": [ 570 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "ConditioningZeroOut" } }, { "id": 190, "type": "StableCascade_StageB_Conditioning", "pos": [ 659, 392 ], "size": { "0": 277.20001220703125, "1": 46 }, "flags": {}, "order": 21, "mode": 0, "inputs": [ { "name": "conditioning", "type": "CONDITIONING", "link": 570 }, { "name": "stage_c", "type": "LATENT", "link": 568 } ], "outputs": [ { "name": "CONDITIONING", "type": "CONDITIONING", "links": [ 571 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "StableCascade_StageB_Conditioning" } }, { "id": 131, "type": "VAEDecode", "pos": [ 692, 858 ], "size": { "0": 210, "1": 46 }, "flags": {}, "order": 23, "mode": 0, "inputs": [ { "name": "samples", "type": "LATENT", "link": 576 }, { "name": "vae", "type": "VAE", "link": 598, "slot_index": 1 } ], "outputs": [ { "name": "IMAGE", "type": "IMAGE", "links": [ 462, 469, 475 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "VAEDecode" } }, { "id": 138, "type": "DeforumFrameWarpNode", "pos": [ 661, 1237 ], "size": { "0": 304.79998779296875, "1": 46 }, "flags": {}, "order": 27, "mode": 0, "inputs": [ { "name": "image", "type": "IMAGE", "link": 440 }, { "name": "deforum_frame_data", "type": "DEFORUM_FRAME_DATA", "link": 441 } ], "outputs": [ { "name": "IMAGE", "type": "IMAGE", "links": [ 455, 488 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumFrameWarpNode" } }, { "id": 142, "type": "DeforumHybridMotionNode", "pos": [ 650, 1028 ], "size": { "0": 315, "1": 98 }, "flags": {}, "order": 25, "mode": 4, "inputs": [ { "name": "image", "type": "IMAGE", "link": 469 }, { "name": "hybrid_image", "type": "IMAGE", "link": 453 }, { "name": "deforum_frame_data", "type": "DEFORUM_FRAME_DATA", "link": 447 } ], "outputs": [ { "name": "IMAGE", "type": "IMAGE", "links": [ 440, 460 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumHybridMotionNode" }, "widgets_values": [ "RAFT" ] }, { "id": 133, "type": "DeforumConditioningBlendNode", "pos": [ 647, 17 ], "size": { "0": 342.5999755859375, "1": 78 }, "flags": {}, "order": 17, "mode": 0, "inputs": [ { "name": "clip", "type": "CLIP", "link": 596 }, { "name": "deforum_frame_data", "type": "DEFORUM_FRAME_DATA", "link": 588 } ], "outputs": [ { "name": "POSITIVE", "type": "CONDITIONING", "links": [ 430, 569 ], "shape": 3, "slot_index": 0 }, { "name": "NEGATIVE", "type": "CONDITIONING", "links": [ 431, 572 ], "shape": 3 } ], "properties": { "Node name for S&R": "DeforumConditioningBlendNode" }, "widgets_values": [ "none" ] }, { "id": 192, "type": "KSampler", "pos": [ 631, 499 ], "size": { "0": 315, "1": 262 }, "flags": {}, "order": 22, "mode": 0, "inputs": [ { "name": "model", "type": "MODEL", "link": 597, "slot_index": 0 }, { "name": "positive", "type": "CONDITIONING", "link": 571 }, { "name": "negative", "type": "CONDITIONING", "link": 572, "slot_index": 2 }, { "name": "latent_image", "type": "LATENT", "link": 573 } ], "outputs": [ { "name": "LATENT", "type": "LATENT", "links": [ 576 ], "slot_index": 0 } ], "properties": { "Node name for S&R": "KSampler" }, "widgets_values": [ 361450159878869, "randomize", 10, 1.1, "ddpm", "normal", 1 ] }, { "id": 126, "type": "DeforumPromptNode", "pos": [ -660, -60 ], "size": { "0": 313.04736328125, "1": 276.55352783203125 }, "flags": {}, "order": 0, "mode": 0, "inputs": [ { "name": "deforum_data", "type": "deforum_data", "link": null } ], "outputs": [ { "name": "deforum_data", "type": "deforum_data", "links": [ 537 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumPromptNode" }, "widgets_values": [ "0:'unreal 3d render of a cyberpunk city, the text saying \"ComfyUI\"'" ] }, { "id": 152, "type": "DeforumTranslationParamsNode", "pos": [ -670, 780 ], "size": { "0": 317.4000244140625, "1": 274 }, "flags": {}, "order": 10, "mode": 0, "inputs": [ { "name": "deforum_data", "type": "deforum_data", "link": 498 } ], "outputs": [ { "name": "deforum_data", "type": "deforum_data", "links": [ 535 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumTranslationParamsNode" }, "widgets_values": [ "0:(0)", "0:(1.0)", "0:(0)", "0:(0)", "0:(10)", "0:(0.5)", "0:(0.5)", "0:(0)", "0:(0)", "0:(0)" ] }, { "id": 153, "type": "DeforumNoiseParamsNode", "pos": [ -310, 990 ], "size": { "0": 272.06170654296875, "1": 298 }, "flags": {}, "order": 15, "mode": 0, "inputs": [ { "name": "deforum_data", "type": "deforum_data", "link": 541 } ], "outputs": [ { "name": "deforum_data", "type": "deforum_data", "links": [ 585 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumNoiseParamsNode" }, "widgets_values": [ true, "0: (0.8)", "0: (0.4)", "0: (5)", "0: (1.0)", "0: (0.0)", "perlin", 8, 8, 4, 0.5 ] }, { "id": 50, "type": "PreviewImage", "pos": [ 1120, 1440 ], "size": { "0": 189.73870849609375, "1": 246 }, "flags": {}, "order": 31, "mode": 0, "inputs": [ { "name": "images", "type": "IMAGE", "link": 456, "slot_index": 0 } ], "properties": { "Node name for S&R": "PreviewImage" } }, { "id": 47, "type": "PreviewImage", "pos": [ 1120, 950 ], "size": { "0": 192.6160888671875, "1": 169.56472778320312 }, "flags": {}, "order": 28, "mode": 4, "inputs": [ { "name": "images", "type": "IMAGE", "link": 460 } ], "properties": { "Node name for S&R": "PreviewImage" } }, { "id": 48, "type": "PreviewImage", "pos": [ 1120, 1160 ], "size": { "0": 188.46986389160156, "1": 246 }, "flags": {}, "order": 29, "mode": 0, "inputs": [ { "name": "images", "type": "IMAGE", "link": 455 } ], "properties": { "Node name for S&R": "PreviewImage" } }, { "id": 179, "type": "StableCascade_EmptyLatentImage", "pos": [ 160, 890 ], "size": { "0": 252, "1": 150 }, "flags": {}, "order": 1, "mode": 0, "inputs": [], "outputs": [ { "name": "stage_c", "type": "LATENT", "links": [], "shape": 3, "slot_index": 0 }, { "name": "stage_b", "type": "LATENT", "links": [ 573 ], "shape": 3, "slot_index": 1 } ], "properties": { "Node name for S&R": "StableCascade_EmptyLatentImage" }, "widgets_values": [ 1024, 1024, 42, 1 ] }, { "id": 170, "type": "Note", "pos": [ 207, -277 ], "size": { "0": 633.6494140625, "1": 221.03656005859375 }, "flags": {}, "order": 2, "mode": 0, "properties": { "text": "" }, "widgets_values": [ "Welcome to Deforum!\n\nThe below graph provides a pluggable Deforum Animation pipeline in ComfyUI. It works frame-by-frame, so the best practice is to enable Extra options, and Auto Queue.\n\nDeforum Parameter and Schedule nodes represent all settings available to consume after chaining up by the Deforum Iterator node, which keeps track of the current frame, generates/gets the cached latent to be denoised in the next pass. The parameters and schedule's are the same as in the auto1111 extension and in the Colab version.\n\nHybrid nodes are currently bypassed (purple), enabling them, and selecting a video transforms the pipeline into Deforum Hybrid. Iteration can be reset to frame 0 with the reset value being set to 1 on the iterator node. Don't forget to switch it back to 0 to use the generated image/latent.\"\n\nDeforum Video Save node dumps its collected frames when the current frame's id reaches/has passed max_frames." ], "color": "#432", "bgcolor": "#653" }, { "id": 62, "type": "PreviewImage", "pos": [ 266, 1378 ], "size": { "0": 309.7149658203125, "1": 251.35919189453125 }, "flags": {}, "order": 8, "mode": 4, "inputs": [ { "name": "images", "type": "IMAGE", "link": 224, "slot_index": 0 } ], "properties": { "Node name for S&R": "PreviewImage" } }, { "id": 88, "type": "DeforumLoadVideo", "pos": [ -104, 1363 ], "size": { "0": 317.5899658203125, "1": 294 }, "flags": {}, "order": 3, "mode": 4, "outputs": [ { "name": "IMAGE", "type": "IMAGE", "links": [ 224, 453 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumLoadVideo" }, "widgets_values": [ "Recording 2024-02-08 214819.mp4", "image" ] }, { "id": 149, "type": "DeforumBaseParamsNode", "pos": [ -310, -60 ], "size": { "0": 281.7219543457031, "1": 448.62408447265625 }, "flags": {}, "order": 12, "mode": 0, "inputs": [ { "name": "deforum_data", "type": "deforum_data", "link": 538 } ], "outputs": [ { "name": "deforum_data", "type": "deforum_data", "links": [ 539 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumBaseParamsNode" }, "widgets_values": [ 1024, 1024, "ddpm", "normal", 20, 6, 1, "Deforum_{timestring}", "random", 1, "output/deforum", 0.4, true, false, false, true, false ] }, { "id": 194, "type": "DeforumIteratorNode", "pos": [ 86, 451 ], "size": { "0": 393, "1": 286 }, "flags": {}, "order": 16, "mode": 0, "inputs": [ { "name": "deforum_data", "type": "deforum_data", "link": 585 }, { "name": "latent", "type": "LATENT", "link": 586 }, { "name": "init_latent", "type": "LATENT", "link": null } ], "outputs": [ { "name": "deforum_frame_data", "type": "DEFORUM_FRAME_DATA", "links": [ 587, 588, 589 ], "shape": 3 }, { "name": "latent", "type": "LATENT", "links": [ 590 ], "shape": 3 }, { "name": "positive_prompt", "type": "STRING", "links": null, "shape": 3 }, { "name": "negative_prompt", "type": "STRING", "links": null, "shape": 3 } ], "properties": { "Node name for S&R": "DeforumIteratorNode" }, "widgets_values": [ "stable_cascade", 847600922662897, "randomize", 0, 0.8, 0.1, false, false ] }, { "id": 102, "type": "DeforumVideoSaveNode", "pos": [ 1383, 973 ], "size": { "0": 315, "1": 150 }, "flags": {}, "order": 26, "mode": 0, "inputs": [ { "name": "image", "type": "IMAGE", "link": 475 }, { "name": "deforum_frame_data", "type": "DEFORUM_FRAME_DATA", "link": 587 } ], "properties": { "Node name for S&R": "DeforumVideoSaveNode" }, "widgets_values": [ "deforum_", 12, "max_frames", 0 ] }, { "id": 16, "type": "PreviewImage", "pos": [ 1383, -37 ], "size": { "0": 894.2135009765625, "1": 941.8845825195312 }, "flags": {}, "order": 24, "mode": 0, "inputs": [ { "name": "images", "type": "IMAGE", "link": 462, "slot_index": 0 } ], "properties": { "Node name for S&R": "PreviewImage" } }, { "id": 154, "type": "DeforumDiffusionParamsNode", "pos": [ -309, 440 ], "size": { "0": 278.891845703125, "1": 274 }, "flags": {}, "order": 13, "mode": 0, "inputs": [ { "name": "deforum_data", "type": "deforum_data", "link": 539 } ], "outputs": [ { "name": "deforum_data", "type": "deforum_data", "links": [ 540 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumDiffusionParamsNode" }, "widgets_values": [ "0: (0.03)", "0: (0.35)", "0: (1.0)", "0: (4)", false, "0: (20)", false, "0:(0)", false, "0:(1)" ] }, { "id": 135, "type": "DeforumGetCachedLatentNode", "pos": [ 169, 800 ], "size": { "0": 218.39999389648438, "1": 26 }, "flags": {}, "order": 4, "mode": 0, "outputs": [ { "name": "LATENT", "type": "LATENT", "links": [ 586 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumGetCachedLatentNode" } }, { "id": 139, "type": "DeforumAddNoiseNode", "pos": [ 644, 1355 ], "size": { "0": 324.6453552246094, "1": 46 }, "flags": {}, "order": 30, "mode": 0, "inputs": [ { "name": "image", "type": "IMAGE", "link": 488, "slot_index": 0 }, { "name": "deforum_frame_data", "type": "DEFORUM_FRAME_DATA", "link": 443, "slot_index": 1 } ], "outputs": [ { "name": "IMAGE", "type": "IMAGE", "links": [ 456, 593 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumAddNoiseNode" } }, { "id": 198, "type": "StableCascade_StageC_VAEEncode", "pos": [ 629, 1461 ], "size": { "0": 315, "1": 78 }, "flags": {}, "order": 32, "mode": 0, "inputs": [ { "name": "image", "type": "IMAGE", "link": 593 }, { "name": "vae", "type": "VAE", "link": 592 } ], "outputs": [ { "name": "stage_c", "type": "LATENT", "links": [ 594 ], "shape": 3, "slot_index": 0 }, { "name": "stage_b", "type": "LATENT", "links": [], "shape": 3, "slot_index": 1 } ], "properties": { "Node name for S&R": "StableCascade_StageC_VAEEncode" }, "widgets_values": [ 32 ] }, { "id": 136, "type": "DeforumCacheLatentNode", "pos": [ 683, 1616 ], "size": { "0": 210, "1": 26 }, "flags": {}, "order": 33, "mode": 0, "inputs": [ { "name": "latent", "type": "LATENT", "link": 594 } ], "outputs": [ { "name": "LATENT", "type": "LATENT", "links": null, "shape": 3 } ], "properties": { "Node name for S&R": "DeforumCacheLatentNode" } }, { "id": 196, "type": "CheckpointLoaderSimple", "pos": [ -285, -251 ], "size": { "0": 336, "1": 98 }, "flags": {}, "order": 5, "mode": 0, "outputs": [ { "name": "MODEL", "type": "MODEL", "links": [ 595 ], "shape": 3, "slot_index": 0 }, { "name": "CLIP", "type": "CLIP", "links": [ 596 ], "shape": 3, "slot_index": 1 }, { "name": "VAE", "type": "VAE", "links": [ 592 ], "shape": 3, "slot_index": 2 } ], "properties": { "Node name for S&R": "CheckpointLoaderSimple" }, "widgets_values": [ "stable_cascade_stage_c.safetensors" ] }, { "id": 197, "type": "CheckpointLoaderSimple", "pos": [ 67, 228 ], "size": { "0": 349.90911865234375, "1": 98 }, "flags": {}, "order": 6, "mode": 0, "outputs": [ { "name": "MODEL", "type": "MODEL", "links": [ 597 ], "shape": 3, "slot_index": 0 }, { "name": "CLIP", "type": "CLIP", "links": null, "shape": 3 }, { "name": "VAE", "type": "VAE", "links": [ 598 ], "shape": 3, "slot_index": 2 } ], "properties": { "Node name for S&R": "CheckpointLoaderSimple" }, "widgets_values": [ "stable_cascade_stage_b.safetensors" ] } ], "links": [ [ 224, 88, 0, 62, 0, "IMAGE" ], [ 430, 133, 0, 130, 2, "CONDITIONING" ], [ 431, 133, 1, 130, 3, "CONDITIONING" ], [ 440, 142, 0, 138, 0, "IMAGE" ], [ 441, 141, 0, 138, 1, "DEFORUM_FRAME_DATA" ], [ 443, 141, 0, 139, 1, "DEFORUM_FRAME_DATA" ], [ 447, 141, 0, 142, 2, "DEFORUM_FRAME_DATA" ], [ 453, 88, 0, 142, 1, "IMAGE" ], [ 455, 138, 0, 48, 0, "IMAGE" ], [ 456, 139, 0, 50, 0, "IMAGE" ], [ 459, 141, 0, 130, 4, "DEFORUM_FRAME_DATA" ], [ 460, 142, 0, 47, 0, "IMAGE" ], [ 462, 131, 0, 16, 0, "IMAGE" ], [ 469, 131, 0, 142, 0, "IMAGE" ], [ 475, 131, 0, 102, 0, "IMAGE" ], [ 488, 138, 0, 139, 0, "IMAGE" ], [ 497, 150, 0, 151, 0, "deforum_data" ], [ 498, 151, 0, 152, 0, "deforum_data" ], [ 535, 152, 0, 155, 0, "deforum_data" ], [ 537, 126, 0, 150, 0, "deforum_data" ], [ 538, 155, 0, 149, 0, "deforum_data" ], [ 539, 149, 0, 154, 0, "deforum_data" ], [ 540, 154, 0, 157, 0, "deforum_data" ], [ 541, 157, 0, 153, 0, "deforum_data" ], [ 568, 130, 0, 190, 1, "LATENT" ], [ 569, 133, 0, 191, 0, "CONDITIONING" ], [ 570, 191, 0, 190, 0, "CONDITIONING" ], [ 571, 190, 0, 192, 1, "CONDITIONING" ], [ 572, 133, 1, 192, 2, "CONDITIONING" ], [ 573, 179, 1, 192, 3, "LATENT" ], [ 576, 192, 0, 131, 0, "LATENT" ], [ 585, 153, 0, 194, 0, "deforum_data" ], [ 586, 135, 0, 194, 1, "LATENT" ], [ 587, 194, 0, 102, 1, "DEFORUM_FRAME_DATA" ], [ 588, 194, 0, 133, 1, "DEFORUM_FRAME_DATA" ], [ 589, 194, 0, 141, 0, "*" ], [ 590, 194, 1, 130, 1, "LATENT" ], [ 592, 196, 2, 198, 1, "VAE" ], [ 593, 139, 0, 198, 0, "IMAGE" ], [ 594, 198, 0, 136, 0, "LATENT" ], [ 595, 196, 0, 130, 0, "MODEL" ], [ 596, 196, 1, 133, 0, "CLIP" ], [ 597, 197, 0, 192, 0, "MODEL" ], [ 598, 197, 2, 131, 1, "VAE" ] ], "groups": [], "config": {}, "extra": { "groupNodes": {} }, "version": 0.4 } ================================================ FILE: examples/deforum_stablecascade_legacy.json ================================================ { "last_node_id": 195, "last_link_id": 591, "nodes": [ { "id": 141, "type": "Reroute", "pos": [ 532, 972 ], "size": [ 75, 26 ], "flags": {}, "order": 21, "mode": 0, "inputs": [ { "name": "", "type": "*", "link": 589 } ], "outputs": [ { "name": "", "type": "DEFORUM_FRAME_DATA", "links": [ 441, 443, 447, 459 ] } ], "properties": { "showOutputText": false, "horizontal": false } }, { "id": 157, "type": "DeforumHybridScheduleNode", "pos": [ -310, 760 ], "size": { "0": 274.891845703125, "1": 178 }, "flags": {}, "order": 17, "mode": 0, "inputs": [ { "name": "deforum_data", "type": "deforum_data", "link": 540 } ], "outputs": [ { "name": "deforum_data", "type": "deforum_data", "links": [ 541 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumHybridScheduleNode" }, "widgets_values": [ "0:(0.5)", "0:(0.5)", "0:(1)", "0:(100)", "0:(0)", "0:(0.8)" ] }, { "id": 155, "type": "DeforumColorParamsNode", "pos": [ -670, 1120 ], "size": { "0": 317.4000244140625, "1": 154 }, "flags": {}, "order": 14, "mode": 0, "inputs": [ { "name": "deforum_data", "type": "deforum_data", "link": 535 } ], "outputs": [ { "name": "deforum_data", "type": "deforum_data", "links": [ 538 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumColorParamsNode" }, "widgets_values": [ "Image", "", 1, false, false ] }, { "id": 150, "type": "DeforumAnimParamsNode", "pos": [ -670, 280 ], "size": { "0": 317.4000244140625, "1": 178 }, "flags": {}, "order": 10, "mode": 0, "inputs": [ { "name": "deforum_data", "type": "deforum_data", "link": 537 } ], "outputs": [ { "name": "deforum_data", "type": "deforum_data", "links": [ 497 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumAnimParamsNode" }, "widgets_values": [ "3D", 100, "wrap", false, "20230129210106", false ] }, { "id": 151, "type": "DeforumDepthParamsNode", "pos": [ -670, 530 ], "size": { "0": 317.4000244140625, "1": 178 }, "flags": {}, "order": 12, "mode": 0, "inputs": [ { "name": "deforum_data", "type": "deforum_data", "link": 497 } ], "outputs": [ { "name": "deforum_data", "type": "deforum_data", "links": [ 498 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumDepthParamsNode" }, "widgets_values": [ true, "Zoe", 0.2, "border", "bicubic", false ] }, { "id": 130, "type": "DeforumKSampler", "pos": [ 650, 152 ], "size": { "0": 325.93902587890625, "1": 106 }, "flags": {}, "order": 23, "mode": 0, "inputs": [ { "name": "model", "type": "MODEL", "link": 577 }, { "name": "latent", "type": "LATENT", "link": 590, "slot_index": 1 }, { "name": "positive", "type": "CONDITIONING", "link": 430 }, { "name": "negative", "type": "CONDITIONING", "link": 431, "slot_index": 3 }, { "name": "deforum_frame_data", "type": "DEFORUM_FRAME_DATA", "link": 459 } ], "outputs": [ { "name": "LATENT", "type": "LATENT", "links": [ 568 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumKSampler" } }, { "id": 191, "type": "ConditioningZeroOut", "pos": [ 692, 320 ], "size": { "0": 211.60000610351562, "1": 26 }, "flags": {}, "order": 22, "mode": 0, "inputs": [ { "name": "conditioning", "type": "CONDITIONING", "link": 569 } ], "outputs": [ { "name": "CONDITIONING", "type": "CONDITIONING", "links": [ 570 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "ConditioningZeroOut" } }, { "id": 190, "type": "StableCascade_StageB_Conditioning", "pos": [ 659, 392 ], "size": { "0": 277.20001220703125, "1": 46 }, "flags": {}, "order": 24, "mode": 0, "inputs": [ { "name": "conditioning", "type": "CONDITIONING", "link": 570 }, { "name": "stage_c", "type": "LATENT", "link": 568 } ], "outputs": [ { "name": "CONDITIONING", "type": "CONDITIONING", "links": [ 571 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "StableCascade_StageB_Conditioning" } }, { "id": 181, "type": "CLIPLoader", "pos": [ 120, 300 ], "size": { "0": 315, "1": 82 }, "flags": {}, "order": 0, "mode": 0, "outputs": [ { "name": "CLIP", "type": "CLIP", "links": [ 567 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "CLIPLoader" }, "widgets_values": [ "cascade_clip.safetensors", "stable_cascade" ] }, { "id": 131, "type": "VAEDecode", "pos": [ 692, 858 ], "size": { "0": 210, "1": 46 }, "flags": {}, "order": 26, "mode": 0, "inputs": [ { "name": "samples", "type": "LATENT", "link": 576 }, { "name": "vae", "type": "VAE", "link": 591, "slot_index": 1 } ], "outputs": [ { "name": "IMAGE", "type": "IMAGE", "links": [ 462, 469, 475 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "VAEDecode" } }, { "id": 137, "type": "VAEEncode", "pos": [ 686, 1459 ], "size": { "0": 210, "1": 46 }, "flags": {}, "order": 35, "mode": 0, "inputs": [ { "name": "pixels", "type": "IMAGE", "link": 489 }, { "name": "vae", "type": "VAE", "link": 574, "slot_index": 1 } ], "outputs": [ { "name": "LATENT", "type": "LATENT", "links": [ 438 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "VAEEncode" } }, { "id": 136, "type": "DeforumCacheLatentNode", "pos": [ 691, 1576 ], "size": { "0": 210, "1": 26 }, "flags": {}, "order": 36, "mode": 0, "inputs": [ { "name": "latent", "type": "LATENT", "link": 438 } ], "outputs": [ { "name": "LATENT", "type": "LATENT", "links": null, "shape": 3 } ], "properties": { "Node name for S&R": "DeforumCacheLatentNode" } }, { "id": 139, "type": "DeforumAddNoiseNode", "pos": [ 644, 1355 ], "size": { "0": 324.6453552246094, "1": 46 }, "flags": {}, "order": 33, "mode": 0, "inputs": [ { "name": "image", "type": "IMAGE", "link": 488, "slot_index": 0 }, { "name": "deforum_frame_data", "type": "DEFORUM_FRAME_DATA", "link": 443, "slot_index": 1 } ], "outputs": [ { "name": "IMAGE", "type": "IMAGE", "links": [ 456, 489 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumAddNoiseNode" } }, { "id": 138, "type": "DeforumFrameWarpNode", "pos": [ 661, 1237 ], "size": { "0": 304.79998779296875, "1": 46 }, "flags": {}, "order": 30, "mode": 0, "inputs": [ { "name": "image", "type": "IMAGE", "link": 440 }, { "name": "deforum_frame_data", "type": "DEFORUM_FRAME_DATA", "link": 441 } ], "outputs": [ { "name": "IMAGE", "type": "IMAGE", "links": [ 455, 488 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumFrameWarpNode" } }, { "id": 142, "type": "DeforumHybridMotionNode", "pos": [ 650, 1028 ], "size": { "0": 315, "1": 98 }, "flags": {}, "order": 28, "mode": 4, "inputs": [ { "name": "image", "type": "IMAGE", "link": 469 }, { "name": "hybrid_image", "type": "IMAGE", "link": 453 }, { "name": "deforum_frame_data", "type": "DEFORUM_FRAME_DATA", "link": 447 } ], "outputs": [ { "name": "IMAGE", "type": "IMAGE", "links": [ 440, 460 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumHybridMotionNode" }, "widgets_values": [ "RAFT" ] }, { "id": 133, "type": "DeforumConditioningBlendNode", "pos": [ 647, 17 ], "size": { "0": 342.5999755859375, "1": 78 }, "flags": {}, "order": 20, "mode": 0, "inputs": [ { "name": "clip", "type": "CLIP", "link": 567 }, { "name": "deforum_frame_data", "type": "DEFORUM_FRAME_DATA", "link": 588 } ], "outputs": [ { "name": "POSITIVE", "type": "CONDITIONING", "links": [ 430, 569 ], "shape": 3, "slot_index": 0 }, { "name": "NEGATIVE", "type": "CONDITIONING", "links": [ 431, 572 ], "shape": 3 } ], "properties": { "Node name for S&R": "DeforumConditioningBlendNode" }, "widgets_values": [ "none" ] }, { "id": 192, "type": "KSampler", "pos": [ 631, 499 ], "size": { "0": 315, "1": 262 }, "flags": {}, "order": 25, "mode": 0, "inputs": [ { "name": "model", "type": "MODEL", "link": 578, "slot_index": 0 }, { "name": "positive", "type": "CONDITIONING", "link": 571 }, { "name": "negative", "type": "CONDITIONING", "link": 572, "slot_index": 2 }, { "name": "latent_image", "type": "LATENT", "link": 573 } ], "outputs": [ { "name": "LATENT", "type": "LATENT", "links": [ 576 ], "slot_index": 0 } ], "properties": { "Node name for S&R": "KSampler" }, "widgets_values": [ 974706146007189, "randomize", 10, 1.1, "ddpm", "normal", 1 ] }, { "id": 126, "type": "DeforumPromptNode", "pos": [ -660, -60 ], "size": { "0": 313.04736328125, "1": 276.55352783203125 }, "flags": {}, "order": 1, "mode": 0, "inputs": [ { "name": "deforum_data", "type": "deforum_data", "link": null } ], "outputs": [ { "name": "deforum_data", "type": "deforum_data", "links": [ 537 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumPromptNode" }, "widgets_values": [ "0:'unreal 3d render of a cyberpunk city, the text saying \"ComfyUI\"'" ] }, { "id": 152, "type": "DeforumTranslationParamsNode", "pos": [ -670, 780 ], "size": { "0": 317.4000244140625, "1": 274 }, "flags": {}, "order": 13, "mode": 0, "inputs": [ { "name": "deforum_data", "type": "deforum_data", "link": 498 } ], "outputs": [ { "name": "deforum_data", "type": "deforum_data", "links": [ 535 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumTranslationParamsNode" }, "widgets_values": [ "0:(0)", "0:(1.0)", "0:(0)", "0:(0)", "0:(10)", "0:(0.5)", "0:(0.5)", "0:(0)", "0:(0)", "0:(0)" ] }, { "id": 153, "type": "DeforumNoiseParamsNode", "pos": [ -310, 990 ], "size": { "0": 272.06170654296875, "1": 298 }, "flags": {}, "order": 18, "mode": 0, "inputs": [ { "name": "deforum_data", "type": "deforum_data", "link": 541 } ], "outputs": [ { "name": "deforum_data", "type": "deforum_data", "links": [ 585 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumNoiseParamsNode" }, "widgets_values": [ true, "0: (0.8)", "0: (0.4)", "0: (5)", "0: (1.0)", "0: (0.0)", "perlin", 8, 8, 4, 0.5 ] }, { "id": 50, "type": "PreviewImage", "pos": [ 1120, 1440 ], "size": { "0": 189.73870849609375, "1": 246 }, "flags": {}, "order": 34, "mode": 0, "inputs": [ { "name": "images", "type": "IMAGE", "link": 456, "slot_index": 0 } ], "properties": { "Node name for S&R": "PreviewImage" } }, { "id": 47, "type": "PreviewImage", "pos": [ 1120, 950 ], "size": { "0": 192.6160888671875, "1": 169.56472778320312 }, "flags": {}, "order": 31, "mode": 4, "inputs": [ { "name": "images", "type": "IMAGE", "link": 460 } ], "properties": { "Node name for S&R": "PreviewImage" } }, { "id": 48, "type": "PreviewImage", "pos": [ 1120, 1160 ], "size": { "0": 188.46986389160156, "1": 246 }, "flags": {}, "order": 32, "mode": 0, "inputs": [ { "name": "images", "type": "IMAGE", "link": 455 } ], "properties": { "Node name for S&R": "PreviewImage" } }, { "id": 179, "type": "StableCascade_EmptyLatentImage", "pos": [ 160, 890 ], "size": [ 252, 150 ], "flags": {}, "order": 2, "mode": 0, "inputs": [], "outputs": [ { "name": "stage_c", "type": "LATENT", "links": [], "shape": 3, "slot_index": 0 }, { "name": "stage_b", "type": "LATENT", "links": [ 573 ], "shape": 3, "slot_index": 1 } ], "properties": { "Node name for S&R": "StableCascade_EmptyLatentImage" }, "widgets_values": [ 1024, 1024, 42, 1 ] }, { "id": 195, "type": "VAELoader", "pos": [ 130, 1100 ], "size": { "0": 315, "1": 58 }, "flags": {}, "order": 3, "mode": 0, "outputs": [ { "name": "VAE", "type": "VAE", "links": [ 591 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "VAELoader" }, "widgets_values": [ "stage_a.safetensors" ] }, { "id": 177, "type": "UNETLoader", "pos": [ 140, 180 ], "size": { "0": 315, "1": 58 }, "flags": {}, "order": 4, "mode": 0, "outputs": [ { "name": "MODEL", "type": "MODEL", "links": [ 578 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "UNETLoader" }, "widgets_values": [ "stage_b_bf16.safetensors" ] }, { "id": 176, "type": "UNETLoader", "pos": [ 140, 60 ], "size": { "0": 315, "1": 58 }, "flags": {}, "order": 5, "mode": 0, "outputs": [ { "name": "MODEL", "type": "MODEL", "links": [ 577 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "UNETLoader" }, "widgets_values": [ "stage_c_bf16.safetensors" ] }, { "id": 184, "type": "VAELoader", "pos": [ 133, 1221 ], "size": { "0": 315, "1": 58 }, "flags": {}, "order": 6, "mode": 0, "outputs": [ { "name": "VAE", "type": "VAE", "links": [ 574 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "VAELoader" }, "widgets_values": [ "effnet_encoder.safetensors" ] }, { "id": 170, "type": "Note", "pos": [ 207, -277 ], "size": { "0": 633.6494140625, "1": 221.03656005859375 }, "flags": {}, "order": 7, "mode": 0, "properties": { "text": "" }, "widgets_values": [ "Welcome to Deforum!\n\nThe below graph provides a pluggable Deforum Animation pipeline in ComfyUI. It works frame-by-frame, so the best practice is to enable Extra options, and Auto Queue.\n\nDeforum Parameter and Schedule nodes represent all settings available to consume after chaining up by the Deforum Iterator node, which keeps track of the current frame, generates/gets the cached latent to be denoised in the next pass. The parameters and schedule's are the same as in the auto1111 extension and in the Colab version.\n\nHybrid nodes are currently bypassed (purple), enabling them, and selecting a video transforms the pipeline into Deforum Hybrid. Iteration can be reset to frame 0 with the reset value being set to 1 on the iterator node. Don't forget to switch it back to 0 to use the generated image/latent.\"\n\nDeforum Video Save node dumps its collected frames when the current frame's id reaches/has passed max_frames." ], "color": "#432", "bgcolor": "#653" }, { "id": 62, "type": "PreviewImage", "pos": [ 266, 1378 ], "size": { "0": 309.7149658203125, "1": 251.35919189453125 }, "flags": {}, "order": 11, "mode": 4, "inputs": [ { "name": "images", "type": "IMAGE", "link": 224, "slot_index": 0 } ], "properties": { "Node name for S&R": "PreviewImage" } }, { "id": 88, "type": "DeforumLoadVideo", "pos": [ -104, 1363 ], "size": { "0": 317.5899658203125, "1": 294 }, "flags": {}, "order": 8, "mode": 4, "outputs": [ { "name": "IMAGE", "type": "IMAGE", "links": [ 224, 453 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumLoadVideo" }, "widgets_values": [ "Recording 2024-02-08 214819.mp4", "image" ] }, { "id": 149, "type": "DeforumBaseParamsNode", "pos": [ -310, -60 ], "size": { "0": 281.7219543457031, "1": 448.62408447265625 }, "flags": {}, "order": 15, "mode": 0, "inputs": [ { "name": "deforum_data", "type": "deforum_data", "link": 538 } ], "outputs": [ { "name": "deforum_data", "type": "deforum_data", "links": [ 539 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumBaseParamsNode" }, "widgets_values": [ 1024, 1024, "ddpm", "normal", 20, 6, 1, "Deforum_{timestring}", "random", 1, "output/deforum", 0.4, true, false, false, true, false ] }, { "id": 194, "type": "DeforumIteratorNode", "pos": [ 86, 451 ], "size": { "0": 393, "1": 286 }, "flags": {}, "order": 19, "mode": 0, "inputs": [ { "name": "deforum_data", "type": "deforum_data", "link": 585 }, { "name": "latent", "type": "LATENT", "link": 586 } ], "outputs": [ { "name": "deforum_frame_data", "type": "DEFORUM_FRAME_DATA", "links": [ 587, 588, 589 ], "shape": 3 }, { "name": "latent", "type": "LATENT", "links": [ 590 ], "shape": 3 }, { "name": "positive_prompt", "type": "STRING", "links": null, "shape": 3 }, { "name": "negative_prompt", "type": "STRING", "links": null, "shape": 3 } ], "properties": { "Node name for S&R": "DeforumIteratorNode" }, "widgets_values": [ "stable_cascade", 828657964052916, "randomize", 0, 0.8, 0.1, false, false ] }, { "id": 102, "type": "DeforumVideoSaveNode", "pos": [ 1383, 973 ], "size": { "0": 315, "1": 150 }, "flags": {}, "order": 29, "mode": 0, "inputs": [ { "name": "image", "type": "IMAGE", "link": 475 }, { "name": "deforum_frame_data", "type": "DEFORUM_FRAME_DATA", "link": 587 } ], "properties": { "Node name for S&R": "DeforumVideoSaveNode" }, "widgets_values": [ "deforum_", 12, "max_frames", 0 ] }, { "id": 16, "type": "PreviewImage", "pos": [ 1383, -37 ], "size": { "0": 894.2135009765625, "1": 941.8845825195312 }, "flags": {}, "order": 27, "mode": 0, "inputs": [ { "name": "images", "type": "IMAGE", "link": 462, "slot_index": 0 } ], "properties": { "Node name for S&R": "PreviewImage" } }, { "id": 154, "type": "DeforumDiffusionParamsNode", "pos": [ -309, 440 ], "size": { "0": 278.891845703125, "1": 274 }, "flags": {}, "order": 16, "mode": 0, "inputs": [ { "name": "deforum_data", "type": "deforum_data", "link": 539 } ], "outputs": [ { "name": "deforum_data", "type": "deforum_data", "links": [ 540 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumDiffusionParamsNode" }, "widgets_values": [ "0: (0.03)", "0: (0.35)", "0: (1.0)", "0: (4)", false, "0: (20)", false, "0:(0)", false, "0:(1)" ] }, { "id": 135, "type": "DeforumGetCachedLatentNode", "pos": [ 169, 800 ], "size": { "0": 218.39999389648438, "1": 26 }, "flags": {}, "order": 9, "mode": 0, "outputs": [ { "name": "LATENT", "type": "LATENT", "links": [ 586 ], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumGetCachedLatentNode" } } ], "links": [ [ 224, 88, 0, 62, 0, "IMAGE" ], [ 430, 133, 0, 130, 2, "CONDITIONING" ], [ 431, 133, 1, 130, 3, "CONDITIONING" ], [ 438, 137, 0, 136, 0, "LATENT" ], [ 440, 142, 0, 138, 0, "IMAGE" ], [ 441, 141, 0, 138, 1, "DEFORUM_FRAME_DATA" ], [ 443, 141, 0, 139, 1, "DEFORUM_FRAME_DATA" ], [ 447, 141, 0, 142, 2, "DEFORUM_FRAME_DATA" ], [ 453, 88, 0, 142, 1, "IMAGE" ], [ 455, 138, 0, 48, 0, "IMAGE" ], [ 456, 139, 0, 50, 0, "IMAGE" ], [ 459, 141, 0, 130, 4, "DEFORUM_FRAME_DATA" ], [ 460, 142, 0, 47, 0, "IMAGE" ], [ 462, 131, 0, 16, 0, "IMAGE" ], [ 469, 131, 0, 142, 0, "IMAGE" ], [ 475, 131, 0, 102, 0, "IMAGE" ], [ 488, 138, 0, 139, 0, "IMAGE" ], [ 489, 139, 0, 137, 0, "IMAGE" ], [ 497, 150, 0, 151, 0, "deforum_data" ], [ 498, 151, 0, 152, 0, "deforum_data" ], [ 535, 152, 0, 155, 0, "deforum_data" ], [ 537, 126, 0, 150, 0, "deforum_data" ], [ 538, 155, 0, 149, 0, "deforum_data" ], [ 539, 149, 0, 154, 0, "deforum_data" ], [ 540, 154, 0, 157, 0, "deforum_data" ], [ 541, 157, 0, 153, 0, "deforum_data" ], [ 567, 181, 0, 133, 0, "CLIP" ], [ 568, 130, 0, 190, 1, "LATENT" ], [ 569, 133, 0, 191, 0, "CONDITIONING" ], [ 570, 191, 0, 190, 0, "CONDITIONING" ], [ 571, 190, 0, 192, 1, "CONDITIONING" ], [ 572, 133, 1, 192, 2, "CONDITIONING" ], [ 573, 179, 1, 192, 3, "LATENT" ], [ 574, 184, 0, 137, 1, "VAE" ], [ 576, 192, 0, 131, 0, "LATENT" ], [ 577, 176, 0, 130, 0, "MODEL" ], [ 578, 177, 0, 192, 0, "MODEL" ], [ 585, 153, 0, 194, 0, "deforum_data" ], [ 586, 135, 0, 194, 1, "LATENT" ], [ 587, 194, 0, 102, 1, "DEFORUM_FRAME_DATA" ], [ 588, 194, 0, 133, 1, "DEFORUM_FRAME_DATA" ], [ 589, 194, 0, 141, 0, "*" ], [ 590, 194, 1, 130, 1, "LATENT" ], [ 591, 195, 0, 131, 1, "VAE" ] ], "groups": [], "config": {}, "extra": { "groupNodes": { "Deforum Parameters": { "nodes": [ { "type": "DeforumBaseParamsNode", "pos": [ -325, 78 ], "size": { "0": 317.4000244140625, "1": 490 }, "flags": {}, "order": 4, "mode": 0, "inputs": [ { "name": "deforum_data", "type": "deforum_data", "link": null } ], "outputs": [ { "name": "deforum_data", "type": "deforum_data", "links": [], "shape": 3 } ], "properties": { "Node name for S&R": "DeforumBaseParamsNode" }, "widgets_values": [ 1366, 1024, "dpmpp_2m_sde", "karras", 14, 6, 1, "Deforum_{timestring}", "iter", 1, "output/deforum", 0.4, true, false, false, true, false ], "index": 0 }, { "type": "DeforumAnimParamsNode", "pos": [ -315, 598 ], "size": { "0": 317.4000244140625, "1": 178 }, "flags": {}, "order": 5, "mode": 0, "inputs": [ { "name": "deforum_data", "type": "deforum_data", "link": null } ], "outputs": [ { "name": "deforum_data", "type": "deforum_data", "links": [], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumAnimParamsNode" }, "widgets_values": [ "3D", 150, "wrap", false, "20230129210106", false ], "index": 1 }, { "type": "DeforumDepthParamsNode", "pos": [ 45, 68 ], "size": { "0": 317.4000244140625, "1": 178 }, "flags": {}, "order": 9, "mode": 0, "inputs": [ { "name": "deforum_data", "type": "deforum_data", "link": null } ], "outputs": [ { "name": "deforum_data", "type": "deforum_data", "links": [], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumDepthParamsNode" }, "widgets_values": [ true, "Zoe", 0.2, "border", "bicubic", false ], "index": 2 }, { "type": "DeforumTranslationParamsNode", "pos": [ 45, 308 ], "size": { "0": 317.4000244140625, "1": 274 }, "flags": {}, "order": 10, "mode": 0, "inputs": [ { "name": "deforum_data", "type": "deforum_data", "link": null } ], "outputs": [ { "name": "deforum_data", "type": "deforum_data", "links": [], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumTranslationParamsNode" }, "widgets_values": [ "0:(0)", "0:(1.0)", "0:(0)", "0:(0)", "0:(0)", "0:(0.5)", "0:(0.5)", "0:(0)", "0:(0)", "0:(0)" ], "index": 3 }, { "type": "DeforumNoiseParamsNode", "pos": [ 45, 638 ], "size": { "0": 317.4000244140625, "1": 298 }, "flags": {}, "order": 15, "mode": 0, "inputs": [ { "name": "deforum_data", "type": "deforum_data", "link": null } ], "outputs": [ { "name": "deforum_data", "type": "deforum_data", "links": [], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumNoiseParamsNode" }, "widgets_values": [ true, "0: (0.4)", "0: (0.1)", "0: (5)", "0: (1.0)", "0: (0.0)", "perlin", 8, 8, 4, 0.5 ], "index": 4 }, { "type": "DeforumDiffusionParamsNode", "pos": [ -315, 858 ], "size": { "0": 317.4000244140625, "1": 274 }, "flags": {}, "order": 16, "mode": 0, "inputs": [ { "name": "deforum_data", "type": "deforum_data", "link": null } ], "outputs": [ { "name": "deforum_data", "type": "deforum_data", "links": [], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumDiffusionParamsNode" }, "widgets_values": [ "0: (0.03)", "0: (0.45)", "0: (1.0)", "0: (5)", false, "0: (25)", false, "0:(0)", false, "0:(1)" ], "index": 5 }, { "type": "DeforumColorParamsNode", "pos": [ 40, 988 ], "size": { "0": 317.4000244140625, "1": 154 }, "flags": {}, "order": 17, "mode": 0, "inputs": [ { "name": "deforum_data", "type": "deforum_data", "link": null } ], "outputs": [ { "name": "deforum_data", "type": "deforum_data", "links": [], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumColorParamsNode" }, "widgets_values": [ "Image", "", 1, false, false ], "index": 6 } ], "links": [ [ null, 1, 0, 0, 112, "deforum_data" ], [ 0, 0, 1, 0, 20, "deforum_data" ], [ 1, 0, 2, 0, 21, "deforum_data" ], [ 2, 0, 3, 0, 42, "deforum_data" ], [ 5, 0, 4, 0, 52, "deforum_data" ], [ 3, 0, 5, 0, 44, "deforum_data" ], [ 4, 0, 6, 0, 51, "deforum_data" ] ], "external": [ [ 6, 0, "deforum_data" ] ] }, "Deforum Sampler": { "nodes": [ { "type": "DeforumPromptNode", "pos": [ 845, 445 ], "size": { "0": 304.5645751953125, "1": 211.12216186523438 }, "flags": {}, "order": 1, "mode": 0, "inputs": [ { "name": "deforum_data", "type": "deforum_data", "link": null } ], "outputs": [ { "name": "deforum_data", "type": "deforum_data", "links": [], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumPromptNode" }, "widgets_values": [ "0:'whirling chaotic tornado, lighthouse, polygons in space, abstract artwork, light illusion'\n25:'chaotic abstract painting'" ], "index": 0 }, { "type": "CheckpointLoaderSimple", "pos": [ 636, -55 ], "size": { "0": 315, "1": 98 }, "flags": {}, "order": 2, "mode": 0, "outputs": [ { "name": "MODEL", "type": "MODEL", "links": [], "shape": 3, "slot_index": 0 }, { "name": "CLIP", "type": "CLIP", "links": [], "shape": 3, "slot_index": 1 }, { "name": "VAE", "type": "VAE", "links": [], "shape": 3 } ], "properties": { "Node name for S&R": "CheckpointLoaderSimple" }, "widgets_values": [ "protovisionXLHighFidelity3D_beta0520Bakedvae.safetensors" ], "index": 1 }, { "type": "DeepCache", "pos": [ 420, 100 ], "size": { "0": 315, "1": 130 }, "flags": {}, "order": 5, "mode": 0, "inputs": [ { "name": "model", "type": "MODEL", "link": null } ], "outputs": [ { "name": "MODEL", "type": "MODEL", "links": [], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeepCache" }, "widgets_values": [ 4, 2, 0, 1000 ], "index": 2 }, { "type": "DeforumIteratorNode", "pos": [ 420, 490 ], "size": { "0": 393, "1": 166 }, "flags": {}, "order": 13, "mode": 0, "inputs": [ { "name": "deforum_data", "type": "deforum_data", "link": null }, { "name": "latent", "type": "LATENT", "link": null } ], "outputs": [ { "name": "deforum_frame_data", "type": "DEFORUM_FRAME_DATA", "links": [], "shape": 3, "slot_index": 0 }, { "name": "latent", "type": "LATENT", "links": [], "shape": 3 }, { "name": "positive_prompt", "type": "STRING", "links": [], "shape": 3 }, { "name": "negative_prompt", "type": "STRING", "links": [], "shape": 3 } ], "properties": { "Node name for S&R": "DeforumIteratorNode" }, "widgets_values": [ 282766464392687, "increment", 0 ], "index": 3 }, { "type": "DeforumKSampler", "pos": [ 460, 310 ], "size": { "0": 317.4000244140625, "1": 106 }, "flags": {}, "order": 14, "mode": 0, "inputs": [ { "name": "model", "type": "MODEL", "link": null }, { "name": "latent", "type": "LATENT", "link": null, "slot_index": 1 }, { "name": "positive", "type": "CONDITIONING", "link": null }, { "name": "negative", "type": "CONDITIONING", "link": null, "slot_index": 3 }, { "name": "deforum_frame_data", "type": "DEFORUM_FRAME_DATA", "link": null } ], "outputs": [ { "name": "LATENT", "type": "LATENT", "links": [], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumKSampler" }, "index": 4 }, { "type": "VAEDecode", "pos": [ 920, 319 ], "size": { "0": 210, "1": 46 }, "flags": {}, "order": 15, "mode": 0, "inputs": [ { "name": "samples", "type": "LATENT", "link": null }, { "name": "vae", "type": "VAE", "link": null, "slot_index": 1 } ], "outputs": [ { "name": "IMAGE", "type": "IMAGE", "links": [], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "VAEDecode" }, "index": 5 }, { "type": "DeforumConditioningBlendNode", "pos": [ 841, 148 ], "size": { "0": 342.5999755859375, "1": 78 }, "flags": {}, "order": 16, "mode": 0, "inputs": [ { "name": "clip", "type": "CLIP", "link": null }, { "name": "deforum_frame_data", "type": "DEFORUM_FRAME_DATA", "link": null, "slot_index": 1 } ], "outputs": [ { "name": "POSITIVE", "type": "CONDITIONING", "links": [], "shape": 3, "slot_index": 0 }, { "name": "NEGATIVE", "type": "CONDITIONING", "links": [], "shape": 3 } ], "properties": { "Node name for S&R": "DeforumConditioningBlendNode" }, "widgets_values": [ "sigmoidal" ], "index": 6 } ], "links": [ [ 1, 0, 2, 0, 115, "MODEL" ], [ null, 0, 3, 0, 113, "deforum_data" ], [ null, 0, 3, 1, 114, "LATENT" ], [ 2, 0, 4, 0, 117, "MODEL" ], [ 3, 1, 4, 1, 118, "LATENT" ], [ 6, 0, 4, 2, 124, "CONDITIONING" ], [ 6, 1, 4, 3, 124, "CONDITIONING" ], [ null, 5, 4, 4, 114, "DEFORUM_FRAME_DATA" ], [ 4, 0, 5, 0, 121, "LATENT" ], [ 1, 2, 5, 1, 115, "VAE" ], [ 1, 1, 6, 0, 115, "CLIP" ], [ 3, 0, 6, 1, 118, "DEFORUM_FRAME_DATA" ] ], "external": [ [ 0, 0, "deforum_data" ], [ 1, 2, "VAE" ], [ 3, 0, "DEFORUM_FRAME_DATA" ], [ 5, 0, "IMAGE" ] ] }, "Deforum Operators": { "nodes": [ { "type": "DeforumGetCachedLatentNode", "pos": [ 1356, 1073 ], "size": { "0": 218.39999389648438, "1": 26 }, "flags": {}, "order": 1, "mode": 0, "outputs": [ { "name": "LATENT", "type": "LATENT", "links": [], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumGetCachedLatentNode" }, "index": 0 }, { "type": "DeforumCacheLatentNode", "pos": [ 1337, 1803 ], "size": { "0": 210, "1": 26 }, "flags": {}, "order": 4, "mode": 0, "inputs": [ { "name": "latent", "type": "LATENT", "link": null } ], "outputs": [ { "name": "LATENT", "type": "LATENT", "links": null, "shape": 3 } ], "properties": { "Node name for S&R": "DeforumCacheLatentNode" }, "index": 1 }, { "type": "VAEEncode", "pos": [ 1344, 1696 ], "size": { "0": 210, "1": 46 }, "flags": {}, "order": 5, "mode": 0, "inputs": [ { "name": "pixels", "type": "IMAGE", "link": null }, { "name": "vae", "type": "VAE", "link": null, "slot_index": 1 } ], "outputs": [ { "name": "LATENT", "type": "LATENT", "links": [], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "VAEEncode" }, "index": 2 }, { "type": "DeforumFrameWarpNode", "pos": [ 1301, 1392 ], "size": { "0": 304.79998779296875, "1": 46 }, "flags": {}, "order": 6, "mode": 0, "inputs": [ { "name": "image", "type": "IMAGE", "link": null }, { "name": "deforum_frame_data", "type": "DEFORUM_FRAME_DATA", "link": null } ], "outputs": [ { "name": "IMAGE", "type": "IMAGE", "links": [], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumFrameWarpNode" }, "index": 3 }, { "type": "DeforumAddNoiseNode", "pos": [ 1301, 1590 ], "size": { "0": 304.79998779296875, "1": 46 }, "flags": {}, "order": 9, "mode": 0, "inputs": [ { "name": "image", "type": "IMAGE", "link": null }, { "name": "deforum_frame_data", "type": "DEFORUM_FRAME_DATA", "link": null, "slot_index": 1 } ], "outputs": [ { "name": "IMAGE", "type": "IMAGE", "links": [], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumAddNoiseNode" }, "index": 4 }, { "type": "DeforumColorMatchNode", "pos": [ 1303, 1157 ], "size": { "0": 304.79998779296875, "1": 46 }, "flags": {}, "order": 11, "mode": 0, "inputs": [ { "name": "image", "type": "IMAGE", "link": null }, { "name": "deforum_frame_data", "type": "DEFORUM_FRAME_DATA", "link": null } ], "outputs": [ { "name": "IMAGE", "type": "IMAGE", "links": [], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumColorMatchNode" }, "index": 5 }, { "type": "Reroute", "pos": [ 1079, 1292 ], "size": [ 75, 26 ], "flags": {}, "order": 12, "mode": 0, "inputs": [ { "name": "", "type": "*", "link": null } ], "outputs": [ { "name": "", "type": "*", "links": null } ], "properties": { "showOutputText": false, "horizontal": false }, "index": 6 }, { "type": "DeforumHybridMotionNode", "pos": [ 1298, 1249 ], "size": { "0": 315, "1": 98 }, "flags": {}, "order": 14, "mode": 0, "inputs": [ { "name": "image", "type": "IMAGE", "link": null }, { "name": "hybrid_image", "type": "IMAGE", "link": null }, { "name": "deforum_frame_data", "type": "DEFORUM_FRAME_DATA", "link": null } ], "outputs": [ { "name": "IMAGE", "type": "IMAGE", "links": [], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumHybridMotionNode" }, "widgets_values": [ "RAFT" ], "index": 7 }, { "type": "DeforumColorMatchNode", "pos": [ 1297, 1492 ], "size": { "0": 304.79998779296875, "1": 46 }, "flags": {}, "order": 15, "mode": 0, "inputs": [ { "name": "image", "type": "IMAGE", "link": null, "slot_index": 0 }, { "name": "deforum_frame_data", "type": "DEFORUM_FRAME_DATA", "link": null } ], "outputs": [ { "name": "IMAGE", "type": "IMAGE", "links": [], "shape": 3, "slot_index": 0 } ], "properties": { "Node name for S&R": "DeforumColorMatchNode" }, "index": 8 } ], "links": [ [ 2, 0, 1, 0, 36, "LATENT" ], [ 8, 0, 2, 0, 109, "IMAGE" ], [ null, 0, 2, 1, 112, "VAE" ], [ 7, 0, 3, 0, 108, "IMAGE" ], [ 6, 0, 3, 1, 98, "DEFORUM_FRAME_DATA" ], [ 8, 0, 4, 0, 109, "IMAGE" ], [ 6, 0, 4, 1, 98, "DEFORUM_FRAME_DATA" ], [ null, 3, 5, 0, 112, "IMAGE" ], [ 6, 0, 5, 1, 98, "DEFORUM_FRAME_DATA" ], [ null, 2, 6, 0, 112, "DEFORUM_FRAME_DATA" ], [ 5, 0, 7, 0, 95, "IMAGE" ], [ null, 0, 7, 1, 88, "IMAGE" ], [ 6, 0, 7, 2, 98, "DEFORUM_FRAME_DATA" ], [ 3, 0, 8, 0, 41, "IMAGE" ], [ 6, 0, 8, 1, 98, "DEFORUM_FRAME_DATA" ] ], "external": [ [ 0, 0, "LATENT" ], [ 3, 0, "IMAGE" ], [ 4, 0, "IMAGE" ], [ 5, 0, "IMAGE" ], [ 6, 0, "DEFORUM_FRAME_DATA" ], [ 7, 0, "IMAGE" ], [ 8, 0, "IMAGE" ] ] } } }, "version": 0.4 } ================================================ FILE: install.py ================================================ import os import shutil import sys import subprocess import threading import locale import traceback import re import os import subprocess import sys import platform if sys.argv[0] == 'install.py': sys.path.append('.') # for portable version comfy_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..')) sys.path.append(comfy_path) # --- def handle_stream(stream, is_stdout): stream.reconfigure(encoding=locale.getpreferredencoding(), errors='replace') for msg in stream: if is_stdout: print(msg, end="", file=sys.stdout) else: print(msg, end="", file=sys.stderr) def process_wrap(cmd_str, cwd=None, handler=None): print(f"[Deforum] EXECUTE: {cmd_str} in '{cwd}'") process = subprocess.Popen(cmd_str, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, bufsize=1) if handler is None: handler = handle_stream stdout_thread = threading.Thread(target=handler, args=(process.stdout, True)) stderr_thread = threading.Thread(target=handler, args=(process.stderr, False)) stdout_thread.start() stderr_thread.start() stdout_thread.join() stderr_thread.join() return process.wait() # --- pip_list = None def get_installed_packages(): global pip_list if pip_list is None: try: result = subprocess.check_output([sys.executable, '-m', 'pip', 'list'], universal_newlines=True) pip_list = set([line.split()[0].lower() for line in result.split('\n') if line.strip()]) except subprocess.CalledProcessError as e: print(f"[ComfyUI-Manager] Failed to retrieve the information of installed pip packages.") return set() return pip_list def is_installed(name): name = name.strip() pattern = r'([^<>!=]+)([<>!=]=?)' match = re.search(pattern, name) if match: name = match.group(1) result = name.lower() in get_installed_packages() return result def is_requirements_installed(file_path): print(f"req_path: {file_path}") if os.path.exists(file_path): with open(file_path, 'r') as file: lines = file.readlines() for line in lines: if not is_installed(line): return False return True def find_path(name: str, path: str = None) -> str: """ Recursively looks at parent folders starting from the given path until it finds the given name. Returns the path as a Path object if found, or None otherwise. """ # If no path is given, use the current working directory if path is None: path = os.getcwd() # Check if the current directory contains the name if name in os.listdir(path): path_name = os.path.join(path, name) print(f"{name} found: {path_name}") return path_name # Get the parent directory parent_directory = os.path.dirname(path) # If the parent directory is the same as the current directory, we've reached the root and stop the search if parent_directory == path: return None # Recursively call the function with the parent directory return find_path(name, parent_directory) def run_git_command(command: str, working_dir: str) -> None: """ Runs a git command in the specified working directory and handles basic errors. """ try: subprocess.run(command, shell=True, check=True, cwd=working_dir) print(f"Successfully executed: {command} in {working_dir}") except subprocess.CalledProcessError as e: print(f"Error executing {command} in {working_dir}: {e}") def clone_or_pull_repo(repo_url: str, repo_dir: str) -> None: """ Clones a new repository or updates it if it already exists. """ if not os.path.exists(repo_dir): os.makedirs(repo_dir, exist_ok=True) run_git_command(f"git clone {repo_url} .", repo_dir) else: run_git_command("git pull", repo_dir) def install(): repositories = [ "https://github.com/ceruleandeep/ComfyUI-LLaVA-Captioner.git", "https://github.com/rgthree/rgthree-comfy.git", "https://github.com/a1lazydog/ComfyUI-AudioScheduler.git", "https://github.com/cubiq/ComfyUI_IPAdapter_plus.git", "https://github.com/Kosinkadink/ComfyUI-VideoHelperSuite.git", "https://github.com/Kosinkadink/ComfyUI-Advanced-ControlNet.git", "https://github.com/WASasquatch/was-node-suite-comfyui.git", "https://github.com/11cafe/comfyui-workspace-manager.git", "https://github.com/cubiq/ComfyUI_essentials.git", "https://github.com/FizzleDorf/ComfyUI_FizzNodes.git", "https://github.com/ltdrdata/ComfyUI-Impact-Pack.git", "https://github.com/Fannovel16/ComfyUI-Frame-Interpolation.git", "https://github.com/Fannovel16/ComfyUI-Video-Matting.git", "https://github.com/crystian/ComfyUI-Crystools.git" # Add more repositories as needed ] comfyui_path = find_path("ComfyUI") custom_nodes_path = os.path.join(comfyui_path, 'custom_nodes') for repo_url in repositories: repo_name = repo_url.split('/')[-1].replace('.git', '') repo_dir = os.path.join(custom_nodes_path, repo_name) clone_or_pull_repo(repo_url, repo_dir) import subprocess def install_packages(): # Install packages from requirements.txt # subprocess.run(["pip", "install", "-r", "requirements.txt"], check=True) # Force reinstall the deforum-studio package from Git subprocess.run(["pip", "install", "--force-reinstall", "git+https://github.com/XmYx/deforum-studio.git"], check=True) def get_cuda_version(): try: cuda_version = subprocess.check_output(["nvcc", "--version"]).decode("utf-8") for line in cuda_version.split('\n'): if "release" in line: return line.split('release')[1].split(',')[0].strip().replace('.', '') except Exception as e: print(f"Error getting CUDA version: {e}") return None def get_torch_version(): try: import torch return torch.__version__.split('+')[0] # Removes the +cuXXX if exists except ImportError: print("PyTorch is not installed. Please install PyTorch before proceeding.") sys.exit(1) def construct_wheel_name(cuda_version, py_version, os_name): # cuda_version: e.g., "110", "102" # py_version: e.g., "38", "37" # os_name: either "linux" or "win" os_map = {"Linux": "manylinux2014_x86_64", "Windows": "win_amd64"} torch_version = get_torch_version().replace('.', '') cuda_str = f"cu{cuda_version}" if cuda_version else "cpu" filename = f"stable_fast-1.0.4+torch{torch_version}{cuda_str}-cp{py_version}-cp{py_version}-{os_map[os_name]}.whl" return filename def install_stable_fast(): cuda_version = get_cuda_version() python_version = f"{sys.version_info.major}{sys.version_info.minor}" os_name = platform.system() wheel_name = construct_wheel_name(cuda_version, python_version, os_name) url = f"https://github.com/chengzeyi/stable-fast/releases/download/v1.0.4/{wheel_name}" print(f"Attempting to install: {wheel_name}") subprocess.run([sys.executable, "-m", "pip", "install", url]) def install_reqs(): # Install 'comfy' dependencies subprocess.check_call([ 'pip', 'install', 'einops>=0.6.0', 'numexpr>=2.8.4', 'matplotlib>=3.7.1', 'pandas>=1.5.3', 'av>=10.0.0', 'pims>=0.6.1', 'imageio-ffmpeg>=0.4.8', 'rich>=13.3.2', 'gdown>=4.7.1', 'py3d>=0.0.87', 'librosa>=0.10.0.post2', 'numpy<2.0.0', 'pydub>=0.16.5', 'opencv-contrib-python>=4.7.0.68', 'loguru>=0.4.0', 'python-decouple>=3.8', 'timm>=0.6.13', 'scikit-image>=0.21.0', 'moviepy<2.0.0.dev1' ]) # Install deforum-studio with 'comfy' extras and no dependencies subprocess.check_call([ 'pip', 'install', '--no-deps', 'git+https://github.com/XmYx/deforum-studio.git#egg=deforum[comfy]' ]) if __name__ == "__main__": print("Installing packages...") try: install_stable_fast() print("Installed Stable Fast 1.0.4") except: print("[warning] Stable Fast Install Failed") try: install_reqs() print("Installation complete.") except Exception as e: print("[warning] deforum backend package install failed, if you encounter any issues, please activate your venv and run:\npip install git+https://github.com/XmYxdeforum-studio.git\nIf you are using ComfyUI portable, you have to locate your python executable and add that's path before the pip install command.") pass ================================================ FILE: node_list.json ================================================ { "DeforumBaseParamsNode": "", "DeforumAnimParamsNode": "", "DeforumTranslationParamsNode": "", "DeforumDepthParamsNode": "", "DeforumNoiseParamsNode": "", "DeforumColorParamsNode": "", "DeforumDiffusionParamsNode": "", "DeforumCadenceParamsNode": "", "DeforumHybridParamsNode": "", "DeforumHybridScheduleNode": "", "DeforumPromptNode": "", "DeforumAreaPromptNode": "", "DeforumSingleSampleNode": "", "DeforumCacheLatentNode": "", "DeforumGetCachedLatentNode": "", "DeforumSeedNode": "", "DeforumIteratorNode": "", "DeforumKSampler": "", "DeforumFrameWarpNode": "", "DeforumColorMatchNode": "", "DeforumAddNoiseNode": "", "DeforumHybridMotionNode": "", "DeforumSetVAEDownscaleRatioNode": "", "DeforumLoadVideo": "", "DeforumVideoSaveNode": "", "DeforumFILMInterpolationNode": "", "DeforumSimpleInterpolationNode": "", "DeforumCadenceNode": "", "DeforumConditioningBlendNode": "" } ================================================ FILE: web/js/deforumIterateNode.js ================================================ import { app } from "../../../scripts/app.js"; import { api } from '../../../scripts/api.js' import { ComfyWidgets } from "../../../scripts/widgets.js"; // Many functions copied from: // https://github.com/Kosinkadink/ComfyUI-VideoHelperSuite // Thank you! // Hijack LiteGraph to sort categories and nodes alphabetically (case-insensitive) (function(LiteGraph) { var originalGetNodeTypesCategories = LiteGraph.getNodeTypesCategories; LiteGraph.getNodeTypesCategories = function(filter) { var categories = originalGetNodeTypesCategories.call(this, filter); // Sort categories case-insensitively return categories.sort((a, b) => a.toLowerCase().localeCompare(b.toLowerCase())); }; var originalGetNodeTypesInCategory = LiteGraph.getNodeTypesInCategory; LiteGraph.getNodeTypesInCategory = function(category, filter) { var nodeTypes = originalGetNodeTypesInCategory.call(this, category, filter); // Sort node types case-insensitively by title return nodeTypes.sort((a, b) => a.title.toLowerCase().localeCompare(b.title.toLowerCase())); }; })(LiteGraph || global.LiteGraph); // Ensure LiteGraph is defined; use global.LiteGraph if it's not directly accessible document.getElementById("comfy-file-input").accept += ",video/webm,video/mp4"; function chainCallback(object, property, callback) { if (object == undefined) { //This should not happen. console.error("Tried to add callback to non-existant object") return; } if (property in object) { const callback_orig = object[property] object[property] = function () { const r = callback_orig.apply(this, arguments); callback.apply(this, arguments); return r }; } else { object[property] = callback; } } async function uploadFile(file) { //TODO: Add uploaded file to cache with Cache.put()? try { // Wrap file in formdata so it includes filename const body = new FormData(); const i = file.webkitRelativePath.lastIndexOf('/'); const subfolder = file.webkitRelativePath.slice(0,i+1) const new_file = new File([file], file.name, { type: file.type, lastModified: file.lastModified, }); body.append("image", new_file); if (i > 0) { body.append("subfolder", subfolder); } const resp = await api.fetchApi("/upload/image", { method: "POST", body, }); if (resp.status === 200) { return resp.status } else { alert(resp.status + " - " + resp.statusText); } } catch (error) { alert(error); } } function fitHeight(node) { node.setSize([node.size[0], node.computeSize([node.size[0], node.size[1]])[1] + 20]) node?.graph?.setDirtyCanvas(true); } function addVideoPreview(nodeType) { chainCallback(nodeType.prototype, "onNodeCreated", function() { var element = document.createElement("div"); const previewNode = this; previewNode.fps = 24 previewNode.size[1] += 45; var previewWidget = this.addDOMWidget("videopreview", "preview", element, { serialize: false, hideOnZoom: false, getValue() { return element.value; }, setValue(v) { element.value = v; }, }); previewWidget.computeSize = function(width) { if (this.aspectRatio && !this.parentEl.hidden) { let height = (previewNode.size[0]-20)/ this.aspectRatio + 10; if (!(height > 0)) { height = 0; } this.computedHeight = height + 50; return [width, height]; } return [width, -4];//no loaded src, widget should not display } //element.style['pointer-events'] = "none" previewWidget.value = {hidden: false, paused: false, params: {}} previewWidget.parentEl = document.createElement("div"); previewWidget.parentEl.className = "deforumVideoSavePreview"; previewWidget.parentEl.style['width'] = "100%" element.appendChild(previewWidget.parentEl); previewWidget.imgEl = document.createElement("img"); previewWidget.imgEl.style['width'] = "100%" previewWidget.imgEl.hidden = true; // Create an audio element for audio playback previewWidget.audioEl = document.createElement("audio"); previewWidget.audioEl.controls = true; // Optional: Show controls previewWidget.audioEl.loop = true; // Enable audio looping by default previewWidget.audioEl.style["width"] = "100%"; // Make audio widget as wide as the node element.appendChild(previewWidget.audioEl); // Append audio element to the DOM previewWidget.parentEl.appendChild(previewWidget.imgEl) let frameIndex = 0; let cachedFrames = this.getCachedFrames(); // Assuming this method exists and retrieves an array of frame data previewWidget.imgEl.onload = () => { previewWidget.aspectRatio = previewWidget.imgEl.naturalWidth / previewWidget.imgEl.naturalHeight; fitHeight(this); }; // Use the audio element's timeline to control the frame index previewWidget.audioEl.addEventListener('timeupdate', function() { // Calculate the current frame based on the audio current time and fps const currentTime = previewWidget.audioEl.currentTime; frameIndex = Math.floor(currentTime * this.fps) % this.getCachedFrames().length; if (this.playing === false) { updateFrame(frameIndex); } }.bind(this)); function updateFrame(index) { const cachedFrames = previewNode.getCachedFrames(); // Get cached frames if (cachedFrames && cachedFrames.length > 0) { previewWidget.imgEl.hidden = false; previewWidget.imgEl.src = cachedFrames[index]; } } this.playing = false; this.playbackInterval = 41.666666666666664; this.startPlayback = function(playbackInterval) { if (this.playing === false) { this.playing = true; const widget = this; // Capture 'this' to use inside setInterval function this.imageSequenceInterval = setInterval(() => { const cachedFrames = this.getCachedFrames(); //const displayFrames = cachedFrames.length > 0 ? cachedFrames : frames; if (cachedFrames && cachedFrames.length > 0) { previewWidget.imgEl.hidden = false; previewWidget.imgEl.src = cachedFrames[frameIndex]; frameIndex = (frameIndex + 1) % cachedFrames.length; } }, this.playbackInterval); // Update frame every 80ms } }; // Function to stop playback this.stopPlayback = function() { const prevIndex = previewWidget.audioEl.currentTime if (this.imageSequenceInterval) { clearInterval(this.imageSequenceInterval); this.imageSequenceInterval = null; // Clear the interval ID this.playing = false; // Mark as not playing } }; this.setPlaybackInterval = function(newInterval) { // Check if the new interval is different from the current playback interval if (this.playbackInterval !== newInterval) { this.playbackInterval = newInterval; // Update the playback interval } console.log("New interval", newInterval) const wasPlaying = this.playing; // Check if audio was playing const currentTime = previewWidget.audioEl.currentTime; // Store current playback time if (wasPlaying) { // Wait for the audio to be loaded this.playing = false; // Mark as not playing clearInterval(this.imageSequenceInterval); this.imageSequenceInterval = null; // Clear the interval ID previewWidget.audioEl.oncanplaythrough = function() { previewWidget.audioEl.currentTime = currentTime; // Seek to the previous playback time previewWidget.audioEl.play(); // Resume playback previewWidget.audioEl.oncanplaythrough = null; // Remove the event listener to avoid memory leaks }; } }; previewWidget.audioEl.addEventListener('play', () => this.startPlayback(previewWidget.playbackInterval)); previewWidget.audioEl.addEventListener('pause', () => this.stopPlayback()); this.updateAudio = function (audioBase64) { //this.cacheAudio(audioBase64); // console.log(this.getCachedAudio().length) // Check if audio was playing const wasPlaying = this.playing; const currentTime = previewWidget.audioEl.currentTime; if (previewWidget.audioEl.src !== audioBase64) { previewWidget.audioEl.src = audioBase64; // If the audio was playing, continue playback if (!previewWidget.audioEl.paused) { previewWidget.audioEl.load(); // Load the new audio source // previewWidget.audioEl.currentTime = currentTime; previewWidget.audioEl.play(); // Resume playback previewWidget.audioEl.currentTime = currentTime; } } }; }); } function addUploadWidget(nodeType, nodeData, widgetName, type="video") { chainCallback(nodeType.prototype, "onNodeCreated", function() { const pathWidget = this.widgets.find((w) => w.name === widgetName); const fileInput = document.createElement("input"); if (type == "folder") { Object.assign(fileInput, { type: "file", style: "display: none", webkitdirectory: true, onchange: async () => { const directory = fileInput.files[0].webkitRelativePath; const i = directory.lastIndexOf('/'); if (i <= 0) { throw "No directory found"; } const path = directory.slice(0,directory.lastIndexOf('/')) if (pathWidget.options.values.includes(path)) { alert("A folder of the same name already exists"); return; } let successes = 0; for(const file of fileInput.files) { if (await uploadFile(file) == 200) { successes++; } else { //Upload failed, but some prior uploads may have succeeded //Stop future uploads to prevent cascading failures //and only add to list if an upload has succeeded if (successes > 0) { break } else { return; } } } pathWidget.options.values.push(path); pathWidget.value = path; if (pathWidget.callback) { pathWidget.callback(path) } }, }); } else if (type == "video") { Object.assign(fileInput, { type: "file", accept: "video/webm,video/mp4,video/mkv,image/gif", style: "display: none", onchange: async () => { if (fileInput.files.length) { if (await uploadFile(fileInput.files[0]) != 200) { //upload failed and file can not be added to options return; } const filename = fileInput.files[0].name; pathWidget.options.values.push(filename); pathWidget.value = filename; if (pathWidget.callback) { pathWidget.callback(filename) } } }, }); } else { throw "Unknown upload type" } document.body.append(fileInput); let uploadWidget = this.addWidget("button", "choose " + type + " to upload", "image", () => { //clear the active click event app.canvas.node_widget = null fileInput.click(); }); uploadWidget.options.serialize = false; }); } function extendNodePrototypeWithFrameCaching(nodeType) { nodeType.prototype.frameCache = []; // Initialize an empty cache for frames nodeType.prototype.audioCache = ''; // Initialize an empty string for audio cache // Method to add frames to the cache nodeType.prototype.cacheFrames = function(frames) { this.frameCache = this.frameCache.concat(frames); }; // Method to cache audio data nodeType.prototype.cacheAudio = function(audioBase64) { this.audioCache = this.audioCache + audioBase64; // Cache the base64 audio data }; // Method to clear both frame and audio caches nodeType.prototype.clearCache = function() { this.frameCache = []; this.audioCache = ''; }; // Method to get cached frames (unchanged) nodeType.prototype.getCachedFrames = function() { return this.frameCache; }; // Method to get cached audio nodeType.prototype.getCachedAudio = function() { return this.audioCache; }; } app.registerExtension({ name: "deforum.deforumIterator", init() { const STRING = ComfyWidgets.STRING; ComfyWidgets.STRING = function (node, inputName, inputData) { const r = STRING.apply(this, arguments); r.widget.dynamicPrompts = inputData?.[1].dynamicPrompts; return r; }; }, beforeRegisterNodeDef(nodeType, nodeData) { if (nodeType.comfyClass === "DeforumIteratorNode") { const onIteratorExecuted = nodeType.prototype.onExecuted nodeType.prototype.onExecuted = function (message) { const r = onIteratorExecuted ? onIteratorExecuted.apply(this, message) : undefined for (const w of this.widgets || []) { if (w.name === "reset_counter") { const counterWidget = w; counterWidget.value = false; } else if (w.name === "reset_latent") { const resetWidget = w; resetWidget.value = false; } } const v = app.nodeOutputs?.[this.id + ""]; if (!this.flags.collapsed && v) { const counter = v["counter"] const max_frames = v["max_frames"] const enableAutorun = v["enable_autoqueue"][0] if (counter[0] >= max_frames[0]) { if (document.getElementById('autoQueueCheckbox').checked === true) { document.getElementById('autoQueueCheckbox').click(); } } if (enableAutorun === true) { if (document.getElementById('autoQueueCheckbox').checked === false) { document.getElementById('autoQueueCheckbox').click(); document.getElementById('extraOptions').style.display = 'block'; } } } return r } const onDrawForeground = nodeType.prototype.onDrawForeground; nodeType.prototype.onDrawForeground = function (ctx) { const r = onDrawForeground?.apply?.(this, arguments); const v = app.nodeOutputs?.[this.id + ""]; if (!this.flags.collapsed && v) { const text = v["counter"] + ""; ctx.save(); ctx.font = "bold 48px sans-serif"; ctx.fillStyle = "dodgerblue"; const sz = ctx.measureText(text); ctx.fillText(text, (this.size[0]) / 2 - sz.width - 5, LiteGraph.NODE_SLOT_HEIGHT * 3); ctx.restore(); } return r; }; } else if (nodeType.comfyClass === "DeforumBigBoneResetNode") { const onResetExecuted = nodeType.prototype.onExecuted nodeType.prototype.onExecuted = function (message) { const r = onResetExecuted ? onResetExecuted.apply(this, message) : undefined for (const w of this.widgets || []) { if (w.name === "reset_deforum") { const counterWidget = w; counterWidget.value = false; } } return r } } else if (nodeType.comfyClass === "DeforumLoadVideo") { addUploadWidget(nodeType, nodeData, "video"); } else if (nodeType.comfyClass === "DeforumVideoSaveNode") { extendNodePrototypeWithFrameCaching(nodeType); addVideoPreview(nodeType); const onVideoSaveExecuted = nodeType.prototype.onExecuted let restoreWidget nodeType.prototype.onExecuted = function (message) { const r = onVideoSaveExecuted ? onVideoSaveExecuted.apply(this, message) : undefined let swapSkipSave = false; for (const w of this.widgets || []) { if (w.name === "dump_now") { const dumpWidget = w; if (dumpWidget.value === true) { swapSkipSave = true } dumpWidget.value = false; this.shouldResetAnimation = true; } else if (w.name === "skip_save") { const saveWidget = w; if (swapSkipSave === true) { saveWidget.value = false; } } else if (w.name === "clear_cache") { const cacheClearWidget = w; if (cacheClearWidget.value === true) { cacheClearWidget.value = false; } }else if (w.name === "restore") { restoreWidget = w } } const output = app.nodeOutputs?.[this.id + ""]; const should_reset = output["should_dump"] const fps = output["fps"] const millisecondsPerFrame = 1000 / fps[0]; this.fps = fps if (output && "frames" in output) { // Safely check if 'frames' is a key in 'output' if (this.playing === false) { //this.playing = true; this.setPlaybackInterval(millisecondsPerFrame); this.cacheFrames(output["frames"]); if ("audio" in output) { this.updateAudio(output["audio"]); } //this.startPlayback(millisecondsPerFrame); } else { this.cacheFrames(output["frames"]); if ("audio" in output) { this.updateAudio(output["audio"]); } } if (should_reset[0] === true) { this.stopPlayback(); this.clearCache(); this.cacheFrames(output["frames"]); restoreWidget.value = false; } else { if (this.getCachedFrames().length < output["counter"]) { restoreWidget.value = true; this.clearCache(); } else { restoreWidget.value = false; } } } return r } const onVideoSaveForeground = nodeType.prototype.onDrawForeground; nodeType.prototype.onDrawForeground = function (ctx) { const r = onVideoSaveForeground?.apply?.(this, arguments); const v = app.nodeOutputs?.[this.id + ""]; if (!this.flags.collapsed && v) { const text = v["counter"] + " cached frame(s)"; ctx.save(); // Set font for measuring text width and for drawing ctx.font = "bold 14px sans-serif"; // Measure text to center it on the rectangle const sz = ctx.measureText(text); const textWidth = sz.width; const textHeight = 14; // Approximation based on font size // Rectangle dimensions and position const rectWidth = textWidth + 20; // Padding around text const rectHeight = textHeight + 10; // Padding around text const rectX = (this.size[0] - rectWidth) / 2; const rectY = LiteGraph.NODE_TITLE_HEIGHT - rectHeight / 2 - 15; // Draw rectangle ctx.fillStyle = "rgba(0, 0, 0, 0.8)"; // Semi-transparent dark rectangle ctx.fillRect(rectX, rectY, rectWidth, rectHeight); // Draw text centered in the rectangle ctx.fillStyle = "white"; // White text color const textX = (this.size[0] - textWidth) / 2; const textY = LiteGraph.NODE_TITLE_HEIGHT + textHeight / 2 - 15; // Adjust based on the font size ctx.fillText(text, textX, textY); ctx.restore(); } return r; }; }; }, }); class FloatingConsole { constructor() { this.element = document.createElement('div'); this.element.id = 'floating-console'; this.titleBar = this.createTitleBar(); this.contentContainer = this.createContentContainer(); this.element.appendChild(this.titleBar); this.element.appendChild(this.contentContainer); document.body.appendChild(this.element); this.dragging = false; this.prevX = 0; this.prevY = 0; this.setupStyles(); this.addEventListeners(); this.addMenuButton(); } setupStyles() { Object.assign(this.element.style, { position: 'fixed', bottom: '10px', right: '10px', width: '300px', maxHeight: '600px', backgroundColor: 'rgba(0, 0, 0, 0.8)', color: 'white', borderRadius: '5px', zIndex: '1000', display: 'none', // Consider starting visible for debugging boxSizing: 'border-box', overflow: 'hidden', resize: 'both', }); // Ensure the content container allows for scrolling overflow content Object.assign(this.contentContainer.style, { overflowY: 'auto', maxHeight: '565px', // Adjust based on titleBar height to prevent overflow }); this.adjustContentContainerSize(); } adjustContentContainerSize() { // Calculate available height for content container const titleBarHeight = this.titleBar.offsetHeight; const consoleHeight = this.element.offsetHeight; const availableHeight = consoleHeight - titleBarHeight; // Update content container's maxHeight to fill available space this.contentContainer.style.maxHeight = `${availableHeight}px`; } createTitleBar() { const bar = document.createElement('div'); bar.textContent = 'Console'; Object.assign(bar.style, { padding: '5px', cursor: 'move', backgroundColor: '#333', borderTopLeftRadius: '5px', borderTopRightRadius: '5px', userSelect: 'none', }); return bar; } createContentContainer() { const container = document.createElement('div'); return container; } addEventListeners() { this.titleBar.addEventListener('mousedown', (e) => { // Mark as dragging this.dragging = true; // Record the initial mouse position this.prevX = e.clientX; this.prevY = e.clientY; if (!this.element.style.left || !this.element.style.top) { // Calculate initial left and top based on current position const rect = this.element.getBoundingClientRect(); this.element.style.right = ''; // Clear 'right' since we're switching to 'left/top' positioning this.element.style.bottom = ''; // Clear 'bottom' as well // Set initial left and top based on the element's current position this.element.style.left = `${rect.left}px`; this.element.style.top = `${rect.top}px`; } }); document.addEventListener('mousemove', (e) => { if (!this.dragging) return; const dx = e.clientX - this.prevX; const dy = e.clientY - this.prevY; const { style } = this.element; style.left = `${parseInt(style.left || 0, 10) + dx}px`; style.top = `${parseInt(style.top || 0, 10) + dy}px`; this.prevX = e.clientX; this.prevY = e.clientY; }); document.addEventListener('mouseup', () => { this.dragging = false; this.adjustContentContainerSize(); }); } addMenuButton() { const menu = document.querySelector(".comfy-menu"); // Create and append the toggle button for the floating console const consoleToggleButton = document.createElement("button"); consoleToggleButton.textContent = "Toggle Console"; consoleToggleButton.onclick = () => { // Check if the console is currently visible and toggle accordingly if (floatingConsole.isVisible()) { floatingConsole.hide(); consoleToggleButton.textContent = "Show Console"; // Update button text } else { floatingConsole.show(); consoleToggleButton.textContent = "Hide Console"; // Update button text } } menu.append(consoleToggleButton); } show() { this.element.style.display = 'block'; } hide() { this.element.style.display = 'none'; } isVisible() { return this.element.style.display !== 'none'; } log(message) { const msgElement = document.createElement('div'); msgElement.textContent = message; this.contentContainer.appendChild(msgElement); this.contentContainer.scrollTop = this.contentContainer.scrollHeight; // Auto-scroll to bottom } clear() { this.contentContainer.innerHTML = ''; } } // Instantiate the floating console const floatingConsole = new FloatingConsole(); // Extend the app plugin to handle console_output events app.registerExtension({ name: "consoleOutput", init() { api.addEventListener('console_output', (event) => { floatingConsole.log(event.detail.message); }); } });