Repository: IuvenisSapiens/ComfyUI_MiniCPM-V-2_6-int4 Branch: main Commit: 7c656fa8a87f Files: 20 Total size: 83.1 KB Directory structure: gitextract_48fg5k5m/ ├── .gitignore ├── LICENSE ├── README.md ├── __init__.py ├── display_text_nodes.py ├── examples/ │ ├── Chat_with_multiple_images_workflow_legacy.json │ ├── Chat_with_multiple_images_workflow_polished.json │ ├── Chat_with_single_image_workflow_legacy.json │ ├── Chat_with_single_image_workflow_polished.json │ ├── Chat_with_text_workflow_legacy.json │ ├── Chat_with_text_workflow_polished.json │ ├── Chat_with_video_workflow_legacy.json │ └── Chat_with_video_workflow_polished.json ├── image_nodes.py ├── nodes_legacy.py ├── nodes_polished.py ├── pyproject.toml ├── requirements.txt └── web/ └── js/ ├── displayText.js └── multipleImagesInput.js ================================================ FILE CONTENTS ================================================ ================================================ FILE: .gitignore ================================================ # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] *$py.class # C extensions *.so # Distribution / packaging .Python build/ develop-eggs/ dist/ downloads/ eggs/ .eggs/ lib/ lib64/ parts/ sdist/ var/ wheels/ share/python-wheels/ *.egg-info/ .installed.cfg *.egg MANIFEST # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. *.manifest *.spec # Installer logs pip-log.txt pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ .tox/ .nox/ .coverage .coverage.* .cache nosetests.xml coverage.xml *.cover *.py,cover .hypothesis/ .pytest_cache/ cover/ # Translations *.mo *.pot # Django stuff: *.log local_settings.py db.sqlite3 db.sqlite3-journal # Flask stuff: instance/ .webassets-cache # Scrapy stuff: .scrapy # Sphinx documentation docs/_build/ # PyBuilder .pybuilder/ target/ # Jupyter Notebook .ipynb_checkpoints # IPython profile_default/ ipython_config.py # pyenv # For a library or package, you might want to ignore these files since the code is # intended to run in multiple environments; otherwise, check them in: # .python-version # pipenv # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. # However, in case of collaboration, if having platform-specific dependencies or dependencies # having no cross-platform support, pipenv may install dependencies that don't work, or not # install all needed dependencies. #Pipfile.lock # poetry # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. # This is especially recommended for binary packages to ensure reproducibility, and is more # commonly ignored for libraries. # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control #poetry.lock # pdm # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. #pdm.lock # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it # in version control. # https://pdm.fming.dev/latest/usage/project/#working-with-version-control .pdm.toml .pdm-python .pdm-build/ # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm __pypackages__/ # Celery stuff celerybeat-schedule celerybeat.pid # SageMath parsed files *.sage.py # Environments .env .venv env/ venv/ ENV/ env.bak/ venv.bak/ # Spyder project settings .spyderproject .spyproject # Rope project settings .ropeproject # mkdocs documentation /site # mypy .mypy_cache/ .dmypy.json dmypy.json # Pyre type checker .pyre/ # pytype static type analyzer .pytype/ # Cython debug symbols cython_debug/ # PyCharm # JetBrains specific template is maintained in a separate JetBrains.gitignore that can # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ ================================================ FILE: LICENSE ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright 2024 OpenBMB Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: README.md ================================================ # ComfyUI_MiniCPM-V-4_5 This is an implementation of [MiniCPM-V-4_5](https://github.com/OpenBMB/MiniCPM-V) by [ComfyUI](https://github.com/comfyanonymous/ComfyUI), including support for text-based queries, video queries, single-image queries, and multi-image queries to generate captions or responses. --- ## Recent Updates - Added `keep_model_loaded` parameter By default, this parameter is set to False, which indicates that the model will be unloaded from GPU memory after each prediction is made. However, if set to True, the model will remain loaded in GPU memory. This is particularly useful when multiple predictions with the same model are needed, eliminating the need to reload it between uses. - Added `seed` parameter This parameter enables the setting of a random seed for the purpose of ensuring reproducibility in results. --- ## Basic Workflow - **Text-based Query**: Users can submit textual queries to request information or generate descriptions. For instance, a user might input a description like "What is the meaning of life?" > Chat_with_text_workflow_legacy preview > ![Chat_with_text_workflow_legacy preview](examples/Chat_with_text_workflow_legacy.png) > Chat_with_text_workflow_polished preview > ![Chat_with_text_workflow_polished preview](examples/Chat_with_text_workflow_polished.png) - **Video Query**: When a user uploads a video, the system can analyze the content and generate a detailed caption for each frame or a summary of the entire video. For example, "Generate a caption for the given video." > Chat_with_video_workflow_legacy preview > ![Chat_with_video_workflow_legacy preview](examples/Chat_with_video_workflow_legacy.png) > Chat_with_video_workflow_polished preview > ![Chat_with_video_workflow_polished preview](examples/Chat_with_video_workflow_polished.png) - **Single-Image Query**: This workflow supports generating a caption for an individual image. A user could upload a photo and ask, "What does this image show?" resulting in a caption such as "A majestic lion pride relaxing on the savannah." > Chat_with_single_image_workflow_legacy preview > ![Chat_with_single_image_workflow_legacy preview](examples/Chat_with_single_image_workflow_legacy.png) > Chat_with_single_image_workflow_polished preview > ![Chat_with_single_image_workflow_polished preview](examples/Chat_with_single_image_workflow_polished.png) - **Multi-Image Query**: For multiple images, the system can provide a collective description or a narrative that ties the images together. For example, "Create a story from the following series of images: one of a couple at a beach, another at a wedding ceremony, and the last one at a baby's christening." > Chat_with_multiple_images_workflow_legacy preview > ![Chat_with_multiple_images_workflow_legacy preview](examples/Chat_with_multiple_images_workflow_legacy.png) > Chat_with_multiple_images_workflow_polished preview > ![Chat_with_multiple_images_workflow_polished preview](examples/Chat_with_multiple_images_workflow_polished.png) ## Installation - Install from [ComfyUI Manager](https://github.com/ltdrdata/ComfyUI-Manager) (search for `minicpm`) - Download or git clone this repository into the `ComfyUI\custom_nodes\` directory and run: ```python pip install -r requirements.txt ``` ## Download Models All the models will be downloaded automatically when running the workflow if they are not found in the `ComfyUI\models\prompt_generator\` directory. ================================================ FILE: __init__.py ================================================ from .nodes_legacy import MiniCPM_VQA from .nodes_polished import MiniCPM_VQA_Polished from .image_nodes import MultipleImagesInput from .display_text_nodes import DisplayText WEB_DIRECTORY = "./web" # A dictionary that contains all nodes you want to export with their names # NOTE: names should be globally unique NODE_CLASS_MAPPINGS = { "MultipleImagesInput": MultipleImagesInput, "MiniCPM_VQA": MiniCPM_VQA, "MiniCPM_VQA_Polished": MiniCPM_VQA_Polished, "DisplayText": DisplayText, } # A dictionary that contains the friendly/humanly readable titles for the nodes NODE_DISPLAY_NAME_MAPPINGS = { "MultipleImagesInput": "Multiple Images Input", "MiniCPM_VQA": "MiniCPM VQA", "MiniCPM_VQA_Polished": "MiniCPM VQA Polished", "DisplayText": "Display Text", } ================================================ FILE: display_text_nodes.py ================================================ class DisplayText: @classmethod def INPUT_TYPES(s): return { "required": { "text": ("STRING", {"forceInput": True}), } } INPUT_IS_LIST = True RETURN_TYPES = ("STRING",) OUTPUT_NODE = True OUTPUT_IS_LIST = (True,) FUNCTION = "display_text" CATEGORY = "Comfyui_MiniCPM-V-4_5" def display_text(self, text): return {"ui": {"text": text}, "result": (text,)} ================================================ FILE: examples/Chat_with_multiple_images_workflow_legacy.json ================================================ { "id": "cf5badb9-49fe-458c-a923-f3b66e937402", "revision": 0, "last_node_id": 57, "last_link_id": 69, "nodes": [ { "id": 45, "type": "LoadImage", "pos": [ -691, 15 ], "size": [ 315, 314 ], "flags": {}, "order": 0, "mode": 0, "inputs": [], "outputs": [ { "name": "IMAGE", "type": "IMAGE", "slot_index": 0, "links": [ 61 ] }, { "name": "MASK", "type": "MASK", "links": null } ], "properties": { "Node name for S&R": "LoadImage", "cnr_id": "comfy-core", "ver": "0.3.44", "widget_ue_connectable": {} }, "widgets_values": [ "ComfyUI_00532_.png", "image" ] }, { "id": 43, "type": "LoadImage", "pos": [ -361, -193 ], "size": [ 315, 314 ], "flags": {}, "order": 1, "mode": 0, "inputs": [], "outputs": [ { "name": "IMAGE", "type": "IMAGE", "slot_index": 0, "links": [ 62 ] }, { "name": "MASK", "type": "MASK", "links": null } ], "properties": { "Node name for S&R": "LoadImage", "cnr_id": "comfy-core", "ver": "0.3.44", "widget_ue_connectable": {} }, "widgets_values": [ "ComfyUI_00509_.png", "image" ] }, { "id": 47, "type": "LoadImage", "pos": [ -360, 161 ], "size": [ 315, 314 ], "flags": {}, "order": 2, "mode": 0, "inputs": [], "outputs": [ { "name": "IMAGE", "type": "IMAGE", "slot_index": 0, "links": [ 63 ] }, { "name": "MASK", "type": "MASK", "links": null } ], "properties": { "Node name for S&R": "LoadImage", "cnr_id": "comfy-core", "ver": "0.3.44", "widget_ue_connectable": {} }, "widgets_values": [ "ComfyUI_00508_.png", "image" ] }, { "id": 7, "type": "Note", "pos": [ 423.2934265136719, -52.42848205566406 ], "size": [ 436.568115234375, 108.88176727294922 ], "flags": { "collapsed": false }, "order": 3, "mode": 0, "inputs": [], "outputs": [], "properties": { "text": "", "widget_ue_connectable": {} }, "widgets_values": [ "当 MiniCPM VQA 同时接收到图像和视频信息时，它会仅处理视频信息而忽略图像信息。如果您想处理图像信息，请断开视频信息的输入。\n\nWhen MiniCPM VQA simultaneously receives both image and video information, it processes only the video information while ignoring the image information. If you want to process the image information, please disconnect the input of the video information." ], "color": "#432", "bgcolor": "#653" }, { "id": 51, "type": "MiniCPM_VQA", "pos": [ -13, -65 ], "size": [ 400, 492 ], "flags": {}, "order": 4, "mode": 0, "inputs": [ { "name": "source_video", "shape": 7, "type": "VIDEO", "link": null }, { "name": "source_image_1st", "shape": 7, "type": "IMAGE", "link": 62 }, { "name": "source_image_2nd", "shape": 7, "type": "IMAGE", "link": 61 }, { "name": "source_image_3rd", "shape": 7, "type": "IMAGE", "link": 63 } ], "outputs": [ { "name": "STRING", "type": "STRING", "links": [ 69 ] } ], "properties": { "Node name for S&R": "MiniCPM_VQA", "aux_id": "zhuanvi/ComfyUI_MiniCPM-V-2_6-int4", "ver": "763f95133a95b611977efaa05ad654ff410670a9", "widget_ue_connectable": {} }, "widgets_values": [ "Compare image 1, image 2 and image 3, tell me about the differences among them.", "MiniCPM-V-4_5-int4", false, 0.8, 100, 0.7, 1.05, 2048, 64, 2, 1888, "randomize" ] }, { "id": 57, "type": "DisplayText", "pos": [ 427.6014404296875, 119.65513610839844 ], "size": [ 431.3169250488281, 210.38330078125 ], "flags": {}, "order": 5, "mode": 0, "inputs": [ { "name": "text", "type": "STRING", "link": 69 } ], "outputs": [ { "name": "STRING", "shape": 6, "type": "STRING", "links": null } ], "properties": { "widget_ue_connectable": {}, "Node name for S&R": "DisplayText" }, "widgets_values": [ "Image 1 and image 2 depict the Earth with different parts of its surface visible. Image 1 shows North America, while image 2 focuses on Africa and Europe. In contrast, image 3 features Saturn with its iconic rings surrounding it. The main difference among these images is the celestial body depicted in each one." ] } ], "links": [ [ 61, 45, 0, 51, 2, "IMAGE" ], [ 62, 43, 0, 51, 1, "IMAGE" ], [ 63, 47, 0, 51, 3, "IMAGE" ], [ 69, 51, 0, 57, 0, "STRING" ] ], "groups": [], "config": {}, "extra": { "ds": { "scale": 0.8378523119015135, "offset": [ 817.9855578825247, 292.40168823944055 ] }, "groupNodes": {}, "ue_links": [], "links_added_by_ue": [], "frontendVersion": "1.25.5", "VHS_latentpreview": false, "VHS_latentpreviewrate": 0, "VHS_MetadataImage": true, "VHS_KeepIntermediate": true }, "version": 0.4 } ================================================ FILE: examples/Chat_with_multiple_images_workflow_polished.json ================================================ { "id": "961bfaa7-3def-4804-8b24-232d21a3abfd", "revision": 0, "last_node_id": 63, "last_link_id": 75, "nodes": [ { "id": 7, "type": "Note", "pos": [ -986, -453 ], "size": [ 717.5083618164062, 88 ], "flags": { "collapsed": false }, "order": 0, "mode": 0, "inputs": [], "outputs": [], "properties": { "text": "", "widget_ue_connectable": {} }, "widgets_values": [ "当 MiniCPM VQA 同时接收到图像和视频信息时，它会仅处理视频信息而忽略图像信息。如果您想处理图像信息，请断开视频信息的输入。\n\nWhen MiniCPM VQA simultaneously receives both image and video information, it processes only the video information while ignoring the image information. If you want to process the image information, please disconnect the input of the video information." ], "color": "#432", "bgcolor": "#653" }, { "id": 55, "type": "LoadImage", "pos": [ -1232, -453 ], "size": [ 210, 314 ], "flags": {}, "order": 1, "mode": 0, "inputs": [], "outputs": [ { "name": "IMAGE", "type": "IMAGE", "links": [ 65 ] }, { "name": "MASK", "type": "MASK", "links": null } ], "properties": { "Node name for S&R": "LoadImage", "cnr_id": "comfy-core", "ver": "0.3.44", "widget_ue_connectable": {} }, "widgets_values": [ "ComfyUI_00509_.png", "image" ] }, { "id": 56, "type": "LoadImage", "pos": [ -1234, -122 ], "size": [ 214.43836975097656, 314 ], "flags": {}, "order": 2, "mode": 0, "inputs": [], "outputs": [ { "name": "IMAGE", "type": "IMAGE", "links": [ 66 ] }, { "name": "MASK", "type": "MASK", "links": null } ], "properties": { "Node name for S&R": "LoadImage", "cnr_id": "comfy-core", "ver": "0.3.44", "widget_ue_connectable": {} }, "widgets_values": [ "ComfyUI_00532_.png", "image" ] }, { "id": 58, "type": "LoadImage", "pos": [ -1232, 227 ], "size": [ 210, 314 ], "flags": {}, "order": 3, "mode": 0, "inputs": [], "outputs": [ { "name": "IMAGE", "type": "IMAGE", "links": [ 68 ] }, { "name": "MASK", "type": "MASK", "links": null } ], "properties": { "Node name for S&R": "LoadImage", "cnr_id": "comfy-core", "ver": "0.3.44", "widget_ue_connectable": {} }, "widgets_values": [ "ComfyUI_00508_.png", "image" ] }, { "id": 59, "type": "PreviewImage", "pos": [ -247, -450 ], "size": [ 321.89825439453125, 978.513916015625 ], "flags": {}, "order": 5, "mode": 0, "inputs": [ { "name": "images", "type": "IMAGE", "link": 69 } ], "outputs": [], "properties": { "Node name for S&R": "PreviewImage", "cnr_id": "comfy-core", "ver": "0.3.44", "widget_ue_connectable": {} }, "widgets_values": [] }, { "id": 54, "type": "MultipleImagesInput", "pos": [ -986, -136 ], "size": [ 210, 122 ], "flags": {}, "order": 4, "mode": 0, "inputs": [ { "name": "image_1", "type": "IMAGE", "link": 65 }, { "name": "image_2", "type": "IMAGE", "link": 66 }, { "name": "image_3", "type": "IMAGE", "link": 68 } ], "outputs": [ { "name": "images", "type": "IMAGE", "slot_index": 0, "links": [ 69, 73 ] } ], "properties": { "Node name for S&R": "MultipleImagesInput", "aux_id": "zhuanvi/ComfyUI_MiniCPM-V-2_6-int4", "ver": "763f95133a95b611977efaa05ad654ff410670a9", "widget_ue_connectable": {} }, "widgets_values": [ 3 ] }, { "id": 61, "type": "MiniCPM_VQA_Polished", "pos": [ -697, -241 ], "size": [ 400, 412 ], "flags": {}, "order": 6, "mode": 0, "inputs": [ { "name": "source_video", "shape": 7, "type": "VIDEO", "link": null }, { "name": "source_image", "shape": 7, "type": "IMAGE", "link": 73 } ], "outputs": [ { "name": "STRING", "type": "STRING", "links": [ 75 ] } ], "properties": { "Node name for S&R": "MiniCPM_VQA_Polished", "aux_id": "zhuanvi/ComfyUI_MiniCPM-V-2_6-int4", "ver": "763f95133a95b611977efaa05ad654ff410670a9", "widget_ue_connectable": {} }, "widgets_values": [ "Compare image 1, image 2 and image 3, tell me about the differences among them.", "MiniCPM-V-4_5-int4", false, 0.8, 100, 0.7, 1.05, 2048, 64, 2, 1574, "randomize" ] }, { "id": 63, "type": "DisplayText", "pos": [ -695.5367431640625, 236.86334228515625 ], "size": [ 398.0273132324219, 251.64744567871094 ], "flags": {}, "order": 7, "mode": 0, "inputs": [ { "name": "text", "type": "STRING", "link": 75 } ], "outputs": [ { "name": "STRING", "shape": 6, "type": "STRING", "links": null } ], "properties": { "widget_ue_connectable": {}, "Node name for S&R": "DisplayText" }, "widgets_values": [ "Image 1 shows a view of Earth from space with the North American continent prominently displayed. Image 2 is similar to image 1 but includes an additional celestial body, which appears to be a moon or another planet, positioned near the top left corner of the frame. Image 3 depicts Earth again, this time with Saturn and its rings visible in the background. The sun is also present in all three images, shining brightly behind Earth.\n\nThe main differences among these images are:\n1. Presence of Additional Celestial Bodies: Image 2 has an extra object (moon/planet) compared to image 1.\n2. Background Elements: Image 3 introduces Saturn and its rings as part of the backdrop, whereas image 1 does not have any such features." ] } ], "links": [ [ 65, 55, 0, 54, 0, "IMAGE" ], [ 66, 56, 0, 54, 1, "IMAGE" ], [ 68, 58, 0, 54, 2, "IMAGE" ], [ 69, 54, 0, 59, 0, "IMAGE" ], [ 73, 54, 0, 61, 1, "IMAGE" ], [ 75, 61, 0, 63, 0, "STRING" ] ], "groups": [], "config": {}, "extra": { "ds": { "scale": 0.6588450000000009, "offset": [ 1525.610289271432, 555.581645176858 ] }, "groupNodes": {}, "ue_links": [], "links_added_by_ue": [], "frontendVersion": "1.25.5", "VHS_latentpreview": false, "VHS_latentpreviewrate": 0, "VHS_MetadataImage": true, "VHS_KeepIntermediate": true }, "version": 0.4 } ================================================ FILE: examples/Chat_with_single_image_workflow_legacy.json ================================================ { "id": "663271fd-c088-42ea-892f-68d527b2a28e", "revision": 0, "last_node_id": 57, "last_link_id": 68, "nodes": [ { "id": 7, "type": "Note", "pos": [ 385, -6 ], "size": [ 681.107421875, 92.63203430175781 ], "flags": { "collapsed": false }, "order": 0, "mode": 0, "inputs": [], "outputs": [], "properties": { "text": "", "widget_ue_connectable": {} }, "widgets_values": [ "当 MiniCPM VQA 同时接收到图像和视频信息时，它会仅处理视频信息而忽略图像信息。如果您想处理图像信息，请断开视频信息的输入。\n\nWhen MiniCPM VQA simultaneously receives both image and video information, it processes only the video information while ignoring the image information. If you want to process the image information, please disconnect the input of the video information." ], "color": "#432", "bgcolor": "#653" }, { "id": 51, "type": "LoadImage", "pos": [ -363, -8 ], "size": [ 308.788818359375, 398.5933532714844 ], "flags": {}, "order": 1, "mode": 0, "inputs": [], "outputs": [ { "name": "IMAGE", "type": "IMAGE", "slot_index": 0, "links": [ 67 ] }, { "name": "MASK", "type": "MASK", "links": null } ], "properties": { "Node name for S&R": "LoadImage", "cnr_id": "comfy-core", "ver": "0.3.44", "widget_ue_connectable": {} }, "widgets_values": [ "ComfyUI_00509_.png", "image" ] }, { "id": 56, "type": "MiniCPM_VQA", "pos": [ -34, -9 ], "size": [ 400, 412 ], "flags": {}, "order": 2, "mode": 0, "inputs": [ { "name": "source_video", "shape": 7, "type": "VIDEO", "link": null }, { "name": "source_image_1st", "shape": 7, "type": "IMAGE", "link": 67 }, { "name": "source_image_2nd", "shape": 7, "type": "IMAGE", "link": null }, { "name": "source_image_3rd", "shape": 7, "type": "IMAGE", "link": null } ], "outputs": [ { "name": "STRING", "type": "STRING", "links": [ 68 ] } ], "properties": { "Node name for S&R": "MiniCPM_VQA", "aux_id": "zhuanvi/ComfyUI_MiniCPM-V-2_6-int4", "ver": "763f95133a95b611977efaa05ad654ff410670a9", "widget_ue_connectable": {} }, "widgets_values": [ "Describe the image in detail", "MiniCPM-V-4_5-int4", true, 0.8, 100, 0.7, 1.05, 2048, 64, 2, 1443, "randomize" ] }, { "id": 57, "type": "DisplayText", "pos": [ 388.22991943359375, 140.32000732421875 ], "size": [ 677.37890625, 259.0337829589844 ], "flags": {}, "order": 3, "mode": 0, "inputs": [ { "name": "text", "type": "STRING", "link": 68 } ], "outputs": [ { "name": "STRING", "shape": 6, "type": "STRING", "links": null } ], "properties": { "widget_ue_connectable": {}, "Node name for S&R": "DisplayText" }, "widgets_values": [ "This image is a highly detailed, computer-generated depiction of Earth from space. The perspective showcases the planet's curvature prominently in the center of the frame against an expansive black background that accentuates its vibrant colors and intricate details.\n\nThe continents are vividly rendered with rich green hues representing landmasses, particularly noticeable on North America where Canada, parts of the United States, Mexico, and Central America stand out clearly. South America occupies the lower right portion of the globe. Above these regions lies the Arctic area, distinguished by patches of white snow covering northern Canada and surrounding areas.\n\nThe oceans appear as deep blue, providing a stark contrast to the earthy tones of the continents. Scattered across various sections of the ocean are small white specks symbolizing clouds, adding depth and realism to the atmospheric portrayal.\n\nIn the upper left corner of the image, swirling patterns resembling whirlwinds or cloud formations add dynamic visual interest. A bright sun glows softly near the top edge of the composition, casting subtle light onto part of the atmosphere above North America. This interplay of natural elements creates a captivating snapshot of our home planet set within the vastness of outer space." ] } ], "links": [ [ 67, 51, 0, 56, 1, "IMAGE" ], [ 68, 56, 0, 57, 0, "STRING" ] ], "groups": [], "config": {}, "extra": { "ds": { "scale": 0.9646149645000006, "offset": [ 411.65231785482786, 170.44936703511385 ] }, "groupNodes": {}, "ue_links": [], "frontendVersion": "1.25.5", "VHS_latentpreview": false, "VHS_latentpreviewrate": 0, "VHS_MetadataImage": true, "VHS_KeepIntermediate": true, "links_added_by_ue": [] }, "version": 0.4 } ================================================ FILE: examples/Chat_with_single_image_workflow_polished.json ================================================ { "id": "f07442f7-7cf5-439d-ba91-1df453dd7521", "revision": 0, "last_node_id": 58, "last_link_id": 69, "nodes": [ { "id": 7, "type": "Note", "pos": [ 385, -6 ], "size": [ 681.107421875, 92.63203430175781 ], "flags": { "collapsed": false }, "order": 0, "mode": 0, "inputs": [], "outputs": [], "properties": { "text": "", "widget_ue_connectable": {} }, "widgets_values": [ "当 MiniCPM VQA 同时接收到图像和视频信息时，它会仅处理视频信息而忽略图像信息。如果您想处理图像信息，请断开视频信息的输入。\n\nWhen MiniCPM VQA simultaneously receives both image and video information, it processes only the video information while ignoring the image information. If you want to process the image information, please disconnect the input of the video information." ], "color": "#432", "bgcolor": "#653" }, { "id": 51, "type": "LoadImage", "pos": [ -348, -2 ], "size": [ 293.6593017578125, 358.04742431640625 ], "flags": {}, "order": 1, "mode": 0, "inputs": [], "outputs": [ { "name": "IMAGE", "type": "IMAGE", "slot_index": 0, "links": [ 67 ] }, { "name": "MASK", "type": "MASK", "links": null } ], "properties": { "Node name for S&R": "LoadImage", "cnr_id": "comfy-core", "ver": "0.3.44", "widget_ue_connectable": {} }, "widgets_values": [ "ComfyUI_00509_.png", "image" ] }, { "id": 57, "type": "MiniCPM_VQA_Polished", "pos": [ -34, -4 ], "size": [ 400, 372 ], "flags": {}, "order": 2, "mode": 0, "inputs": [ { "name": "source_video", "shape": 7, "type": "VIDEO", "link": null }, { "name": "source_image", "shape": 7, "type": "IMAGE", "link": 67 } ], "outputs": [ { "name": "STRING", "type": "STRING", "links": [ 69 ] } ], "properties": { "Node name for S&R": "MiniCPM_VQA_Polished", "aux_id": "zhuanvi/ComfyUI_MiniCPM-V-2_6-int4", "ver": "763f95133a95b611977efaa05ad654ff410670a9", "widget_ue_connectable": {} }, "widgets_values": [ "Describe the image in detail", "MiniCPM-V-4_5-int4", true, 0.8, 100, 0.7, 1.05, 2048, 64, 2, 1845, "randomize" ] }, { "id": 58, "type": "DisplayText", "pos": [ 390.8650207519531, 141.06495666503906 ], "size": [ 673.6586303710938, 218.28106689453125 ], "flags": {}, "order": 3, "mode": 0, "inputs": [ { "name": "text", "type": "STRING", "link": 69 } ], "outputs": [ { "name": "STRING", "shape": 6, "type": "STRING", "links": null } ], "properties": { "widget_ue_connectable": {}, "Node name for S&R": "DisplayText" }, "widgets_values": [ "This image is a highly detailed and realistic depiction of planet Earth, showcasing the continents with remarkable clarity. Prominently featured are North America and South America at the center-left portion of the globe. The northern regions of these continents exhibit brown landmasses indicative of their diverse topography. Surrounding these areas are patches of green representing lush vegetation or forests, interspersed with white clouds that drift across the atmosphere.\n\nThe oceanic expanse around the continents appears as deep blue, adding contrast to the vibrant colors on the land. Towards the upper right corner of the image, there's an intriguing element resembling either another celestial body or perhaps atmospheric phenomena such as a solar flare emitting bright light. This adds a sense of depth and mystery to the overall composition.\n\nOverall, this captivating portrayal emphasizes both the beauty and complexity of our home planet while hinting at its enigmatic surroundings in space." ] } ], "links": [ [ 67, 51, 0, 57, 1, "IMAGE" ], [ 69, 57, 0, 58, 0, "STRING" ] ], "groups": [], "config": {}, "extra": { "ds": { "scale": 1.0610764609500007, "offset": [ 391.6576442287418, 120.97620455457698 ] }, "groupNodes": {}, "ue_links": [], "frontendVersion": "1.25.5", "VHS_latentpreview": false, "VHS_latentpreviewrate": 0, "VHS_MetadataImage": true, "VHS_KeepIntermediate": true, "links_added_by_ue": [] }, "version": 0.4 } ================================================ FILE: examples/Chat_with_text_workflow_legacy.json ================================================ { "id": "b3951e8d-a7bd-48a8-bf05-57af5373eb27", "revision": 0, "last_node_id": 53, "last_link_id": 56, "nodes": [ { "id": 7, "type": "Note", "pos": [ 398, -255 ], "size": [ 560.1107788085938, 103.60144805908203 ], "flags": { "collapsed": false }, "order": 0, "mode": 0, "inputs": [], "outputs": [], "properties": { "text": "", "widget_ue_connectable": {} }, "widgets_values": [ "当 MiniCPM VQA 同时接收到图像和视频信息时，它会仅处理视频信息而忽略图像信息。如果您想处理图像信息，请断开视频信息的输入。\n\nWhen MiniCPM VQA simultaneously receives both image and video information, it processes only the video information while ignoring the image information. If you want to process the image information, please disconnect the input of the video information." ], "color": "#432", "bgcolor": "#653" }, { "id": 52, "type": "MiniCPM_VQA", "pos": [ -21, -256 ], "size": [ 403.634765625, 428.1322326660156 ], "flags": {}, "order": 1, "mode": 0, "inputs": [ { "name": "source_video", "shape": 7, "type": "VIDEO", "link": null }, { "name": "source_image_1st", "shape": 7, "type": "IMAGE", "link": null }, { "name": "source_image_2nd", "shape": 7, "type": "IMAGE", "link": null }, { "name": "source_image_3rd", "shape": 7, "type": "IMAGE", "link": null } ], "outputs": [ { "name": "STRING", "type": "STRING", "links": [ 56 ] } ], "properties": { "Node name for S&R": "MiniCPM_VQA", "aux_id": "zhuanvi/ComfyUI_MiniCPM-V-2_6-int4", "ver": "763f95133a95b611977efaa05ad654ff410670a9", "widget_ue_connectable": {} }, "widgets_values": [ "Why is the sky blue?", "MiniCPM-V-4_5-int4", false, 0.8, 100, 0.7, 1.05, 2048, 64, 2, 1437, "randomize" ] }, { "id": 53, "type": "DisplayText", "pos": [ 402.0209045410156, -102.32914733886719 ], "size": [ 555.4002075195312, 272.1246032714844 ], "flags": {}, "order": 2, "mode": 0, "inputs": [ { "name": "text", "type": "STRING", "link": 56 } ], "outputs": [ { "name": "STRING", "shape": 6, "type": "STRING", "links": null } ], "properties": { "widget_ue_connectable": {}, "Node name for S&R": "DisplayText" }, "widgets_values": [ "The sky appears blue because of a phenomenon called Rayleigh scattering. When sunlight passes through the Earth's atmosphere, it collides with particles such as nitrogen and oxygen molecules. These collisions cause the light to scatter in different directions. Blue light has a shorter wavelength than other colors of light, so it is scattered more easily by these particles. This results in our eyes perceiving the color blue when we look up at the sky on a clear day." ] } ], "links": [ [ 56, 52, 0, 53, 0, "STRING" ] ], "groups": [], "config": {}, "extra": { "ds": { "scale": 1.2839025177495011, "offset": [ 89.1900206229874, 335.4725883983666 ] }, "groupNodes": {}, "ue_links": [], "frontendVersion": "1.25.5", "VHS_latentpreview": false, "VHS_latentpreviewrate": 0, "VHS_MetadataImage": true, "VHS_KeepIntermediate": true, "links_added_by_ue": [] }, "version": 0.4 } ================================================ FILE: examples/Chat_with_text_workflow_polished.json ================================================ { "id": "7efe6818-5b98-4dbe-aae3-317073142f92", "revision": 0, "last_node_id": 56, "last_link_id": 59, "nodes": [ { "id": 7, "type": "Note", "pos": [ 390, -254 ], "size": [ 443.0672302246094, 104.75543212890625 ], "flags": { "collapsed": false }, "order": 0, "mode": 0, "inputs": [], "outputs": [], "properties": { "text": "", "widget_ue_connectable": {} }, "widgets_values": [ "当 MiniCPM VQA 同时接收到图像和视频信息时，它会仅处理视频信息而忽略图像信息。如果您想处理图像信息，请断开视频信息的输入。\n\nWhen MiniCPM VQA simultaneously receives both image and video information, it processes only the video information while ignoring the image information. If you want to process the image information, please disconnect the input of the video information." ], "color": "#432", "bgcolor": "#653" }, { "id": 53, "type": "MiniCPM_VQA_Polished", "pos": [ -28, -258 ], "size": [ 400, 412 ], "flags": {}, "order": 1, "mode": 0, "inputs": [ { "name": "source_video", "shape": 7, "type": "VIDEO", "link": null }, { "name": "source_image", "shape": 7, "type": "IMAGE", "link": null } ], "outputs": [ { "name": "STRING", "type": "STRING", "links": [ 59 ] } ], "properties": { "Node name for S&R": "MiniCPM_VQA_Polished", "aux_id": "zhuanvi/ComfyUI_MiniCPM-V-2_6-int4", "ver": "763f95133a95b611977efaa05ad654ff410670a9", "widget_ue_connectable": {} }, "widgets_values": [ "Why is the sky blue?", "MiniCPM-V-4_5-int4", false, 0.8, 100, 0.7, 1.05, 2048, 64, 2, 1432, "randomize" ] }, { "id": 56, "type": "DisplayText", "pos": [ 392.57000732421875, -93.56874084472656 ], "size": [ 442.1517333984375, 229.07498168945312 ], "flags": {}, "order": 2, "mode": 0, "inputs": [ { "name": "text", "type": "STRING", "link": 59 } ], "outputs": [ { "name": "STRING", "shape": 6, "type": "STRING", "links": null } ], "properties": { "widget_ue_connectable": {}, "Node name for S&R": "DisplayText" }, "widgets_values": [ "The sky is blue because of a phenomenon called Rayleigh scattering. When sunlight enters the Earth's atmosphere, it interacts with gas molecules such as nitrogen and oxygen which scatter shorter wavelengths (blue light) more than longer wavelengths (red light). This causes the sky to appear blue when viewed from space or even in some cases during sunrise/sunset depending on how much dust particles are present within our atmosphere" ] } ], "links": [ [ 59, 53, 0, 56, 0, "STRING" ] ], "groups": [], "config": {}, "extra": { "ds": { "scale": 1.1671841070450013, "offset": [ 189.45751021231808, 351.168856958881 ] }, "groupNodes": {}, "ue_links": [], "frontendVersion": "1.25.5", "VHS_latentpreview": false, "VHS_latentpreviewrate": 0, "VHS_MetadataImage": true, "VHS_KeepIntermediate": true, "links_added_by_ue": [] }, "version": 0.4 } ================================================ FILE: examples/Chat_with_video_workflow_legacy.json ================================================ { "id": "f047c436-5a6d-49eb-87d0-0ace2b958f96", "revision": 0, "last_node_id": 59, "last_link_id": 70, "nodes": [ { "id": 55, "type": "LoadVideo", "pos": [ -578.4146118164062, -150.24563598632812 ], "size": [ 469.4351806640625, 397.9642639160156 ], "flags": {}, "order": 0, "mode": 0, "inputs": [], "outputs": [ { "name": "VIDEO", "type": "VIDEO", "links": [ 66 ] } ], "properties": { "Node name for S&R": "LoadVideo", "cnr_id": "comfy-core", "ver": "0.3.44", "widget_ue_connectable": {} }, "widgets_values": [ "AnimateDiff_00002.mp4", "image" ] }, { "id": 59, "type": "DisplayText", "pos": [ -574.1065673828125, 301.9283447265625 ], "size": [ 893.9513549804688, 157.5222625732422 ], "flags": {}, "order": 3, "mode": 0, "inputs": [ { "name": "text", "type": "STRING", "link": 70 } ], "outputs": [ { "name": "STRING", "shape": 6, "type": "STRING", "links": null } ], "properties": { "widget_ue_connectable": {}, "Node name for S&R": "DisplayText" }, "widgets_values": [ "The video features a striking sunset scene. The sun is prominently positioned near the horizon, casting an intense glow of orange and yellow hues across the sky. This creates a dramatic backdrop with rich gradients from bright yellows at the sun to deeper oranges and reds further away.\n\nIn the foreground, there's a silhouette of a leafless tree on the right side of the frame. Its intricate branches stand out starkly against the vibrant colors of the setting sun, adding depth and contrast to the composition. Above the tree, several birds are captured in mid-flight, their small shapes creating dynamic movement within the serene landscape.\n\nThe overall atmosphere evokes tranquility and beauty as day transitions into night. There are no visible texts or human figures in the image, allowing nature’s elements—the sun, trees, and birds—to dominate the visual narrative." ] }, { "id": 54, "type": "MiniCPM_VQA", "pos": [ -92.34504699707031, -160.6724853515625 ], "size": [ 414.95574951171875, 412 ], "flags": {}, "order": 2, "mode": 0, "inputs": [ { "name": "source_video", "shape": 7, "type": "VIDEO", "link": 66 }, { "name": "source_image_1st", "shape": 7, "type": "IMAGE", "link": null }, { "name": "source_image_2nd", "shape": 7, "type": "IMAGE", "link": null }, { "name": "source_image_3rd", "shape": 7, "type": "IMAGE", "link": null } ], "outputs": [ { "name": "STRING", "type": "STRING", "links": [ 70 ] } ], "properties": { "Node name for S&R": "MiniCPM_VQA", "aux_id": "zhuanvi/ComfyUI_MiniCPM-V-2_6-int4", "ver": "763f95133a95b611977efaa05ad654ff410670a9", "widget_ue_connectable": {} }, "widgets_values": [ "Describe the video in detail", "MiniCPM-V-4_5-int4", false, 0.8, 100, 0.7, 1.05, 2048, 64, 2, 607, "randomize" ] }, { "id": 7, "type": "Note", "pos": [ -563.6396484375, -307.6985168457031 ], "size": [ 845.676513671875, 88.83624267578125 ], "flags": { "collapsed": false }, "order": 1, "mode": 0, "inputs": [], "outputs": [], "properties": { "text": "", "widget_ue_connectable": {} }, "widgets_values": [ "当 MiniCPM VQA 同时接收到图像和视频信息时，它会仅处理视频信息而忽略图像信息。如果您想处理图像信息，请断开视频信息的输入。\n\nWhen MiniCPM VQA simultaneously receives both image and video information, it processes only the video information while ignoring the image information. If you want to process the image information, please disconnect the input of the video information." ], "color": "#432", "bgcolor": "#653" } ], "links": [ [ 66, 55, 0, 54, 0, "VIDEO" ], [ 70, 54, 0, 59, 0, "STRING" ] ], "groups": [], "config": {}, "extra": { "ds": { "scale": 0.8769226950000013, "offset": [ 833.5007984425921, 361.1394852284463 ] }, "groupNodes": {}, "ue_links": [], "links_added_by_ue": [], "frontendVersion": "1.25.5", "VHS_latentpreview": false, "VHS_latentpreviewrate": 0, "VHS_MetadataImage": true, "VHS_KeepIntermediate": true }, "version": 0.4 } ================================================ FILE: examples/Chat_with_video_workflow_polished.json ================================================ { "id": "7ef5c0e0-b61f-4ad8-a2e7-9eb86aaa15e6", "revision": 0, "last_node_id": 58, "last_link_id": 69, "nodes": [ { "id": 55, "type": "LoadVideo", "pos": [ -484.74945068359375, -126.60325622558594 ], "size": [ 409.98541259765625, 399.4589538574219 ], "flags": {}, "order": 0, "mode": 0, "inputs": [], "outputs": [ { "name": "VIDEO", "type": "VIDEO", "links": [ 68 ] } ], "properties": { "Node name for S&R": "LoadVideo", "cnr_id": "comfy-core", "ver": "0.3.44", "widget_ue_connectable": {} }, "widgets_values": [ "AnimateDiff_00002.mp4", "Video" ] }, { "id": 56, "type": "MiniCPM_VQA_Polished", "pos": [ -60.45558547973633, -124.85325622558594 ], "size": [ 411.7491149902344, 394.8070373535156 ], "flags": {}, "order": 2, "mode": 0, "inputs": [ { "name": "source_video", "shape": 7, "type": "VIDEO", "link": 68 }, { "name": "source_image", "shape": 7, "type": "IMAGE", "link": null } ], "outputs": [ { "name": "STRING", "type": "STRING", "links": [ 69 ] } ], "properties": { "Node name for S&R": "MiniCPM_VQA_Polished", "aux_id": "zhuanvi/ComfyUI_MiniCPM-V-2_6-int4", "ver": "763f95133a95b611977efaa05ad654ff410670a9", "widget_ue_connectable": {} }, "widgets_values": [ "Describe the video in detail", "MiniCPM-V-4_5-int4", true, 0.8, 100, 0.7, 1.05, 2048, 64, 2, 978, "randomize" ] }, { "id": 58, "type": "DisplayText", "pos": [ -480.8397216796875, 326.099853515625 ], "size": [ 829.0487670898438, 111.2472915649414 ], "flags": {}, "order": 3, "mode": 0, "inputs": [ { "name": "text", "type": "STRING", "link": 69 } ], "outputs": [ { "name": "STRING", "shape": 6, "type": "STRING", "links": null } ], "properties": { "widget_ue_connectable": {}, "Node name for S&R": "DisplayText" }, "widgets_values": [ "The video captures a stunning sunset scene. The sun is depicted as an enormous, glowing orb in the sky, casting vibrant hues of orange and red across the atmosphere. This creates a dramatic backdrop for the silhouette of a solitary tree standing on what appears to be flat land. Birds can be seen flying around the setting sun, adding life to the serene landscape. There are no other objects like buildings or vehicles visible, emphasizing the natural beauty and tranquility of the moment." ] }, { "id": 7, "type": "Note", "pos": [ -485.3626403808594, -276.0425720214844 ], "size": [ 832.9252319335938, 88 ], "flags": { "collapsed": false }, "order": 1, "mode": 0, "inputs": [], "outputs": [], "properties": { "text": "", "widget_ue_connectable": {} }, "widgets_values": [ "当 MiniCPM VQA 同时接收到图像和视频信息时，它会仅处理视频信息而忽略图像信息。如果您想处理图像信息，请断开视频信息的输入。\n\nWhen MiniCPM VQA simultaneously receives both image and video information, it processes only the video information while ignoring the image information. If you want to process the image information, please disconnect the input of the video information." ], "color": "#432", "bgcolor": "#653" } ], "links": [ [ 68, 55, 0, 56, 0, "VIDEO" ], [ 69, 56, 0, 58, 0, "STRING" ] ], "groups": [], "config": {}, "extra": { "ds": { "scale": 0.9646149645000006, "offset": [ 630.1225666298221, 338.759734215204 ] }, "groupNodes": {}, "ue_links": [], "links_added_by_ue": [], "frontendVersion": "1.25.5", "VHS_latentpreview": false, "VHS_latentpreviewrate": 0, "VHS_MetadataImage": true, "VHS_KeepIntermediate": true }, "version": 0.4 } ================================================ FILE: image_nodes.py ================================================ class MultipleImagesInput: @classmethod def INPUT_TYPES(s): return { "required": { "inputcount": ("INT", {"default": 2, "min": 2, "max": 1000, "step": 1}), "image_1": ("IMAGE",), "image_2": ("IMAGE",), }, } RETURN_TYPES = ("IMAGE",) RETURN_NAMES = ("images",) FUNCTION = "combine" CATEGORY = "Comfyui_MiniCPM-V-4_5" DESCRIPTION = """ Creates an image batch from multiple images. You can set how many inputs the node has, with the **inputcount** and clicking update. """ def combine(self, inputcount, **kwargs): from nodes import ImageBatch image_batch_node = ImageBatch() image = kwargs["image_1"] for c in range(1, inputcount): new_image = kwargs[f"image_{c + 1}"] (image,) = image_batch_node.batch(image, new_image) return (image,) ================================================ FILE: nodes_legacy.py ================================================ import os import torch import folder_paths from transformers import AutoTokenizer, AutoModel from torchvision.transforms.v2 import ToPILImage from comfy.comfy_types import IO from comfy_api.input import VideoInput class MiniCPM_VQA: def __init__(self): self.model_checkpoint = None self.tokenizer = None self.model = None self.device = ( torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu") ) self.bf16_support = ( torch.cuda.is_available() and torch.cuda.get_device_capability(self.device)[0] >= 8 ) @classmethod def INPUT_TYPES(s): return { "required": { "text": ("STRING", {"default": "", "multiline": True}), "model": ( ["MiniCPM-V-4_5-int4", "MiniCPM-V-4_5"], {"default": "MiniCPM-V-4_5-int4"}, ), "keep_model_loaded": ("BOOLEAN", {"default": False}), "top_p": ( "FLOAT", { "default": 0.8, }, ), "top_k": ( "INT", { "default": 100, }, ), "temperature": ( "FLOAT", {"default": 0.7, "min": 0, "max": 1, "step": 0.1}, ), "repetition_penalty": ( "FLOAT", { "default": 1.05, }, ), "max_new_tokens": ( "INT", { "default": 2048, }, ), "video_max_num_frames": ( "INT", { "default": 64, }, ), # if cuda OOM set a smaller number "video_max_slice_nums": ( "INT", { "default": 2, }, ), # use 1 if cuda OOM and video resolution > 448*448 "seed": ("INT", {"default": -1}), # add seed parameter, default is -1 }, "optional": { "source_video": (IO.VIDEO,), "source_image_1st": ("IMAGE",), "source_image_2nd": ("IMAGE",), "source_image_3rd": ("IMAGE",), }, } RETURN_TYPES = ("STRING",) FUNCTION = "inference" CATEGORY = "Comfyui_MiniCPM-V-4_5" def encode_video(self, source_video: VideoInput, MAX_NUM_FRAMES): def uniform_sample(l, n): # noqa: E741 gap = len(l) / n idxs = [int(i * gap + gap / 2) for i in range(n)] return [l[i] for i in idxs] components = source_video.get_components() vr = components.images avg_fps = float(components.frame_rate) print("Get average FPS(frame per second):", avg_fps) sample_fps = round(avg_fps / 1) # FPS duration = len(vr) / avg_fps print("Total duration:", duration, "seconds") width = vr[0].shape[1] height = vr[0].shape[0] print("Video resolution(width x height):", width, "x", height) frame_idx = [i for i in range(0, len(vr), sample_fps)] if len(frame_idx) > MAX_NUM_FRAMES: frame_idx = uniform_sample(frame_idx, MAX_NUM_FRAMES) frames = [vr[idx] for idx in frame_idx] frames = [ToPILImage()(v.permute([2, 0, 1])).convert("RGB") for v in frames] print("num frames:", len(frames)) return frames def inference( self, text, model, keep_model_loaded, top_p, top_k, temperature, repetition_penalty, max_new_tokens, video_max_num_frames, video_max_slice_nums, seed, source_image_1st=None, source_image_2nd=None, source_image_3rd=None, source_video: VideoInput = None, ): if seed != -1: torch.manual_seed(seed) model_id = f"openbmb/{model}" self.model_checkpoint = os.path.join( folder_paths.models_dir, "prompt_generator", os.path.basename(model_id) ) if not os.path.exists(self.model_checkpoint): from huggingface_hub import snapshot_download snapshot_download( repo_id=model_id, local_dir=self.model_checkpoint, local_dir_use_symlinks=False, ) if self.tokenizer is None: self.tokenizer = AutoTokenizer.from_pretrained( self.model_checkpoint, trust_remote_code=True, low_cpu_mem_usage=True, ) if self.model is None: self.model = AutoModel.from_pretrained( self.model_checkpoint, trust_remote_code=True, low_cpu_mem_usage=True, attn_implementation="sdpa", torch_dtype=torch.bfloat16 if self.bf16_support else torch.float16, ) with torch.no_grad(): if source_video: frames = self.encode_video(source_video, video_max_num_frames) msgs = [{"role": "user", "content": frames + [text]}] elif ( source_image_1st is not None and source_image_2nd is not None and source_image_3rd is not None ): image1 = ToPILImage()( source_image_1st.permute([0, 3, 1, 2])[0] ).convert("RGB") image2 = ToPILImage()( source_image_2nd.permute([0, 3, 1, 2])[0] ).convert("RGB") image3 = ToPILImage()( source_image_3rd.permute([0, 3, 1, 2])[0] ).convert("RGB") msgs = [{"role": "user", "content": [image1, image2, image3, text]}] elif ( source_image_1st is not None and source_image_2nd is not None and source_image_3rd is None ): image1 = ToPILImage()( source_image_1st.permute([0, 3, 1, 2])[0] ).convert("RGB") image2 = ToPILImage()( source_image_2nd.permute([0, 3, 1, 2])[0] ).convert("RGB") msgs = [{"role": "user", "content": [image1, image2, text]}] elif ( source_image_1st is not None and source_image_2nd is None and source_image_3rd is not None ): image1 = ToPILImage()( source_image_1st.permute([0, 3, 1, 2])[0] ).convert("RGB") image3 = ToPILImage()( source_image_3rd.permute([0, 3, 1, 2])[0] ).convert("RGB") msgs = [{"role": "user", "content": [image1, image3, text]}] elif ( source_image_1st is None and source_image_2nd is not None and source_image_3rd is not None ): image2 = ToPILImage()( source_image_2nd.permute([0, 3, 1, 2])[0] ).convert("RGB") image3 = ToPILImage()( source_image_3rd.permute([0, 3, 1, 2])[0] ).convert("RGB") msgs = [{"role": "user", "content": [image2, image3, text]}] elif ( source_image_1st is not None and source_image_2nd is None and source_image_3rd is None ): image = ToPILImage()(source_image_1st.permute([0, 3, 1, 2])[0]).convert( "RGB" ) msgs = [{"role": "user", "content": [image, text]}] elif ( source_image_1st is None and source_image_2nd is not None and source_image_3rd is None ): image = ToPILImage()(source_image_2nd.permute([0, 3, 1, 2])[0]).convert( "RGB" ) msgs = [{"role": "user", "content": [image, text]}] elif ( source_image_1st is None and source_image_2nd is None and source_image_3rd is not None ): image = ToPILImage()(source_image_3rd.permute([0, 3, 1, 2])[0]).convert( "RGB" ) msgs = [{"role": "user", "content": [image, text]}] else: msgs = [{"role": "user", "content": [text]}] # raise ValueError("Either image or video must be provided") params = {"use_image_id": False, "max_slice_nums": video_max_slice_nums} # offload model to CPU # self.model = self.model.to(torch.device("cpu")) # self.model.eval() result = self.model.chat( image=None, msgs=msgs, tokenizer=self.tokenizer, sampling=True, top_k=top_k, top_p=top_p, temperature=temperature, repetition_penalty=repetition_penalty, max_new_tokens=max_new_tokens, **params, ) # offload model to GPU # self.model = self.model.to(torch.device("cpu")) # self.model.eval() if not keep_model_loaded: del self.tokenizer # release tokenizer memory del self.model # release model memory self.tokenizer = None # set tokenizer to None self.model = None # set model to None torch.cuda.empty_cache() # release GPU memory torch.cuda.ipc_collect() return (result,) ================================================ FILE: nodes_polished.py ================================================ import os import torch import folder_paths from transformers import AutoTokenizer, AutoModel from torchvision.transforms.v2 import ToPILImage from comfy.comfy_types import IO from comfy_api.input import VideoInput class MiniCPM_VQA_Polished: def __init__(self): self.model_checkpoint = None self.tokenizer = None self.model = None self.device = ( torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu") ) self.bf16_support = ( torch.cuda.is_available() and torch.cuda.get_device_capability(self.device)[0] >= 8 ) @classmethod def INPUT_TYPES(s): return { "required": { "text": ("STRING", {"default": "", "multiline": True}), "model": ( ["MiniCPM-V-4_5-int4", "MiniCPM-V-4_5"], {"default": "MiniCPM-V-4_5-int4"}, ), "keep_model_loaded": ("BOOLEAN", {"default": False}), "top_p": ( "FLOAT", { "default": 0.8, }, ), "top_k": ( "INT", { "default": 100, }, ), "temperature": ( "FLOAT", {"default": 0.7, "min": 0, "max": 1, "step": 0.1}, ), "repetition_penalty": ( "FLOAT", { "default": 1.05, }, ), "max_new_tokens": ( "INT", { "default": 2048, }, ), "video_max_num_frames": ( "INT", { "default": 64, }, ), # if cuda OOM set a smaller number "video_max_slice_nums": ( "INT", { "default": 2, }, ), # use 1 if cuda OOM and video resolution > 448*448 "seed": ("INT", {"default": -1}), # add seed parameter, default is -1 }, "optional": { "source_video": (IO.VIDEO,), "source_image": ("IMAGE",), }, } RETURN_TYPES = ("STRING",) FUNCTION = "inference" CATEGORY = "Comfyui_MiniCPM-V-4_5" def encode_video(self, source_video: VideoInput, MAX_NUM_FRAMES): def uniform_sample(l, n): # noqa: E741 gap = len(l) / n idxs = [int(i * gap + gap / 2) for i in range(n)] return [l[i] for i in idxs] components = source_video.get_components() vr = components.images avg_fps = float(components.frame_rate) print("Get average FPS(frame per second):", avg_fps) sample_fps = round(avg_fps / 1) # FPS duration = len(vr) / avg_fps print("Total duration:", duration, "seconds") width = vr[0].shape[1] height = vr[0].shape[0] print("Video resolution(width x height):", width, "x", height) frame_idx = [i for i in range(0, len(vr), sample_fps)] if len(frame_idx) > MAX_NUM_FRAMES: frame_idx = uniform_sample(frame_idx, MAX_NUM_FRAMES) frames = [vr[idx] for idx in frame_idx] frames = [ToPILImage()(v.permute([2, 0, 1])).convert("RGB") for v in frames] print("num frames:", len(frames)) return frames def inference( self, text, model, keep_model_loaded, top_p, top_k, temperature, repetition_penalty, max_new_tokens, video_max_num_frames, video_max_slice_nums, seed, source_image=None, source_video: VideoInput = None, ): if seed != -1: torch.manual_seed(seed) model_id = f"openbmb/{model}" self.model_checkpoint = os.path.join( folder_paths.models_dir, "prompt_generator", os.path.basename(model_id) ) if not os.path.exists(self.model_checkpoint): from huggingface_hub import snapshot_download snapshot_download( repo_id=model_id, local_dir=self.model_checkpoint, local_dir_use_symlinks=False, ) if self.tokenizer is None: self.tokenizer = AutoTokenizer.from_pretrained( self.model_checkpoint, trust_remote_code=True, low_cpu_mem_usage=True, ) if self.model is None: self.model = AutoModel.from_pretrained( self.model_checkpoint, trust_remote_code=True, low_cpu_mem_usage=True, attn_implementation="sdpa", torch_dtype=torch.bfloat16 if self.bf16_support else torch.float16, ) with torch.no_grad(): if source_video: print("source_video:", source_video) frames = self.encode_video(source_video, video_max_num_frames) msgs = [{"role": "user", "content": frames + [text]}] elif source_image is not None: images = source_image.permute([0, 3, 1, 2]) images = [ToPILImage()(img).convert("RGB") for img in images] msgs = [{"role": "user", "content": images + [text]}] else: msgs = [{"role": "user", "content": [text]}] # raise ValueError("Either image or video must be provided") params = {"use_image_id": False, "max_slice_nums": video_max_slice_nums} # offload model to CPU # self.model = self.model.to(torch.device("cpu")) # self.model.eval() result = self.model.chat( image=None, msgs=msgs, tokenizer=self.tokenizer, sampling=True, top_k=top_k, top_p=top_p, temperature=temperature, repetition_penalty=repetition_penalty, max_new_tokens=max_new_tokens, **params, ) # offload model to GPU # self.model = self.model.to(torch.device("cpu")) # self.model.eval() if not keep_model_loaded: del self.tokenizer # release tokenizer memory del self.model # release model memory self.tokenizer = None # set tokenizer to None self.model = None # set model to None torch.cuda.empty_cache() # release GPU memory torch.cuda.ipc_collect() return (result,) ================================================ FILE: pyproject.toml ================================================ [project] name = "ComfyUI_MiniCPM-V-4_5" description = "This is an implementation of [MiniCPM-V-4_5](https://github.com/OpenBMB/MiniCPM-V) by [ComfyUI](https://github.com/comfyanonymous/ComfyUI), including support for text-based queries, video queries, single-image queries, and multi-image queries to generate captions or responses." version = "1.0.0" license = { file = "LICENSE" } dependencies = ["torch", "torchvision", "numpy", "pillow", "huggingface_hub", "transformers", "decord", "bitsandbytes","accelerate"] [project.urls] Repository = "https://github.com/IuvenisSapiens/ComfyUI_MiniCPM-V-4_5" [tool.comfy] PublisherId = "IuvenisSapiens" DisplayName = "ComfyUI_MiniCPM-V-4_5" Icon = "favicon.ico" ================================================ FILE: requirements.txt ================================================ torch torchvision torchaudio numpy pillow huggingface_hub transformers bitsandbytes accelerate ================================================ FILE: web/js/displayText.js ================================================ const app = window.comfyAPI.app.app; const ComfyWidgets = window.comfyAPI.widgets.ComfyWidgets; app.registerExtension({ name: "Comfyui_MiniCPM-V-4_5.DisplayTextNode", async beforeRegisterNodeDef(nodeType, nodeData, app) { if (nodeData.name === "DisplayText") { function populate(text) { if (this.widgets) { this.widgets.forEach((widget) => widget.onRemove?.()); this.widgets = []; } const v = [...text]; if (!v[0]) { v.shift(); } for (const list of v) { const w = ComfyWidgets["STRING"]( this, "text", ["STRING", { multiline: true }], app ).widget; // w.inputEl.readOnly = true; // w.inputEl.style.opacity = 0.6; w.value = list; } requestAnimationFrame(() => { const sz = this.computeSize(); if (sz[0] < this.size[0]) { sz[0] = this.size[0]; } if (sz[1] < this.size[1]) { sz[1] = this.size[1]; } this.onResize?.(sz); app.graph.setDirtyCanvas(true, false); }); } const onExecuted = nodeType.prototype.onExecuted; nodeType.prototype.onExecuted = function (message) { onExecuted?.apply(this, arguments); populate.call(this, message.text); }; const onConfigure = nodeType.prototype.onConfigure; nodeType.prototype.onConfigure = function () { onConfigure?.apply(this, arguments); if (this.widgets_values?.length) { populate.call( this, this.widgets_values.slice(+this.widgets_values.length > 1) ); } }; } }, }); ================================================ FILE: web/js/multipleImagesInput.js ================================================ const app = window.comfyAPI.app.app; app.registerExtension({ name: "Comfyui_MiniCPM-V-4_5.MultipleImagesInput", async beforeRegisterNodeDef(nodeType, nodeData, app) { if (!nodeData?.category?.startsWith("Comfyui_MiniCPM-V-int4")) { return; } switch (nodeData.name) { case "MultipleImagesInput": nodeType.prototype.onNodeCreated = function () { this._type = "IMAGE"; this.inputs_offset = nodeData.name.includes("selective") ? 1 : 0; this.addWidget("button", "Update inputs", null, () => { if (!this.inputs) { this.inputs = []; } const target_number_of_inputs = this.widgets.find( (w) => w.name === "inputcount" )["value"]; if (target_number_of_inputs === this.inputs.length) return; // already set, do nothing if (target_number_of_inputs < this.inputs.length) { for ( let i = this.inputs.length; i >= this.inputs_offset + target_number_of_inputs; i-- ) this.removeInput(i); } else { for ( let i = this.inputs.length + 1 - this.inputs_offset; i <= target_number_of_inputs; ++i ) this.addInput(`image_${i}`, this._type); } }); }; break; } }, async setup() { const originalComputeVisibleNodes = LGraphCanvas.prototype.computeVisibleNodes; LGraphCanvas.prototype.computeVisibleNodes = function () { const visibleNodesSet = new Set( originalComputeVisibleNodes.apply(this, arguments) ); for (const node of this.graph._nodes) { if ( (node.type === "SetNode" || node.type === "GetNode") && node.drawConnection ) { visibleNodesSet.add(node); } } return Array.from(visibleNodesSet); }; }, });