[
  {
    "path": "Diffutoon_color_jupyter.ipynb",
    "content": "{\n  \"cells\": [\n    {\n      \"cell_type\": \"markdown\",\n      \"metadata\": {\n        \"id\": \"view-in-github\"\n      },\n      \"source\": [\n        \"[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/camenduru/Diffutoon-jupyter/blob/main/Diffutoon_color_jupyter.ipynb)\"\n      ]\n    },\n    {\n      \"cell_type\": \"code\",\n      \"execution_count\": null,\n      \"metadata\": {\n        \"id\": \"VjYy0F2gZIPR\"\n      },\n      \"outputs\": [],\n      \"source\": [\n        \"# https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/Diffutoon/Diffutoon.ipynb modified\\n\",\n        \"\\n\",\n        \"%cd /content\\n\",\n        \"!git clone https://github.com/Artiprocher/DiffSynth-Studio\\n\",\n        \"%cd /content/DiffSynth-Studio\\n\",\n        \"\\n\",\n        \"!pip install -q einops transformers controlnet-aux==0.0.7 sentencepiece imageio imageio-ffmpeg\\n\",\n        \"\\n\",\n        \"!apt -y install -qq aria2\\n\",\n        \"!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://civitai.com/api/download/models/229575 -d /content/DiffSynth-Studio/models/stable_diffusion -o aingdiffusion_v12.safetensors\\n\",\n        \"!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/guoyww/animatediff/resolve/main/mm_sd_v15_v2.ckpt -d /content/DiffSynth-Studio/models/AnimateDiff -o mm_sd_v15_v2.ckpt\\n\",\n        \"!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/lllyasviel/ControlNet-v1-1/resolve/main/control_v11p_sd15_lineart.pth -d /content/DiffSynth-Studio/models/ControlNet -o control_v11p_sd15_lineart.pth\\n\",\n        \"!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/lllyasviel/ControlNet-v1-1/resolve/main/control_v11f1e_sd15_tile.pth -d /content/DiffSynth-Studio/models/ControlNet -o control_v11f1e_sd15_tile.pth\\n\",\n        \"!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/lllyasviel/ControlNet-v1-1/resolve/main/control_v11f1p_sd15_depth.pth -d /content/DiffSynth-Studio/models/ControlNet -o control_v11f1p_sd15_depth.pth\\n\",\n        \"!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/lllyasviel/ControlNet-v1-1/resolve/main/control_v11p_sd15_softedge.pth -d /content/DiffSynth-Studio/models/ControlNet -o control_v11p_sd15_softedge.pth\\n\",\n        \"!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/lllyasviel/Annotators/resolve/main/dpt_hybrid-midas-501f0c75.pt -d /content/DiffSynth-Studio/models/Annotators -o dpt_hybrid-midas-501f0c75.pt\\n\",\n        \"!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/lllyasviel/Annotators/resolve/main/ControlNetHED.pth -d /content/DiffSynth-Studio/models/Annotators -o ControlNetHED.pth\\n\",\n        \"!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/lllyasviel/Annotators/resolve/main/sk_model.pth -d /content/DiffSynth-Studio/models/Annotators -o sk_model.pth\\n\",\n        \"!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/lllyasviel/Annotators/resolve/main/sk_model2.pth -d /content/DiffSynth-Studio/models/Annotators -o sk_model2.pth\\n\",\n        \"!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M \\\"https://civitai.com/api/download/models/25820?type=Model&format=PickleTensor&size=full&fp=fp16\\\" -d /content/DiffSynth-Studio/models/textual_inversion -o verybadimagenegative_v1.3.pt\\n\",\n        \"!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/camenduru/Diffutoon/resolve/main/input_video.mp4 -d /content -o input_video.mp4\"\n      ]\n    },\n    {\n      \"cell_type\": \"code\",\n      \"execution_count\": null,\n      \"metadata\": {},\n      \"outputs\": [],\n      \"source\": [\n        \"config_stage_1_template = {\\n\",\n        \"    \\\"models\\\": {\\n\",\n        \"        \\\"model_list\\\": [\\n\",\n        \"            \\\"models/stable_diffusion/aingdiffusion_v12.safetensors\\\",\\n\",\n        \"            \\\"models/ControlNet/control_v11p_sd15_softedge.pth\\\",\\n\",\n        \"            \\\"models/ControlNet/control_v11f1p_sd15_depth.pth\\\"\\n\",\n        \"        ],\\n\",\n        \"        \\\"textual_inversion_folder\\\": \\\"models/textual_inversion\\\",\\n\",\n        \"        \\\"device\\\": \\\"cuda\\\",\\n\",\n        \"        \\\"lora_alphas\\\": [],\\n\",\n        \"        \\\"controlnet_units\\\": [\\n\",\n        \"            {\\n\",\n        \"                \\\"processor_id\\\": \\\"softedge\\\",\\n\",\n        \"                \\\"model_path\\\": \\\"models/ControlNet/control_v11p_sd15_softedge.pth\\\",\\n\",\n        \"                \\\"scale\\\": 0.5\\n\",\n        \"            },\\n\",\n        \"            {\\n\",\n        \"                \\\"processor_id\\\": \\\"depth\\\",\\n\",\n        \"                \\\"model_path\\\": \\\"models/ControlNet/control_v11f1p_sd15_depth.pth\\\",\\n\",\n        \"                \\\"scale\\\": 0.5\\n\",\n        \"            }\\n\",\n        \"        ]\\n\",\n        \"    },\\n\",\n        \"    \\\"data\\\": {\\n\",\n        \"        \\\"input_frames\\\": {\\n\",\n        \"            \\\"video_file\\\": \\\"/content/input_video.mp4\\\",\\n\",\n        \"            \\\"image_folder\\\": None,\\n\",\n        \"            \\\"height\\\": 512,\\n\",\n        \"            \\\"width\\\": 512,\\n\",\n        \"            \\\"start_frame_id\\\": 0,\\n\",\n        \"            \\\"end_frame_id\\\": 30\\n\",\n        \"        },\\n\",\n        \"        \\\"controlnet_frames\\\": [\\n\",\n        \"            {\\n\",\n        \"                \\\"video_file\\\": \\\"/content/input_video.mp4\\\",\\n\",\n        \"                \\\"image_folder\\\": None,\\n\",\n        \"                \\\"height\\\": 512,\\n\",\n        \"                \\\"width\\\": 512,\\n\",\n        \"                \\\"start_frame_id\\\": 0,\\n\",\n        \"                \\\"end_frame_id\\\": 30\\n\",\n        \"            },\\n\",\n        \"            {\\n\",\n        \"                \\\"video_file\\\": \\\"/content/input_video.mp4\\\",\\n\",\n        \"                \\\"image_folder\\\": None,\\n\",\n        \"                \\\"height\\\": 512,\\n\",\n        \"                \\\"width\\\": 512,\\n\",\n        \"                \\\"start_frame_id\\\": 0,\\n\",\n        \"                \\\"end_frame_id\\\": 30\\n\",\n        \"            }\\n\",\n        \"        ],\\n\",\n        \"        \\\"output_folder\\\": \\\"data/examples/diffutoon_edit/color_video\\\",\\n\",\n        \"        \\\"fps\\\": 25\\n\",\n        \"    },\\n\",\n        \"    \\\"smoother_configs\\\": [\\n\",\n        \"        {\\n\",\n        \"            \\\"processor_type\\\": \\\"FastBlend\\\",\\n\",\n        \"            \\\"config\\\": {}\\n\",\n        \"        }\\n\",\n        \"    ],\\n\",\n        \"    \\\"pipeline\\\": {\\n\",\n        \"        \\\"seed\\\": 0,\\n\",\n        \"        \\\"pipeline_inputs\\\": {\\n\",\n        \"            \\\"prompt\\\": \\\"best quality, perfect anime illustration, orange clothes, night, a girl is dancing, smile, solo, black silk stockings\\\",\\n\",\n        \"            \\\"negative_prompt\\\": \\\"verybadimagenegative_v1.3\\\",\\n\",\n        \"            \\\"cfg_scale\\\": 7.0,\\n\",\n        \"            \\\"clip_skip\\\": 1,\\n\",\n        \"            \\\"denoising_strength\\\": 0.9,\\n\",\n        \"            \\\"num_inference_steps\\\": 20,\\n\",\n        \"            \\\"animatediff_batch_size\\\": 8,\\n\",\n        \"            \\\"animatediff_stride\\\": 4,\\n\",\n        \"            \\\"unet_batch_size\\\": 8,\\n\",\n        \"            \\\"controlnet_batch_size\\\": 8,\\n\",\n        \"            \\\"cross_frame_attention\\\": True,\\n\",\n        \"            \\\"smoother_progress_ids\\\": [-1],\\n\",\n        \"            # The following parameters will be overwritten. You don't need to modify them.\\n\",\n        \"            \\\"input_frames\\\": [],\\n\",\n        \"            \\\"num_frames\\\": 30,\\n\",\n        \"            \\\"width\\\": 512,\\n\",\n        \"            \\\"height\\\": 512,\\n\",\n        \"            \\\"controlnet_frames\\\": []\\n\",\n        \"        }\\n\",\n        \"    }\\n\",\n        \"}\\n\",\n        \"\\n\",\n        \"from diffsynth import SDVideoPipelineRunner\\n\",\n        \"\\n\",\n        \"config_stage_1 = config_stage_1_template.copy()\\n\",\n        \"config_stage_1[\\\"data\\\"][\\\"input_frames\\\"] = {\\n\",\n        \"    \\\"video_file\\\": \\\"/content/input_video.mp4\\\",\\n\",\n        \"    \\\"image_folder\\\": None,\\n\",\n        \"    \\\"height\\\": 512,\\n\",\n        \"    \\\"width\\\": 512,\\n\",\n        \"    \\\"start_frame_id\\\": 0,\\n\",\n        \"    \\\"end_frame_id\\\": 30\\n\",\n        \"}\\n\",\n        \"config_stage_1[\\\"data\\\"][\\\"controlnet_frames\\\"] = [config_stage_1[\\\"data\\\"][\\\"input_frames\\\"], config_stage_1[\\\"data\\\"][\\\"input_frames\\\"]]\\n\",\n        \"config_stage_1[\\\"data\\\"][\\\"output_folder\\\"] = \\\"/content/color_video\\\"\\n\",\n        \"config_stage_1[\\\"data\\\"][\\\"fps\\\"] = 25\\n\",\n        \"config_stage_1[\\\"pipeline\\\"][\\\"pipeline_inputs\\\"][\\\"prompt\\\"] = \\\"best quality, perfect anime illustration, orange clothes, night, a girl is dancing, smile, solo, black silk stockings\\\"\\n\",\n        \"\\n\",\n        \"runner = SDVideoPipelineRunner()\\n\",\n        \"runner.run(config_stage_1)\"\n      ]\n    },\n    {\n      \"cell_type\": \"code\",\n      \"execution_count\": null,\n      \"metadata\": {},\n      \"outputs\": [],\n      \"source\": [\n        \"# If T4, at this point, restart and run the next cell.\"\n      ]\n    },\n    {\n      \"cell_type\": \"code\",\n      \"execution_count\": null,\n      \"metadata\": {},\n      \"outputs\": [],\n      \"source\": [\n        \"config_stage_2_template = {\\n\",\n        \"    \\\"models\\\": {\\n\",\n        \"        \\\"model_list\\\": [\\n\",\n        \"            \\\"models/stable_diffusion/aingdiffusion_v12.safetensors\\\",\\n\",\n        \"            \\\"models/AnimateDiff/mm_sd_v15_v2.ckpt\\\",\\n\",\n        \"            \\\"models/ControlNet/control_v11f1e_sd15_tile.pth\\\",\\n\",\n        \"            \\\"models/ControlNet/control_v11p_sd15_lineart.pth\\\"\\n\",\n        \"        ],\\n\",\n        \"        \\\"textual_inversion_folder\\\": \\\"models/textual_inversion\\\",\\n\",\n        \"        \\\"device\\\": \\\"cuda\\\",\\n\",\n        \"        \\\"lora_alphas\\\": [],\\n\",\n        \"        \\\"controlnet_units\\\": [\\n\",\n        \"            {\\n\",\n        \"                \\\"processor_id\\\": \\\"tile\\\",\\n\",\n        \"                \\\"model_path\\\": \\\"models/ControlNet/control_v11f1e_sd15_tile.pth\\\",\\n\",\n        \"                \\\"scale\\\": 0.5\\n\",\n        \"            },\\n\",\n        \"            {\\n\",\n        \"                \\\"processor_id\\\": \\\"lineart\\\",\\n\",\n        \"                \\\"model_path\\\": \\\"models/ControlNet/control_v11p_sd15_lineart.pth\\\",\\n\",\n        \"                \\\"scale\\\": 0.5\\n\",\n        \"            }\\n\",\n        \"        ]\\n\",\n        \"    },\\n\",\n        \"    \\\"data\\\": {\\n\",\n        \"        \\\"input_frames\\\": {\\n\",\n        \"            \\\"video_file\\\": \\\"/content/input_video.mp4\\\",\\n\",\n        \"            \\\"image_folder\\\": None,\\n\",\n        \"            \\\"height\\\": 1024,\\n\",\n        \"            \\\"width\\\": 1024,\\n\",\n        \"            \\\"start_frame_id\\\": 0,\\n\",\n        \"            \\\"end_frame_id\\\": 30\\n\",\n        \"        },\\n\",\n        \"        \\\"controlnet_frames\\\": [\\n\",\n        \"            {\\n\",\n        \"                \\\"video_file\\\": \\\"/content/input_video.mp4\\\",\\n\",\n        \"                \\\"image_folder\\\": None,\\n\",\n        \"                \\\"height\\\": 1024,\\n\",\n        \"                \\\"width\\\": 1024,\\n\",\n        \"                \\\"start_frame_id\\\": 0,\\n\",\n        \"                \\\"end_frame_id\\\": 30\\n\",\n        \"            },\\n\",\n        \"            {\\n\",\n        \"                \\\"video_file\\\": \\\"/content/input_video.mp4\\\",\\n\",\n        \"                \\\"image_folder\\\": None,\\n\",\n        \"                \\\"height\\\": 1024,\\n\",\n        \"                \\\"width\\\": 1024,\\n\",\n        \"                \\\"start_frame_id\\\": 0,\\n\",\n        \"                \\\"end_frame_id\\\": 30\\n\",\n        \"            }\\n\",\n        \"        ],\\n\",\n        \"        \\\"output_folder\\\": \\\"/content/output\\\",\\n\",\n        \"        \\\"fps\\\": 25\\n\",\n        \"    },\\n\",\n        \"    \\\"pipeline\\\": {\\n\",\n        \"        \\\"seed\\\": 0,\\n\",\n        \"        \\\"pipeline_inputs\\\": {\\n\",\n        \"            \\\"prompt\\\": \\\"best quality, perfect anime illustration, light, a girl is dancing, smile, solo\\\",\\n\",\n        \"            \\\"negative_prompt\\\": \\\"verybadimagenegative_v1.3\\\",\\n\",\n        \"            \\\"cfg_scale\\\": 7.0,\\n\",\n        \"            \\\"clip_skip\\\": 2,\\n\",\n        \"            \\\"denoising_strength\\\": 1.0,\\n\",\n        \"            \\\"num_inference_steps\\\": 10,\\n\",\n        \"            \\\"animatediff_batch_size\\\": 16,\\n\",\n        \"            \\\"animatediff_stride\\\": 8,\\n\",\n        \"            \\\"unet_batch_size\\\": 1,\\n\",\n        \"            \\\"controlnet_batch_size\\\": 1,\\n\",\n        \"            \\\"cross_frame_attention\\\": False,\\n\",\n        \"            # The following parameters will be overwritten. You don't need to modify them.\\n\",\n        \"            \\\"input_frames\\\": [],\\n\",\n        \"            \\\"num_frames\\\": 30,\\n\",\n        \"            \\\"width\\\": 1536,\\n\",\n        \"            \\\"height\\\": 1536,\\n\",\n        \"            \\\"controlnet_frames\\\": []\\n\",\n        \"        }\\n\",\n        \"    }\\n\",\n        \"}\\n\",\n        \"\\n\",\n        \"from diffsynth import SDVideoPipelineRunner\\n\",\n        \"\\n\",\n        \"config_stage_2 = config_stage_2_template.copy()\\n\",\n        \"config_stage_2[\\\"data\\\"][\\\"input_frames\\\"] = {\\n\",\n        \"    \\\"video_file\\\": \\\"/content/input_video.mp4\\\",\\n\",\n        \"    \\\"image_folder\\\": None,\\n\",\n        \"    \\\"height\\\": 1024,\\n\",\n        \"    \\\"width\\\": 1024,\\n\",\n        \"    \\\"start_frame_id\\\": 0,\\n\",\n        \"    \\\"end_frame_id\\\": 30\\n\",\n        \"}\\n\",\n        \"config_stage_2[\\\"data\\\"][\\\"controlnet_frames\\\"][0] = {\\n\",\n        \"    \\\"video_file\\\": \\\"/content/color_video/video.mp4\\\",\\n\",\n        \"    \\\"image_folder\\\": None,\\n\",\n        \"    \\\"height\\\": config_stage_2[\\\"data\\\"][\\\"input_frames\\\"][\\\"height\\\"],\\n\",\n        \"    \\\"width\\\": config_stage_2[\\\"data\\\"][\\\"input_frames\\\"][\\\"width\\\"],\\n\",\n        \"    \\\"start_frame_id\\\": None,\\n\",\n        \"    \\\"end_frame_id\\\": None\\n\",\n        \"}\\n\",\n        \"config_stage_2[\\\"data\\\"][\\\"controlnet_frames\\\"][1] = config_stage_2[\\\"data\\\"][\\\"input_frames\\\"]\\n\",\n        \"config_stage_2[\\\"data\\\"][\\\"output_folder\\\"] = \\\"/content/edit_video\\\"\\n\",\n        \"config_stage_2[\\\"data\\\"][\\\"fps\\\"] = 25\\n\",\n        \"\\n\",\n        \"runner = SDVideoPipelineRunner()\\n\",\n        \"runner.run(config_stage_2)\"\n      ]\n    },\n    {\n      \"cell_type\": \"code\",\n      \"execution_count\": null,\n      \"metadata\": {},\n      \"outputs\": [],\n      \"source\": [\n        \"import moviepy.editor\\n\",\n        \"moviepy.editor.ipython_display(\\\"/content/edit_video/video.mp4\\\")\"\n      ]\n    }\n  ],\n  \"metadata\": {\n    \"accelerator\": \"GPU\",\n    \"colab\": {\n      \"gpuType\": \"T4\",\n      \"provenance\": []\n    },\n    \"kernelspec\": {\n      \"display_name\": \"Python 3\",\n      \"name\": \"python3\"\n    },\n    \"language_info\": {\n      \"name\": \"python\"\n    }\n  },\n  \"nbformat\": 4,\n  \"nbformat_minor\": 0\n}\n"
  },
  {
    "path": "Diffutoon_jupyter.ipynb",
    "content": "{\n  \"cells\": [\n    {\n      \"cell_type\": \"markdown\",\n      \"metadata\": {\n        \"id\": \"view-in-github\"\n      },\n      \"source\": [\n        \"[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/camenduru/Diffutoon-jupyter/blob/main/Diffutoon_jupyter.ipynb)\"\n      ]\n    },\n    {\n      \"cell_type\": \"code\",\n      \"execution_count\": null,\n      \"metadata\": {\n        \"id\": \"VjYy0F2gZIPR\"\n      },\n      \"outputs\": [],\n      \"source\": [\n        \"# https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/Diffutoon/Diffutoon.ipynb modified\\n\",\n        \"\\n\",\n        \"%cd /content\\n\",\n        \"!git clone https://github.com/Artiprocher/DiffSynth-Studio\\n\",\n        \"%cd /content/DiffSynth-Studio\\n\",\n        \"\\n\",\n        \"!pip install -q einops transformers controlnet-aux==0.0.7 sentencepiece imageio imageio-ffmpeg\\n\",\n        \"\\n\",\n        \"!apt -y install -qq aria2\\n\",\n        \"!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://civitai.com/api/download/models/229575 -d /content/DiffSynth-Studio/models/stable_diffusion -o aingdiffusion_v12.safetensors\\n\",\n        \"!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/guoyww/animatediff/resolve/main/mm_sd_v15_v2.ckpt -d /content/DiffSynth-Studio/models/AnimateDiff -o mm_sd_v15_v2.ckpt\\n\",\n        \"!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/lllyasviel/ControlNet-v1-1/resolve/main/control_v11p_sd15_lineart.pth -d /content/DiffSynth-Studio/models/ControlNet -o control_v11p_sd15_lineart.pth\\n\",\n        \"!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/lllyasviel/ControlNet-v1-1/resolve/main/control_v11f1e_sd15_tile.pth -d /content/DiffSynth-Studio/models/ControlNet -o control_v11f1e_sd15_tile.pth\\n\",\n        \"!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/lllyasviel/ControlNet-v1-1/resolve/main/control_v11f1p_sd15_depth.pth -d /content/DiffSynth-Studio/models/ControlNet -o control_v11f1p_sd15_depth.pth\\n\",\n        \"!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/lllyasviel/ControlNet-v1-1/resolve/main/control_v11p_sd15_softedge.pth -d /content/DiffSynth-Studio/models/ControlNet -o control_v11p_sd15_softedge.pth\\n\",\n        \"!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/lllyasviel/Annotators/resolve/main/dpt_hybrid-midas-501f0c75.pt -d /content/DiffSynth-Studio/models/Annotators -o dpt_hybrid-midas-501f0c75.pt\\n\",\n        \"!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/lllyasviel/Annotators/resolve/main/ControlNetHED.pth -d /content/DiffSynth-Studio/models/Annotators -o ControlNetHED.pth\\n\",\n        \"!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/lllyasviel/Annotators/resolve/main/sk_model.pth -d /content/DiffSynth-Studio/models/Annotators -o sk_model.pth\\n\",\n        \"!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/lllyasviel/Annotators/resolve/main/sk_model2.pth -d /content/DiffSynth-Studio/models/Annotators -o sk_model2.pth\\n\",\n        \"!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M \\\"https://civitai.com/api/download/models/25820?type=Model&format=PickleTensor&size=full&fp=fp16\\\" -d /content/DiffSynth-Studio/models/textual_inversion -o verybadimagenegative_v1.3.pt\\n\",\n        \"!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/camenduru/Diffutoon/resolve/main/input_video.mp4 -d /content -o input_video.mp4\"\n      ]\n    },\n    {\n      \"cell_type\": \"code\",\n      \"execution_count\": null,\n      \"metadata\": {},\n      \"outputs\": [],\n      \"source\": [\n        \"config_stage_2_template = {\\n\",\n        \"    \\\"models\\\": {\\n\",\n        \"        \\\"model_list\\\": [\\n\",\n        \"            \\\"models/stable_diffusion/aingdiffusion_v12.safetensors\\\",\\n\",\n        \"            \\\"models/AnimateDiff/mm_sd_v15_v2.ckpt\\\",\\n\",\n        \"            \\\"models/ControlNet/control_v11f1e_sd15_tile.pth\\\",\\n\",\n        \"            \\\"models/ControlNet/control_v11p_sd15_lineart.pth\\\"\\n\",\n        \"        ],\\n\",\n        \"        \\\"textual_inversion_folder\\\": \\\"models/textual_inversion\\\",\\n\",\n        \"        \\\"device\\\": \\\"cuda\\\",\\n\",\n        \"        \\\"lora_alphas\\\": [],\\n\",\n        \"        \\\"controlnet_units\\\": [\\n\",\n        \"            {\\n\",\n        \"                \\\"processor_id\\\": \\\"tile\\\",\\n\",\n        \"                \\\"model_path\\\": \\\"models/ControlNet/control_v11f1e_sd15_tile.pth\\\",\\n\",\n        \"                \\\"scale\\\": 0.5\\n\",\n        \"            },\\n\",\n        \"            {\\n\",\n        \"                \\\"processor_id\\\": \\\"lineart\\\",\\n\",\n        \"                \\\"model_path\\\": \\\"models/ControlNet/control_v11p_sd15_lineart.pth\\\",\\n\",\n        \"                \\\"scale\\\": 0.5\\n\",\n        \"            }\\n\",\n        \"        ]\\n\",\n        \"    },\\n\",\n        \"    \\\"data\\\": {\\n\",\n        \"        \\\"input_frames\\\": {\\n\",\n        \"            \\\"video_file\\\": \\\"/content/input_video.mp4\\\",\\n\",\n        \"            \\\"image_folder\\\": None,\\n\",\n        \"            \\\"height\\\": 1024,\\n\",\n        \"            \\\"width\\\": 1024,\\n\",\n        \"            \\\"start_frame_id\\\": 0,\\n\",\n        \"            \\\"end_frame_id\\\": 30\\n\",\n        \"        },\\n\",\n        \"        \\\"controlnet_frames\\\": [\\n\",\n        \"            {\\n\",\n        \"                \\\"video_file\\\": \\\"/content/input_video.mp4\\\",\\n\",\n        \"                \\\"image_folder\\\": None,\\n\",\n        \"                \\\"height\\\": 1024,\\n\",\n        \"                \\\"width\\\": 1024,\\n\",\n        \"                \\\"start_frame_id\\\": 0,\\n\",\n        \"                \\\"end_frame_id\\\": 30\\n\",\n        \"            },\\n\",\n        \"            {\\n\",\n        \"                \\\"video_file\\\": \\\"/content/input_video.mp4\\\",\\n\",\n        \"                \\\"image_folder\\\": None,\\n\",\n        \"                \\\"height\\\": 1024,\\n\",\n        \"                \\\"width\\\": 1024,\\n\",\n        \"                \\\"start_frame_id\\\": 0,\\n\",\n        \"                \\\"end_frame_id\\\": 30\\n\",\n        \"            }\\n\",\n        \"        ],\\n\",\n        \"        \\\"output_folder\\\": \\\"/content/output\\\",\\n\",\n        \"        \\\"fps\\\": 25\\n\",\n        \"    },\\n\",\n        \"    \\\"pipeline\\\": {\\n\",\n        \"        \\\"seed\\\": 0,\\n\",\n        \"        \\\"pipeline_inputs\\\": {\\n\",\n        \"            \\\"prompt\\\": \\\"best quality, perfect anime illustration, light, a girl is dancing, smile, solo\\\",\\n\",\n        \"            \\\"negative_prompt\\\": \\\"verybadimagenegative_v1.3\\\",\\n\",\n        \"            \\\"cfg_scale\\\": 7.0,\\n\",\n        \"            \\\"clip_skip\\\": 2,\\n\",\n        \"            \\\"denoising_strength\\\": 1.0,\\n\",\n        \"            \\\"num_inference_steps\\\": 10,\\n\",\n        \"            \\\"animatediff_batch_size\\\": 16,\\n\",\n        \"            \\\"animatediff_stride\\\": 8,\\n\",\n        \"            \\\"unet_batch_size\\\": 1,\\n\",\n        \"            \\\"controlnet_batch_size\\\": 1,\\n\",\n        \"            \\\"cross_frame_attention\\\": False,\\n\",\n        \"            # The following parameters will be overwritten. You don't need to modify them.\\n\",\n        \"            \\\"input_frames\\\": [],\\n\",\n        \"            \\\"num_frames\\\": 30,\\n\",\n        \"            \\\"width\\\": 1536,\\n\",\n        \"            \\\"height\\\": 1536,\\n\",\n        \"            \\\"controlnet_frames\\\": []\\n\",\n        \"        }\\n\",\n        \"    }\\n\",\n        \"}\"\n      ]\n    },\n    {\n      \"cell_type\": \"code\",\n      \"execution_count\": null,\n      \"metadata\": {},\n      \"outputs\": [],\n      \"source\": [\n        \"from diffsynth import SDVideoPipelineRunner\\n\",\n        \"\\n\",\n        \"config = config_stage_2_template.copy()\\n\",\n        \"config[\\\"data\\\"][\\\"input_frames\\\"] = {\\n\",\n        \"    \\\"video_file\\\": \\\"/content/input_video.mp4\\\",\\n\",\n        \"    \\\"image_folder\\\": None,\\n\",\n        \"    \\\"height\\\": 1024,\\n\",\n        \"    \\\"width\\\": 1024,\\n\",\n        \"    \\\"start_frame_id\\\": 0,\\n\",\n        \"    \\\"end_frame_id\\\": 30\\n\",\n        \"}\\n\",\n        \"config[\\\"data\\\"][\\\"controlnet_frames\\\"] = [config[\\\"data\\\"][\\\"input_frames\\\"], config[\\\"data\\\"][\\\"input_frames\\\"]]\\n\",\n        \"config[\\\"data\\\"][\\\"output_folder\\\"] = \\\"/content/toon_video\\\"\\n\",\n        \"config[\\\"data\\\"][\\\"fps\\\"] = 25\\n\",\n        \"\\n\",\n        \"runner = SDVideoPipelineRunner()\\n\",\n        \"runner.run(config)\"\n      ]\n    },\n    {\n      \"cell_type\": \"code\",\n      \"execution_count\": null,\n      \"metadata\": {},\n      \"outputs\": [],\n      \"source\": [\n        \"import moviepy.editor\\n\",\n        \"moviepy.editor.ipython_display(\\\"/content/toon_video/video.mp4\\\")\"\n      ]\n    }\n  ],\n  \"metadata\": {\n    \"accelerator\": \"GPU\",\n    \"colab\": {\n      \"gpuType\": \"T4\",\n      \"provenance\": []\n    },\n    \"kernelspec\": {\n      \"display_name\": \"Python 3\",\n      \"name\": \"python3\"\n    },\n    \"language_info\": {\n      \"name\": \"python\"\n    }\n  },\n  \"nbformat\": 4,\n  \"nbformat_minor\": 0\n}\n"
  },
  {
    "path": "README.md",
    "content": "🐣 Please follow me for new updates https://twitter.com/camenduru <br />\n🔥 Please join our discord server https://discord.gg/k5BwmmvJJU <br />\n🥳 Please join my patreon community https://patreon.com/camenduru <br />\n\n### 🍊 Jupyter Notebook\n\n| Notebook | Info\n| --- | --- |\n[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/camenduru/Diffutoon-jupyter/blob/main/Diffutoon_jupyter.ipynb) | Diffutoon_jupyter\n[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/camenduru/Diffutoon-jupyter/blob/main/Diffutoon_color_jupyter.ipynb) | Diffutoon_color_jupyter\n\n### 🧬 Code\nhttps://github.com/modelscope/DiffSynth-Studio/tree/main/examples/Diffutoon\n\n### 📄 Paper\nhttps://arxiv.org/abs/2401.16224\n\n### 🌐 Page\nhttps://ecnu-cilab.github.io/DiffutoonProjectPage/\n\n### 🖼 Output\n\nDiffutoon_color_jupyter\n\nhttps://github.com/camenduru/Diffutoon-jupyter/assets/54370274/e8741f4b-8925-488b-95b6-3812d219af06\n\n### 🏢 Sponsor\nhttps://runpod.io\n"
  }
]