MagicTime_jupyter.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github"
      },
      "source": [
        "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/camenduru/MagicTime-jupyter/blob/main/MagicTime_jupyter.ipynb)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "VjYy0F2gZIPR"
      },
      "outputs": [],
      "source": [
        "%cd /content\n",
        "!git clone -b dev https://github.com/camenduru/MagicTime\n",
        "%cd /content/MagicTime\n",
        "\n",
        "!pip install -q xformers==0.0.25 einops omegaconf gradio accelerate==0.28.0 diffusers==0.11.1 transformers==4.38.2 jax==0.4.19 jaxlib==0.4.19 ms-swift\n",
        "!apt -y install -qq aria2\n",
        "!git clone -b fp16 https://huggingface.co/runwayml/stable-diffusion-v1-5 /content/MagicTime/ckpts/Base_Model/stable-diffusion-v1-5\n",
        "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/spaces/BestWishYsh/MagicTime/resolve/main/ckpts/Base_Model/motion_module/motion_module.ckpt -d /content/MagicTime/ckpts/Base_Model/motion_module -o motion_module.ckpt\n",
        "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/spaces/BestWishYsh/MagicTime/resolve/main/ckpts/DreamBooth/RcnzCartoon.safetensors -d /content/MagicTime/ckpts/DreamBooth -o RcnzCartoon.safetensors\n",
        "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/spaces/BestWishYsh/MagicTime/resolve/main/ckpts/DreamBooth/RealisticVisionV60B1_v51VAE.safetensors -d /content/MagicTime/ckpts/DreamBooth -o RealisticVisionV60B1_v51VAE.safetensors\n",
        "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/spaces/BestWishYsh/MagicTime/resolve/main/ckpts/DreamBooth/ToonYou_beta6.safetensors -d /content/MagicTime/ckpts/DreamBooth -o ToonYou_beta6.safetensors\n",
        "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/spaces/BestWishYsh/MagicTime/resolve/main/ckpts/Magic_Weights/magic_adapter_s/magic_adapter_s.ckpt -d /content/MagicTime/ckpts/Magic_Weights/magic_adapter_s -o magic_adapter_s.ckpt\n",
        "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/spaces/BestWishYsh/MagicTime/raw/main/ckpts/Magic_Weights/magic_adapter_t/configuration.json -d /content/MagicTime/ckpts/Magic_Weights/magic_adapter_t -o configuration.json\n",
        "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/spaces/BestWishYsh/MagicTime/raw/main/ckpts/Magic_Weights/magic_adapter_t/default/adapter_config.json -d /content/MagicTime/ckpts/Magic_Weights/magic_adapter_t/default -o adapter_config.json\n",
        "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/spaces/BestWishYsh/MagicTime/resolve/main/ckpts/Magic_Weights/magic_adapter_t/default/adapter_model.bin -d /content/MagicTime/ckpts/Magic_Weights/magic_adapter_t/default -o adapter_model.bin\n",
        "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/spaces/BestWishYsh/MagicTime/raw/main/ckpts/Magic_Weights/magic_text_encoder/configuration.json -d /content/MagicTime/ckpts/Magic_Weights/magic_text_encoder -o configuration.json\n",
        "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/spaces/BestWishYsh/MagicTime/resolve/main/ckpts/Magic_Weights/magic_text_encoder/default/adapter_model.bin -d /content/MagicTime/ckpts/Magic_Weights/magic_text_encoder/default -o adapter_model.bin\n",
        "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/spaces/BestWishYsh/MagicTime/raw/main/ckpts/Magic_Weights/magic_text_encoder/default/adapter_config.json -d /content/MagicTime/ckpts/Magic_Weights/magic_text_encoder/default -o adapter_config.json\n",
        "\n",
        "# %cd /content/MagicTime\n",
        "# !python inference_magictime.py --config sample_configs/RealisticVision.yaml\n",
        "# !python inference_magictime.py --config sample_configs/RcnzCartoon.yaml\n",
        "# !python inference_magictime.py --config sample_configs/ToonYou.yaml"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "%cd /content/MagicTime\n",
        "from transformers import CLIPTextModel, CLIPTokenizer\n",
        "from diffusers import AutoencoderKL, DDIMScheduler\n",
        "from diffusers.utils.import_utils import is_xformers_available\n",
        "from utils.unet import UNet3DConditionModel\n",
        "from utils.pipeline_magictime import MagicTimePipeline\n",
        "from utils.util import save_videos_grid\n",
        "from utils.util import load_weights\n",
        "\n",
        "tokenizer    = CLIPTokenizer.from_pretrained(\"/content/MagicTime/ckpts/Base_Model/stable-diffusion-v1-5\", subfolder=\"tokenizer\")\n",
        "text_encoder = CLIPTextModel.from_pretrained(\"/content/MagicTime/ckpts/Base_Model/stable-diffusion-v1-5\", subfolder=\"text_encoder\").cuda()\n",
        "vae          = AutoencoderKL.from_pretrained(\"/content/MagicTime/ckpts/Base_Model/stable-diffusion-v1-5\", subfolder=\"vae\").cuda()\n",
        "unet = UNet3DConditionModel.from_pretrained_2d(\"/content/MagicTime/ckpts/Base_Model/stable-diffusion-v1-5\", subfolder=\"unet\",\n",
        "unet_additional_kwargs = {\n",
        "    \"use_inflated_groupnorm\": True,\n",
        "    \"use_motion_module\": True,\n",
        "    \"motion_module_resolutions\": [1, 2, 4, 8],\n",
        "    \"motion_module_mid_block\": False,\n",
        "    \"motion_module_type\": \"Vanilla\",\n",
        "    \"motion_module_kwargs\": {\n",
        "        \"num_attention_heads\": 8,\n",
        "        \"num_transformer_block\": 1,\n",
        "        \"attention_block_types\": [\"Temporal_Self\", \"Temporal_Self\"],\n",
        "        \"temporal_position_encoding\": True,\n",
        "        \"temporal_position_encoding_max_len\": 32,\n",
        "        \"temporal_attention_dim_div\": 1,\n",
        "        \"zero_initialize\": True\n",
        "    },\n",
        "    \"noise_scheduler_kwargs\": {\n",
        "        \"beta_start\": 0.00085,\n",
        "        \"beta_end\": 0.012,\n",
        "        \"beta_schedule\": \"linear\",\n",
        "        \"steps_offset\": 1,\n",
        "        \"clip_sample\": False\n",
        "    }\n",
        "}).cuda()\n",
        "\n",
        "if is_xformers_available():\n",
        "    unet.enable_xformers_memory_efficient_attention()\n",
        "\n",
        "pipeline = MagicTimePipeline(\n",
        "    vae=vae, text_encoder=text_encoder, tokenizer=tokenizer, unet=unet,\n",
        "    scheduler=DDIMScheduler(beta_start=0.00085,\n",
        "                            beta_end=0.012,\n",
        "                            beta_schedule=\"linear\",\n",
        "                            steps_offset=1,\n",
        "                            clip_sample=False)\n",
        ").to(\"cuda\")\n",
        "\n",
        "dreambooth = \"ToonYou_beta6\" # @param [\"ToonYou_beta6\", \"RealisticVisionV60B1_v51VAE\", \"RcnzCartoon\"]\n",
        "\n",
        "pipeline = load_weights(\n",
        "    pipeline,\n",
        "    motion_module_path=\"/content/MagicTime/ckpts/Base_Model/motion_module/motion_module.ckpt\",\n",
        "    dreambooth_model_path=f\"/content/MagicTime/ckpts/DreamBooth/{dreambooth}.safetensors\",\n",
        "    magic_adapter_s_path=\"/content/MagicTime/ckpts/Magic_Weights/magic_adapter_s/magic_adapter_s.ckpt\",\n",
        "    magic_adapter_t_path=\"/content/MagicTime/ckpts/Magic_Weights/magic_adapter_t\",\n",
        "    magic_text_encoder_path=\"/content/MagicTime/ckpts/Magic_Weights/magic_text_encoder\",\n",
        ").to(\"cuda\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "import torch\n",
        "random_seed = torch.randint(0, 2 ** 32 - 1, (1,)).item()\n",
        "torch.manual_seed(torch.randint(0, 2 ** 32 - 1, (1,)).item())\n",
        "\n",
        "prompt = 'An ice cube is melting.' # @param {type:\"string\"}\n",
        "num_inference_steps = 25 # @param {type:\"number\"}\n",
        "guidance_scale = 8.5 # @param {type:\"number\"}\n",
        "width = 512 # @param {type:\"number\"}\n",
        "height = 512 # @param {type:\"number\"}\n",
        "video_length = 16 # @param {type:\"number\"}\n",
        "sample = pipeline(prompt,\n",
        "                  num_inference_steps=num_inference_steps,\n",
        "                  guidance_scale=guidance_scale,\n",
        "                  width=width,\n",
        "                  height=height,\n",
        "                  video_length=video_length,\n",
        "                  ).videos\n",
        "save_videos_grid(sample, f\"/content/MagicTime/output-{random_seed}.gif\")\n",
        "\n",
        "from IPython.display import Image\n",
        "Image(open(f\"/content/MagicTime/output-{random_seed}.gif\", 'rb').read())"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "gpuType": "T4",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}