PSchroedl/lipsync pipeline feature

cmd/lipsync/Dockerfile.debug

@@ -0,0 +1,11 @@
+# A Dockerfile for attaching a debugger to the app using debugpy. For more information, see
+# https://code.visualstudio.com/docs/python/debugging#_command-line-debugging.
+FROM livepeer/ai-runner:lipsync
+
+RUN pip install debugpy
+
+# Expose the debugpy port and start the app as usual.
+CMD ["python", "-m", "debugpy", "--listen", "0.0.0.0:5678", "-m", "uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"]
+
+# If you want to wait for the debugger to attach before starting the app, use the --wait-for-client option.
+# CMD ["python", "-m", "debugpy", "--listen", "0.0.0.0:5678", "--wait-for-client", "-m", "uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"]

cmd/lipsync/Dockerfile.lipsync

@@ -0,0 +1,35 @@
+FROM livepeer/ai-runner:latest
+
+# Install necessary build tools
+RUN apt-get update && apt-get install -y \
+    ffmpeg libsm6 libxext6 python3-dev ninja-build cuda-toolkit-12-1 \
+    gawk bison gcc make wget tar
+
+# Download and install GLIBC 2.35
+# RUN wget -c https://ftp.gnu.org/gnu/glibc/glibc-2.35.tar.gz && \
+#     tar -zxvf glibc-2.35.tar.gz && \
+#     cd glibc-2.35 && \
+#     mkdir glibc-build && \
+#     cd glibc-build && \
+#     ../configure --prefix=/opt/glibc-2.35 && \
+#     make && \
+#     make install
+
+# # Set the LD_LIBRARY_PATH to use the new GLIBC
+# ENV LD_LIBRARY_PATH=/opt/glibc-2.35/lib:$LD_LIBRARY_PATH
+
+# # Verify the installation
+# RUN /opt/glibc-2.35/lib/ld-2.35.so --version
+
+# Download and install GLIBC 2.35
+RUN wget http://mirrors.edge.kernel.org/ubuntu/pool/main/g/glibc/libc6_2.35-0ubuntu3_amd64.deb && \
+    dpkg -i libc6_2.35-0ubuntu3_amd64.deb || apt-get install -f -y
+
+# Verify the installation
+RUN ldd --version
+
+COPY ./runner/app/ /app/app
+COPY ./cmd/lipsync/run_real3dportrait.sh /app/run_real3dportrait.sh
+
+# Expose the debugpy port and start the app as usual.
+CMD ["python", "-m", "debugpy", "--listen", "0.0.0.0:5678", "-m", "uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"]

cmd/lipsync/README.md

@@ -0,0 +1,75 @@
+# Setup for lipsync pipeline
+
+Real3DPortrait https://github.com/yerfor/Real3DPortrait is utilized for the lipsync pipeline
+
+Conda environment is built and configured on the host at /models/models--yerfor--Real3DPortrait/anaconda3 to mimic the paths that will be encountered by the containerized app.
+
+### Setup
+
+On the host machine, run :
+
+
+``` sudo ./setup_real3dportrait_env.sh ~```
+
+Models are expected to be stored at ~/.lpData/models/ on the host machine, and the final step of this script copies the new conda environment to ~/.lpData/models/models/models--yerfor--Real3DPortrait/anaconda3.
+
+### Usage
+
+```
+git clone https://github.com/livepeer/ai-worker.git
+cd ai-worker/
+docker build -f ./cmd/lipsync/Dockerfile.lipsync -t livepeer/ai-runner:lipsync .
+docker run --name lipsync-public -e PIPELINE=lipsync -e MODEL_ID=real3dportrait -e HUGGINGFACE_TOKEN={your token or env var} --gpus 0 -p 8005:8000 -v ~/.lpData/models:/models livepeer/ai-runner:lipsync
+```
+
+### Implementation details
+
+The lipsync pipeline in ai-worker/runner invokes a helper script ( cmd/lipsync/run_real3dportrait.sh ) to activate the conda environment inside a shell running in the container, Then, the real3dportrait python inference script is invoked with the parameters provided to the helper script.
+
+### Debugging the lipsync pipeline
+
+Launch.json for Visual Studio Code: 
+```
+{
+    "version": "0.2.0",
+    "configurations": [
+
+        {
+            "name": "Python: Uvicorn",
+            "type": "debugpy",
+            "request": "launch",
+            "module": "uvicorn",
+            "args": [
+                "app.main:app",
+                "--log-config", "app/cfg/uvicorn_logging_config.json",
+                "--host", "0.0.0.0",
+                "--port", "8000"
+            ],
+            "env": {
+                "PIPELINE": "lipsync",
+                "MODEL_ID": "real3dportrait",
+                "HUGGINGFACE_TOKEN": "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"
+            },
+            "python": "python",
+            "console": "integratedTerminal"
+        }
+    ]
+}
+
+```
+
+Changes to reflect use of the lipsync debug dockerfile need to be made to .devcontainer:
+
+```
+diff --git a/runner/.devcontainer/devcontainer.json b/runner/.devcontainer/devcontainer.json
+index 3253288..f5c94d6 100644
+--- a/runner/.devcontainer/devcontainer.json
++++ b/runner/.devcontainer/devcontainer.json
+@@ -5,7 +5,11 @@
+
+        // Image to use for the dev container. More info: https://containers.dev/guide/dockerfile.
+        "build": {
+-               "dockerfile": "../Dockerfile"
++               "context": "../..",
++               "dockerfile": "../../cmd/lipsync/Dockerfile.debug",
+```

cmd/lipsync/run_real3dportrait.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+activate_conda_env() {
+  local env_path="$1"
+  if [ -f "${env_path}/etc/profile.d/conda.sh" ]; then
+    source "${env_path}/etc/profile.d/conda.sh"
+    eval "$($env_path}/bin/conda shell.bash hook)"
+    conda activate real3dportrait
+  else
+    echo "Conda activation script not found at ${env_path}/etc/profile.d/conda.sh"
+    exit 1
+  fi
+}
+
+activate_conda_env "/models/models--yerfor--Real3DPortrait/anaconda3"
+
+# Debugging: Check which Python is being used - 3.9 is used by the conda env for real3dportrait
+which python
+python --version
+
+# Set PYTHONPATH
+export PYTHONPATH="/models/models--yerfor--Real3DPortrait:$PYTHONPATH"
+cd /models/models--yerfor--Real3DPortrait
+
+# Run the Real3DPortrait inference command
+python /models/models--yerfor--Real3DPortrait/inference/real3d_infer.py \
+  --src_img "$1" \
+  --drv_aud "$2" \
+  --out_name "$3" \
+  --drv_pose "$4" \
+  --out_mode final
+

cmd/lipsync/setup_real3dportrait_env.sh

@@ -0,0 +1,128 @@
+#!/bin/bash
+
+# Check if the root directory is passed as an argument
+if [ -z "$1" ]; then
+  echo "Please provide the root directory as an argument."
+  exit 1
+fi
+
+ROOT_DIR=$(eval echo $1)
+
+# Define variables
+CONDA_INSTALLER=Anaconda3-2024.06-1-Linux-x86_64.sh
+CONDA_DIR=/models/models--yerfor--Real3DPortrait/anaconda3
+REPO_URL=https://github.com/yerfor/Real3DPortrait.git
+REPO_DIR=$ROOT_DIR/.lpData/models/models--yerfor--Real3DPortrait
+ENV_NAME=real3dportrait
+
+# Step 1: Download and install Anaconda
+if [ ! -f $CONDA_INSTALLER ]; then
+  wget https://repo.anaconda.com/archive/$CONDA_INSTALLER
+fi
+
+# Install Anaconda silently
+bash ./$CONDA_INSTALLER -b -p $CONDA_DIR
+
+# Initialize conda
+eval "$($CONDA_DIR/bin/conda shell.bash hook)"
+
+# Create the models directory if it doesn't exist
+mkdir -p $ROOT_DIR/.lpData/models
+
+# Clone the repository containing inference.py
+if [ ! -d $REPO_DIR ]; then
+  git clone $REPO_URL $REPO_DIR
+fi
+
+# Navigate to the repository directory
+cd $REPO_DIR
+
+# Step 2: Create and activate conda environment in the specified directory
+conda create -p $CONDA_DIR/envs/$ENV_NAME python=3.9 -y
+source $CONDA_DIR/bin/activate $CONDA_DIR/envs/$ENV_NAME
+
+# Step 3: Install dependencies
+conda install -p $CONDA_DIR/envs/$ENV_NAME conda-forge::ffmpeg -y
+conda install pytorch torchvision torchaudio pytorch-cuda=12.1 -c pytorch -c nvidia -y
+conda install -p $CONDA_DIR/envs/$ENV_NAME pytorch3d::pytorch3d -y
+
+# Install additional dependencies
+pip install cython
+pip install openmim==0.3.9
+mim install mmcv==2.1.0
+pip install -r docs/prepare_env/requirements.txt -v
+
+# Handle potential dependency conflict
+pip install -r docs/prepare_env/requirements.txt -v --use-deprecated=legacy-resolver
+
+
+
+# Required for downloading real3dportrait checkpoints
+pip install gdown
+
+# Create the required directories
+mkdir -p deep_3drecon/BFM
+mkdir -p checkpoints/240126_real3dportrait_orig/audio2secc_vae
+mkdir -p checkpoints/240126_real3dportrait_orig/secc2plane_torso_orig
+mkdir -p checkpoints/pretrained_ckpts
+
+echo "Directory structure created."
+
+# Download 3DMM BFM Model
+echo "Downloading 3DMM BFM Model..."
+cd deep_3drecon/BFM
+gdown https://drive.google.com/uc?id=1SPM3IHsyNAaVMwqZZGV6QVaV7I2Hly0v
+gdown https://drive.google.com/uc?id=1MSldX9UChKEb3AXLVTPzZQcsbGD4VmGF
+gdown https://drive.google.com/uc?id=180ciTvm16peWrcpl4DOekT9eUQ-lJfMU
+gdown https://drive.google.com/uc?id=1KX9MyGueFB3M-X0Ss152x_johyTXHTfU
+gdown https://drive.google.com/uc?id=19-NyZn_I0_mkF-F5GPyFMwQJ_-WecZIL
+gdown https://drive.google.com/uc?id=11ouQ7Wr2I-JKStp2Fd1afedmWeuifhof
+gdown https://drive.google.com/uc?id=18ICIvQoKX-7feYWP61RbpppzDuYTptCq
+gdown https://drive.google.com/uc?id=1VktuY46m0v_n_d4nvOupauJkK4LF6mHE
+cd ../..
+
+# Download Pre-trained Real3D-Portrait models
+echo "Downloading Pre-trained Real3D-Portrait models..."
+cd checkpoints
+gdown https://drive.google.com/uc?id=1gz8A6xestHp__GbZT5qozb43YaybRJhZ
+gdown https://drive.google.com/uc?id=1gSUIw2AkkKnlLJnNfS2FCqtaVw9tw3QF
+unzip 240210_real3dportrait_orig.zip
+unzip pretrained_ckpts.zip
+ls
+cd ..
+# Navigate to the repository directory
+cd $REPO_DIR
+# Check if all required files are in place
+required_files=(
+    "deep_3drecon/BFM/01_MorphableModel.mat"
+    "deep_3drecon/BFM/BFM_exp_idx.mat"
+    "deep_3drecon/BFM/BFM_front_idx.mat"
+    "deep_3drecon/BFM/BFM_model_front.mat"
+    "deep_3drecon/BFM/Exp_Pca.bin"
+    "deep_3drecon/BFM/facemodel_info.mat"
+    "deep_3drecon/BFM/std_exp.txt"
+    "deep_3drecon/reconstructor_opt.pkl"
+    "checkpoints/240210_real3dportrait_orig/audio2secc_vae/config.yaml"
+    "checkpoints/240210_real3dportrait_orig/audio2secc_vae/model_ckpt_steps_400000.ckpt"
+    "checkpoints/240210_real3dportrait_orig/secc2plane_torso_orig/config.yaml"
+    "checkpoints/240210_real3dportrait_orig/secc2plane_torso_orig/model_ckpt_steps_100000.ckpt"
+    "checkpoints/pretrained_ckpts/mit_b0.pth"
+)
+
+all_files_exist=true
+
+for file in "${required_files[@]}"; do
+    if [ ! -f "$file" ]; then
+        echo "File not found: $file"
+        all_files_exist=false
+    fi
+done
+
+if $all_files_exist; then
+    echo "All files are in the correct places."
+else
+    echo "Some files are missing."
+fi
+
+cd ..
+mv $CONDA_DIR $REPO_DIR/anaconda3

runner/app/main.py

@@ -1,5 +1,7 @@
-from fastapi import FastAPI
+from http.client import HTTPException
+from fastapi import FastAPI, HTTPException
 from fastapi.routing import APIRoute
+from fastapi.responses import FileResponse
 from contextlib import asynccontextmanager
 import os
 import logging
@@ -47,6 +49,9 @@ def load_pipeline(pipeline: str, model_id: str) -> any:
         case "upscale":
             from app.pipelines.upscale import UpscalePipeline
             return UpscalePipeline(model_id)
+        case "lipsync":
+            from app.pipelines.lipsync import LipsyncPipeline
+            return LipsyncPipeline()
         case _:
             raise EnvironmentError(
                 f"{pipeline} is not a valid pipeline for model {model_id}"
@@ -73,6 +78,10 @@ def load_route(pipeline: str) -> any:
             from app.routes import upscale
 
             return upscale.router
+        case "lipsync":
+            from app.routes import lipsync
+
+            return lipsync.router
         case _:
             raise EnvironmentError(f"{pipeline} is not a valid pipeline")