From fd1ab5820999e2a6c6b5df301ff6c2075c1a6c9e Mon Sep 17 00:00:00 2001 From: Jason Stone Date: Fri, 26 Jul 2024 01:30:54 +0300 Subject: [PATCH 01/38] update to the frame interpolation pipeline, there is some minor issue with creating go api bindings because of openapi json sceme having a null option. --- runner/app/main.py | 18 ++- runner/app/pipelines/frame_interpolation.py | 114 +++++++++++++- runner/app/pipelines/image_to_video.py | 6 +- runner/app/pipelines/upscale.py | 4 +- runner/app/pipelines/utils/__init__.py | 4 + runner/app/pipelines/utils/utils.py | 160 ++++++++++++++++++++ runner/app/routes/frame_interpolation.py | 120 +++++++++++++++ runner/dl_checkpoints.sh | 3 + runner/gen_openapi.py | 9 +- runner/openapi.json | 151 ++++++++++++++++-- runner/requirements.txt | 3 + 11 files changed, 566 insertions(+), 26 deletions(-) create mode 100644 runner/app/routes/frame_interpolation.py diff --git a/runner/app/main.py b/runner/app/main.py index 6f511420..c4dd6b8e 100644 --- a/runner/app/main.py +++ b/runner/app/main.py @@ -1,5 +1,7 @@ import logging import os +import sys +import cv2 from contextlib import asynccontextmanager from app.routes import health @@ -15,8 +17,8 @@ async def lifespan(app: FastAPI): app.include_router(health.router) - pipeline = os.environ["PIPELINE"] - model_id = os.environ["MODEL_ID"] + pipeline = os.environ.get("PIPELINE", "") # Default to + model_id = os.environ.get("MODEL_ID", "") # Provide a default if necessary app.pipeline = load_pipeline(pipeline, model_id) app.include_router(load_route(pipeline)) @@ -46,8 +48,10 @@ def load_pipeline(pipeline: str, model_id: str) -> any: from app.pipelines.audio_to_text import AudioToTextPipeline return AudioToTextPipeline(model_id) - case "frame-interpolation": - raise NotImplementedError("frame-interpolation pipeline not implemented") + case "FILMPipeline": + from app.pipelines.frame_interpolation import FILMPipeline + + return FILMPipeline(model_id) case "upscale": from app.pipelines.upscale import UpscalePipeline @@ -76,8 +80,10 @@ def load_route(pipeline: str) -> any: from app.routes import audio_to_text return audio_to_text.router - case "frame-interpolation": - raise NotImplementedError("frame-interpolation pipeline not implemented") + case "FILMPipeline": + from app.routes import frame_interpolation + + return frame_interpolation.router case "upscale": from app.routes import upscale diff --git a/runner/app/pipelines/frame_interpolation.py b/runner/app/pipelines/frame_interpolation.py index 396c8067..755afee7 100644 --- a/runner/app/pipelines/frame_interpolation.py +++ b/runner/app/pipelines/frame_interpolation.py @@ -1,5 +1,113 @@ -from app.pipelines.base import Pipeline +import torch +from torchvision.transforms import v2 +from tqdm import tqdm +import bisect +import numpy as np +from app.pipelines.utils.utils import get_model_dir -class FrameInterpolationPipeline(Pipeline): - pass +class FILMPipeline: + model: torch.jit.ScriptModule + + def __init__(self, model_id: str): + self.model_id = model_id + model_dir = get_model_dir() # Get the directory where models are stored + model_path = f"{model_dir}/{model_id}" # Construct the full path to the model file + + self.model = torch.jit.load(model_path, map_location="cpu") + self.model.eval() + + def to(self, *args, **kwargs): + self.model = self.model.to(*args, **kwargs) + return self + + @property + def device(self) -> torch.device: + # Checking device for ScriptModule requires checking one of its parameters + params = self.model.parameters() + return next(params).device + + @property + def dtype(self) -> torch.dtype: + # Checking device for ScriptModule requires checking one of its parameters + params = self.model.parameters() + return next(params).dtype + + def __call__( + self, + reader, + writer, + inter_frames: int = 2, + ): + transforms = v2.Compose( + [ + v2.ToDtype(torch.uint8, scale=True), + ] + ) + + writer.open() + + while True: + frame_1 = reader.get_frame() + # If the first frame read is None then there are no more frames + if frame_1 is None: + break + + frame_2 = reader.get_frame() + # If the second frame read is None there there is a final frame + if frame_2 is None: + writer.write_frame(transforms(frame_1)) + break + + # frame_1 and frame_2 must be tensors with n c h w format + frame_1 = frame_1.unsqueeze(0) + frame_2 = frame_2.unsqueeze(0) + + frames = inference( + self.model, frame_1, frame_2, inter_frames, self.device, self.dtype + ) + + frames = [transforms(frame.detach().cpu()) for frame in frames] + for frame in frames: + writer.write_frame(frame) + + writer.close() + + +def inference( + model, img_batch_1, img_batch_2, inter_frames, device, dtype +) -> torch.Tensor: + results = [img_batch_1, img_batch_2] + + idxes = [0, inter_frames + 1] + remains = list(range(1, inter_frames + 1)) + + splits = torch.linspace(0, 1, inter_frames + 2) + + for _ in tqdm(range(len(remains)), "Generating in-between frames"): + starts = splits[idxes[:-1]] + ends = splits[idxes[1:]] + distances = ( + (splits[None, remains] - starts[:, None]) + / (ends[:, None] - starts[:, None]) + - 0.5 + ).abs() + matrix = torch.argmin(distances).item() + start_i, step = np.unravel_index(matrix, distances.shape) + end_i = start_i + 1 + + x0 = results[start_i].to(device=device, dtype=dtype) + x1 = results[end_i].to(device=device, dtype=dtype) + + dt = x0.new_full((1, 1), (splits[remains[step]] - splits[idxes[start_i]])) / ( + splits[idxes[end_i]] - splits[idxes[start_i]] + ) + + with torch.no_grad(): + prediction = model(x0, x1, dt) + insert_position = bisect.bisect_left(idxes, remains[step]) + idxes.insert(insert_position, remains[step]) + results.insert(insert_position, prediction.clamp(0, 1).float()) + del remains[step] + + return results \ No newline at end of file diff --git a/runner/app/pipelines/image_to_video.py b/runner/app/pipelines/image_to_video.py index f605cb2f..988c075b 100644 --- a/runner/app/pipelines/image_to_video.py +++ b/runner/app/pipelines/image_to_video.py @@ -6,10 +6,14 @@ import PIL import torch from app.pipelines.base import Pipeline -from app.pipelines.utils import SafetyChecker, get_model_dir, get_torch_device from diffusers import StableVideoDiffusionPipeline from huggingface_hub import file_download from PIL import ImageFile +from app.pipelines.utils import ( + SafetyChecker, + get_model_dir, + get_torch_device +) ImageFile.LOAD_TRUNCATED_IMAGES = True diff --git a/runner/app/pipelines/upscale.py b/runner/app/pipelines/upscale.py index 0cba865e..6fd3cefe 100644 --- a/runner/app/pipelines/upscale.py +++ b/runner/app/pipelines/upscale.py @@ -87,7 +87,7 @@ def __init__(self, model_id: str): elif deepcache_enabled: logger.warning( "DeepCache is not supported for Lightning or Turbo models. " - "TextToImagePipeline will NOT be optimized with DeepCache for %s", + "UpscalingPiepline will NOT be optimized with DeepCache for %s", model_id, ) @@ -112,7 +112,7 @@ def __call__( ] if num_inference_steps is None or num_inference_steps < 1: - del kwargs["num_inference_steps"] + kwargs.pop("num_inference_steps", None) output = self.ldm(prompt, image=image, **kwargs) diff --git a/runner/app/pipelines/utils/__init__.py b/runner/app/pipelines/utils/__init__.py index 844b86e9..2c083d20 100644 --- a/runner/app/pipelines/utils/__init__.py +++ b/runner/app/pipelines/utils/__init__.py @@ -9,4 +9,8 @@ is_turbo_model, split_prompt, validate_torch_device, + frames_compactor, + video_shredder, + DirectoryReader, + DirectoryWriter ) diff --git a/runner/app/pipelines/utils/utils.py b/runner/app/pipelines/utils/utils.py index dbc44d48..5c9ad6a5 100644 --- a/runner/app/pipelines/utils/utils.py +++ b/runner/app/pipelines/utils/utils.py @@ -5,11 +5,18 @@ import re from pathlib import Path from typing import Optional +import glob +import tempfile +from io import BytesIO +from typing import List, Union import numpy as np import torch from diffusers.pipelines.stable_diffusion import StableDiffusionSafetyChecker from PIL import Image +from torchvision.transforms import v2 +import cv2 +from torchaudio.io import StreamWriter from torch import dtype as TorchDtype from transformers import CLIPFeatureExtractor @@ -111,6 +118,108 @@ def split_prompt( return prompt_dict +def frames_compactor( + frames: Union[List[np.ndarray], List[torch.Tensor]], + output_path: str, + fps: float, + codec: str = "MJPEG", + is_directory: bool = False, + width: int = None, + height: int = None +) -> None: + """ + Generate a video from a list of frames. Frames can be from a directory or in-memory. + + Args: + frames (List[np.ndarray] | List[torch.Tensor]): List of frames as NumPy arrays or PyTorch tensors. + output_path (str): Path to save the output video file. + fps (float): Frames per second for the video. + codec (str): Codec used for video compression (default is "XVID"). + is_directory (bool): If True, treat `frames` as a directory path containing image files. + width (int): Width of the video. Must be provided if `frames` are in-memory. + height (int): Height of the video. Must be provided if `frames` are in-memory. + + Returns: + None + """ + if is_directory: + # Read frames from a directory + frames = [cv2.imread(os.path.join(frames, file)) for file in sorted(os.listdir(frames))] + else: + # Convert torch tensors to numpy arrays if necessary + if isinstance(frames[0], torch.Tensor): + frames = [frame.permute(1, 2, 0).cpu().numpy() for frame in frames] + + # Ensure frames are numpy arrays and are uint8 type + frames = [frame.astype(np.uint8) for frame in frames] + + # Check if frames are consistent + if not frames: + raise ValueError("No frames to process.") + + if width is None or height is None: + # Use dimensions of the first frame if not provided + height, width = frames[0].shape[:2] + + # Define the codec and create VideoWriter object + fourcc = cv2.VideoWriter_fourcc(*codec) + video_writer = cv2.VideoWriter(output_path, fourcc, fps, (width, height)) + + # Write frames to the video file + for frame in frames: + # Ensure each frame has the correct size + if frame.shape[1] != width or frame.shape[0] != height: + frame = cv2.resize(frame, (width, height)) + video_writer.write(frame) + + # Release the video writer + video_writer.release() + +def video_shredder(video_data, is_file_path=True) -> np.ndarray: + """ + Extract frames from a video file or in-memory video data and return them as a NumPy array. + + Args: + video_data (str or BytesIO): Path to the input video file or in-memory video data. + is_file_path (bool): Indicates if video_data is a file path (True) or in-memory data (False). + + Returns: + np.ndarray: Array of frames with shape (num_frames, height, width, channels). + """ + if is_file_path: + # Handle file-based video input + video_capture = cv2.VideoCapture(video_data) + else: + # Handle in-memory video input + # Create a temporary file to store in-memory video data + with tempfile.NamedTemporaryFile(delete=False, suffix='.mp4') as temp_file: + temp_file.write(video_data.getvalue()) + temp_file_path = temp_file.name + + # Open the temporary video file + video_capture = cv2.VideoCapture(temp_file_path) + + if not video_capture.isOpened(): + raise ValueError("Error opening video data") + + frames = [] + success, frame = video_capture.read() + + while success: + frames.append(frame) + success, frame = video_capture.read() + + video_capture.release() + + # Delete the temporary file if it was created + if not is_file_path: + os.remove(temp_file_path) + + # Convert list of frames to a NumPy array + frames_array = np.array(frames) + print(f"Extracted {frames_array.shape[0]} frames from video in shape of {frames_array.shape}") + + return frames_array class SafetyChecker: """Checks images for unsafe or inappropriate content using a pretrained model. @@ -167,3 +276,54 @@ def check_nsfw_images( clip_input=safety_checker_input.pixel_values.to(self._dtype), ) return images, has_nsfw_concept + +class DirectoryReader: + def __init__(self, dir: str): + self.paths = sorted( + glob.glob(os.path.join(dir, "*")), + key=lambda x: int(os.path.basename(x).split(".")[0]), + ) + self.nb_frames = len(self.paths) + self.idx = 0 + + assert self.nb_frames > 0, "no frames found in directory" + + first_img = Image.open(self.paths[0]) + self.height = first_img.height + self.width = first_img.width + + def get_resolution(self): + return self.height, self.width + + def reset(self): + self.idx = 0 # Reset the index counter to 0 + + def get_frame(self): + if self.idx >= self.nb_frames: + return None + + path = self.paths[self.idx] + self.idx += 1 + + img = Image.open(path) + transforms = v2.Compose([v2.ToImage(), v2.ToDtype(torch.float32, scale=True)]) + + return transforms(img) + +class DirectoryWriter: + def __init__(self, dir: str): + self.dir = dir + self.idx = 0 + + def open(self): + return + + def close(self): + return + + def write_frame(self, frame: torch.Tensor): + path = f"{self.dir}/{self.idx}.png" + self.idx += 1 + + transforms = v2.Compose([v2.ToPILImage()]) + transforms(frame.squeeze(0)).save(path) \ No newline at end of file diff --git a/runner/app/routes/frame_interpolation.py b/runner/app/routes/frame_interpolation.py new file mode 100644 index 00000000..a1ae75bc --- /dev/null +++ b/runner/app/routes/frame_interpolation.py @@ -0,0 +1,120 @@ +# app/routes/film_interpolate.py + +import logging +import os +import torch +import glob +from typing import Annotated, Optional +from fastapi import APIRouter, Depends, File, Form, UploadFile, status +from fastapi.responses import JSONResponse +from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer +from PIL import Image, ImageFile + +from app.dependencies import get_pipeline +from app.pipelines.frame_interpolation import FILMPipeline +from app.pipelines.utils.utils import DirectoryReader, DirectoryWriter, get_torch_device, get_model_dir +from app.routes.util import HTTPError, ImageResponse, http_error, image_to_data_url + +ImageFile.LOAD_TRUNCATED_IMAGES = True + +router = APIRouter() + +logger = logging.getLogger(__name__) + +RESPONSES = { + status.HTTP_400_BAD_REQUEST: {"model": HTTPError}, + status.HTTP_401_UNAUTHORIZED: {"model": HTTPError}, + status.HTTP_500_INTERNAL_SERVER_ERROR: {"model": HTTPError}, +} + +@router.post("/frame_interpolation", response_model=ImageResponse, responses=RESPONSES) +@router.post( + "/frame_interpolation/", + response_model=ImageResponse, + responses=RESPONSES, + include_in_schema=False, +) +async def frame_interpolation( + model_id: Annotated[str, Form()], + image1: Annotated[Optional[UploadFile], File()]=None, + image2: Annotated[Optional[UploadFile], File()]=None, + image_dir: Annotated[Optional[str], Form()]="", + inter_frames: Annotated[int, Form()] = 2, + token: HTTPAuthorizationCredentials = Depends(HTTPBearer(auto_error=False)), +): + auth_token = os.environ.get("AUTH_TOKEN") + if auth_token: + if not token or token.credentials != auth_token: + return JSONResponse( + status_code=status.HTTP_401_UNAUTHORIZED, + headers={"WWW-Authenticate": "Bearer"}, + content=http_error("Invalid bearer token"), + ) + + + # Initialize FILMPipeline + film_pipeline = FILMPipeline(model_id) + film_pipeline.to(device=get_torch_device(),dtype=torch.float16) + + # Prepare directories for input and output + temp_input_dir = "temp_input" + temp_output_dir = "temp_output" + os.makedirs(temp_input_dir, exist_ok=True) + os.makedirs(temp_output_dir, exist_ok=True) + + try: + if os.path.isdir(image_dir): + if image1 and image2: + logger.info("Both directory and individual images provided. Directory will be used, and images will be ignored.") + reader = DirectoryReader(image_dir) + else: + if not (image1 and image2): + return JSONResponse( + status_code=status.HTTP_400_BAD_REQUEST, + content=http_error("Either a directory or two images must be provided."), + ) + + image1_path = os.path.join(temp_input_dir, "0.png") + image2_path = os.path.join(temp_input_dir, "1.png") + + with open(image1_path, "wb") as f: + f.write(await image1.read()) + with open(image2_path, "wb") as f: + f.write(await image2.read()) + + reader = DirectoryReader(temp_input_dir) + + writer = DirectoryWriter(temp_output_dir) + # Perform interpolation + film_pipeline(reader, writer, inter_frames=inter_frames) + + writer.close() + reader.reset() + + # Collect output frames + output_frames = [] + for frame_path in sorted(glob.glob(os.path.join(temp_output_dir, "*.png"))): + frame = Image.open(frame_path) + output_frames.append(frame) + + output_images = [{"url": image_to_data_url(frame),"seed":0, "nsfw":False} for frame in output_frames] + + except Exception as e: + logger.error(f"FILMPipeline error: {e}") + logger.exception(e) + return JSONResponse( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + content=http_error("FILMPipeline error"), + ) + + finally: + # Clean up temporary directories + for file_path in glob.glob(os.path.join(temp_input_dir, "*")): + os.remove(file_path) + os.rmdir(temp_input_dir) + + for file_path in glob.glob(os.path.join(temp_output_dir, "*")): + os.remove(file_path) + os.rmdir(temp_output_dir) + + return {"images": output_images} diff --git a/runner/dl_checkpoints.sh b/runner/dl_checkpoints.sh index 822590d4..1528a60b 100755 --- a/runner/dl_checkpoints.sh +++ b/runner/dl_checkpoints.sh @@ -59,6 +59,9 @@ function download_all_models() { # Download image-to-video models. huggingface-cli download stabilityai/stable-video-diffusion-img2vid-xt --include "*.fp16.safetensors" "*.json" --cache-dir models + + #Download frame-interpolation model. + wget -O models/film_net_fp16.pt https://github.com/dajes/frame-interpolation-pytorch/releases/download/v1.0.2/film_net_fp16.pt } # Enable HF transfer acceleration. diff --git a/runner/gen_openapi.py b/runner/gen_openapi.py index 7fde5ee3..198102db 100644 --- a/runner/gen_openapi.py +++ b/runner/gen_openapi.py @@ -11,6 +11,7 @@ image_to_image, image_to_video, text_to_image, + frame_interpolation, upscale, ) from fastapi.openapi.utils import get_openapi @@ -68,7 +69,7 @@ def translate_to_gateway(openapi): openapi["components"]["schemas"]["VideoResponse"]["title"] = "VideoResponse" return openapi - + def write_openapi(fname, entrypoint="runner"): """Write OpenAPI schema to file. @@ -83,8 +84,10 @@ def write_openapi(fname, entrypoint="runner"): app.include_router(text_to_image.router) app.include_router(image_to_image.router) app.include_router(image_to_video.router) - app.include_router(upscale.router) app.include_router(audio_to_text.router) + app.include_router(frame_interpolation.router) + app.include_router(upscale.router) + use_route_names_as_operation_ids(app) @@ -144,3 +147,5 @@ def write_openapi(fname, entrypoint="runner"): args = parser.parse_args() write_openapi(f"openapi.{args.type.lower()}", args.entrypoint) + + diff --git a/runner/openapi.json b/runner/openapi.json index 4345e565..c103d2f6 100644 --- a/runner/openapi.json +++ b/runner/openapi.json @@ -249,15 +249,15 @@ ] } }, - "/upscale": { + "/audio-to-text": { "post": { - "summary": "Upscale", - "operationId": "upscale", + "summary": "Audio To Text", + "operationId": "audio_to_text", "requestBody": { "content": { "multipart/form-data": { "schema": { - "$ref": "#/components/schemas/Body_upscale_upscale_post" + "$ref": "#/components/schemas/Body_audio_to_text_audio_to_text_post" } } }, @@ -269,7 +269,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ImageResponse" + "$ref": "#/components/schemas/TextResponse" } } } @@ -294,6 +294,16 @@ } } }, + "413": { + "description": "Request Entity Too Large", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPError" + } + } + } + }, "500": { "description": "Internal Server Error", "content": { @@ -322,15 +332,15 @@ ] } }, - "/audio-to-text": { + "/frame_interpolation": { "post": { - "summary": "Audio To Text", - "operationId": "audio_to_text", + "summary": "Frame Interpolation", + "operationId": "frame_interpolation", "requestBody": { "content": { "multipart/form-data": { "schema": { - "$ref": "#/components/schemas/Body_audio_to_text_audio_to_text_post" + "$ref": "#/components/schemas/Body_frame_interpolation_frame_interpolation_post" } } }, @@ -342,7 +352,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/TextResponse" + "$ref": "#/components/schemas/ImageResponse" } } } @@ -367,8 +377,71 @@ } } }, - "413": { - "description": "Request Entity Too Large", + "500": { + "description": "Internal Server Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPError" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + }, + "security": [ + { + "HTTPBearer": [] + } + ] + } + }, + "/upscale": { + "post": { + "summary": "Upscale", + "operationId": "upscale", + "requestBody": { + "content": { + "multipart/form-data": { + "schema": { + "$ref": "#/components/schemas/Body_upscale_upscale_post" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ImageResponse" + } + } + } + }, + "400": { + "description": "Bad Request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPError" + } + } + } + }, + "401": { + "description": "Unauthorized", "content": { "application/json": { "schema": { @@ -440,6 +513,60 @@ ], "title": "Body_audio_to_text_audio_to_text_post" }, + "Body_frame_interpolation_frame_interpolation_post": { + "properties": { + "model_id": { + "type": "string", + "title": "Model Id" + }, + "image1": { + "anyOf": [ + { + "type": "string", + "format": "binary" + }, + { + "type": "null" + } + ], + "title": "Image1" + }, + "image2": { + "anyOf": [ + { + "type": "string", + "format": "binary" + }, + { + "type": "null" + } + ], + "title": "Image2" + }, + "image_dir": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Image Dir", + "default": "" + }, + "inter_frames": { + "type": "integer", + "title": "Inter Frames", + "default": 2 + } + }, + "type": "object", + "required": [ + "model_id" + ], + "title": "Body_frame_interpolation_frame_interpolation_post" + }, "Body_image_to_image_image_to_image_post": { "properties": { "prompt": { diff --git a/runner/requirements.txt b/runner/requirements.txt index 3852800d..0a75fd8a 100644 --- a/runner/requirements.txt +++ b/runner/requirements.txt @@ -6,6 +6,8 @@ pydantic==2.7.2 Pillow==10.3.0 python-multipart==0.0.9 uvicorn==0.30.0 +setuptools +torch huggingface_hub==0.23.2 xformers==0.0.23 triton>=2.1.0 @@ -17,3 +19,4 @@ numpy==1.26.4 av==12.1.0 sentencepiece== 0.2.0 protobuf==5.27.2 +opencv-python==4.10.0.84 From 65d2c5390698a225a1a0c45be05bae5d509b20b6 Mon Sep 17 00:00:00 2001 From: Jason Stone Date: Fri, 26 Jul 2024 11:51:56 +0300 Subject: [PATCH 02/38] minor changes to requirements --- runner/requirements.txt | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/runner/requirements.txt b/runner/requirements.txt index 0a75fd8a..c02b564e 100644 --- a/runner/requirements.txt +++ b/runner/requirements.txt @@ -1,13 +1,15 @@ diffusers==0.29.2 accelerate==0.30.1 -transformers==4.41.1 +transformers==4.43.1 fastapi==0.111.0 pydantic==2.7.2 Pillow==10.3.0 python-multipart==0.0.9 uvicorn==0.30.0 -setuptools -torch +setuptools==71.1.0 +torch==2.4.0+cu121 +torchaudio==2.4.0+cu121 +torchvision==0.19.0+cu121 huggingface_hub==0.23.2 xformers==0.0.23 triton>=2.1.0 From 796796dfee1bcec784e65147288969eec9fda70a Mon Sep 17 00:00:00 2001 From: Jason Stone Date: Fri, 26 Jul 2024 12:18:07 +0300 Subject: [PATCH 03/38] update to requrements to fetch from --index-url --- runner/requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/runner/requirements.txt b/runner/requirements.txt index c02b564e..82877c24 100644 --- a/runner/requirements.txt +++ b/runner/requirements.txt @@ -7,9 +7,9 @@ Pillow==10.3.0 python-multipart==0.0.9 uvicorn==0.30.0 setuptools==71.1.0 -torch==2.4.0+cu121 -torchaudio==2.4.0+cu121 -torchvision==0.19.0+cu121 +torch --index-url https://download.pytorch.org/whl/cu121 +torchvision --index-url https://download.pytorch.org/whl/cu121 +torchaudio --index-url https://download.pytorch.org/whl/cu121 huggingface_hub==0.23.2 xformers==0.0.23 triton>=2.1.0 From b5eb66df646a293d5a842b7cb5aa27437f2f4699 Mon Sep 17 00:00:00 2001 From: Jason Stone Date: Fri, 26 Jul 2024 14:12:57 +0300 Subject: [PATCH 04/38] simple patch to solve the go api bindings issue --- runner/app/routes/frame_interpolation.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/runner/app/routes/frame_interpolation.py b/runner/app/routes/frame_interpolation.py index a1ae75bc..6abf6a66 100644 --- a/runner/app/routes/frame_interpolation.py +++ b/runner/app/routes/frame_interpolation.py @@ -36,9 +36,9 @@ ) async def frame_interpolation( model_id: Annotated[str, Form()], - image1: Annotated[Optional[UploadFile], File()]=None, - image2: Annotated[Optional[UploadFile], File()]=None, - image_dir: Annotated[Optional[str], Form()]="", + image1: Annotated[UploadFile, File()]=None, + image2: Annotated[UploadFile, File()]=None, + image_dir: Annotated[str, Form()]="", inter_frames: Annotated[int, Form()] = 2, token: HTTPAuthorizationCredentials = Depends(HTTPBearer(auto_error=False)), ): From 522ca4f3d31f7e694b38e3079deafb7cb01f6532 Mon Sep 17 00:00:00 2001 From: Jason Stone Date: Fri, 26 Jul 2024 14:47:44 +0300 Subject: [PATCH 05/38] checking if it works in my system --- runner/openapi.json | 31 +----- worker/runner.gen.go | 230 ++++++++++++++++++++++++++++++++++++++----- 2 files changed, 209 insertions(+), 52 deletions(-) diff --git a/runner/openapi.json b/runner/openapi.json index c103d2f6..2e2bb4e7 100644 --- a/runner/openapi.json +++ b/runner/openapi.json @@ -520,38 +520,17 @@ "title": "Model Id" }, "image1": { - "anyOf": [ - { - "type": "string", - "format": "binary" - }, - { - "type": "null" - } - ], + "type": "string", + "format": "binary", "title": "Image1" }, "image2": { - "anyOf": [ - { - "type": "string", - "format": "binary" - }, - { - "type": "null" - } - ], + "type": "string", + "format": "binary", "title": "Image2" }, "image_dir": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], + "type": "string", "title": "Image Dir", "default": "" }, diff --git a/worker/runner.gen.go b/worker/runner.gen.go index 0dbe8036..788a7e82 100644 --- a/worker/runner.gen.go +++ b/worker/runner.gen.go @@ -37,6 +37,15 @@ type BodyAudioToTextAudioToTextPost struct { ModelId *string `json:"model_id,omitempty"` } +// BodyFrameInterpolationFrameInterpolationPost defines model for Body_frame_interpolation_frame_interpolation_post. +type BodyFrameInterpolationFrameInterpolationPost struct { + Image1 *openapi_types.File `json:"image1,omitempty"` + Image2 *openapi_types.File `json:"image2,omitempty"` + ImageDir *string `json:"image_dir,omitempty"` + InterFrames *int `json:"inter_frames,omitempty"` + ModelId string `json:"model_id"` +} + // BodyImageToImageImageToImagePost defines model for Body_image_to_image_image_to_image_post. type BodyImageToImageImageToImagePost struct { GuidanceScale *float32 `json:"guidance_scale,omitempty"` @@ -155,6 +164,9 @@ type Chunk struct { // AudioToTextMultipartRequestBody defines body for AudioToText for multipart/form-data ContentType. type AudioToTextMultipartRequestBody = BodyAudioToTextAudioToTextPost +// FrameInterpolationMultipartRequestBody defines body for FrameInterpolation for multipart/form-data ContentType. +type FrameInterpolationMultipartRequestBody = BodyFrameInterpolationFrameInterpolationPost + // ImageToImageMultipartRequestBody defines body for ImageToImage for multipart/form-data ContentType. type ImageToImageMultipartRequestBody = BodyImageToImageImageToImagePost @@ -305,6 +317,9 @@ type ClientInterface interface { // AudioToTextWithBody request with any body AudioToTextWithBody(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) + // FrameInterpolationWithBody request with any body + FrameInterpolationWithBody(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) + // Health request Health(ctx context.Context, reqEditors ...RequestEditorFn) (*http.Response, error) @@ -335,6 +350,18 @@ func (c *Client) AudioToTextWithBody(ctx context.Context, contentType string, bo return c.Client.Do(req) } +func (c *Client) FrameInterpolationWithBody(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewFrameInterpolationRequestWithBody(c.Server, contentType, body) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + func (c *Client) Health(ctx context.Context, reqEditors ...RequestEditorFn) (*http.Response, error) { req, err := NewHealthRequest(c.Server) if err != nil { @@ -436,6 +463,35 @@ func NewAudioToTextRequestWithBody(server string, contentType string, body io.Re return req, nil } +// NewFrameInterpolationRequestWithBody generates requests for FrameInterpolation with any type of body +func NewFrameInterpolationRequestWithBody(server string, contentType string, body io.Reader) (*http.Request, error) { + var err error + + serverURL, err := url.Parse(server) + if err != nil { + return nil, err + } + + operationPath := fmt.Sprintf("/frame_interpolation") + if operationPath[0] == '/' { + operationPath = "." + operationPath + } + + queryURL, err := serverURL.Parse(operationPath) + if err != nil { + return nil, err + } + + req, err := http.NewRequest("POST", queryURL.String(), body) + if err != nil { + return nil, err + } + + req.Header.Add("Content-Type", contentType) + + return req, nil +} + // NewHealthRequest generates requests for Health func NewHealthRequest(server string) (*http.Request, error) { var err error @@ -636,6 +692,9 @@ type ClientWithResponsesInterface interface { // AudioToTextWithBodyWithResponse request with any body AudioToTextWithBodyWithResponse(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*AudioToTextResponse, error) + // FrameInterpolationWithBodyWithResponse request with any body + FrameInterpolationWithBodyWithResponse(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*FrameInterpolationResponse, error) + // HealthWithResponse request HealthWithResponse(ctx context.Context, reqEditors ...RequestEditorFn) (*HealthResponse, error) @@ -681,6 +740,32 @@ func (r AudioToTextResponse) StatusCode() int { return 0 } +type FrameInterpolationResponse struct { + Body []byte + HTTPResponse *http.Response + JSON200 *ImageResponse + JSON400 *HTTPError + JSON401 *HTTPError + JSON422 *HTTPValidationError + JSON500 *HTTPError +} + +// Status returns HTTPResponse.Status +func (r FrameInterpolationResponse) Status() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Status + } + return http.StatusText(0) +} + +// StatusCode returns HTTPResponse.StatusCode +func (r FrameInterpolationResponse) StatusCode() int { + if r.HTTPResponse != nil { + return r.HTTPResponse.StatusCode + } + return 0 +} + type HealthResponse struct { Body []byte HTTPResponse *http.Response @@ -816,6 +901,15 @@ func (c *ClientWithResponses) AudioToTextWithBodyWithResponse(ctx context.Contex return ParseAudioToTextResponse(rsp) } +// FrameInterpolationWithBodyWithResponse request with arbitrary body returning *FrameInterpolationResponse +func (c *ClientWithResponses) FrameInterpolationWithBodyWithResponse(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*FrameInterpolationResponse, error) { + rsp, err := c.FrameInterpolationWithBody(ctx, contentType, body, reqEditors...) + if err != nil { + return nil, err + } + return ParseFrameInterpolationResponse(rsp) +} + // HealthWithResponse request returning *HealthResponse func (c *ClientWithResponses) HealthWithResponse(ctx context.Context, reqEditors ...RequestEditorFn) (*HealthResponse, error) { rsp, err := c.Health(ctx, reqEditors...) @@ -930,6 +1024,60 @@ func ParseAudioToTextResponse(rsp *http.Response) (*AudioToTextResponse, error) return response, nil } +// ParseFrameInterpolationResponse parses an HTTP response from a FrameInterpolationWithResponse call +func ParseFrameInterpolationResponse(rsp *http.Response) (*FrameInterpolationResponse, error) { + bodyBytes, err := io.ReadAll(rsp.Body) + defer func() { _ = rsp.Body.Close() }() + if err != nil { + return nil, err + } + + response := &FrameInterpolationResponse{ + Body: bodyBytes, + HTTPResponse: rsp, + } + + switch { + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: + var dest ImageResponse + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON200 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 400: + var dest HTTPError + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON400 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 401: + var dest HTTPError + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON401 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 422: + var dest HTTPValidationError + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON422 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 500: + var dest HTTPError + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON500 = &dest + + } + + return response, nil +} + // ParseHealthResponse parses an HTTP response from a HealthWithResponse call func ParseHealthResponse(rsp *http.Response) (*HealthResponse, error) { bodyBytes, err := io.ReadAll(rsp.Body) @@ -1177,6 +1325,9 @@ type ServerInterface interface { // Audio To Text // (POST /audio-to-text) AudioToText(w http.ResponseWriter, r *http.Request) + // Frame Interpolation + // (POST /frame_interpolation) + FrameInterpolation(w http.ResponseWriter, r *http.Request) // Health // (GET /health) Health(w http.ResponseWriter, r *http.Request) @@ -1204,6 +1355,12 @@ func (_ Unimplemented) AudioToText(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusNotImplemented) } +// Frame Interpolation +// (POST /frame_interpolation) +func (_ Unimplemented) FrameInterpolation(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotImplemented) +} + // Health // (GET /health) func (_ Unimplemented) Health(w http.ResponseWriter, r *http.Request) { @@ -1260,6 +1417,23 @@ func (siw *ServerInterfaceWrapper) AudioToText(w http.ResponseWriter, r *http.Re handler.ServeHTTP(w, r.WithContext(ctx)) } +// FrameInterpolation operation middleware +func (siw *ServerInterfaceWrapper) FrameInterpolation(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + ctx = context.WithValue(ctx, HTTPBearerScopes, []string{}) + + handler := http.Handler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + siw.Handler.FrameInterpolation(w, r) + })) + + for _, middleware := range siw.HandlerMiddlewares { + handler = middleware(handler) + } + + handler.ServeHTTP(w, r.WithContext(ctx)) +} + // Health operation middleware func (siw *ServerInterfaceWrapper) Health(w http.ResponseWriter, r *http.Request) { ctx := r.Context() @@ -1459,6 +1633,9 @@ func HandlerWithOptions(si ServerInterface, options ChiServerOptions) http.Handl r.Group(func(r chi.Router) { r.Post(options.BaseURL+"/audio-to-text", wrapper.AudioToText) }) + r.Group(func(r chi.Router) { + r.Post(options.BaseURL+"/frame_interpolation", wrapper.FrameInterpolation) + }) r.Group(func(r chi.Router) { r.Get(options.BaseURL+"/health", wrapper.Health) }) @@ -1481,32 +1658,33 @@ func HandlerWithOptions(si ServerInterface, options ChiServerOptions) http.Handl // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+xZ227bOBN+FYL/f+nEhzabhe+SbLcNtoegdrsXRWAw0thmK5FaHtJ6A7/7gkNZomQp", - "cpDGC2R9Zcsaznxz+IZD+o5GMs2kAGE0Hd9RHS0hZfj17OrylVJSue+ZkhkowwHfpHrhPgw3CdAxfacX", - "tEfNKnMP2iguFnS97lEFf1muIKbjL7jkulcsKXQX6+TNV4gMXffouYxXM2ZjLmdGzgz8MLWnTGqzDQpl", - "3Je5VCkzdExvuGBqRQOrKLIFtUdTGUMy47FbHsOc2cStD1a+cwLkMu7006MIPN3Nm7Yw8JQtwIn6L7XH", - "5kAsLI+ZiGCmI+YgBC6dHp+UyF7ncmSCcgUEYdMbUA4CWrk/pJco0hBSj/AeLMMQC6oh3YgekageFbBg", - "ht/CLFMyzUyrjve5HLnyck2qbOpzoGcZqCaFw0CfTQk6qMkVqC2tXBhYePdQrZiDAoyZgUxXlQ4GNbUb", - "YTJB4SalJbjNyna/NJuDWc2iJUTfKpaNslCanqAYuUCxQs2NlAkwgXoA4tDixD03gdNGgViYZcXY4PjX", - "wNZGYqscatTLNl75sq1zcAcqdbLwlscg64/NLJzXUvdLCef3lkQtgS+W1So6OQ3WvfHvm5Y+hqmP4lQq", - "DZdidmOjb2DqSoaj01CLkyTnKFnRFhJAcg0zZhezlsIYjAICOGFyZhekvUa6OTU6eTil9k6T7zyuhWI4", - "GL0sLf2J77dX1ijSwYz28m5jhs2wsRefzVz416qzK/enJ8+qnT6sITbmriHRb6bTq5ZBMAbDeOK+/V/B", - "nI7p//rlONnPZ8l+MezVAebLA2ClrRYgn1nCY+Y6SSckbiDVXdjq+tYllt+8pgIIU4qt0IcQbV1BE25g", - "iVlebIqgilcbZmy1KumHP2i4/6FA0+BZbgylgQb7yK2PoDMpNLSwU+8csXcQcxbGyY82TXHaaj06zHUV", - "VgNub2kLr9Dz7yEZ3rvnR3VXq5JQ7pNKOud8izLaa0REgWceeINHU/hh2hMRLa34tnsiUDxMxIVfX09E", - "j7pzRuigg9HpofFCOajAu4oTLU5OJWb3iinmHXmqI0o5M+0wJf3HTw8nz+3wUExFDxyDcqdqNV2t2YbC", - "7tx7EhlV2MvE6sOcjr/cbcXqbgvidUDktzJCMw1Url+9gNYtg5P/oRRFzGTqfu2ivvPDm8olg0jtsN99", - "dnNje5ubK5bW9psHbjz19rY5V3nFHRtRbj50qYK3wSHfabcc2a2tOjspaMPSLHQ1wD0t3ndAN6GgMxY4", - "4TFugUc6RVZxs5q4OHrkbnA5B6ZAFXd+yEH/U6FkaUxG104HF3PpKa0jxTMszjE9E4RlWcJ9tRIjibKC", - "nF2SjGeQcOGTsSlqfgsZgHLvP1oh0NAtKO11DY6HxwMXLZmBYBmnY/oCf+rRjJklwu7jzdmRkUeb0G/O", - "Gy4tCOIy3tzzTWWeDxdB0MbNvLjLSmFA4KrUJoZnTJm+O5gcxcyw8g60qxx3u9hbV3PoOiH+4IsNvRoN", - "BjVcQVD7X7ULz66gKnsz2q5mbGKjCLSe24SUYj368idCKEf4BvvnLCYffT683eF+7H4SzJqlVPxviNHw", - "8MV+DOfOklfCcLMiUynJW6YWPuqj0U8FsXWW2YZTipDivHOyr+RfCgNKsIRMQN2CIuWhcNOicK8Mm9OX", - "6/V1j2qbpkytNswmU0mQ225pf4mHH5wqoaEX+LMRfULOhaevXSm3Dp3KIaI3OBa6DlfcmTS3OBxV8onl", - "iXvcDhene+5y1ZPjoc21t7lDh3loh/H/RE2lP3PVSIk3op2kxHlyX6Rsv7PdMymrU/SBlAdSPgEpPbWQ", - "lG7G3mGjDE7291LycTN39e7gsB0emPdMmOeKu7Yb5v8XtVPuUy7wtDtg499XB+YdmPdMmLdh0dqvcmo0", - "LqpaKq7VLhJpY3Ih09QKblbkNTPwna1o/vcWXubpcb8fK2Dp0cK/PU7y5ceRW07X1+t/AgAA///2pVcb", - "EigAAA==", + "H4sIAAAAAAAC/+xZWW/bOhb+KwRnHp14aTMZ+C1Jt2C6BI3beSgCg5GObbYSqUtSaX0D//cLHsoSJVOR", + "jTS+QK6fvOgs31m+w0X3NJJpJgUIo+n4nupoASnDr2dXl6+Vksp+z5TMQBkO+CTVc/thuEmAjukHPac9", + "apaZ/aGN4mJOV6seVfBHzhXEdPwNVW56pUppu9STt98hMnTVo+cyXk5ZHnM5NXJq4Jdp/MqkNpugUMZ+", + "mUmVMkPH9JYLppbU84oiG1B7NJUxJFMeW/UYZixPrL6n+cEKkMu4M06Hwot0u2ja0jBTLIUpFwZUJhNm", + "uBTB/8Ip4Smbw/DhnFw6mUBSUHu0hfaoVXsac9WaU9Qlr7gKqtvwXKi6ZmHkGbAy5I2TKW1YzTmoZll3", + "rmSp3CzmTjVpK6xLj5HFl8bPcDnnOY+ZiGCqI2bheFk5PT6pUL4t5Mg1ypUQRJ7eusSgly0q217YB7AM", + "fSyuyN2IHsHAHhUwZ4bfwTRTMs1Mq42PhRy5cnIhU3nqaqCnGaiQwaFnL08JBqjJFagNq14nolkxAwWY", + "MwNZvauHg0HD7FqYXKNwyGgFbq3ZHpdmMzDLabSA6EfNs1E5VK6vUYxcoFhp5lbKBJhAOwA1Ol3b3yFw", + "2igQc7OoORsc/9fztZbYaIcGE7N1VK5tm3zcgkqdLLzjMcjmzzALZ43S/aeC86alUAvg80W9i05OPb13", + "7nlI9TFMfRSnUolD7DaPfoBpGhmOTn0rVpKco2TNmk8AyTVMWT6ftjTGwBvsH60wOcvnpL1Hujk1Otmd", + "UnunyU8eN1IxHIxeVp7+j883NRsU6WBGe3u3MSPPcLCXnw9sMP6O7uyq/enJsxqnuw3EYO0ChX43mVy1", + "7PBjMIwn9tu/FczomP6rX50T+sUhoV/u4psAC3UPWOWrBchXlvAYN06dkLiBVHdha9pbVVheOUslEKYU", + "W2IMPtqmgRBuYIlZXKyboI5XG2byelfST/+j/vqHAqF9aLUwVA4C/pFbn0FnUmhoYafeOmMfIObMz5Pb", + "2oTytDF6tF/rOqwAbudpA6/Qs58+GT7a34+arrlKfLkvKunc9ucoo51FRORF5oAHIprAL9NeiGiRix/b", + "FwLF/UJcOP1mIXrUHiD9AC2MzgiNEypAedHVgmgJciKxuldMMRfIUx1Rqj3TFrukf/jp4eS5HR7KXdGO", + "26AiqEZP13s20Nida08ioxp7mVh+mtHxt/uNXN1vQLzxiPxeRugmQOXmnRpo3bJxcn9UooiZTOy/XdS3", + "cThXhaSXqS3Wu69239g+5qrbmjJROy48zfG2Plc1rnjCC1Hh3g+phjcQkJu0G4FsN1atnxS0YWnmh+rh", + "npTPO6AbX9A684JwGDfAI52iXHGzvLZ5dMjtxuUcmAJVXuYiB91fpZGFMRldrfCebSYdpXWkeIbNOaZn", + "grAsS7jrVmIkUbkgZ5ck4xkkXLhirJua30EGoOzzz7kQ6OgOlHa2BsfD44HNlsxAsIzTMX2Bf/VoxswC", + "YffxSvTIyKN16tfnDVsWBHEZry9wJ7Koh80gaGP3vLjKSmFAoFaaJ4ZnTJm+PZgcxcyw6nK7qx23u7Fd", + "1WtoJyH+4ZoNoxoNBg1cXlL737VNz7agamsz+q5X7DqPItB6liekEuvRl78RQrWFD/g/ZzH57Orh/A73", + "4/eLYLlZSMX/hBgdD1/sx3ERLHktDDdLMpGSvGdq7rI+Gv1WEBtnmU04lQgpzzsn+yo+XsQLlpBrUHeg", + "SHUoXI8oXCv94fTtZnXTozpPU6aWa2aTiSTIbavaD9ypt08GXCIua7JPOyB2eguw52FRP4AdpkX7tDgQ", + "dVeiItFInWlI1wXeVeAhEAIEdVcZ9Am73r8s2bbnV35oBUSMBk9xdkNSXnGG5w5SrThgPPHE2eI9x2HO", + "HObMM5kz7sXxRLorkgYp8QVGJynx+LcvUra/YtkzKeuH3gMpD6R8AlI6aiEp7ZF4i4XSu4h7kJKPOyLX", + "r/oOy+GBec+Eeba5G6th8Xq3nXJfCoGnXQGDb5sPzDsw75kwb82ildOyZjQq1T2Vt+AXicxjciHTNBfc", + "LMlbZuAnW9LibTTevetxvx8rYOnR3D09Tgr148iq09XN6q8AAAD///BmUXaaLQAA", } // GetSwagger returns the content of the embedded swagger specification file From c1b5ca14b94e458e50d064d829b5775eb4184ce1 Mon Sep 17 00:00:00 2001 From: Jason <83615043+JJassonn69@users.noreply.github.com> Date: Fri, 26 Jul 2024 14:53:33 +0300 Subject: [PATCH 06/38] Create docker-image.yml --- .github/workflows/docker-image.yml | 77 ++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 .github/workflows/docker-image.yml diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml new file mode 100644 index 00000000..3bf96140 --- /dev/null +++ b/.github/workflows/docker-image.yml @@ -0,0 +1,77 @@ +name: Build Docker image for ai-runner + +on: + pull_request: + paths: + - "runner/**" + - "!runner/.devcontainer/**" + push: + branches: + - main + tags: + - '*' + paths: + - "runner/**" + - "!runner/.devcontainer/**" + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + docker: + name: Docker image generation + if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository + permissions: + packages: write + contents: read + runs-on: ubuntu-20.04 + steps: + - name: Check out code + uses: actions/checkout@v4.1.1 + with: + fetch-depth: 0 + # Check https://github.com/livepeer/go-livepeer/pull/1891 + # for ref value discussion + ref: ${{ github.event.pull_request.head.sha }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ secrets.CI_DOCKERHUB_USERNAME }} + password: ${{ secrets.CI_DOCKERHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v5 + with: + images: | + livepeer/ai-runner + tags: | + type=sha + type=ref,event=pr + type=ref,event=tag + type=sha,format=long + type=ref,event=branch + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{version}},prefix=v + type=semver,pattern={{major}}.{{minor}},prefix=v + type=raw,value=latest,enable={{is_default_branch}} + type=raw,value=${{ github.event.pull_request.head.ref }} + type=raw,value=stable,enable=${{ startsWith(github.event.ref, 'refs/tags/v') }} + + - name: Build and push runner docker image + uses: docker/build-push-action@v5 + with: + context: "{{defaultContext}}:runner" + platforms: linux/amd64 + push: true + tags: ${{ steps.meta.outputs.tags }} + file: "Dockerfile" + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=registry,ref=livepeerci/build:cache + cache-to: type=registry,ref=livepeerci/build:cache,mode=max From c9bf8d2f097b39d151798b87aac17d2b5b9a7c05 Mon Sep 17 00:00:00 2001 From: Jason <83615043+JJassonn69@users.noreply.github.com> Date: Fri, 26 Jul 2024 14:58:30 +0300 Subject: [PATCH 07/38] Delete .github/workflows/docker-image.yml testing workflow --- .github/workflows/docker-image.yml | 77 ------------------------------ 1 file changed, 77 deletions(-) delete mode 100644 .github/workflows/docker-image.yml diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml deleted file mode 100644 index 3bf96140..00000000 --- a/.github/workflows/docker-image.yml +++ /dev/null @@ -1,77 +0,0 @@ -name: Build Docker image for ai-runner - -on: - pull_request: - paths: - - "runner/**" - - "!runner/.devcontainer/**" - push: - branches: - - main - tags: - - '*' - paths: - - "runner/**" - - "!runner/.devcontainer/**" - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -jobs: - docker: - name: Docker image generation - if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository - permissions: - packages: write - contents: read - runs-on: ubuntu-20.04 - steps: - - name: Check out code - uses: actions/checkout@v4.1.1 - with: - fetch-depth: 0 - # Check https://github.com/livepeer/go-livepeer/pull/1891 - # for ref value discussion - ref: ${{ github.event.pull_request.head.sha }} - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to DockerHub - uses: docker/login-action@v3 - with: - username: ${{ secrets.CI_DOCKERHUB_USERNAME }} - password: ${{ secrets.CI_DOCKERHUB_TOKEN }} - - - name: Extract metadata (tags, labels) for Docker - id: meta - uses: docker/metadata-action@v5 - with: - images: | - livepeer/ai-runner - tags: | - type=sha - type=ref,event=pr - type=ref,event=tag - type=sha,format=long - type=ref,event=branch - type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} - type=semver,pattern={{version}},prefix=v - type=semver,pattern={{major}}.{{minor}},prefix=v - type=raw,value=latest,enable={{is_default_branch}} - type=raw,value=${{ github.event.pull_request.head.ref }} - type=raw,value=stable,enable=${{ startsWith(github.event.ref, 'refs/tags/v') }} - - - name: Build and push runner docker image - uses: docker/build-push-action@v5 - with: - context: "{{defaultContext}}:runner" - platforms: linux/amd64 - push: true - tags: ${{ steps.meta.outputs.tags }} - file: "Dockerfile" - labels: ${{ steps.meta.outputs.labels }} - cache-from: type=registry,ref=livepeerci/build:cache - cache-to: type=registry,ref=livepeerci/build:cache,mode=max From d1b5d3cd00c2a3ff2d1d5fe527689bcd3b8fe672 Mon Sep 17 00:00:00 2001 From: Jason <83615043+JJassonn69@users.noreply.github.com> Date: Fri, 26 Jul 2024 15:04:59 +0300 Subject: [PATCH 08/38] Create validate-openapi-on-push.yaml --- .../workflows/validate-openapi-on-push.yaml | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 .github/workflows/validate-openapi-on-push.yaml diff --git a/.github/workflows/validate-openapi-on-push.yaml b/.github/workflows/validate-openapi-on-push.yaml new file mode 100644 index 00000000..4809d94c --- /dev/null +++ b/.github/workflows/validate-openapi-on-push.yaml @@ -0,0 +1,44 @@ +name: Check OpenAPI spec and Golang bindings + +on: + push: + +jobs: + check-openapi-and-bindings: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + cache: "pip" + + - name: Install Python dependencies + run: | + python -m pip install --upgrade pip + pip install -r runner/requirements.txt + + - name: Generate AI OpenAPI specification + working-directory: runner + run: | + python gen_openapi.py + + - name: Check for OpenAPI spec changes + run: | + if ! git diff --exit-code; then + echo "::error:: OpenAPI spec has changed. Please run 'python gen_openapi.py' in the 'runner' directory and commit the changes." + exit 1 + fi + + - name: Generate Go bindings + run: make + + - name: Check for Go bindings changes + run: | + if ! git diff --exit-code; then + echo "::error::Go bindings have changed. Please run 'make' at the root of the repository and commit the changes." + exit 1 + fi From 8f82e52bbfb35317420fa9a7e25a49bf89582c25 Mon Sep 17 00:00:00 2001 From: Jason <83615043+JJassonn69@users.noreply.github.com> Date: Fri, 26 Jul 2024 15:12:14 +0300 Subject: [PATCH 09/38] Create docker-create-ai-runner.yaml --- .../workflows/docker-create-ai-runner.yaml | 77 +++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 .github/workflows/docker-create-ai-runner.yaml diff --git a/.github/workflows/docker-create-ai-runner.yaml b/.github/workflows/docker-create-ai-runner.yaml new file mode 100644 index 00000000..3bf96140 --- /dev/null +++ b/.github/workflows/docker-create-ai-runner.yaml @@ -0,0 +1,77 @@ +name: Build Docker image for ai-runner + +on: + pull_request: + paths: + - "runner/**" + - "!runner/.devcontainer/**" + push: + branches: + - main + tags: + - '*' + paths: + - "runner/**" + - "!runner/.devcontainer/**" + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + docker: + name: Docker image generation + if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository + permissions: + packages: write + contents: read + runs-on: ubuntu-20.04 + steps: + - name: Check out code + uses: actions/checkout@v4.1.1 + with: + fetch-depth: 0 + # Check https://github.com/livepeer/go-livepeer/pull/1891 + # for ref value discussion + ref: ${{ github.event.pull_request.head.sha }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ secrets.CI_DOCKERHUB_USERNAME }} + password: ${{ secrets.CI_DOCKERHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v5 + with: + images: | + livepeer/ai-runner + tags: | + type=sha + type=ref,event=pr + type=ref,event=tag + type=sha,format=long + type=ref,event=branch + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{version}},prefix=v + type=semver,pattern={{major}}.{{minor}},prefix=v + type=raw,value=latest,enable={{is_default_branch}} + type=raw,value=${{ github.event.pull_request.head.ref }} + type=raw,value=stable,enable=${{ startsWith(github.event.ref, 'refs/tags/v') }} + + - name: Build and push runner docker image + uses: docker/build-push-action@v5 + with: + context: "{{defaultContext}}:runner" + platforms: linux/amd64 + push: true + tags: ${{ steps.meta.outputs.tags }} + file: "Dockerfile" + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=registry,ref=livepeerci/build:cache + cache-to: type=registry,ref=livepeerci/build:cache,mode=max From 6d1d32acb0946c74197ad7ce612c6614ee7d2d49 Mon Sep 17 00:00:00 2001 From: Jason <83615043+JJassonn69@users.noreply.github.com> Date: Fri, 26 Jul 2024 15:21:34 +0300 Subject: [PATCH 10/38] Update docker-create-ai-runner.yaml --- .github/workflows/docker-create-ai-runner.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-create-ai-runner.yaml b/.github/workflows/docker-create-ai-runner.yaml index 3bf96140..fb7feafb 100644 --- a/.github/workflows/docker-create-ai-runner.yaml +++ b/.github/workflows/docker-create-ai-runner.yaml @@ -1,7 +1,7 @@ name: Build Docker image for ai-runner on: - pull_request: + member: paths: - "runner/**" - "!runner/.devcontainer/**" From 4812c2ab53816f3e5f17de1d24fe869eed9e2f7f Mon Sep 17 00:00:00 2001 From: Jason <83615043+JJassonn69@users.noreply.github.com> Date: Fri, 26 Jul 2024 16:07:59 +0300 Subject: [PATCH 11/38] Frame interpolation (#5) * update to the frame interpolation pipeline, there is some minor issue with creating go api bindings because of openapi json sceme having a null option. * minor changes to requirements * update to requrements to fetch from --index-url * simple patch to solve the go api bindings issue * checking if it works in my system --------- Co-authored-by: Jason Stone --- runner/app/main.py | 18 +- runner/app/pipelines/frame_interpolation.py | 114 +++++++++- runner/app/pipelines/image_to_video.py | 6 +- runner/app/pipelines/upscale.py | 4 +- runner/app/pipelines/utils/__init__.py | 4 + runner/app/pipelines/utils/utils.py | 160 ++++++++++++++ runner/app/routes/frame_interpolation.py | 120 ++++++++++ runner/dl_checkpoints.sh | 3 + runner/gen_openapi.py | 9 +- runner/openapi.json | 130 ++++++++++- runner/requirements.txt | 7 +- worker/runner.gen.go | 230 +++++++++++++++++--- 12 files changed, 752 insertions(+), 53 deletions(-) create mode 100644 runner/app/routes/frame_interpolation.py diff --git a/runner/app/main.py b/runner/app/main.py index 6f511420..c4dd6b8e 100644 --- a/runner/app/main.py +++ b/runner/app/main.py @@ -1,5 +1,7 @@ import logging import os +import sys +import cv2 from contextlib import asynccontextmanager from app.routes import health @@ -15,8 +17,8 @@ async def lifespan(app: FastAPI): app.include_router(health.router) - pipeline = os.environ["PIPELINE"] - model_id = os.environ["MODEL_ID"] + pipeline = os.environ.get("PIPELINE", "") # Default to + model_id = os.environ.get("MODEL_ID", "") # Provide a default if necessary app.pipeline = load_pipeline(pipeline, model_id) app.include_router(load_route(pipeline)) @@ -46,8 +48,10 @@ def load_pipeline(pipeline: str, model_id: str) -> any: from app.pipelines.audio_to_text import AudioToTextPipeline return AudioToTextPipeline(model_id) - case "frame-interpolation": - raise NotImplementedError("frame-interpolation pipeline not implemented") + case "FILMPipeline": + from app.pipelines.frame_interpolation import FILMPipeline + + return FILMPipeline(model_id) case "upscale": from app.pipelines.upscale import UpscalePipeline @@ -76,8 +80,10 @@ def load_route(pipeline: str) -> any: from app.routes import audio_to_text return audio_to_text.router - case "frame-interpolation": - raise NotImplementedError("frame-interpolation pipeline not implemented") + case "FILMPipeline": + from app.routes import frame_interpolation + + return frame_interpolation.router case "upscale": from app.routes import upscale diff --git a/runner/app/pipelines/frame_interpolation.py b/runner/app/pipelines/frame_interpolation.py index 396c8067..755afee7 100644 --- a/runner/app/pipelines/frame_interpolation.py +++ b/runner/app/pipelines/frame_interpolation.py @@ -1,5 +1,113 @@ -from app.pipelines.base import Pipeline +import torch +from torchvision.transforms import v2 +from tqdm import tqdm +import bisect +import numpy as np +from app.pipelines.utils.utils import get_model_dir -class FrameInterpolationPipeline(Pipeline): - pass +class FILMPipeline: + model: torch.jit.ScriptModule + + def __init__(self, model_id: str): + self.model_id = model_id + model_dir = get_model_dir() # Get the directory where models are stored + model_path = f"{model_dir}/{model_id}" # Construct the full path to the model file + + self.model = torch.jit.load(model_path, map_location="cpu") + self.model.eval() + + def to(self, *args, **kwargs): + self.model = self.model.to(*args, **kwargs) + return self + + @property + def device(self) -> torch.device: + # Checking device for ScriptModule requires checking one of its parameters + params = self.model.parameters() + return next(params).device + + @property + def dtype(self) -> torch.dtype: + # Checking device for ScriptModule requires checking one of its parameters + params = self.model.parameters() + return next(params).dtype + + def __call__( + self, + reader, + writer, + inter_frames: int = 2, + ): + transforms = v2.Compose( + [ + v2.ToDtype(torch.uint8, scale=True), + ] + ) + + writer.open() + + while True: + frame_1 = reader.get_frame() + # If the first frame read is None then there are no more frames + if frame_1 is None: + break + + frame_2 = reader.get_frame() + # If the second frame read is None there there is a final frame + if frame_2 is None: + writer.write_frame(transforms(frame_1)) + break + + # frame_1 and frame_2 must be tensors with n c h w format + frame_1 = frame_1.unsqueeze(0) + frame_2 = frame_2.unsqueeze(0) + + frames = inference( + self.model, frame_1, frame_2, inter_frames, self.device, self.dtype + ) + + frames = [transforms(frame.detach().cpu()) for frame in frames] + for frame in frames: + writer.write_frame(frame) + + writer.close() + + +def inference( + model, img_batch_1, img_batch_2, inter_frames, device, dtype +) -> torch.Tensor: + results = [img_batch_1, img_batch_2] + + idxes = [0, inter_frames + 1] + remains = list(range(1, inter_frames + 1)) + + splits = torch.linspace(0, 1, inter_frames + 2) + + for _ in tqdm(range(len(remains)), "Generating in-between frames"): + starts = splits[idxes[:-1]] + ends = splits[idxes[1:]] + distances = ( + (splits[None, remains] - starts[:, None]) + / (ends[:, None] - starts[:, None]) + - 0.5 + ).abs() + matrix = torch.argmin(distances).item() + start_i, step = np.unravel_index(matrix, distances.shape) + end_i = start_i + 1 + + x0 = results[start_i].to(device=device, dtype=dtype) + x1 = results[end_i].to(device=device, dtype=dtype) + + dt = x0.new_full((1, 1), (splits[remains[step]] - splits[idxes[start_i]])) / ( + splits[idxes[end_i]] - splits[idxes[start_i]] + ) + + with torch.no_grad(): + prediction = model(x0, x1, dt) + insert_position = bisect.bisect_left(idxes, remains[step]) + idxes.insert(insert_position, remains[step]) + results.insert(insert_position, prediction.clamp(0, 1).float()) + del remains[step] + + return results \ No newline at end of file diff --git a/runner/app/pipelines/image_to_video.py b/runner/app/pipelines/image_to_video.py index f605cb2f..988c075b 100644 --- a/runner/app/pipelines/image_to_video.py +++ b/runner/app/pipelines/image_to_video.py @@ -6,10 +6,14 @@ import PIL import torch from app.pipelines.base import Pipeline -from app.pipelines.utils import SafetyChecker, get_model_dir, get_torch_device from diffusers import StableVideoDiffusionPipeline from huggingface_hub import file_download from PIL import ImageFile +from app.pipelines.utils import ( + SafetyChecker, + get_model_dir, + get_torch_device +) ImageFile.LOAD_TRUNCATED_IMAGES = True diff --git a/runner/app/pipelines/upscale.py b/runner/app/pipelines/upscale.py index 0cba865e..6fd3cefe 100644 --- a/runner/app/pipelines/upscale.py +++ b/runner/app/pipelines/upscale.py @@ -87,7 +87,7 @@ def __init__(self, model_id: str): elif deepcache_enabled: logger.warning( "DeepCache is not supported for Lightning or Turbo models. " - "TextToImagePipeline will NOT be optimized with DeepCache for %s", + "UpscalingPiepline will NOT be optimized with DeepCache for %s", model_id, ) @@ -112,7 +112,7 @@ def __call__( ] if num_inference_steps is None or num_inference_steps < 1: - del kwargs["num_inference_steps"] + kwargs.pop("num_inference_steps", None) output = self.ldm(prompt, image=image, **kwargs) diff --git a/runner/app/pipelines/utils/__init__.py b/runner/app/pipelines/utils/__init__.py index 844b86e9..2c083d20 100644 --- a/runner/app/pipelines/utils/__init__.py +++ b/runner/app/pipelines/utils/__init__.py @@ -9,4 +9,8 @@ is_turbo_model, split_prompt, validate_torch_device, + frames_compactor, + video_shredder, + DirectoryReader, + DirectoryWriter ) diff --git a/runner/app/pipelines/utils/utils.py b/runner/app/pipelines/utils/utils.py index dbc44d48..5c9ad6a5 100644 --- a/runner/app/pipelines/utils/utils.py +++ b/runner/app/pipelines/utils/utils.py @@ -5,11 +5,18 @@ import re from pathlib import Path from typing import Optional +import glob +import tempfile +from io import BytesIO +from typing import List, Union import numpy as np import torch from diffusers.pipelines.stable_diffusion import StableDiffusionSafetyChecker from PIL import Image +from torchvision.transforms import v2 +import cv2 +from torchaudio.io import StreamWriter from torch import dtype as TorchDtype from transformers import CLIPFeatureExtractor @@ -111,6 +118,108 @@ def split_prompt( return prompt_dict +def frames_compactor( + frames: Union[List[np.ndarray], List[torch.Tensor]], + output_path: str, + fps: float, + codec: str = "MJPEG", + is_directory: bool = False, + width: int = None, + height: int = None +) -> None: + """ + Generate a video from a list of frames. Frames can be from a directory or in-memory. + + Args: + frames (List[np.ndarray] | List[torch.Tensor]): List of frames as NumPy arrays or PyTorch tensors. + output_path (str): Path to save the output video file. + fps (float): Frames per second for the video. + codec (str): Codec used for video compression (default is "XVID"). + is_directory (bool): If True, treat `frames` as a directory path containing image files. + width (int): Width of the video. Must be provided if `frames` are in-memory. + height (int): Height of the video. Must be provided if `frames` are in-memory. + + Returns: + None + """ + if is_directory: + # Read frames from a directory + frames = [cv2.imread(os.path.join(frames, file)) for file in sorted(os.listdir(frames))] + else: + # Convert torch tensors to numpy arrays if necessary + if isinstance(frames[0], torch.Tensor): + frames = [frame.permute(1, 2, 0).cpu().numpy() for frame in frames] + + # Ensure frames are numpy arrays and are uint8 type + frames = [frame.astype(np.uint8) for frame in frames] + + # Check if frames are consistent + if not frames: + raise ValueError("No frames to process.") + + if width is None or height is None: + # Use dimensions of the first frame if not provided + height, width = frames[0].shape[:2] + + # Define the codec and create VideoWriter object + fourcc = cv2.VideoWriter_fourcc(*codec) + video_writer = cv2.VideoWriter(output_path, fourcc, fps, (width, height)) + + # Write frames to the video file + for frame in frames: + # Ensure each frame has the correct size + if frame.shape[1] != width or frame.shape[0] != height: + frame = cv2.resize(frame, (width, height)) + video_writer.write(frame) + + # Release the video writer + video_writer.release() + +def video_shredder(video_data, is_file_path=True) -> np.ndarray: + """ + Extract frames from a video file or in-memory video data and return them as a NumPy array. + + Args: + video_data (str or BytesIO): Path to the input video file or in-memory video data. + is_file_path (bool): Indicates if video_data is a file path (True) or in-memory data (False). + + Returns: + np.ndarray: Array of frames with shape (num_frames, height, width, channels). + """ + if is_file_path: + # Handle file-based video input + video_capture = cv2.VideoCapture(video_data) + else: + # Handle in-memory video input + # Create a temporary file to store in-memory video data + with tempfile.NamedTemporaryFile(delete=False, suffix='.mp4') as temp_file: + temp_file.write(video_data.getvalue()) + temp_file_path = temp_file.name + + # Open the temporary video file + video_capture = cv2.VideoCapture(temp_file_path) + + if not video_capture.isOpened(): + raise ValueError("Error opening video data") + + frames = [] + success, frame = video_capture.read() + + while success: + frames.append(frame) + success, frame = video_capture.read() + + video_capture.release() + + # Delete the temporary file if it was created + if not is_file_path: + os.remove(temp_file_path) + + # Convert list of frames to a NumPy array + frames_array = np.array(frames) + print(f"Extracted {frames_array.shape[0]} frames from video in shape of {frames_array.shape}") + + return frames_array class SafetyChecker: """Checks images for unsafe or inappropriate content using a pretrained model. @@ -167,3 +276,54 @@ def check_nsfw_images( clip_input=safety_checker_input.pixel_values.to(self._dtype), ) return images, has_nsfw_concept + +class DirectoryReader: + def __init__(self, dir: str): + self.paths = sorted( + glob.glob(os.path.join(dir, "*")), + key=lambda x: int(os.path.basename(x).split(".")[0]), + ) + self.nb_frames = len(self.paths) + self.idx = 0 + + assert self.nb_frames > 0, "no frames found in directory" + + first_img = Image.open(self.paths[0]) + self.height = first_img.height + self.width = first_img.width + + def get_resolution(self): + return self.height, self.width + + def reset(self): + self.idx = 0 # Reset the index counter to 0 + + def get_frame(self): + if self.idx >= self.nb_frames: + return None + + path = self.paths[self.idx] + self.idx += 1 + + img = Image.open(path) + transforms = v2.Compose([v2.ToImage(), v2.ToDtype(torch.float32, scale=True)]) + + return transforms(img) + +class DirectoryWriter: + def __init__(self, dir: str): + self.dir = dir + self.idx = 0 + + def open(self): + return + + def close(self): + return + + def write_frame(self, frame: torch.Tensor): + path = f"{self.dir}/{self.idx}.png" + self.idx += 1 + + transforms = v2.Compose([v2.ToPILImage()]) + transforms(frame.squeeze(0)).save(path) \ No newline at end of file diff --git a/runner/app/routes/frame_interpolation.py b/runner/app/routes/frame_interpolation.py new file mode 100644 index 00000000..6abf6a66 --- /dev/null +++ b/runner/app/routes/frame_interpolation.py @@ -0,0 +1,120 @@ +# app/routes/film_interpolate.py + +import logging +import os +import torch +import glob +from typing import Annotated, Optional +from fastapi import APIRouter, Depends, File, Form, UploadFile, status +from fastapi.responses import JSONResponse +from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer +from PIL import Image, ImageFile + +from app.dependencies import get_pipeline +from app.pipelines.frame_interpolation import FILMPipeline +from app.pipelines.utils.utils import DirectoryReader, DirectoryWriter, get_torch_device, get_model_dir +from app.routes.util import HTTPError, ImageResponse, http_error, image_to_data_url + +ImageFile.LOAD_TRUNCATED_IMAGES = True + +router = APIRouter() + +logger = logging.getLogger(__name__) + +RESPONSES = { + status.HTTP_400_BAD_REQUEST: {"model": HTTPError}, + status.HTTP_401_UNAUTHORIZED: {"model": HTTPError}, + status.HTTP_500_INTERNAL_SERVER_ERROR: {"model": HTTPError}, +} + +@router.post("/frame_interpolation", response_model=ImageResponse, responses=RESPONSES) +@router.post( + "/frame_interpolation/", + response_model=ImageResponse, + responses=RESPONSES, + include_in_schema=False, +) +async def frame_interpolation( + model_id: Annotated[str, Form()], + image1: Annotated[UploadFile, File()]=None, + image2: Annotated[UploadFile, File()]=None, + image_dir: Annotated[str, Form()]="", + inter_frames: Annotated[int, Form()] = 2, + token: HTTPAuthorizationCredentials = Depends(HTTPBearer(auto_error=False)), +): + auth_token = os.environ.get("AUTH_TOKEN") + if auth_token: + if not token or token.credentials != auth_token: + return JSONResponse( + status_code=status.HTTP_401_UNAUTHORIZED, + headers={"WWW-Authenticate": "Bearer"}, + content=http_error("Invalid bearer token"), + ) + + + # Initialize FILMPipeline + film_pipeline = FILMPipeline(model_id) + film_pipeline.to(device=get_torch_device(),dtype=torch.float16) + + # Prepare directories for input and output + temp_input_dir = "temp_input" + temp_output_dir = "temp_output" + os.makedirs(temp_input_dir, exist_ok=True) + os.makedirs(temp_output_dir, exist_ok=True) + + try: + if os.path.isdir(image_dir): + if image1 and image2: + logger.info("Both directory and individual images provided. Directory will be used, and images will be ignored.") + reader = DirectoryReader(image_dir) + else: + if not (image1 and image2): + return JSONResponse( + status_code=status.HTTP_400_BAD_REQUEST, + content=http_error("Either a directory or two images must be provided."), + ) + + image1_path = os.path.join(temp_input_dir, "0.png") + image2_path = os.path.join(temp_input_dir, "1.png") + + with open(image1_path, "wb") as f: + f.write(await image1.read()) + with open(image2_path, "wb") as f: + f.write(await image2.read()) + + reader = DirectoryReader(temp_input_dir) + + writer = DirectoryWriter(temp_output_dir) + # Perform interpolation + film_pipeline(reader, writer, inter_frames=inter_frames) + + writer.close() + reader.reset() + + # Collect output frames + output_frames = [] + for frame_path in sorted(glob.glob(os.path.join(temp_output_dir, "*.png"))): + frame = Image.open(frame_path) + output_frames.append(frame) + + output_images = [{"url": image_to_data_url(frame),"seed":0, "nsfw":False} for frame in output_frames] + + except Exception as e: + logger.error(f"FILMPipeline error: {e}") + logger.exception(e) + return JSONResponse( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + content=http_error("FILMPipeline error"), + ) + + finally: + # Clean up temporary directories + for file_path in glob.glob(os.path.join(temp_input_dir, "*")): + os.remove(file_path) + os.rmdir(temp_input_dir) + + for file_path in glob.glob(os.path.join(temp_output_dir, "*")): + os.remove(file_path) + os.rmdir(temp_output_dir) + + return {"images": output_images} diff --git a/runner/dl_checkpoints.sh b/runner/dl_checkpoints.sh index 822590d4..1528a60b 100755 --- a/runner/dl_checkpoints.sh +++ b/runner/dl_checkpoints.sh @@ -59,6 +59,9 @@ function download_all_models() { # Download image-to-video models. huggingface-cli download stabilityai/stable-video-diffusion-img2vid-xt --include "*.fp16.safetensors" "*.json" --cache-dir models + + #Download frame-interpolation model. + wget -O models/film_net_fp16.pt https://github.com/dajes/frame-interpolation-pytorch/releases/download/v1.0.2/film_net_fp16.pt } # Enable HF transfer acceleration. diff --git a/runner/gen_openapi.py b/runner/gen_openapi.py index 7fde5ee3..198102db 100644 --- a/runner/gen_openapi.py +++ b/runner/gen_openapi.py @@ -11,6 +11,7 @@ image_to_image, image_to_video, text_to_image, + frame_interpolation, upscale, ) from fastapi.openapi.utils import get_openapi @@ -68,7 +69,7 @@ def translate_to_gateway(openapi): openapi["components"]["schemas"]["VideoResponse"]["title"] = "VideoResponse" return openapi - + def write_openapi(fname, entrypoint="runner"): """Write OpenAPI schema to file. @@ -83,8 +84,10 @@ def write_openapi(fname, entrypoint="runner"): app.include_router(text_to_image.router) app.include_router(image_to_image.router) app.include_router(image_to_video.router) - app.include_router(upscale.router) app.include_router(audio_to_text.router) + app.include_router(frame_interpolation.router) + app.include_router(upscale.router) + use_route_names_as_operation_ids(app) @@ -144,3 +147,5 @@ def write_openapi(fname, entrypoint="runner"): args = parser.parse_args() write_openapi(f"openapi.{args.type.lower()}", args.entrypoint) + + diff --git a/runner/openapi.json b/runner/openapi.json index 4345e565..2e2bb4e7 100644 --- a/runner/openapi.json +++ b/runner/openapi.json @@ -249,15 +249,15 @@ ] } }, - "/upscale": { + "/audio-to-text": { "post": { - "summary": "Upscale", - "operationId": "upscale", + "summary": "Audio To Text", + "operationId": "audio_to_text", "requestBody": { "content": { "multipart/form-data": { "schema": { - "$ref": "#/components/schemas/Body_upscale_upscale_post" + "$ref": "#/components/schemas/Body_audio_to_text_audio_to_text_post" } } }, @@ -269,7 +269,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ImageResponse" + "$ref": "#/components/schemas/TextResponse" } } } @@ -294,6 +294,16 @@ } } }, + "413": { + "description": "Request Entity Too Large", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPError" + } + } + } + }, "500": { "description": "Internal Server Error", "content": { @@ -322,15 +332,15 @@ ] } }, - "/audio-to-text": { + "/frame_interpolation": { "post": { - "summary": "Audio To Text", - "operationId": "audio_to_text", + "summary": "Frame Interpolation", + "operationId": "frame_interpolation", "requestBody": { "content": { "multipart/form-data": { "schema": { - "$ref": "#/components/schemas/Body_audio_to_text_audio_to_text_post" + "$ref": "#/components/schemas/Body_frame_interpolation_frame_interpolation_post" } } }, @@ -342,7 +352,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/TextResponse" + "$ref": "#/components/schemas/ImageResponse" } } } @@ -367,8 +377,71 @@ } } }, - "413": { - "description": "Request Entity Too Large", + "500": { + "description": "Internal Server Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPError" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + }, + "security": [ + { + "HTTPBearer": [] + } + ] + } + }, + "/upscale": { + "post": { + "summary": "Upscale", + "operationId": "upscale", + "requestBody": { + "content": { + "multipart/form-data": { + "schema": { + "$ref": "#/components/schemas/Body_upscale_upscale_post" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ImageResponse" + } + } + } + }, + "400": { + "description": "Bad Request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPError" + } + } + } + }, + "401": { + "description": "Unauthorized", "content": { "application/json": { "schema": { @@ -440,6 +513,39 @@ ], "title": "Body_audio_to_text_audio_to_text_post" }, + "Body_frame_interpolation_frame_interpolation_post": { + "properties": { + "model_id": { + "type": "string", + "title": "Model Id" + }, + "image1": { + "type": "string", + "format": "binary", + "title": "Image1" + }, + "image2": { + "type": "string", + "format": "binary", + "title": "Image2" + }, + "image_dir": { + "type": "string", + "title": "Image Dir", + "default": "" + }, + "inter_frames": { + "type": "integer", + "title": "Inter Frames", + "default": 2 + } + }, + "type": "object", + "required": [ + "model_id" + ], + "title": "Body_frame_interpolation_frame_interpolation_post" + }, "Body_image_to_image_image_to_image_post": { "properties": { "prompt": { diff --git a/runner/requirements.txt b/runner/requirements.txt index 3852800d..82877c24 100644 --- a/runner/requirements.txt +++ b/runner/requirements.txt @@ -1,11 +1,15 @@ diffusers==0.29.2 accelerate==0.30.1 -transformers==4.41.1 +transformers==4.43.1 fastapi==0.111.0 pydantic==2.7.2 Pillow==10.3.0 python-multipart==0.0.9 uvicorn==0.30.0 +setuptools==71.1.0 +torch --index-url https://download.pytorch.org/whl/cu121 +torchvision --index-url https://download.pytorch.org/whl/cu121 +torchaudio --index-url https://download.pytorch.org/whl/cu121 huggingface_hub==0.23.2 xformers==0.0.23 triton>=2.1.0 @@ -17,3 +21,4 @@ numpy==1.26.4 av==12.1.0 sentencepiece== 0.2.0 protobuf==5.27.2 +opencv-python==4.10.0.84 diff --git a/worker/runner.gen.go b/worker/runner.gen.go index 0dbe8036..788a7e82 100644 --- a/worker/runner.gen.go +++ b/worker/runner.gen.go @@ -37,6 +37,15 @@ type BodyAudioToTextAudioToTextPost struct { ModelId *string `json:"model_id,omitempty"` } +// BodyFrameInterpolationFrameInterpolationPost defines model for Body_frame_interpolation_frame_interpolation_post. +type BodyFrameInterpolationFrameInterpolationPost struct { + Image1 *openapi_types.File `json:"image1,omitempty"` + Image2 *openapi_types.File `json:"image2,omitempty"` + ImageDir *string `json:"image_dir,omitempty"` + InterFrames *int `json:"inter_frames,omitempty"` + ModelId string `json:"model_id"` +} + // BodyImageToImageImageToImagePost defines model for Body_image_to_image_image_to_image_post. type BodyImageToImageImageToImagePost struct { GuidanceScale *float32 `json:"guidance_scale,omitempty"` @@ -155,6 +164,9 @@ type Chunk struct { // AudioToTextMultipartRequestBody defines body for AudioToText for multipart/form-data ContentType. type AudioToTextMultipartRequestBody = BodyAudioToTextAudioToTextPost +// FrameInterpolationMultipartRequestBody defines body for FrameInterpolation for multipart/form-data ContentType. +type FrameInterpolationMultipartRequestBody = BodyFrameInterpolationFrameInterpolationPost + // ImageToImageMultipartRequestBody defines body for ImageToImage for multipart/form-data ContentType. type ImageToImageMultipartRequestBody = BodyImageToImageImageToImagePost @@ -305,6 +317,9 @@ type ClientInterface interface { // AudioToTextWithBody request with any body AudioToTextWithBody(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) + // FrameInterpolationWithBody request with any body + FrameInterpolationWithBody(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) + // Health request Health(ctx context.Context, reqEditors ...RequestEditorFn) (*http.Response, error) @@ -335,6 +350,18 @@ func (c *Client) AudioToTextWithBody(ctx context.Context, contentType string, bo return c.Client.Do(req) } +func (c *Client) FrameInterpolationWithBody(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewFrameInterpolationRequestWithBody(c.Server, contentType, body) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + func (c *Client) Health(ctx context.Context, reqEditors ...RequestEditorFn) (*http.Response, error) { req, err := NewHealthRequest(c.Server) if err != nil { @@ -436,6 +463,35 @@ func NewAudioToTextRequestWithBody(server string, contentType string, body io.Re return req, nil } +// NewFrameInterpolationRequestWithBody generates requests for FrameInterpolation with any type of body +func NewFrameInterpolationRequestWithBody(server string, contentType string, body io.Reader) (*http.Request, error) { + var err error + + serverURL, err := url.Parse(server) + if err != nil { + return nil, err + } + + operationPath := fmt.Sprintf("/frame_interpolation") + if operationPath[0] == '/' { + operationPath = "." + operationPath + } + + queryURL, err := serverURL.Parse(operationPath) + if err != nil { + return nil, err + } + + req, err := http.NewRequest("POST", queryURL.String(), body) + if err != nil { + return nil, err + } + + req.Header.Add("Content-Type", contentType) + + return req, nil +} + // NewHealthRequest generates requests for Health func NewHealthRequest(server string) (*http.Request, error) { var err error @@ -636,6 +692,9 @@ type ClientWithResponsesInterface interface { // AudioToTextWithBodyWithResponse request with any body AudioToTextWithBodyWithResponse(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*AudioToTextResponse, error) + // FrameInterpolationWithBodyWithResponse request with any body + FrameInterpolationWithBodyWithResponse(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*FrameInterpolationResponse, error) + // HealthWithResponse request HealthWithResponse(ctx context.Context, reqEditors ...RequestEditorFn) (*HealthResponse, error) @@ -681,6 +740,32 @@ func (r AudioToTextResponse) StatusCode() int { return 0 } +type FrameInterpolationResponse struct { + Body []byte + HTTPResponse *http.Response + JSON200 *ImageResponse + JSON400 *HTTPError + JSON401 *HTTPError + JSON422 *HTTPValidationError + JSON500 *HTTPError +} + +// Status returns HTTPResponse.Status +func (r FrameInterpolationResponse) Status() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Status + } + return http.StatusText(0) +} + +// StatusCode returns HTTPResponse.StatusCode +func (r FrameInterpolationResponse) StatusCode() int { + if r.HTTPResponse != nil { + return r.HTTPResponse.StatusCode + } + return 0 +} + type HealthResponse struct { Body []byte HTTPResponse *http.Response @@ -816,6 +901,15 @@ func (c *ClientWithResponses) AudioToTextWithBodyWithResponse(ctx context.Contex return ParseAudioToTextResponse(rsp) } +// FrameInterpolationWithBodyWithResponse request with arbitrary body returning *FrameInterpolationResponse +func (c *ClientWithResponses) FrameInterpolationWithBodyWithResponse(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*FrameInterpolationResponse, error) { + rsp, err := c.FrameInterpolationWithBody(ctx, contentType, body, reqEditors...) + if err != nil { + return nil, err + } + return ParseFrameInterpolationResponse(rsp) +} + // HealthWithResponse request returning *HealthResponse func (c *ClientWithResponses) HealthWithResponse(ctx context.Context, reqEditors ...RequestEditorFn) (*HealthResponse, error) { rsp, err := c.Health(ctx, reqEditors...) @@ -930,6 +1024,60 @@ func ParseAudioToTextResponse(rsp *http.Response) (*AudioToTextResponse, error) return response, nil } +// ParseFrameInterpolationResponse parses an HTTP response from a FrameInterpolationWithResponse call +func ParseFrameInterpolationResponse(rsp *http.Response) (*FrameInterpolationResponse, error) { + bodyBytes, err := io.ReadAll(rsp.Body) + defer func() { _ = rsp.Body.Close() }() + if err != nil { + return nil, err + } + + response := &FrameInterpolationResponse{ + Body: bodyBytes, + HTTPResponse: rsp, + } + + switch { + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: + var dest ImageResponse + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON200 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 400: + var dest HTTPError + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON400 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 401: + var dest HTTPError + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON401 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 422: + var dest HTTPValidationError + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON422 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 500: + var dest HTTPError + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON500 = &dest + + } + + return response, nil +} + // ParseHealthResponse parses an HTTP response from a HealthWithResponse call func ParseHealthResponse(rsp *http.Response) (*HealthResponse, error) { bodyBytes, err := io.ReadAll(rsp.Body) @@ -1177,6 +1325,9 @@ type ServerInterface interface { // Audio To Text // (POST /audio-to-text) AudioToText(w http.ResponseWriter, r *http.Request) + // Frame Interpolation + // (POST /frame_interpolation) + FrameInterpolation(w http.ResponseWriter, r *http.Request) // Health // (GET /health) Health(w http.ResponseWriter, r *http.Request) @@ -1204,6 +1355,12 @@ func (_ Unimplemented) AudioToText(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusNotImplemented) } +// Frame Interpolation +// (POST /frame_interpolation) +func (_ Unimplemented) FrameInterpolation(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotImplemented) +} + // Health // (GET /health) func (_ Unimplemented) Health(w http.ResponseWriter, r *http.Request) { @@ -1260,6 +1417,23 @@ func (siw *ServerInterfaceWrapper) AudioToText(w http.ResponseWriter, r *http.Re handler.ServeHTTP(w, r.WithContext(ctx)) } +// FrameInterpolation operation middleware +func (siw *ServerInterfaceWrapper) FrameInterpolation(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + ctx = context.WithValue(ctx, HTTPBearerScopes, []string{}) + + handler := http.Handler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + siw.Handler.FrameInterpolation(w, r) + })) + + for _, middleware := range siw.HandlerMiddlewares { + handler = middleware(handler) + } + + handler.ServeHTTP(w, r.WithContext(ctx)) +} + // Health operation middleware func (siw *ServerInterfaceWrapper) Health(w http.ResponseWriter, r *http.Request) { ctx := r.Context() @@ -1459,6 +1633,9 @@ func HandlerWithOptions(si ServerInterface, options ChiServerOptions) http.Handl r.Group(func(r chi.Router) { r.Post(options.BaseURL+"/audio-to-text", wrapper.AudioToText) }) + r.Group(func(r chi.Router) { + r.Post(options.BaseURL+"/frame_interpolation", wrapper.FrameInterpolation) + }) r.Group(func(r chi.Router) { r.Get(options.BaseURL+"/health", wrapper.Health) }) @@ -1481,32 +1658,33 @@ func HandlerWithOptions(si ServerInterface, options ChiServerOptions) http.Handl // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+xZ227bOBN+FYL/f+nEhzabhe+SbLcNtoegdrsXRWAw0thmK5FaHtJ6A7/7gkNZomQp", - "cpDGC2R9Zcsaznxz+IZD+o5GMs2kAGE0Hd9RHS0hZfj17OrylVJSue+ZkhkowwHfpHrhPgw3CdAxfacX", - "tEfNKnMP2iguFnS97lEFf1muIKbjL7jkulcsKXQX6+TNV4gMXffouYxXM2ZjLmdGzgz8MLWnTGqzDQpl", - "3Je5VCkzdExvuGBqRQOrKLIFtUdTGUMy47FbHsOc2cStD1a+cwLkMu7006MIPN3Nm7Yw8JQtwIn6L7XH", - "5kAsLI+ZiGCmI+YgBC6dHp+UyF7ncmSCcgUEYdMbUA4CWrk/pJco0hBSj/AeLMMQC6oh3YgekageFbBg", - "ht/CLFMyzUyrjve5HLnyck2qbOpzoGcZqCaFw0CfTQk6qMkVqC2tXBhYePdQrZiDAoyZgUxXlQ4GNbUb", - "YTJB4SalJbjNyna/NJuDWc2iJUTfKpaNslCanqAYuUCxQs2NlAkwgXoA4tDixD03gdNGgViYZcXY4PjX", - "wNZGYqscatTLNl75sq1zcAcqdbLwlscg64/NLJzXUvdLCef3lkQtgS+W1So6OQ3WvfHvm5Y+hqmP4lQq", - "DZdidmOjb2DqSoaj01CLkyTnKFnRFhJAcg0zZhezlsIYjAICOGFyZhekvUa6OTU6eTil9k6T7zyuhWI4", - "GL0sLf2J77dX1ijSwYz28m5jhs2wsRefzVz416qzK/enJ8+qnT6sITbmriHRb6bTq5ZBMAbDeOK+/V/B", - "nI7p//rlONnPZ8l+MezVAebLA2ClrRYgn1nCY+Y6SSckbiDVXdjq+tYllt+8pgIIU4qt0IcQbV1BE25g", - "iVlebIqgilcbZmy1KumHP2i4/6FA0+BZbgylgQb7yK2PoDMpNLSwU+8csXcQcxbGyY82TXHaaj06zHUV", - "VgNub2kLr9Dz7yEZ3rvnR3VXq5JQ7pNKOud8izLaa0REgWceeINHU/hh2hMRLa34tnsiUDxMxIVfX09E", - "j7pzRuigg9HpofFCOajAu4oTLU5OJWb3iinmHXmqI0o5M+0wJf3HTw8nz+3wUExFDxyDcqdqNV2t2YbC", - "7tx7EhlV2MvE6sOcjr/cbcXqbgvidUDktzJCMw1Url+9gNYtg5P/oRRFzGTqfu2ivvPDm8olg0jtsN99", - "dnNje5ubK5bW9psHbjz19rY5V3nFHRtRbj50qYK3wSHfabcc2a2tOjspaMPSLHQ1wD0t3ndAN6GgMxY4", - "4TFugUc6RVZxs5q4OHrkbnA5B6ZAFXd+yEH/U6FkaUxG104HF3PpKa0jxTMszjE9E4RlWcJ9tRIjibKC", - "nF2SjGeQcOGTsSlqfgsZgHLvP1oh0NAtKO11DY6HxwMXLZmBYBmnY/oCf+rRjJklwu7jzdmRkUeb0G/O", - "Gy4tCOIy3tzzTWWeDxdB0MbNvLjLSmFA4KrUJoZnTJm+O5gcxcyw8g60qxx3u9hbV3PoOiH+4IsNvRoN", - "BjVcQVD7X7ULz66gKnsz2q5mbGKjCLSe24SUYj368idCKEf4BvvnLCYffT683eF+7H4SzJqlVPxviNHw", - "8MV+DOfOklfCcLMiUynJW6YWPuqj0U8FsXWW2YZTipDivHOyr+RfCgNKsIRMQN2CIuWhcNOicK8Mm9OX", - "6/V1j2qbpkytNswmU0mQ225pf4mHH5wqoaEX+LMRfULOhaevXSm3Dp3KIaI3OBa6DlfcmTS3OBxV8onl", - "iXvcDhene+5y1ZPjoc21t7lDh3loh/H/RE2lP3PVSIk3op2kxHlyX6Rsv7PdMymrU/SBlAdSPgEpPbWQ", - "lG7G3mGjDE7291LycTN39e7gsB0emPdMmOeKu7Yb5v8XtVPuUy7wtDtg499XB+YdmPdMmLdh0dqvcmo0", - "LqpaKq7VLhJpY3Ih09QKblbkNTPwna1o/vcWXubpcb8fK2Dp0cK/PU7y5ceRW07X1+t/AgAA///2pVcb", - "EigAAA==", + "H4sIAAAAAAAC/+xZWW/bOhb+KwRnHp14aTMZ+C1Jt2C6BI3beSgCg5GObbYSqUtSaX0D//cLHsoSJVOR", + "jTS+QK6fvOgs31m+w0X3NJJpJgUIo+n4nupoASnDr2dXl6+Vksp+z5TMQBkO+CTVc/thuEmAjukHPac9", + "apaZ/aGN4mJOV6seVfBHzhXEdPwNVW56pUppu9STt98hMnTVo+cyXk5ZHnM5NXJq4Jdp/MqkNpugUMZ+", + "mUmVMkPH9JYLppbU84oiG1B7NJUxJFMeW/UYZixPrL6n+cEKkMu4M06Hwot0u2ja0jBTLIUpFwZUJhNm", + "uBTB/8Ip4Smbw/DhnFw6mUBSUHu0hfaoVXsac9WaU9Qlr7gKqtvwXKi6ZmHkGbAy5I2TKW1YzTmoZll3", + "rmSp3CzmTjVpK6xLj5HFl8bPcDnnOY+ZiGCqI2bheFk5PT6pUL4t5Mg1ypUQRJ7eusSgly0q217YB7AM", + "fSyuyN2IHsHAHhUwZ4bfwTRTMs1Mq42PhRy5cnIhU3nqaqCnGaiQwaFnL08JBqjJFagNq14nolkxAwWY", + "MwNZvauHg0HD7FqYXKNwyGgFbq3ZHpdmMzDLabSA6EfNs1E5VK6vUYxcoFhp5lbKBJhAOwA1Ol3b3yFw", + "2igQc7OoORsc/9fztZbYaIcGE7N1VK5tm3zcgkqdLLzjMcjmzzALZ43S/aeC86alUAvg80W9i05OPb13", + "7nlI9TFMfRSnUolD7DaPfoBpGhmOTn0rVpKco2TNmk8AyTVMWT6ftjTGwBvsH60wOcvnpL1Hujk1Otmd", + "UnunyU8eN1IxHIxeVp7+j883NRsU6WBGe3u3MSPPcLCXnw9sMP6O7uyq/enJsxqnuw3EYO0ChX43mVy1", + "7PBjMIwn9tu/FczomP6rX50T+sUhoV/u4psAC3UPWOWrBchXlvAYN06dkLiBVHdha9pbVVheOUslEKYU", + "W2IMPtqmgRBuYIlZXKyboI5XG2byelfST/+j/vqHAqF9aLUwVA4C/pFbn0FnUmhoYafeOmMfIObMz5Pb", + "2oTytDF6tF/rOqwAbudpA6/Qs58+GT7a34+arrlKfLkvKunc9ucoo51FRORF5oAHIprAL9NeiGiRix/b", + "FwLF/UJcOP1mIXrUHiD9AC2MzgiNEypAedHVgmgJciKxuldMMRfIUx1Rqj3TFrukf/jp4eS5HR7KXdGO", + "26AiqEZP13s20Nida08ioxp7mVh+mtHxt/uNXN1vQLzxiPxeRugmQOXmnRpo3bJxcn9UooiZTOy/XdS3", + "cThXhaSXqS3Wu69239g+5qrbmjJROy48zfG2Plc1rnjCC1Hh3g+phjcQkJu0G4FsN1atnxS0YWnmh+rh", + "npTPO6AbX9A684JwGDfAI52iXHGzvLZ5dMjtxuUcmAJVXuYiB91fpZGFMRldrfCebSYdpXWkeIbNOaZn", + "grAsS7jrVmIkUbkgZ5ck4xkkXLhirJua30EGoOzzz7kQ6OgOlHa2BsfD44HNlsxAsIzTMX2Bf/VoxswC", + "YffxSvTIyKN16tfnDVsWBHEZry9wJ7Koh80gaGP3vLjKSmFAoFaaJ4ZnTJm+PZgcxcyw6nK7qx23u7Fd", + "1WtoJyH+4ZoNoxoNBg1cXlL737VNz7agamsz+q5X7DqPItB6liekEuvRl78RQrWFD/g/ZzH57Orh/A73", + "4/eLYLlZSMX/hBgdD1/sx3ERLHktDDdLMpGSvGdq7rI+Gv1WEBtnmU04lQgpzzsn+yo+XsQLlpBrUHeg", + "SHUoXI8oXCv94fTtZnXTozpPU6aWa2aTiSTIbavaD9ypt08GXCIua7JPOyB2eguw52FRP4AdpkX7tDgQ", + "dVeiItFInWlI1wXeVeAhEAIEdVcZ9Am73r8s2bbnV35oBUSMBk9xdkNSXnGG5w5SrThgPPHE2eI9x2HO", + "HObMM5kz7sXxRLorkgYp8QVGJynx+LcvUra/YtkzKeuH3gMpD6R8AlI6aiEp7ZF4i4XSu4h7kJKPOyLX", + "r/oOy+GBec+Eeba5G6th8Xq3nXJfCoGnXQGDb5sPzDsw75kwb82ildOyZjQq1T2Vt+AXicxjciHTNBfc", + "LMlbZuAnW9LibTTevetxvx8rYOnR3D09Tgr148iq09XN6q8AAAD///BmUXaaLQAA", } // GetSwagger returns the content of the embedded swagger specification file From 9db6034261b4191e0df7eab4355b42acb7dbbe48 Mon Sep 17 00:00:00 2001 From: Jason <83615043+JJassonn69@users.noreply.github.com> Date: Fri, 26 Jul 2024 16:23:29 +0300 Subject: [PATCH 12/38] Update docker-create-ai-runner.yaml --- .github/workflows/docker-create-ai-runner.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-create-ai-runner.yaml b/.github/workflows/docker-create-ai-runner.yaml index fb7feafb..ccd819ba 100644 --- a/.github/workflows/docker-create-ai-runner.yaml +++ b/.github/workflows/docker-create-ai-runner.yaml @@ -49,7 +49,7 @@ jobs: uses: docker/metadata-action@v5 with: images: | - livepeer/ai-runner + jjassonn69/ai-runner tags: | type=sha type=ref,event=pr From 3ed779228ad70c72ff1696ff70be2de9117fc610 Mon Sep 17 00:00:00 2001 From: Jason <83615043+JJassonn69@users.noreply.github.com> Date: Fri, 26 Jul 2024 16:43:26 +0300 Subject: [PATCH 13/38] Delete .github/workflows/ai-runner-docker.yaml --- .github/workflows/ai-runner-docker.yaml | 77 ------------------------- 1 file changed, 77 deletions(-) delete mode 100644 .github/workflows/ai-runner-docker.yaml diff --git a/.github/workflows/ai-runner-docker.yaml b/.github/workflows/ai-runner-docker.yaml deleted file mode 100644 index 3bf96140..00000000 --- a/.github/workflows/ai-runner-docker.yaml +++ /dev/null @@ -1,77 +0,0 @@ -name: Build Docker image for ai-runner - -on: - pull_request: - paths: - - "runner/**" - - "!runner/.devcontainer/**" - push: - branches: - - main - tags: - - '*' - paths: - - "runner/**" - - "!runner/.devcontainer/**" - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -jobs: - docker: - name: Docker image generation - if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository - permissions: - packages: write - contents: read - runs-on: ubuntu-20.04 - steps: - - name: Check out code - uses: actions/checkout@v4.1.1 - with: - fetch-depth: 0 - # Check https://github.com/livepeer/go-livepeer/pull/1891 - # for ref value discussion - ref: ${{ github.event.pull_request.head.sha }} - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to DockerHub - uses: docker/login-action@v3 - with: - username: ${{ secrets.CI_DOCKERHUB_USERNAME }} - password: ${{ secrets.CI_DOCKERHUB_TOKEN }} - - - name: Extract metadata (tags, labels) for Docker - id: meta - uses: docker/metadata-action@v5 - with: - images: | - livepeer/ai-runner - tags: | - type=sha - type=ref,event=pr - type=ref,event=tag - type=sha,format=long - type=ref,event=branch - type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} - type=semver,pattern={{version}},prefix=v - type=semver,pattern={{major}}.{{minor}},prefix=v - type=raw,value=latest,enable={{is_default_branch}} - type=raw,value=${{ github.event.pull_request.head.ref }} - type=raw,value=stable,enable=${{ startsWith(github.event.ref, 'refs/tags/v') }} - - - name: Build and push runner docker image - uses: docker/build-push-action@v5 - with: - context: "{{defaultContext}}:runner" - platforms: linux/amd64 - push: true - tags: ${{ steps.meta.outputs.tags }} - file: "Dockerfile" - labels: ${{ steps.meta.outputs.labels }} - cache-from: type=registry,ref=livepeerci/build:cache - cache-to: type=registry,ref=livepeerci/build:cache,mode=max From d032b0f221c3e2f1a891634a51d24f18155bceb6 Mon Sep 17 00:00:00 2001 From: Jason <83615043+JJassonn69@users.noreply.github.com> Date: Fri, 26 Jul 2024 16:43:44 +0300 Subject: [PATCH 14/38] Delete .github/workflows/validate-openapi-on-pr.yaml --- .github/workflows/validate-openapi-on-pr.yaml | 44 ------------------- 1 file changed, 44 deletions(-) delete mode 100644 .github/workflows/validate-openapi-on-pr.yaml diff --git a/.github/workflows/validate-openapi-on-pr.yaml b/.github/workflows/validate-openapi-on-pr.yaml deleted file mode 100644 index 2674dd01..00000000 --- a/.github/workflows/validate-openapi-on-pr.yaml +++ /dev/null @@ -1,44 +0,0 @@ -name: Check OpenAPI spec and Golang bindings - -on: - pull_request: - -jobs: - check-openapi-and-bindings: - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: "3.11" - cache: "pip" - - - name: Install Python dependencies - run: | - python -m pip install --upgrade pip - pip install -r runner/requirements.txt - - - name: Generate AI OpenAPI specification - working-directory: runner - run: | - python gen_openapi.py - - - name: Check for OpenAPI spec changes - run: | - if ! git diff --exit-code; then - echo "::error:: OpenAPI spec has changed. Please run 'python gen_openapi.py' in the 'runner' directory and commit the changes." - exit 1 - fi - - - name: Generate Go bindings - run: make - - - name: Check for Go bindings changes - run: | - if ! git diff --exit-code; then - echo "::error::Go bindings have changed. Please run 'make' at the root of the repository and commit the changes." - exit 1 - fi From 9606ba619f3b6f40599602b2ac5978767a42fe31 Mon Sep 17 00:00:00 2001 From: Jason <83615043+JJassonn69@users.noreply.github.com> Date: Fri, 26 Jul 2024 16:51:49 +0300 Subject: [PATCH 15/38] Update docker-create-ai-runner.yaml --- .github/workflows/docker-create-ai-runner.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-create-ai-runner.yaml b/.github/workflows/docker-create-ai-runner.yaml index ccd819ba..3013d245 100644 --- a/.github/workflows/docker-create-ai-runner.yaml +++ b/.github/workflows/docker-create-ai-runner.yaml @@ -1,7 +1,7 @@ name: Build Docker image for ai-runner on: - member: + check_run: paths: - "runner/**" - "!runner/.devcontainer/**" From 32ebe0f2112788305f2e1268fd866eac63836269 Mon Sep 17 00:00:00 2001 From: Jason <83615043+JJassonn69@users.noreply.github.com> Date: Fri, 26 Jul 2024 16:52:24 +0300 Subject: [PATCH 16/38] Update validate-openapi-on-push.yaml --- .github/workflows/validate-openapi-on-push.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validate-openapi-on-push.yaml b/.github/workflows/validate-openapi-on-push.yaml index 4809d94c..886d24e6 100644 --- a/.github/workflows/validate-openapi-on-push.yaml +++ b/.github/workflows/validate-openapi-on-push.yaml @@ -1,7 +1,7 @@ name: Check OpenAPI spec and Golang bindings on: - push: + check_run: jobs: check-openapi-and-bindings: From ce33b203a001f2b131338450c4067e3604b93ca9 Mon Sep 17 00:00:00 2001 From: Jason <83615043+JJassonn69@users.noreply.github.com> Date: Fri, 26 Jul 2024 16:52:42 +0300 Subject: [PATCH 17/38] Update trigger-upstream-openapi-sync.yaml --- .github/workflows/trigger-upstream-openapi-sync.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/trigger-upstream-openapi-sync.yaml b/.github/workflows/trigger-upstream-openapi-sync.yaml index 9e006f7d..054ed1a7 100644 --- a/.github/workflows/trigger-upstream-openapi-sync.yaml +++ b/.github/workflows/trigger-upstream-openapi-sync.yaml @@ -1,7 +1,7 @@ name: Trigger upstream OpenAPI sync on: - push: + check_run: paths: - "runner/openapi.json" workflow_dispatch: From ca1d2f222ea3409c0f52b5b8bfc3f1c768e1c811 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 26 Jul 2024 17:02:27 +0300 Subject: [PATCH 18/38] chore(deps): bump github.com/docker/docker (#6) Bumps the go_modules group with 1 update in the / directory: [github.com/docker/docker](https://github.com/docker/docker). Updates `github.com/docker/docker` from 24.0.7+incompatible to 24.0.9+incompatible - [Release notes](https://github.com/docker/docker/releases) - [Commits](https://github.com/docker/docker/compare/v24.0.7...v24.0.9) --- updated-dependencies: - dependency-name: github.com/docker/docker dependency-type: direct:production dependency-group: go_modules ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index d95246f2..84c614e8 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.21.5 require ( github.com/deepmap/oapi-codegen/v2 v2.2.0 github.com/docker/cli v24.0.5+incompatible - github.com/docker/docker v24.0.7+incompatible + github.com/docker/docker v24.0.9+incompatible github.com/docker/go-connections v0.4.0 github.com/getkin/kin-openapi v0.124.0 github.com/go-chi/chi/v5 v5.0.12 diff --git a/go.sum b/go.sum index 7b4d8b69..f7291a7b 100644 --- a/go.sum +++ b/go.sum @@ -17,8 +17,8 @@ github.com/docker/cli v24.0.5+incompatible h1:WeBimjvS0eKdH4Ygx+ihVq1Q++xg36M/rM github.com/docker/cli v24.0.5+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk= github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v24.0.7+incompatible h1:Wo6l37AuwP3JaMnZa226lzVXGA3F9Ig1seQen0cKYlM= -github.com/docker/docker v24.0.7+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v24.0.9+incompatible h1:HPGzNmwfLZWdxHqK9/II92pyi1EpYKsAqcl4G0Of9v0= +github.com/docker/docker v24.0.9+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= From 56d802d1a367c190341fd72db9e66f2944b7641b Mon Sep 17 00:00:00 2001 From: Jason <83615043+JJassonn69@users.noreply.github.com> Date: Fri, 26 Jul 2024 17:06:22 +0300 Subject: [PATCH 19/38] Update docker-create-ai-runner.yaml --- .github/workflows/docker-create-ai-runner.yaml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/docker-create-ai-runner.yaml b/.github/workflows/docker-create-ai-runner.yaml index 3013d245..af35018c 100644 --- a/.github/workflows/docker-create-ai-runner.yaml +++ b/.github/workflows/docker-create-ai-runner.yaml @@ -1,10 +1,7 @@ name: Build Docker image for ai-runner on: - check_run: - paths: - - "runner/**" - - "!runner/.devcontainer/**" + workflow_dispatch: push: branches: - main From 01f229afe368124ff8d8c004d96a5025acd21dfa Mon Sep 17 00:00:00 2001 From: Jason <83615043+JJassonn69@users.noreply.github.com> Date: Fri, 26 Jul 2024 17:11:36 +0300 Subject: [PATCH 20/38] Update docker-create-ai-runner.yaml --- .github/workflows/docker-create-ai-runner.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker-create-ai-runner.yaml b/.github/workflows/docker-create-ai-runner.yaml index af35018c..b35e9fd8 100644 --- a/.github/workflows/docker-create-ai-runner.yaml +++ b/.github/workflows/docker-create-ai-runner.yaml @@ -1,7 +1,10 @@ name: Build Docker image for ai-runner on: - workflow_dispatch: + check_run: + paths: + - "runner/**" + - "!runner/.devcontainer/**" push: branches: - main @@ -10,6 +13,7 @@ on: paths: - "runner/**" - "!runner/.devcontainer/**" + workflow_dispatch: concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} From 580224c3927166560a6bb3aaed654762a63ffd1e Mon Sep 17 00:00:00 2001 From: Jason <83615043+JJassonn69@users.noreply.github.com> Date: Fri, 26 Jul 2024 17:16:26 +0300 Subject: [PATCH 21/38] Update docker-create-ai-runner.yaml --- .github/workflows/docker-create-ai-runner.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/docker-create-ai-runner.yaml b/.github/workflows/docker-create-ai-runner.yaml index b35e9fd8..4b233ff4 100644 --- a/.github/workflows/docker-create-ai-runner.yaml +++ b/.github/workflows/docker-create-ai-runner.yaml @@ -22,7 +22,6 @@ concurrency: jobs: docker: name: Docker image generation - if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository permissions: packages: write contents: read From 4f9f57862395c472f9aa2b9b4b443dcbce460325 Mon Sep 17 00:00:00 2001 From: Jason <83615043+JJassonn69@users.noreply.github.com> Date: Fri, 26 Jul 2024 17:39:28 +0300 Subject: [PATCH 22/38] Update docker-create-ai-runner.yaml --- .github/workflows/docker-create-ai-runner.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-create-ai-runner.yaml b/.github/workflows/docker-create-ai-runner.yaml index 4b233ff4..d71aa858 100644 --- a/.github/workflows/docker-create-ai-runner.yaml +++ b/.github/workflows/docker-create-ai-runner.yaml @@ -73,5 +73,5 @@ jobs: tags: ${{ steps.meta.outputs.tags }} file: "Dockerfile" labels: ${{ steps.meta.outputs.labels }} - cache-from: type=registry,ref=livepeerci/build:cache - cache-to: type=registry,ref=livepeerci/build:cache,mode=max + cache-from: type=registry,ref=jjassonn69/build:cache + cache-to: type=registry,ref=jjassonn69/build:cache,mode=max From fa07ec6cd35b7cb2fa4d9880df2ed2bbf9dae483 Mon Sep 17 00:00:00 2001 From: Jason <83615043+JJassonn69@users.noreply.github.com> Date: Fri, 26 Jul 2024 18:12:03 +0300 Subject: [PATCH 23/38] Frame interpolation merging from branch (#7) * update to the frame interpolation pipeline, there is some minor issue with creating go api bindings because of openapi json sceme having a null option. * minor changes to requirements * update to requrements to fetch from --index-url * simple patch to solve the go api bindings issue * checking if it works in my system --------- Co-authored-by: Jason Stone From 672f5fd5aa970192bb5e652e1f653743c20d3baa Mon Sep 17 00:00:00 2001 From: Jason <83615043+JJassonn69@users.noreply.github.com> Date: Fri, 26 Jul 2024 18:18:48 +0300 Subject: [PATCH 24/38] Delete .github/workflows/trigger-upstream-openapi-sync.yaml --- .../trigger-upstream-openapi-sync.yaml | 30 ------------------- 1 file changed, 30 deletions(-) delete mode 100644 .github/workflows/trigger-upstream-openapi-sync.yaml diff --git a/.github/workflows/trigger-upstream-openapi-sync.yaml b/.github/workflows/trigger-upstream-openapi-sync.yaml deleted file mode 100644 index 054ed1a7..00000000 --- a/.github/workflows/trigger-upstream-openapi-sync.yaml +++ /dev/null @@ -1,30 +0,0 @@ -name: Trigger upstream OpenAPI sync - -on: - check_run: - paths: - - "runner/openapi.json" - workflow_dispatch: - -jobs: - trigger-upstream-openapi-sync: - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Trigger docs AI OpenAPI spec update - uses: peter-evans/repository-dispatch@v3 - with: - token: ${{ secrets.DOCS_TRIGGER_PAT }} - repository: livepeer/docs - event-type: update-ai-openapi - client-payload: '{"sha": "${{ github.sha }}"}' - - - name: Trigger SDK generation - uses: peter-evans/repository-dispatch@v3 - with: - token: ${{ secrets.SDKS_TRIGGER_PAT }} - repository: livepeer/livepeer-ai-sdks - event-type: update-ai-openapi - client-payload: '{"sha": "${{ github.sha }}"}' From ef155aa2219f625eba187b50dbd550b1ae81737e Mon Sep 17 00:00:00 2001 From: Jason Stone Date: Fri, 26 Jul 2024 18:22:36 +0300 Subject: [PATCH 25/38] test-examples for frame-interpolation --- .../app/test_examples/continuous-creation.py | 154 +++++++++++++++++ .../generate-interpolate-upscale.py | 159 ++++++++++++++++++ .../test_examples/test-film-interpolate.py | 45 +++++ runner/app/test_examples/test-optimized.py | 69 ++++++++ .../test_examples/utility-func-compactor.py | 89 ++++++++++ .../app/test_examples/utility-function-vid.py | 62 +++++++ runner/uvicorn.env | 4 + 7 files changed, 582 insertions(+) create mode 100644 runner/app/test_examples/continuous-creation.py create mode 100644 runner/app/test_examples/generate-interpolate-upscale.py create mode 100644 runner/app/test_examples/test-film-interpolate.py create mode 100644 runner/app/test_examples/test-optimized.py create mode 100644 runner/app/test_examples/utility-func-compactor.py create mode 100644 runner/app/test_examples/utility-function-vid.py create mode 100644 runner/uvicorn.env diff --git a/runner/app/test_examples/continuous-creation.py b/runner/app/test_examples/continuous-creation.py new file mode 100644 index 00000000..54473930 --- /dev/null +++ b/runner/app/test_examples/continuous-creation.py @@ -0,0 +1,154 @@ +import torch +import numpy as np +import os +import shutil +import time +import sys +sys.path.append('C://Users//ganes//ai-worker//runner') +from PIL import PngImagePlugin, Image +from diffusers import StableVideoDiffusionPipeline +from app.pipelines.frame_interpolation import FILMPipeline +from app.pipelines.utils import DirectoryReader, frames_compactor, DirectoryWriter + +# Increase the max text chunk size for PNG images +PngImagePlugin.MAX_TEXT_CHUNK = 100 * (1024**2) + +def move_file_with_retry(src_file_path: str, dst_file_path: str, retries: int = 5, delay: float = 1.0): + for attempt in range(retries): + try: + shutil.move(src_file_path, dst_file_path) + return + except PermissionError: + print(f"Attempt {attempt + 1} failed: File is in use. Retrying in {delay} seconds...") + time.sleep(delay) + raise PermissionError(f"Failed to move file after {retries} attempts.") + +def get_last_file_sorted_by_name(directory: str) -> str: + """ + Get the last file in the directory when sorted by filename. + + Args: + directory (str): Path to the directory. + + Returns: + str: Path to the last file in the sorted list, or None if directory is empty. + """ + try: + # List all files in the directory + files = [f for f in os.listdir(directory) if os.path.isfile(os.path.join(directory, f))] + + if not files: + print("No files found in the directory.") + return None + + # Sort files by name + files.sort() + + # Get the last file in the sorted list + last_file = files[-1] + + return os.path.join(directory, last_file) + + except Exception as e: + print(f"An error occurred: {e}") + return None + +def main(): + # Initialize pipelines + repo_id = "stabilityai/stable-video-diffusion-img2vid-xt" + svd_xt_pipeline = StableVideoDiffusionPipeline.from_pretrained( + repo_id, torch_dtype=torch.float16, variant="fp16", cache_dir="G:/ai-models/models" + ) + svd_xt_pipeline.enable_model_cpu_offload() + + film_pipeline = FILMPipeline("G:/ai-models/models/film_net_fp16.pt").to(device="cuda", dtype=torch.float16) + + # Load initial input image + image_path = "G:/ai-models/models/gif_frames/donut_motion.png" + image = Image.open(image_path) + + fps = 24.0 + inter_frames = 4 + rounds = 2 # Number of rounds of generation and interpolation + base_output_dir = "G:/ai-models/models" + + all_frames_dir = os.path.join(base_output_dir, "all_interpolated_frames") + os.makedirs(all_frames_dir, exist_ok=True) + + last_frame_for_next_round = os.path.join(base_output_dir, "last_frame_for_next_round.png") + + for round_num in range(1, rounds + 1): + svd_xt_output_dir = os.path.join(base_output_dir, f"svd_xt_output_round_{round_num}") + os.makedirs(svd_xt_output_dir, exist_ok=True) + + # Generate frames using SVD pipeline + generator = torch.manual_seed(42) + frames = svd_xt_pipeline(image, decode_chunk_size=8, generator=generator, output_type="np").frames[0] + + # Save SVD frames to directory + film_writer = DirectoryWriter(svd_xt_output_dir) + for idx, frame in enumerate(frames): + film_writer.write_frame(torch.tensor(frame).permute(2, 0, 1)) + + # Read saved frames for interpolation + film_reader = DirectoryReader(svd_xt_output_dir) + height, width = film_reader.get_resolution() + + # Interpolate frames using FILM pipeline + film_pipeline(film_reader, film_writer, inter_frames=inter_frames) + + # Close reader and writer + film_writer.close() + film_reader.reset() + + # Deleting the SVD generated images. + for i in range(len(frames)): + os.remove(os.path.join(svd_xt_output_dir, f"{i}.png")) + print(f"Deleted the first {len(frames)} frames in directory: {svd_xt_output_dir}") + + # Save the last frame separately for the next round + last_frame_path = get_last_file_sorted_by_name(svd_xt_output_dir) + if last_frame_path: + shutil.copy2(last_frame_path, last_frame_for_next_round) + else: + print("No frames found to copy.") + + # Move all interpolated frames to a common directory with a unique naming scheme + for file_name in sorted(os.listdir(svd_xt_output_dir)): + src_file_path = os.path.join(svd_xt_output_dir, file_name) + dst_file_name = f"round_{round_num:03d}_frame_{file_name}" + dst_file_path = os.path.join(all_frames_dir, dst_file_name) + + move_file_with_retry(src_file_path, dst_file_path) + + # Clean up the source directory after moving frames + for file_name in os.listdir(svd_xt_output_dir): + os.remove(os.path.join(svd_xt_output_dir, file_name)) + os.rmdir(svd_xt_output_dir) + + # Ensure all operations on last frame are complete before opening it again + time.sleep(1) # Small delay to ensure file system operations are complete + + # Prepare for next round + image = Image.open(last_frame_for_next_round) + + # Compile all interpolated frames from all rounds into a final video + video_output_dir = "G:/ai-models/models/video_out" + os.makedirs(video_output_dir, exist_ok=True) + + film_output_path = os.path.join(video_output_dir, "output.avi") + frames_compactor(frames=all_frames_dir, output_path=film_output_path, fps=fps, codec="MJPG", is_directory=True) + + # Clean up all frames in the directories after video generation + for file_name in os.listdir(all_frames_dir): + file_path = os.path.join(all_frames_dir, file_name) + if os.path.isfile(file_path): + os.remove(file_path) + os.rmdir(all_frames_dir) + + print(f"All frames deleted from directories.") + print(f"Video generated at: {film_output_path}") + return film_output_path + +if __name__ == "__main__": + main() diff --git a/runner/app/test_examples/generate-interpolate-upscale.py b/runner/app/test_examples/generate-interpolate-upscale.py new file mode 100644 index 00000000..69954a0a --- /dev/null +++ b/runner/app/test_examples/generate-interpolate-upscale.py @@ -0,0 +1,159 @@ +import logging +import os +os.environ["MODEL_DIR"] = "G://ai-models//models" +import shutil +import time +import sys +sys.path.append('C://Users//ganes//ai-worker//runner') +from typing import List, Optional, Tuple +import PIL +import torch +from diffusers import StableVideoDiffusionPipeline +from PIL import PngImagePlugin, Image, ImageFile + +from app.pipelines.base import Pipeline +from app.pipelines.frame_interpolation import FILMPipeline +from app.pipelines.upscale import UpscalePipeline +from app.pipelines.utils import DirectoryReader, frames_compactor, DirectoryWriter, SafetyChecker, get_model_dir, get_torch_device, is_lightning_model, is_turbo_model +from huggingface_hub import file_download + +# Increase the max text chunk size for PNG images +PngImagePlugin.MAX_TEXT_CHUNK = 100 * (1024**2) +ImageFile.LOAD_TRUNCATED_IMAGES = True + +logger = logging.getLogger(__name__) + +# Helper function to move files with retry mechanism +def move_file_with_retry(src_file_path: str, dst_file_path: str, retries: int = 5, delay: float = 1.0): + for attempt in range(retries): + try: + shutil.move(src_file_path, dst_file_path) + return + except PermissionError: + print(f"Attempt {attempt + 1} failed: File is in use. Retrying in {delay} seconds...") + time.sleep(delay) + raise PermissionError(f"Failed to move file after {retries} attempts.") + +# Helper function to get the last file in a directory sorted by filename +def get_last_file_sorted_by_name(directory: str) -> str: + try: + files = [f for f in os.listdir(directory) if os.path.isfile(os.path.join(directory, f))] + if not files: + print("No files found in the directory.") + return None + files.sort() + last_file = files[-1] + return os.path.join(directory, last_file) + except Exception as e: + print(f"An error occurred: {e}") + return None + +def main(): + # Initialize SVD and FILM pipelines + repo_id = "stabilityai/stable-video-diffusion-img2vid-xt" + svd_xt_pipeline = StableVideoDiffusionPipeline.from_pretrained( + repo_id, torch_dtype=torch.float16, variant="fp16", cache_dir="G:/ai-models/models" + ) + svd_xt_pipeline.enable_model_cpu_offload() + + film_pipeline = FILMPipeline("G:/ai-models/models/film_net_fp16.pt").to(device="cuda", dtype=torch.float16) + + # Load initial input image + image_path = "G:/ai-models/models/gif_frames/donut_motion.png" + image = Image.open(image_path) + + fps = 24.0 + inter_frames = 4 + rounds = 2 # Number of rounds of generation and interpolation + base_output_dir = "G:/ai-models/models" + + all_frames_dir = os.path.join(base_output_dir, "all_interpolated_frames") + os.makedirs(all_frames_dir, exist_ok=True) + + last_frame_for_next_round = os.path.join(base_output_dir, "last_frame_for_next_round.png") + + for round_num in range(1, rounds + 1): + svd_xt_output_dir = os.path.join(base_output_dir, f"svd_xt_output_round_{round_num}") + os.makedirs(svd_xt_output_dir, exist_ok=True) + + # Generate frames using SVD pipeline + generator = torch.manual_seed(42) + frames = svd_xt_pipeline(image, decode_chunk_size=8, generator=generator, output_type="np").frames[0] + + # Save SVD frames to directory + film_writer = DirectoryWriter(svd_xt_output_dir) + for idx, frame in enumerate(frames): + film_writer.write_frame(torch.tensor(frame).permute(2, 0, 1)) + + # Read saved frames for interpolation + film_reader = DirectoryReader(svd_xt_output_dir) + height, width = film_reader.get_resolution() + + # Interpolate frames using FILM pipeline + film_pipeline(film_reader, film_writer, inter_frames=inter_frames) + + # Close reader and writer + film_writer.close() + film_reader.reset() + + # Deleting the SVD generated images. + for i in range(len(frames)): + os.remove(os.path.join(svd_xt_output_dir, f"{i}.png")) + print(f"Deleted the first {len(frames)} frames in directory: {svd_xt_output_dir}") + + # Save the last frame separately for the next round + last_frame_path = get_last_file_sorted_by_name(svd_xt_output_dir) + if last_frame_path: + shutil.copy2(last_frame_path, last_frame_for_next_round) + else: + print("No frames found to copy.") + + # Initialize Upscale pipeline and Upscale the last frame before passing to the next round + upscale_pipeline = UpscalePipeline("stabilityai/stable-diffusion-x4-upscaler", torch_dtype=torch.float16) + upscale_pipeline.enable_model_cpu_offload() + upscale_pipeline.sfast_enabled() + upscaled_image, _ = upscale_pipeline("", image=Image.open(last_frame_for_next_round),) + print('Upscaling of the seed image before next round.') + print(upscaled_image[0].shape) + exit + upscaled_image[0].save(last_frame_for_next_round) + + # Move all interpolated frames to a common directory with a unique naming scheme + for file_name in sorted(os.listdir(svd_xt_output_dir)): + src_file_path = os.path.join(svd_xt_output_dir, file_name) + dst_file_name = f"round_{round_num:03d}_frame_{file_name}" + dst_file_path = os.path.join(all_frames_dir, dst_file_name) + + move_file_with_retry(src_file_path, dst_file_path) + + # Clean up the source directory after moving frames + for file_name in os.listdir(svd_xt_output_dir): + os.remove(os.path.join(svd_xt_output_dir, file_name)) + os.rmdir(svd_xt_output_dir) + + # Ensure all operations on last frame are complete before opening it again + time.sleep(1) # Small delay to ensure file system operations are complete + + # Prepare for next round + image = Image.open(last_frame_for_next_round) + + # Compile all interpolated frames from all rounds into a final video + video_output_dir = "G:/ai-models/models/video_out" + os.makedirs(video_output_dir, exist_ok=True) + + film_output_path = os.path.join(video_output_dir, "output.avi") + frames_compactor(frames=all_frames_dir, output_path=film_output_path, fps=fps, codec="MJPG", is_directory=True) + + # Clean up all frames in the directories after video generation + for file_name in os.listdir(all_frames_dir): + file_path = os.path.join(all_frames_dir, file_name) + if os.path.isfile(file_path): + os.remove(file_path) + os.rmdir(all_frames_dir) + + print(f"All frames deleted from directories.") + print(f"Video generated at: {film_output_path}") + return film_output_path + +if __name__ == "__main__": + main() diff --git a/runner/app/test_examples/test-film-interpolate.py b/runner/app/test_examples/test-film-interpolate.py new file mode 100644 index 00000000..195a3a4b --- /dev/null +++ b/runner/app/test_examples/test-film-interpolate.py @@ -0,0 +1,45 @@ +import os +import requests + +# Define the URL of the FastAPI application +URL = "http://localhost:8000/FILMPipeline" + +# Test with two images +def test_with_two_images(): + image1_path = "G:/ai-models/models/all_interpolated_frames/round_003_frame_74.png" + image2_path = "G:/ai-models/models/all_interpolated_frames/round_003_frame_36.png" + + with open(image1_path, "rb") as image1, open(image2_path, "rb") as image2: + files = { + "image1": ("image1.png", image1, "image/png"), + "image2": ("image2.png", image2, "image/png"), + } + data = { + "inter_frames": 2, + "model_id": "film_net_fp16.pt" + } + response = requests.post(URL, files=files, data=data) + + print("Test with two images") + print(response.status_code) + print(response.json()) + +# Test with a directory of images +def test_with_image_directory(): + image_dir = "path/to/image_directory" + + data = { + "inter_frames": 2, + "model_path": "path/to/film_net_fp16.pt", + "image_dir": image_dir + } + response = requests.post(URL, data=data) + + print("Test with image directory") + print(response.status_code) + print(response.json()) + +if __name__ == "__main__": + # Ensure that the FastAPI server is running before executing these tests + test_with_two_images() + test_with_image_directory() diff --git a/runner/app/test_examples/test-optimized.py b/runner/app/test_examples/test-optimized.py new file mode 100644 index 00000000..db200286 --- /dev/null +++ b/runner/app/test_examples/test-optimized.py @@ -0,0 +1,69 @@ +import torch +import numpy as np +import os +import sys +sys.path.append('C://Users//ganes//ai-worker//runner') +from diffusers import StableVideoDiffusionPipeline +from PIL import PngImagePlugin, Image +from app.pipelines.frame_interpolation import FILMPipeline +from app.pipelines.utils import DirectoryReader, frames_compactor, DirectoryWriter + +# Increase the max text chunk size for PNG images +PngImagePlugin.MAX_TEXT_CHUNK = 100 * (1024**2) + +def main(): + # Initialize pipelines + repo_id = "stabilityai/stable-video-diffusion-img2vid-xt" + svd_xt_pipeline = StableVideoDiffusionPipeline.from_pretrained( + repo_id, torch_dtype=torch.float16, variant="fp16", cache_dir="G:/ai-models/models" + ) + svd_xt_pipeline.enable_model_cpu_offload() + + film_pipeline = FILMPipeline("G:/ai-models/models/film_net_fp16.pt").to(device="cuda", dtype=torch.float16) + + # Load input image + image_path = "G:/ai-models/models/gif_frames/rocket.png" + image = Image.open(image_path) + + # Generate frames using SVD pipeline + generator = torch.manual_seed(42) + frames = svd_xt_pipeline(image, decode_chunk_size=8, generator=generator, output_type="np").frames[0] + + fps = 24.0 + inter_frames = 2 + svd_xt_output_dir = "G:/ai-models/models/svd_xt_output" + video_output_dir = "G:/ai-models/models/video_out" + + # Save SVD frames to directory + film_writer = DirectoryWriter(svd_xt_output_dir) + for frame in frames: + film_writer.write_frame(torch.tensor(frame).permute(2, 0, 1)) + + # Read saved frames for interpolation + film_reader = DirectoryReader(svd_xt_output_dir) + height, width = film_reader.get_resolution() + + # Interpolate frames using FILM pipeline + film_pipeline(film_reader, film_writer, inter_frames=inter_frames) + + # Delete original SVD frames since interpolated frames are also in the same directory. + for i in range(len(frames)): + os.remove(os.path.join(svd_xt_output_dir, f"{i}.png")) + print(f"Deleted the first {len(frames)} frames in directory: {svd_xt_output_dir}") + + # Compile interpolated frames into a video + film_output_path = os.path.join(video_output_dir, "output.avi") + frames_compactor(frames=svd_xt_output_dir, output_path=film_output_path, fps=fps, codec="MJPG", is_directory=True) + + # Clean up all frames in the directory after video generation + for file_name in os.listdir(svd_xt_output_dir): + file_path = os.path.join(svd_xt_output_dir, file_name) + if os.path.isfile(file_path): + os.remove(file_path) + print(f"All frames deleted from directory: {svd_xt_output_dir}") + + print(f"Video generated at: {film_output_path}") + return film_output_path + +if __name__ == "__main__": + main() diff --git a/runner/app/test_examples/utility-func-compactor.py b/runner/app/test_examples/utility-func-compactor.py new file mode 100644 index 00000000..6dac4309 --- /dev/null +++ b/runner/app/test_examples/utility-func-compactor.py @@ -0,0 +1,89 @@ +import subprocess +import numpy as np +import torch +import os +from typing import List, Union +from pathlib import Path +import cv2 + +def generate_video_from_frames( + frames: Union[List[np.ndarray], List[torch.Tensor]], + output_path: str, + fps: float, + codec: str = "MJPEG", + is_directory: bool = False, + width: int = None, + height: int = None +) -> None: + """ + Generate a video from a list of frames. Frames can be from a directory or in-memory. + + Args: + frames (List[np.ndarray] | List[torch.Tensor]): List of frames as NumPy arrays or PyTorch tensors. + output_path (str): Path to save the output video file. + fps (float): Frames per second for the video. + codec (str): Codec used for video compression (default is "MJPEG"). + is_directory (bool): If True, treat `frames` as a directory path containing image files. + width (int): Width of the video. Must be provided if `frames` are in-memory. + height (int): Height of the video. Must be provided if `frames` are in-memory. + + Returns: + None + """ + if is_directory: + # Read frames from a directory + frames = [cv2.imread(os.path.join(frames, file)) for file in sorted(os.listdir(frames))] + else: + # Convert torch tensors to numpy arrays if necessary + if isinstance(frames[0], torch.Tensor): + frames = [frame.permute(1, 2, 0).cpu().numpy() for frame in frames] + + # Ensure frames are numpy arrays and are uint8 type + frames = [frame.astype(np.uint8) for frame in frames] + + # Check if frames are consistent + if not frames: + raise ValueError("No frames to process.") + + if width is None or height is None: + # Use dimensions of the first frame if not provided + height, width = frames[0].shape[:2] + + # Write frames to a temporary directory + temp_dir = Path("temp_frames") + temp_dir.mkdir(exist_ok=True) + for i, frame in enumerate(frames): + cv2.imwrite(str(temp_dir / f"frame_{i:05d}.png"), frame) + + # Build ffmpeg command + ffmpeg_cmd = [ + 'ffmpeg', '-y', '-framerate', str(fps), + '-i', str(temp_dir / 'frame_%05d.png'), + '-c:v', codec, '-pix_fmt', 'yuv420p', + output_path + ] + + # Run ffmpeg command + subprocess.run(ffmpeg_cmd, check=True) + + # Clean up temporary frames + for file in temp_dir.glob("*.png"): + file.unlink() + temp_dir.rmdir() + + print(f"Video saved to {output_path}") + +# Example usage +if __name__ == "__main__": + # Example with in-memory frames (as np.ndarray) + # Assuming `in_memory_frames` is a list of numpy arrays + + # Example with frames from a directory + frames_directory = "G:/ai-models/models/svd_xt_output" + generate_video_from_frames( + frames=frames_directory, + output_path="G:/ai-models/models/video_out/output.mp4", + fps=24.0, + codec="mpeg4", + is_directory=True + ) diff --git a/runner/app/test_examples/utility-function-vid.py b/runner/app/test_examples/utility-function-vid.py new file mode 100644 index 00000000..adf194cc --- /dev/null +++ b/runner/app/test_examples/utility-function-vid.py @@ -0,0 +1,62 @@ +import cv2 +import numpy as np +import tempfile +import os +from io import BytesIO + +def extract_frames_from_video(video_data, is_file_path=True) -> np.ndarray: + """ + Extract frames from a video file or in-memory video data and return them as a NumPy array. + + Args: + video_data (str or BytesIO): Path to the input video file or in-memory video data. + is_file_path (bool): Indicates if video_data is a file path (True) or in-memory data (False). + + Returns: + np.ndarray: Array of frames with shape (num_frames, height, width, channels). + """ + if is_file_path: + # Handle file-based video input + video_capture = cv2.VideoCapture(video_data) + else: + # Handle in-memory video input + # Create a temporary file to store in-memory video data + with tempfile.NamedTemporaryFile(delete=False, suffix='.mp4') as temp_file: + temp_file.write(video_data.getvalue()) + temp_file_path = temp_file.name + + # Open the temporary video file + video_capture = cv2.VideoCapture(temp_file_path) + + if not video_capture.isOpened(): + raise ValueError("Error opening video data") + + frames = [] + success, frame = video_capture.read() + + while success: + frames.append(frame) + success, frame = video_capture.read() + + video_capture.release() + + # Delete the temporary file if it was created + if not is_file_path: + os.remove(temp_file_path) + + # Convert list of frames to a NumPy array + frames_array = np.array(frames) + print(f"Extracted {frames_array.shape[0]} frames from video in shape of {frames_array.shape}") + + return frames_array + +# Example usage +if __name__ == "__main__": + # File path example + video_file_path = "C:/Users/ganes/Desktop/Generated videos/output.mp4" + + + # In-memory video example + with open(video_file_path, "rb") as f: + video_data = BytesIO(f.read()) + frames_array_from_memory = extract_frames_from_video(video_data, is_file_path=False) diff --git a/runner/uvicorn.env b/runner/uvicorn.env new file mode 100644 index 00000000..77869fee --- /dev/null +++ b/runner/uvicorn.env @@ -0,0 +1,4 @@ +# myenvfile.env +MODEL_ID="" +MODEL_DIR="" +PIPELINE=FILMPipeline From 950cdf9beabc8c0e612ec7e952a2d39b666bfb1e Mon Sep 17 00:00:00 2001 From: Jason Stone Date: Sun, 28 Jul 2024 20:32:38 +0300 Subject: [PATCH 26/38] update to sfast optimization to i2i and t2i and upscale pipelines --- runner/app/pipelines/image_to_image.py | 30 +++++++++++++++++++---- runner/app/pipelines/text_to_image.py | 33 ++++++++++++++++++++------ runner/app/pipelines/upscale.py | 30 +++++++++++++++++++---- 3 files changed, 76 insertions(+), 17 deletions(-) diff --git a/runner/app/pipelines/image_to_image.py b/runner/app/pipelines/image_to_image.py index 4080c919..d5c821a5 100644 --- a/runner/app/pipelines/image_to_image.py +++ b/runner/app/pipelines/image_to_image.py @@ -1,6 +1,7 @@ import logging import os from enum import Enum +import time from typing import List, Optional, Tuple import PIL @@ -30,6 +31,7 @@ logger = logging.getLogger(__name__) +SFAST_WARMUP_ITERATIONS = 2 # Model warm-up iterations when SFAST is enabled. class ModelName(Enum): """Enumeration mapping model names to their corresponding IDs.""" @@ -142,11 +144,29 @@ def __init__(self, model_id: str): # Warm-up the pipeline. # TODO: Not yet supported for ImageToImagePipeline. if os.getenv("SFAST_WARMUP", "true").lower() == "true": - logger.warning( - "The 'SFAST_WARMUP' flag is not yet supported for the " - "ImageToImagePipeline and will be ignored. As a result the first " - "call may be slow if 'SFAST' is enabled." - ) + warmup_kwargs = { + "prompt":"A warmed up pipeline is a happy pipeline a short poem by ricksta", + "image": PIL.Image.new("RGB", (576, 1024)), + "strength": 0.8, + "negative_prompt": "No blurry or weird artifacts", + "num_images_per_prompt":4, + } + + logger.info("Warming up ImageToImagePipeline pipeline...") + total_time = 0 + for ii in range(SFAST_WARMUP_ITERATIONS): + t = time.time() + try: + self.ldm(**warmup_kwargs).images + except Exception as e: + logger.error(f"ImageToImagePipeline warmup error: {e}") + raise e + iteration_time = time.time() - t + total_time += iteration_time + logger.info( + "Warmup iteration %s took %s seconds", ii + 1, iteration_time + ) + logger.info("Total warmup time: %s seconds", total_time) if deepcache_enabled and not ( is_lightning_model(model_id) or is_turbo_model(model_id) diff --git a/runner/app/pipelines/text_to_image.py b/runner/app/pipelines/text_to_image.py index e2d6c692..85f37cdb 100644 --- a/runner/app/pipelines/text_to_image.py +++ b/runner/app/pipelines/text_to_image.py @@ -1,6 +1,7 @@ import logging import os from enum import Enum +import time from typing import List, Optional, Tuple import PIL @@ -26,6 +27,7 @@ logger = logging.getLogger(__name__) +SFAST_WARMUP_ITERATIONS = 2 # Model warm-up iterations when SFAST is enabled. class ModelName(Enum): """Enumeration mapping model names to their corresponding IDs.""" @@ -151,14 +153,31 @@ def __init__(self, model_id: str): self.ldm = compile_model(self.ldm) - # Warm-up the pipeline. - # TODO: Not yet supported for TextToImagePipeline. if os.getenv("SFAST_WARMUP", "true").lower() == "true": - logger.warning( - "The 'SFAST_WARMUP' flag is not yet supported for the " - "TextToImagePipeline and will be ignored. As a result the first " - "call may be slow if 'SFAST' is enabled." - ) + # Retrieve default model params. + # TODO: Retrieve defaults from Pydantic class in route. + warmup_kwargs = { + "prompt": "A happy pipe in the line looking at the wall with words sfast", + "num_images_per_prompt": 4, + "negative_prompt": "No blurry or weird artifacts", + } + + logger.info("Warming up TextToImagePipeline pipeline...") + total_time = 0 + for ii in range(SFAST_WARMUP_ITERATIONS): + t = time.time() + try: + self.ldm(**warmup_kwargs).images + except Exception as e: + # FIXME: When out of memory, pipeline is corrupted. + logger.error(f"TextToImagePipeline warmup error: {e}") + raise e + iteration_time = time.time() - t + total_time += iteration_time + logger.info( + "Warmup iteration %s took %s seconds", ii + 1, iteration_time + ) + logger.info("Total warmup time: %s seconds", total_time) if deepcache_enabled and not ( is_lightning_model(model_id) or is_turbo_model(model_id) diff --git a/runner/app/pipelines/upscale.py b/runner/app/pipelines/upscale.py index 6fd3cefe..b743434b 100644 --- a/runner/app/pipelines/upscale.py +++ b/runner/app/pipelines/upscale.py @@ -1,5 +1,6 @@ import logging import os +import time from typing import List, Optional, Tuple import PIL @@ -21,6 +22,7 @@ logger = logging.getLogger(__name__) +SFAST_WARMUP_ITERATIONS = 2 # Model warm-up iterations when SFAST is enabled. class UpscalePipeline(Pipeline): def __init__(self, model_id: str): @@ -68,11 +70,29 @@ def __init__(self, model_id: str): # Warm-up the pipeline. # TODO: Not yet supported for UpscalePipeline. if os.getenv("SFAST_WARMUP", "true").lower() == "true": - logger.warning( - "The 'SFAST_WARMUP' flag is not yet supported for the " - "UpscalePipeline and will be ignored. As a result the first " - "call may be slow if 'SFAST' is enabled." - ) + # Retrieve default model params. + # TODO: Retrieve defaults from Pydantic class in route. + warmup_kwargs = { + "prompt": "Upscaling the pipeline with sfast enabled", + "image": PIL.Image.new("RGB", (576, 1024)), + } + + logger.info("Warming up ImageToVideoPipeline pipeline...") + total_time = 0 + for ii in range(SFAST_WARMUP_ITERATIONS): + t = time.time() + try: + self.ldm(**warmup_kwargs).images + except Exception as e: + # FIXME: When out of memory, pipeline is corrupted. + logger.error(f"ImageToVideoPipeline warmup error: {e}") + raise e + iteration_time = time.time() - t + total_time += iteration_time + logger.info( + "Warmup iteration %s took %s seconds", ii + 1, iteration_time + ) + logger.info("Total warmup time: %s seconds", total_time) if deepcache_enabled and not ( is_lightning_model(model_id) or is_turbo_model(model_id) From b48692b6499dc93a50eca8081939654bc69ecb5d Mon Sep 17 00:00:00 2001 From: Jason Stone Date: Sun, 28 Jul 2024 23:07:09 +0300 Subject: [PATCH 27/38] changes to extra files --- runner/app/.gitignore/base64-image.py | 62 +++++++++++++++++++ .../continuous-creation.py | 0 .../generate-interpolate-upscale.py | 0 .../test-film-interpolate.py | 0 .../test-optimized.py | 0 .../utility-func-compactor.py | 0 .../utility-function-vid.py | 0 runner/app/pipelines/optim/sfast.py | 2 +- runner/uvicorn.env | 4 -- 9 files changed, 63 insertions(+), 5 deletions(-) create mode 100644 runner/app/.gitignore/base64-image.py rename runner/app/{test_examples => .gitignore}/continuous-creation.py (100%) rename runner/app/{test_examples => .gitignore}/generate-interpolate-upscale.py (100%) rename runner/app/{test_examples => .gitignore}/test-film-interpolate.py (100%) rename runner/app/{test_examples => .gitignore}/test-optimized.py (100%) rename runner/app/{test_examples => .gitignore}/utility-func-compactor.py (100%) rename runner/app/{test_examples => .gitignore}/utility-function-vid.py (100%) delete mode 100644 runner/uvicorn.env diff --git a/runner/app/.gitignore/base64-image.py b/runner/app/.gitignore/base64-image.py new file mode 100644 index 00000000..9116fdbe --- /dev/null +++ b/runner/app/.gitignore/base64-image.py @@ -0,0 +1,62 @@ +import json +import base64 +from PIL import Image, UnidentifiedImageError +from io import BytesIO +import os + +def extract_base64_string(data_url): + # Remove the 'data:image/png;base64,' prefix + base64_str = data_url.split(',', 1)[1] + return base64_str + +def add_padding(base64_string): + # Add padding if necessary + missing_padding = len(base64_string) % 4 + if missing_padding: + base64_string += '=' * (4 - missing_padding) + return base64_string + +def convert_base64_to_image(base64_string, output_path): + try: + # Add padding to the base64 string + base64_string = add_padding(base64_string) + + # Decode the base64 string to bytes + image_data = base64.b64decode(base64_string) + + # Convert bytes to an image + image = Image.open(BytesIO(image_data)) + + # Save the image to a file + image.save(output_path) + print(f"Image saved successfully to {output_path}") + except (base64.binascii.Error, UnidentifiedImageError) as e: + print(f"Failed to decode and save image: {e}") + +def extract_and_convert_images(json_file_path, output_dir): + try: + # Read the JSON file + with open(json_file_path, 'r') as file: + data = json.load(file) + + # Create the output directory if it doesn't exist + os.makedirs(output_dir, exist_ok=True) + + # Extract the base64 strings from the URLs + if 'images' in data: + for idx, image_info in enumerate(data['images']): + if 'url' in image_info: + data_url = image_info['url'] + base64_string = extract_base64_string(data_url) + + output_image_path = os.path.join(output_dir, f'image_{idx}.png') + convert_base64_to_image(base64_string, output_image_path) + else: + print("Invalid JSON schema or missing 'url' field") + except json.JSONDecodeError as e: + print(f"Failed to parse JSON file: {e}") + +# Example usage +json_file_path = 'G:/ai-models/models/response_1722196176417.json' # Path to your JSON file +output_image_path = 'G:/ai-models/models/output_image.jpg' # Path to save the output image +extract_and_convert_images(json_file_path, output_image_path) diff --git a/runner/app/test_examples/continuous-creation.py b/runner/app/.gitignore/continuous-creation.py similarity index 100% rename from runner/app/test_examples/continuous-creation.py rename to runner/app/.gitignore/continuous-creation.py diff --git a/runner/app/test_examples/generate-interpolate-upscale.py b/runner/app/.gitignore/generate-interpolate-upscale.py similarity index 100% rename from runner/app/test_examples/generate-interpolate-upscale.py rename to runner/app/.gitignore/generate-interpolate-upscale.py diff --git a/runner/app/test_examples/test-film-interpolate.py b/runner/app/.gitignore/test-film-interpolate.py similarity index 100% rename from runner/app/test_examples/test-film-interpolate.py rename to runner/app/.gitignore/test-film-interpolate.py diff --git a/runner/app/test_examples/test-optimized.py b/runner/app/.gitignore/test-optimized.py similarity index 100% rename from runner/app/test_examples/test-optimized.py rename to runner/app/.gitignore/test-optimized.py diff --git a/runner/app/test_examples/utility-func-compactor.py b/runner/app/.gitignore/utility-func-compactor.py similarity index 100% rename from runner/app/test_examples/utility-func-compactor.py rename to runner/app/.gitignore/utility-func-compactor.py diff --git a/runner/app/test_examples/utility-function-vid.py b/runner/app/.gitignore/utility-function-vid.py similarity index 100% rename from runner/app/test_examples/utility-function-vid.py rename to runner/app/.gitignore/utility-function-vid.py diff --git a/runner/app/pipelines/optim/sfast.py b/runner/app/pipelines/optim/sfast.py index c449aadb..001a2b53 100644 --- a/runner/app/pipelines/optim/sfast.py +++ b/runner/app/pipelines/optim/sfast.py @@ -1,6 +1,6 @@ """This module provides a function to enable StableFast optimization for the pipeline. -For more information, see the DeepCache project on GitHub: https://github.com/chengzeyi/stable-fast +For more information, see the Stable Fast project on GitHub: https://github.com/chengzeyi/stable-fast """ import logging diff --git a/runner/uvicorn.env b/runner/uvicorn.env deleted file mode 100644 index 77869fee..00000000 --- a/runner/uvicorn.env +++ /dev/null @@ -1,4 +0,0 @@ -# myenvfile.env -MODEL_ID="" -MODEL_DIR="" -PIPELINE=FILMPipeline From 2d25c46eecc26f177127d764835f78734bc161ec Mon Sep 17 00:00:00 2001 From: Jason Stone Date: Sun, 28 Jul 2024 23:18:03 +0300 Subject: [PATCH 28/38] added git ignore to the files to remove unnecessary files --- .gitignore | 1 + runner/app/{.gitignore => tests-examples}/base64-image.py | 0 runner/app/{.gitignore => tests-examples}/continuous-creation.py | 0 .../generate-interpolate-upscale.py | 0 .../app/{.gitignore => tests-examples}/test-film-interpolate.py | 0 runner/app/{.gitignore => tests-examples}/test-optimized.py | 0 .../app/{.gitignore => tests-examples}/utility-func-compactor.py | 0 .../app/{.gitignore => tests-examples}/utility-function-vid.py | 0 8 files changed, 1 insertion(+) rename runner/app/{.gitignore => tests-examples}/base64-image.py (100%) rename runner/app/{.gitignore => tests-examples}/continuous-creation.py (100%) rename runner/app/{.gitignore => tests-examples}/generate-interpolate-upscale.py (100%) rename runner/app/{.gitignore => tests-examples}/test-film-interpolate.py (100%) rename runner/app/{.gitignore => tests-examples}/test-optimized.py (100%) rename runner/app/{.gitignore => tests-examples}/utility-func-compactor.py (100%) rename runner/app/{.gitignore => tests-examples}/utility-function-vid.py (100%) diff --git a/.gitignore b/.gitignore index 0a9429b0..1832d101 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ output aiModels.json models checkpoints +C:/Users/ganes/ai-worker/runner/app/tests-examples # IDE .vscode diff --git a/runner/app/.gitignore/base64-image.py b/runner/app/tests-examples/base64-image.py similarity index 100% rename from runner/app/.gitignore/base64-image.py rename to runner/app/tests-examples/base64-image.py diff --git a/runner/app/.gitignore/continuous-creation.py b/runner/app/tests-examples/continuous-creation.py similarity index 100% rename from runner/app/.gitignore/continuous-creation.py rename to runner/app/tests-examples/continuous-creation.py diff --git a/runner/app/.gitignore/generate-interpolate-upscale.py b/runner/app/tests-examples/generate-interpolate-upscale.py similarity index 100% rename from runner/app/.gitignore/generate-interpolate-upscale.py rename to runner/app/tests-examples/generate-interpolate-upscale.py diff --git a/runner/app/.gitignore/test-film-interpolate.py b/runner/app/tests-examples/test-film-interpolate.py similarity index 100% rename from runner/app/.gitignore/test-film-interpolate.py rename to runner/app/tests-examples/test-film-interpolate.py diff --git a/runner/app/.gitignore/test-optimized.py b/runner/app/tests-examples/test-optimized.py similarity index 100% rename from runner/app/.gitignore/test-optimized.py rename to runner/app/tests-examples/test-optimized.py diff --git a/runner/app/.gitignore/utility-func-compactor.py b/runner/app/tests-examples/utility-func-compactor.py similarity index 100% rename from runner/app/.gitignore/utility-func-compactor.py rename to runner/app/tests-examples/utility-func-compactor.py diff --git a/runner/app/.gitignore/utility-function-vid.py b/runner/app/tests-examples/utility-function-vid.py similarity index 100% rename from runner/app/.gitignore/utility-function-vid.py rename to runner/app/tests-examples/utility-function-vid.py From eb5ae468328a709025902aaecc42577ee327cefb Mon Sep 17 00:00:00 2001 From: Jason Stone Date: Sun, 28 Jul 2024 23:22:25 +0300 Subject: [PATCH 29/38] files not removed checking again --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 1832d101..abd3b40b 100644 --- a/.gitignore +++ b/.gitignore @@ -8,7 +8,7 @@ output aiModels.json models checkpoints -C:/Users/ganes/ai-worker/runner/app/tests-examples +tests-examples # IDE .vscode From e9b39650e680368e6bd3db3ff58004962b7fc7a5 Mon Sep 17 00:00:00 2001 From: Jason Stone Date: Sun, 28 Jul 2024 23:25:13 +0300 Subject: [PATCH 30/38] still in test phase --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index abd3b40b..73953d48 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ cache __pycache__ input output +runner\app\tests-examples # AI Configuration files and model storage folders. aiModels.json From 19261b6ecfff1a280f07c0dd607e4beb4881dbb3 Mon Sep 17 00:00:00 2001 From: Jason Stone Date: Sun, 28 Jul 2024 23:32:40 +0300 Subject: [PATCH 31/38] test-test --- .gitignore | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 73953d48..05c24bab 100644 --- a/.gitignore +++ b/.gitignore @@ -3,13 +3,12 @@ cache __pycache__ input output -runner\app\tests-examples +runner/app/tests-examples # AI Configuration files and model storage folders. aiModels.json models checkpoints -tests-examples # IDE .vscode From cb6498e499bf095c37d698b4edcaaaea9b60bbef Mon Sep 17 00:00:00 2001 From: Jason <83615043+JJassonn69@users.noreply.github.com> Date: Sun, 28 Jul 2024 23:34:27 +0300 Subject: [PATCH 32/38] Update .gitignore --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index 05c24bab..0a9429b0 100644 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,6 @@ cache __pycache__ input output -runner/app/tests-examples # AI Configuration files and model storage folders. aiModels.json From b91da520e5c32034118a69e8dba7a8aac48a2344 Mon Sep 17 00:00:00 2001 From: Jason <83615043+JJassonn69@users.noreply.github.com> Date: Sun, 28 Jul 2024 23:34:53 +0300 Subject: [PATCH 33/38] Delete runner/app/tests-examples directory --- runner/app/tests-examples/base64-image.py | 62 ------- .../app/tests-examples/continuous-creation.py | 154 ----------------- .../generate-interpolate-upscale.py | 159 ------------------ .../tests-examples/test-film-interpolate.py | 45 ----- runner/app/tests-examples/test-optimized.py | 69 -------- .../tests-examples/utility-func-compactor.py | 89 ---------- .../tests-examples/utility-function-vid.py | 62 ------- 7 files changed, 640 deletions(-) delete mode 100644 runner/app/tests-examples/base64-image.py delete mode 100644 runner/app/tests-examples/continuous-creation.py delete mode 100644 runner/app/tests-examples/generate-interpolate-upscale.py delete mode 100644 runner/app/tests-examples/test-film-interpolate.py delete mode 100644 runner/app/tests-examples/test-optimized.py delete mode 100644 runner/app/tests-examples/utility-func-compactor.py delete mode 100644 runner/app/tests-examples/utility-function-vid.py diff --git a/runner/app/tests-examples/base64-image.py b/runner/app/tests-examples/base64-image.py deleted file mode 100644 index 9116fdbe..00000000 --- a/runner/app/tests-examples/base64-image.py +++ /dev/null @@ -1,62 +0,0 @@ -import json -import base64 -from PIL import Image, UnidentifiedImageError -from io import BytesIO -import os - -def extract_base64_string(data_url): - # Remove the 'data:image/png;base64,' prefix - base64_str = data_url.split(',', 1)[1] - return base64_str - -def add_padding(base64_string): - # Add padding if necessary - missing_padding = len(base64_string) % 4 - if missing_padding: - base64_string += '=' * (4 - missing_padding) - return base64_string - -def convert_base64_to_image(base64_string, output_path): - try: - # Add padding to the base64 string - base64_string = add_padding(base64_string) - - # Decode the base64 string to bytes - image_data = base64.b64decode(base64_string) - - # Convert bytes to an image - image = Image.open(BytesIO(image_data)) - - # Save the image to a file - image.save(output_path) - print(f"Image saved successfully to {output_path}") - except (base64.binascii.Error, UnidentifiedImageError) as e: - print(f"Failed to decode and save image: {e}") - -def extract_and_convert_images(json_file_path, output_dir): - try: - # Read the JSON file - with open(json_file_path, 'r') as file: - data = json.load(file) - - # Create the output directory if it doesn't exist - os.makedirs(output_dir, exist_ok=True) - - # Extract the base64 strings from the URLs - if 'images' in data: - for idx, image_info in enumerate(data['images']): - if 'url' in image_info: - data_url = image_info['url'] - base64_string = extract_base64_string(data_url) - - output_image_path = os.path.join(output_dir, f'image_{idx}.png') - convert_base64_to_image(base64_string, output_image_path) - else: - print("Invalid JSON schema or missing 'url' field") - except json.JSONDecodeError as e: - print(f"Failed to parse JSON file: {e}") - -# Example usage -json_file_path = 'G:/ai-models/models/response_1722196176417.json' # Path to your JSON file -output_image_path = 'G:/ai-models/models/output_image.jpg' # Path to save the output image -extract_and_convert_images(json_file_path, output_image_path) diff --git a/runner/app/tests-examples/continuous-creation.py b/runner/app/tests-examples/continuous-creation.py deleted file mode 100644 index 54473930..00000000 --- a/runner/app/tests-examples/continuous-creation.py +++ /dev/null @@ -1,154 +0,0 @@ -import torch -import numpy as np -import os -import shutil -import time -import sys -sys.path.append('C://Users//ganes//ai-worker//runner') -from PIL import PngImagePlugin, Image -from diffusers import StableVideoDiffusionPipeline -from app.pipelines.frame_interpolation import FILMPipeline -from app.pipelines.utils import DirectoryReader, frames_compactor, DirectoryWriter - -# Increase the max text chunk size for PNG images -PngImagePlugin.MAX_TEXT_CHUNK = 100 * (1024**2) - -def move_file_with_retry(src_file_path: str, dst_file_path: str, retries: int = 5, delay: float = 1.0): - for attempt in range(retries): - try: - shutil.move(src_file_path, dst_file_path) - return - except PermissionError: - print(f"Attempt {attempt + 1} failed: File is in use. Retrying in {delay} seconds...") - time.sleep(delay) - raise PermissionError(f"Failed to move file after {retries} attempts.") - -def get_last_file_sorted_by_name(directory: str) -> str: - """ - Get the last file in the directory when sorted by filename. - - Args: - directory (str): Path to the directory. - - Returns: - str: Path to the last file in the sorted list, or None if directory is empty. - """ - try: - # List all files in the directory - files = [f for f in os.listdir(directory) if os.path.isfile(os.path.join(directory, f))] - - if not files: - print("No files found in the directory.") - return None - - # Sort files by name - files.sort() - - # Get the last file in the sorted list - last_file = files[-1] - - return os.path.join(directory, last_file) - - except Exception as e: - print(f"An error occurred: {e}") - return None - -def main(): - # Initialize pipelines - repo_id = "stabilityai/stable-video-diffusion-img2vid-xt" - svd_xt_pipeline = StableVideoDiffusionPipeline.from_pretrained( - repo_id, torch_dtype=torch.float16, variant="fp16", cache_dir="G:/ai-models/models" - ) - svd_xt_pipeline.enable_model_cpu_offload() - - film_pipeline = FILMPipeline("G:/ai-models/models/film_net_fp16.pt").to(device="cuda", dtype=torch.float16) - - # Load initial input image - image_path = "G:/ai-models/models/gif_frames/donut_motion.png" - image = Image.open(image_path) - - fps = 24.0 - inter_frames = 4 - rounds = 2 # Number of rounds of generation and interpolation - base_output_dir = "G:/ai-models/models" - - all_frames_dir = os.path.join(base_output_dir, "all_interpolated_frames") - os.makedirs(all_frames_dir, exist_ok=True) - - last_frame_for_next_round = os.path.join(base_output_dir, "last_frame_for_next_round.png") - - for round_num in range(1, rounds + 1): - svd_xt_output_dir = os.path.join(base_output_dir, f"svd_xt_output_round_{round_num}") - os.makedirs(svd_xt_output_dir, exist_ok=True) - - # Generate frames using SVD pipeline - generator = torch.manual_seed(42) - frames = svd_xt_pipeline(image, decode_chunk_size=8, generator=generator, output_type="np").frames[0] - - # Save SVD frames to directory - film_writer = DirectoryWriter(svd_xt_output_dir) - for idx, frame in enumerate(frames): - film_writer.write_frame(torch.tensor(frame).permute(2, 0, 1)) - - # Read saved frames for interpolation - film_reader = DirectoryReader(svd_xt_output_dir) - height, width = film_reader.get_resolution() - - # Interpolate frames using FILM pipeline - film_pipeline(film_reader, film_writer, inter_frames=inter_frames) - - # Close reader and writer - film_writer.close() - film_reader.reset() - - # Deleting the SVD generated images. - for i in range(len(frames)): - os.remove(os.path.join(svd_xt_output_dir, f"{i}.png")) - print(f"Deleted the first {len(frames)} frames in directory: {svd_xt_output_dir}") - - # Save the last frame separately for the next round - last_frame_path = get_last_file_sorted_by_name(svd_xt_output_dir) - if last_frame_path: - shutil.copy2(last_frame_path, last_frame_for_next_round) - else: - print("No frames found to copy.") - - # Move all interpolated frames to a common directory with a unique naming scheme - for file_name in sorted(os.listdir(svd_xt_output_dir)): - src_file_path = os.path.join(svd_xt_output_dir, file_name) - dst_file_name = f"round_{round_num:03d}_frame_{file_name}" - dst_file_path = os.path.join(all_frames_dir, dst_file_name) - - move_file_with_retry(src_file_path, dst_file_path) - - # Clean up the source directory after moving frames - for file_name in os.listdir(svd_xt_output_dir): - os.remove(os.path.join(svd_xt_output_dir, file_name)) - os.rmdir(svd_xt_output_dir) - - # Ensure all operations on last frame are complete before opening it again - time.sleep(1) # Small delay to ensure file system operations are complete - - # Prepare for next round - image = Image.open(last_frame_for_next_round) - - # Compile all interpolated frames from all rounds into a final video - video_output_dir = "G:/ai-models/models/video_out" - os.makedirs(video_output_dir, exist_ok=True) - - film_output_path = os.path.join(video_output_dir, "output.avi") - frames_compactor(frames=all_frames_dir, output_path=film_output_path, fps=fps, codec="MJPG", is_directory=True) - - # Clean up all frames in the directories after video generation - for file_name in os.listdir(all_frames_dir): - file_path = os.path.join(all_frames_dir, file_name) - if os.path.isfile(file_path): - os.remove(file_path) - os.rmdir(all_frames_dir) - - print(f"All frames deleted from directories.") - print(f"Video generated at: {film_output_path}") - return film_output_path - -if __name__ == "__main__": - main() diff --git a/runner/app/tests-examples/generate-interpolate-upscale.py b/runner/app/tests-examples/generate-interpolate-upscale.py deleted file mode 100644 index 69954a0a..00000000 --- a/runner/app/tests-examples/generate-interpolate-upscale.py +++ /dev/null @@ -1,159 +0,0 @@ -import logging -import os -os.environ["MODEL_DIR"] = "G://ai-models//models" -import shutil -import time -import sys -sys.path.append('C://Users//ganes//ai-worker//runner') -from typing import List, Optional, Tuple -import PIL -import torch -from diffusers import StableVideoDiffusionPipeline -from PIL import PngImagePlugin, Image, ImageFile - -from app.pipelines.base import Pipeline -from app.pipelines.frame_interpolation import FILMPipeline -from app.pipelines.upscale import UpscalePipeline -from app.pipelines.utils import DirectoryReader, frames_compactor, DirectoryWriter, SafetyChecker, get_model_dir, get_torch_device, is_lightning_model, is_turbo_model -from huggingface_hub import file_download - -# Increase the max text chunk size for PNG images -PngImagePlugin.MAX_TEXT_CHUNK = 100 * (1024**2) -ImageFile.LOAD_TRUNCATED_IMAGES = True - -logger = logging.getLogger(__name__) - -# Helper function to move files with retry mechanism -def move_file_with_retry(src_file_path: str, dst_file_path: str, retries: int = 5, delay: float = 1.0): - for attempt in range(retries): - try: - shutil.move(src_file_path, dst_file_path) - return - except PermissionError: - print(f"Attempt {attempt + 1} failed: File is in use. Retrying in {delay} seconds...") - time.sleep(delay) - raise PermissionError(f"Failed to move file after {retries} attempts.") - -# Helper function to get the last file in a directory sorted by filename -def get_last_file_sorted_by_name(directory: str) -> str: - try: - files = [f for f in os.listdir(directory) if os.path.isfile(os.path.join(directory, f))] - if not files: - print("No files found in the directory.") - return None - files.sort() - last_file = files[-1] - return os.path.join(directory, last_file) - except Exception as e: - print(f"An error occurred: {e}") - return None - -def main(): - # Initialize SVD and FILM pipelines - repo_id = "stabilityai/stable-video-diffusion-img2vid-xt" - svd_xt_pipeline = StableVideoDiffusionPipeline.from_pretrained( - repo_id, torch_dtype=torch.float16, variant="fp16", cache_dir="G:/ai-models/models" - ) - svd_xt_pipeline.enable_model_cpu_offload() - - film_pipeline = FILMPipeline("G:/ai-models/models/film_net_fp16.pt").to(device="cuda", dtype=torch.float16) - - # Load initial input image - image_path = "G:/ai-models/models/gif_frames/donut_motion.png" - image = Image.open(image_path) - - fps = 24.0 - inter_frames = 4 - rounds = 2 # Number of rounds of generation and interpolation - base_output_dir = "G:/ai-models/models" - - all_frames_dir = os.path.join(base_output_dir, "all_interpolated_frames") - os.makedirs(all_frames_dir, exist_ok=True) - - last_frame_for_next_round = os.path.join(base_output_dir, "last_frame_for_next_round.png") - - for round_num in range(1, rounds + 1): - svd_xt_output_dir = os.path.join(base_output_dir, f"svd_xt_output_round_{round_num}") - os.makedirs(svd_xt_output_dir, exist_ok=True) - - # Generate frames using SVD pipeline - generator = torch.manual_seed(42) - frames = svd_xt_pipeline(image, decode_chunk_size=8, generator=generator, output_type="np").frames[0] - - # Save SVD frames to directory - film_writer = DirectoryWriter(svd_xt_output_dir) - for idx, frame in enumerate(frames): - film_writer.write_frame(torch.tensor(frame).permute(2, 0, 1)) - - # Read saved frames for interpolation - film_reader = DirectoryReader(svd_xt_output_dir) - height, width = film_reader.get_resolution() - - # Interpolate frames using FILM pipeline - film_pipeline(film_reader, film_writer, inter_frames=inter_frames) - - # Close reader and writer - film_writer.close() - film_reader.reset() - - # Deleting the SVD generated images. - for i in range(len(frames)): - os.remove(os.path.join(svd_xt_output_dir, f"{i}.png")) - print(f"Deleted the first {len(frames)} frames in directory: {svd_xt_output_dir}") - - # Save the last frame separately for the next round - last_frame_path = get_last_file_sorted_by_name(svd_xt_output_dir) - if last_frame_path: - shutil.copy2(last_frame_path, last_frame_for_next_round) - else: - print("No frames found to copy.") - - # Initialize Upscale pipeline and Upscale the last frame before passing to the next round - upscale_pipeline = UpscalePipeline("stabilityai/stable-diffusion-x4-upscaler", torch_dtype=torch.float16) - upscale_pipeline.enable_model_cpu_offload() - upscale_pipeline.sfast_enabled() - upscaled_image, _ = upscale_pipeline("", image=Image.open(last_frame_for_next_round),) - print('Upscaling of the seed image before next round.') - print(upscaled_image[0].shape) - exit - upscaled_image[0].save(last_frame_for_next_round) - - # Move all interpolated frames to a common directory with a unique naming scheme - for file_name in sorted(os.listdir(svd_xt_output_dir)): - src_file_path = os.path.join(svd_xt_output_dir, file_name) - dst_file_name = f"round_{round_num:03d}_frame_{file_name}" - dst_file_path = os.path.join(all_frames_dir, dst_file_name) - - move_file_with_retry(src_file_path, dst_file_path) - - # Clean up the source directory after moving frames - for file_name in os.listdir(svd_xt_output_dir): - os.remove(os.path.join(svd_xt_output_dir, file_name)) - os.rmdir(svd_xt_output_dir) - - # Ensure all operations on last frame are complete before opening it again - time.sleep(1) # Small delay to ensure file system operations are complete - - # Prepare for next round - image = Image.open(last_frame_for_next_round) - - # Compile all interpolated frames from all rounds into a final video - video_output_dir = "G:/ai-models/models/video_out" - os.makedirs(video_output_dir, exist_ok=True) - - film_output_path = os.path.join(video_output_dir, "output.avi") - frames_compactor(frames=all_frames_dir, output_path=film_output_path, fps=fps, codec="MJPG", is_directory=True) - - # Clean up all frames in the directories after video generation - for file_name in os.listdir(all_frames_dir): - file_path = os.path.join(all_frames_dir, file_name) - if os.path.isfile(file_path): - os.remove(file_path) - os.rmdir(all_frames_dir) - - print(f"All frames deleted from directories.") - print(f"Video generated at: {film_output_path}") - return film_output_path - -if __name__ == "__main__": - main() diff --git a/runner/app/tests-examples/test-film-interpolate.py b/runner/app/tests-examples/test-film-interpolate.py deleted file mode 100644 index 195a3a4b..00000000 --- a/runner/app/tests-examples/test-film-interpolate.py +++ /dev/null @@ -1,45 +0,0 @@ -import os -import requests - -# Define the URL of the FastAPI application -URL = "http://localhost:8000/FILMPipeline" - -# Test with two images -def test_with_two_images(): - image1_path = "G:/ai-models/models/all_interpolated_frames/round_003_frame_74.png" - image2_path = "G:/ai-models/models/all_interpolated_frames/round_003_frame_36.png" - - with open(image1_path, "rb") as image1, open(image2_path, "rb") as image2: - files = { - "image1": ("image1.png", image1, "image/png"), - "image2": ("image2.png", image2, "image/png"), - } - data = { - "inter_frames": 2, - "model_id": "film_net_fp16.pt" - } - response = requests.post(URL, files=files, data=data) - - print("Test with two images") - print(response.status_code) - print(response.json()) - -# Test with a directory of images -def test_with_image_directory(): - image_dir = "path/to/image_directory" - - data = { - "inter_frames": 2, - "model_path": "path/to/film_net_fp16.pt", - "image_dir": image_dir - } - response = requests.post(URL, data=data) - - print("Test with image directory") - print(response.status_code) - print(response.json()) - -if __name__ == "__main__": - # Ensure that the FastAPI server is running before executing these tests - test_with_two_images() - test_with_image_directory() diff --git a/runner/app/tests-examples/test-optimized.py b/runner/app/tests-examples/test-optimized.py deleted file mode 100644 index db200286..00000000 --- a/runner/app/tests-examples/test-optimized.py +++ /dev/null @@ -1,69 +0,0 @@ -import torch -import numpy as np -import os -import sys -sys.path.append('C://Users//ganes//ai-worker//runner') -from diffusers import StableVideoDiffusionPipeline -from PIL import PngImagePlugin, Image -from app.pipelines.frame_interpolation import FILMPipeline -from app.pipelines.utils import DirectoryReader, frames_compactor, DirectoryWriter - -# Increase the max text chunk size for PNG images -PngImagePlugin.MAX_TEXT_CHUNK = 100 * (1024**2) - -def main(): - # Initialize pipelines - repo_id = "stabilityai/stable-video-diffusion-img2vid-xt" - svd_xt_pipeline = StableVideoDiffusionPipeline.from_pretrained( - repo_id, torch_dtype=torch.float16, variant="fp16", cache_dir="G:/ai-models/models" - ) - svd_xt_pipeline.enable_model_cpu_offload() - - film_pipeline = FILMPipeline("G:/ai-models/models/film_net_fp16.pt").to(device="cuda", dtype=torch.float16) - - # Load input image - image_path = "G:/ai-models/models/gif_frames/rocket.png" - image = Image.open(image_path) - - # Generate frames using SVD pipeline - generator = torch.manual_seed(42) - frames = svd_xt_pipeline(image, decode_chunk_size=8, generator=generator, output_type="np").frames[0] - - fps = 24.0 - inter_frames = 2 - svd_xt_output_dir = "G:/ai-models/models/svd_xt_output" - video_output_dir = "G:/ai-models/models/video_out" - - # Save SVD frames to directory - film_writer = DirectoryWriter(svd_xt_output_dir) - for frame in frames: - film_writer.write_frame(torch.tensor(frame).permute(2, 0, 1)) - - # Read saved frames for interpolation - film_reader = DirectoryReader(svd_xt_output_dir) - height, width = film_reader.get_resolution() - - # Interpolate frames using FILM pipeline - film_pipeline(film_reader, film_writer, inter_frames=inter_frames) - - # Delete original SVD frames since interpolated frames are also in the same directory. - for i in range(len(frames)): - os.remove(os.path.join(svd_xt_output_dir, f"{i}.png")) - print(f"Deleted the first {len(frames)} frames in directory: {svd_xt_output_dir}") - - # Compile interpolated frames into a video - film_output_path = os.path.join(video_output_dir, "output.avi") - frames_compactor(frames=svd_xt_output_dir, output_path=film_output_path, fps=fps, codec="MJPG", is_directory=True) - - # Clean up all frames in the directory after video generation - for file_name in os.listdir(svd_xt_output_dir): - file_path = os.path.join(svd_xt_output_dir, file_name) - if os.path.isfile(file_path): - os.remove(file_path) - print(f"All frames deleted from directory: {svd_xt_output_dir}") - - print(f"Video generated at: {film_output_path}") - return film_output_path - -if __name__ == "__main__": - main() diff --git a/runner/app/tests-examples/utility-func-compactor.py b/runner/app/tests-examples/utility-func-compactor.py deleted file mode 100644 index 6dac4309..00000000 --- a/runner/app/tests-examples/utility-func-compactor.py +++ /dev/null @@ -1,89 +0,0 @@ -import subprocess -import numpy as np -import torch -import os -from typing import List, Union -from pathlib import Path -import cv2 - -def generate_video_from_frames( - frames: Union[List[np.ndarray], List[torch.Tensor]], - output_path: str, - fps: float, - codec: str = "MJPEG", - is_directory: bool = False, - width: int = None, - height: int = None -) -> None: - """ - Generate a video from a list of frames. Frames can be from a directory or in-memory. - - Args: - frames (List[np.ndarray] | List[torch.Tensor]): List of frames as NumPy arrays or PyTorch tensors. - output_path (str): Path to save the output video file. - fps (float): Frames per second for the video. - codec (str): Codec used for video compression (default is "MJPEG"). - is_directory (bool): If True, treat `frames` as a directory path containing image files. - width (int): Width of the video. Must be provided if `frames` are in-memory. - height (int): Height of the video. Must be provided if `frames` are in-memory. - - Returns: - None - """ - if is_directory: - # Read frames from a directory - frames = [cv2.imread(os.path.join(frames, file)) for file in sorted(os.listdir(frames))] - else: - # Convert torch tensors to numpy arrays if necessary - if isinstance(frames[0], torch.Tensor): - frames = [frame.permute(1, 2, 0).cpu().numpy() for frame in frames] - - # Ensure frames are numpy arrays and are uint8 type - frames = [frame.astype(np.uint8) for frame in frames] - - # Check if frames are consistent - if not frames: - raise ValueError("No frames to process.") - - if width is None or height is None: - # Use dimensions of the first frame if not provided - height, width = frames[0].shape[:2] - - # Write frames to a temporary directory - temp_dir = Path("temp_frames") - temp_dir.mkdir(exist_ok=True) - for i, frame in enumerate(frames): - cv2.imwrite(str(temp_dir / f"frame_{i:05d}.png"), frame) - - # Build ffmpeg command - ffmpeg_cmd = [ - 'ffmpeg', '-y', '-framerate', str(fps), - '-i', str(temp_dir / 'frame_%05d.png'), - '-c:v', codec, '-pix_fmt', 'yuv420p', - output_path - ] - - # Run ffmpeg command - subprocess.run(ffmpeg_cmd, check=True) - - # Clean up temporary frames - for file in temp_dir.glob("*.png"): - file.unlink() - temp_dir.rmdir() - - print(f"Video saved to {output_path}") - -# Example usage -if __name__ == "__main__": - # Example with in-memory frames (as np.ndarray) - # Assuming `in_memory_frames` is a list of numpy arrays - - # Example with frames from a directory - frames_directory = "G:/ai-models/models/svd_xt_output" - generate_video_from_frames( - frames=frames_directory, - output_path="G:/ai-models/models/video_out/output.mp4", - fps=24.0, - codec="mpeg4", - is_directory=True - ) diff --git a/runner/app/tests-examples/utility-function-vid.py b/runner/app/tests-examples/utility-function-vid.py deleted file mode 100644 index adf194cc..00000000 --- a/runner/app/tests-examples/utility-function-vid.py +++ /dev/null @@ -1,62 +0,0 @@ -import cv2 -import numpy as np -import tempfile -import os -from io import BytesIO - -def extract_frames_from_video(video_data, is_file_path=True) -> np.ndarray: - """ - Extract frames from a video file or in-memory video data and return them as a NumPy array. - - Args: - video_data (str or BytesIO): Path to the input video file or in-memory video data. - is_file_path (bool): Indicates if video_data is a file path (True) or in-memory data (False). - - Returns: - np.ndarray: Array of frames with shape (num_frames, height, width, channels). - """ - if is_file_path: - # Handle file-based video input - video_capture = cv2.VideoCapture(video_data) - else: - # Handle in-memory video input - # Create a temporary file to store in-memory video data - with tempfile.NamedTemporaryFile(delete=False, suffix='.mp4') as temp_file: - temp_file.write(video_data.getvalue()) - temp_file_path = temp_file.name - - # Open the temporary video file - video_capture = cv2.VideoCapture(temp_file_path) - - if not video_capture.isOpened(): - raise ValueError("Error opening video data") - - frames = [] - success, frame = video_capture.read() - - while success: - frames.append(frame) - success, frame = video_capture.read() - - video_capture.release() - - # Delete the temporary file if it was created - if not is_file_path: - os.remove(temp_file_path) - - # Convert list of frames to a NumPy array - frames_array = np.array(frames) - print(f"Extracted {frames_array.shape[0]} frames from video in shape of {frames_array.shape}") - - return frames_array - -# Example usage -if __name__ == "__main__": - # File path example - video_file_path = "C:/Users/ganes/Desktop/Generated videos/output.mp4" - - - # In-memory video example - with open(video_file_path, "rb") as f: - video_data = BytesIO(f.read()) - frames_array_from_memory = extract_frames_from_video(video_data, is_file_path=False) From 12a925eec6b197dda8c6f11e97d368088ba88bc5 Mon Sep 17 00:00:00 2001 From: Jason Stone Date: Mon, 29 Jul 2024 16:41:33 +0300 Subject: [PATCH 34/38] update to directory reader as it now reads almost any naming convention --- runner/app/pipelines/utils/utils.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/runner/app/pipelines/utils/utils.py b/runner/app/pipelines/utils/utils.py index 5c9ad6a5..f2deeae8 100644 --- a/runner/app/pipelines/utils/utils.py +++ b/runner/app/pipelines/utils/utils.py @@ -277,11 +277,21 @@ def check_nsfw_images( ) return images, has_nsfw_concept +def natural_sort_key(s): + """ + Sort in a natural order, separating strings into a list of strings and integers. + This handles leading zeros and case insensitivity. + """ + return [ + int(text) if text.isdigit() else text.lower() + for text in re.split(r'([0-9]+)', os.path.basename(s)) + ] + class DirectoryReader: def __init__(self, dir: str): self.paths = sorted( glob.glob(os.path.join(dir, "*")), - key=lambda x: int(os.path.basename(x).split(".")[0]), + key=natural_sort_key ) self.nb_frames = len(self.paths) self.idx = 0 @@ -306,9 +316,9 @@ def get_frame(self): self.idx += 1 img = Image.open(path) - transforms = v2.Compose([v2.ToImage(), v2.ToDtype(torch.float32, scale=True)]) + transforms = v2.Compose([v2.ToTensor(), v2.ConvertImageDtype(torch.float32)]) - return transforms(img) + return transforms(img) class DirectoryWriter: def __init__(self, dir: str): From cd45ee0c34bd352e3d75be130d77d90fe10fc8f5 Mon Sep 17 00:00:00 2001 From: Jason Stone Date: Tue, 30 Jul 2024 13:20:24 +0300 Subject: [PATCH 35/38] changing files similar to main to make easy to merge --- ...e-ai-runner.yaml => ai-runner-docker.yaml} | 10 +++---- .../trigger-upstream-openapi-sync.yaml | 30 +++++++++++++++++++ ...-push.yaml => validate-openapi-on-pr.yaml} | 4 +-- 3 files changed, 37 insertions(+), 7 deletions(-) rename .github/workflows/{docker-create-ai-runner.yaml => ai-runner-docker.yaml} (88%) create mode 100644 .github/workflows/trigger-upstream-openapi-sync.yaml rename .github/workflows/{validate-openapi-on-push.yaml => validate-openapi-on-pr.yaml} (97%) diff --git a/.github/workflows/docker-create-ai-runner.yaml b/.github/workflows/ai-runner-docker.yaml similarity index 88% rename from .github/workflows/docker-create-ai-runner.yaml rename to .github/workflows/ai-runner-docker.yaml index d71aa858..161eb5cf 100644 --- a/.github/workflows/docker-create-ai-runner.yaml +++ b/.github/workflows/ai-runner-docker.yaml @@ -1,7 +1,7 @@ name: Build Docker image for ai-runner on: - check_run: + pull_request: paths: - "runner/**" - "!runner/.devcontainer/**" @@ -13,7 +13,6 @@ on: paths: - "runner/**" - "!runner/.devcontainer/**" - workflow_dispatch: concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} @@ -22,6 +21,7 @@ concurrency: jobs: docker: name: Docker image generation + if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository permissions: packages: write contents: read @@ -49,7 +49,7 @@ jobs: uses: docker/metadata-action@v5 with: images: | - jjassonn69/ai-runner + livepeer/ai-runner tags: | type=sha type=ref,event=pr @@ -73,5 +73,5 @@ jobs: tags: ${{ steps.meta.outputs.tags }} file: "Dockerfile" labels: ${{ steps.meta.outputs.labels }} - cache-from: type=registry,ref=jjassonn69/build:cache - cache-to: type=registry,ref=jjassonn69/build:cache,mode=max + cache-from: type=registry,ref=livepeerci/build:cache + cache-to: type=registry,ref=livepeerci/build:cache,mode=max \ No newline at end of file diff --git a/.github/workflows/trigger-upstream-openapi-sync.yaml b/.github/workflows/trigger-upstream-openapi-sync.yaml new file mode 100644 index 00000000..5c7bdeb1 --- /dev/null +++ b/.github/workflows/trigger-upstream-openapi-sync.yaml @@ -0,0 +1,30 @@ +name: Trigger upstream OpenAPI sync + +on: + push: + paths: + - "runner/openapi.json" + workflow_dispatch: + +jobs: + trigger-upstream-openapi-sync: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Trigger docs AI OpenAPI spec update + uses: peter-evans/repository-dispatch@v3 + with: + token: ${{ secrets.DOCS_TRIGGER_PAT }} + repository: livepeer/docs + event-type: update-ai-openapi + client-payload: '{"sha": "${{ github.sha }}"}' + + - name: Trigger SDK generation + uses: peter-evans/repository-dispatch@v3 + with: + token: ${{ secrets.SDKS_TRIGGER_PAT }} + repository: livepeer/livepeer-ai-sdks + event-type: update-ai-openapi + client-payload: '{"sha": "${{ github.sha }}"}' \ No newline at end of file diff --git a/.github/workflows/validate-openapi-on-push.yaml b/.github/workflows/validate-openapi-on-pr.yaml similarity index 97% rename from .github/workflows/validate-openapi-on-push.yaml rename to .github/workflows/validate-openapi-on-pr.yaml index 886d24e6..08cc39e4 100644 --- a/.github/workflows/validate-openapi-on-push.yaml +++ b/.github/workflows/validate-openapi-on-pr.yaml @@ -1,7 +1,7 @@ name: Check OpenAPI spec and Golang bindings on: - check_run: + pull_request: jobs: check-openapi-and-bindings: @@ -41,4 +41,4 @@ jobs: if ! git diff --exit-code; then echo "::error::Go bindings have changed. Please run 'make' at the root of the repository and commit the changes." exit 1 - fi + fi \ No newline at end of file From 088571894314e6f8885a315a21b4b8a118d77ae3 Mon Sep 17 00:00:00 2001 From: Jason Stone Date: Tue, 30 Jul 2024 14:40:38 +0300 Subject: [PATCH 36/38] update to upscale warmup params to fix OOM --- runner/app/pipelines/upscale.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runner/app/pipelines/upscale.py b/runner/app/pipelines/upscale.py index b743434b..3d7c4e39 100644 --- a/runner/app/pipelines/upscale.py +++ b/runner/app/pipelines/upscale.py @@ -74,7 +74,7 @@ def __init__(self, model_id: str): # TODO: Retrieve defaults from Pydantic class in route. warmup_kwargs = { "prompt": "Upscaling the pipeline with sfast enabled", - "image": PIL.Image.new("RGB", (576, 1024)), + "image": PIL.Image.new("RGB", (400, 400)), # anything higher than this size cause the model to OOM } logger.info("Warming up ImageToVideoPipeline pipeline...") From 3de64b37f95b4c7eead7e70c97a4835e6ded44de Mon Sep 17 00:00:00 2001 From: Jason Stone Date: Wed, 31 Jul 2024 07:05:55 +0300 Subject: [PATCH 37/38] naming wrong in the info section for error msg --- runner/app/pipelines/upscale.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runner/app/pipelines/upscale.py b/runner/app/pipelines/upscale.py index 3d7c4e39..174b0d14 100644 --- a/runner/app/pipelines/upscale.py +++ b/runner/app/pipelines/upscale.py @@ -77,7 +77,7 @@ def __init__(self, model_id: str): "image": PIL.Image.new("RGB", (400, 400)), # anything higher than this size cause the model to OOM } - logger.info("Warming up ImageToVideoPipeline pipeline...") + logger.info("Warming up Upscale pipeline...") total_time = 0 for ii in range(SFAST_WARMUP_ITERATIONS): t = time.time() @@ -85,7 +85,7 @@ def __init__(self, model_id: str): self.ldm(**warmup_kwargs).images except Exception as e: # FIXME: When out of memory, pipeline is corrupted. - logger.error(f"ImageToVideoPipeline warmup error: {e}") + logger.error(f"Upscale pipeline warmup error: {e}") raise e iteration_time = time.time() - t total_time += iteration_time From 5a26e9c66c521ce2cca2cebce7a79a7cdb7a61ee Mon Sep 17 00:00:00 2001 From: Jason Stone Date: Tue, 15 Oct 2024 02:49:00 +0000 Subject: [PATCH 38/38] clean up to isolate single Sfast PR, and dependencies requirements --- runner/app/main.py | 18 +- runner/app/pipelines/frame_interpolation.py | 115 +------------ runner/app/pipelines/utils/__init__.py | 6 +- runner/app/pipelines/utils/utils.py | 174 +------------------- runner/app/routes/frame_interpolation.py | 120 -------------- runner/gateway.openapi.yaml | 42 ++--- runner/gen_openapi.py | 1 - runner/openapi.yaml | 42 ++--- runner/requirements.txt | 5 - worker/runner.gen.go | 66 ++++---- 10 files changed, 86 insertions(+), 503 deletions(-) delete mode 100644 runner/app/routes/frame_interpolation.py diff --git a/runner/app/main.py b/runner/app/main.py index d86a68a3..1f542e54 100644 --- a/runner/app/main.py +++ b/runner/app/main.py @@ -1,7 +1,5 @@ import logging import os -import sys -import cv2 from contextlib import asynccontextmanager from app.routes import health @@ -17,8 +15,8 @@ async def lifespan(app: FastAPI): app.include_router(health.router) - pipeline = os.environ.get("PIPELINE", "") # Default to - model_id = os.environ.get("MODEL_ID", "") # Provide a default if necessary + pipeline = os.environ.get("PIPELINE", "") + model_id = os.environ.get("MODEL_ID", "") app.pipeline = load_pipeline(pipeline, model_id) app.include_router(load_route(pipeline)) @@ -46,10 +44,8 @@ def load_pipeline(pipeline: str, model_id: str) -> any: from app.pipelines.audio_to_text import AudioToTextPipeline return AudioToTextPipeline(model_id) - case "FILMPipeline": - from app.pipelines.frame_interpolation import FILMPipeline - - return FILMPipeline(model_id) + case "frame-interpolation": + raise NotImplementedError("frame-interpolation pipeline not implemented") case "upscale": from app.pipelines.upscale import UpscalePipeline @@ -85,10 +81,8 @@ def load_route(pipeline: str) -> any: from app.routes import audio_to_text return audio_to_text.router - case "FILMPipeline": - from app.routes import frame_interpolation - - return frame_interpolation.router + case "frame-interpolation": + raise NotImplementedError("frame-interpolation pipeline not implemented") case "upscale": from app.routes import upscale diff --git a/runner/app/pipelines/frame_interpolation.py b/runner/app/pipelines/frame_interpolation.py index 755afee7..ad5822c4 100644 --- a/runner/app/pipelines/frame_interpolation.py +++ b/runner/app/pipelines/frame_interpolation.py @@ -1,113 +1,4 @@ -import torch -from torchvision.transforms import v2 -from tqdm import tqdm -import bisect -import numpy as np -from app.pipelines.utils.utils import get_model_dir +from app.pipelines.base import Pipeline - -class FILMPipeline: - model: torch.jit.ScriptModule - - def __init__(self, model_id: str): - self.model_id = model_id - model_dir = get_model_dir() # Get the directory where models are stored - model_path = f"{model_dir}/{model_id}" # Construct the full path to the model file - - self.model = torch.jit.load(model_path, map_location="cpu") - self.model.eval() - - def to(self, *args, **kwargs): - self.model = self.model.to(*args, **kwargs) - return self - - @property - def device(self) -> torch.device: - # Checking device for ScriptModule requires checking one of its parameters - params = self.model.parameters() - return next(params).device - - @property - def dtype(self) -> torch.dtype: - # Checking device for ScriptModule requires checking one of its parameters - params = self.model.parameters() - return next(params).dtype - - def __call__( - self, - reader, - writer, - inter_frames: int = 2, - ): - transforms = v2.Compose( - [ - v2.ToDtype(torch.uint8, scale=True), - ] - ) - - writer.open() - - while True: - frame_1 = reader.get_frame() - # If the first frame read is None then there are no more frames - if frame_1 is None: - break - - frame_2 = reader.get_frame() - # If the second frame read is None there there is a final frame - if frame_2 is None: - writer.write_frame(transforms(frame_1)) - break - - # frame_1 and frame_2 must be tensors with n c h w format - frame_1 = frame_1.unsqueeze(0) - frame_2 = frame_2.unsqueeze(0) - - frames = inference( - self.model, frame_1, frame_2, inter_frames, self.device, self.dtype - ) - - frames = [transforms(frame.detach().cpu()) for frame in frames] - for frame in frames: - writer.write_frame(frame) - - writer.close() - - -def inference( - model, img_batch_1, img_batch_2, inter_frames, device, dtype -) -> torch.Tensor: - results = [img_batch_1, img_batch_2] - - idxes = [0, inter_frames + 1] - remains = list(range(1, inter_frames + 1)) - - splits = torch.linspace(0, 1, inter_frames + 2) - - for _ in tqdm(range(len(remains)), "Generating in-between frames"): - starts = splits[idxes[:-1]] - ends = splits[idxes[1:]] - distances = ( - (splits[None, remains] - starts[:, None]) - / (ends[:, None] - starts[:, None]) - - 0.5 - ).abs() - matrix = torch.argmin(distances).item() - start_i, step = np.unravel_index(matrix, distances.shape) - end_i = start_i + 1 - - x0 = results[start_i].to(device=device, dtype=dtype) - x1 = results[end_i].to(device=device, dtype=dtype) - - dt = x0.new_full((1, 1), (splits[remains[step]] - splits[idxes[start_i]])) / ( - splits[idxes[end_i]] - splits[idxes[start_i]] - ) - - with torch.no_grad(): - prediction = model(x0, x1, dt) - insert_position = bisect.bisect_left(idxes, remains[step]) - idxes.insert(insert_position, remains[step]) - results.insert(insert_position, prediction.clamp(0, 1).float()) - del remains[step] - - return results \ No newline at end of file +class FrameInterpolationPipeline(Pipeline): + pass \ No newline at end of file diff --git a/runner/app/pipelines/utils/__init__.py b/runner/app/pipelines/utils/__init__.py index 3431abc4..87ed4774 100644 --- a/runner/app/pipelines/utils/__init__.py +++ b/runner/app/pipelines/utils/__init__.py @@ -11,9 +11,5 @@ is_lightning_model, is_turbo_model, split_prompt, - validate_torch_device, - frames_compactor, - video_shredder, - DirectoryReader, - DirectoryWriter + validate_torch_device ) diff --git a/runner/app/pipelines/utils/utils.py b/runner/app/pipelines/utils/utils.py index 075da109..89cf8a9a 100644 --- a/runner/app/pipelines/utils/utils.py +++ b/runner/app/pipelines/utils/utils.py @@ -5,20 +5,13 @@ import os import re from pathlib import Path -from typing import Optional -import glob -import tempfile -from io import BytesIO -from typing import List, Union, Dict, Optional, Union +from typing import Any, Dict, List, Optional import numpy as np import torch from diffusers.pipelines.pipeline_utils import DiffusionPipeline from diffusers.pipelines.stable_diffusion import StableDiffusionSafetyChecker from PIL import Image -from torchvision.transforms import v2 -import cv2 -from torchaudio.io import StreamWriter from torch import dtype as TorchDtype from transformers import CLIPImageProcessor @@ -148,109 +141,6 @@ def split_prompt( return prompt_dict -def frames_compactor( - frames: Union[List[np.ndarray], List[torch.Tensor]], - output_path: str, - fps: float, - codec: str = "MJPEG", - is_directory: bool = False, - width: int = None, - height: int = None -) -> None: - """ - Generate a video from a list of frames. Frames can be from a directory or in-memory. - - Args: - frames (List[np.ndarray] | List[torch.Tensor]): List of frames as NumPy arrays or PyTorch tensors. - output_path (str): Path to save the output video file. - fps (float): Frames per second for the video. - codec (str): Codec used for video compression (default is "XVID"). - is_directory (bool): If True, treat `frames` as a directory path containing image files. - width (int): Width of the video. Must be provided if `frames` are in-memory. - height (int): Height of the video. Must be provided if `frames` are in-memory. - - Returns: - None - """ - if is_directory: - # Read frames from a directory - frames = [cv2.imread(os.path.join(frames, file)) for file in sorted(os.listdir(frames))] - else: - # Convert torch tensors to numpy arrays if necessary - if isinstance(frames[0], torch.Tensor): - frames = [frame.permute(1, 2, 0).cpu().numpy() for frame in frames] - - # Ensure frames are numpy arrays and are uint8 type - frames = [frame.astype(np.uint8) for frame in frames] - - # Check if frames are consistent - if not frames: - raise ValueError("No frames to process.") - - if width is None or height is None: - # Use dimensions of the first frame if not provided - height, width = frames[0].shape[:2] - - # Define the codec and create VideoWriter object - fourcc = cv2.VideoWriter_fourcc(*codec) - video_writer = cv2.VideoWriter(output_path, fourcc, fps, (width, height)) - - # Write frames to the video file - for frame in frames: - # Ensure each frame has the correct size - if frame.shape[1] != width or frame.shape[0] != height: - frame = cv2.resize(frame, (width, height)) - video_writer.write(frame) - - # Release the video writer - video_writer.release() - -def video_shredder(video_data, is_file_path=True) -> np.ndarray: - """ - Extract frames from a video file or in-memory video data and return them as a NumPy array. - - Args: - video_data (str or BytesIO): Path to the input video file or in-memory video data. - is_file_path (bool): Indicates if video_data is a file path (True) or in-memory data (False). - - Returns: - np.ndarray: Array of frames with shape (num_frames, height, width, channels). - """ - if is_file_path: - # Handle file-based video input - video_capture = cv2.VideoCapture(video_data) - else: - # Handle in-memory video input - # Create a temporary file to store in-memory video data - with tempfile.NamedTemporaryFile(delete=False, suffix='.mp4') as temp_file: - temp_file.write(video_data.getvalue()) - temp_file_path = temp_file.name - - # Open the temporary video file - video_capture = cv2.VideoCapture(temp_file_path) - - if not video_capture.isOpened(): - raise ValueError("Error opening video data") - - frames = [] - success, frame = video_capture.read() - - while success: - frames.append(frame) - success, frame = video_capture.read() - - video_capture.release() - - # Delete the temporary file if it was created - if not is_file_path: - os.remove(temp_file_path) - - # Convert list of frames to a NumPy array - frames_array = np.array(frames) - print(f"Extracted {frames_array.shape[0]} frames from video in shape of {frames_array.shape}") - - return frames_array - class SafetyChecker: """Checks images for unsafe or inappropriate content using a pretrained model. @@ -307,68 +197,6 @@ def check_nsfw_images( ) return images, has_nsfw_concept - -def natural_sort_key(s): - """ - Sort in a natural order, separating strings into a list of strings and integers. - This handles leading zeros and case insensitivity. - """ - return [ - int(text) if text.isdigit() else text.lower() - for text in re.split(r'([0-9]+)', os.path.basename(s)) - ] - -class DirectoryReader: - def __init__(self, dir: str): - self.paths = sorted( - glob.glob(os.path.join(dir, "*")), - key=natural_sort_key - ) - self.nb_frames = len(self.paths) - self.idx = 0 - - assert self.nb_frames > 0, "no frames found in directory" - - first_img = Image.open(self.paths[0]) - self.height = first_img.height - self.width = first_img.width - - def get_resolution(self): - return self.height, self.width - - def reset(self): - self.idx = 0 # Reset the index counter to 0 - - def get_frame(self): - if self.idx >= self.nb_frames: - return None - - path = self.paths[self.idx] - self.idx += 1 - - img = Image.open(path) - transforms = v2.Compose([v2.ToImage(), v2.ToDtype(torch.float32, scale=True)]) - - return transforms(img) - -class DirectoryWriter: - def __init__(self, dir: str): - self.dir = dir - self.idx = 0 - - def open(self): - return - - def close(self): - return - - def write_frame(self, frame: torch.Tensor): - path = f"{self.dir}/{self.idx}.png" - self.idx += 1 - - transforms = v2.Compose([v2.ToPILImage()]) - transforms(frame.squeeze(0)).save(path) - class LoraLoadingError(Exception): """Exception raised for errors during LoRa loading.""" diff --git a/runner/app/routes/frame_interpolation.py b/runner/app/routes/frame_interpolation.py deleted file mode 100644 index 6abf6a66..00000000 --- a/runner/app/routes/frame_interpolation.py +++ /dev/null @@ -1,120 +0,0 @@ -# app/routes/film_interpolate.py - -import logging -import os -import torch -import glob -from typing import Annotated, Optional -from fastapi import APIRouter, Depends, File, Form, UploadFile, status -from fastapi.responses import JSONResponse -from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer -from PIL import Image, ImageFile - -from app.dependencies import get_pipeline -from app.pipelines.frame_interpolation import FILMPipeline -from app.pipelines.utils.utils import DirectoryReader, DirectoryWriter, get_torch_device, get_model_dir -from app.routes.util import HTTPError, ImageResponse, http_error, image_to_data_url - -ImageFile.LOAD_TRUNCATED_IMAGES = True - -router = APIRouter() - -logger = logging.getLogger(__name__) - -RESPONSES = { - status.HTTP_400_BAD_REQUEST: {"model": HTTPError}, - status.HTTP_401_UNAUTHORIZED: {"model": HTTPError}, - status.HTTP_500_INTERNAL_SERVER_ERROR: {"model": HTTPError}, -} - -@router.post("/frame_interpolation", response_model=ImageResponse, responses=RESPONSES) -@router.post( - "/frame_interpolation/", - response_model=ImageResponse, - responses=RESPONSES, - include_in_schema=False, -) -async def frame_interpolation( - model_id: Annotated[str, Form()], - image1: Annotated[UploadFile, File()]=None, - image2: Annotated[UploadFile, File()]=None, - image_dir: Annotated[str, Form()]="", - inter_frames: Annotated[int, Form()] = 2, - token: HTTPAuthorizationCredentials = Depends(HTTPBearer(auto_error=False)), -): - auth_token = os.environ.get("AUTH_TOKEN") - if auth_token: - if not token or token.credentials != auth_token: - return JSONResponse( - status_code=status.HTTP_401_UNAUTHORIZED, - headers={"WWW-Authenticate": "Bearer"}, - content=http_error("Invalid bearer token"), - ) - - - # Initialize FILMPipeline - film_pipeline = FILMPipeline(model_id) - film_pipeline.to(device=get_torch_device(),dtype=torch.float16) - - # Prepare directories for input and output - temp_input_dir = "temp_input" - temp_output_dir = "temp_output" - os.makedirs(temp_input_dir, exist_ok=True) - os.makedirs(temp_output_dir, exist_ok=True) - - try: - if os.path.isdir(image_dir): - if image1 and image2: - logger.info("Both directory and individual images provided. Directory will be used, and images will be ignored.") - reader = DirectoryReader(image_dir) - else: - if not (image1 and image2): - return JSONResponse( - status_code=status.HTTP_400_BAD_REQUEST, - content=http_error("Either a directory or two images must be provided."), - ) - - image1_path = os.path.join(temp_input_dir, "0.png") - image2_path = os.path.join(temp_input_dir, "1.png") - - with open(image1_path, "wb") as f: - f.write(await image1.read()) - with open(image2_path, "wb") as f: - f.write(await image2.read()) - - reader = DirectoryReader(temp_input_dir) - - writer = DirectoryWriter(temp_output_dir) - # Perform interpolation - film_pipeline(reader, writer, inter_frames=inter_frames) - - writer.close() - reader.reset() - - # Collect output frames - output_frames = [] - for frame_path in sorted(glob.glob(os.path.join(temp_output_dir, "*.png"))): - frame = Image.open(frame_path) - output_frames.append(frame) - - output_images = [{"url": image_to_data_url(frame),"seed":0, "nsfw":False} for frame in output_frames] - - except Exception as e: - logger.error(f"FILMPipeline error: {e}") - logger.exception(e) - return JSONResponse( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - content=http_error("FILMPipeline error"), - ) - - finally: - # Clean up temporary directories - for file_path in glob.glob(os.path.join(temp_input_dir, "*")): - os.remove(file_path) - os.rmdir(temp_input_dir) - - for file_path in glob.glob(os.path.join(temp_output_dir, "*")): - os.remove(file_path) - os.rmdir(temp_output_dir) - - return {"images": output_images} diff --git a/runner/gateway.openapi.yaml b/runner/gateway.openapi.yaml index a8bba1b1..1d2a8ce9 100644 --- a/runner/gateway.openapi.yaml +++ b/runner/gateway.openapi.yaml @@ -2,7 +2,7 @@ openapi: 3.1.0 info: title: Livepeer AI Runner description: An application to run AI pipelines - version: '' + version: v1.0.0 servers: - url: https://dream-gateway.livepeer.cloud description: Livepeer Cloud Community Gateway @@ -153,18 +153,18 @@ paths: security: - HTTPBearer: [] x-speakeasy-name-override: imageToVideo - /upscale: + /audio-to-text: post: tags: - generate - summary: Upscale - description: Upscale an image by increasing its resolution. - operationId: genUpscale + summary: Audio To Text + description: Transcribe audio files to text. + operationId: genAudioToText requestBody: content: multipart/form-data: schema: - $ref: '#/components/schemas/Body_genUpscale' + $ref: '#/components/schemas/Body_genAudioToText' required: true responses: '200': @@ -172,7 +172,7 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/ImageResponse' + $ref: '#/components/schemas/TextResponse' x-speakeasy-name-override: data '400': description: Bad Request @@ -186,6 +186,12 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPError' + '413': + description: Request Entity Too Large + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPError' '500': description: Internal Server Error content: @@ -200,19 +206,19 @@ paths: $ref: '#/components/schemas/HTTPValidationError' security: - HTTPBearer: [] - x-speakeasy-name-override: upscale - /audio-to-text: + x-speakeasy-name-override: audioToText + /upscale: post: tags: - generate - summary: Audio To Text - description: Transcribe audio files to text. - operationId: genAudioToText + summary: Upscale + description: Upscale an image by increasing its resolution. + operationId: genUpscale requestBody: content: multipart/form-data: schema: - $ref: '#/components/schemas/Body_genAudioToText' + $ref: '#/components/schemas/Body_genUpscale' required: true responses: '200': @@ -220,7 +226,7 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/TextResponse' + $ref: '#/components/schemas/ImageResponse' x-speakeasy-name-override: data '400': description: Bad Request @@ -234,12 +240,6 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPError' - '413': - description: Request Entity Too Large - content: - application/json: - schema: - $ref: '#/components/schemas/HTTPError' '500': description: Internal Server Error content: @@ -254,7 +254,7 @@ paths: $ref: '#/components/schemas/HTTPValidationError' security: - HTTPBearer: [] - x-speakeasy-name-override: audioToText + x-speakeasy-name-override: upscale /segment-anything-2: post: tags: diff --git a/runner/gen_openapi.py b/runner/gen_openapi.py index e4e7498d..afbd5799 100644 --- a/runner/gen_openapi.py +++ b/runner/gen_openapi.py @@ -11,7 +11,6 @@ image_to_video, segment_anything_2, text_to_image, - frame_interpolation, upscale, llm ) diff --git a/runner/openapi.yaml b/runner/openapi.yaml index 377777a9..f48b551b 100644 --- a/runner/openapi.yaml +++ b/runner/openapi.yaml @@ -2,7 +2,7 @@ openapi: 3.1.0 info: title: Livepeer AI Runner description: An application to run AI pipelines - version: '' + version: v1.0.0 servers: - url: https://dream-gateway.livepeer.cloud description: Livepeer Cloud Community Gateway @@ -164,18 +164,18 @@ paths: security: - HTTPBearer: [] x-speakeasy-name-override: imageToVideo - /upscale: + /audio-to-text: post: tags: - generate - summary: Upscale - description: Upscale an image by increasing its resolution. - operationId: genUpscale + summary: Audio To Text + description: Transcribe audio files to text. + operationId: genAudioToText requestBody: content: multipart/form-data: schema: - $ref: '#/components/schemas/Body_genUpscale' + $ref: '#/components/schemas/Body_genAudioToText' required: true responses: '200': @@ -183,7 +183,7 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/ImageResponse' + $ref: '#/components/schemas/TextResponse' x-speakeasy-name-override: data '400': description: Bad Request @@ -197,6 +197,12 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPError' + '413': + description: Request Entity Too Large + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPError' '500': description: Internal Server Error content: @@ -211,19 +217,19 @@ paths: $ref: '#/components/schemas/HTTPValidationError' security: - HTTPBearer: [] - x-speakeasy-name-override: upscale - /audio-to-text: + x-speakeasy-name-override: audioToText + /upscale: post: tags: - generate - summary: Audio To Text - description: Transcribe audio files to text. - operationId: genAudioToText + summary: Upscale + description: Upscale an image by increasing its resolution. + operationId: genUpscale requestBody: content: multipart/form-data: schema: - $ref: '#/components/schemas/Body_genAudioToText' + $ref: '#/components/schemas/Body_genUpscale' required: true responses: '200': @@ -231,7 +237,7 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/TextResponse' + $ref: '#/components/schemas/ImageResponse' x-speakeasy-name-override: data '400': description: Bad Request @@ -245,12 +251,6 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPError' - '413': - description: Request Entity Too Large - content: - application/json: - schema: - $ref: '#/components/schemas/HTTPError' '500': description: Internal Server Error content: @@ -265,7 +265,7 @@ paths: $ref: '#/components/schemas/HTTPValidationError' security: - HTTPBearer: [] - x-speakeasy-name-override: audioToText + x-speakeasy-name-override: upscale /segment-anything-2: post: tags: diff --git a/runner/requirements.txt b/runner/requirements.txt index 4ed63b65..87f72e43 100644 --- a/runner/requirements.txt +++ b/runner/requirements.txt @@ -6,10 +6,6 @@ pydantic==2.7.2 Pillow==10.3.0 python-multipart==0.0.9 uvicorn==0.30.0 -setuptools==71.1.0 -torch --index-url https://download.pytorch.org/whl/cu121 -torchvision --index-url https://download.pytorch.org/whl/cu121 -torchaudio --index-url https://download.pytorch.org/whl/cu121 huggingface_hub==0.23.2 xformers==0.0.23 triton>=2.1.0 @@ -21,6 +17,5 @@ numpy==1.26.4 av==12.1.0 sentencepiece== 0.2.0 protobuf==5.27.2 -opencv-python==4.10.0.84 bitsandbytes==0.43.3 psutil==6.0.0 diff --git a/worker/runner.gen.go b/worker/runner.gen.go index 03812d9d..105cd33f 100644 --- a/worker/runner.gen.go +++ b/worker/runner.gen.go @@ -2005,36 +2005,36 @@ func HandlerWithOptions(si ServerInterface, options ChiServerOptions) http.Handl // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+xba28bN9b+K8S8L9AEkCzLbZqFgX5w0jQx1k4DW2laJIZAzRyNWHPIKS+W1Kz/+4KH", + "H4sIAAAAAAAC/+xba28bN9b+K8S8L9AEkCzZbZqFgX5w0jQx1k4DW2laJIZAzRyNWHPIKS+W1Kz/+4KH", "MyPORbds4u6m+hRHvJzn3A/JMx+jWGa5FCCMjk4/RjqeQUbxz7M35y+Uksr9nYCOFcsNkyI6dSME3BBR", "oHMpNJBMJsCPol6UK5mDMgxwj0yn7eWjGRTLM9CapuDWGWY4RKfRpU7d/5a5+482iok0ur/vRQr+sExB", "Ep2+x11vVksqoNU6OfkdYhPd96JnMlmOUxBnNmFyJEewMA5QHSV1g22cb3MuaQIJwXEyZRyIkWQCxCgq", "3MwJJA77VKqMmug0mjBB1TLgBsm2+elFKK8xSzzVKbXcrY96DQivbJoykZKfaFzImJz/SKyGhEylqnDg", "9JoU/dRkqyg964EwuwS2Qa7nGU1hJPGftmBTyxIqYhjrmHKo8fr06EmT2RcillbRFHTBqpEkBQGKGiAs", "w4GYSw18STgTt5C4GWYGxMDCkFzJLDfk0YylM1DkjnLrdqJLoiCxcbEF+cNSzszycSiulwVOco04K36F", - "zSagHL+sZHCNifi9jXTI2XRJ5szMEFrOcuBMwGY78fLrsBPcd7xBjsO2HH+EVAGCmc9Y7GGUciyRMk1y", - "q2cowjlVicZZTDDDKPdzjpr4yHYxcal88Nhg02fkQl6dkUcXct6/ouKWnCU0N9SNPi4UT0VCmNEklspH", - "mMQ5wRxYOjNo+J6Jgiln++TFgmY5h1PykXyIODUgTD+WQjNtQMTLAY+zvkPX18mCf4hOyfDouEc+RAIU", - "+10PcrYA3qfK9MvRk/tQABfI2Bdz5BY/O/lyLxKQUsPuYOyNfwuI0cpNHunH6F6WJUDmM2rc/2ARc5sA", - "mSqZdYj4PBVSOQuakrpBkg/2+PjbmAxD2K8LaOSNh9aF3mZj79fjHFQXD8MmC6/R1IiclgEhjBE5qIK9", - "GhCbkXM/+Q2oFhwmDKTeehGPmIICZM1AXrfl4fHxejwJCMm00zEuPCKXUoH/m1htKXdRCyjGrCJEFaGo", + "zSagHL+sZHCNifi9jXTI2XRJ5szMEFrOcuBMwGY78fLrsBPcd7xBjsdtOf4IqQIEM5+x2MMo5VgiZZrk", + "Vs9QhHOqEo2zmGCGUe7nHDXxke1i4lL54LHBps/Ihbw6I48u5Lx/RcUtOUtobqgbfVwonoqEMKNJLJWP", + "MIlzgjmwdGbQ8D0TBVPO9smLBc1yDqfkI/kQcWpAmH4shWbagIiXAx5nfYeur5MF/xCdkuOjYY98iAQo", + "9rse5GwBvE+V6ZejJ/ehAC6QsS/myC1+dvLlXiQgpYbdwdgb/xYQo5WbPNKP0b0sS4DMZ9S4/8Ei5jYB", + "MlUy6xDxeSqkchY0JXWDJB/scPhtTI5D2K8LaOSNh9aF3mZj79fjHFQXD8dNFl6jqRE5LQNCGCNyUAV7", + "NSA2I+d+8htQLThMGEi99SIeMQUFyJqBvG7Lx8PhejwJCMm00zEuPCKXUoH/m1htKXdRCyjGrCJEFaGo", "ZGViDdFczkGRCoXbJrEcPXeyJNooEKmZtfgr55NrRN3FXSjeXaxik02u16mmUzDLcTyD+LYmPKMsNKX3", "BpSLiYQSv4zgMjRFbViGcX/ajF0uLFieuDwsp1MQ2hmZVGRGVTa1PIR57Xd9jmAqsBMpOVCBaAGStkSu", - "oXBLRUUiM+Lj2xpRuMmd8i51VZPC8dE/1oRrOfXp3CcJJgWhec7ZKskpKHXsNfPo2I0Ma4nsuqTZis2N", - "vJ+XCvSJraMAqGX27RXALywB2a4Apg0X+r7XUQxOFc1Ao/tqiKVIUBi1rHXntg85/WmNlc8wSdRoPnna", - "SdXPJEwQDP56B6Kv/OZddHcuEKpoRf3+GG0/sTr4PMnHw9g/+WTSzR5PbHwLpoliePK0CeNtSdCpmLkf", + "oXBLRUUiM+Lj2xpRuMmd8i51VZPC8Ogfa8K1nPp07pMEk4LQPOdsleQUlDr2mnk0dCPHtUR2XdJsxeZG", + "3s9LBfrE1lEA1DL79grgF5aAbFcA04YLfd/rKAanimag0X01xFIkKIxa1rpz24ec/rTGymeYJGo0nzzt", + "pOpnEiYIBn+9A9FXfvMuujsXCFW0on5/jLafWB18nuTjYeyffDLpZo8nNr4F00RxfPK0CeNtSdCpmLkf", "HSgncppJK4xTgN/TF7ezevpBnfnA6YYKp3R/Zi7SFivnjHMXGpjAoZYKL/20Zwi6xliYCCTTMKY2Ha9x", - "4uOTVlVTsYCLCU2SlevWGPbFFXlVK1OLElWBhmzCschau9aXRyJWQHXJdy0hIIAzm5L14WB7sjt58j+c", - "6w5ZqJTEnCUN6x0en3zXFQ9x5l7h8B3u3abayDXbUoxPHRtSzMXFZTuzzJg2Ui3roe/9TRitixldoYsu", - "xkbegmja/PdBpKALMvJzugS7NvbuEjpXtdgOFZVRQLMamSnlGupZn2bdprXUBrJxdQ/TgfMap5DOi5de", - "ZCDLnf6tgkYMfLraYhRM2rHy6DAHp+YNVnANaQbCnImlmTGRnrRNYiIXHZdVhGMYId8RqhRdkpTdgSBU", - "E0omclFeGxTRFrXac17w62+//kZ8Tg5t/plcrD2nt4mfl1lfe/Cfmuepvh0zkVvTyZ+c9xVoyS2mNjeZ", - "4OQGU2aZsxhjMx7wKMkV3DFptfsjYTGuZqaILr3VHQZGx+Hi1eIdefTqh3c/nDz5HgPT9dllrfq8dJTP", + "4uFJq6qpWMDFhCbJynVrDPviiryqlalFiapAQzbhWGStXevLIxEroLrku5YQEMCZTcn6cLA92Z08+R/O", + "dYcsVEpizpKG9R4PT77rioc4c69w+A73blNt5JptKcanjg0p5uLisp1ZZkwbqZb10Pf+JozWxYyu0EUX", + "YyNvQTRt/vsgUtAFGfk5XYJdG3t3CZ2rWmyHisoooFmNzJRyDfWsT7Nu01pqA9m4uofpwHmNU0jnxUsv", + "MpDlTv9WQSMGPl1tMQom7Vh5dJiDU/MGK7iGNANhzsTSzJhIT9omMZGLjssqwjGMkO8IVYouScruQBCq", + "CSUTuSivDYpoi1rtOS/49bdffyM+J4c2/0wu1p7T28TPy6yvPfhPzfNU346ZyK3p5E/O+wq05BZTm5tM", + "cHKDKbPMWYyxGQ94lOQK7pi02v2RsBhXM1NEl97qDgOj4/Hi1eIdefTqh3c/nDz5HgPT9dllrfq8dJTP", "EeZ/3Uk5s9zFcn07ltZUgtyQFc5dPW6ht5Kgry0UGKtcceGKdrehRlw0m7DUOmF60Xuz0j0ipwaE+29i", "Y8fXBIwBVaw0Mypc3mEi5RCoocZViZz87JF3+blwRsXZnzCOpVSJ3o+9XDJhCK5kghrQVRlV7bs6hlCR", - "Anl/3BveFCaCqwu6BBY5xMZPn4CfoEC7H91PXn0Jy1zGlELX65aCFnnueehiNCTWdobXi5PCy+W04KpQ", - "RMMX5jNQQIDGBXzCnOLIo197vz1e5cDakRenNZEFIR2BcToB3gHsAn+v6toatBLNkDCRsBjlT91USJW0", - "Iilmu6rvuDZlQuPbcEobrifbBdeb8ZjLlJk9rMUv08SKvvMAPZPc1blonn4vwoQ2rvaTUwcRYxyOh+iu", + "Ank/7B3fFCaCqwu6BBY5xMZPn4CfoEC7H91PXn0Jy1zGlELX65aCFnnueehiNCTWdobXi5PCy+W04KpQ", + "RMMX5jNQQIDGBXzCnOLIo197vz1e5cDakRenNZEFIR2BcToB3gHsAn+v6toatBLNMWEiYTHKn7qpkCpp", + "RVLMdlXfsDZlQuPbcEobrifbBdeb8ZjLlJk9rMUv08SKvvMAPZPc1blonn4vwoQ2rvaTUwcRYxyOh+iu", "vBNdeOptPe9aQbRywob88Tavbk/raeOvutv9PAHReraST79D3HIQePrkb3TptZM0D7df284de982lc7Z", "4b+vRqM3ax4i3dCOL5EJGMo4vvZx/vM0On3/Mfp/BdPoNPq/weoNdFA8gA6qR8X7m/aFndsKkoIyE9WV", "3VGL84JswPGKnTW8/kI5S3C7iut1rDADGf60iZPmfvcrLJ6TFRBMnchDiLa5QRduoNzMnpdmX8erDTW2", @@ -2049,20 +2049,20 @@ var swaggerSpec = []string{ "KpbFDUPTHj62IN7chyE5RjId1UfxgLeqvbBxuvOohj+spiJmMnK/bqtEHB+eVDEzkNQOlwD4jrtX4dfV", "edLoH8LWoG11V9lI4+bWSr89z+TNkq/sNfIgtpzRC6ihzGoC6ZCYrz47Tjw4gIbvYhkGI0oMy0AbmuVt", "Ma0vTnGDwoNw1+31qRsvKK3ZsxxubVzKOxDeqNpri/xMONEBCyTpBdWSIIas2CpmltdOmV4Yr0ajN8+A", - "KlDVFw0Y5/xP1SYzY/Lo3u3hzpEdWig68LxPuiisrCBn59W1vw6rKXYHOYBy41dWCCR0B0r7vZxQZQ6C", - "5iw6jb49Gh4dOx1SM0PEA2zD7xvZLxWZS92l0Oq7g+CbBP+0VZw8ZF640nniyupmH7+TN2jzTCbYOOHO", - "0iCQkM+AVJmBS0H9hBq6+h5kmwd1fTRwX9evy3f4g/cGZPvk+LiBIhD44HfteN4VQu2whLQbScziGXhq", - "OVlN60XffUYIq5vdDvrPaEKuvPQ93eHD0H0rqDUzqdifkCDh4bcPQ7hglrwQxpWEIynJBVWpl/rJyWcF", - "0bribsNZTSHVNfiTh1L+uTCgBOXkGtQdqBJBEL6wWggD1/ub+5tepG2WUbUsvyQiI0nKmE1T7aJmmQRd", - "tFz0dQ70Fqhe9gXNoC/vQCmWYMytuWYvGszwRh0vCQB5r4cOf+EefUGPDa/0d3XY+1AkBUTkBktCF0Cr", - "x9zuCHqW53xZvujW2q0xjFJ3AHDVRFBktkJqozP6C8fUGrUHDqr1R4ZDVF0fVQ8Bbd+A5lvjRpJU/RF7", - "RjRWd4wwCNxVHyN0BoGXXS34e/l+2bL6ML7vqT2w79cPLwffP/j+F/D9qvX703y/dIxeNOA828Hh8TRs", - "8VKTEk5Fah2Q6j6v5e6+JXm9l4ciXvTn83kfvd0qDiKWib9N28/nHckHdvXw3f3g6AdH/3yOXrT07+nd", - "zpfRqYvGjT4t+jv7J+t9vGgFLdoEsJuXig2ZvKN19Atn8xbFB3bzegPGwdEPjv75HL30vtK4yckn+L1u", - "O0gvGricvcPJ/mWjUQFr+qAvQXdGgeABaOdEv//FZP2J6XCIP7j9V+L2+NT+H5zhTeB+6Ow2+Eij082L", - "RvEqt5PJsvwWGlsCjSarb+E6XX7Vav6F831J6ODvB3//Svw9+ExjT0+3oTNoBKCRXOM7ufId9TmXNiHP", - "ZZZZwcySvKQG5nQZFQ2w+HqrTweDRAHN+qkfPeLF8qPYLceGizX7Xxt8UFm3bbWRxnkDmrPBBAwdVPze", - "39z/OwAA//87pybUPU8AAA==", + "KlDVFw0Y5/xP1SYzY/Lo3u3hzpEdWig68LxPuiisrCBn59W1vw6rKXYHOYBy41dWCCR0B0r7ve6Oj4ZH", + "QydamYOgOYtOo2+Pjo+GTpPUzBD3AJvx+0b2S3XmUneptfr6IPgywT9wFecPmRcOdZ644rrZze+kDto8", + "kwm2T7gTNQgk5PMgVWbgElE/oYauvgrZ5kddnw7c17Xssh7+4H0C2T4ZDhsoArEPfteO510h1I5MSLuR", + "yiyehKeWk9W0XvTdZ4Swut/toP+MJuTKS9/TPX4Yum8FtWYmFfsTEiR8/O3DEC6YJS+EcYXhSEpyQVXq", + "pX5y8llBtC6623BWU0h1Gf7koZR/LgwoQTm5BnUHqkQQBDGsGcLw9f7m/qYXaZtlVC3L74nISJIyctNU", + "u9hZpkIXMxd9nQO9BaqXfUEz6Ms7UIolGHlrrtmLBjO8V8erAkDe66HDX7tHX9Bjw4v9XR32PhRJARG5", + "wcLQBdDqSbc7gp7lOV+W77q1pmsMo9QdA1xNEZSarZDa6I/+wjG1Ru2Bg2r9qeEQVddH1UNA2zeg+Qa5", + "kSRVl8SeEY3VHSMMAnfVJwmdQeBlVyP+Xr5fNq4+jO97ag/s+/UjzMH3D77/BXy/agD/NN8vHaMXDTjP", + "dnB4PBNbvNqkhFORWgekutVrubtvTF7v5aGIF/35fN5Hb7eKg4hl4u/U9vN5R/KBXT18fT84+sHRP5+j", + "F439e3q382V06qJ9o0+LLs/+yXofLxpCi2YB7OmlYkMm72gg/cLZvEXxgd283oZxcPSDo38+Ry+9rzRu", + "cvIJfq/bDtKLBi5n73Cyf9loV8CaPuhO0J1RIHgG2jnR738xWX9oOhziD27/lbg9Prj/B2d4E7gfOrsN", + "PtXodPOiXbzK7WSyLL+IxsZAo8nqi7hOl181nH/hfF8SOvj7wd+/En8PPtbY09Nt6AwaAWgk1/harnxN", + "fc6lTchzmWVWMLMkL6mBOV1GRRssvuHq08EgUUCzfupHj3ix/Ch2y7HtYs3+1wYfVNZtW22kcd6A5mww", + "AUMHFb/3N/f/DgAA//8K/20eQ08AAA==", } // GetSwagger returns the content of the embedded swagger specification file