diff --git a/.ci/ignore_convert_execution.txt b/.ci/ignore_convert_execution.txt index a12fecd8ae4..07e7d38e74c 100644 --- a/.ci/ignore_convert_execution.txt +++ b/.ci/ignore_convert_execution.txt @@ -63,4 +63,5 @@ notebooks/llm-rag-llamaindex/llm-rag-llamaindex.ipynb notebooks/llm-agent-functioncall/llm-agent-functioncall-qwen.ipynb notebooks/llm-agent-react/llm-agent-rag-llamaindex.ipynb notebooks/mllama-3.2/mllama-3.2.ipynb -notebooks/llm-agent-react/llm-agent-react.ipynb \ No newline at end of file +notebooks/llm-agent-react/llm-agent-react.ipynb +notebooks/multilora-image-generation/multilora-image-generation.ipynb \ No newline at end of file diff --git a/.ci/ignore_treon_docker.txt b/.ci/ignore_treon_docker.txt index 4eafb07efea..7254c4c6797 100644 --- a/.ci/ignore_treon_docker.txt +++ b/.ci/ignore_treon_docker.txt @@ -83,4 +83,5 @@ notebooks/stable-fast-3d/stable-fast-3d.ipynb notebooks/mllama-3.2/mllama-3.2.ipynb notebooks/sam2-image-segmentation/segment-anything-2-image.ipynb notebooks/pixtral/pixtral.ipynb -notebooks/llm-agent-react/llm-agent-react.ipynb \ No newline at end of file +notebooks/llm-agent-react/llm-agent-react.ipynb +notebooks/multilora-image-generation/multilora-image-generation.ipynb \ No newline at end of file diff --git a/.ci/skipped_notebooks.yml b/.ci/skipped_notebooks.yml index bce2986bf49..93db10381ba 100644 --- a/.ci/skipped_notebooks.yml +++ b/.ci/skipped_notebooks.yml @@ -582,4 +582,11 @@ - notebook: notebooks/llm-agent-react/llm-agent-react.ipynb skips: - os: - - macos-12 \ No newline at end of file + - macos-12 +- notebook: notebooks/multilora-image-generation/multilora-image-generation.ipynb + skips: + - os: + - macos-12 + - ubuntu-20.04 + - ubuntu-22.04 + - windows-2019 \ No newline at end of file diff --git a/.ci/spellcheck/.pyspelling.wordlist.txt b/.ci/spellcheck/.pyspelling.wordlist.txt index ac65e23c115..92a65e46779 100644 --- a/.ci/spellcheck/.pyspelling.wordlist.txt +++ b/.ci/spellcheck/.pyspelling.wordlist.txt @@ -99,6 +99,7 @@ Cifar cityscape Cityscapes Civitai +CivitAI classname ClassProbabilityMap CLI @@ -651,6 +652,7 @@ PIL PNDM png Pointilism +pointillistic PointNet pos Postfuse @@ -768,6 +770,7 @@ Ruizhongtai Runtime runtime runtimes +Safetensors SageMaker sagittal SALICON diff --git a/notebooks/multilora-image-generation/README.md b/notebooks/multilora-image-generation/README.md new file mode 100644 index 00000000000..8a11495f294 --- /dev/null +++ b/notebooks/multilora-image-generation/README.md @@ -0,0 +1,34 @@ +# Multi LoRA Image Generation + +LoRA, or [Low-Rank Adaptation](https://arxiv.org/abs/2106.09685), is a popular and lightweight training technique used for fine-tuning Large Language and Stable Diffusion Models without needing full model training. Full fine-tuning of larger models (consisting of billions of parameters) is inherently expensive and time-consuming. LoRA works by adding a smaller number of new weights to the model for training, rather than retraining the entire parameter space of the model. This makes training with LoRA much faster, memory-efficient, and produces smaller model weights (a few hundred MBs), which are easier to store and share. + +At its core, LoRA leverages the concept of low-rank matrix factorization. Instead of updating all the parameters in a neural network, LoRA decomposes the parameter space into two low-rank matrices. This decomposition allows the model to capture essential information with fewer parameters, significantly reducing the amount of data and computation required for fine-tuning. + +![](https://github.com/user-attachments/assets/bf823c71-13b4-402c-a7b4-d6fc30a60d88) + +By incorporating LoRA into Stable Diffusion models, we can enhance their ability to understand complex relationships and patterns in data. This approach opens up numerous possibilities: +* **Art and Design**: Artists can fine-tune models to generate images that align with their unique styles, creating personalized artwork effortlessly. +* **Content Creation**: Businesses can customize image generation models to produce branded visuals, enhancing marketing and media production. +* **Entertainment**: Game developers and filmmakers can use fine-tuned models to create realistic and imaginative worlds, streamlining the creative process. + +In this tutorial we explore possibilities to use LoRA with OpenVINO Generative API. + +## Notebook Contents + +This notebook demonstrates how to perform image generation using OpenVINO GenAI and LoRA adapters. + +The tutorial consists of following steps: +- Convert model using Optimum Intel +- Load and configure LoRA adapters +- Run inference with OpenVINO GenAI Text2ImagePipeline +- Interactive demo + +In interactive demonstration you can try to generate images using Stable Diffusion XL and make stylization using different LoRA adapters. Example of generated images with the same prompt and different adapters can be found bellow. + +![](https://github.com/user-attachments/assets/2138a109-add0-473a-b9dc-015e4b0415ce) + + +## Installation Instructions +We recommend running the notebook in a virtual environment. You only need a Jupyter server to start. +For details, please refer to [Installation Guide](../../README.md). + diff --git a/notebooks/multilora-image-generation/multilora-image-generation.ipynb b/notebooks/multilora-image-generation/multilora-image-generation.ipynb index da4c65ff509..f7c39697dee 100644 --- a/notebooks/multilora-image-generation/multilora-image-generation.ipynb +++ b/notebooks/multilora-image-generation/multilora-image-generation.ipynb @@ -538,6 +538,22 @@ "pygments_lexer": "ipython3", "version": "3.11.4" }, + "openvino_notebooks": { + "imageUrl": "https://github.com/user-attachments/assets/bf823c71-13b4-402c-a7b4-d6fc30a60d88", + "tags": { + "categories": [ + "Model Demos", + "AI Trends" + ], + "libraries": [], + "other": [ + "Stable Diffusion" + ], + "tasks": [ + "Text-to-Image" + ] + } + }, "widgets": { "application/vnd.jupyter.widget-state+json": { "state": {