update doc and install

.travis/codestyle/clang_format.hook

@@ -0,0 +1,4 @@
+#!/bin/bash
+set -e
+
+clang-format $@

.travis/codestyle/cpplint_pre_commit.hook

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+TOTAL_ERRORS=0
+if [[ ! $TRAVIS_BRANCH ]]; then
+  # install cpplint on local machine.
+  if [[ ! $(which cpplint) ]]; then
+    pip install cpplint
+  fi
+  # diff files on local machine. 
+  files=$(git diff --cached --name-status | awk '$1 != "D" {print $2}')
+else
+  # diff files between PR and latest commit on Travis CI. 
+  branch_ref=$(git rev-parse "$TRAVIS_BRANCH")
+  head_ref=$(git rev-parse HEAD)
+  files=$(git diff --name-status $branch_ref $head_ref | awk '$1 != "D" {print $2}')
+fi
+# The trick to remove deleted files: https://stackoverflow.com/a/2413151
+for file in $files; do
+    if [[ $file =~ ^(patches/.*) ]]; then
+        continue;
+    else
+        cpplint --filter=-readability/fn_size,-build/include_what_you_use,-build/c++11 $file;
+        TOTAL_ERRORS=$(expr $TOTAL_ERRORS + $?);
+    fi
+done
+
+exit $TOTAL_ERRORS

.travis/precommit.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+function abort(){
+    echo "Your commit not fit PaddlePaddle code style" 1>&2
+    echo "Please use pre-commit scripts to auto-format your code" 1>&2
+    exit 1
+}

README.md

@@ -18,20 +18,31 @@ PaddleMIX是基于飞桨的跨模态大模型开发套件，聚合图像、文
 
 **2023.7.31 发布 PaddleMIX v0.1**
 * 首次发布PaddleMIX跨模态大模型开发套件，融合PPdiffusers多模态扩散模型工具箱能力，广泛支持PaddleNLP大语言模型
-* 新增EVA-CLIP，BLIP-2，miniGPT-4，Stable Diffusion，ControlNet等xx个跨模态大模型
+* 新增EVA-CLIP，BLIP-2，miniGPT-4，Stable Diffusion，ControlNet等12个跨模态大模型
 
 ## 主要特性
 
-- **丰富的多模态功能:** 覆盖图文预训练，文生图，跨模态视觉任务，实现图像编辑、数据标注、数据清理等多样功能
+- **丰富的多模态功能:** 覆盖图文预训练，文生图，跨模态视觉任务，实现图像编辑、图像描述、数据标注等多样功能
 - **简洁的开发体验:** 模型统一开发接口，高效实现自定义模型开发和功能实现
-- **高效的训推流程:** 全量模型打通训练推理一站式开发流程，BLIP-2，Stable Diffusion等重点模型训推性能达到业界领先
+- **高效的训推流程:** 全量模型打通训练推理一站式开发流程，BLIP-2，Stable Diffusion等重点模型训推性能业界领先
 - **超大规模训练支持:** 可训练千亿规模图文预训练模型，百亿规模文生图底座模型
 
 ## 任务展示
 
 1. 图像描述（Image Caption）
+
+  <div align="center">
+  <img src="./docs/demo/caption.png"  height = "300" caption='' />
+  <p></p>
+  </div>
+
 2. 文图生成（Text-to-Image Generation）
-3. 文本引导的图像编辑（Text-Guided Image Inpainting）
+
+  <div align="center">
+  <img src="./docs/demo/text2img.png"  height = "300" caption='' />
+  <p></p>
+  </div>
+
 
 ## 安装
 
@@ -51,27 +62,37 @@ pip install -e .
 
 ## 教程
 
-- 快速开始
-- API文档
-- 训练微调
-- 推理部署
+- [快速开始](application/README.md)
+- [训练微调](docs/train_tutorial.md)
+- [推理部署](deploy/README.md)
 
 ## 特色应用
 
-1. 二维码模型
-2. Generative FIll
-3. Mix叠图
+1. 艺术风格二维码模型
+
+<div align="center">
+<img src="https://github.com/PaddlePaddle/Paddle/assets/22989727/ba091291-a1ee-49dc-a1af-fc501c62bfc8" height = "300",caption='' />
+<p>[体验专区](https://aistudio.baidu.com/aistudio/community/)</p>
+</div>
+
+2. Mix叠图
+
+<div align="center">
+<img src="https://github.com/PaddlePaddle/Paddle/assets/22989727/a71be5a0-b0f3-4aa8-bc20-740ea8ae6785" height = "300",caption='' />
+<p>[体验专区](https://aistudio.baidu.com/aistudio/community/)</p>
+</div>
+
 
 ## 模型库
 
 <table align="center">
   <tbody>
     <tr align="center" valign="center">
       <td>
-        <b>视觉语言模型（paddlevlp）</b>
+        <b>多模态预训练</b>
       </td>
       <td>
-        <b>扩散类模型（ppdiffusers）</b>
+        <b>扩散类模型</b>
       </td>
     </tr>
     <tr valign="top">
@@ -97,8 +118,15 @@ pip install -e .
         </ul>
           <li><b>文生图</b></li>
         <ul>
-           <li><a href="ppdiffusers/examples">Stable Diffusion</a></li>
-            <li><a href="ppdiffusers/examples">ControlNet</a></li>
+           <li><a href="ppdiffusers/examples/stable_diffusion">Stable Diffusion</a></li>
+            <li><a href="ppdiffusers/examples/controlnet">ControlNet</a></li>
+            <li><a href="ppdiffusers/examples/text_to_image_laion400m">LDM</a></li>
+            <li><a href="ppdiffusers/ppdiffusers/pipelines/unidiffuser">Unidiffuser</a></li>
+        </ul>
+        </ul>
+          <li><b>文生视频</b></li>
+        <ul>
+           <li><a href="ppdiffusers/ppdiffusers/pipelines/lvdm">LVDM</a></li>
         </ul>
       </td>
     </tr>

paddlevlp/trainer/trainer.py

@@ -76,7 +76,6 @@ def training_step(self, model, inputs) -> paddle.Tensor:
 
         if self.args.max_grad_norm > 0.0:
             grad_norms = clip_grad_norm(model, self.args.max_grad_norm)
-
         if self.rank == 0 and self.args.tensorboard:
             self.logstep += 1
             self.writer.add_scalar("train/loss", loss.item(), self.logstep)

scripts/build_wheel.sh

@@ -0,0 +1,136 @@
+#!/usr/bin/env bash
+
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#=================================================
+#                   Utils
+#=================================================
+
+
+# directory config
+DIST_DIR="dist"
+BUILD_DIR="build"
+EGG_DIR="paddlemix.egg-info"
+
+# command line log config
+RED='\033[0;31m'
+BLUE='\033[0;34m'
+GREEN='\033[1;32m'
+BOLD='\033[1m'
+NONE='\033[0m'
+
+function python_version_check() {
+  PY_MAIN_VERSION=`python -V 2>&1 | awk '{print $2}' | awk -F '.' '{print $1}'`
+  PY_SUB_VERSION=`python -V 2>&1 | awk '{print $2}' | awk -F '.' '{print $2}'`
+  echo -e "find python version ${PY_MAIN_VERSION}.${PY_SUB_VERSION}"
+  if [ $PY_MAIN_VERSION -ne "3" -o $PY_SUB_VERSION -lt "5" ]; then
+    echo -e "${RED}FAIL:${NONE} please use Python >= 3.5 !"
+    exit 1
+  fi
+}
+
+function init() {
+    echo -e "${BLUE}[init]${NONE} removing building directory..."
+    rm -rf $DIST_DIR $BUILD_DIR $EGG_DIR
+    if [ `pip list | grep paddlemix | wc -l` -gt 0  ]; then
+      echo -e "${BLUE}[init]${NONE} uninstalling paddlemix..."
+      pip uninstall -y paddlemix
+    fi
+    echo -e "${BLUE}[init]${NONE} ${GREEN}init success\n"
+}
+
+function build_and_install() {
+  echo -e "${BLUE}[build]${NONE} building paddlemix wheel..."
+  # add ppdiffusers as dependency to paddlemix
+  cp requirements.txt requirements.bak
+  echo 'ppdiffusers==0.16.3' >> requirements.txt 
+  python setup.py sdist bdist_wheel
+  if [ $? -ne 0 ]; then
+    echo -e "${RED}[FAIL]${NONE} build paddlemix wheel failed !"
+    exit 1
+  fi
+  echo -e "${BLUE}[build]${NONE} ${GREEN}build paddldet wheel success\n"
+  mv requirements.bak requirements.txt
+
+  echo -e "${BLUE}[install]${NONE} installing paddlemix..."
+  cd $DIST_DIR
+
+  find . -name "paddlemix*.whl" | xargs pip install
+  if [ $? -ne 0 ]; then
+    cd ..
+    echo -e "${RED}[FAIL]${NONE} install paddlemix wheel failed !"
+    exit 1
+  fi
+  echo -e "${BLUE}[install]${NONE} ${GREEN}paddlemix install success\n"
+  cd ..
+}
+
+function unittest() {
+  echo -e "${BLUE}[unittest]${NONE} run unittests..."
+
+  # NOTE: perform unittests make sure installed paddlemix is used
+  python -m unittest discover -v 
+
+  echo -e "${BLUE}[unittest]${NONE} ${GREEN}unittests success\n${NONE}"
+}
+
+function cleanup() {
+  rm -rf $BUILD_DIR $EGG_DIR
+  pip uninstall -y paddlemix
+}
+
+function abort() {
+  echo -e "${RED}[FAIL]${NONE} build wheel and unittest failed !
+          please check your code" 1>&2
+
+  cur_dir=`basename "$pwd"`
+  if [ cur_dir==$TEST_DIR -o cur_dir==$DIST_DIR ]; then
+    cd ..
+  fi
+
+  rm -rf $BUILD_DIR $EGG_DIR $DIST_DIR $TEST_DIR
+  pip uninstall -y paddlemix
+}
+
+python_version_check
+
+trap 'abort' 0
+set -e
+
+init
+build_and_install
+unittest
+cleanup
+
+# get Paddle version
+PADDLE_VERSION=`python -c "import paddle; print(paddle.version.full_version)"`
+PADDLE_COMMIT=`python -c "import paddle; print(paddle.version.commit)"`
+PADDLE_COMMIT=`git rev-parse --short $PADDLE_COMMIT`
+
+# get PaddleMIX branch
+PPDET_BRANCH=`git rev-parse --abbrev-ref HEAD`
+PPDET_COMMIT=`git rev-parse --short HEAD`
+
+# get Python version
+PYTHON_VERSION=`python -c "import platform; print(platform.python_version())"`
+
+echo -e "\n${GREEN}paddlemix wheel compiled and checked success !${NONE}
+        ${BLUE}Python version:${NONE} $PYTHON_VERSION
+        ${BLUE}Paddle version:${NONE} $PADDLE_VERSION ($PADDLE_COMMIT)
+        ${BLUE}paddlemix branch:${NONE} $PPDET_BRANCH ($PPDET_COMMIT)\n"
+
+echo -e "${GREEN}wheel saved under${NONE} ${RED}${BOLD}./dist"
+
+trap : 0

paddlevlp/tests/__init__.py → tests/__init__.py

@@ -10,4 +10,4 @@
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
-# limitations under the License.
+# limitations under the License.

paddlevlp/tests/models/__init__.py → tests/models/__init__.py

@@ -10,4 +10,4 @@
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
-# limitations under the License.
+# limitations under the License.