From b44321f9c3dc97f52a3394c8d76a9e7801ecb90d Mon Sep 17 00:00:00 2001 From: Zhichang Yu Date: Fri, 6 Dec 2024 20:47:22 +0800 Subject: [PATCH 1/2] Introduced NEED_MIRROR (#3907) ### What problem does this PR solve? Introduced NEED_MIRROR ### Type of change - [x] Refactoring --- .github/workflows/tests.yml | 4 +- Dockerfile | 42 ++++++++++--------- README.md | 4 +- README_id.md | 4 +- README_ja.md | 4 +- README_ko.md | 4 +- README_zh.md | 10 +++-- .../develop/launch_ragflow_from_source.md | 2 +- 8 files changed, 40 insertions(+), 34 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 4e467fdad61..9eb2e30718e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -53,11 +53,11 @@ jobs: run: | RUNNER_WORKSPACE_PREFIX=${RUNNER_WORKSPACE_PREFIX:-$HOME} sudo docker pull ubuntu:22.04 - sudo docker build --progress=plain --build-arg LIGHTEN=1 -f Dockerfile -t infiniflow/ragflow:dev-slim . + sudo docker build --progress=plain --build-arg LIGHTEN=1 --build-arg NEED_MIRROR=1 -f Dockerfile -t infiniflow/ragflow:dev-slim . - name: Build ragflow:dev run: | - sudo docker build --progress=plain -f Dockerfile -t infiniflow/ragflow:dev . + sudo docker build --progress=plain --build-arg NEED_MIRROR=1 -f Dockerfile -t infiniflow/ragflow:dev . - name: Start ragflow:dev-slim run: | diff --git a/Dockerfile b/Dockerfile index 002955f63d2..1e0c46ce812 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,6 +3,7 @@ FROM ubuntu:22.04 AS base USER root SHELL ["/bin/bash", "-c"] +ARG NEED_MIRROR=0 ARG LIGHTEN=0 ENV LIGHTEN=${LIGHTEN} @@ -16,7 +17,7 @@ RUN --mount=type=bind,from=infiniflow/ragflow_deps:latest,source=/huggingface.co /huggingface.co/InfiniFlow/deepdoc \ | tar -xf - --strip-components=3 -C /ragflow/rag/res/deepdoc RUN --mount=type=bind,from=infiniflow/ragflow_deps:latest,source=/huggingface.co,target=/huggingface.co \ - if [ "$LIGHTEN" == "0" ]; then \ + if [ "$LIGHTEN" != "1" ]; then \ (tar -cf - \ /huggingface.co/BAAI/bge-large-zh-v1.5 \ /huggingface.co/BAAI/bge-reranker-v2-m3 \ @@ -35,25 +36,27 @@ RUN --mount=type=bind,from=infiniflow/ragflow_deps:latest,source=/,target=/deps ENV TIKA_SERVER_JAR="file:///ragflow/tika-server-standard-3.0.0.jar" # Setup apt +# cv2 requires libGL.so.1 RUN --mount=type=cache,id=ragflow_apt,target=/var/cache/apt,sharing=locked \ - sed -i 's|http://archive.ubuntu.com|https://mirrors.tuna.tsinghua.edu.cn|g' /etc/apt/sources.list && \ + if [ "$NEED_MIRROR" == "1" ]; then \ + sed -i 's|http://archive.ubuntu.com|https://mirrors.tuna.tsinghua.edu.cn|g' /etc/apt/sources.list; \ + fi; \ rm -f /etc/apt/apt.conf.d/docker-clean && \ echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache && \ - apt update && apt --no-install-recommends install -y ca-certificates && \ - rm -rf /var/lib/apt/lists/* - -# cv2 requires libGL.so.1 -RUN --mount=type=cache,id=ragflow_apt,target=/var/cache/apt,sharing=locked \ - apt update && DEBIAN_FRONTEND=noninteractive apt install -y curl libpython3-dev nginx libglib2.0-0 libglx-mesa0 pkg-config libicu-dev libgdiplus default-jdk python3-pip pipx \ - libatk-bridge2.0-0 libgtk-4-1 libnss3 xdg-utils unzip libgbm-dev wget git nginx libgl1 vim less && \ - rm -rf /var/lib/apt/lists/* + apt update && \ + apt --no-install-recommends install -y ca-certificates && \ + apt update && \ + DEBIAN_FRONTEND=noninteractive apt install -y curl libpython3-dev nginx libglib2.0-0 libglx-mesa0 pkg-config libicu-dev libgdiplus default-jdk python3-pip pipx \ + libatk-bridge2.0-0 libgtk-4-1 libnss3 xdg-utils unzip libgbm-dev wget git nginx libgl1 vim less -RUN pip3 config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple && \ - pip3 config set global.trusted-host pypi.tuna.tsinghua.edu.cn && \ - pipx install poetry && \ - pipx runpip poetry config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple && \ - pipx runpip poetry config set global.trusted-host pypi.tuna.tsinghua.edu.cn && \ - /root/.local/bin/poetry self add poetry-plugin-pypi-mirror +RUN if [ "$NEED_MIRROR" == "1" ]; then \ + pip3 config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple && \ + pip3 config set global.trusted-host pypi.tuna.tsinghua.edu.cn; \ + fi; \ + pipx install poetry; \ + if [ "$NEED_MIRROR" == "1" ]; then \ + pipx inject poetry poetry-plugin-pypi-mirror; \ + fi ENV PYTHONDONTWRITEBYTECODE=1 DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=1 ENV PATH=/root/.local/bin:$PATH @@ -62,7 +65,6 @@ ENV POETRY_NO_INTERACTION=1 ENV POETRY_VIRTUALENVS_IN_PROJECT=true ENV POETRY_VIRTUALENVS_CREATE=true ENV POETRY_REQUESTS_TIMEOUT=15 -ENV POETRY_PYPI_MIRROR_URL=https://pypi.tuna.tsinghua.edu.cn/simple/ # nodejs 12.22 on Ubuntu 22.04 is too old RUN --mount=type=cache,id=ragflow_apt,target=/var/cache/apt,sharing=locked \ @@ -70,8 +72,7 @@ RUN --mount=type=cache,id=ragflow_apt,target=/var/cache/apt,sharing=locked \ apt purge -y nodejs npm && \ apt autoremove && \ apt update && \ - apt install -y nodejs cargo && \ - rm -rf /var/lib/apt/lists/* + apt install -y nodejs cargo # Add dependencies of selenium RUN --mount=type=bind,from=infiniflow/ragflow_deps:latest,source=/chrome-linux64-121-0-6167-85,target=/chrome-linux64.zip \ @@ -103,6 +104,9 @@ WORKDIR /ragflow COPY pyproject.toml poetry.toml poetry.lock ./ RUN --mount=type=cache,id=ragflow_poetry,target=/root/.cache/pypoetry,sharing=locked \ + if [ "$NEED_MIRROR" == "1" ]; then \ + export POETRY_PYPI_MIRROR_URL=https://pypi.tuna.tsinghua.edu.cn/simple/; \ + fi; \ if [ "$LIGHTEN" == "1" ]; then \ poetry install --no-root; \ else \ diff --git a/README.md b/README.md index d65f8fa2e73..8be1526b35e 100644 --- a/README.md +++ b/README.md @@ -289,14 +289,14 @@ docker build -f Dockerfile -t infiniflow/ragflow:dev . 1. Install Poetry, or skip this step if it is already installed: ```bash - curl -sSL https://install.python-poetry.org | python3 - + pipx install poetry + export POETRY_VIRTUALENVS_CREATE=true POETRY_VIRTUALENVS_IN_PROJECT=true ``` 2. Clone the source code and install Python dependencies: ```bash git clone https://github.com/infiniflow/ragflow.git cd ragflow/ - export POETRY_VIRTUALENVS_CREATE=true POETRY_VIRTUALENVS_IN_PROJECT=true ~/.local/bin/poetry install --sync --no-root --with=full # install RAGFlow dependent python modules ``` diff --git a/README_id.md b/README_id.md index b12315e40c4..fd06d63b8cb 100644 --- a/README_id.md +++ b/README_id.md @@ -264,14 +264,14 @@ docker build -f Dockerfile -t infiniflow/ragflow:dev . 1. Instal Poetry, atau lewati langkah ini jika sudah terinstal: ```bash - curl -sSL https://install.python-poetry.org | python3 - + pipx install poetry + export POETRY_VIRTUALENVS_CREATE=true POETRY_VIRTUALENVS_IN_PROJECT=true ``` 2. Clone kode sumber dan instal dependensi Python: ```bash git clone https://github.com/infiniflow/ragflow.git cd ragflow/ - export POETRY_VIRTUALENVS_CREATE=true POETRY_VIRTUALENVS_IN_PROJECT=true ~/.local/bin/poetry install --sync --no-root # install modul python RAGFlow ``` diff --git a/README_ja.md b/README_ja.md index 8fcda98523d..22135c88e39 100644 --- a/README_ja.md +++ b/README_ja.md @@ -245,14 +245,14 @@ docker build -f Dockerfile -t infiniflow/ragflow:dev . 1. Poetry をインストールする。すでにインストールされている場合は、このステップをスキップしてください: ```bash - curl -sSL https://install.python-poetry.org | python3 - + pipx install poetry + export POETRY_VIRTUALENVS_CREATE=true POETRY_VIRTUALENVS_IN_PROJECT=true ``` 2. ソースコードをクローンし、Python の依存関係をインストールする: ```bash git clone https://github.com/infiniflow/ragflow.git cd ragflow/ - export POETRY_VIRTUALENVS_CREATE=true POETRY_VIRTUALENVS_IN_PROJECT=true ~/.local/bin/poetry install --sync --no-root # install RAGFlow dependent python modules ``` diff --git a/README_ko.md b/README_ko.md index ec33c78db36..87cde3d4ec0 100644 --- a/README_ko.md +++ b/README_ko.md @@ -247,14 +247,14 @@ docker build -f Dockerfile -t infiniflow/ragflow:dev . 1. Poetry를 설치하거나 이미 설치된 경우 이 단계를 건너뜁니다: ```bash - curl -sSL https://install.python-poetry.org | python3 - + pipx install poetry + export POETRY_VIRTUALENVS_CREATE=true POETRY_VIRTUALENVS_IN_PROJECT=true ``` 2. 소스 코드를 클론하고 Python 의존성을 설치합니다: ```bash git clone https://github.com/infiniflow/ragflow.git cd ragflow/ - export POETRY_VIRTUALENVS_CREATE=true POETRY_VIRTUALENVS_IN_PROJECT=true ~/.local/bin/poetry install --sync --no-root # install RAGFlow dependent python modules ``` diff --git a/README_zh.md b/README_zh.md index c03e05ed5a9..f7a5b1b25a4 100644 --- a/README_zh.md +++ b/README_zh.md @@ -235,7 +235,7 @@ RAGFlow 默认使用 Elasticsearch 存储文本和向量数据. 如果要切换 ```bash git clone https://github.com/infiniflow/ragflow.git cd ragflow/ -docker build --build-arg LIGHTEN=1 -f Dockerfile -t infiniflow/ragflow:dev-slim . +docker build --build-arg LIGHTEN=1 --build-arg NEED_MIRROR=1 -f Dockerfile -t infiniflow/ragflow:dev-slim . ``` ## 🔧 源码编译 Docker 镜像（包含 embedding 模型） @@ -245,21 +245,23 @@ docker build --build-arg LIGHTEN=1 -f Dockerfile -t infiniflow/ragflow:dev-slim ```bash git clone https://github.com/infiniflow/ragflow.git cd ragflow/ -docker build -f Dockerfile -t infiniflow/ragflow:dev . +docker build --build-arg NEED_MIRROR=1 -f Dockerfile -t infiniflow/ragflow:dev . ``` ## 🔨 以源代码启动服务 1. 安装 Poetry。如已经安装，可跳过本步骤： ```bash - curl -sSL https://install.python-poetry.org | python3 - + pipx install poetry + pipx inject poetry poetry-plugin-pypi-mirror + export POETRY_VIRTUALENVS_CREATE=true POETRY_VIRTUALENVS_IN_PROJECT=true + export POETRY_PYPI_MIRROR_URL=https://pypi.tuna.tsinghua.edu.cn/simple/ ``` 2. 下载源代码并安装 Python 依赖： ```bash git clone https://github.com/infiniflow/ragflow.git cd ragflow/ - export POETRY_VIRTUALENVS_CREATE=true POETRY_VIRTUALENVS_IN_PROJECT=true ~/.local/bin/poetry install --sync --no-root # install RAGFlow dependent python modules ``` diff --git a/docs/guides/develop/launch_ragflow_from_source.md b/docs/guides/develop/launch_ragflow_from_source.md index 05af7c668d0..9a0e34bb5a4 100644 --- a/docs/guides/develop/launch_ragflow_from_source.md +++ b/docs/guides/develop/launch_ragflow_from_source.md @@ -38,7 +38,7 @@ cd ragflow/ 1. Install Poetry: ```bash - curl -sSL https://install.python-poetry.org | python3 - + pipx install poetry ``` 2. Configure Poetry: From 885418f3b0869ca1e811f8fd6e6fc7a397a02653 Mon Sep 17 00:00:00 2001 From: Zhichang Yu Date: Fri, 6 Dec 2024 21:10:06 +0800 Subject: [PATCH 2/2] Fix release.yml --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 38cc5aeabb5..4454f48077b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -50,10 +50,10 @@ jobs: - name: Move the existing mutable tag # https://github.com/softprops/action-gh-release/issues/171 run: | + git fetch --tags if [[ $GITHUB_EVENT_NAME == 'schedule' ]]; then # Determine if a given tag exists and matches a specific Git commit. # actions/checkout@v4 fetch-tags doesn't work when triggered by schedule - git fetch --tags if [ "$(git rev-parse -q --verify "refs/tags/$RELEASE_TAG")" = "$GITHUB_SHA" ]; then echo "mutable tag $RELEASE_TAG exists and matches $GITHUB_SHA" else