Skip to content
Package and Release Conda Environment

Package and Release Conda Environment #22

Sign in to view logs

Package and Release Conda Environment

Package and Release Conda Environment #22

Workflow file for this run

.github/workflows/package_and_release.yml at 7c57307
name: Package and Release Conda Environment
on:
workflow_dispatch:
push:
tags:
- 'v*' # 例如，v1.0, v2.0 等
jobs:
build:
runs-on: ubuntu-latest
steps:
- name: 初始磁盘空间检查
run: |
echo "=== 系统磁盘使用情况 ==="
df -h
echo "=== /mnt 目录使用情况 ==="
du -sh /mnt/* 2>/dev/null || true
echo "=== 当前目录使用情况 ==="
du -sh ./* 2>/dev/null || true
- name: 检出存储库
uses: actions/checkout@v4 # 使用最新的v4版本
- name: 创建 /mnt/miniconda 目录并设置权限
run: |
sudo mkdir -p /mnt/miniconda
sudo chmod 777 /mnt/miniconda
ls -ld /mnt/miniconda
echo "=== 创建目录后的磁盘空间 ==="
df -h
- name: 下载 Miniconda 安装脚本
run: |
wget -P /mnt/miniconda https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh
echo "=== Miniconda下载后的磁盘空间 ==="
df -h
du -sh /mnt/miniconda/*
- name: 安装 Miniconda
run: |
bash /mnt/miniconda/Miniconda3-latest-Linux-x86_64.sh -b -p /mnt/miniconda/miniconda3
echo "=== Miniconda安装后的磁盘空间 ==="
df -h
du -sh /mnt/miniconda/*
- name: 配置 Conda
run: |
source /mnt/miniconda/miniconda3/etc/profile.d/conda.sh
conda init bash
# - name: 安装 Miniconda
# uses: conda-incubator/setup-miniconda@v3
# with:
# miniforge-variant: Miniforge
# python-version: '3.10.8'
# base-environment-directory: '/mnt/miniconda'
# installer-url: "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-x86_64.sh"
# auto-activate-base: false
- name: 清理 Docker 和 Conda 缓存
run: |
echo "=== 清理前的磁盘空间 ==="
df -h
docker system prune -af || true
conda clean -a -y
echo "=== 清理后的磁盘空间 ==="
df -h
- name: 设置并安装依赖
run: |
echo "=== 安装依赖前的磁盘空间 ==="
df -h
conda create --prefix /mnt/miniconda/miniconda3/envs/VisRAG python=3.10.8 -y
# conda activate /mnt/miniconda/miniconda3/envs/VisRAG
# conda init bash
# source ~/.bashrc
# conda activate VisRAG
# source /mnt/miniconda/miniconda3/envs/VisRAG/bin/activate
# conda install -c nvidia/label/cuda-11.8.0 cuda-toolkit -y
# conda install -c conda-forge conda-pack -y
# pip install -r requirements.txt
# pip install .
conda run --prefix /mnt/miniconda/miniconda3/envs/VisRAG conda install -c nvidia/label/cuda-11.8.0 cuda-toolkit -y
conda run --prefix /mnt/miniconda/miniconda3/envs/VisRAG conda install -c conda-forge conda-pack -y
# conda run --prefix /mnt/miniconda/miniconda3/envs/VisRAG pip install -r requirements.txt
# conda run --prefix /mnt/miniconda/miniconda3/envs/VisRAG pip install .
# cd timm_modified
# pip install .
# conda run --prefix /mnt/miniconda/miniconda3/envs/VisRAG pip install .
# cd ..
echo "=== 安装依赖后的磁盘空间 ==="
df -h
du -sh /mnt/miniconda/miniconda3/envs/VisRAG
- name: 打包 Conda 环境
run: |
# conda run -n VisRAG conda pack -n VisRAG -o VisRAG_env.tar.gz
echo "=== 打包前的磁盘空间 ==="
df -h
# conda run conda pack -p /mnt/miniconda/miniconda3/envs/VisRAG -o /mnt/miniconda/VisRAG_env.tar.gz
conda run --prefix /mnt/miniconda/miniconda3/envs/VisRAG conda pack -p /mnt/miniconda/miniconda3/envs/VisRAG -o /mnt/miniconda/VisRAG_env.tar.gz
df -h
ls -lh /mnt/miniconda/VisRAG_env.tar.gz
- name: 检查文件大小并处理
run: |
FILE_SIZE=$(stat -c %s /mnt/miniconda/VisRAG_env.tar.gz)
SIZE_2GB=2147483648 # 2GB in bytes
if [ $FILE_SIZE -gt $SIZE_2GB ]; then
# 如果大于2GB，需要分割
TOTAL_PARTS=$(( ($FILE_SIZE + $SIZE_2GB - 1) / $SIZE_2GB )) # 向上取整计算需要的部分数
echo "需要分割成 $TOTAL_PARTS 个部分"
# 循环分割文件
for ((i=1; i<=$TOTAL_PARTS; i++)); do
if [ $i -lt $TOTAL_PARTS ]; then
# 不是最后一个文件，分割2GB
dd if=/mnt/miniconda/VisRAG_env.tar.gz of=/mnt/miniconda/VisRAG_env.tar.gz.part$i bs=1M count=2048 skip=$(((i-1)*2048))
else
# 最后一个文件，获取剩余部分
dd if=/mnt/miniconda/VisRAG_env.tar.gz of=/mnt/miniconda/VisRAG_env.tar.gz.part$i bs=1M skip=$(((i-1)*2048))
fi
done
# 生成合并命令
MERGE_PARTS=$(ls /mnt/miniconda/VisRAG_env.tar.gz.part* | sort -V | tr '\n' ' ')
echo "MERGE_CMD=cat $MERGE_PARTS > VisRAG_env.tar.gz" >> $GITHUB_ENV
echo "SPLIT_FILE=true" >> $GITHUB_ENV
echo "TOTAL_PARTS=$TOTAL_PARTS" >> $GITHUB_ENV
else
# 如果小于2GB，直接上传
echo "SPLIT_FILE=false" >> $GITHUB_ENV
fi
- name: 上传完整文件到 Release (小于2GB)
if: env.SPLIT_FILE == 'false'
uses: softprops/action-gh-release@v2
with:
tag_name: ${{ github.ref_name }}
name: VisRAG Environment for ${{ github.ref_name }}
body: "Conda environment package for version ${{ github.ref_name }}"
files: /mnt/miniconda/VisRAG_env.tar.gz
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: 上传分割文件到 Release (大于2GB)
if: env.SPLIT_FILE == 'true'
uses: softprops/action-gh-release@v2
with:
tag_name: ${{ github.ref_name }}
name: VisRAG Environment for ${{ github.ref_name }}
body: |
Conda environment package for version ${{ github.ref_name }}
File has been split into ${{ env.TOTAL_PARTS }} parts due to size limitations.
Please download all parts and combine them using:
```bash
${{ env.MERGE_CMD }}
```
files: /mnt/miniconda/VisRAG_env.tar.gz.part*
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
# - name: 上传环境到 GitHub Release
# uses: softprops/action-gh-release@v2 # 使用有效的发布 Action 版本
# with:
# tag_name: ${{ github.ref_name }}
# name: VisRAG Environment for ${{ github.ref_name }}
# body: "Conda environment package for version ${{ github.ref_name }}"
# files: /mnt/miniconda/VisRAG_env.tar.gz
# env:
# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: 最终磁盘空间检查
if: always()
run: |
echo "=== 最终磁盘使用情况 ==="
df -h
echo "=== /mnt 目录最终使用情况 ==="
du -sh /mnt/* 2>/dev/null || true