Linux用実行バイナリの自動ビルドを追加 (#95) (#96)

* linux build workflow

* add note about GitHub Actions disk capacity

* fix typo (misreplace)

* Nuitka cache working

* run build-linux on push master

* follow upstream (added build args)

* rm speakers.json form build script in docker

* nvidia build

* Makefile: fix target name for nvidia build in docker

* fix gpu build: copy libtorch "*.so.*" instead "*.so"

* add commit id to dist archive

* copy libtorch *.so and *.so.*

* replace link path: libcore.so to libtorch

* install patchelf in build env

* fix libcore.so path to patchelf

* add exec perm to run

* versioned artifact name for release

* fix artifact name; variable

* Makefile: compile-python-env base image arg

* artifact name as matrix

* artifact name: linux-cpu and linux-nvidia

* add comment about artifact name.

.github/workflows/build.yml

@@ -0,0 +1,111 @@
+name: build
+on:
+  push:
+    branches:
+      - master
+  release:
+    types:
+      - created
+
+env:
+  IMAGE_NAME: ${{ secrets.DOCKERHUB_USERNAME }}/voicevox_engine
+
+jobs:
+  # Build Linux binary (push only buildcache image)
+  build-linux:
+    strategy:
+      matrix:
+        include:
+        - os: ubuntu-latest
+          tag: build-cpu-ubuntu20.04
+          runtime_tag: cpu-ubuntu20.04 # for cache use
+          target: build-env
+          base_image: ubuntu:focal
+          base_runtime_image: ubuntu:focal
+          inference_device: cpu
+          libtorch_url: https://download.pytorch.org/libtorch/cpu/libtorch-cxx11-abi-shared-with-deps-1.9.0%2Bcpu.zip
+          artifact_name: linux-cpu
+          nuitka_cache_path: nuitka_cache
+        - os: ubuntu-latest
+          tag: build-nvidia-ubuntu20.04
+          runtime_tag: nvidia-ubuntu20.04 # for cache use
+          target: build-env
+          base_image: ubuntu:focal
+          base_runtime_image: nvidia/cuda:11.4.1-cudnn8-runtime-ubuntu20.04
+          inference_device: nvidia
+          libtorch_url: https://download.pytorch.org/libtorch/cu111/libtorch-cxx11-abi-shared-with-deps-1.9.0%2Bcu111.zip
+          artifact_name: linux-nvidia
+          nuitka_cache_path: nuitka_cache
+
+    runs-on: ${{ matrix.os }}
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Setup Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      # NOTE: `load: true` may silently fail when the GitHub Actions disk (14GB) is full.
+      # https://docs.github.com/ja/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources
+      - name: Create binary build environment with Docker
+        uses: docker/build-push-action@v2
+        env:
+          IMAGE_TAG: ${{ env.IMAGE_NAME }}:${{ matrix.tag }}${{ (matrix.tag != '' && '-') || '' }}latest
+          RUNTIME_IMAGE_TAG: ${{ env.IMAGE_NAME }}:${{ matrix.runtime_tag }}${{ (matrix.runtime_tag != '' && '-') || '' }}latest
+        with:
+          context: .
+          builder: ${{ steps.buildx.outputs.name }}
+          file: ./Dockerfile
+          build-args: |
+            BASE_IMAGE=${{ matrix.base_image }}
+            BASE_RUNTIME_IMAGE=${{ matrix.base_runtime_image }}
+            INFERENCE_DEVICE=${{ matrix.inference_device }}
+            LIBTORCH_URL=${{ matrix.libtorch_url }}
+          target: ${{ matrix.target }}
+          load: true
+          tags: |
+            ${{ env.IMAGE_TAG }}
+          cache-from: |
+            type=registry,ref=${{ env.IMAGE_TAG }}-buildcache
+            type=registry,ref=${{ env.RUNTIME_IMAGE_TAG }}-buildcache
+          cache-to: type=registry,ref=${{ env.IMAGE_TAG }}-buildcache,mode=max
+
+      # Build run.py with Nuitka in Docker
+      - name: Cache Nuitka (ccache, module-cache)
+        uses: actions/cache@v2
+        id: nuitka-cache
+        with:
+          path: ${{ matrix.nuitka_cache_path }}
+          key: ${{ runner.os }}-nuitka-${{ matrix.target }}-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-nuitka-${{ matrix.target }}-
+
+      - name: Build run.py with Nuitka in Docker
+        env:
+          IMAGE_TAG: ${{ env.IMAGE_NAME }}:${{ matrix.tag }}${{ (matrix.tag != '' && '-') || '' }}latest
+        run: |
+          docker run --rm \
+            -v "$(readlink -f "${{ matrix.nuitka_cache_path }}"):/home/user/.cache/Nuitka" \
+            -v "$(readlink -f "build"):/opt/voicevox_engine_build" \
+            "${{ env.IMAGE_TAG }}"
+
+      # FIXME: versioned name may be useful; but
+      # actions/download-artifact and dawidd6/download-artifact do not support
+      # wildcard / forward-matching yet.
+      # Currently, It is good to use static artifact name for future binary test workflow.
+      # https://github.com/actions/toolkit/blob/ea81280a4d48fb0308d40f8f12ae00d117f8acb9/packages/artifact/src/internal/artifact-client.ts#L147
+      # https://github.com/dawidd6/action-download-artifact/blob/af92a8455a59214b7b932932f2662fdefbd78126/main.js#L113
+      - uses: actions/upload-artifact@v2
+        # env:
+        #   VERSIONED_ARTIFACT_NAME: |
+        #     ${{ format('{0}-{1}', matrix.artifact_name, (github.event.release.tag_name != '' && github.event.release.tag_name) || github.sha) }}
+        with:
+          name: ${{ matrix.artifact_name }}
+          path: build/run.dist/

.gitignore

@@ -14,3 +14,6 @@ __pycache__/
 venv/
 # JetBrains IDE project settings folder
 .idea/
+
+/build
+/cache

Dockerfile

@@ -218,7 +218,7 @@ RUN <<EOF
     gosu user pip3 install -r /tmp/requirements.txt
 
     # Install voicevox_core
-    cd /opt/voicevox_core_example/example/python 
+    cd /opt/voicevox_core_example/example/python
     gosu user pip3 install .
 
     # FIXME: remove first execution delay
@@ -236,3 +236,79 @@ CMD [ "gosu", "user", "/opt/python/bin/python3", "./run.py", "--voicevox_dir", "
 # Enable use_gpu
 FROM runtime-env AS runtime-nvidia-env
 CMD [ "gosu", "user", "/opt/python/bin/python3", "./run.py", "--use_gpu", "--voicevox_dir", "/opt/voicevox_core/", "--voicelib_dir", "/opt/voicevox_core/", "--host", "0.0.0.0" ]
+
+# Binary build environment (common to CPU, GPU)
+FROM runtime-env AS build-env
+
+# Install ccache for Nuitka cache
+# chrpath: required for nuitka build; 'RPATH' settings in used shared
+RUN <<EOF
+    apt-get update
+    apt-get install -y \
+        ccache \
+        chrpath \
+        patchelf
+    apt-get clean
+    rm -rf /var/lib/apt/lists/*
+EOF
+
+# Install Python build dependencies
+ADD ./requirements-dev.txt /tmp/
+RUN <<EOF
+    gosu user /opt/python/bin/pip3 install -r /tmp/requirements-dev.txt
+EOF
+
+# Create build script
+RUN <<EOF
+    set -eux
+
+    cat <<EOD > /build.sh
+        #!/bin/bash
+        set -eux
+
+        # chown general user c.z. mounted directory may be owned by root
+        mkdir -p /opt/voicevox_engine_build
+        chown -R user:user /opt/voicevox_engine_build
+
+        mkdir -p /home/user/.cache/Nuitka
+        chown -R user:user /home/user/.cache/Nuitka
+
+        cd /opt/voicevox_engine_build
+
+        LIBRARY_PATH="/opt/voicevox_core:\${LIBRARY_PATH:-}" \
+            gosu user /opt/python/bin/python3 -m nuitka \
+                --output-dir=/opt/voicevox_engine_build \
+                --standalone \
+                --plugin-enable=numpy \
+                --follow-import-to=numpy \
+                --follow-import-to=aiofiles \
+                --include-package=uvicorn \
+                --include-package-data=pyopenjtalk \
+                --include-package-data=resampy \
+                --include-data-file=/opt/voicevox_engine/VERSION.txt=./ \
+                --include-data-file=/opt/libtorch/lib/*.so=./ \
+                --include-data-file=/opt/libtorch/lib/*.so.*=./ \
+                --include-data-file=/opt/voicevox_core/*.so=./ \
+                --include-data-file=/opt/voicevox_core/*.bin=./ \
+                --include-data-file=/opt/voicevox_core/metas.json=./ \
+                --include-data-file=/home/user/.local/lib/python*/site-packages/llvmlite/binding/*.so=./ \
+                --follow-imports \
+                --no-prefer-source-code \
+                /opt/voicevox_engine/run.py
+
+        # replace libcore.so link for libtorch to relative path
+        cat <<EOT | xargs -I '%' patchelf --replace-needed "%" "./%" /opt/voicevox_engine_build/run.dist/libcore.so
+            libc10.so
+            libtorch_cuda.so
+            libtorch_cuda_cpp.so
+            libtorch_cpu.so
+            libtorch_cuda_cu.so
+            libtorch.so
+EOT
+
+        chmod +x /opt/voicevox_engine_build/run.dist/run
+EOD
+    chmod +x /build.sh
+EOF
+
+CMD [ "bash", "/build.sh" ]

Makefile

@@ -6,6 +6,7 @@ build-linux-docker-ubuntu:
 		-t hiroshiba/voicevox_engine:cpu-ubuntu20.04-latest \
 		--target runtime-env \
 		--progress plain \
+		--build-arg BASE_IMAGE=ubuntu:focal \
 		--build-arg BASE_RUNTIME_IMAGE=ubuntu:focal \
 		--build-arg LIBTORCH_URL=https://download.pytorch.org/libtorch/cpu/libtorch-cxx11-abi-shared-with-deps-1.9.0%2Bcpu.zip \
 		--build-arg INFERENCE_DEVICE=cpu
@@ -22,6 +23,7 @@ build-linux-docker-nvidia:
 		-t hiroshiba/voicevox_engine:nvidia-ubuntu20.04-latest \
 		--target runtime-nvidia-env \
 		--progress plain \
+		--build-arg BASE_IMAGE=ubuntu:focal \
 		--build-arg BASE_RUNTIME_IMAGE=nvidia/cuda:11.4.1-cudnn8-runtime-ubuntu20.04 \
 		--build-arg LIBTORCH_URL=https://download.pytorch.org/libtorch/cu111/libtorch-cxx11-abi-shared-with-deps-1.9.0%2Bcu111.zip \
 		--build-arg INFERENCE_DEVICE=nvidia
@@ -39,9 +41,47 @@ build-linux-docker-compile-python-env:
 		-t hiroshiba/voicevox_engine:compile-python-env \
 		--target compile-python-env \
 		--progress plain \
-		--build-arg BASE_RUNTIME_IMAGE=ubuntu:focal
+		--build-arg BASE_IMAGE=ubuntu:focal
 
 .PHONY: run-linux-docker-compile-python-env
 run-linux-docker-compile-python-env:
 	docker run --rm -it \
 		hiroshiba/voicevox_engine:compile-python-env $(CMD)
+
+# Build linux binary in Docker
+.PHONY: build-linux-docker-build
+build-linux-docker-build:
+	docker buildx build . \
+		-t hiroshiba/voicevox_engine:build-cpu-ubuntu20.04-latest \
+		--target build-env \
+		--progress plain \
+		--build-arg BASE_IMAGE=ubuntu:focal \
+		--build-arg BASE_RUNTIME_IMAGE=ubuntu:focal \
+		--build-arg LIBTORCH_URL=https://download.pytorch.org/libtorch/cpu/libtorch-cxx11-abi-shared-with-deps-1.9.0%2Bcpu.zip \
+		--build-arg INFERENCE_DEVICE=cpu
+
+.PHONY: run-linux-docker-build
+run-linux-docker-build:
+	docker run --rm -it \
+		-v "$(shell pwd)/cache/Nuitka:/home/user/.cache/Nuitka" \
+		-v "$(shell pwd)/build:/opt/voicevox_engine_build" \
+		hiroshiba/voicevox_engine:build-cpu-ubuntu20.04-latest $(CMD)
+
+
+.PHONY: build-linux-docker-build-nvidia
+build-linux-docker-build-nvidia:
+	docker buildx build . \
+		-t hiroshiba/voicevox_engine:build-nvidia-ubuntu20.04-latest \
+		--target build-env \
+		--progress plain \
+		--build-arg BASE_IMAGE=ubuntu:focal \
+		--build-arg BASE_RUNTIME_IMAGE=nvidia/cuda:11.4.1-cudnn8-runtime-ubuntu20.04 \
+		--build-arg LIBTORCH_URL=https://download.pytorch.org/libtorch/cu111/libtorch-cxx11-abi-shared-with-deps-1.9.0%2Bcu111.zip \
+		--build-arg INFERENCE_DEVICE=nvidia
+
+.PHONY: run-linux-docker-build-nvidia
+run-linux-docker-build-nvidia:
+	docker run --rm -it \
+		-v "$(shell pwd)/cache/Nuitka:/home/user/.cache/Nuitka" \
+		-v "$(shell pwd)/build:/opt/voicevox_engine_build" \
+		hiroshiba/voicevox_engine:build-nvidia-ubuntu20.04-latest $(CMD)