From b8aa41a11f5535c61291ef223c56bbe55228da97 Mon Sep 17 00:00:00 2001 From: sabonerune <102559104+sabonerune@users.noreply.github.com> Date: Mon, 23 Oct 2023 17:51:38 +0900 Subject: [PATCH 1/7] =?UTF-8?q?BLD:=20PyInstaller=E3=82=92v6=E3=81=B8?= =?UTF-8?q?=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/build.yml | 79 +++++++++++++++--------- poetry.lock | 61 +++++++++--------- pyproject.toml | 6 +- requirements-dev.txt | 10 +-- run.py | 3 +- run.spec | 46 +++----------- voicevox_engine/setting/SettingLoader.py | 4 +- voicevox_engine/user_dict.py | 4 +- voicevox_engine/utility/__init__.py | 5 +- voicevox_engine/utility/path_utility.py | 43 +++++++++---- 10 files changed, 136 insertions(+), 125 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9f88c238e..453f59ea8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -453,28 +453,29 @@ jobs: # Replace version & specify dynamic libraries $sed -i "s/__version__ = \"latest\"/__version__ = \"${{ needs.config.outputs.version_or_latest }}\"/" voicevox_engine/__init__.py - if [[ ${{ matrix.os }} == windows-* ]]; then - LIBCORE_PATH=download/core/voicevox_core.dll - LIBONNXRUNTIME_PATH=download/onnxruntime/lib/onnxruntime.dll - elif [[ ${{ matrix.os }} == macos-* ]]; then - LIBCORE_PATH=download/core/libvoicevox_core.dylib - LIBONNXRUNTIME_PATH=download/onnxruntime/lib/libonnxruntime.dylib - else - LIBCORE_PATH=download/core/libvoicevox_core.so - LIBONNXRUNTIME_PATH=download/onnxruntime/lib/libonnxruntime.so - fi - CORE_MODEL_DIR_PATH="download/core/model" \ - LIBCORE_PATH="$LIBCORE_PATH" \ - LIBONNXRUNTIME_PATH="$LIBONNXRUNTIME_PATH" \ pyinstaller --noconfirm run.spec + - name: Gather files to engine root + run: | + set -eux + + mv licenses.json dist/run/ + mv speaker_info dist/run/ + mv engine_manifest.json dist/run/ + mv engine_manifest_assets dist/run/ + mv download/core/model dist/run/ + - name: Gather DLL dependencies to dist/run/ (Windows) if: startsWith(matrix.os, 'windows-') run: | set -eux # Move DLL dependencies (cache already saved) + mv download/core/voicevox_core.dll dist/run/ + rm -rf download/core + + mv download/onnxruntime/lib/onnxruntime.dll dist/run/ if [ -f "download/onnxruntime/lib/onnxruntime_providers_cuda.dll" ]; then # ONNX Runtime providers (PyInstaller does not copy dynamic loaded libraries) @@ -509,32 +510,50 @@ jobs: rm -rf download/directml fi - - name: Gather DLL dependencies to dist/run/ (Linux CUDA) - if: startsWith(matrix.os, 'ubuntu-') && endsWith(matrix.target, 'nvidia') + - name: Gather DLL dependencies to dist/run/ (Linux) + if: startsWith(matrix.os, 'ubuntu-') run: | set -eux # Move DLL dependencies (cache already saved) + cp download/core/libvoicevox_core.so dist/run/ + rm -rf download/core + + cp download/onnxruntime/lib/libonnxruntime.so dist/run/ - # ONNX Runtime providers (PyInstaller does not copy dynamic loaded libraries) - patchelf --set-rpath '$ORIGIN' "$(pwd)/download/onnxruntime/lib"/libonnxruntime_providers_*.so - mv download/onnxruntime/lib/libonnxruntime_*.so dist/run/ + if [[ ${{ matrix.target }} == *-nvidia ]]; then + # ONNX Runtime providers (PyInstaller does not copy dynamic loaded libraries) + patchelf --set-rpath '$ORIGIN' "$(pwd)/download/onnxruntime/lib"/libonnxruntime_providers_*.so + mv download/onnxruntime/lib/libonnxruntime_*.so dist/run/ - # CUDA - mv download/cuda/bin/libcublas.so.* dist/run/ - mv download/cuda/bin/libcublasLt.so.* dist/run/ - mv download/cuda/bin/libcudart.so.* dist/run/ - mv download/cuda/bin/libcufft.so.* dist/run/ - mv download/cuda/bin/libcurand.so.* dist/run/ + # CUDA + mv download/cuda/bin/libcublas.so.* dist/run/ + mv download/cuda/bin/libcublasLt.so.* dist/run/ + mv download/cuda/bin/libcudart.so.* dist/run/ + mv download/cuda/bin/libcufft.so.* dist/run/ + mv download/cuda/bin/libcurand.so.* dist/run/ - # cuDNN - mv download/cudnn/bin/libcudnn.so.* dist/run/ - mv download/cudnn/bin/libcudnn_*_infer.so.* dist/run/ + # cuDNN + mv download/cudnn/bin/libcudnn.so.* dist/run/ + mv download/cudnn/bin/libcudnn_*_infer.so.* dist/run/ + + # Remove source directories to reduce disk usage (already cached) + rm -rf download/onnxruntime + rm -rf download/cuda + rm -rf download/cudnn + fi + + - name: Gather DLL dependencies to dist/run/ (Mac) + if: startsWith(matrix.os, 'macos-') + run: | + set -eux + + # Move DLL dependencies (cache already saved) + cp download/core/libvoicevox_core.dylib dist/run/ + rm -rf download/core - # Remove source directories to reduce disk usage (already cached) + cp download/onnxruntime/lib/libonnxruntime.dylib dist/run/ rm -rf download/onnxruntime - rm -rf download/cuda - rm -rf download/cudnn - name: Set @rpath to @executable_path if: startsWith(matrix.os, 'macos-') diff --git a/poetry.lock b/poetry.lock index 6e7957c9a..128c9d660 100644 --- a/poetry.lock +++ b/poetry.lock @@ -13,13 +13,13 @@ files = [ [[package]] name = "altgraph" -version = "0.17.3" +version = "0.17.4" description = "Python graph (network) package" optional = false python-versions = "*" files = [ - {file = "altgraph-0.17.3-py2.py3-none-any.whl", hash = "sha256:c8ac1ca6772207179ed8003ce7687757c04b0b71536f81e2ac5755c6226458fe"}, - {file = "altgraph-0.17.3.tar.gz", hash = "sha256:ad33358114df7c9416cdb8fa1eaa5852166c505118717021c6a8c7c7abbd03dd"}, + {file = "altgraph-0.17.4-py2.py3-none-any.whl", hash = "sha256:642743b4750de17e655e6711601b077bc6598dbfa3ba5fa2b2a35ce12b508dff"}, + {file = "altgraph-0.17.4.tar.gz", hash = "sha256:1b5afbb98f6c4dcadb2e2ae6ab9fa994bbb8c1d75f4fa96d340f9437ae454406"}, ] [[package]] @@ -1020,13 +1020,13 @@ testing = ["pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", [[package]] name = "macholib" -version = "1.16.2" +version = "1.16.3" description = "Mach-O header analysis and editing" optional = false python-versions = "*" files = [ - {file = "macholib-1.16.2-py2.py3-none-any.whl", hash = "sha256:44c40f2cd7d6726af8fa6fe22549178d3a4dfecc35a9cd15ea916d9c83a688e0"}, - {file = "macholib-1.16.2.tar.gz", hash = "sha256:557bbfa1bb255c20e9abafe7ed6cd8046b48d9525db2f9b77d3122a63a2a8bf8"}, + {file = "macholib-1.16.3-py2.py3-none-any.whl", hash = "sha256:0e315d7583d38b8c77e815b1ecbdbf504a8258d8b3e17b61165c6feb60d18f2c"}, + {file = "macholib-1.16.3.tar.gz", hash = "sha256:07ae9e15e8e4cd9a788013d81f5908b3609aa76f9b1421bae9c4d7606ec86a30"}, ] [package.dependencies] @@ -1612,46 +1612,47 @@ files = [ [[package]] name = "pyinstaller" -version = "5.13.2" +version = "6.2.0" description = "PyInstaller bundles a Python application and all its dependencies into a single package." optional = false -python-versions = "<3.13,>=3.7" +python-versions = "<3.13,>=3.8" files = [ - {file = "pyinstaller-5.13.2-py3-none-macosx_10_13_universal2.whl", hash = "sha256:16cbd66b59a37f4ee59373a003608d15df180a0d9eb1a29ff3bfbfae64b23d0f"}, - {file = "pyinstaller-5.13.2-py3-none-manylinux2014_aarch64.whl", hash = "sha256:8f6dd0e797ae7efdd79226f78f35eb6a4981db16c13325e962a83395c0ec7420"}, - {file = "pyinstaller-5.13.2-py3-none-manylinux2014_i686.whl", hash = "sha256:65133ed89467edb2862036b35d7c5ebd381670412e1e4361215e289c786dd4e6"}, - {file = "pyinstaller-5.13.2-py3-none-manylinux2014_ppc64le.whl", hash = "sha256:7d51734423685ab2a4324ab2981d9781b203dcae42839161a9ee98bfeaabdade"}, - {file = "pyinstaller-5.13.2-py3-none-manylinux2014_s390x.whl", hash = "sha256:2c2fe9c52cb4577a3ac39626b84cf16cf30c2792f785502661286184f162ae0d"}, - {file = "pyinstaller-5.13.2-py3-none-manylinux2014_x86_64.whl", hash = "sha256:c63ef6133eefe36c4b2f4daf4cfea3d6412ece2ca218f77aaf967e52a95ac9b8"}, - {file = "pyinstaller-5.13.2-py3-none-musllinux_1_1_aarch64.whl", hash = "sha256:aadafb6f213549a5906829bb252e586e2cf72a7fbdb5731810695e6516f0ab30"}, - {file = "pyinstaller-5.13.2-py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:b2e1c7f5cceb5e9800927ddd51acf9cc78fbaa9e79e822c48b0ee52d9ce3c892"}, - {file = "pyinstaller-5.13.2-py3-none-win32.whl", hash = "sha256:421cd24f26144f19b66d3868b49ed673176765f92fa9f7914cd2158d25b6d17e"}, - {file = "pyinstaller-5.13.2-py3-none-win_amd64.whl", hash = "sha256:ddcc2b36052a70052479a9e5da1af067b4496f43686ca3cdda99f8367d0627e4"}, - {file = "pyinstaller-5.13.2-py3-none-win_arm64.whl", hash = "sha256:27cd64e7cc6b74c5b1066cbf47d75f940b71356166031deb9778a2579bb874c6"}, - {file = "pyinstaller-5.13.2.tar.gz", hash = "sha256:c8e5d3489c3a7cc5f8401c2d1f48a70e588f9967e391c3b06ddac1f685f8d5d2"}, + {file = "pyinstaller-6.2.0-py3-none-macosx_10_13_universal2.whl", hash = "sha256:a1adbd3cf25dc90926d783eae0f444d65cdfecc7bcdf6da522c3ae3ff47b4c25"}, + {file = "pyinstaller-6.2.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:29d164394f1e949072f78a64c1e040f1c47b7f4aff08514c7666a031c8b44996"}, + {file = "pyinstaller-6.2.0-py3-none-manylinux2014_i686.whl", hash = "sha256:ba602a38d7403de89c38b8956b221ce6de0280730d269bab522492fcad82ee33"}, + {file = "pyinstaller-6.2.0-py3-none-manylinux2014_ppc64le.whl", hash = "sha256:ebac06d99b80d2035594c3cc2fb5f2612d86289edd0510dbcbeb20a873f51d5a"}, + {file = "pyinstaller-6.2.0-py3-none-manylinux2014_s390x.whl", hash = "sha256:fcfabc0ff1d38a4262c051dea3fdc1f7f106405c1f1b491b4c79cd28df19cab6"}, + {file = "pyinstaller-6.2.0-py3-none-manylinux2014_x86_64.whl", hash = "sha256:104430686149b2f1c135b2c17aa2967c85d54ef77dc92feb4e179ec846c0c467"}, + {file = "pyinstaller-6.2.0-py3-none-musllinux_1_1_aarch64.whl", hash = "sha256:e87fd60292b53bb9965cb5a84122875469a2bd475fd0d0db0052a3f1be351f75"}, + {file = "pyinstaller-6.2.0-py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:8ec9d6c98972bb922cedb16a6638257aa66e5deadd79e2953f3464696237c413"}, + {file = "pyinstaller-6.2.0-py3-none-win32.whl", hash = "sha256:e5561e9a9b946d835c8dbc11ae4c16cc21e62bc77d10cc043406dc2992dfb4c6"}, + {file = "pyinstaller-6.2.0-py3-none-win_amd64.whl", hash = "sha256:3b586196277c4c54b69880650984c39c28bb6258c2b4b64200032e6ac69d53a0"}, + {file = "pyinstaller-6.2.0-py3-none-win_arm64.whl", hash = "sha256:d0c87b605bf13c3a04dfaa1d2fa7cd36765b8137000eeadccba865e1d6a19bf0"}, + {file = "pyinstaller-6.2.0.tar.gz", hash = "sha256:1ce77043929bf525be38289d78feecde0fcf15506215eda6500176a8715c5047"}, ] [package.dependencies] altgraph = "*" macholib = {version = ">=1.8", markers = "sys_platform == \"darwin\""} +packaging = ">=22.0" pefile = {version = ">=2022.5.30", markers = "sys_platform == \"win32\""} pyinstaller-hooks-contrib = ">=2021.4" pywin32-ctypes = {version = ">=0.2.1", markers = "sys_platform == \"win32\""} setuptools = ">=42.0.0" [package.extras] -encryption = ["tinyaes (>=1.0.0)"] +completion = ["argcomplete"] hook-testing = ["execnet (>=1.5.0)", "psutil", "pytest (>=2.7.3)"] [[package]] name = "pyinstaller-hooks-contrib" -version = "2023.7" +version = "2023.10" description = "Community maintained hooks for PyInstaller" optional = false python-versions = ">=3.7" files = [ - {file = "pyinstaller-hooks-contrib-2023.7.tar.gz", hash = "sha256:0c436a4c3506020e34116a8a7ddfd854c1ad6ddca9a8cd84500bd6e69c9e68f9"}, - {file = "pyinstaller_hooks_contrib-2023.7-py2.py3-none-any.whl", hash = "sha256:3c10df14c0f71ab388dfbf1625375b087e7330d9444cbfd2b310ba027fa0cff0"}, + {file = "pyinstaller-hooks-contrib-2023.10.tar.gz", hash = "sha256:4b4a998036abb713774cb26534ca06b7e6e09e4c628196017a10deb11a48747f"}, + {file = "pyinstaller_hooks_contrib-2023.10-py2.py3-none-any.whl", hash = "sha256:6dc1786a8f452941245d5bb85893e2a33632ebdcbc4c23eea41f2ee08281b0c0"}, ] [[package]] @@ -2048,19 +2049,19 @@ files = [ [[package]] name = "setuptools" -version = "68.1.2" +version = "69.0.2" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "setuptools-68.1.2-py3-none-any.whl", hash = "sha256:3d8083eed2d13afc9426f227b24fd1659489ec107c0e86cec2ffdde5c92e790b"}, - {file = "setuptools-68.1.2.tar.gz", hash = "sha256:3d4dfa6d95f1b101d695a6160a7626e15583af71a5f52176efa5d39a054d475d"}, + {file = "setuptools-69.0.2-py3-none-any.whl", hash = "sha256:1e8fdff6797d3865f37397be788a4e3cba233608e9b509382a2777d25ebde7f2"}, + {file = "setuptools-69.0.2.tar.gz", hash = "sha256:735896e78a4742605974de002ac60562d286fa8051a7e2299445e8e8fbb01aa6"}, ] [package.extras] -docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5,<=7.1.2)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] -testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] [[package]] name = "shellingham" @@ -2431,4 +2432,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = "~3.11" -content-hash = "b3ef9f8c5445b3e481d666a4a3b6a73d44fa1159646cf64f480a19aa1999d0ee" +content-hash = "eb3e0209e98c6df8760ef8dae1ccbd175af6a28e09ea5efc5e84b566b6c5b8d0" diff --git a/pyproject.toml b/pyproject.toml index 88926aa0a..c69cf96af 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,14 +51,14 @@ pyyaml = "^6.0" pyworld = "^0.3.0" requests = "^2.28.1" jinja2 = "^3.1.2" -pyopenjtalk = {git = "https://github.com/VOICEVOX/pyopenjtalk", rev = "b35fc89fe42948a28e33aed886ea145a51113f88"} +pyopenjtalk = { git = "https://github.com/VOICEVOX/pyopenjtalk", rev = "b35fc89fe42948a28e33aed886ea145a51113f88" } semver = "^3.0.0" platformdirs = "^3.10.0" soxr = "^0.3.6" [tool.poetry.group.dev.dependencies] cython = "^0.29.34,>=0.29.33" # NOTE: for Python 3.11 -pyinstaller = "^5.13" +pyinstaller = "^6.2.0" pre-commit = "^2.16.0" atomicwrites = "^1.4.0" colorama = "^0.4.4" @@ -74,7 +74,7 @@ mypy = "^1.6.0" pytest = "^6.2.5" coveralls = "^3.2.0" poetry = "^1.3.1" -httpx = "^0.25.0" # NOTE: required by fastapi.testclient.TestClient +httpx = "^0.25.0" # NOTE: required by fastapi.testclient.TestClient [tool.poetry.group.license.dependencies] pip-licenses = "^4.2.0" diff --git a/requirements-dev.txt b/requirements-dev.txt index 51806c4da..a42435195 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,5 +1,5 @@ aiofiles==0.7.0 ; python_version >= "3.11" and python_version < "3.12" -altgraph==0.17.3 ; python_version >= "3.11" and python_version < "3.12" +altgraph==0.17.4 ; python_version >= "3.11" and python_version < "3.12" anyio==3.7.1 ; python_version >= "3.11" and python_version < "3.12" asgiref==3.7.2 ; python_version >= "3.11" and python_version < "3.12" atomicwrites==1.4.1 ; python_version >= "3.11" and python_version < "3.12" @@ -30,7 +30,7 @@ jeepney==0.8.0 ; python_version >= "3.11" and python_version < "3.12" and sys_pl jinja2==3.1.2 ; python_version >= "3.11" and python_version < "3.12" jsonschema==4.17.3 ; python_version >= "3.11" and python_version < "3.12" keyring==24.2.0 ; python_version >= "3.11" and python_version < "3.12" -macholib==1.16.2 ; python_version >= "3.11" and python_version < "3.12" and sys_platform == "darwin" +macholib==1.16.3 ; python_version >= "3.11" and python_version < "3.12" and sys_platform == "darwin" markupsafe==2.1.3 ; python_version >= "3.11" and python_version < "3.12" more-itertools==10.1.0 ; python_version >= "3.11" and python_version < "3.12" msgpack==1.0.5 ; python_version >= "3.11" and python_version < "3.12" @@ -48,8 +48,8 @@ pre-commit==2.21.0 ; python_version >= "3.11" and python_version < "3.12" ptyprocess==0.7.0 ; python_version >= "3.11" and python_version < "3.12" pycparser==2.21 ; python_version >= "3.11" and python_version < "3.12" pydantic==1.10.12 ; python_version >= "3.11" and python_version < "3.12" -pyinstaller-hooks-contrib==2023.7 ; python_version >= "3.11" and python_version < "3.12" -pyinstaller==5.13.2 ; python_version >= "3.11" and python_version < "3.12" +pyinstaller-hooks-contrib==2023.10 ; python_version >= "3.11" and python_version < "3.12" +pyinstaller==6.2.0 ; python_version >= "3.11" and python_version < "3.12" pyopenjtalk @ git+https://github.com/VOICEVOX/pyopenjtalk@b35fc89fe42948a28e33aed886ea145a51113f88 ; python_version >= "3.11" and python_version < "3.12" pyproject-hooks==1.0.0 ; python_version >= "3.11" and python_version < "3.12" pyrsistent==0.19.3 ; python_version >= "3.11" and python_version < "3.12" @@ -62,7 +62,7 @@ requests-toolbelt==1.0.0 ; python_version >= "3.11" and python_version < "3.12" requests==2.31.0 ; python_version >= "3.11" and python_version < "3.12" secretstorage==3.3.3 ; python_version >= "3.11" and python_version < "3.12" and sys_platform == "linux" semver==3.0.1 ; python_version >= "3.11" and python_version < "3.12" -setuptools==68.1.2 ; python_version >= "3.11" and python_version < "3.12" +setuptools==69.0.2 ; python_version >= "3.11" and python_version < "3.12" shellingham==1.5.3 ; python_version >= "3.11" and python_version < "3.12" six==1.16.0 ; python_version >= "3.11" and python_version < "3.12" sniffio==1.3.0 ; python_version >= "3.11" and python_version < "3.12" diff --git a/run.py b/run.py index 85ae2002a..950a5fc08 100644 --- a/run.py +++ b/run.py @@ -82,6 +82,7 @@ engine_root, get_latest_core_version, get_save_dir, + internal_root, ) @@ -209,7 +210,7 @@ async def block_origin_middleware(request: Request, call_next): metas_store = MetasStore(root_dir / "speaker_info") - setting_ui_template = Jinja2Templates(directory=engine_root() / "ui_template") + setting_ui_template = Jinja2Templates(directory=internal_root() / "ui_template") # キャッシュを有効化 # モジュール側でlru_cacheを指定するとキャッシュを制御しにくいため、HTTPサーバ側で指定する diff --git a/run.spec b/run.spec index 9f73c6b07..bfe052265 100644 --- a/run.spec +++ b/run.spec @@ -1,50 +1,21 @@ # -*- mode: python ; coding: utf-8 -*- # このファイルはPyInstallerによって自動生成されたもので、それをカスタマイズして使用しています。 from PyInstaller.utils.hooks import collect_data_files -import os datas = [ - ('engine_manifest_assets', 'engine_manifest_assets'), - ('speaker_info', 'speaker_info'), - ('engine_manifest.json', '.'), - ('default.csv', '.'), - ('licenses.json', '.'), - ('presets.yaml', '.'), - ('default_setting.yaml', '.'), - ('ui_template', 'ui_template'), + ("default.csv", "."), + ("presets.yaml", "."), + ("default_setting.yaml", "."), + ("ui_template", "ui_template"), ] -datas += collect_data_files('pyopenjtalk') - -core_model_dir_path = os.environ.get('CORE_MODEL_DIR_PATH') -if core_model_dir_path: - print('CORE_MODEL_DIR_PATH is found:', core_model_dir_path) - if not os.path.isdir(core_model_dir_path): - raise Exception("CORE_MODEL_DIR_PATH was found, but it is not directory!") - datas += [(core_model_dir_path, "model")] - -# コアとONNX Runtimeはバイナリであるが、`binaries`に加えると -# 依存関係のパスがPyInstallerに書き換えらるので、`datas`に加える -# 参考: https://github.com/VOICEVOX/voicevox_engine/pull/446#issuecomment-1210052318 -libcore_path = os.environ.get('LIBCORE_PATH') -if libcore_path: - print('LIBCORE_PATH is found:', libcore_path) - if not os.path.isfile(libcore_path): - raise Exception("LIBCORE_PATH was found, but it is not file!") - datas += [(libcore_path, ".")] - -libonnxruntime_path = os.environ.get('LIBONNXRUNTIME_PATH') -if libonnxruntime_path: - print('LIBONNXRUNTIME_PATH is found:', libonnxruntime_path) - if not os.path.isfile(libonnxruntime_path): - raise Exception("LIBCORE_PATH was found, but it is not file!") - datas += [(libonnxruntime_path, ".")] +datas += collect_data_files("pyopenjtalk") block_cipher = None a = Analysis( - ['run.py'], + ["run.py"], pathex=[], binaries=[], datas=datas, @@ -66,7 +37,7 @@ exe = EXE( a.scripts, [], exclude_binaries=True, - name='run', + name="run", debug=False, bootloader_ignore_signals=False, strip=False, @@ -77,6 +48,7 @@ exe = EXE( target_arch=None, codesign_identity=None, entitlements_file=None, + contents_directory="engine_internal", ) coll = COLLECT( @@ -87,5 +59,5 @@ coll = COLLECT( strip=False, upx=True, upx_exclude=[], - name='run', + name="run", ) diff --git a/voicevox_engine/setting/SettingLoader.py b/voicevox_engine/setting/SettingLoader.py index 3f1669e26..536fc58ce 100644 --- a/voicevox_engine/setting/SettingLoader.py +++ b/voicevox_engine/setting/SettingLoader.py @@ -2,10 +2,10 @@ import yaml -from ..utility import engine_root, get_save_dir +from ..utility import get_save_dir, internal_root from .Setting import Setting -DEFAULT_SETTING_PATH: Path = engine_root() / "default_setting.yaml" +DEFAULT_SETTING_PATH: Path = internal_root() / "default_setting.yaml" USER_SETTING_PATH: Path = get_save_dir() / "setting.yml" diff --git a/voicevox_engine/user_dict.py b/voicevox_engine/user_dict.py index 7db07a721..00f498289 100644 --- a/voicevox_engine/user_dict.py +++ b/voicevox_engine/user_dict.py @@ -13,9 +13,9 @@ from .model import UserDictWord, WordTypes from .part_of_speech_data import MAX_PRIORITY, MIN_PRIORITY, part_of_speech_data -from .utility import engine_root, get_save_dir, mutex_wrapper +from .utility import get_save_dir, internal_root, mutex_wrapper -root_dir = engine_root() +root_dir = internal_root() save_dir = get_save_dir() if not save_dir.is_dir(): diff --git a/voicevox_engine/utility/__init__.py b/voicevox_engine/utility/__init__.py index d40fea3e6..7ed74f118 100644 --- a/voicevox_engine/utility/__init__.py +++ b/voicevox_engine/utility/__init__.py @@ -5,7 +5,7 @@ ) from .core_version_utility import get_latest_core_version, parse_core_version from .mutex_utility import mutex_wrapper -from .path_utility import delete_file, engine_root, get_save_dir +from .path_utility import delete_file, engine_root, get_save_dir, internal_root __all__ = [ "ConnectBase64WavesException", @@ -13,8 +13,9 @@ "decode_base64_waves", "get_latest_core_version", "parse_core_version", + "mutex_wrapper", "delete_file", "engine_root", "get_save_dir", - "mutex_wrapper", + "internal_root", ] diff --git a/voicevox_engine/utility/path_utility.py b/voicevox_engine/utility/path_utility.py index 7c46ad40b..8a09146fc 100644 --- a/voicevox_engine/utility/path_utility.py +++ b/voicevox_engine/utility/path_utility.py @@ -2,18 +2,43 @@ import sys import traceback from pathlib import Path +from typing import Literal from platformdirs import user_data_dir +def runtime_type() -> Literal["nuitka", "pyinstaller", "python"]: + """ + コンパイルに使用したライブラリ名を返す。 + コンパイルしていない場合は"python"を返す。 + """ + # nuitkaビルドをした際はグローバルに__compiled__が含まれる + if "__compiled__" in globals(): + return "nuitka" + + # pyinstallerでビルドをした際はsys.frozenが設定される + elif getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"): + return "pyinstaller" + + return "python" + + def engine_root() -> Path: - if is_development(): - root_dir = Path(__file__).parents[2] + runtime = runtime_type() + if runtime == "nuitka": + root_dir = Path(sys.argv[0]).parent + + elif runtime == "pyinstaller": + root_dir = Path(sys.executable).parent - # Nuitka/Pyinstallerでビルドされている場合 else: - root_dir = Path(sys.argv[0]).parent + root_dir = Path(__file__).parents[2] + + return root_dir.resolve(strict=True) + +def internal_root() -> Path: + root_dir = Path(__file__).parents[2] return root_dir.resolve(strict=True) @@ -22,15 +47,7 @@ def is_development() -> bool: 開発版かどうか判定する関数 Nuitka/Pyinstallerでコンパイルされていない場合は開発環境とする。 """ - # nuitkaビルドをした際はグローバルに__compiled__が含まれる - if "__compiled__" in globals(): - return False - - # pyinstallerでビルドをした際はsys.frozenが設定される - elif getattr(sys, "frozen", False): - return False - - return True + return runtime_type() == "python" def get_save_dir(): From a6c810c85fa5dbb9785a771ac7c8d236af15ac7d Mon Sep 17 00:00:00 2001 From: sabonerune <102559104+sabonerune@users.noreply.github.com> Date: Wed, 13 Dec 2023 17:25:44 +0900 Subject: [PATCH 2/7] =?UTF-8?q?FIX:=20runtime=5Ftype=E3=82=92=E3=83=97?= =?UTF-8?q?=E3=83=A9=E3=82=A4=E3=83=99=E3=83=BC=E3=83=88=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- voicevox_engine/utility/path_utility.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/voicevox_engine/utility/path_utility.py b/voicevox_engine/utility/path_utility.py index 8a09146fc..d0ccbe006 100644 --- a/voicevox_engine/utility/path_utility.py +++ b/voicevox_engine/utility/path_utility.py @@ -7,7 +7,7 @@ from platformdirs import user_data_dir -def runtime_type() -> Literal["nuitka", "pyinstaller", "python"]: +def _runtime_type() -> Literal["nuitka", "pyinstaller", "python"]: """ コンパイルに使用したライブラリ名を返す。 コンパイルしていない場合は"python"を返す。 @@ -24,7 +24,7 @@ def runtime_type() -> Literal["nuitka", "pyinstaller", "python"]: def engine_root() -> Path: - runtime = runtime_type() + runtime = _runtime_type() if runtime == "nuitka": root_dir = Path(sys.argv[0]).parent @@ -47,7 +47,7 @@ def is_development() -> bool: 開発版かどうか判定する関数 Nuitka/Pyinstallerでコンパイルされていない場合は開発環境とする。 """ - return runtime_type() == "python" + return _runtime_type() == "python" def get_save_dir(): From 4104c2befaa6c82314fb16a6895cf94228fa8e01 Mon Sep 17 00:00:00 2001 From: sabonerune <102559104+sabonerune@users.noreply.github.com> Date: Wed, 13 Dec 2023 18:36:31 +0900 Subject: [PATCH 3/7] =?UTF-8?q?FIX:=20docstring=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- voicevox_engine/utility/path_utility.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/voicevox_engine/utility/path_utility.py b/voicevox_engine/utility/path_utility.py index d0ccbe006..6c5c36ca2 100644 --- a/voicevox_engine/utility/path_utility.py +++ b/voicevox_engine/utility/path_utility.py @@ -24,6 +24,10 @@ def _runtime_type() -> Literal["nuitka", "pyinstaller", "python"]: def engine_root() -> Path: + """ + 開発環境ではリポジトリのルートディレクトリを返す。 + コンパイル後は実行ファイルがあるディレクトリを返す。 + """ runtime = _runtime_type() if runtime == "nuitka": root_dir = Path(sys.argv[0]).parent @@ -38,6 +42,10 @@ def engine_root() -> Path: def internal_root() -> Path: + """ + コンパイル時に収集された実行ファイル内部用のルートディレクトリを返す。 + 開発環境ではリポジトリのルートディレクトリを返す。 + """ root_dir = Path(__file__).parents[2] return root_dir.resolve(strict=True) From 2bb7885f2e839ab6bc7719f07845f5d2feff8112 Mon Sep 17 00:00:00 2001 From: sabonerune <102559104+sabonerune@users.noreply.github.com> Date: Fri, 15 Dec 2023 19:22:46 +0900 Subject: [PATCH 4/7] =?UTF-8?q?FIX:=20spec=E3=83=95=E3=82=A1=E3=82=A4?= =?UTF-8?q?=E3=83=AB=E5=86=85=E3=81=A7=E3=83=95=E3=82=A1=E3=82=A4=E3=83=AB?= =?UTF-8?q?=E3=81=AE=E7=A7=BB=E5=8B=95=E5=87=A6=E7=90=86=E3=82=92=E3=81=99?= =?UTF-8?q?=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/build.yml | 81 ++++++++++++++----------------------- run.spec | 34 ++++++++++++++++ 2 files changed, 65 insertions(+), 50 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 453f59ea8..31e431aed 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -453,18 +453,21 @@ jobs: # Replace version & specify dynamic libraries $sed -i "s/__version__ = \"latest\"/__version__ = \"${{ needs.config.outputs.version_or_latest }}\"/" voicevox_engine/__init__.py + if [[ ${{ matrix.os }} == windows-* ]]; then + LIBCORE_PATH=download/core/voicevox_core.dll + LIBONNXRUNTIME_PATH=download/onnxruntime/lib/onnxruntime.dll + elif [[ ${{ matrix.os }} == macos-* ]]; then + LIBCORE_PATH=download/core/libvoicevox_core.dylib + LIBONNXRUNTIME_PATH=download/onnxruntime/lib/libonnxruntime.dylib + else + LIBCORE_PATH=download/core/libvoicevox_core.so + LIBONNXRUNTIME_PATH=download/onnxruntime/lib/libonnxruntime.so + fi - pyinstaller --noconfirm run.spec - - - name: Gather files to engine root - run: | - set -eux - - mv licenses.json dist/run/ - mv speaker_info dist/run/ - mv engine_manifest.json dist/run/ - mv engine_manifest_assets dist/run/ - mv download/core/model dist/run/ + pyinstaller --noconfirm run.spec -- \ + --libcore_path="$LIBCORE_PATH" \ + --libonnxruntime_path="$LIBONNXRUNTIME_PATH" \ + --core_model_dir_path="download/core/model" - name: Gather DLL dependencies to dist/run/ (Windows) if: startsWith(matrix.os, 'windows-') @@ -472,10 +475,6 @@ jobs: set -eux # Move DLL dependencies (cache already saved) - mv download/core/voicevox_core.dll dist/run/ - rm -rf download/core - - mv download/onnxruntime/lib/onnxruntime.dll dist/run/ if [ -f "download/onnxruntime/lib/onnxruntime_providers_cuda.dll" ]; then # ONNX Runtime providers (PyInstaller does not copy dynamic loaded libraries) @@ -510,50 +509,32 @@ jobs: rm -rf download/directml fi - - name: Gather DLL dependencies to dist/run/ (Linux) - if: startsWith(matrix.os, 'ubuntu-') + - name: Gather DLL dependencies to dist/run/ (Linux CUDA) + if: startsWith(matrix.os, 'ubuntu-') && endsWith(matrix.target, 'nvidia') run: | set -eux # Move DLL dependencies (cache already saved) - cp download/core/libvoicevox_core.so dist/run/ - rm -rf download/core - - cp download/onnxruntime/lib/libonnxruntime.so dist/run/ - if [[ ${{ matrix.target }} == *-nvidia ]]; then - # ONNX Runtime providers (PyInstaller does not copy dynamic loaded libraries) - patchelf --set-rpath '$ORIGIN' "$(pwd)/download/onnxruntime/lib"/libonnxruntime_providers_*.so - mv download/onnxruntime/lib/libonnxruntime_*.so dist/run/ + # ONNX Runtime providers (PyInstaller does not copy dynamic loaded libraries) + patchelf --set-rpath '$ORIGIN' "$(pwd)/download/onnxruntime/lib"/libonnxruntime_providers_*.so + mv download/onnxruntime/lib/libonnxruntime_*.so dist/run/ - # CUDA - mv download/cuda/bin/libcublas.so.* dist/run/ - mv download/cuda/bin/libcublasLt.so.* dist/run/ - mv download/cuda/bin/libcudart.so.* dist/run/ - mv download/cuda/bin/libcufft.so.* dist/run/ - mv download/cuda/bin/libcurand.so.* dist/run/ + # CUDA + mv download/cuda/bin/libcublas.so.* dist/run/ + mv download/cuda/bin/libcublasLt.so.* dist/run/ + mv download/cuda/bin/libcudart.so.* dist/run/ + mv download/cuda/bin/libcufft.so.* dist/run/ + mv download/cuda/bin/libcurand.so.* dist/run/ - # cuDNN - mv download/cudnn/bin/libcudnn.so.* dist/run/ - mv download/cudnn/bin/libcudnn_*_infer.so.* dist/run/ - - # Remove source directories to reduce disk usage (already cached) - rm -rf download/onnxruntime - rm -rf download/cuda - rm -rf download/cudnn - fi - - - name: Gather DLL dependencies to dist/run/ (Mac) - if: startsWith(matrix.os, 'macos-') - run: | - set -eux - - # Move DLL dependencies (cache already saved) - cp download/core/libvoicevox_core.dylib dist/run/ - rm -rf download/core + # cuDNN + mv download/cudnn/bin/libcudnn.so.* dist/run/ + mv download/cudnn/bin/libcudnn_*_infer.so.* dist/run/ - cp download/onnxruntime/lib/libonnxruntime.dylib dist/run/ + # Remove source directories to reduce disk usage (already cached) rm -rf download/onnxruntime + rm -rf download/cuda + rm -rf download/cudnn - name: Set @rpath to @executable_path if: startsWith(matrix.os, 'macos-') diff --git a/run.spec b/run.spec index df2b910ca..345fa2d92 100644 --- a/run.spec +++ b/run.spec @@ -1,7 +1,29 @@ # -*- mode: python ; coding: utf-8 -*- # このファイルはPyInstallerによって自動生成されたもので、それをカスタマイズして使用しています。 +from argparse import ArgumentParser +from pathlib import Path +from shutil import copy2, copytree + from PyInstaller.utils.hooks import collect_data_files +parser = ArgumentParser() +parser.add_argument("--libcore_path", type=Path, required=True) +parser.add_argument("--libonnxruntime_path", type=Path, required=True) +parser.add_argument("--core_model_dir_path", type=Path, required=True) +options = parser.parse_args() + +libonnxruntime_path: Path = options.libonnxruntime_path +if not libonnxruntime_path.is_file(): + raise Exception(f"libonnxruntime_path: {libonnxruntime_path} is not file") + +libcore_path: Path = options.libcore_path +if not libcore_path.is_file(): + raise Exception(f"libcore_path: {libcore_path} is not file") + +core_model_dir_path: Path = options.core_model_dir_path +if not core_model_dir_path.is_dir(): + raise Exception(f"core_model_dir_path: {core_model_dir_path} is not dir") + datas = [ ("default.csv", "."), ("presets.yaml", "."), @@ -60,3 +82,15 @@ coll = COLLECT( upx_exclude=[], name="run", ) + +# 実行ファイル作成後の処理 +target_dir = Path(DISTPATH) / "run" + +copy2(libonnxruntime_path, target_dir) +copy2(libcore_path, target_dir) +copytree(core_model_dir_path, target_dir / "model") + +copytree("speaker_info", target_dir / "speaker_info") +copy2("engine_manifest.json", target_dir) +copytree("engine_manifest_assets", target_dir / "engine_manifest_assets") +copy2("licenses.json", target_dir) From 658c1bf8be7f64a3c8331ab3f7bd911cb05f5dad Mon Sep 17 00:00:00 2001 From: sabonerune <102559104+sabonerune@users.noreply.github.com> Date: Sat, 16 Dec 2023 22:42:28 +0900 Subject: [PATCH 5/7] =?UTF-8?q?FIX:=20=E3=83=87=E3=83=95=E3=82=A9=E3=83=AB?= =?UTF-8?q?=E3=83=88=E3=81=A7=E5=AD=98=E5=9C=A8=E3=81=97=E3=81=AA=E3=81=84?= =?UTF-8?q?=E3=83=95=E3=82=A1=E3=82=A4=E3=83=AB=E3=82=92=E7=84=A1=E8=A6=96?= =?UTF-8?q?=E3=81=99=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- run.spec | 34 +++++++++++++++++++++------------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/run.spec b/run.spec index 345fa2d92..010fe436f 100644 --- a/run.spec +++ b/run.spec @@ -7,21 +7,21 @@ from shutil import copy2, copytree from PyInstaller.utils.hooks import collect_data_files parser = ArgumentParser() -parser.add_argument("--libcore_path", type=Path, required=True) -parser.add_argument("--libonnxruntime_path", type=Path, required=True) -parser.add_argument("--core_model_dir_path", type=Path, required=True) +parser.add_argument("--libcore_path", type=Path) +parser.add_argument("--libonnxruntime_path", type=Path) +parser.add_argument("--core_model_dir_path", type=Path) options = parser.parse_args() -libonnxruntime_path: Path = options.libonnxruntime_path -if not libonnxruntime_path.is_file(): +libonnxruntime_path: Path | None = options.libonnxruntime_path +if libonnxruntime_path is not None and not libonnxruntime_path.is_file(): raise Exception(f"libonnxruntime_path: {libonnxruntime_path} is not file") -libcore_path: Path = options.libcore_path -if not libcore_path.is_file(): +libcore_path: Path | None = options.libcore_path +if libcore_path is not None and not libcore_path.is_file(): raise Exception(f"libcore_path: {libcore_path} is not file") -core_model_dir_path: Path = options.core_model_dir_path -if not core_model_dir_path.is_dir(): +core_model_dir_path: Path | None = options.core_model_dir_path +if core_model_dir_path is not None and not core_model_dir_path.is_dir(): raise Exception(f"core_model_dir_path: {core_model_dir_path} is not dir") datas = [ @@ -86,11 +86,19 @@ coll = COLLECT( # 実行ファイル作成後の処理 target_dir = Path(DISTPATH) / "run" -copy2(libonnxruntime_path, target_dir) -copy2(libcore_path, target_dir) -copytree(core_model_dir_path, target_dir / "model") +if libonnxruntime_path is not None: + copy2(libonnxruntime_path, target_dir) + +if libcore_path is not None: + copy2(libcore_path, target_dir) + +if core_model_dir_path is not None: + copytree(core_model_dir_path, target_dir / "model") + +license_file_path = Path("licenses.json") +if license_file_path.is_file(): + copy2("licenses.json", target_dir) copytree("speaker_info", target_dir / "speaker_info") copy2("engine_manifest.json", target_dir) copytree("engine_manifest_assets", target_dir / "engine_manifest_assets") -copy2("licenses.json", target_dir) From e49b4ad5588d0198a20c612bdb86435d927b3cf3 Mon Sep 17 00:00:00 2001 From: Hiroshiba Date: Mon, 18 Dec 2023 04:32:55 +0900 Subject: [PATCH 6/7] Update .github/workflows/build.yml --- .github/workflows/build.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 31e431aed..ba6a453db 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -465,9 +465,9 @@ jobs: fi pyinstaller --noconfirm run.spec -- \ - --libcore_path="$LIBCORE_PATH" \ - --libonnxruntime_path="$LIBONNXRUNTIME_PATH" \ - --core_model_dir_path="download/core/model" + --libcore_path="$LIBCORE_PATH" \ + --libonnxruntime_path="$LIBONNXRUNTIME_PATH" \ + --core_model_dir_path="download/core/model" - name: Gather DLL dependencies to dist/run/ (Windows) if: startsWith(matrix.os, 'windows-') From 9acdfa3188367c8af241776f0fe8d247d24f9012 Mon Sep 17 00:00:00 2001 From: Hiroshiba Date: Mon, 18 Dec 2023 04:33:05 +0900 Subject: [PATCH 7/7] Update run.spec --- run.spec | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/run.spec b/run.spec index 010fe436f..65642c61d 100644 --- a/run.spec +++ b/run.spec @@ -84,17 +84,19 @@ coll = COLLECT( ) # 実行ファイル作成後の処理 + +# 実行ファイルと同じrootディレクトリ target_dir = Path(DISTPATH) / "run" +# 動的ライブラリをコピー if libonnxruntime_path is not None: copy2(libonnxruntime_path, target_dir) - if libcore_path is not None: copy2(libcore_path, target_dir) - if core_model_dir_path is not None: copytree(core_model_dir_path, target_dir / "model") +# 互換性維持のために必要なファイルをコピー license_file_path = Path("licenses.json") if license_file_path.is_file(): copy2("licenses.json", target_dir)