From f9288907c54e9d8062dca505d34fe9c1c01a1cea Mon Sep 17 00:00:00 2001 From: Junduo Dong Date: Tue, 15 Oct 2024 00:10:41 +0800 Subject: [PATCH 01/15] feat(ci): support building python on windows Signed-off-by: Junduo Dong --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fa2d3e92d3..17859fdf6a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -215,10 +215,11 @@ jobs: name: Python CI # Fix python 3.6 install issue, see # https://github.com/rwth-i6/returnn/commit/38ecab17d781c4b74db6a174c8097187380b4ddc - runs-on: ubuntu-20.04 + runs-on: ${{ matrix.os }} strategy: matrix: python-version: [3.7, 3.12] + os: [ubuntu-20.04, windows-2022] steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} From 2647e8eb958bab43e1eae02a45669b3dae05b9b8 Mon Sep 17 00:00:00 2001 From: Junduo Dong Date: Sat, 2 Nov 2024 20:27:36 +0800 Subject: [PATCH 02/15] Empty-Commit From 939a37ebaca4625b1a62c071bd8c210d1714b92d Mon Sep 17 00:00:00 2001 From: Junduo Dong Date: Sat, 2 Nov 2024 20:27:38 +0800 Subject: [PATCH 03/15] Empty-Commit From 3ee566631426644502ff5440c7fa6a82a953258b Mon Sep 17 00:00:00 2001 From: Junduo Dong Date: Sat, 2 Nov 2024 20:40:56 +0800 Subject: [PATCH 04/15] fix(ci): use (git) bash to run python steps on Windows Signed-off-by: Junduo Dong --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 17859fdf6a..dd8c7183c1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -229,6 +229,7 @@ jobs: - name: Install bazel run: ./ci/run_ci.sh install_bazel - name: Run Python CI + shell: bash run: ./ci/run_ci.sh python go: From 8085249dba27ee448e49bfe25a26563efeace92a Mon Sep 17 00:00:00 2001 From: Junduo Dong Date: Sat, 2 Nov 2024 21:02:02 +0800 Subject: [PATCH 05/15] fix(ci): add conda/bin to Windows system path Signed-off-by: Junduo Dong --- .github/workflows/ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dd8c7183c1..dbb09d1dda 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -226,6 +226,10 @@ jobs: uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} + # https://github.com/actions/runner-images/blob/main/images/windows/Windows2022-Readme.md + - name: Add conda bin to path + run: echo "C:\Miniconda\bin" >> "$GITHUB_PATH" + if: matrix.os == 'windows-2022' - name: Install bazel run: ./ci/run_ci.sh install_bazel - name: Run Python CI From 5e9a7539ad8732d35a4ba249c0e535e9c490c711 Mon Sep 17 00:00:00 2001 From: Junduo Dong Date: Sat, 2 Nov 2024 21:18:22 +0800 Subject: [PATCH 06/15] remove conda/bin from python ci windows runner Signed-off-by: Junduo Dong --- .github/workflows/ci.yml | 4 ---- ci/deploy.sh | 4 ++-- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dbb09d1dda..dd8c7183c1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -226,10 +226,6 @@ jobs: uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - # https://github.com/actions/runner-images/blob/main/images/windows/Windows2022-Readme.md - - name: Add conda bin to path - run: echo "C:\Miniconda\bin" >> "$GITHUB_PATH" - if: matrix.os == 'windows-2022' - name: Install bazel run: ./ci/run_ci.sh install_bazel - name: Run Python CI diff --git a/ci/deploy.sh b/ci/deploy.sh index 7021e581b4..9bb53caa7a 100755 --- a/ci/deploy.sh +++ b/ci/deploy.sh @@ -47,9 +47,8 @@ VERSIONS=("3.7" "3.11" "3.12") -source $(conda info --base)/etc/profile.d/conda.sh - create_py_envs() { + source $(conda info --base)/etc/profile.d/conda.sh for version in "${VERSIONS[@]}"; do conda create -y --name "py$version" python="$version" done @@ -94,6 +93,7 @@ deploy_jars() { } deploy_python() { + source $(conda info --base)/etc/profile.d/conda.sh if command -v pyenv; then pyenv local system fi From eb6af9ac90ccc52865fb7003bf25de6ed68ea5eb Mon Sep 17 00:00:00 2001 From: Junduo Dong Date: Sat, 2 Nov 2024 21:24:03 +0800 Subject: [PATCH 07/15] fix(cpp): add missing header Signed-off-by: Junduo Dong --- cpp/fury/thirdparty/MurmurHash3.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/fury/thirdparty/MurmurHash3.cc b/cpp/fury/thirdparty/MurmurHash3.cc index c9d144692d..7267187cc1 100644 --- a/cpp/fury/thirdparty/MurmurHash3.cc +++ b/cpp/fury/thirdparty/MurmurHash3.cc @@ -19,6 +19,7 @@ #define FORCE_INLINE __forceinline #include +#include #define ROTL32(x, y) _rotl(x, y) #define ROTL64(x, y) _rotl64(x, y) From 06372146cbbe78525e8fc0a6714e42f7777ef8f5 Mon Sep 17 00:00:00 2001 From: Junduo Dong Date: Sat, 2 Nov 2024 21:30:50 +0800 Subject: [PATCH 08/15] fix(cpp): rename FuryLogLevel::ERROR to FuryLogLevel::ERR I don't know why this enum make build failed... but successed if renaming it to FuryLogLevel::ERR Signed-off-by: Junduo Dong --- cpp/fury/util/logging.cc | 4 ++-- cpp/fury/util/logging.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cpp/fury/util/logging.cc b/cpp/fury/util/logging.cc index 8c2a09c3bf..ebe8c34bc7 100644 --- a/cpp/fury/util/logging.cc +++ b/cpp/fury/util/logging.cc @@ -52,7 +52,7 @@ std::string GetCallTrace() { std::unordered_map log_level_to_str = { {FuryLogLevel::DEBUG, "DEBUG"}, {FuryLogLevel::INFO, "INFO"}, - {FuryLogLevel::WARNING, "WARNING"}, {FuryLogLevel::ERROR, "ERROR"}, + {FuryLogLevel::WARNING, "WARNING"}, {FuryLogLevel::ERR, "ERROR"}, {FuryLogLevel::FATAL, "FATAL"}, }; @@ -77,7 +77,7 @@ FuryLogLevel FuryLog::GetLogLevel() { } else if (data == "warning") { severity_threshold = FuryLogLevel::WARNING; } else if (data == "error") { - severity_threshold = FuryLogLevel::ERROR; + severity_threshold = FuryLogLevel::ERR; } else if (data == "fatal") { severity_threshold = FuryLogLevel::FATAL; } else { diff --git a/cpp/fury/util/logging.h b/cpp/fury/util/logging.h index 1770e3feb5..addb21027a 100644 --- a/cpp/fury/util/logging.h +++ b/cpp/fury/util/logging.h @@ -35,7 +35,7 @@ enum class FuryLogLevel { DEBUG = -1, INFO = 0, WARNING = 1, - ERROR = 2, + ERR = 2, FATAL = 3 }; From 8efdc8362939b3400715e92106f4d515a5c2f5e1 Mon Sep 17 00:00:00 2001 From: Junduo Dong Date: Sat, 2 Nov 2024 21:39:29 +0800 Subject: [PATCH 09/15] fix(python): rename pyx built dynamic lib name to '*.pyd' * pyfury/_serialization.pyd * pyfury/_util.pyd * pyfury/format/_format.pyd * pyfury/lib/mmh3/mmh3.pyd Signed-off-by: Junduo Dong --- BUILD | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/BUILD b/BUILD index 5fa5b66f0e..de2a0bb28b 100644 --- a/BUILD +++ b/BUILD @@ -102,10 +102,19 @@ genrule( set -e set -x WORK_DIR=$$(pwd) - cp -f $(location python/pyfury/_util.so) "$$WORK_DIR/python/pyfury" - cp -f $(location python/pyfury/lib/mmh3/mmh3.so) "$$WORK_DIR/python/pyfury/lib/mmh3" - cp -f $(location python/pyfury/format/_format.so) "$$WORK_DIR/python/pyfury/format" - cp -f $(location python/pyfury/_serialization.so) "$$WORK_DIR/python/pyfury" + u_name=`uname -s` + if [ "$${u_name: 0: 4}" == "MING" ] + then + cp -f $(location python/pyfury/_util.so) "$$WORK_DIR/python/pyfury/_util.pyd" + cp -f $(location python/pyfury/lib/mmh3/mmh3.so) "$$WORK_DIR/python/pyfury/lib/mmh3/mmh3.pyd" + cp -f $(location python/pyfury/format/_format.so) "$$WORK_DIR/python/pyfury/format/_format.pyd" + cp -f $(location python/pyfury/_serialization.so) "$$WORK_DIR/python/pyfury/_serialization.pyd" + else + cp -f $(location python/pyfury/_util.so) "$$WORK_DIR/python/pyfury" + cp -f $(location python/pyfury/lib/mmh3/mmh3.so) "$$WORK_DIR/python/pyfury/lib/mmh3" + cp -f $(location python/pyfury/format/_format.so) "$$WORK_DIR/python/pyfury/format" + cp -f $(location python/pyfury/_serialization.so) "$$WORK_DIR/python/pyfury" + fi echo $$(date) > $@ """, local = 1, From 5ec5b12bfa92bb073ca203848ab2a8e8e28dc908 Mon Sep 17 00:00:00 2001 From: Junduo Dong Date: Sat, 2 Nov 2024 21:58:10 +0800 Subject: [PATCH 10/15] fix(python): rename .so to .pyd on windows Signed-off-by: Junduo Dong --- BUILD | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/BUILD b/BUILD index de2a0bb28b..4012d3f41b 100644 --- a/BUILD +++ b/BUILD @@ -103,7 +103,8 @@ genrule( set -x WORK_DIR=$$(pwd) u_name=`uname -s` - if [ "$${u_name: 0: 4}" == "MING" ] + os_name=$${u_name: 0: 4} + if [ "$${u_name: 0: 4}" == "MING" ] || [ "$${u_name: 0: 4}" == "MSYS" ] then cp -f $(location python/pyfury/_util.so) "$$WORK_DIR/python/pyfury/_util.pyd" cp -f $(location python/pyfury/lib/mmh3/mmh3.so) "$$WORK_DIR/python/pyfury/lib/mmh3/mmh3.pyd" From 4d1508927c29be233413ec6bf34186748f68714c Mon Sep 17 00:00:00 2001 From: Junduo Dong Date: Sat, 2 Nov 2024 22:10:47 +0800 Subject: [PATCH 11/15] fix(misc): add *.pyd to .gitignore Signed-off-by: Junduo Dong --- .gitignore | 1 + BUILD | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 53cfc24de2..f4faab088a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ **/*.cpp **/*.so **/*.dylib +**/*.pyd bazel-* .whl python/.cache diff --git a/BUILD b/BUILD index 4012d3f41b..505b944e2a 100644 --- a/BUILD +++ b/BUILD @@ -104,7 +104,7 @@ genrule( WORK_DIR=$$(pwd) u_name=`uname -s` os_name=$${u_name: 0: 4} - if [ "$${u_name: 0: 4}" == "MING" ] || [ "$${u_name: 0: 4}" == "MSYS" ] + if [ "${os_name}" == "MING" ] || [ "${os_name}" == "MSYS" ] then cp -f $(location python/pyfury/_util.so) "$$WORK_DIR/python/pyfury/_util.pyd" cp -f $(location python/pyfury/lib/mmh3/mmh3.so) "$$WORK_DIR/python/pyfury/lib/mmh3/mmh3.pyd" From fa649b976534c75250af060063275c522269cc38 Mon Sep 17 00:00:00 2001 From: Junduo Dong Date: Sat, 2 Nov 2024 22:15:46 +0800 Subject: [PATCH 12/15] fix wrong shell syntax Signed-off-by: Junduo Dong --- BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/BUILD b/BUILD index 505b944e2a..7ea9cfa090 100644 --- a/BUILD +++ b/BUILD @@ -104,7 +104,7 @@ genrule( WORK_DIR=$$(pwd) u_name=`uname -s` os_name=$${u_name: 0: 4} - if [ "${os_name}" == "MING" ] || [ "${os_name}" == "MSYS" ] + if [ "$(os_name)" == "MING" ] || [ "$(os_name)" == "MSYS" ] then cp -f $(location python/pyfury/_util.so) "$$WORK_DIR/python/pyfury/_util.pyd" cp -f $(location python/pyfury/lib/mmh3/mmh3.so) "$$WORK_DIR/python/pyfury/lib/mmh3/mmh3.pyd" From f14f0f469b027e9c549eb4d0e94868f942d832b4 Mon Sep 17 00:00:00 2001 From: Junduo Dong Date: Sat, 2 Nov 2024 22:19:48 +0800 Subject: [PATCH 13/15] fix wrong shell syntax Signed-off-by: Junduo Dong --- BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/BUILD b/BUILD index 7ea9cfa090..5f0d73717a 100644 --- a/BUILD +++ b/BUILD @@ -104,7 +104,7 @@ genrule( WORK_DIR=$$(pwd) u_name=`uname -s` os_name=$${u_name: 0: 4} - if [ "$(os_name)" == "MING" ] || [ "$(os_name)" == "MSYS" ] + if [ "$os_name" == "MING" ] || [ "$os_name" == "MSYS" ] then cp -f $(location python/pyfury/_util.so) "$$WORK_DIR/python/pyfury/_util.pyd" cp -f $(location python/pyfury/lib/mmh3/mmh3.so) "$$WORK_DIR/python/pyfury/lib/mmh3/mmh3.pyd" From 64b91bbe40e7d823d3ad1a1516a682a61389625a Mon Sep 17 00:00:00 2001 From: Junduo Dong Date: Sat, 2 Nov 2024 22:22:22 +0800 Subject: [PATCH 14/15] fix wrong shell syntax Signed-off-by: Junduo Dong --- BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/BUILD b/BUILD index 5f0d73717a..3bbfa80c14 100644 --- a/BUILD +++ b/BUILD @@ -104,7 +104,7 @@ genrule( WORK_DIR=$$(pwd) u_name=`uname -s` os_name=$${u_name: 0: 4} - if [ "$os_name" == "MING" ] || [ "$os_name" == "MSYS" ] + if [ "$$(os_name)" == "MING" ] || [ "$$(os_name)" == "MSYS" ] then cp -f $(location python/pyfury/_util.so) "$$WORK_DIR/python/pyfury/_util.pyd" cp -f $(location python/pyfury/lib/mmh3/mmh3.so) "$$WORK_DIR/python/pyfury/lib/mmh3/mmh3.pyd" From 6582f9c8e41708f45efda1079581627ec4f1d090 Mon Sep 17 00:00:00 2001 From: Junduo Dong Date: Sat, 2 Nov 2024 22:31:48 +0800 Subject: [PATCH 15/15] revert Signed-off-by: Junduo Dong --- BUILD | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/BUILD b/BUILD index 3bbfa80c14..d13bfe44fc 100644 --- a/BUILD +++ b/BUILD @@ -103,8 +103,7 @@ genrule( set -x WORK_DIR=$$(pwd) u_name=`uname -s` - os_name=$${u_name: 0: 4} - if [ "$$(os_name)" == "MING" ] || [ "$$(os_name)" == "MSYS" ] + if [ "$${u_name: 0: 4}" == "MING" ] || [ "$${u_name: 0: 4}" == "MSYS" ] then cp -f $(location python/pyfury/_util.so) "$$WORK_DIR/python/pyfury/_util.pyd" cp -f $(location python/pyfury/lib/mmh3/mmh3.so) "$$WORK_DIR/python/pyfury/lib/mmh3/mmh3.pyd"