From a83dc11b85ceb2e0d12a78086012b773e2da0052 Mon Sep 17 00:00:00 2001 From: Spotlight Date: Thu, 16 May 2024 11:27:35 -0400 Subject: [PATCH 1/7] Merge universal2 debloat script into prep This additionally skips building the wheel if it has already been built (e.g. for rapid development). --- scripts/macos-universal2/build.sh | 16 +++++++-- scripts/macos-universal2/debloat-qt.py | 30 ----------------- scripts/macos-universal2/prep-PyQt.sh | 46 ++++++++++++++++++++++---- 3 files changed, 53 insertions(+), 39 deletions(-) delete mode 100644 scripts/macos-universal2/debloat-qt.py diff --git a/scripts/macos-universal2/build.sh b/scripts/macos-universal2/build.sh index 2e35579..4f3cb6b 100755 --- a/scripts/macos-universal2/build.sh +++ b/scripts/macos-universal2/build.sh @@ -1,8 +1,21 @@ #!/bin/bash +set -e + +# Run everything relative to our script directory. +cd "$(dirname "$0")" + +# Activate a virtual environment so we don't pollute the system environment. python3 -m venv --upgrade-deps venv source venv/bin/activate -python3 -m pip install wheel PyQt6 +python3 -m pip install wheel + +# Before we install anything further, create our custom universal2 version of +# the PyQt6 frameworks (PyQt6_Qt6), and then install PyQt6 itself. bash prep-PyQt.sh +python3 -m pip install PyQt6 + +# Lastly, build our client. +# We'll override our default setup.py with one adjusted for macOS. cd ../../client python3 -m pip install -r requirements.txt py2app GitPython python3 _version.py @@ -11,7 +24,6 @@ py2applet --make-setup app.py icon.icns "icon.png" "taskbarDark.png" "taskbarLig # build universal binary sed -i '' -e "s/)/ name='NSO-RPC')/" setup.py python3 setup.py py2app -O2 --arch=universal2 -python3 ../scripts/macos-universal2/debloat-qt.py # arm64 requires codesigning to run codesign --deep --force --sign - dist/NSO-RPC.app/Contents/MacOS/* open dist diff --git a/scripts/macos-universal2/debloat-qt.py b/scripts/macos-universal2/debloat-qt.py deleted file mode 100644 index e320581..0000000 --- a/scripts/macos-universal2/debloat-qt.py +++ /dev/null @@ -1,30 +0,0 @@ - -import shutil -import os -import sys - -removeFiles = [ - f"./dist/NSO-RPC.app/Contents/Resources/lib/python{sys.version_info.major}.{sys.version_info.minor}/PyQt6/Qt6/lib/QtQuick3DRuntimeRender.framework", - f"./dist/NSO-RPC.app/Contents/Resources/lib/python{sys.version_info.major}.{sys.version_info.minor}/PyQt6/Qt6/lib/QtQuickParticles.framework", - f"./dist/NSO-RPC.app/Contents/Resources/lib/python{sys.version_info.major}.{sys.version_info.minor}/PyQt6/Qt6/lib/QtSpatialAudio.framework", - f"./dist/NSO-RPC.app/Contents/Resources/lib/python{sys.version_info.major}.{sys.version_info.minor}/PyQt6/Qt6/lib/QtShaderTools.framework", - f"./dist/NSO-RPC.app/Contents/Resources/lib/python{sys.version_info.major}.{sys.version_info.minor}/PyQt6/Qt6/lib/QtQuickTest.framework", - f"./dist/NSO-RPC.app/Contents/Resources/lib/python{sys.version_info.major}.{sys.version_info.minor}/PyQt6/Qt6/lib/QtBluetooth.framework", - f"./dist/NSO-RPC.app/Contents/Resources/lib/python{sys.version_info.major}.{sys.version_info.minor}/PyQt6/Qt6/lib/QtDesigner.framework", - f"./dist/NSO-RPC.app/Contents/Resources/lib/python{sys.version_info.major}.{sys.version_info.minor}/PyQt6/Qt6/lib/QtQuick.framework", - f"./dist/NSO-RPC.app/Contents/Resources/lib/python{sys.version_info.major}.{sys.version_info.minor}/PyQt6/Qt6/lib/QtHelp.framework", - f"./dist/NSO-RPC.app/Contents/Resources/lib/python{sys.version_info.major}.{sys.version_info.minor}/PyQt6/Qt6/lib/QtPdf.framework", - f"./dist/NSO-RPC.app/Contents/Resources/lib/python{sys.version_info.major}.{sys.version_info.minor}/PyQt6/Qt6/lib/QtQml.framework", - f"./dist/NSO-RPC.app/Contents/Resources/lib/python{sys.version_info.major}.{sys.version_info.minor}/PyQt6/Qt6/plugins/sqldrivers", - f"./dist/NSO-RPC.app/Contents/Resources/lib/python{sys.version_info.major}.{sys.version_info.minor}/PyQt6/Qt6/plugins/multimedia", - f"./dist/NSO-RPC.app/Contents/Resources/lib/python{sys.version_info.major}.{sys.version_info.minor}/PyQt6/Qt6/qml", -] - -for file in removeFiles: - try: - rmFile = shutil.rmtree(file) - print(f"Removing: {file}") - except NotADirectoryError: - os.remove(file) - except FileNotFoundError: - pass diff --git a/scripts/macos-universal2/prep-PyQt.sh b/scripts/macos-universal2/prep-PyQt.sh index fca384c..0168c83 100644 --- a/scripts/macos-universal2/prep-PyQt.sh +++ b/scripts/macos-universal2/prep-PyQt.sh @@ -1,18 +1,27 @@ #!/bin/bash +set -e + # Provided by @spotlightishere on Github # https://github.com/MCMi460/NSO-RPC/pull/86#issuecomment-1605700512 +# If we already have a universal2 wheel available, install and process no further. +# https://stackoverflow.com/a/6364244 +if compgen -G "./PyQt6_*universal2.whl"; then + python3 -m pip install PyQt6_*universal2.whl --force-reinstall + exit 0 +fi + # Download and unpack python3 -m pip download --only-binary=:all: --platform=macosx_13_0_x86_64 PyQt6_Qt6 python3 -m pip download --only-binary=:all: --platform=macosx_13_0_arm64 PyQt6_Qt6 -python3 -m wheel unpack PyQt6_*arm64.whl --dest arm64 -python3 -m wheel unpack PyQt6_*x86_64.whl --dest x86_64 +python3 -m wheel unpack PyQt6_Qt6*arm64.whl --dest arm64 +python3 -m wheel unpack PyQt6_Qt6*x86_64.whl --dest x86_64 # We'll use x86_64 as our basis. # As of writing, PyQt6_Qt6 specifies a minimum of 10.14 for x86_64, and 11.0 for arm64. # We'll reuse this tag; it should be updated if this ever changes in the future. -python3 -m wheel tags --platform-tag macosx_10_14_universal2 PyQt6_*x86_64.whl -python3 -m wheel unpack PyQt6_*universal2.whl --dest universal +python3 -m wheel tags --platform-tag macosx_10_14_universal2 PyQt6_Qt6*x86_64.whl +python3 -m wheel unpack PyQt6_Qt6*universal2.whl --dest universal # https://stackoverflow.com/a/46020381 merge_frameworks() { @@ -22,9 +31,32 @@ merge_frameworks() { } export -f merge_frameworks -# Iterate through all frameworks and libraries, and lipo together +# Iterate through all frameworks and libraries, and lipo together. find universal -perm +111 -type f -exec sh -c 'merge_frameworks "$1"' _ {} \; -python3 -m wheel pack universal/PyQt6_* -# Finally, install our universal python3.11 -m wheel. +# We can now debloat our created universal wheel by removing +# frameworks and libraries irrelevant to NSO-RPC. +debloat_paths=( + "Qt6/lib/QtQuick3DRuntimeRender.framework" + "Qt6/lib/QtQuickParticles.framework" + "Qt6/lib/QtSpatialAudio.framework" + "Qt6/lib/QtShaderTools.framework" + "Qt6/lib/QtQuickTest.framework" + "Qt6/lib/QtBluetooth.framework" + "Qt6/lib/QtDesigner.framework" + "Qt6/lib/QtQuick.framework" + "Qt6/lib/QtHelp.framework" + "Qt6/lib/QtPdf.framework" + "Qt6/lib/QtQml.framework" + "Qt6/plugins/sqldrivers" + "Qt6/plugins/multimedia" + "Qt6/qml" +) + +for debloat_path in "${debloat_paths[@]}"; do + rm -rf universal/PyQt6_Qt6*/PyQt6/$debloat_path +done + +# Finally, pack and install our universal2 wheel. +python3 -m wheel pack universal/PyQt6_* python3 -m pip install PyQt6_*universal2.whl --force-reinstall From 22a8ce8601ad51b632c2127b772da8503d84793e Mon Sep 17 00:00:00 2001 From: Spotlight Date: Thu, 16 May 2024 11:40:01 -0400 Subject: [PATCH 2/7] Enforce x86_64 for build environment This ensures that x86_64 is leveraged for our pure x86_64 builds even on arm64 runners. --- .github/workflows/actions.yml | 7 +++++-- scripts/build.sh | 8 ++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml index 71ed01c..f110ff8 100644 --- a/.github/workflows/actions.yml +++ b/.github/workflows/actions.yml @@ -64,12 +64,15 @@ jobs: - uses: actions/setup-python@v5 with: python-version: 3.11.4 + # We initially use `arch -x86_64` to ensure that we use an x86_64 version + # of Python, regardless of the host architecture. + # Subsequent invocations will all use the x86_64 `python3` binary within the venv. - name: Build run: > cd scripts && - ./build.sh && + arch -x86_64 /bin/bash ./build.sh && cd ../client/dist && - bash ../../scripts/tests/macos_test.sh && + arch -x86_64 /bin/bash ../../scripts/tests/macos_test.sh && rm output.log && ln -s /Applications "Applications (admin)" && hdiutil create -fs HFS+ -srcfolder . -volname NSO-RPC mac-installer.dmg && diff --git a/scripts/build.sh b/scripts/build.sh index 1da5d20..2c5ee1a 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -1,7 +1,15 @@ #!/bin/bash +set -e + +# Run everything relative to our script directory. +cd "$(dirname "$0")" + +# Activate a virtual environment so we don't pollute the system environment. python3 -m venv --upgrade-deps venv source venv/bin/activate cd ../client + +# As this is an x86_64 only version, ensure `py2app` outputs x86_64. python3 -m pip install -r requirements.txt pyqt6 py2app GitPython python3 _version.py rm setup.py From c96363fbb1dbb23d11ad8bdffbe4c3988b5a1f49 Mon Sep 17 00:00:00 2001 From: Spotlight Date: Thu, 16 May 2024 11:57:25 -0400 Subject: [PATCH 3/7] Ensure setup.py is removed when building --- scripts/build.sh | 6 +++++- scripts/macos-universal2/build.sh | 7 +++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/scripts/build.sh b/scripts/build.sh index 2c5ee1a..c8e3f5a 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -12,7 +12,11 @@ cd ../client # As this is an x86_64 only version, ensure `py2app` outputs x86_64. python3 -m pip install -r requirements.txt pyqt6 py2app GitPython python3 _version.py -rm setup.py + +# Recreate our setup.py with macOS-specific options. +if [ -f setup.py ]; then + rm setup.py +fi py2applet --make-setup app.py icon.icns "icon.png" "taskbarDark.png" "taskbarLight.png" "version.txt" sed -i '' -e "s/)/ name='NSO-RPC')/" setup.py python3 setup.py py2app --arch=x86_64 -O2 diff --git a/scripts/macos-universal2/build.sh b/scripts/macos-universal2/build.sh index 4f3cb6b..74079ea 100755 --- a/scripts/macos-universal2/build.sh +++ b/scripts/macos-universal2/build.sh @@ -15,11 +15,14 @@ bash prep-PyQt.sh python3 -m pip install PyQt6 # Lastly, build our client. -# We'll override our default setup.py with one adjusted for macOS. cd ../../client python3 -m pip install -r requirements.txt py2app GitPython python3 _version.py -rm setup.py + +# Recreate our setup.py with macOS-specific options. +if [ -f setup.py ]; then + rm setup.py +fi py2applet --make-setup app.py icon.icns "icon.png" "taskbarDark.png" "taskbarLight.png" "version.txt" # build universal binary sed -i '' -e "s/)/ name='NSO-RPC')/" setup.py From b2b1f370716299acb91bf1c25fefb182f42c8d5e Mon Sep 17 00:00:00 2001 From: Spotlight Date: Thu, 16 May 2024 12:04:32 -0400 Subject: [PATCH 4/7] Resolve test invocation issues --- scripts/tests/macos_test.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) mode change 100644 => 100755 scripts/tests/macos_test.sh diff --git a/scripts/tests/macos_test.sh b/scripts/tests/macos_test.sh old mode 100644 new mode 100755 index c1a6d42..08b88cb --- a/scripts/tests/macos_test.sh +++ b/scripts/tests/macos_test.sh @@ -1,11 +1,14 @@ #!/bin/bash +set -e +cd "$(dirname "$0")" + RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[0;33m' NC='\033[0m' # Start the application in the background -./NSO-RPC.app/Contents/MacOS/NSO-RPC > output.log 2>&1 & +../../client/dist/NSO-RPC.app/Contents/MacOS/NSO-RPC > output.log 2>&1 & APP_PID=$! sleep 10 From 1f823b923b0a3c3b2a877412aace110f38b41666 Mon Sep 17 00:00:00 2001 From: Spotlight Date: Thu, 16 May 2024 12:09:21 -0400 Subject: [PATCH 5/7] Leverage open for test invocation --- .github/workflows/actions.yml | 35 ++++++++++++++++++++--------------- scripts/tests/macos_test.sh | 8 ++++++-- 2 files changed, 26 insertions(+), 17 deletions(-) diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml index f110ff8..aee82ee 100644 --- a/.github/workflows/actions.yml +++ b/.github/workflows/actions.yml @@ -58,6 +58,8 @@ jobs: files: scripts/linux.sh build-macos: name: Build NSO-RPC - MacOS + # We cannot use arm64 macOS runners until setup-python permits specifying an architecture. + # See also: https://github.com/actions/setup-python/issues/547 runs-on: macos-12 steps: - uses: actions/checkout@v4 @@ -69,11 +71,13 @@ jobs: # Subsequent invocations will all use the x86_64 `python3` binary within the venv. - name: Build run: > - cd scripts && - arch -x86_64 /bin/bash ./build.sh && - cd ../client/dist && - arch -x86_64 /bin/bash ../../scripts/tests/macos_test.sh && - rm output.log && + arch -x86_64 /bin/bash ./scripts/build.sh + - name: Test + run: > + arch -x86_64 /bin/bash ./scripts/tests/macos_test.sh + - name: Create Distributions + run: > + cd ./client/dist && ln -s /Applications "Applications (admin)" && hdiutil create -fs HFS+ -srcfolder . -volname NSO-RPC mac-installer.dmg && zip -yr mac-portable.zip NSO-RPC.app/ @@ -86,22 +90,23 @@ jobs: client/dist/mac-portable.zip build-universal2: name: Build NSO-RPC - Universal2 - runs-on: macos-12 + runs-on: macos-latest steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: 3.11.4 - name: Install Python 3.11.4 and build NSO-RPC run: > curl https://www.python.org/ftp/python/3.11.4/python-3.11.4-macos11.pkg -o python-3.11.4-macos11.pkg && - sudo installer -verbose -pkg python-3.11.4-macos11.pkg -target / && + sudo installer -verbose -pkg python-3.11.4-macos11.pkg -target / + - name: Build + run: > alias python3=python3.11 && - cd scripts/macos-universal2 && - bash ./build.sh && - cd ../../client/dist && - bash ../../scripts/tests/macos_test.sh && - rm output.log && + bash ./scripts/macos-universal2/build.sh + - name: Test + run: > + bash ./scripts/tests/macos_test.sh + - name: Create Distributions + run: > + cd client/dist && ln -s /Applications "Applications (admin)" && hdiutil create -fs HFS+ -srcfolder . -volname NSO-RPC mac-universal2-installer.dmg && zip -yr mac-universal2-portable.zip NSO-RPC.app/ diff --git a/scripts/tests/macos_test.sh b/scripts/tests/macos_test.sh index 08b88cb..bafc949 100755 --- a/scripts/tests/macos_test.sh +++ b/scripts/tests/macos_test.sh @@ -8,8 +8,8 @@ YELLOW='\033[0;33m' NC='\033[0m' # Start the application in the background -../../client/dist/NSO-RPC.app/Contents/MacOS/NSO-RPC > output.log 2>&1 & -APP_PID=$! +open --stdout output.log --stderr output.log ../../client/dist/NSO-RPC.app +APP_PID=$(pgrep -n NSO-RPC) sleep 10 kill $APP_PID @@ -24,5 +24,9 @@ if echo "$output" | grep -q "Launch error"; then exit 1 else echo -e "${GREEN}Test Passed!${NC}" + # TODO(spotlightishere): Resolve issues with test invocation + if [ -f output.log ]; then + rm output.log + fi exit 0 fi From 6f1b13c7eb9225033f16eaa8b0093787663627f0 Mon Sep 17 00:00:00 2001 From: Spotlight Date: Fri, 17 May 2024 09:43:36 -0400 Subject: [PATCH 6/7] Enforce x86_64 Python for x86_64-only build --- .github/workflows/actions.yml | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml index aee82ee..a3e711f 100644 --- a/.github/workflows/actions.yml +++ b/.github/workflows/actions.yml @@ -20,7 +20,6 @@ jobs: - pyqt5 steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 with: python-version: 3.11.4 @@ -34,7 +33,7 @@ jobs: run: mv client/dist/NSO-RPC.exe client/dist/NSO-RPC-qt5.exe - name: Upload Build if: github.event_name != 'pull_request' - uses: softprops/action-gh-release@v2.0.4 + uses: softprops/action-gh-release@v2 with: files: | client/dist/NSO-RPC*.exe @@ -53,19 +52,18 @@ jobs: continue-on-error: false - name: Upload Build if: github.event_name != 'pull_request' - uses: softprops/action-gh-release@v2.0.4 + uses: softprops/action-gh-release@v2 with: files: scripts/linux.sh build-macos: name: Build NSO-RPC - MacOS - # We cannot use arm64 macOS runners until setup-python permits specifying an architecture. - # See also: https://github.com/actions/setup-python/issues/547 - runs-on: macos-12 + runs-on: macos-latest steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: python-version: 3.11.4 + architecture: x64 # We initially use `arch -x86_64` to ensure that we use an x86_64 version # of Python, regardless of the host architecture. # Subsequent invocations will all use the x86_64 `python3` binary within the venv. @@ -83,7 +81,7 @@ jobs: zip -yr mac-portable.zip NSO-RPC.app/ - name: Upload Build if: github.event_name != 'pull_request' - uses: softprops/action-gh-release@v2.0.4 + uses: softprops/action-gh-release@v2 with: files: | client/dist/mac-installer.dmg @@ -93,7 +91,7 @@ jobs: runs-on: macos-latest steps: - uses: actions/checkout@v4 - - name: Install Python 3.11.4 and build NSO-RPC + - name: Install Universal Python 3.11.4 run: > curl https://www.python.org/ftp/python/3.11.4/python-3.11.4-macos11.pkg -o python-3.11.4-macos11.pkg && sudo installer -verbose -pkg python-3.11.4-macos11.pkg -target / @@ -112,7 +110,7 @@ jobs: zip -yr mac-universal2-portable.zip NSO-RPC.app/ - name: Upload NSO-RPC Universal2 Build if: github.event_name != 'pull_request' - uses: softprops/action-gh-release@v2.0.4 + uses: softprops/action-gh-release@v2 with: files: | client/dist/mac-universal2-installer.dmg @@ -133,6 +131,6 @@ jobs: hash-type: sha256 file-name: checksums.txt get-assets: true - - uses: softprops/action-gh-release@v2.0.4 + - uses: softprops/action-gh-release@v2 with: files: checksums.txt From 2722564d497e1bbd4f3fcef92402ee95b49bcf5f Mon Sep 17 00:00:00 2001 From: Spotlight Date: Fri, 17 May 2024 10:42:18 -0400 Subject: [PATCH 7/7] Enforce universal2 Python --- scripts/macos-universal2/build.sh | 8 ++++++++ scripts/macos-universal2/prep-PyQt.sh | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/scripts/macos-universal2/build.sh b/scripts/macos-universal2/build.sh index 74079ea..9573161 100755 --- a/scripts/macos-universal2/build.sh +++ b/scripts/macos-universal2/build.sh @@ -4,6 +4,14 @@ set -e # Run everything relative to our script directory. cd "$(dirname "$0")" +# Within GitHub Actions and similar, we should use the Python.org +# copy of Python 3.11 available. This permits a universal2 framework for py2app. +# (Otherwise, GitHub's default runners include a single architecture version.) +if [[ "$CI" == "true" ]]; then + shopt -s expand_aliases + alias python3=/Library/Frameworks/Python.framework/Versions/3.11/bin/python3.11 +fi + # Activate a virtual environment so we don't pollute the system environment. python3 -m venv --upgrade-deps venv source venv/bin/activate diff --git a/scripts/macos-universal2/prep-PyQt.sh b/scripts/macos-universal2/prep-PyQt.sh index 0168c83..12a271d 100644 --- a/scripts/macos-universal2/prep-PyQt.sh +++ b/scripts/macos-universal2/prep-PyQt.sh @@ -4,6 +4,14 @@ set -e # Provided by @spotlightishere on Github # https://github.com/MCMi460/NSO-RPC/pull/86#issuecomment-1605700512 +# Within GitHub Actions and similar, we should use the Python.org +# copy of Python 3.11 available. This permits a universal2 framework for py2app. +# (Otherwise, GitHub's default runners include a single architecture version.) +if [[ "$CI" == "true" ]]; then + shopt -s expand_aliases + alias python3=/Library/Frameworks/Python.framework/Versions/3.11/bin/python3.11 +fi + # If we already have a universal2 wheel available, install and process no further. # https://stackoverflow.com/a/6364244 if compgen -G "./PyQt6_*universal2.whl"; then