Refactor cross compilation scripts

.ci/build_appimage.sh

@@ -7,9 +7,12 @@ sudo ()
 }
 
 # Build
-if [[ "$1" != "0" ]]; then
-    .ci/common/build.sh appimage_build linux || exit 1
+if [[ "$1" == "" ]]; then
+    PLATFORM=linux
+else
+    PLATFORM="linux_$1"
 fi
+.ci/common/build.sh appimage_build $PLATFORM || exit 1
 
 repo_dir=$(pwd)
 cd appimage_build
@@ -45,45 +48,63 @@ cmake .. &&
 make -j$(nproc --all) &&
 mv src/linuxdeploy-plugin-appimage ../.. &&
 cd ../.. &&
-rm -rf plugin-appimage &&
-
-# Build AppImageKit
-sudo apt install -y snapd squashfs-tools &&
-sudo snap install docker &&
-git clone https://github.com/AppImage/AppImageKit --recurse-submodules &&
-cd AppImageKit &&
-sudo env ARCH=$(arch) bash ci/build.sh
-sudo cp out/appimagetool /usr/bin/ &&
-sudo cp out/digest /usr/bin/ &&
-sudo cp out/validate /usr/bin/ &&
-cd .. &&
-sudo mkdir -p /usr/lib/appimagekit &&
-sudo ln -s /usr/bin/mksquashfs /usr/lib/appimagekit/mksquashfs &&
+rm -rf plugin-appimage
+
+# Download appimagetool
+wget https://github.com/AppImage/appimagetool/releases/download/continuous/appimagetool-x86_64.AppImage &&
+mv appimagetool-*.AppImage appimagetool
+chmod +x appimagetool
+export PATH=$(pwd):$PATH
 
 # Install patchelf from PyPI (see https://github.com/linuxdeploy/linuxdeploy-plugin-qt/issues/133#issuecomment-1608168363)
 sudo apt install -y python3-pip
+python3 -m venv .venv
+source .venv/bin/activate
 pip3 install patchelf
-export PATH=$PATH:~/.local/bin
+patchelf --version
+
+# Use custom ldd and strip
+if [[ "$PLATFORM" == "linux_aarch64" ]]; then
+    ln -s /usr/bin/${BUILD_TOOLCHAIN_PREFIX}strip strip
+    sudo cp ../.ci/bin/xldd /usr/bin/${BUILD_TOOLCHAIN_PREFIX}ldd
+    ln -s /usr/bin/${BUILD_TOOLCHAIN_PREFIX}ldd ldd
+    export CT_XLDD_ROOT="$BUILD_SYSROOT_PATH"
+fi
+
+# Set LD_LIBRARY_PATH (directories with *.so files)
+# TODO: Installing with cmake is probably a better idea
+LD_LIBRARY_PATH=""
+
+for file in $(find . -type f -name "*.so*"); do
+    dir=$(dirname "$file")
+    if [[ ":$LD_LIBRARY_PATH:" != *":$dir:"* ]]; then
+        LD_LIBRARY_PATH="$LD_LIBRARY_PATH:`readlink -f $dir`"
+    fi
+done
+
+LD_LIBRARY_PATH=${LD_LIBRARY_PATH#:}
+LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$QT_ROOT_DIR/lib" # Qt
+export LD_LIBRARY_PATH
+echo "LD_LIBRARY_PATH set to: $LD_LIBRARY_PATH"
 
 # Build AppImage
+if [[ "$PLATFORM" == "linux_aarch64" ]]; then
+    wget https://github.com/AppImage/type2-runtime/releases/download/continuous/runtime-aarch64
+    export ARCH=arm_aarch64
+    APPIMAGE_ARCH=aarch64
+    export LDAI_RUNTIME_FILE=runtime-aarch64
+    export QEMU_LD_PREFIX="$BUILD_SYSROOT_PATH"
+    export PATH="$QT_ROOT_DIR/bin:$PATH" # use cross Qt
+    export PATH="$PATH:$QT_HOST_PATH/libexec"
+else
+    APPIMAGE_ARCH=x86_64
+fi
+
 export QML_SOURCES_PATHS=$(pwd)/src &&
-export EXTRA_QT_PLUGINS="svg;" &&
-export LDAI_UPDATE_INFORMATION="${appimage_zsync_prefix}${app_name}*-${APPIMAGE_ARCH-$(arch)}.AppImage.zsync"
+export EXTRA_QT_MODULES="qml;svg;" &&
+export LDAI_UPDATE_INFORMATION="${appimage_zsync_prefix}${app_name}*-${APPIMAGE_ARCH}.AppImage.zsync"
 echo "AppImage update information: ${LDAI_UPDATE_INFORMATION}"
 
-case "$(qmake -query QMAKE_XSPEC)" in
-    linux-arm-gnueabi-g++)
-        wget https://github.com/AppImage/AppImageKit/releases/download/continuous/runtime-armhf
-        export ARCH=arm
-        export LDAI_RUNTIME_FILE=runtime-armhf
-        ;;
-    linux-aarch64-gnu-g++)
-        wget https://github.com/AppImage/AppImageKit/releases/download/continuous/runtime-aarch64
-        export ARCH=arm_aarch64
-        export LDAI_RUNTIME_FILE=runtime-aarch64
-        ;;
-esac
-
 ./linuxdeploy --appdir AppDir -e src/app/${executable_name} -i $repo_dir/res/${executable_name}.png -d $repo_dir/release/appimage.desktop --plugin qt --output appimage
 
 mv *.AppImage* $repo_dir

.ci/build_qt6.sh

@@ -3,102 +3,56 @@
 sudo apt install -y lsb-release
 
 qt_version="$1"
-qt_modules="$(echo $2 | tr ' ' ',')"
+
+if [[ "$2" == "" ]]; then
+    qt_modules=""
+else
+    qt_modules=",$(echo $2 | tr ' ' ',')"
+fi
+
 target_arch="$3"
 root_path="$(pwd)"
-sysroot_path="${root_path}/sysroot"
-sysroot_ubuntu_version="$(lsb_release -rs).1"
-sysroot_ubuntu_codename="$(lsb_release -cs)"
-host_prefix="${root_path}/qt-host"
-cross_prefix="${root_path}/qt-cross"
+#host_prefix="${root_path}/qt-host"
+host_prefix="$QT_HOST_PATH"
+cross_prefix="$QT_CROSS_PATH"
 target_prefix="/usr/local/qt"
-toolchain_config="${root_path}/.ci/qt6-toolchain.cmake"
-
-case "$target_arch" in
-    aarch64)
-        target_arch_name="armv8-a"
-        toolchain_name="aarch64-linux-gnu"
-        target_platform="linux-aarch64-gnu-g++"
-        ;;
-    armv7)
-        target_arch_name="armv7-a"
-        toolchain_name="arm-linux-gnueabihf"
-        target_platform="linux-arm-gnueabi-g++"
-        ;;
-esac
-
-toolchain_prefix="${toolchain_name}-"
-
-case "$target_arch" in
-    aarch64)
-        target_arch_debian_name="arm64"
-        ;;
-    armv7)
-        target_arch_debian_name="armhf"
-        ;;
-esac
 
 echo "Qt version to build: ${qt_version}"
 echo "Qt modules: ${qt_modules}"
-echo "Target architecture: ${target_arch} (${target_arch_name})"
-
-# Install dependencies
-${root_path}/.ci/qt6_deps.sh || exit 1
-${root_path}/.ci/install_cross_compiler.sh "${target_arch}" || exit 1
-sudo apt install -y qemu-user-static || exit 1
-sudo apt install -y symlinks || exit 1
+echo "Target architecture: ${target_arch} (${BUILD_ARCH_NAME})"
 
 # Clone Qt
 git clone https://github.com/qt/qt5 qt || exit 1
 cd qt
-git checkout "v$qt_version" || exit 1
-./init-repository --module-subset=qtbase,qttools,qtdeclarative,qtsvg,${qt_modules} || exit 1
+git checkout $(git tag | grep '^v6\.8\.[0-9]*$' | sort -V | tail -n 1) || exit 1
+./init-repository --module-subset=qtbase,qttools,qtdeclarative,qtsvg${qt_modules} || exit 1
 
 # Build Qt (host)
-mkdir host-build
-cd host-build
-echo "Building host Qt..."
-../configure -release -nomake examples -nomake tests -opensource -confirm-license -prefix "$host_prefix" || exit 1
-cmake --build . --parallel $(nproc --all) || exit 1
-echo "Installing host Qt..."
-cmake --install . || exit 1
-cd ..
-rm -rf host-build
-
-# Prepare sysroot
-echo "Preparing sysroot..."
-curl "https://cdimage.ubuntu.com/ubuntu-base/releases/${sysroot_ubuntu_codename}/release/ubuntu-base-${sysroot_ubuntu_version}-base-${target_arch_debian_name}.tar.gz" > ./ubuntu-base.tar.gz || exit 1
-mkdir -p "$sysroot_path"
-sudo tar -xvzf ubuntu-base.tar.gz -C "$sysroot_path" || exit 1
-sudo update-binfmts --enable qemu-arm || exit 1
-sudo mount -o bind /dev "${sysroot_path}/dev" || exit 1
-sudo cp /etc/resolv.conf "${sysroot_path}/etc" || exit 1
-sudo chmod 1777 "${sysroot_path}/tmp" || exit 1
-sudo cp "${root_path}/.ci/qt6_deps.sh" "${sysroot_path}/"
-sudo chroot "$sysroot_path" /bin/bash -c "/qt6_deps.sh" || exit 1
-sudo chroot "$sysroot_path" /bin/bash -c "apt install -y symlinks && symlinks -rc /" || exit 1
+#mkdir host-build
+#cd host-build
+#echo "Building host Qt..."
+#../configure -release -nomake examples -nomake tests -opensource -confirm-license -prefix "$host_prefix" || exit 1
+#cmake --build . --parallel $(nproc --all) || exit 1
+#echo "Installing host Qt..."
+#cmake --install . || exit 1
+#cd ..
+#rm -rf host-build
 
 # Build Qt (cross)
 mkdir cross-build
 cd cross-build
 echo "Cross-compiling Qt..."
-export BUILD_SYSROOT_PATH=${sysroot_path}
-export BUILD_TOOLCHAIN_NAME=${toolchain_name}
-export BUILD_TOOLCHAIN_PREFIX=${toolchain_prefix}
-export BUILD_ARCH_NAME=${target_arch_name}
-../configure -release -opengl es2 -nomake examples -nomake tests -qt-host-path "$host_prefix" -xplatform "$target_platform" \
-             -device-option CROSS_COMPILE="$toolchain_prefix" -sysroot "$sysroot_path" -opensource -confirm-license \
-             -prefix "$target_prefix" -extprefix "$cross_prefix" -- -DCMAKE_TOOLCHAIN_FILE="$toolchain_config" \
+../configure -release -opengl es2 -nomake examples -nomake tests -qt-host-path "$host_prefix" -xplatform "$BUILD_PLATFORM" \
+             -device-option CROSS_COMPILE="$BUILD_TOOLCHAIN_PREFIX" -sysroot "$BUILD_SYSROOT_PATH" -opensource -confirm-license \
+             -prefix "$target_prefix" -extprefix "$cross_prefix" -- -DCMAKE_TOOLCHAIN_FILE="$BUILD_TOOLCHAIN_CONFIG" \
              -DQT_FEATURE_xcb=ON -DFEATURE_xcb_xlib=ON -DQT_FEATURE_xlib=ON || exit 1
 cmake --build . --parallel $(nproc --all) || exit 1
 echo "Installing cross-compiled Qt..."
 cmake --install . || exit 1
 cd ..
-rm -rf cross-build
 
 # Cleanup
-sudo umount "${sysroot_path}/dev" || exit 1
 cd ..
 rm -rf qt
 # Required for cache
-sudo chmod 777 -R ${sysroot_path}
+sudo chmod 777 -R ${BUILD_SYSROOT_PATH}

.ci/common/build.sh

@@ -12,6 +12,8 @@ mkdir -p "$BUILD_DIR"
 
 if [[ "$PLATFORM" == "win64" ]] || [[ "$PLATFORM" == "win32" ]]; then
 	cmake -B "$BUILD_DIR" -G "Ninja" -DCMAKE_BUILD_TYPE=Release -DSCRATCHCPP_PLAYER_BUILD_UNIT_TESTS=OFF || exit 3
+elif [[ "$PLATFORM" == "linux_aarch64" ]]; then
+	cmake -B "$BUILD_DIR" -DCMAKE_BUILD_TYPE=Release -DSCRATCHCPP_PLAYER_BUILD_UNIT_TESTS=OFF -DCMAKE_PREFIX_PATH="$QT_ROOT_DIR" -DCMAKE_TOOLCHAIN_FILE="$BUILD_TOOLCHAIN_CONFIG" -DCMAKE_FIND_ROOT_PATH="$QT_ROOT_DIR" -DQT_HOST_PATH="$QT_HOST_PATH" -DQT_HOST_CMAKE_DIR="$QT_HOST_PATH/lib/cmake"
 else
 	cmake -B "$BUILD_DIR" -DCMAKE_BUILD_TYPE=Release -DSCRATCHCPP_PLAYER_BUILD_UNIT_TESTS=OFF || exit 3
 fi

.ci/install_cross_compiler.sh

@@ -5,6 +5,4 @@ case "$1" in
     aarch64)
         sudo apt install -y g++-aarch64-linux-gnu
         ;;
-    armv7)
-        sudo apt install -y g++-arm-linux-gnueabihf
 esac

.ci/prepare_cross_build.sh

@@ -1,36 +1,52 @@
 #!/bin/bash
 
+sudo apt install -y lsb-release
+
 target_arch="$1"
+root_path="$(pwd)"
+sysroot_path="${root_path}/sysroot"
+sysroot_ubuntu_version="$(lsb_release -rs).1"
+sysroot_ubuntu_codename="$(lsb_release -cs)"
+cross_prefix="${root_path}/qt-cross"
+toolchain_config="${root_path}/.ci/qt6-toolchain.cmake"
 
 case "$target_arch" in
     aarch64)
-        toolchain_prefix="aarch64-linux-gnu-"
-        echo "APPIMAGE_ARCH=aarch64" >> "${GITHUB_ENV}"
-        ;;
-    armv7)
-        toolchain_prefix="arm-linux-gnueabihf-"
-        echo "APPIMAGE_ARCH=armhf" >> "${GITHUB_ENV}"
+        target_arch_name="armv8-a"
+        target_arch_debian_name="arm64"
+        toolchain_name="aarch64-linux-gnu"
+        target_platform="linux-aarch64-gnu-g++"
         ;;
 esac
 
-echo "$(pwd)/qt-cross/bin:$(pwd)/qt-host/libexec" >> $GITHUB_PATH
-echo "LD_LIBRARY_PATH=$(pwd)/qt-cross/lib:$(pwd)/qt-host/lib" >> "${GITHUB_ENV}"
-.ci/install-cross-compiler.sh "$target_arch"
-.ci/qt6-dependencies.sh
-if [[ "$target_arch" == "armv7" ]]; then
-    echo "QMAKE_CC=arm-linux-gnueabihf-gcc
-    QMAKE_CXX=arm-linux-gnueabihf-g++
-    QMAKE_LINK=arm-linux-gnueabihf-g++
-    QMAKE_LINK_SHLIB=arm-linux-gnueabihf-g++
-    QMAKE_AR=arm-linux-gnueabihf-ar cqs
-    QMAKE_OBJCOPY=arm-linux-gnueabihf-objcopy
-    QMAKE_NM=arm-linux-gnueabihf-nm -P
-    QMAKE_STRIP=arm-linux-gnueabihf-strip" >> .qmake.conf
-fi
-
-# Prepare cross-tools for linuxdeploy
-sudo cp /usr/bin/${toolchain_prefix}strip strip
-sudo mv /usr/bin/ldd /usr/bin/ldd-amd64
-sudo cp .ci/bin/xldd /usr/bin/${toolchain_prefix}ldd
-sudo ln -s /usr/bin/${toolchain_prefix}ldd /usr/bin/ldd
-echo "CT_XLDD_ROOT=$(pwd)/sysroot" >> "${GITHUB_ENV}"
+toolchain_prefix="${toolchain_name}-"
+
+echo "Target architecture: ${target_arch} (${target_arch_name})"
+
+# Install dependencies
+${root_path}/.ci/install_cross_compiler.sh "${target_arch}" || exit 1
+sudo apt install -y qemu-user-static || exit 1
+
+# Prepare sysroot
+echo "Preparing sysroot..."
+curl "https://cdimage.ubuntu.com/ubuntu-base/releases/${sysroot_ubuntu_codename}/release/ubuntu-base-${sysroot_ubuntu_version}-base-${target_arch_debian_name}.tar.gz" > ./ubuntu-base.tar.gz || exit 1
+mkdir -p "$sysroot_path"
+sudo tar -xvzf ubuntu-base.tar.gz -C "$sysroot_path" || exit 1
+rm ubuntu-base.tar.gz
+sudo update-binfmts --enable qemu-arm || exit 1
+sudo mount -o bind /dev "${sysroot_path}/dev" || exit 1
+sudo cp /etc/resolv.conf "${sysroot_path}/etc" || exit 1
+sudo chmod 1777 "${sysroot_path}/tmp" || exit 1
+sudo cp "${root_path}/.ci/qt6_deps.sh" "${sysroot_path}/"
+sudo chroot "$sysroot_path" /bin/bash -c "/qt6_deps.sh" || exit 1
+sudo chroot "$sysroot_path" /bin/bash -c "apt update && apt install -y symlinks libssl-dev && symlinks -rc /" || exit 1
+sudo umount "${sysroot_path}/dev" || exit 1
+
+# Prepare environment
+echo "BUILD_SYSROOT_PATH=$sysroot_path" >> "$GITHUB_ENV"
+echo "BUILD_TOOLCHAIN_NAME=$toolchain_name" >> "$GITHUB_ENV"
+echo "BUILD_TOOLCHAIN_PREFIX=$toolchain_prefix" >> "$GITHUB_ENV"
+echo "BUILD_TOOLCHAIN_CONFIG=$toolchain_config" >> "$GITHUB_ENV"
+echo "BUILD_ARCH_NAME=$target_arch_name" >> "$GITHUB_ENV"
+echo "BUILD_PLATFORM=$target_platform" >> "$GITHUB_ENV"
+echo "QT_ROOT_DIR=$cross_prefix" >> "$GITHUB_ENV"