Reenable build of loadtest apps on Windows arm64 CI (#802)

Set CMP-0149 before product() command so it works and the desired Windows SDK is chosen. Fixes Windows arm64 build. Set threads limit for vcpkg builds to avoid memory exhaustion. Add CMake configuration option for Windows and macOS to search for dependencies (assimp, glew, SDL2) locally instead of using the binaries in the repo. Defaults to OFF.
KhronosGroup · Nov 23, 2023 · 6c131f7 · 6c131f7
1 parent 1c5dc9c
commit 6c131f7
Show file tree

Hide file tree

Showing 12 changed files with 116 additions and 49 deletions.
diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml
@@ -70,7 +70,7 @@ jobs:
             # Can set to OpenGL once issue is resolved.
             options: {
               config: 'Debug,Release',
-              doc: ON, jni: ON, loadtests: OFF, tests: ON, tools: ON, tools_cts: ON,
+              doc: ON, jni: ON, loadtests: OpenGL, tests: ON, tools: ON, tools_cts: ON,
               package: YES
             }
     runs-on: ${{ matrix.os }}
@@ -110,6 +110,10 @@ jobs:
       OPENGL_ES_EMULATOR: C:/Imagination/Windows_x86_64
       OPENGL_ES_EMULATOR_WIN: C:\Imagination\Windows_x86_64
       PVR_SDK_HOME: https://github.com/powervr-graphics/Native_SDK/raw/master/lib/Windows_x86_64/
+      # This is set to avoid the compiler running out of heap space
+      # when vcpkg is compiling glew. Default is logical cores + 1.
+      # Runners have 1 thread per core and 2 cores.
+      VCPKG_MAX_CONCURRENCY: 2
       VULKAN_SDK_VER: 1.3.243.0
 
     steps:
@@ -191,6 +195,7 @@ jobs:
         cat buildtrees\glew\install-arm64-windows-rel-out.log
         popd
         echo "CMAKE_TOOLCHAIN_FILE=$env:VCPKG_INSTALLATION_ROOT/scripts/buildsystems/vcpkg.cmake" >> $env:GITHUB_ENV
+        echo "LOADTESTS_USE_LOCAL_DEPENDENCIES=ON" >> $env:GITHUB_ENV
 
     - name: Install Dependencies
       # This script only installs what's needed by ON FEATUREs.

diff --git a/.travis.yml b/.travis.yml
@@ -17,10 +17,10 @@ language: cpp
 
 addons:
   homebrew:
+    update: false
     packages:
     - git-lfs
     - doxygen
-    #update: true
 
 env:
   global:
@@ -43,7 +43,8 @@ env:
     # add the list of tests to the cmake test runner.
     - CONFIGURATION=Debug,Release PLATFORM=macOS ARCHS=x86_64
       FEATURE_DOC=ON FEATURE_JNI=ON FEATURE_LOADTESTS=OpenGL+Vulkan FEATURE_TOOLS=ON
-      FEATURE_TOOLS_CTS=ON SUPPORT_SSE=ON SUPPORT_OPENCL=OFF DEPLOY_DOCS=YES PACKAGE=YES
+      FEATURE_TOOLS_CTS=ON LOADTESTS_USE_LOCAL_DEPENDENCIES=ON SUPPORT_SSE=ON
+      SUPPORT_OPENCL=OFF DEPLOY_DOCS=YES PACKAGE=YES
     - CONFIGURATION=Release PLATFORM=macOS ARCHS=arm64
       FEATURE_DOC=ON FEATURE_JNI=ON FEATURE_LOADTESTS=OpenGL+Vulkan FEATURE_TESTS=OFF
       FEATURE_TOOLS=ON FEATURE_TOOLS_CTS=OFF SUPPORT_SSE=ON SUPPORT_OPENCL=OFF PACKAGE=YES
@@ -216,9 +217,7 @@ install:
     fi
     ;;
   osx)
-    if [ "$PLATFORM" = "macOS" -a "$ARCHS" = $(uname -m) ]; then
-        # We're building for the host processor type. Safe to use
-        # brew installed libraries instead of repo included ones.
+    if [ "$LOADTESTS_USE_LOCAL_DEPENDENCIES" = "ON" ]; then
         brew install sdl2
         brew install assimp
     fi

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -10,6 +10,16 @@ find_package(Bash REQUIRED)
 
 include(cmake/version.cmake)
 
+if(POLICY CMP0149)
+    # Ignore CMAKE_SYSTEM_VERSION and select either latest available
+    # Windows SDK or that specified in WindowsSDKVersion environment
+    # variable  Needed because OLD policy picks SDK that matches
+    # system version, CI uses Windows Server 2022 and its matching
+    # SDK lacks the arm64 glu32.lib which causes builds to fail.
+    # MUST be set before project() command.
+    cmake_policy(SET CMP0149 NEW)
+endif()
+
 project(KTX-Software
     VERSION ${KTX_VERSION}
     DESCRIPTION "Libraries and tools to create and read KTX image texture files."
@@ -113,6 +123,18 @@ set( KTX_FEATURE_LOADTEST_APPS
 set_property( CACHE KTX_FEATURE_LOADTEST_APPS
               PROPERTY STRINGS OFF OpenGL Vulkan OpenGL+Vulkan
 )
+if(WIN32 OR APPLE)
+    if(NOT IOS)
+        CMAKE_DEPENDENT_OPTION( KTX_LOADTEST_APPS_USE_LOCAL_DEPENDENCIES
+            "Look for locally installed dependencies. Fail build if not found. When OFF use binaries from the KTX-Software repo, if available for the target processor. Useful when building for Windows arm64 using vcpkg packages."
+            OFF
+            KTX_FEATURE_LOADTEST_APPS
+            OFF
+        )
+    endif()
+else()
+    set( KTX_LOADTEST_APPS_USE_LOCAL_DEPENDENCIES ON )
+endif()
 
 # Platform specific settings
 

diff --git a/ci_scripts/build_macos.sh b/ci_scripts/build_macos.sh
@@ -28,6 +28,7 @@ FEATURE_LOADTESTS=${FEATURE_LOADTESTS:-OpenGL+Vulkan}
 FEATURE_TESTS=${FEATURE_TESTS:-ON}
 FEATURE_TOOLS=${FEATURE_TOOLS:-ON}
 FEATURE_TOOLS_CTS=${FEATURE_TOOLS_CTS:-ON}
+LOADTESTS_USE_LOCAL_DEPENDENCIES=${LOADTESTS_USE_LOCAL_DEPENDENCIES:-OFF}
 PACKAGE=${PACKAGE:-NO}
 SUPPORT_SSE=${SUPPORT_SSE:-ON}
 SUPPORT_OPENCL=${SUPPORT_OPENCL:-OFF}
@@ -73,9 +74,10 @@ cmake_args=("-G" "Xcode" \
   "-D" "KTX_FEATURE_TESTS=$FEATURE_TESTS" \
   "-D" "KTX_FEATURE_TOOLS=$FEATURE_TOOLS" \
   "-D" "KTX_FEATURE_TOOLS_CTS=$FEATURE_TOOLS_CTS" \
+  "-D" "KTX_LOADTEST_APPS_USE_LOCAL_DEPENDENCIES=$LOADTESTS_USE_LOCAL_DEPENDENCIES"
+  "-D" "KTX_WERROR=$WERROR"
   "-D" "BASISU_SUPPORT_OPENCL=$SUPPORT_OPENCL" \
   "-D" "BASISU_SUPPORT_SSE=$SUPPORT_SSE" \
-  "-D" "KTX_WERROR=$WERROR"
 )
 if [ "$ARCHS" = "x86_64" ]; then cmake_args+=("-D" "ASTCENC_ISA_SSE41=ON"); fi
 if [ -n "$MACOS_CERTIFICATES_P12" ]; then

diff --git a/ci_scripts/build_win.ps1 b/ci_scripts/build_win.ps1
@@ -45,16 +45,25 @@ $CMAKE_GEN = Set-ConfigVariable CMAKE_GEN "Visual Studio 17 2022"
 $CMAKE_TOOLSET = Set-ConfigVariable CMAKE_TOOLSET ""
 $FEATURE_DOC = Set-ConfigVariable FEATURE_DOC "OFF"
 $FEATURE_JNI = Set-ConfigVariable FEATURE_JNI "OFF"
-$FEATURE_LOADTESTS = Set-ConfigVariable FEATURE_LOADTESTS "OpenGL+Vulkan"
+if ($ARCH -eq 'x64') {
+  $FEATURE_LOADTESTS = Set-ConfigVariable FEATURE_LOADTESTS "OpenGL+Vulkan"
+} else {
+  $FEATURE_LOADTESTS = Set-ConfigVariable FEATURE_LOADTESTS "OpenGL"
+}
 $FEATURE_TESTS = Set-ConfigVariable FEATURE_TESTS "ON"
 $FEATURE_TOOLS = Set-ConfigVariable FEATURE_TOOLS "ON"
 $FEATURE_TOOLS_CTS = Set-ConfigVariable FEATURE_TOOLS_CTS "ON"
+$LOADTESTS_USE_LOCAL_DEPENDENCIES = Set-ConfigVariable LOADTESTS_USE_LOCAL_DEPENDENCIES "OFF"
 $PACKAGE = Set-ConfigVariable PACKAGE "NO"
 $SUPPORT_SSE = Set-ConfigVariable SUPPORT_SSE "ON"
 $SUPPORT_OPENCL = Set-ConfigVariable SUPPORT_OPENCL "OFF"
 $WERROR = Set-ConfigVariable WERROR "OFF"
-$OPENGL_ES_EMULATOR = Set-ConfigVariable OPENGL_ES_EMULATOR `
-  "c:/Imagination` Technologies/PowerVR_Graphics/PowerVR_Tools/PVRVFrame/Library/Windows_x86_64"
+if ($ARCH -eq 'x64') {
+  $OPENGL_ES_EMULATOR = Set-ConfigVariable OPENGL_ES_EMULATOR `
+    "c:/Imagination` Technologies/PowerVR_Graphics/PowerVR_Tools/PVRVFrame/Library/Windows_x86_64"
+} else {
+  $OPENGL_ES_EMULATOR = Set-ConfigVariable OPENGL_ES_EMULATOR ""
+}
 $CODE_SIGN_KEY_VAULT = Set-ConfigVariable CODE_SIGN_KEY_VAULT ""
 $CODE_SIGN_TIMESTAMP_URL = Set-ConfigVariable CODE_SIGN_TIMESTAMP_URL ""
 $LOCAL_KEY_VAULT_SIGNING_IDENTITY = Set-ConfigVariable LOCAL_KEY_VAULT_SIGNING_IDENTITY ""
@@ -93,9 +102,10 @@ $cmake_args += @(
   "-D", "KTX_FEATURE_TESTS=$FEATURE_TESTS"
   "-D", "KTX_FEATURE_TOOLS=$FEATURE_TOOLS"
   "-D", "KTX_FEATURE_TOOLS_CTS=$FEATURE_TOOLS_CTS"
+  "-D", "KTX_LOADTEST_APPS_USE_LOCAL_DEPENDENCIES=$LOADTESTS_USE_LOCAL_DEPENDENCIES"
+  "-D", "KTX_WERROR=$WERROR"
   "-D", "BASISU_SUPPORT_SSE=$SUPPORT_SSE"
   "-D", "BASISU_SUPPORT_OPENCL=$SUPPORT_OPENCL"
-  "-D", "KTX_WERROR=$WERROR"
   "-D", "CODE_SIGN_KEY_VAULT=$CODE_SIGN_KEY_VAULT"
 )
 if ($CODE_SIGN_KEY_VAULT) {

diff --git a/lib/ktxint.h b/lib/ktxint.h
@@ -308,8 +308,8 @@ KTX_error_code printKTX2Info2(ktxStream* src, KTX_header2* header);
 #include <shellapi.h>
 #include <stdlib.h>
 
-// For Windows, we convert the UTF-8 path and mode to UTF-16 path and use _wfopen
-// which correctly handles unicode characters.
+// For Windows, we convert the UTF-8 path and mode to UTF-16 path and use
+// _wfopen which correctly handles unicode characters.
 static inline FILE* ktxFOpenUTF8(char const* path, char const* mode) {
     int wpLen = MultiByteToWideChar(CP_UTF8, 0, path, -1, NULL, 0);
     int wmLen = MultiByteToWideChar(CP_UTF8, 0, mode, -1, NULL, 0);
@@ -320,20 +320,22 @@ static inline FILE* ktxFOpenUTF8(char const* path, char const* mode) {
         wchar_t* wmode = (wchar_t*)malloc(wmLen * sizeof(wchar_t));
         MultiByteToWideChar(CP_UTF8, 0, path, -1, wpath, wpLen);
         MultiByteToWideChar(CP_UTF8, 0, mode, -1, wmode, wmLen);
-        // Returned errmo_t value is also set in the global errno.
+        // Returned errno_t value is also set in the global errno.
         // Apps use that for error detail as libktx only returns
         // KTX_FILE_OPEN_FAILED.
         (void)_wfopen_s(&fp, wpath, wmode);
         free(wpath);
         free(wmode);
         return fp;
     } else {
-        assert(KTX_FALSE && "ktxFOpenUTF8 called with zero length path or mode.");
+        assert(KTX_FALSE
+               && "ktxFOpenUTF8 called with zero length path or mode.");
         return NULL;
     }
 }
 #else
-// For other platforms there is no need for any conversion, they support UTF-8 natively
+// For other platforms there is no need for any conversion, they
+// support UTF-8 natively.
 static inline FILE* ktxFOpenUTF8(char const* path, char const* mode) {
     return fopen(path, mode);
 }

diff --git a/lib/writer1.c b/lib/writer1.c
@@ -848,9 +848,9 @@ ktxTexture1_WriteKTX2ToStdioStream(ktxTexture1* This, FILE* dstsstr)
  * The file name must be encoded in utf-8. On Windows convert unicode names
  * to utf-8 with @c WideCharToMultiByte(CP_UTF8, ...) before calling.
  *
- * Callers are strongly urged to include a KTXwriter item in the texture's metadata.
- * It can be added by code, similar to the following, prior to calling this
- * function.
+ * Callers are strongly urged to include a KTXwriter item in the texture's
+ * metadata. It can be added by code, similar to the following, prior to
+ * calling this function.
  * @code
  *     char writer[100];
  *     snprintf(writer, sizeof(writer), "%s version %s", appName, appVer);

diff --git a/lib/writer2.c b/lib/writer2.c
@@ -584,9 +584,9 @@ ktxTexture2_WriteToStream(ktxTexture2* This, ktxStream* dststr)
  * @~English
  * @brief Write a ktxTexture object to a stdio stream in KTX format.
  *
- * Callers are strongly urged to include a KTXwriter item in the texture's metadata.
- * It can be added by code, similar to the following, prior to calling this
- * function.
+ * Callers are strongly urged to include a KTXwriter item in the texture's
+ * metadata. It can be added by code, similar to the following, prior to
+ * calling this function.
  * @code
  *     char writer[100];
  *     snprintf(writer, sizeof(writer), "%s version %s", appName, appVer);
@@ -635,9 +635,9 @@ ktxTexture2_WriteToStdioStream(ktxTexture2* This, FILE* dstsstr)
  * The file name must be encoded in utf-8. On Windows convert unicode names
  * to utf-8 with @c WideCharToMultiByte(CP_UTF8, ...) before calling.
  *
- * Callers are strongly urged to include a KTXwriter item in the texture's metadata.
- * It can be added by code, similar to the following, prior to calling this
- * function.
+ * Callers are strongly urged to include a KTXwriter item in the texture's
+ * metadata. It can be added by code, similar to the following, prior to
+ * calling this function.
  * @code
  *     char writer[100];
  *     snprintf(writer, sizeof(writer), "%s version %s", appName, appVer);
@@ -689,9 +689,9 @@ ktxTexture2_WriteToNamedFile(ktxTexture2* This, const char* const dstname)
  * Memory is allocated by the function and the caller is responsible for
  * freeing it.
  *
- * Callers are strongly urged to include a KTXwriter item in the texture's metadata.
- * It can be added by code, similar to the following, prior to calling this
- * function.
+ * Callers are strongly urged to include a KTXwriter item in the texture's
+ * metadata. It can be added by code, similar to the following, prior to
+ * calling this function.
  * @code
  *     char writer[100];
  *     snprintf(writer, sizeof(writer), "%s version %s", appName, appVer);

diff --git a/tests/loadtests/CMakeLists.txt b/tests/loadtests/CMakeLists.txt
@@ -1,24 +1,35 @@
 # Copyright 2017-2020 The Khronos Group Inc.
 # SPDX-License-Identifier: Apache-2.0
 
-#include(CMakePrintHelpers)
+include(CMakePrintHelpers)
 
-#cmake_print_variables(CMAKE_GENERATOR_PLATFORM CMAKE_SYSTEM_VERSION CMAKE_VS_WINDOWS_TARGET_PLATFORM_VERSION)
+if(WIN32)
+    cmake_print_variables(
+        CMAKE_SYSTEM_VERSION
+        CMAKE_VS_WINDOWS_TARGET_PLATFORM_VERSION
+    )
+endif()
 
-set( fp_pref ${CMAKE_FIND_PACKAGE_PREFER_CONFIG} )
-# Prefer CONFIG for vcpkg
-set( CMAKE_FIND_PACKAGE_PREFER_CONFIG TRUE )
-find_package(SDL2 QUIET)
-find_package(assimp QUIET)
-if(${KTX_FEATURE_LOADTEST_APPS} MATCHES "OpenGL")
-    if(NOT IOS)
+if(NOT IOS)
+    set( fp_pref ${CMAKE_FIND_PACKAGE_PREFER_CONFIG} )
+    # Prefer CONFIG for vcpkg
+    set( CMAKE_FIND_PACKAGE_PREFER_CONFIG TRUE )
+    if(${KTX_FEATURE_LOADTEST_APPS} MATCHES "OpenGL")
         find_package(OpenGL REQUIRED)
     endif()
-    if(WIN32)
-        find_package(GLEW QUIET)
+    if(KTX_LOADTEST_APPS_USE_LOCAL_DEPENDENCIES)
+        find_package(SDL2 REQUIRED)
+        if(NOT EMSCRIPTEN)
+            # Tests that require assimp are omitted from loadtests
+            # when building for the web.
+            find_package(assimp REQUIRED)
+        endif()
+        if(${KTX_FEATURE_LOADTEST_APPS} MATCHES "OpenGL" AND WIN32)
+            find_package(GLEW REQUIRED)
+        endif()
     endif()
+    set( CMAKE_FIND_PACKAGE_PREFER_CONFIG ${fp_pref} )
 endif()
-set( CMAKE_FIND_PACKAGE_PREFER_CONFIG ${fp_pref} )
 
 # We use our own local copy of GL headers to ensure we have glcorearb.h.
 set( OPENGL_INCLUDE_DIR  ${CMAKE_SOURCE_DIR}/other_include )

diff --git a/tests/loadtests/glloadtests.cmake b/tests/loadtests/glloadtests.cmake
@@ -223,10 +223,16 @@ function( create_gl_target target version sources common_resources test_images
                   COMMENT "Copy KTX library to build destination"
               )
             endif()
-            add_custom_command( TARGET ${target} POST_BUILD
-                COMMAND ${CMAKE_COMMAND} -E copy "${PROJECT_SOURCE_DIR}/other_lib/mac/$<CONFIG>/libSDL2.dylib" "$<TARGET_BUNDLE_CONTENT_DIR:${target}>/Frameworks/libSDL2.dylib"
-                COMMENT "Copy SDL2 library to build destination"
-            )
+            # No need to copy when there is a TARGET. The BREW SDL
+            # library has no LC_RPATH setting so the binary will
+            # only search for it where it was during linking.
+            # The vcpkg SDL target copies the library.
+            if(NOT TARGET SDL2::SDL2)
+                add_custom_command( TARGET ${target} POST_BUILD
+                    COMMAND ${CMAKE_COMMAND} -E copy "${PROJECT_SOURCE_DIR}/other_lib/mac/$<CONFIG>/libSDL2.dylib" "$<TARGET_BUNDLE_CONTENT_DIR:${target}>/Frameworks/libSDL2.dylib"
+                    COMMENT "Copy SDL2 library to build destination"
+                )
+            endif()
 
             # Specify destination for cmake --install.
             install(TARGETS ${target}

diff --git a/tests/loadtests/vkloadtests.cmake b/tests/loadtests/vkloadtests.cmake
@@ -330,9 +330,18 @@ if(APPLE)
             COMMAND ${CMAKE_COMMAND} -E copy "${Vulkan_LIBRARY_DIR}/libVkLayer*.dylib" "$<TARGET_BUNDLE_CONTENT_DIR:vkloadtests>/Frameworks/"
             COMMAND ${CMAKE_COMMAND} -E copy "${Vulkan_LIBRARY_REAL_PATH_NAME}" "$<TARGET_BUNDLE_CONTENT_DIR:vkloadtests>/Frameworks/"
             COMMAND ${CMAKE_COMMAND} -E create_symlink "${Vulkan_LIBRARY_REAL_FILE_NAME}" "$<TARGET_BUNDLE_CONTENT_DIR:vkloadtests>/Frameworks/${Vulkan_LIBRARY_SONAME_FILE_NAME}"
-            COMMAND ${CMAKE_COMMAND} -E copy "${PROJECT_SOURCE_DIR}/other_lib/mac/$<CONFIG>/libSDL2.dylib" "$<TARGET_BUNDLE_CONTENT_DIR:vkloadtests>/Frameworks/libSDL2.dylib"
             COMMENT "Copy libraries & frameworks to build destination"
         )
+        # No need to copy when there is a TARGET. The BREW SDL
+        # library has no LC_RPATH setting so the binary will
+        # only search for it where it was during linking.
+        # The vcpkg SDL target copies the library.
+        if(NOT TARGET SDL2::SDL2)
+            add_custom_command( TARGET vkloadtests POST_BUILD
+                COMMAND ${CMAKE_COMMAND} -E copy "${PROJECT_SOURCE_DIR}/other_lib/mac/$<CONFIG>/libSDL2.dylib" "$<TARGET_BUNDLE_CONTENT_DIR:vkloadtests>/Frameworks/libSDL2.dylib"
+                COMMENT "Copy repo's SDL2 library to build destination"
+            )
+        endif()
 
         # Specify destination for cmake --install.
         install(TARGETS vkloadtests

diff --git a/utils/platform_utils.h b/utils/platform_utils.h
@@ -22,8 +22,8 @@
 #endif
 
 #if defined(_WIN32) && !defined(_UNICODE)
-// For Windows, we convert the UTF-8 path to a UTF-16 path to force using the APIs
-// that correctly handle unicode characters
+// For Windows, we convert the UTF-8 path to a UTF-16 path to force using
+// the APIs that correctly handle unicode characters.
 inline std::wstring DecodeUTF8Path(std::string path) {
     std::wstring result;
     int len = MultiByteToWideChar(CP_UTF8, 0, path.c_str(), static_cast<int>(path.length()), NULL, 0);
@@ -35,7 +35,8 @@ inline std::wstring DecodeUTF8Path(std::string path) {
     return result;
 }
 #else
-// For other platforms there is no need for any conversion, they support UTF-8 natively
+// For other platforms there is no need for any conversion, they
+// support UTF-8 natively.
 inline std::string DecodeUTF8Path(std::string path) {
     return path;
 }
@@ -63,7 +64,7 @@ inline void InitUTF8CLI(int& argc, char* argv[]) {
 inline FILE* fopenUTF8(const std::string& path, const std::string& mode) {
 #if defined(_WIN32)
     FILE* fp;
-    // Returned errmo_t value is also set in the global errno.
+    // Returned errno_t value is also set in the global errno.
     (void)_wfopen_s(&fp, DecodeUTF8Path(path).c_str(), DecodeUTF8Path(mode).c_str());
     return fp;
 #else