Allow output mismatches on MSVC builds for BasisLZ tests

See: BinomialLLC/basis_universal#60

clitests/CMakeLists.txt

@@ -250,27 +250,35 @@ set(CASELIST
  tests/encode/encode_error_normal_mode.json
 )
 
-set(CASE_REGEN_GOLDEN_COMMANDS)
+set(CASE_REGEN_GOLDEN_OUTPUTS)
 
 if(NOT DEFINED KTX_TOOLS_PATH)
  message(FATAL_ERROR "KTX_TOOLS_PATH not defined")
 endif()
 
+if(MSVC)
+ set(CLITEST_ARGS "--msvc")
+else()
+ set(CLITEST_ARGS "")
+endif()
+
 foreach(CASE_FILE ${CASELIST})
  get_filename_component(CASE_DIR ${CASE_FILE} DIRECTORY)
  get_filename_component(CASE_NAME ${CASE_FILE} NAME_WE)
  string(REGEX REPLACE "^tests/" "ktxToolTests/" CASE_TRIMMED_DIR "${CASE_DIR}/")
  string(REPLACE "/" "." CASE_GROUP "${CASE_TRIMMED_DIR}")
  set(FULL_CASE_NAME "${CASE_GROUP}${CASE_NAME}")
 
- list(APPEND CASE_REGEN_GOLDEN_COMMANDS
- COMMAND ${PYTHON_EXECUTABLE} clitest.py ${CASE_FILE} --executable-path ${KTX_TOOLS_PATH} --regen-golden)
+ list(APPEND CASE_REGEN_GOLDEN_OUTPUTS ${CASE_FILE})
+ add_custom_command(
+ OUTPUT ${CASE_FILE}
+ WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+ COMMAND ${PYTHON_EXECUTABLE} clitest.py ${CASE_FILE} ${CLITEST_ARGS} --executable-path ${KTX_TOOLS_PATH} --regen-golden)
 
  add_test(NAME ${FULL_CASE_NAME}
- COMMAND ${PYTHON_EXECUTABLE} clitest.py ${CASE_FILE} --executable-path ${KTX_TOOLS_PATH}
+ COMMAND ${PYTHON_EXECUTABLE} clitest.py ${CASE_FILE} ${CLITEST_ARGS} --executable-path ${KTX_TOOLS_PATH}
  WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
 endforeach()
 
 add_custom_target(clitests_regen_golden
- WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
- ${CASE_REGEN_GOLDEN_COMMANDS})
+ DEPENDS ${CASE_REGEN_GOLDEN_OUTPUTS})

clitests/clitest.py

@@ -55,6 +55,8 @@ def cmdArgs(self):
  help='Regenerate reference files')
  parser.add_argument('-e', '--executable-path', action='store', required=True,
  help='Path to use for the executed command')
+ parser.add_argument('--msvc', action='store_true',
+ help='Indicates that test runs against MSVC build')
 
  cli_args, unknown_args = parser.parse_known_args()
 
@@ -108,14 +110,17 @@ def cmdArgs(self):
 
  # Execute subcases
 
- print(f"Executing test case '{cli_args.json_test_file}'...")
+ if not cli_args.regen_golden:
+ print(f"Executing test case '{cli_args.json_test_file}'...")
 
  failed = False
  messages = []
  subcases_passed = 0
  subcases_skipped = 0
  subcases_failed = 0
 
+ ref_not_updated = {}
+
  for subcase_index, subcase in enumerate(subcases):
  skip_subcase = False
 
@@ -134,7 +139,8 @@ def cmdArgs(self):
  subcase_failed = False
  subcase_messages = []
 
- messages.append(f" {ctx.eval(testcase['command'])}")
+ if not cli_args.regen_golden:
+ messages.append(f" {ctx.eval(testcase['command'])}")
 
  # Run command
 
@@ -212,8 +218,7 @@ def cmdArgs(self):
  old_output_ref_contains_regex = bool(re.findall(r'`.*`', old_output_ref))
 
  if old_output_ref_contains_regex:
- subcase_messages.append(f"Warning: reference file '{output_ref_filename}' was not updated as it contains a regex")
- subcase_failed = True
+ ref_not_updated[output_ref_filename] = f"NOTE: reference file '{output_ref_filename}' was not updated as it contains a regex"
  else:
  os.makedirs(os.path.dirname(output_ref_filename), exist_ok=True)
  output_ref_file = open(output_ref_filename, 'w+', newline='\n', encoding='utf-8')
@@ -253,7 +258,10 @@ def cmdArgs(self):
  if not cmd_failed and cli_args.regen_golden:
  os.makedirs(os.path.dirname(output_ref), exist_ok=True)
  if os.path.isfile(output_cur):
- shutil.copyfile(output_cur, output_ref)
+ if cli_args.msvc and 'allowOutputMismatchOnMSVC' in testcase:
+ ref_not_updated[output_ref] = f"NOTE: reference file '{output_ref}' was not updated on MSVC build as it has MSVC output mismatches allowed"
+ else:
+ shutil.copyfile(output_cur, output_ref)
  else:
  subcase_failed = True
  subcase_messages.append(f"stdout:\n{output['stdout']}")
@@ -270,8 +278,11 @@ def cmdArgs(self):
  files_found = False
 
  if files_found and not filecmp.cmp(output_cur, output_ref, shallow=False):
- subcase_messages.append(f"Mismatch between output file '{output_cur}' and reference file '{output_ref}'")
- subcase_failed = True
+ if cli_args.msvc and 'allowOutputMismatchOnMSVC' in testcase:
+ messages.append(f" WARNING: allowed mismatch on MSVC build between output file '{output_cur}' and reference file '{output_ref}'")
+ else:
+ subcase_messages.append(f"Mismatch between output file '{output_cur}' and reference file '{output_ref}'")
+ subcase_failed = True
 
 
  # Handle subcase failure
@@ -293,11 +304,15 @@ def cmdArgs(self):
  for message in messages:
  print(message)
 
- print("Subcase summary:")
- print(f" Passed: {subcases_passed}")
- print(f" Skipped: {subcases_skipped}")
- print(f" Failed: {subcases_failed}")
- print(f" Total: {len(subcases)}")
+ for ref in ref_not_updated:
+ print(ref_not_updated[ref])
+
+ if not cli_args.regen_golden:
+ print("Subcase summary:")
+ print(f" Passed: {subcases_passed}")
+ print(f" Skipped: {subcases_skipped}")
+ print(f" Failed: {subcases_failed}")
+ print(f" Total: {len(subcases)}")
 
- if failed and not cli_args.regen_golden:
+ if failed:
  exit(1)

clitests/tests/create/encode_blze.json

@@ -2,6 +2,7 @@
  "description": "Test create with default behavior of BasisLZ encoding.",
  "command": "ktx create --raw --format ${subcase} --width 8 --height 8 --testrun --threads 1 --encode basis-lz input/raw/raw_${subcase}_2D_8x8.raw output/create/encode_blze/output_${subcase}.ktx2",
  "status": 0,
+ "allowOutputMismatchOnMSVC": "BasisLZ compression relies on undefined order of std::unordered_map containers.",
  "outputs": {
  "output/create/encode_blze/output_${subcase}.ktx2": "golden/create/encode_blze/output_${subcase}.ktx2"
  },

clitests/tests/create/encode_blze_params.json

@@ -2,6 +2,7 @@
  "description": "Test create with BasisLZ encoding with various parameter combinations.",
  "command": "ktx create --raw --format ${subcase} --width 8 --height 8 --testrun --threads 1 --encode basis-lz ${select[args]} input/raw/raw_${subcase}_2D_8x8.raw output/create/encode_blze_params/output_${subcase}_${select[id]}.ktx2",
  "status": 0,
+ "allowOutputMismatchOnMSVC": "BasisLZ compression relies on undefined order of std::unordered_map containers.",
  "outputs": {
  "output/create/encode_blze_params/output_${subcase}_${select[id]}.ktx2": "golden/create/encode_blze_params/output_${subcase}_${select[id]}.ktx2"
  },

clitests/tests/encode/encode_blze.json

@@ -2,6 +2,7 @@
  "description": "Test default behavior of BasisLZ encoding.",
  "command": "ktx encode --testrun --threads 1 --codec basis-lz input/valid/valid_${subcase}.ktx2 output/encode/encode_blze/output_${subcase}.ktx2",
  "status": 0,
+ "allowOutputMismatchOnMSVC": "BasisLZ compression relies on undefined order of std::unordered_map containers.",
  "outputs": {
  "output/encode/encode_blze/output_${subcase}.ktx2": "golden/encode/encode_blze/output_${subcase}.ktx2"
  },

clitests/tests/encode/encode_blze_params.json

@@ -2,6 +2,7 @@
  "description": "Test BasisLZ encoding with various parameter combinations.",
  "command": "ktx encode --testrun --threads 1 --codec basis-lz ${select[args]} input/encode/large_${subcase}_ZSTD.ktx2 output/encode/encode_blze_params/output_${subcase}_${select[id]}.ktx2",
  "status": 0,
+ "allowOutputMismatchOnMSVC": "BasisLZ compression relies on undefined order of std::unordered_map containers.",
  "outputs": {
  "output/encode/encode_blze_params/output_${subcase}_${select[id]}.ktx2": "golden/encode/encode_blze_params/output_${subcase}_${select[id]}.ktx2"
  },