Changes for CI tests

appveyor.yml

@@ -92,18 +92,24 @@ environment:
     APPVEYOR_BUILD_WORKER_IMAGE: macos-monterey
     HOMEBREW_NO_INSTALL_CLEANUP: 1
     CC: clang
+    CFLAGS: "-I/usr/local/include"
+    LDFLAGS: "-L/usr/local/lib"
     CONFIGURE_OPTIONS: ""
   - TARGET: macos-x64-gcc
     BUILD_ENVIRONMENT: xcode
     APPVEYOR_BUILD_WORKER_IMAGE: macos-monterey
     HOMEBREW_NO_INSTALL_CLEANUP: 1
+    CC: gcc
+    CFLAGS: "-I/usr/local/include"
+    LDFLAGS: "-L/usr/local/lib"
     CONFIGURE_OPTIONS: ""
   - TARGET: macos-x64-gcc-python
     BUILD_ENVIRONMENT: xcode
     APPVEYOR_BUILD_WORKER_IMAGE: macos-monterey
     PYTHON: "/usr/local/opt/python@3.11/bin/python3"
     PYTHON_CONFIG: "/usr/local/opt/python@3.11/bin/python3-config"
     HOMEBREW_NO_INSTALL_CLEANUP: 1
+    CC: gcc
     CFLAGS: "-I/usr/local/include"
     LDFLAGS: "-L/usr/local/lib"
     CONFIGURE_OPTIONS: "--enable-python"
@@ -113,6 +119,7 @@ environment:
     PYTHON: "/usr/local/opt/python@3.11/bin/python3"
     PYTHON_CONFIG: "/usr/local/opt/python@3.11/bin/python3-config"
     HOMEBREW_NO_INSTALL_CLEANUP: 1
+    CC: gcc
     CFLAGS: "-I/usr/local/include"
     LDFLAGS: "-L/usr/local/lib"
     CONFIGURE_OPTIONS: "--disable-dependency-tracking --prefix=/usr/local --enable-python --with-pyprefix"
@@ -148,6 +155,12 @@ environment:
     HOMEBREW_NO_INSTALL_CLEANUP: 1
     PYTHON_VERSION: 3.11
     TOXENV: py311
+  - TARGET: macos-tox-py312
+    BUILD_ENVIRONMENT: python-tox
+    APPVEYOR_BUILD_WORKER_IMAGE: macos-monterey
+    HOMEBREW_NO_INSTALL_CLEANUP: 1
+    PYTHON_VERSION: 3.12
+    TOXENV: py312
   - TARGET: cygwin64-gcc
     BUILD_ENVIRONMENT: cygwin64
     APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2022
@@ -193,6 +206,7 @@ install:
     git clone https://github.com/libyal/vstools.git ..\vstools )
 - ps: If (($env:BUILD_ENVIRONMENT -eq "msbuild") -And (Test-Path ".\synctestdata.ps1")) {
     .\synctestdata.ps1 }
+- sh: if ( test ${BUILD_ENVIRONMENT} = "python-tox" || test ${BUILD_ENVIRONMENT} = "xcode" ) && test -f "./synctestdata.sh"; then ./synctestdata.sh; fi
 - ps: If ($env:BUILD_ENVIRONMENT -eq "msbuild") {
     If ( ( "vs2008", "vs2010", "vs2012", "vs2013", "vs2015" ).Contains( $env:TARGET ) ) {
       .\syncdokan.ps1 -UseLegacyVersion }

configure.ac

@@ -2,7 +2,7 @@ AC_PREREQ([2.71])
 
 AC_INIT(
  [libfsntfs],
- [20230930],
+ [20231007],
  [joachim.metz@gmail.com])
 
 AC_CONFIG_SRCDIR(

libfsntfs.nuspec

@@ -2,15 +2,15 @@
 <package >
   <metadata>
     <id>libfsntfs</id>
-    <version>20230930</version>
+    <version>20231007</version>
     <authors>Joachim Metz</authors>
     <owners>joachimmetz</owners>
     <license type="expression">LGPL-3.0-or-later</license>
     <projectUrl>https://github.com/libyal/libfsntfs</projectUrl>
     <requireLicenseAcceptance>false</requireLicenseAcceptance>
     <title>libfsntfs</title>
     <description>Library to access the Windows New Technology File System (NTFS) format</description>
-    <releaseNotes>Release of libfsntfs 20230930</releaseNotes>
+    <releaseNotes>Release of libfsntfs 20231007</releaseNotes>
     <copyright>Copyright (C) 2010-2023</copyright>
     <tags>native</tags>
   </metadata>

m4/libcrypto.m4

@@ -1,6 +1,6 @@
 dnl Checks for libcrypto required headers and functions
 dnl
-dnl Version: 20230701
+dnl Version: 20231007
 
 dnl Function to detect whether openssl/evp.h can be used in combination with zlib.h
 AC_DEFUN([AX_LIBCRYPTO_CHECK_OPENSSL_EVP_ZLIB_COMPATIBILE],
@@ -687,6 +687,10 @@ AC_DEFUN([AX_LIBCRYPTO_CHECK_LIB],
       ac_cv_libcrypto_CPPFLAGS="$openssl_CFLAGS"
       ac_cv_libcrypto_LIBADD="$openssl_LIBS"
 
+      AS_IF(
+        [test "x$ac_cv_libcrypto_LIBADD" = x],
+        [ac_cv_libcrypto_LIBADD="-lcrypto"])
+
       dnl On Cygwin also link zlib since libcrypto relies on it
       AS_CASE(
         [$host],

tests/fsntfs_test_libcpath.h

@@ -1,7 +1,7 @@
 /*
- * The internal libcpath header
+ * The libcpath header wrapper
  *
- * Copyright (C) 2010-2022, Joachim Metz <joachim.metz@gmail.com>
+ * Copyright (C) 2010-2023, Joachim Metz <joachim.metz@gmail.com>
  *
  * Refer to AUTHORS for acknowledgements.
  *

tests/runtests.py

@@ -2,29 +2,36 @@
 #
 # Script to run Python test scripts.
 #
-# Copyright (C) 2010-2023, Joachim Metz <joachim.metz@gmail.com>
-#
-# Refer to AUTHORS for acknowledgements.
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Lesser General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+# Version: 20231009
 
 import glob
 import os
 import sys
 import unittest
 
 
+test_profile = ".pyfsntfs"
+input_glob = "*"
+option_sets = ["offset"]
+
+
+def ReadIgnoreList(test_profile):
+  """Reads the test profile ignore file if it exists.
+
+  Args:
+    test_profile (str): test profile.
+
+  Returns:
+    set[str]: ignore list.
+  """
+  ignore_file_path = os.path.join("tests", "input", test_profile, "ignore")
+  if os.path.isfile(ignore_file_path):
+    with open(ignore_file_path, "r", encoding="utf-8") as file_object:
+      return set([line.strip() for line in file_object.readlines()])
+
+  return set()
+
+
 if __name__ == "__main__":
   print(f"Using Python version {sys.version!s}")
 
@@ -33,30 +40,38 @@
 
   test_scripts = test_loader.discover("tests", pattern="*.py")
 
-  test_profile = ".pyfsntfs"
-  input_glob = "*"
-
-  ignore_list = set()
-
-  ignore_file_path = f"tests/input/{test_profile}/ignore"
-  if os.path.isfile(ignore_file_path):
-    with open(ignore_file_path, "r", encoding="utf-8") as file_object:
-      ignore_list = set([line.strip() for line in file_object.readlines()])
+  ignore_list = ReadIgnoreList(test_profile)
 
+  test_set = None
   source_file = None
 
-  for test_set in glob.glob("tests/input/*"):
-    test_set = test_set.rsplit('/', maxsplit=1)[-1]
+  for test_set in glob.glob(os.path.join("tests", "input", "*")):
+    test_set = test_set.rsplit(os.path.sep, maxsplit=1)[-1]
     if not test_set or test_set[0] == '.' or test_set in ignore_list:
       continue
 
-    source_files = glob.glob(f"tests/input/{test_set:s}/{input_glob:s}")
+    source_files = glob.glob(os.path.join(
+        "tests", "input", test_set, input_glob))
     if source_files:
       source_file = source_files[0]
       break
 
   setattr(unittest, "source", source_file)
 
+  if source_file:
+    for option_set in option_sets:
+      test_file = os.path.basename(source_file)
+      test_options_file_path = os.path.join(
+          "tests", "input", test_profile, test_set,
+          f"{test_file:s}.{option_set:s}")
+      if os.path.isfile(test_options_file_path):
+        with open(test_options_file_path, "r", encoding="utf-8") as file_object:
+          lines = [line.strip() for line in file_object.readlines()]
+          if lines[0] == "# libyal test data options":
+            for line in lines[1:]:
+              key, value = line.split("=", maxsplit=1)
+              setattr(unittest, key, value)
+
   test_results = test_runner.run(test_scripts)
   if not test_results.wasSuccessful():
     sys.exit(1)

tests/test_fsntfsinfo.sh

@@ -1,15 +1,15 @@
 #!/usr/bin/env bash
 # Info tool testing script
 #
-# Version: 20230410
+# Version: 20231005
 
 EXIT_SUCCESS=0;
 EXIT_FAILURE=1;
 EXIT_IGNORE=77;
 
 PROFILES=("fsntfsinfo" "fsntfsinfo_fs" "fsntfsinfo_mft" "fsntfsinfo_usn");
 OPTIONS_PER_PROFILE=("" "-H" "-Eall" "-U");
-OPTION_SETS="offset";
+OPTION_SETS=("offset");
 
 INPUT_GLOB="*";
 
@@ -71,7 +71,7 @@ do
 
 	IGNORE_LIST=$(read_ignore_list "${TEST_PROFILE_DIRECTORY}");
 
-	IFS=" " read -a OPTIONS <<< ${OPTIONS_PER_PROFILE[${PROFILE_INDEX}]};
+	IFS=" " read -a PROFILE_OPTIONS <<< ${OPTIONS_PER_PROFILE[${PROFILE_INDEX}]};
 
 	RESULT=${EXIT_SUCCESS};
 
@@ -89,8 +89,49 @@ do
 		fi
 		TEST_SET_DIRECTORY=$(get_test_set_directory "${TEST_PROFILE_DIRECTORY}" "${TEST_SET_INPUT_DIRECTORY}");
 
-		run_test_on_test_set_with_options "${TEST_SET_DIRECTORY}" "fsntfsinfo" "with_stdout_reference" "${OPTION_SETS}" "${TEST_EXECUTABLE}" "${OPTIONS[@]}";
-		RESULT=$?;
+		RESULT=${EXIT_SUCCESS};
+
+		if test -f "${TEST_SET_DIRECTORY}/files";
+		then
+			IFS="" read -a INPUT_FILES <<< $(cat ${TEST_SET_DIRECTORY}/files | sed "s?^?${TEST_SET_INPUT_DIRECTORY}/?");
+		else
+			IFS="" read -a INPUT_FILES <<< $(ls -1d ${TEST_SET_INPUT_DIRECTORY}/${INPUT_GLOB});
+		fi
+		for INPUT_FILE in "${INPUT_FILES[@]}";
+		do
+			TESTED_WITH_OPTIONS=0;
+
+			for OPTION_SET in ${OPTION_SETS[@]};
+			do
+				TEST_DATA_OPTION_FILE=$(get_test_data_option_file "${TEST_SET_DIRECTORY}" "${INPUT_FILE}" "${OPTION_SET}");
+
+				if test -f ${TEST_DATA_OPTION_FILE};
+				then
+					TESTED_WITH_OPTIONS=1;
+
+					IFS=" " read -a OPTIONS <<< $(read_test_data_option_file "${TEST_SET_DIRECTORY}" "${INPUT_FILE}" "${OPTION_SET}");
+
+					run_test_on_input_file "${TEST_SET_DIRECTORY}" "fsntfsinfo" "with_stdout_reference" "${OPTION_SET}" "${TEST_EXECUTABLE}" "${INPUT_FILE}" "${PROFILE_OPTIONS[@]}" "${OPTIONS[@]}";
+					RESULT=$?;
+
+					if test ${RESULT} -ne ${EXIT_SUCCESS};
+					then
+						break;
+					fi
+				fi
+			done
+
+			if test ${TESTED_WITH_OPTIONS} -eq 0;
+			then
+				run_test_on_input_file "${TEST_SET_DIRECTORY}" "fsntfsinfo" "with_stdout_reference" "" "${TEST_EXECUTABLE}" "${INPUT_FILE}" "${PROFILE_OPTIONS[@]}";
+				RESULT=$?;
+			fi
+
+			if test ${RESULT} -ne ${EXIT_SUCCESS};
+			then
+				break;
+			fi
+		done
 
 		# Ignore failures due to corrupted data.
 		if test "${TEST_SET}" = "corrupted";

tests/test_fsntfsinfo_bodyfile.sh

@@ -1,15 +1,15 @@
 #!/usr/bin/env bash
 # Info tool testing script
 #
-# Version: 20230410
+# Version: 20231005
 
 EXIT_SUCCESS=0;
 EXIT_FAILURE=1;
 EXIT_IGNORE=77;
 
 PROFILES=("fsntfsinfo_bodyfile_fs" "fsntfsinfo_bodyfile_mft");
 OPTIONS_PER_PROFILE=("-Bbodyfile -H" "-Bbodyfile -Eall");
-OPTION_SETS="offset";
+OPTION_SETS=("offset");
 
 INPUT_GLOB="*";
 
@@ -107,7 +107,7 @@ do
 
 	IGNORE_LIST=$(read_ignore_list "${TEST_PROFILE_DIRECTORY}");
 
-	IFS=" " read -a OPTIONS <<< ${OPTIONS_PER_PROFILE[${PROFILE_INDEX}]};
+	IFS=" " read -a PROFILE_OPTIONS <<< ${OPTIONS_PER_PROFILE[${PROFILE_INDEX}]};
 
 	RESULT=${EXIT_SUCCESS};
 
@@ -125,8 +125,49 @@ do
 		fi
 		TEST_SET_DIRECTORY=$(get_test_set_directory "${TEST_PROFILE_DIRECTORY}" "${TEST_SET_INPUT_DIRECTORY}");
 
-		run_test_on_test_set_with_options "${TEST_SET_DIRECTORY}" "fsntfsinfo" "with_callback" "${OPTION_SETS}" "${TEST_EXECUTABLE}" "${OPTIONS[@]}";
-		RESULT=$?;
+		RESULT=${EXIT_SUCCESS};
+
+		if test -f "${TEST_SET_DIRECTORY}/files";
+		then
+			IFS="" read -a INPUT_FILES <<< $(cat ${TEST_SET_DIRECTORY}/files | sed "s?^?${TEST_SET_INPUT_DIRECTORY}/?");
+		else
+			IFS="" read -a INPUT_FILES <<< $(ls -1d ${TEST_SET_INPUT_DIRECTORY}/${INPUT_GLOB});
+		fi
+		for INPUT_FILE in "${INPUT_FILES[@]}";
+		do
+			TESTED_WITH_OPTIONS=0;
+
+			for OPTION_SET in ${OPTION_SETS[@]};
+			do
+				TEST_DATA_OPTION_FILE=$(get_test_data_option_file "${TEST_SET_DIRECTORY}" "${INPUT_FILE}" "${OPTION_SET}");
+
+				if test -f ${TEST_DATA_OPTION_FILE};
+				then
+					TESTED_WITH_OPTIONS=1;
+
+					IFS=" " read -a OPTIONS <<< $(read_test_data_option_file "${TEST_SET_DIRECTORY}" "${INPUT_FILE}" "${OPTION_SET}");
+
+					run_test_on_input_file "${TEST_SET_DIRECTORY}" "fsntfsinfo" "with_callback" "${OPTION_SET}" "${TEST_EXECUTABLE}" "${INPUT_FILE}" "${PROFILE_OPTIONS[@]}" "${OPTIONS[@]}";
+					RESULT=$?;
+
+					if test ${RESULT} -ne ${EXIT_SUCCESS};
+					then
+						break;
+					fi
+				fi
+			done
+
+			if test ${TESTED_WITH_OPTIONS} -eq 0;
+			then
+				run_test_on_input_file "${TEST_SET_DIRECTORY}" "fsntfsinfo" "with_callback" "" "${TEST_EXECUTABLE}" "${INPUT_FILE}" "${PROFILE_OPTIONS[@]}";
+				RESULT=$?;
+			fi
+
+			if test ${RESULT} -ne ${EXIT_SUCCESS};
+			then
+				break;
+			fi
+		done
 
 		# Ignore failures due to corrupted data.
 		if test "${TEST_SET}" = "corrupted";