Add hash entry parsing

* Also fix potential 404 errors on Ubuntu updates

.github/workflows/Linux.yml

@@ -71,7 +71,7 @@ jobs:
         echo '#define VERSION_STRING L"${{steps.set_version.outputs.version}}"' > version.h
 
     - name: Set up Linux environment
-      run: sudo apt install -y ${{ matrix.TARGET_PKGS }}
+      run: sudo apt-get update && sudo apt-get -y --no-install-recommends install ${{ matrix.TARGET_PKGS }}
 
     - name: Set up EDK2
       run: |

boot.c

@@ -103,6 +103,7 @@ EFI_STATUS EFIAPI efi_main(EFI_HANDLE BaseImageHandle, EFI_SYSTEM_TABLE *SystemT
 	EFI_LOADED_IMAGE_PROTOCOL* LoadedImage;
 	EFI_SIMPLE_FILE_SYSTEM_PROTOCOL* Volume;
 	EFI_FILE_HANDLE Root;
+	HASH_LIST HashList = { 0 };
 	INTN SecureBootStatus;
 #if defined(EFI_DEBUG)
 	UINTN Event;
@@ -154,7 +155,11 @@ EFI_STATUS EFIAPI efi_main(EFI_HANDLE BaseImageHandle, EFI_SYSTEM_TABLE *SystemT
 		goto out;
 	}
 
-	Status = Parse(Root, HASH_FILE);
+	Status = Parse(Root, HASH_FILE, &HashList);
+	if (EFI_ERROR(Status))
+		goto out;
+
+	Print(L"Found %d entries\n", HashList.Size);
 
 out:
 	// If running in test mode, close QEMU by invoking shutdown

boot.h

@@ -84,6 +84,9 @@ extern EFI_SYSTEM_TABLE*    MainSystemTable;
 /* Maximum size allowed for hash file we process */
 #define HASH_FILE_SIZE_MAX  (64 * 1024 * 1024)
 
+/* Maximum number of lines allowed in a hash file */
+#define HASH_FILE_LINES_MAX 100000
+
 /* Maximum size for the File Info structure we query */
 #define FILE_INFO_SIZE      (SIZE_OF_EFI_FILE_INFO + PATH_MAX * sizeof(CHAR16))
 
@@ -149,6 +152,31 @@ extern EFI_SYSTEM_TABLE*    MainSystemTable;
 #define COMPARE_GUID CompareGuid
 #endif
 
+/* Hash entry, comprised of the (hexascii) hash value and the path it applies to */
+typedef struct {
+	CHAR8*      Hash;
+	CHAR8*      Path;
+} HASH_ENTRY;
+
+/* Hash list of <Size> Hash entries */
+typedef struct {
+	HASH_ENTRY* Entry;
+	UINTN       Size;
+	CHAR8*      Buffer;
+} HASH_LIST;
+
+/* Check for a valid lowercase hex ASCII value */
+static __inline BOOLEAN IsValidHexAscii(CHAR8 c)
+{
+	return ((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f'));
+}
+
+/* Check for a valid whitespace character */
+static __inline BOOLEAN IsWhiteSpace(CHAR8 c)
+{
+	return (c == ' ' || c == '\t');
+}
+
 /*
  * Secure string length, that asserts if the string is NULL or if
  * the length is larger than a predetermined value (STRING_MAX)
@@ -194,6 +222,7 @@ extern INTN GetSecureBootStatus(VOID);
 
   @param[in]  Root   A file handle to the root directory.
   @param[in]  Path   A pointer to the CHAR16 string.
+  @param[out] List   A pointer to the HASH_LIST structure to populate.
 
   @retval EFI_SUCCESS           The file was successfully parsed and the hash list is populated.
   @retval EFI_INVALID_PARAMETER One or more of the input parameters are invalid.
@@ -202,4 +231,4 @@ extern INTN GetSecureBootStatus(VOID);
   @retval EFI_END_OF_FILE       The hash list file could not be read.
   @retval EFI_ABORTED           The hash list file contains invalid data.
 **/
-extern EFI_STATUS Parse(CONST EFI_FILE_HANDLE Root, CONST CHAR16* Path);
+extern EFI_STATUS Parse(CONST EFI_FILE_HANDLE Root, CONST CHAR16* Path, HASH_LIST* List);

parse.c

@@ -23,6 +23,7 @@
 
   @param[in]  Root   A file handle to the root directory.
   @param[in]  Path   A pointer to the CHAR16 string.
+  @param[out] List   A pointer to the HASH_LIST structure to populate.
 
   @retval EFI_SUCCESS           The file was successfully parsed and the hash list is populated.
   @retval EFI_INVALID_PARAMETER One or more of the input parameters are invalid.
@@ -31,15 +32,16 @@
   @retval EFI_END_OF_FILE       The hash list file could not be read.
   @retval EFI_ABORTED           The hash list file contains invalid data.
 **/
-EFI_STATUS Parse(CONST EFI_FILE_HANDLE Root, CONST CHAR16* Path)
+EFI_STATUS Parse(CONST EFI_FILE_HANDLE Root, CONST CHAR16* Path, HASH_LIST* List)
 {
 	EFI_STATUS Status;
 	EFI_FILE_HANDLE File;
 	EFI_FILE_INFO* Info = NULL;
 	CHAR8* HashFile = NULL;
-	UINTN i, Size, HashFileSize, NumLines;
+	HASH_ENTRY* HashList = NULL;
+	UINTN i, c, Size, HashFileSize, NumLines, HashListSize;
 
-	if (Root == NULL || Path == NULL)
+	if (Root == NULL || Path == NULL || List == NULL)
 		return EFI_INVALID_PARAMETER;
 
 	// Look for the hash file on the boot partition
@@ -94,23 +96,125 @@ EFI_STATUS Parse(CONST EFI_FILE_HANDLE Root, CONST CHAR16* Path)
 	// Compute the maximum number of lines/entries we need to allocate
 	NumLines = 1;	// We added a line break
 	for (i = 0; i < HashFileSize - 1; i++) {
-		if (HashFile[i] == 0x0A)
+		if (HashFile[i] == 0x0A) {
 			NumLines++;
-		if (HashFile[i] == 0x0D) {
+		} else if (HashFile[i] == 0x0D) {
 			// Convert to UNIX style
 			HashFile[i] = 0x0A;
 			// Don't double count lines with DOS style ending
 			if (HashFile[i + 1] != 0x0A)
 				NumLines++;
+		} else if (HashFile[i] < ' ' && HashFile[i] != '\t') {
+			// Do not allow any NUL or control characters besides TAB
+			Status = EFI_ABORTED;
+			PrintError(L"'%s' contains invalid data", HASH_FILE);
+			goto out;
 		}
 	}
 
-	Print(L"Read %d line(s)\n", NumLines);
+	// Don't allow files with more than a specific set of entries
+	if (NumLines > HASH_FILE_LINES_MAX) {
+		Status = EFI_UNSUPPORTED;
+		PrintError(L"'%s' contains too many lines", HASH_FILE);
+		goto out;
+	}
+
+	// Allocate a array of hash entries
+	HashList = AllocateZeroPool(NumLines * sizeof(HASH_ENTRY));
+	if (HashList == NULL) {
+		Status = EFI_OUT_OF_RESOURCES;
+		PrintError(L"Unable to allocate memory");
+		goto out;
+	}
+
+	// Now parse the file to populate the array
+	HashListSize = 0;
+	for (i = 0; i < HashFileSize; ) {
+		// Ignore whitespaces, control characters or anything non-ASCII
+		// (such as BOMs) that may precede a hash entry or a comment.
+		while ((HashFile[i] <= 0x20 || HashFile[i] >= 0x80) && i < HashFileSize)
+			i++;
+		if (i >= HashFileSize)
+			break;
+
+		// Ignore comments
+		if (HashFile[i] == '#') {
+			// Note that because we added a terminating 0x0A to the file,
+			// we cannot overflow on the while loop below.
+			while (HashFile[i++] != 0x0A);
+			continue;
+		}
+
+		// Check for a valid hash, which should be HASH_HEXASCII_SIZE
+		// hexascii followed by whitespace.
+		if (i + HASH_HEXASCII_SIZE >= HashFileSize ||
+			(!IsWhiteSpace(HashFile[i + HASH_HEXASCII_SIZE]))) {
+			HashFile[MIN(HashFileSize - 1, i + HASH_HEXASCII_SIZE)] = 0x00;
+			Status = EFI_ABORTED;
+			PrintError(L"Invalid hash data after '%a'", &HashFile[i]);
+			goto out;
+		}
+
+		// NUL-terminate the hash value, add it to our array and validate it
+		HashFile[i + HASH_HEXASCII_SIZE] = 0x00;
+		HashList[HashListSize].Hash = &HashFile[i];
+		for (; HashFile[i] != 0x00; i++) {
+			// Convert A-F to lowercase
+			if (HashFile[i] >= 'A' && HashFile[i] <= 'F')
+				HashFile[i] += 0x20;
+			if (!IsValidHexAscii(HashFile[i])) {
+				Status = EFI_ABORTED;
+				PrintError(L"Invalid hash data in '%a'", HashList[HashListSize].Hash);
+				goto out;
+			}
+		}
+
+		// Skip data between hash and path
+		while (++i < HashFileSize && HashFile[i] < 0x21) {
+			// Anything other than whitespace is illegal
+			if (!IsWhiteSpace(HashFile[i])) {
+				Status = EFI_ABORTED;
+				PrintError(L"Invalid hash data after '%a'", HashList[HashListSize].Hash);
+				goto out;
+			}
+		}
+
+		// Start of path value
+		c = i;
+		while (i < HashFileSize && HashFile[i] != 0x0A) {
+			if (HashFile[i] == '/') {
+				// Convert slashes to backslashes
+				HashFile[i] = '\\';
+			} else if (HashFile[i] < ' ') {
+				// Anything lower than space (including TAB) is illegal
+				i = c;
+				break;
+			}
+			i++;
+		}
+		// Check for a path parsing error above or an illegal path length
+		if (i == c || i > c + PATH_MAX) {
+			Status = EFI_ABORTED;
+			PrintError(L"Invalid path after '%a'", HashList[HashListSize].Hash);
+			goto out;
+		}
+		// NUL-terminate the path.
+		// Note that we can't overflow here since we added an extra 0x0A to our file.
+		HashFile[i++] = 0x00;
+		HashList[HashListSize].Path = &HashFile[c];
+		HashListSize++;
+	}
+
+	List->Size = HashListSize;
+	List->Buffer = HashFile;
+	List->Entry = HashList;
 
 out:
 	SafeFree(Info);
-	if (EFI_ERROR(Status))
+	if (EFI_ERROR(Status)) {
 		SafeFree(HashFile);
+		SafeFree(HashList);
+	}
 
 	return Status;
 }

test/004 Hash list min size.txt

@@ -1 +1 @@
-Read 1 line(s)
+Found 1 entries

test/006 Hash list too many lines.txt

@@ -0,0 +1 @@
+[FAIL] 'md5sum.txt' contains too many lines: [3] Unsupported

test/006 setup.sh

@@ -0,0 +1,2 @@
+#!/bin/env bash
+tr '\0' '\n' < /dev/zero | head -c 100001 > image/md5sum.txt

test/007 Garbage entry.txt

@@ -0,0 +1 @@
+[FAIL] Invalid hash data in 'This entry is invalid and should': [21] Aborted

test/007 setup.sh

@@ -0,0 +1,2 @@
+#!/bin/env bash
+echo "This entry is invalid and should fail" > image/md5sum.txt

test/008 Hash containing NUL.txt

@@ -0,0 +1 @@
+[FAIL] 'md5sum.txt' contains invalid data: [21] Aborted

test/008 setup.sh

@@ -0,0 +1,2 @@
+#!/bin/env bash
+echo "001122334455	6778899aabbccddeeff This hash contains a NUL" | tr '\11' '\0'  > image/md5sum.txt

test/009 Hash containing space.txt

@@ -0,0 +1 @@
+[FAIL] Invalid hash data in '001122334455 6778899aabbccddeeff': [21] Aborted

test/009 setup.sh

@@ -0,0 +1,2 @@
+#!/bin/env bash
+echo "001122334455 6778899aabbccddeeff This hash contains a space" > image/md5sum.txt

test/010 Hash containing an invalid character.txt

@@ -0,0 +1 @@
+[FAIL] Invalid hash data in '00112233445566778899azbbccddeeff': [21] Aborted

test/010 setup.sh

@@ -0,0 +1,2 @@
+#!/bin/env bash
+echo "00112233445566778899azbbccddeeff This hash contains an invalid character" > image/md5sum.txt

test/011 Hash too short.txt

@@ -0,0 +1 @@
+[FAIL] Invalid hash data in '00112233445566778899aabbccddeef ': [21] Aborted

test/011 setup.sh

@@ -0,0 +1,2 @@
+#!/bin/env bash
+echo "00112233445566778899aabbccddeef  Hash is too short"  > image/md5sum.txt

test/012 Hash too long.txt

@@ -0,0 +1 @@
+[FAIL] Invalid hash data after '00112233445566778899aabbccddeeff': [21] Aborted

test/012 setup.sh

@@ -0,0 +1,2 @@
+#!/bin/env bash
+echo "00112233445566778899aabbccddeeffa Hash is too long"  > image/md5sum.txt

test/013 Hash containing uppercase.txt

@@ -0,0 +1 @@
+Found 1 entries

test/013 setup.sh

@@ -0,0 +1,2 @@
+#!/bin/env bash
+echo "00112233445566778899AABBCCDDEEFF Uppercase Hash"  > image/md5sum.txt

test/014 Path containing TAB.txt

@@ -0,0 +1 @@
+[FAIL] Invalid path after '00112233445566778899aabbccddeeff': [21] Aborted

test/014 setup.sh

@@ -0,0 +1,2 @@
+#!/bin/env bash
+echo "00112233445566778899aabbccddeeff This file	name contains a TAB"  > image/md5sum.txt

test/015 Path containing NUL.txt

@@ -0,0 +1 @@
+[FAIL] 'md5sum.txt' contains invalid data: [21] Aborted

test/015 setup.sh

@@ -0,0 +1,2 @@
+#!/bin/env bash
+echo "00112233445566778899aabbccddeeff This	path contains a NUL" | tr '\11' '\0'  > image/md5sum.txt

test/016 Path too long.txt

@@ -0,0 +1 @@
+[FAIL] Invalid path after '00112233445566778899aabbccddeeff': [21] Aborted

test/016 setup.sh

@@ -0,0 +1,3 @@
+#!/bin/env bash
+echo -n "00112233445566778899aabbccddeeff " > image/md5sum.txt
+cat /dev/zero | tr '\0' 'a' | head -c 513 - >> image/md5sum.txt

test/017 Max Path size.txt

@@ -0,0 +1 @@
+Found 1 entries

test/017 setup.sh

@@ -0,0 +1,3 @@
+#!/bin/env bash
+echo -n "00112233445566778899aabbccddeeff " > image/md5sum.txt
+cat /dev/zero | tr '\0' 'a' | head -c 512 - >> image/md5sum.txt

test/018 Comment preceded by whitespaces.txt

@@ -0,0 +1 @@
+Found 1 entries

test/018 setup.sh

@@ -0,0 +1,3 @@
+#!/bin/env bash
+echo " 	# Comment preceded by whitespaces" > image/md5sum.txt
+echo "0123456789abcdef0123456789abcdef file1" >> image/md5sum.txt

test/019 Comment followed by whitespaces.txt

@@ -0,0 +1 @@
+Found 1 entries

test/019 setup.sh

@@ -0,0 +1,3 @@
+#!/bin/env bash
+echo "# Comment followed by whitespaces 	" > image/md5sum.txt
+echo "0123456789abcdef0123456789abcdef file1" >> image/md5sum.txt

test/020 Comment in the middle.txt

@@ -0,0 +1 @@
+Found 2 entries

test/020 setup.sh

@@ -0,0 +1,4 @@
+#!/bin/env bash
+echo "00112233445566778899aabbccddeeff file1" > image/md5sum.txt
+echo "# Comment in the middle" >> image/md5sum.txt
+echo "0123456789abcdef0123456789abcdef file2" >> image/md5sum.txt

test/021 Unterminated hash list.txt

@@ -0,0 +1 @@
+Found 2 entries

test/021 setup.sh

@@ -0,0 +1,3 @@
+#!/bin/env bash
+echo "00112233445566778899aabbccddeeff file1" > image/md5sum.txt
+echo -n "0123456789abcdef0123456789abcdef file2" >> image/md5sum.txt