Impose sorted order on the VCF data JSONs for more determinism

Signed-off-by: jakub-nt <175944085+jakub-nt@users.noreply.github.com>

cfbs/masterfiles/analyze.py

@@ -1,7 +1,7 @@
 # TODO merge this with ENT-12099 branch cfbs analyze.py
 import os
 
-from cfbs.utils import file_sha256
+from cfbs.utils import dict_sorted_by_key, file_sha256
 
 IGNORED_PATH_COMPONENTS = [".git/", ".gitignore", ".gitattributes"]
 # ignore a path iff it contains a component (single file or directory) from this list
@@ -52,3 +52,22 @@ def versions_checksums_files(
             )
 
     return versions_dict, checksums_dict, files_dict
+
+
+def finalize_vcf(versions_dict, checksums_dict, files_dict):
+    # sort checksums
+    sorted_checksums_dict = dict_sorted_by_key(checksums_dict["checksums"])
+    checksums_dict["checksums"] = sorted_checksums_dict
+
+    # sort files, alphabetically
+    sorted_files_dict = dict_sorted_by_key(files_dict["files"])
+    files_dict["files"] = sorted_files_dict
+
+    # sort files of each version
+    working_dict = versions_dict["versions"]
+    for k in working_dict.keys():
+        sorted_dict = dict_sorted_by_key(working_dict[k]["files"])
+        working_dict[k]["files"] = sorted_dict
+    versions_dict["versions"] = working_dict
+
+    return versions_dict, checksums_dict, files_dict

cfbs/masterfiles/generate_vcf_download.py

@@ -1,5 +1,9 @@
 from cfbs.utils import write_json
-from cfbs.masterfiles.analyze import initialize_vcf, versions_checksums_files
+from cfbs.masterfiles.analyze import (
+    finalize_vcf,
+    initialize_vcf,
+    versions_checksums_files,
+)
 
 
 def generate_vcf_download(dir_path, downloaded_versions):
@@ -16,6 +20,10 @@ def generate_vcf_download(dir_path, downloaded_versions):
             files_dir_path, version, versions_dict, checksums_dict, files_dict
         )
 
+    versions_dict, checksums_dict, files_dict = finalize_vcf(
+        versions_dict, checksums_dict, files_dict
+    )
+
     write_json("versions.json", versions_dict)
     write_json("checksums.json", checksums_dict)
     write_json("files.json", files_dict)

cfbs/masterfiles/generate_vcf_git_checkout.py

@@ -5,7 +5,11 @@
 
 from cfbs.git import git_exists
 from cfbs.utils import write_json
-from cfbs.masterfiles.analyze import initialize_vcf, versions_checksums_files
+from cfbs.masterfiles.analyze import (
+    finalize_vcf,
+    initialize_vcf,
+    versions_checksums_files,
+)
 
 DIR_PATH = "."
 """The path of the working directory."""
@@ -79,6 +83,10 @@ def generate_vcf_git_checkout(interesting_tags=None):
             MPF_PATH, tag, versions_dict, checksums_dict, files_dict
         )
 
+    versions_dict, checksums_dict, files_dict = finalize_vcf(
+        versions_dict, checksums_dict, files_dict
+    )
+
     write_json("versions-git.json", versions_dict)
     write_json("checksums-git.json", checksums_dict)
     write_json("files-git.json", files_dict)

cfbs/utils.py

@@ -236,6 +236,12 @@ def deduplicate_list(l):
     return list(OrderedDict.fromkeys(l))
 
 
+def dict_sorted_by_key(the_dict):
+    sorted_dict = OrderedDict(sorted(the_dict.items()))
+
+    return sorted_dict
+
+
 def cfbs_filename() -> str:
     return "cfbs.json"