Merge pull request #214 from jakub-nt/ENT-12486-new-format

ENT-12486: Improve the release-information JSON file format

cfbs/masterfiles/analyze.py

@@ -23,27 +23,19 @@ def versions_checksums_files(
 
             if version not in versions_dict["versions"]:
                 versions_dict["versions"][version] = {}
-            if "files" not in versions_dict["versions"][version]:
-                versions_dict["versions"][version]["files"] = {}
-            versions_dict["versions"][version]["files"][tarball_relpath] = file_checksum
+            versions_dict["versions"][version][tarball_relpath] = file_checksum
 
             if not file_checksum in checksums_dict["checksums"]:
-                checksums_dict["checksums"][file_checksum] = []
-            checksums_dict["checksums"][file_checksum].append(
-                {
-                    "file": tarball_relpath,
-                    "version": version,
-                }
-            )
+                checksums_dict["checksums"][file_checksum] = {}
+            if not tarball_relpath in checksums_dict["checksums"][file_checksum]:
+                checksums_dict["checksums"][file_checksum][tarball_relpath] = []
+            checksums_dict["checksums"][file_checksum][tarball_relpath].append(version)
 
             if not tarball_relpath in files_dict["files"]:
-                files_dict["files"][tarball_relpath] = []
-            files_dict["files"][tarball_relpath].append(
-                {
-                    "checksum": file_checksum,
-                    "version": version,
-                }
-            )
+                files_dict["files"][tarball_relpath] = {}
+            if not file_checksum in files_dict["files"][tarball_relpath]:
+                files_dict["files"][tarball_relpath][file_checksum] = []
+            files_dict["files"][tarball_relpath][file_checksum].append(version)
 
     return versions_dict, checksums_dict, files_dict
 
@@ -53,37 +45,40 @@ def finalize_vcf(versions_dict, checksums_dict, files_dict):
 
     # checksums.json:
     working_dict = checksums_dict["checksums"]
-    # sort each list, first by version descending, then by filepath alphabetically
-    for k in working_dict.keys():
-        working_dict[k] = sorted(
-            working_dict[k],
-            key=lambda d: (
-                version_as_comparable_list_negated(d["version"]),
-                d["file"],
-            ),
-        )
+    for c in working_dict.keys():
+        for f in working_dict[c].keys():
+            # sort each version list, descending
+            working_dict[c][f] = sorted(
+                working_dict[c][f],
+                key=lambda v: version_as_comparable_list(v),
+                reverse=True,
+            )
+        # sort filepaths, alphabetically
+        working_dict[c] = dict_sorted_by_key(working_dict[c])
     # sort checksums
     checksums_dict["checksums"] = dict_sorted_by_key(working_dict)
 
     # files.json:
     working_dict = files_dict["files"]
     # sort each list, first by version descending, then by checksum
-    for k in working_dict.keys():
-        working_dict[k] = sorted(
-            working_dict[k],
-            key=lambda d: (
-                version_as_comparable_list_negated(d["version"]),
-                d["checksum"],
-            ),
-        )
+    for f in working_dict.keys():
+        for c in working_dict[f].keys():
+            # sort each version list, descending
+            working_dict[f][c] = sorted(
+                working_dict[f][c],
+                key=lambda v: version_as_comparable_list(v),
+                reverse=True,
+            )
+        # sort checksums
+        working_dict[f] = dict_sorted_by_key(working_dict[f])
     # sort files, alphabetically
     files_dict["files"] = dict_sorted_by_key(working_dict)
 
     # versions.json:
     working_dict = versions_dict["versions"]
     # sort files of each version
-    for k in working_dict.keys():
-        working_dict[k]["files"] = dict_sorted_by_key(working_dict[k]["files"])
+    for v in working_dict.keys():
+        working_dict[v] = dict_sorted_by_key(working_dict[v])
     # sort version numbers, in decreasing order
     versions_dict["versions"] = OrderedDict(
         sorted(

cfbs/masterfiles/check_download_matches_git.py

@@ -22,25 +22,25 @@ def check_download_matches_git(versions):
     differing_count = 0
 
     for version in versions:
-        download_version_dict = download_versions_dict["versions"][version]["files"]
-        git_version_dict = git_versions_dict["versions"][version]["files"]
+        dl_version_files_dict = download_versions_dict["versions"][version]
+        git_version_files_dict = git_versions_dict["versions"][version]
 
         # normalize downloaded version dictionary filepaths
         # necessary because the downloaded version and git version dictionaries have filepaths of different forms
         new_download_dict = {}
-        for key, value in download_version_dict.items():
+        for key, value in dl_version_files_dict.items():
             if key.startswith("masterfiles/"):
                 key = key[12:]
             new_download_dict[key] = value
-        download_version_dict = new_download_dict
+        dl_version_files_dict = new_download_dict
 
         version_diffs_dict = {}
         version_diffs_dict["files_only_in_downloads"] = []
         version_diffs_dict["files_only_in_git"] = []
         version_diffs_dict["files_with_different_content"] = []
 
         only_dl, only_git, value_diff = dict_diff(
-            download_version_dict, git_version_dict
+            dl_version_files_dict, git_version_files_dict
         )
 
         for filepath in only_dl: