chore: verify checksums during dist.ipfs.io sync (#8587)

* only copy files from dist.ipfs.io which have associated checksum files (also copy the checksum files)
* verify checksums before copying
* also, ignore path from sha512sum output, which sometimes has absolute path on dist.ipfs.io website

Co-authored-by: Marcin Rataj <lidel@lidel.org>

.github/workflows/sync-release-assets.yml

@@ -72,11 +72,9 @@ jobs:
               }
               num_synced += 1
 
-              const github_assets = [];
-              github_map = {};
+              const github_assets = new Set()
               for (const asset of release.assets) {
-                github_assets.push(asset.name);
-                github_map[asset.name] = true;
+                github_assets.add(asset.name)
               }
 
               // fetch asset info from dist.ipfs.io
@@ -90,38 +88,75 @@ jobs:
               }
               await exec.exec('ipfs', ['ls', p], options)
 
-              const dist_assets = []
-              missing_files = []
+              const dist_assets = new Set()
+              const missing_files = []
               for (const raw_line of stdout.split("\n")) {
                 line = raw_line.trim();
                 if (line.length != 0) {
                   file = line.split(/(\s+)/).filter( function(e) { return e.trim().length > 0; } )[2]
-                  dist_assets.push(file)
-                  if (!github_map[file]) {
+                  dist_assets.add(file)
+                  if (!github_assets.has(file)) {
                     missing_files.push(file)
                   }
                 }
               }
 
               // if dist.ipfs.io has files not found in github, copy them over
               for (const file of missing_files) {
+                file_sha = file + ".sha512"
+                file_cid = file + ".cid"
+
+                // skip files that don't have .cid and .sha512 checksum files
+                if (!dist_assets.has(file_sha) || !dist_assets.has(file_cid)) {
+                  if (!file.endsWith('.cid') && !file.endsWith('.sha512')) { // silent skip of .sha512.sha512 :)
+                    console.log(`skipping "${file}" as dist.ipfs.io does not provide .cid and .sha512 checksum files for it`)
+                  }
+                  continue
+                }
+
                 console.log("fetching", file, "from dist.ipfs.io")
                 await exec.exec('ipfs', ['get', p + '/' + file])
+                await exec.exec('ipfs', ['get', p + '/' + file_sha])
+                await exec.exec('ipfs', ['get', p + '/' + file_cid])
+                console.log("verifying contents of", file)
+
+                // compute sha512 output for file
+                let sha_stdout = ''
+                const sha_options = {}
+                sha_options.listeners = {
+                  stdout: (data) => {
+                    sha_stdout += data.toString();
+                  }
+                }
+                await exec.exec('sha512sum', [file], sha_options)
+                // read expected sha512 output
+                const sha_data = await fs.readFile(file_sha, "utf8")
+                const digest = (s) => s.split(' ').shift()
+                if (digest(sha_data) != digest(sha_stdout)) {
+                  console.log(`${file}.sha512: ${sha_data}`)
+                  console.log(`sha512sum ${file}: ${sha_stdout}`)
+                  throw "checksum verification failed for " + file
+                }
+
                 console.log("uploading", file, "to github release", release.tag_name)
-                resp = await github.repos.uploadReleaseAsset({
+                const uploadReleaseAsset = async (file) => github.repos.uploadReleaseAsset({
                   owner: context.repo.owner,
                   repo: context.repo.repo,
                   release_id: release.id,
                   headers: {
                     "content-type": "application/octet-stream",
                     "content-length": `${(await fs.stat(file)).size}`
-                    },
+                  },
                   name: file,
-                  data: await fs.readFile(file),
+                  data: await fs.readFile(file)
                 })
+                await uploadReleaseAsset(file)
+                await uploadReleaseAsset(file_sha)
+                await uploadReleaseAsset(file_cid)
+
               }
               // summary of assets on both sides
-              release_assets.push({ tag: release.tag_name, github_assets: github_assets, dist_assets: dist_assets })
+              release_assets.push({ tag: release.tag_name, github_assets, dist_assets })
             }
             console.log(release_assets)
             return release_assets