From 540a8f7d6299d2c0e03408997469974df39eba8b Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Sat, 24 Nov 2018 00:56:13 +1100 Subject: [PATCH 1/5] release: check_assets.js script for inspecting assets prior to promotion --- setup/www/tools/promote/check_assets.js | 463 ++++++++++++++++++ setup/www/tools/promote/expected_assets/v10.x | 40 ++ setup/www/tools/promote/expected_assets/v11.x | 40 ++ setup/www/tools/promote/expected_assets/v6.x | 46 ++ setup/www/tools/promote/expected_assets/v8.x | 44 ++ 5 files changed, 633 insertions(+) create mode 100755 setup/www/tools/promote/check_assets.js create mode 100644 setup/www/tools/promote/expected_assets/v10.x create mode 100644 setup/www/tools/promote/expected_assets/v11.x create mode 100644 setup/www/tools/promote/expected_assets/v6.x create mode 100644 setup/www/tools/promote/expected_assets/v8.x diff --git a/setup/www/tools/promote/check_assets.js b/setup/www/tools/promote/check_assets.js new file mode 100755 index 000000000..d12658308 --- /dev/null +++ b/setup/www/tools/promote/check_assets.js @@ -0,0 +1,463 @@ +#!/usr/bin/env node + +const fs = require('fs').promises +const path = require('path') +const childProcess = require('child_process') +const assert = require('assert') + +const versionRe = /^v\d+\.\d+\.\d+$/ + +const stagingDir = process.argv[2] +const distDir = process.argv[3] + +// `check_assets.js test` kicks it into test mode +if (stagingDir === 'test') { + test() +} else { + checkArgs().then(execute).catch(console.error) +} + +// must provide a valid staging directory that exists and a valid dist dir name +async function checkArgs () { + let bad = false + let dirStat + try { + dirStat = await fs.stat(stagingDir) + } catch (e) {} + + if (!dirStat || !dirStat.isDirectory()) { + bad = true + console.error('Staging directory not found') + } + + if (!versionRe.test(path.basename(stagingDir))) { + bad = true + console.error(`Bad staging directory name: ${stagingDir}`) + } + + if (!distDir || !versionRe.test(path.basename(distDir))) { + bad = true + console.error(`Bad dist directory name: ${distDir}`) + } + + if (bad) { + console.error('Usage: check_assets.js ') + process.exit(1) + } +} + +// equivalent to `find -maxdepth 2` +async function lsDepth2 (dir, isStaging) { + let assets = [] + + let subdirs + try { + subdirs = await ls(dir, isStaging) + } catch (e) { + return assets + } + await Promise.all(subdirs.map((dir) => ls(dir, false))) + assets.sort() + return assets + + // convenience fs.stat that preserves the original path across async calls + async function statPlus (file) { + const stat = await fs.stat(file) + return { stat: stat, path: file } + } + + // ls plus a stat on each file so we know if there are directories + // each entry is pushed to `assets` with directories having a trailing '/' + // return the directory entries in an array + async function ls (subdir, onlyDone) { + let files = await fs.readdir(subdir) + if (onlyDone) { + files = files.filter((f) => { + // include only files with an additional file by the same name with a + // .done extension, this is how staging is organised + return files.includes(`${f}.done`) + }) + } + let stats = await Promise.all(files.map((a) => statPlus(path.join(subdir, a)))) + stats.forEach((s) => { + assets.push(`${path.relative(dir, s.path)}${s.stat.isDirectory() ? '/' : ''}`) + }) + return stats.filter((s) => s.stat.isDirectory()).map((s) => s.path) + } +} + +function versionToLine (version) { + return version.replace(/^(v\d+)\.[\d.]+$/g, '$1.x') +} + +async function loadExpectedAssets (version, line) { + if (!line) { + line = versionToLine(version) + } + try { + const templateFile = path.join(__dirname, 'expected_assets', line) + let files = await fs.readFile(templateFile, 'utf8') + return files.replace(/{VERSION}/g, version).split(/\n/g) + } catch (e) { } + return null +} + +function intersection (a1, a2) { + return a1.filter((e) => a2.includes(e)) +} + +function union (a1, a2) { + let u = [].concat(a1) + a2.forEach((e) => { + if (!a1.includes(e)) { + u.push(e) + } + }) + return u +} + +async function execute () { + console.log('... Checking assets') + + let caution = false + const version = path.basename(stagingDir) + const line = versionToLine(version) + const lineI = parseInt(line.replace(/[v.x]/g, ''), 10) + let expectedLineI + + const stagingAssets = await lsDepth2(stagingDir, true) + const distAssets = await lsDepth2(distDir, false) + let expectedAssets + + // load expected asset list, use a prior version if one isn't available for this line + for (let i = 0; expectedAssets == null && i < 2; i++) { + expectedLineI = lineI - i + expectedAssets = await loadExpectedAssets(version, `v${expectedLineI}.x`) + } + + // No expected asset list available for this line, wut? + if (!expectedAssets) { + console.log(` \u001b[31m\u001b[1m✖\u001b[22m\u001b[39m No expected asset list is available for ${line}, does one need to be created?`) + console.log(` https://github.com/nodejs/build/tree/master/setup/www/tools/promote/expected_assets/${line}`) + process.exit(0) + } + + // Picked an older release line that did have an expected asset list, better warn + if (expectedLineI !== lineI) { + console.log(` \u001b[33m\u001b[1m⚠\u001b[22m\u001b[39m No expected asset list is available for ${line}, using the list for v${expectedLineI}.x instead`) + console.log(` Should a new list be created for ${line}?`) + console.log(` https://github.com/nodejs/build/tree/master/setup/www/tools/promote/expected_assets/${line}`) + } + + console.log(`... Expecting a total of ${expectedAssets.length} assets for ${line}`) + console.log(`... ${stagingAssets.length} assets waiting in staging`) + + // what might be overwritten by promotion? + const stagingDistIntersection = intersection(stagingAssets, distAssets) + if (stagingDistIntersection.length) { + caution = true + console.log(` \u001b[33m\u001b[1m⚠\u001b[22m\u001b[39m ${stagingDistIntersection.length} assets already promoted will be overwritten, is this OK?`) + if (stagingDistIntersection.length <= 10) { + stagingDistIntersection.forEach((a) => console.log(` • ${a}`)) + } + } else { + console.log(`... ${distAssets.length} assets already promoted`) + } + + const stagingDistUnion = union(stagingAssets, distAssets) + let notInActual = [] + let stagingNotInExpected = [] + + for (let asset of expectedAssets) { + if (!stagingDistUnion.includes(asset)) { + notInActual.push(asset) + } + } + + for (let asset of stagingAssets) { + if (!expectedAssets.includes(asset)) { + stagingNotInExpected.push(asset) + } + } + + if (!notInActual.length && !stagingNotInExpected.length) { // perfect staging state, we have everything we need + console.log(` \u001b[32m\u001b[1m✓\u001b[22m\u001b[39m Complete set of expected assets in place for ${line}`) + } else if (notInActual.length) { // missing some assets and they're not in staging, are you impatient? + caution = true + console.log(` \u001b[33m\u001b[1m⚠\u001b[22m\u001b[39m The following assets are expected for ${line} but are currently missing from staging:`) + notInActual.forEach((a) => console.log(` • ${a}`)) + } else if (stagingNotInExpected.length) { // bogus unexpected files found, not good + caution = true + console.log(` \u001b[31m\u001b[1m✖\u001b[22m\u001b[39m The following assets were found but are not expected for ${line}:`) + stagingNotInExpected.forEach((a) => console.log(` • ${a}`)) + console.log(` Does the expected assets list for ${line} need to be updated?`) + console.log(` https://github.com/nodejs/build/tree/master/setup/www/tools/promote/expected_assets/${line}`) + } + + if (caution) { + console.log(' \u001b[33mPromote if you are certain this is the the correct course of action\u001b[39m') + } +} + +/* TESTS ***********************************************************************/ + +function test () { + // makes a staging or dist directory given a version string + async function makeFixture (version, isStaging, dir, assets) { + if (!assets) { + assets = await loadExpectedAssets(version) + } + for (let asset of assets) { + let absoluteAsset = path.join(dir, asset) + if (asset.endsWith('/')) { + await fs.mkdir(absoluteAsset, { recursive: true }) + } else { + await fs.writeFile(absoluteAsset, asset, 'utf8') + } + let dashPos = isStaging && asset.indexOf('/') + if (isStaging && (dashPos === -1 || dashPos === asset.length - 1)) { + // in staging, all files and directories that are ready have a + // duplicate empty file with .done in the name + await fs.writeFile(absoluteAsset.replace(/\/?$/, '.done'), asset, 'utf8') + } + } + return assets + } + + async function rimraf (dir) { + const stat = await fs.stat(dir) + if (!stat.isDirectory()) { + await fs.unlink(dir) + return + } + const ls = await fs.readdir(dir) + for (let f of ls) { + f = path.join(dir, f) + const stat = await fs.stat(f) + if (stat.isDirectory()) { + await rimraf(f) + } else { + await fs.unlink(f) + } + } + await fs.rmdir(dir) + } + + // execute check_assets.js with the given staging and dist dirs, collect output + async function executeMe (stagingDir, distDir) { + return new Promise((resolve, reject) => { + const args = [ '--no-warnings', __filename, stagingDir, distDir ] + childProcess.execFile(process.execPath, args, (err, stdout, stderr) => { + if (err) { + return reject(err) + } + if (stderr) { + console.log('STDERR:', stderr) + } + resolve(stdout) + }) + }) + } + + // TEST 1: Everything is in staging, nothing in dist, good to go + async function test1 () { + const version = 'v8.13.0' + const testDir = await fs.mkdtemp(`${__filename}_`) + const fixtureStagingDir = path.join(testDir, `staging/${version}`) + const fixtureDistDir = path.join(testDir, `dist/${version}`) + await makeFixture(version, true, fixtureStagingDir) + const stdout = await executeMe(fixtureStagingDir, fixtureDistDir) + console.log('\nTest 1 Assertions ???') + console.log(`STDOUT------------------------------------------\n\n${stdout}\n------------------------------------------------`) + assert.equal(stdout, '... Checking assets\n' + + '... Expecting a total of 44 assets for v8.x\n' + + '... 44 assets waiting in staging\n' + + '... 0 assets already promoted\n' + + ' \u001b[32m\u001b[1m✓\u001b[22m\u001b[39m Complete set of expected assets in place for v8.x\n') + console.log('Test 1 Assertions ✓✓✓') + await rimraf(testDir) + } + + // TEST 2: Not quite everything is in staging, missing two assets, nothing in dist + async function test2 () { + const version = 'v10.1.0' + const testDir = await fs.mkdtemp(`${__filename}_`) + const fixtureStagingDir = path.join(testDir, `staging/${version}`) + const fixtureDistDir = path.join(testDir, `dist/${version}`) + await makeFixture(version, true, fixtureStagingDir) + await Promise.all([ + fs.unlink(path.join(fixtureStagingDir, 'node-v10.1.0-linux-armv6l.tar.gz')), + fs.unlink(path.join(fixtureStagingDir, 'node-v10.1.0-linux-armv6l.tar.gz.done')), + fs.unlink(path.join(fixtureStagingDir, 'node-v10.1.0-linux-armv6l.tar.xz')), + fs.unlink(path.join(fixtureStagingDir, 'node-v10.1.0-linux-armv6l.tar.xz.done')), + fs.unlink(path.join(fixtureStagingDir, 'node-v10.1.0.pkg')), + fs.unlink(path.join(fixtureStagingDir, 'node-v10.1.0.pkg.done')) + ]) + const stdout = await executeMe(fixtureStagingDir, fixtureDistDir) + console.log('\nTest 2 Assertions ???') + console.log(`STDOUT------------------------------------------\n\n${stdout}\n------------------------------------------------`) + assert.equal(stdout, '... Checking assets\n' + + '... Expecting a total of 40 assets for v10.x\n' + + '... 37 assets waiting in staging\n' + + '... 0 assets already promoted\n' + + ' \u001b[33m\u001b[1m⚠\u001b[22m\u001b[39m The following assets are expected for v10.x but are currently missing from staging:\n' + + ' • node-v10.1.0-linux-armv6l.tar.gz\n' + + ' • node-v10.1.0-linux-armv6l.tar.xz\n' + + ' • node-v10.1.0.pkg\n' + + ' \u001b[33mPromote if you are certain this is the the correct course of action\u001b[39m\n' + ) + console.log('Test 2 Assertions ✓✓✓') + await rimraf(testDir) + } + + // TEST 3: Everything in staging, 3 files in dist + async function test3 () { + const version = 'v6.0.1' + const testDir = await fs.mkdtemp(`${__filename}_`) + const fixtureStagingDir = path.join(testDir, `staging/${version}`) + const fixtureDistDir = path.join(testDir, `dist/${version}`) + await makeFixture(version, true, fixtureStagingDir) + const distAssets = await makeFixture(version, false, fixtureDistDir) + await Promise.all( + distAssets.filter((a) => { + return a !== 'node-v6.0.1-headers.tar.gz' && + a !== 'node-v6.0.1.tar.gz' && + a !== 'node-v6.0.1-x86.msi' && + // deleting all directories, so we don't need to delete the children + !/^win-x64\/.+/.test(a) && + !/^win-x86\/.+/.test(a) && + !/^docs\/.+/.test(a) + }).map((e) => rimraf(path.join(fixtureDistDir, e))) + ) + const stdout = await executeMe(fixtureStagingDir, fixtureDistDir) + console.log('\nTest 3 Assertions ???') + console.log(`STDOUT------------------------------------------\n\n${stdout}\n------------------------------------------------`) + assert.equal(stdout, '... Checking assets\n' + + '... Expecting a total of 46 assets for v6.x\n' + + '... 46 assets waiting in staging\n' + + ' \u001b[33m\u001b[1m⚠\u001b[22m\u001b[39m 3 assets already promoted will be overwritten, is this OK?\n' + + ' • node-v6.0.1-headers.tar.gz\n' + + ' • node-v6.0.1-x86.msi\n' + + ' • node-v6.0.1.tar.gz\n' + + ' \u001b[32m\u001b[1m✓\u001b[22m\u001b[39m Complete set of expected assets in place for v6.x\n' + + ' \u001b[33mPromote if you are certain this is the the correct course of action\u001b[39m\n') + console.log('Test 3 Assertions ✓✓✓') + await rimraf(testDir) + } + + // TEST 4: Everything in staging and everything in dist + async function test4 () { + const version = 'v11.11.11' + const testDir = await fs.mkdtemp(`${__filename}_`) + const fixtureStagingDir = path.join(testDir, `staging/${version}`) + const fixtureDistDir = path.join(testDir, `dist/${version}`) + await makeFixture(version, true, fixtureStagingDir) + await makeFixture(version, false, fixtureDistDir) + const stdout = await executeMe(fixtureStagingDir, fixtureDistDir) + console.log('\nTest 4 Assertions ???') + console.log(`STDOUT------------------------------------------\n\n${stdout}\n------------------------------------------------`) + assert.equal(stdout, '... Checking assets\n' + + '... Expecting a total of 40 assets for v11.x\n' + + '... 40 assets waiting in staging\n' + + ' \u001b[33m\u001b[1m⚠\u001b[22m\u001b[39m 40 assets already promoted will be overwritten, is this OK?\n' + + ' \u001b[32m\u001b[1m✓\u001b[22m\u001b[39m Complete set of expected assets in place for v11.x\n' + + ' \u001b[33mPromote if you are certain this is the the correct course of action\u001b[39m\n') + console.log('Test 4 Assertions ✓✓✓') + await rimraf(testDir) + } + + // TEST 5: No expected files list is available for this version, it has to guess + async function test5 () { + const version = 'v9.9.9' + const testDir = await fs.mkdtemp(`${__filename}_`) + const fixtureStagingDir = path.join(testDir, `staging/${version}`) + const fixtureDistDir = path.join(testDir, `dist/${version}`) + // use the 10.x list, which is missing the x86 files, it'll check the 9.x + const expectedAssets = await loadExpectedAssets(version, 'v10.x') + await makeFixture(version, true, fixtureStagingDir, expectedAssets) + const stdout = await executeMe(fixtureStagingDir, fixtureDistDir) + console.log('\nTest 5 Assertions ???') + console.log(`STDOUT------------------------------------------\n\n${stdout}\n------------------------------------------------`) + assert.equal(stdout, '... Checking assets\n' + + ' \u001b[33m\u001b[1m⚠\u001b[22m\u001b[39m No expected asset list is available for v9.x, using the list for v8.x instead\n' + + ' Should a new list be created for v9.x?\n' + + ' https://github.com/nodejs/build/tree/master/setup/www/tools/promote/expected_assets/v9.x\n' + + '... Expecting a total of 44 assets for v9.x\n' + + '... 40 assets waiting in staging\n' + + '... 0 assets already promoted\n' + + ' \u001b[33m\u001b[1m⚠\u001b[22m\u001b[39m The following assets are expected for v9.x but are currently missing from staging:\n' + + ' • node-v9.9.9-linux-x86.tar.gz\n' + + ' • node-v9.9.9-linux-x86.tar.xz\n' + + ' • node-v9.9.9-sunos-x86.tar.gz\n' + + ' • node-v9.9.9-sunos-x86.tar.xz\n' + + ' \u001b[33mPromote if you are certain this is the the correct course of action\u001b[39m\n') + console.log('Test 5 Assertions ✓✓✓') + await rimraf(testDir) + } + + // TEST 6: Everything is in dist except for the armv6l files, but they are in staging + async function test6 () { + const version = 'v10.0.0' + const testDir = await fs.mkdtemp(`${__filename}_`) + const fixtureStagingDir = path.join(testDir, `staging/${version}`) + const fixtureDistDir = path.join(testDir, `dist/${version}`) + await fs.mkdir(fixtureStagingDir, { recursive: true }) + await makeFixture(version, true, fixtureStagingDir, [ + 'node-v10.0.0-linux-armv6l.tar.gz', + 'node-v10.0.0-linux-armv6l.tar.xz' + ]) + await makeFixture(version, false, fixtureDistDir) + await Promise.all([ + fs.unlink(path.join(fixtureDistDir, 'node-v10.0.0-linux-armv6l.tar.gz')), + fs.unlink(path.join(fixtureDistDir, 'node-v10.0.0-linux-armv6l.tar.xz')) + ]) + const stdout = await executeMe(fixtureStagingDir, fixtureDistDir) + console.log('\nTest 6 Assertions ???') + console.log(`STDOUT------------------------------------------\n\n${stdout}\n------------------------------------------------`) + assert.equal(stdout, '... Checking assets\n' + + '... Expecting a total of 40 assets for v10.x\n' + + '... 2 assets waiting in staging\n' + + '... 38 assets already promoted\n' + + ' \u001b[32m\u001b[1m✓\u001b[22m\u001b[39m Complete set of expected assets in place for v10.x\n') + console.log('Test 6 Assertions ✓✓✓') + await rimraf(testDir) + } + + // TEST 7: Some unexpected files in staging + async function test7 () { + const version = 'v10.0.0' + const testDir = await fs.mkdtemp(`${__filename}_`) + const fixtureStagingDir = path.join(testDir, `staging/${version}`) + const fixtureDistDir = path.join(testDir, `dist/${version}`) + await fs.mkdir(fixtureStagingDir, { recursive: true }) + await makeFixture(version, true, fixtureStagingDir, [ + 'ooolaalaa.tar.gz', + 'whatdis.tar.xz' + ]) + await makeFixture(version, false, fixtureDistDir) + const stdout = await executeMe(fixtureStagingDir, fixtureDistDir) + console.log('\nTest 7 Assertions ???') + console.log(`STDOUT------------------------------------------\n\n${stdout}\n------------------------------------------------`) + assert.equal(stdout, '... Checking assets\n' + + '... Expecting a total of 40 assets for v10.x\n' + + '... 2 assets waiting in staging\n' + + '... 40 assets already promoted\n' + + ' \u001b[31m\u001b[1m✖\u001b[22m\u001b[39m The following assets were found but are not expected for v10.x:\n' + + ' • ooolaalaa.tar.gz\n' + + ' • whatdis.tar.xz\n' + + ' Does the expected assets list for v10.x need to be updated?\n' + + ' https://github.com/nodejs/build/tree/master/setup/www/tools/promote/expected_assets/v10.x\n' + + ' \u001b[33mPromote if you are certain this is the the correct course of action\u001b[39m\n') + console.log('Test 7 Assertions ✓✓✓') + await rimraf(testDir) + } + + // run in parallel and just print failures to stderr + test1().catch(console.error) + test2().catch(console.error) + test3().catch(console.error) + test4().catch(console.error) + test5().catch(console.error) + test6().catch(console.error) + test7().catch(console.error) +} diff --git a/setup/www/tools/promote/expected_assets/v10.x b/setup/www/tools/promote/expected_assets/v10.x new file mode 100644 index 000000000..97d3a0fa1 --- /dev/null +++ b/setup/www/tools/promote/expected_assets/v10.x @@ -0,0 +1,40 @@ +docs/ +docs/api/ +node-{VERSION}-aix-ppc64.tar.gz +node-{VERSION}-darwin-x64.tar.gz +node-{VERSION}-darwin-x64.tar.xz +node-{VERSION}-headers.tar.gz +node-{VERSION}-headers.tar.xz +node-{VERSION}-linux-arm64.tar.gz +node-{VERSION}-linux-arm64.tar.xz +node-{VERSION}-linux-armv6l.tar.gz +node-{VERSION}-linux-armv6l.tar.xz +node-{VERSION}-linux-armv7l.tar.gz +node-{VERSION}-linux-armv7l.tar.xz +node-{VERSION}-linux-ppc64le.tar.gz +node-{VERSION}-linux-ppc64le.tar.xz +node-{VERSION}-linux-s390x.tar.gz +node-{VERSION}-linux-s390x.tar.xz +node-{VERSION}-linux-x64.tar.gz +node-{VERSION}-linux-x64.tar.xz +node-{VERSION}.pkg +node-{VERSION}-sunos-x64.tar.gz +node-{VERSION}-sunos-x64.tar.xz +node-{VERSION}.tar.gz +node-{VERSION}.tar.xz +node-{VERSION}-win-x64.7z +node-{VERSION}-win-x64.zip +node-{VERSION}-win-x86.7z +node-{VERSION}-win-x86.zip +node-{VERSION}-x64.msi +node-{VERSION}-x86.msi +win-x64/ +win-x64/node.exe +win-x64/node.lib +win-x64/node_pdb.7z +win-x64/node_pdb.zip +win-x86/ +win-x86/node.exe +win-x86/node.lib +win-x86/node_pdb.7z +win-x86/node_pdb.zip \ No newline at end of file diff --git a/setup/www/tools/promote/expected_assets/v11.x b/setup/www/tools/promote/expected_assets/v11.x new file mode 100644 index 000000000..97d3a0fa1 --- /dev/null +++ b/setup/www/tools/promote/expected_assets/v11.x @@ -0,0 +1,40 @@ +docs/ +docs/api/ +node-{VERSION}-aix-ppc64.tar.gz +node-{VERSION}-darwin-x64.tar.gz +node-{VERSION}-darwin-x64.tar.xz +node-{VERSION}-headers.tar.gz +node-{VERSION}-headers.tar.xz +node-{VERSION}-linux-arm64.tar.gz +node-{VERSION}-linux-arm64.tar.xz +node-{VERSION}-linux-armv6l.tar.gz +node-{VERSION}-linux-armv6l.tar.xz +node-{VERSION}-linux-armv7l.tar.gz +node-{VERSION}-linux-armv7l.tar.xz +node-{VERSION}-linux-ppc64le.tar.gz +node-{VERSION}-linux-ppc64le.tar.xz +node-{VERSION}-linux-s390x.tar.gz +node-{VERSION}-linux-s390x.tar.xz +node-{VERSION}-linux-x64.tar.gz +node-{VERSION}-linux-x64.tar.xz +node-{VERSION}.pkg +node-{VERSION}-sunos-x64.tar.gz +node-{VERSION}-sunos-x64.tar.xz +node-{VERSION}.tar.gz +node-{VERSION}.tar.xz +node-{VERSION}-win-x64.7z +node-{VERSION}-win-x64.zip +node-{VERSION}-win-x86.7z +node-{VERSION}-win-x86.zip +node-{VERSION}-x64.msi +node-{VERSION}-x86.msi +win-x64/ +win-x64/node.exe +win-x64/node.lib +win-x64/node_pdb.7z +win-x64/node_pdb.zip +win-x86/ +win-x86/node.exe +win-x86/node.lib +win-x86/node_pdb.7z +win-x86/node_pdb.zip \ No newline at end of file diff --git a/setup/www/tools/promote/expected_assets/v6.x b/setup/www/tools/promote/expected_assets/v6.x new file mode 100644 index 000000000..957941824 --- /dev/null +++ b/setup/www/tools/promote/expected_assets/v6.x @@ -0,0 +1,46 @@ +docs/ +docs/api/ +node-{VERSION}-aix-ppc64.tar.gz +node-{VERSION}-darwin-x64.tar.gz +node-{VERSION}-darwin-x64.tar.xz +node-{VERSION}-headers.tar.gz +node-{VERSION}-headers.tar.xz +node-{VERSION}-linux-arm64.tar.gz +node-{VERSION}-linux-arm64.tar.xz +node-{VERSION}-linux-armv6l.tar.gz +node-{VERSION}-linux-armv6l.tar.xz +node-{VERSION}-linux-armv7l.tar.gz +node-{VERSION}-linux-armv7l.tar.xz +node-{VERSION}-linux-ppc64le.tar.gz +node-{VERSION}-linux-ppc64le.tar.xz +node-{VERSION}-linux-ppc64.tar.gz +node-{VERSION}-linux-ppc64.tar.xz +node-{VERSION}-linux-s390x.tar.gz +node-{VERSION}-linux-s390x.tar.xz +node-{VERSION}-linux-x64.tar.gz +node-{VERSION}-linux-x64.tar.xz +node-{VERSION}-linux-x86.tar.gz +node-{VERSION}-linux-x86.tar.xz +node-{VERSION}.pkg +node-{VERSION}-sunos-x64.tar.gz +node-{VERSION}-sunos-x64.tar.xz +node-{VERSION}-sunos-x86.tar.gz +node-{VERSION}-sunos-x86.tar.xz +node-{VERSION}.tar.gz +node-{VERSION}.tar.xz +node-{VERSION}-win-x64.7z +node-{VERSION}-win-x64.zip +node-{VERSION}-win-x86.7z +node-{VERSION}-win-x86.zip +node-{VERSION}-x64.msi +node-{VERSION}-x86.msi +win-x64/ +win-x64/node.exe +win-x64/node.lib +win-x64/node_pdb.7z +win-x64/node_pdb.zip +win-x86/ +win-x86/node.exe +win-x86/node.lib +win-x86/node_pdb.7z +win-x86/node_pdb.zip \ No newline at end of file diff --git a/setup/www/tools/promote/expected_assets/v8.x b/setup/www/tools/promote/expected_assets/v8.x new file mode 100644 index 000000000..e90dd2160 --- /dev/null +++ b/setup/www/tools/promote/expected_assets/v8.x @@ -0,0 +1,44 @@ +docs/ +docs/api/ +node-{VERSION}-aix-ppc64.tar.gz +node-{VERSION}-darwin-x64.tar.gz +node-{VERSION}-darwin-x64.tar.xz +node-{VERSION}-headers.tar.gz +node-{VERSION}-headers.tar.xz +node-{VERSION}-linux-arm64.tar.gz +node-{VERSION}-linux-arm64.tar.xz +node-{VERSION}-linux-armv6l.tar.gz +node-{VERSION}-linux-armv6l.tar.xz +node-{VERSION}-linux-armv7l.tar.gz +node-{VERSION}-linux-armv7l.tar.xz +node-{VERSION}-linux-ppc64le.tar.gz +node-{VERSION}-linux-ppc64le.tar.xz +node-{VERSION}-linux-s390x.tar.gz +node-{VERSION}-linux-s390x.tar.xz +node-{VERSION}-linux-x64.tar.gz +node-{VERSION}-linux-x64.tar.xz +node-{VERSION}-linux-x86.tar.gz +node-{VERSION}-linux-x86.tar.xz +node-{VERSION}.pkg +node-{VERSION}-sunos-x64.tar.gz +node-{VERSION}-sunos-x64.tar.xz +node-{VERSION}-sunos-x86.tar.gz +node-{VERSION}-sunos-x86.tar.xz +node-{VERSION}.tar.gz +node-{VERSION}.tar.xz +node-{VERSION}-win-x64.7z +node-{VERSION}-win-x64.zip +node-{VERSION}-win-x86.7z +node-{VERSION}-win-x86.zip +node-{VERSION}-x64.msi +node-{VERSION}-x86.msi +win-x64/ +win-x64/node.exe +win-x64/node.lib +win-x64/node_pdb.7z +win-x64/node_pdb.zip +win-x86/ +win-x86/node.exe +win-x86/node.lib +win-x86/node_pdb.7z +win-x86/node_pdb.zip \ No newline at end of file From 31c13b2a479f5a1ea29f2d9a5153eedf36eab0bb Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Sat, 24 Nov 2018 10:55:26 +1100 Subject: [PATCH 2/5] release: account for SHASUMS files, lots of refactoring --- setup/www/tools/promote/check_assets.js | 418 ++++++++++++++++-------- 1 file changed, 275 insertions(+), 143 deletions(-) diff --git a/setup/www/tools/promote/check_assets.js b/setup/www/tools/promote/check_assets.js index d12658308..0269f90f6 100755 --- a/setup/www/tools/promote/check_assets.js +++ b/setup/www/tools/promote/check_assets.js @@ -6,6 +6,11 @@ const childProcess = require('child_process') const assert = require('assert') const versionRe = /^v\d+\.\d+\.\d+$/ +const additionalDistAssets = [ + 'SHASUMS256.txt', + 'SHASUMS256.txt.asc', + 'SHASUMS256.txt.sig' +] const stagingDir = process.argv[2] const distDir = process.argv[3] @@ -120,13 +125,17 @@ async function execute () { console.log('... Checking assets') let caution = false + let update = false + const version = path.basename(stagingDir) const line = versionToLine(version) const lineI = parseInt(line.replace(/[v.x]/g, ''), 10) let expectedLineI + // load asset lists const stagingAssets = await lsDepth2(stagingDir, true) - const distAssets = await lsDepth2(distDir, false) + let distAssets = await lsDepth2(distDir, false) + distAssets = distAssets.filter((a) => !additionalDistAssets.includes(a)) let expectedAssets // load expected asset list, use a prior version if one isn't available for this line @@ -149,11 +158,17 @@ async function execute () { console.log(` https://github.com/nodejs/build/tree/master/setup/www/tools/promote/expected_assets/${line}`) } + // generate comparison lists + const stagingDistIntersection = intersection(stagingAssets, distAssets) + const stagingDistUnion = union(stagingAssets, distAssets) + let notInActual = expectedAssets.filter((a) => !stagingDistUnion.includes(a)) + let stagingNotInExpected = stagingAssets.filter((a) => !expectedAssets.includes(a)) + let distNotInExpected = distAssets.filter((a) => !expectedAssets.includes(a)) + console.log(`... Expecting a total of ${expectedAssets.length} assets for ${line}`) console.log(`... ${stagingAssets.length} assets waiting in staging`) // what might be overwritten by promotion? - const stagingDistIntersection = intersection(stagingAssets, distAssets) if (stagingDistIntersection.length) { caution = true console.log(` \u001b[33m\u001b[1m⚠\u001b[22m\u001b[39m ${stagingDistIntersection.length} assets already promoted will be overwritten, is this OK?`) @@ -164,36 +179,35 @@ async function execute () { console.log(`... ${distAssets.length} assets already promoted`) } - const stagingDistUnion = union(stagingAssets, distAssets) - let notInActual = [] - let stagingNotInExpected = [] - - for (let asset of expectedAssets) { - if (!stagingDistUnion.includes(asset)) { - notInActual.push(asset) - } - } - - for (let asset of stagingAssets) { - if (!expectedAssets.includes(asset)) { - stagingNotInExpected.push(asset) - } - } - - if (!notInActual.length && !stagingNotInExpected.length) { // perfect staging state, we have everything we need + if (!notInActual.length) { // perfect staging state, we have everything we need console.log(` \u001b[32m\u001b[1m✓\u001b[22m\u001b[39m Complete set of expected assets in place for ${line}`) - } else if (notInActual.length) { // missing some assets and they're not in staging, are you impatient? + } else { // missing some assets and they're not in staging, are you impatient? caution = true console.log(` \u001b[33m\u001b[1m⚠\u001b[22m\u001b[39m The following assets are expected for ${line} but are currently missing from staging:`) notInActual.forEach((a) => console.log(` • ${a}`)) - } else if (stagingNotInExpected.length) { // bogus unexpected files found, not good + } + + // bogus unexpected files found in staging, not good + if (stagingNotInExpected.length) { caution = true - console.log(` \u001b[31m\u001b[1m✖\u001b[22m\u001b[39m The following assets were found but are not expected for ${line}:`) + update = true + console.log(` \u001b[31m\u001b[1m✖\u001b[22m\u001b[39m The following assets were found in staging but are not expected for ${line}:`) stagingNotInExpected.forEach((a) => console.log(` • ${a}`)) + } + + // bogus unexpected files found in dist, not good + if (distNotInExpected.length) { + caution = true + update = true + console.log(` \u001b[31m\u001b[1m✖\u001b[22m\u001b[39m The following assets were already promoted but are not expected for ${line}:`) + distNotInExpected.forEach((a) => console.log(` • ${a}`)) + } + + // do we need to provide final notices? + if (update) { console.log(` Does the expected assets list for ${line} need to be updated?`) console.log(` https://github.com/nodejs/build/tree/master/setup/www/tools/promote/expected_assets/${line}`) } - if (caution) { console.log(' \u001b[33mPromote if you are certain this is the the correct course of action\u001b[39m') } @@ -259,44 +273,42 @@ function test () { }) } - // TEST 1: Everything is in staging, nothing in dist, good to go - async function test1 () { - const version = 'v8.13.0' + async function runTest (number, version, expectedStdout, setup) { const testDir = await fs.mkdtemp(`${__filename}_`) const fixtureStagingDir = path.join(testDir, `staging/${version}`) const fixtureDistDir = path.join(testDir, `dist/${version}`) - await makeFixture(version, true, fixtureStagingDir) + await setup(fixtureStagingDir, fixtureDistDir) + const stdout = await executeMe(fixtureStagingDir, fixtureDistDir) - console.log('\nTest 1 Assertions ???') + console.log(`\nTest ${number} Assertions ???`) console.log(`STDOUT------------------------------------------\n\n${stdout}\n------------------------------------------------`) - assert.equal(stdout, '... Checking assets\n' + + assert.equal(stdout, expectedStdout) + console.log(`Test ${number} Assertions ✓✓✓`) + + await rimraf(testDir) + } + // TEST 1: Everything is in staging, nothing in dist, good to go + async function test1 () { + const version = 'v8.13.0' + const expectedStdout = + '... Checking assets\n' + '... Expecting a total of 44 assets for v8.x\n' + '... 44 assets waiting in staging\n' + '... 0 assets already promoted\n' + - ' \u001b[32m\u001b[1m✓\u001b[22m\u001b[39m Complete set of expected assets in place for v8.x\n') - console.log('Test 1 Assertions ✓✓✓') - await rimraf(testDir) + ' \u001b[32m\u001b[1m✓\u001b[22m\u001b[39m Complete set of expected assets in place for v8.x\n' + + async function setup (fixtureStagingDir, fixtureDistDir) { + await makeFixture(version, true, fixtureStagingDir) + } + + return runTest(1, version, expectedStdout, setup) } // TEST 2: Not quite everything is in staging, missing two assets, nothing in dist async function test2 () { const version = 'v10.1.0' - const testDir = await fs.mkdtemp(`${__filename}_`) - const fixtureStagingDir = path.join(testDir, `staging/${version}`) - const fixtureDistDir = path.join(testDir, `dist/${version}`) - await makeFixture(version, true, fixtureStagingDir) - await Promise.all([ - fs.unlink(path.join(fixtureStagingDir, 'node-v10.1.0-linux-armv6l.tar.gz')), - fs.unlink(path.join(fixtureStagingDir, 'node-v10.1.0-linux-armv6l.tar.gz.done')), - fs.unlink(path.join(fixtureStagingDir, 'node-v10.1.0-linux-armv6l.tar.xz')), - fs.unlink(path.join(fixtureStagingDir, 'node-v10.1.0-linux-armv6l.tar.xz.done')), - fs.unlink(path.join(fixtureStagingDir, 'node-v10.1.0.pkg')), - fs.unlink(path.join(fixtureStagingDir, 'node-v10.1.0.pkg.done')) - ]) - const stdout = await executeMe(fixtureStagingDir, fixtureDistDir) - console.log('\nTest 2 Assertions ???') - console.log(`STDOUT------------------------------------------\n\n${stdout}\n------------------------------------------------`) - assert.equal(stdout, '... Checking assets\n' + + const expectedStdout = + '... Checking assets\n' + '... Expecting a total of 40 assets for v10.x\n' + '... 37 assets waiting in staging\n' + '... 0 assets already promoted\n' + @@ -305,34 +317,27 @@ function test () { ' • node-v10.1.0-linux-armv6l.tar.xz\n' + ' • node-v10.1.0.pkg\n' + ' \u001b[33mPromote if you are certain this is the the correct course of action\u001b[39m\n' - ) - console.log('Test 2 Assertions ✓✓✓') - await rimraf(testDir) + + async function setup (fixtureStagingDir, fixtureDistDir) { + await makeFixture(version, true, fixtureStagingDir) + await Promise.all([ + fs.unlink(path.join(fixtureStagingDir, 'node-v10.1.0-linux-armv6l.tar.gz')), + fs.unlink(path.join(fixtureStagingDir, 'node-v10.1.0-linux-armv6l.tar.gz.done')), + fs.unlink(path.join(fixtureStagingDir, 'node-v10.1.0-linux-armv6l.tar.xz')), + fs.unlink(path.join(fixtureStagingDir, 'node-v10.1.0-linux-armv6l.tar.xz.done')), + fs.unlink(path.join(fixtureStagingDir, 'node-v10.1.0.pkg')), + fs.unlink(path.join(fixtureStagingDir, 'node-v10.1.0.pkg.done')) + ]) + } + + return runTest(2, version, expectedStdout, setup) } // TEST 3: Everything in staging, 3 files in dist async function test3 () { const version = 'v6.0.1' - const testDir = await fs.mkdtemp(`${__filename}_`) - const fixtureStagingDir = path.join(testDir, `staging/${version}`) - const fixtureDistDir = path.join(testDir, `dist/${version}`) - await makeFixture(version, true, fixtureStagingDir) - const distAssets = await makeFixture(version, false, fixtureDistDir) - await Promise.all( - distAssets.filter((a) => { - return a !== 'node-v6.0.1-headers.tar.gz' && - a !== 'node-v6.0.1.tar.gz' && - a !== 'node-v6.0.1-x86.msi' && - // deleting all directories, so we don't need to delete the children - !/^win-x64\/.+/.test(a) && - !/^win-x86\/.+/.test(a) && - !/^docs\/.+/.test(a) - }).map((e) => rimraf(path.join(fixtureDistDir, e))) - ) - const stdout = await executeMe(fixtureStagingDir, fixtureDistDir) - console.log('\nTest 3 Assertions ???') - console.log(`STDOUT------------------------------------------\n\n${stdout}\n------------------------------------------------`) - assert.equal(stdout, '... Checking assets\n' + + const expectedStdout = + '... Checking assets\n' + '... Expecting a total of 46 assets for v6.x\n' + '... 46 assets waiting in staging\n' + ' \u001b[33m\u001b[1m⚠\u001b[22m\u001b[39m 3 assets already promoted will be overwritten, is this OK?\n' + @@ -340,45 +345,51 @@ function test () { ' • node-v6.0.1-x86.msi\n' + ' • node-v6.0.1.tar.gz\n' + ' \u001b[32m\u001b[1m✓\u001b[22m\u001b[39m Complete set of expected assets in place for v6.x\n' + - ' \u001b[33mPromote if you are certain this is the the correct course of action\u001b[39m\n') - console.log('Test 3 Assertions ✓✓✓') - await rimraf(testDir) + ' \u001b[33mPromote if you are certain this is the the correct course of action\u001b[39m\n' + + async function setup (fixtureStagingDir, fixtureDistDir) { + await makeFixture(version, true, fixtureStagingDir) + const distAssets = await makeFixture(version, false, fixtureDistDir) + await Promise.all( + distAssets.filter((a) => { + return a !== 'node-v6.0.1-headers.tar.gz' && + a !== 'node-v6.0.1.tar.gz' && + a !== 'node-v6.0.1-x86.msi' && + // deleting all directories, so we don't need to delete the children + !/^win-x64\/.+/.test(a) && + !/^win-x86\/.+/.test(a) && + !/^docs\/.+/.test(a) + }).map((e) => rimraf(path.join(fixtureDistDir, e))) + ) + } + + return runTest(3, version, expectedStdout, setup) } // TEST 4: Everything in staging and everything in dist async function test4 () { const version = 'v11.11.11' - const testDir = await fs.mkdtemp(`${__filename}_`) - const fixtureStagingDir = path.join(testDir, `staging/${version}`) - const fixtureDistDir = path.join(testDir, `dist/${version}`) - await makeFixture(version, true, fixtureStagingDir) - await makeFixture(version, false, fixtureDistDir) - const stdout = await executeMe(fixtureStagingDir, fixtureDistDir) - console.log('\nTest 4 Assertions ???') - console.log(`STDOUT------------------------------------------\n\n${stdout}\n------------------------------------------------`) - assert.equal(stdout, '... Checking assets\n' + + const expectedStdout = + '... Checking assets\n' + '... Expecting a total of 40 assets for v11.x\n' + '... 40 assets waiting in staging\n' + ' \u001b[33m\u001b[1m⚠\u001b[22m\u001b[39m 40 assets already promoted will be overwritten, is this OK?\n' + ' \u001b[32m\u001b[1m✓\u001b[22m\u001b[39m Complete set of expected assets in place for v11.x\n' + - ' \u001b[33mPromote if you are certain this is the the correct course of action\u001b[39m\n') - console.log('Test 4 Assertions ✓✓✓') - await rimraf(testDir) + ' \u001b[33mPromote if you are certain this is the the correct course of action\u001b[39m\n' + + async function setup (fixtureStagingDir, fixtureDistDir) { + await makeFixture(version, true, fixtureStagingDir) + await makeFixture(version, false, fixtureDistDir) + } + + return runTest(4, version, expectedStdout, setup) } // TEST 5: No expected files list is available for this version, it has to guess async function test5 () { const version = 'v9.9.9' - const testDir = await fs.mkdtemp(`${__filename}_`) - const fixtureStagingDir = path.join(testDir, `staging/${version}`) - const fixtureDistDir = path.join(testDir, `dist/${version}`) - // use the 10.x list, which is missing the x86 files, it'll check the 9.x - const expectedAssets = await loadExpectedAssets(version, 'v10.x') - await makeFixture(version, true, fixtureStagingDir, expectedAssets) - const stdout = await executeMe(fixtureStagingDir, fixtureDistDir) - console.log('\nTest 5 Assertions ???') - console.log(`STDOUT------------------------------------------\n\n${stdout}\n------------------------------------------------`) - assert.equal(stdout, '... Checking assets\n' + + const expectedStdout = + '... Checking assets\n' + ' \u001b[33m\u001b[1m⚠\u001b[22m\u001b[39m No expected asset list is available for v9.x, using the list for v8.x instead\n' + ' Should a new list be created for v9.x?\n' + ' https://github.com/nodejs/build/tree/master/setup/www/tools/promote/expected_assets/v9.x\n' + @@ -390,74 +401,195 @@ function test () { ' • node-v9.9.9-linux-x86.tar.xz\n' + ' • node-v9.9.9-sunos-x86.tar.gz\n' + ' • node-v9.9.9-sunos-x86.tar.xz\n' + - ' \u001b[33mPromote if you are certain this is the the correct course of action\u001b[39m\n') - console.log('Test 5 Assertions ✓✓✓') - await rimraf(testDir) + ' \u001b[33mPromote if you are certain this is the the correct course of action\u001b[39m\n' + + async function setup (fixtureStagingDir, fixtureDistDir) { + // use the 10.x list, which is missing the x86 files, it'll check the 9.x + const expectedAssets = await loadExpectedAssets(version, 'v10.x') + await makeFixture(version, true, fixtureStagingDir, expectedAssets) + } + + return runTest(5, version, expectedStdout, setup) } // TEST 6: Everything is in dist except for the armv6l files, but they are in staging async function test6 () { const version = 'v10.0.0' - const testDir = await fs.mkdtemp(`${__filename}_`) - const fixtureStagingDir = path.join(testDir, `staging/${version}`) - const fixtureDistDir = path.join(testDir, `dist/${version}`) - await fs.mkdir(fixtureStagingDir, { recursive: true }) - await makeFixture(version, true, fixtureStagingDir, [ - 'node-v10.0.0-linux-armv6l.tar.gz', - 'node-v10.0.0-linux-armv6l.tar.xz' - ]) - await makeFixture(version, false, fixtureDistDir) - await Promise.all([ - fs.unlink(path.join(fixtureDistDir, 'node-v10.0.0-linux-armv6l.tar.gz')), - fs.unlink(path.join(fixtureDistDir, 'node-v10.0.0-linux-armv6l.tar.xz')) - ]) - const stdout = await executeMe(fixtureStagingDir, fixtureDistDir) - console.log('\nTest 6 Assertions ???') - console.log(`STDOUT------------------------------------------\n\n${stdout}\n------------------------------------------------`) - assert.equal(stdout, '... Checking assets\n' + + const expectedStdout = + '... Checking assets\n' + '... Expecting a total of 40 assets for v10.x\n' + '... 2 assets waiting in staging\n' + '... 38 assets already promoted\n' + - ' \u001b[32m\u001b[1m✓\u001b[22m\u001b[39m Complete set of expected assets in place for v10.x\n') - console.log('Test 6 Assertions ✓✓✓') - await rimraf(testDir) + ' \u001b[32m\u001b[1m✓\u001b[22m\u001b[39m Complete set of expected assets in place for v10.x\n' + + async function setup (fixtureStagingDir, fixtureDistDir) { + await fs.mkdir(fixtureStagingDir, { recursive: true }) + await makeFixture(version, true, fixtureStagingDir, [ + 'node-v10.0.0-linux-armv6l.tar.gz', + 'node-v10.0.0-linux-armv6l.tar.xz' + ]) + await makeFixture(version, false, fixtureDistDir) + await Promise.all([ + fs.unlink(path.join(fixtureDistDir, 'node-v10.0.0-linux-armv6l.tar.gz')), + fs.unlink(path.join(fixtureDistDir, 'node-v10.0.0-linux-armv6l.tar.xz')) + ]) + } + + return runTest(6, version, expectedStdout, setup) } // TEST 7: Some unexpected files in staging async function test7 () { const version = 'v10.0.0' - const testDir = await fs.mkdtemp(`${__filename}_`) - const fixtureStagingDir = path.join(testDir, `staging/${version}`) - const fixtureDistDir = path.join(testDir, `dist/${version}`) - await fs.mkdir(fixtureStagingDir, { recursive: true }) - await makeFixture(version, true, fixtureStagingDir, [ - 'ooolaalaa.tar.gz', - 'whatdis.tar.xz' - ]) - await makeFixture(version, false, fixtureDistDir) - const stdout = await executeMe(fixtureStagingDir, fixtureDistDir) - console.log('\nTest 7 Assertions ???') - console.log(`STDOUT------------------------------------------\n\n${stdout}\n------------------------------------------------`) - assert.equal(stdout, '... Checking assets\n' + + const expectedStdout = + '... Checking assets\n' + '... Expecting a total of 40 assets for v10.x\n' + '... 2 assets waiting in staging\n' + '... 40 assets already promoted\n' + - ' \u001b[31m\u001b[1m✖\u001b[22m\u001b[39m The following assets were found but are not expected for v10.x:\n' + + ' \u001b[32m\u001b[1m✓\u001b[22m\u001b[39m Complete set of expected assets in place for v10.x\n' + + ' \u001b[31m\u001b[1m✖\u001b[22m\u001b[39m The following assets were found in staging but are not expected for v10.x:\n' + ' • ooolaalaa.tar.gz\n' + ' • whatdis.tar.xz\n' + ' Does the expected assets list for v10.x need to be updated?\n' + ' https://github.com/nodejs/build/tree/master/setup/www/tools/promote/expected_assets/v10.x\n' + - ' \u001b[33mPromote if you are certain this is the the correct course of action\u001b[39m\n') - console.log('Test 7 Assertions ✓✓✓') - await rimraf(testDir) + ' \u001b[33mPromote if you are certain this is the the correct course of action\u001b[39m\n' + + async function setup (fixtureStagingDir, fixtureDistDir) { + await fs.mkdir(fixtureStagingDir, { recursive: true }) + await makeFixture(version, true, fixtureStagingDir, [ + 'ooolaalaa.tar.gz', + 'whatdis.tar.xz' + ]) + await makeFixture(version, false, fixtureDistDir) + } + + return runTest(7, version, expectedStdout, setup) + } + + // TEST 8: Unexpected files in subdirectories, only check 2 levels deep + async function test8 () { + const version = 'v11.11.1' + const expectedStdout = + '... Checking assets\n' + + '... Expecting a total of 40 assets for v11.x\n' + + '... 42 assets waiting in staging\n' + + '... 0 assets already promoted\n' + + ' \u001b[32m\u001b[1m✓\u001b[22m\u001b[39m Complete set of expected assets in place for v11.x\n' + + ' \u001b[31m\u001b[1m✖\u001b[22m\u001b[39m The following assets were found in staging but are not expected for v11.x:\n' + + ' • docs/bar\n' + + ' • docs/foo\n' + + ' Does the expected assets list for v11.x need to be updated?\n' + + ' https://github.com/nodejs/build/tree/master/setup/www/tools/promote/expected_assets/v11.x\n' + + ' \u001b[33mPromote if you are certain this is the the correct course of action\u001b[39m\n' + + async function setup (fixtureStagingDir, fixtureDistDir) { + await fs.mkdir(fixtureStagingDir, { recursive: true }) + await makeFixture(version, true, fixtureStagingDir) + await Promise.all([ + fs.writeFile(path.join(fixtureStagingDir, 'docs/foo'), 'foo'), + fs.writeFile(path.join(fixtureStagingDir, 'docs/bar'), 'foo'), + fs.writeFile(path.join(fixtureStagingDir, 'docs/api/baz'), 'bar') + ]) + } + + return runTest(8, version, expectedStdout, setup) + } + + // TEST 9: Some unexpected files in dist + async function test9 () { + const version = 'v6.7.8' + const expectedStdout = + '... Checking assets\n' + + '... Expecting a total of 46 assets for v6.x\n' + + '... 46 assets waiting in staging\n' + + '... 2 assets already promoted\n' + + ' \u001b[32m\u001b[1m✓\u001b[22m\u001b[39m Complete set of expected assets in place for v6.x\n' + + ' \u001b[31m\u001b[1m✖\u001b[22m\u001b[39m The following assets were already promoted but are not expected for v6.x:\n' + + ' • ooolaalaa.tar.gz\n' + + ' • whatdis.tar.xz\n' + + ' Does the expected assets list for v6.x need to be updated?\n' + + ' https://github.com/nodejs/build/tree/master/setup/www/tools/promote/expected_assets/v6.x\n' + + ' \u001b[33mPromote if you are certain this is the the correct course of action\u001b[39m\n' + + async function setup (fixtureStagingDir, fixtureDistDir) { + await fs.mkdir(fixtureDistDir, { recursive: true }) + await makeFixture(version, false, fixtureDistDir, [ + 'ooolaalaa.tar.gz', + 'whatdis.tar.xz' + ]) + await makeFixture(version, true, fixtureStagingDir) + } + + return runTest(9, version, expectedStdout, setup) + } + + // TEST 10: SHASUMS in dist already, shouldn't bother us + async function test10 () { + const version = 'v8.0.0' + const expectedStdout = + '... Checking assets\n' + + '... Expecting a total of 44 assets for v8.x\n' + + '... 44 assets waiting in staging\n' + + ' \u001b[33m\u001b[1m⚠\u001b[22m\u001b[39m 4 assets already promoted will be overwritten, is this OK?\n' + + ' • docs/\n' + + ' • docs/api/\n' + + ' • node-v8.0.0-linux-armv6l.tar.gz\n' + + ' • node-v8.0.0-linux-armv6l.tar.xz\n' + + ' \u001b[32m\u001b[1m✓\u001b[22m\u001b[39m Complete set of expected assets in place for v8.x\n' + + ' \u001b[33mPromote if you are certain this is the the correct course of action\u001b[39m\n' + + async function setup (fixtureStagingDir, fixtureDistDir) { + await fs.mkdir(fixtureDistDir, { recursive: true }) + await Promise.all([ + makeFixture(version, true, fixtureStagingDir), + makeFixture(version, false, fixtureDistDir, [ + 'SHASUMS256.txt', + 'SHASUMS256.txt.asc', + 'SHASUMS256.txt.sig', + 'docs/', + 'docs/api/', + 'docs/api/bar', + 'node-v8.0.0-linux-armv6l.tar.gz', + 'node-v8.0.0-linux-armv6l.tar.xz' + ]) + ]) + } + + return runTest(10, version, expectedStdout, setup) } // run in parallel and just print failures to stderr - test1().catch(console.error) - test2().catch(console.error) - test3().catch(console.error) - test4().catch(console.error) - test5().catch(console.error) - test6().catch(console.error) - test7().catch(console.error) + let failures = 0 + let passes = 0 + const tests = [ + test1, + test2, + test3, + test4, + test5, + test6, + test7, + test8, + test9, + test10 + ] + + tests.forEach((test) => { + test() + .then(() => passes++) + .catch((err) => { + failures++ + console.error(err) + }) + }) + + process.on('exit', () => { + if (failures) { + console.error(`\n\u001b[31mThere were ${failures} test failures\u001b[39m`) + } else if (passes === tests.length) { + console.error('\n\u001b[32mAll tests have passed!\u001b[39m') + } else { + console.error(`\n\u001b[31mNot all tests seem to have run, something\'s wrong\u001b[39m`) + } + }) } From 3c4752f4ca9e8511bba05ceb698fad1499c29e03 Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Tue, 27 Nov 2018 17:02:30 +1100 Subject: [PATCH 3/5] fixup! release: account for SHASUMS files, lots of refactoring --- setup/www/tools/promote/check_assets.js | 33 ++++++++++++++----------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/setup/www/tools/promote/check_assets.js b/setup/www/tools/promote/check_assets.js index 0269f90f6..2d178d20c 100755 --- a/setup/www/tools/promote/check_assets.js +++ b/setup/www/tools/promote/check_assets.js @@ -26,23 +26,28 @@ if (stagingDir === 'test') { async function checkArgs () { let bad = false let dirStat - try { - dirStat = await fs.stat(stagingDir) - } catch (e) {} - if (!dirStat || !dirStat.isDirectory()) { - bad = true - console.error('Staging directory not found') - } + if (stagingDir && distDir) { + try { + dirStat = await fs.stat(stagingDir) + } catch (e) {} - if (!versionRe.test(path.basename(stagingDir))) { - bad = true - console.error(`Bad staging directory name: ${stagingDir}`) - } + if (!dirStat || !dirStat.isDirectory()) { + bad = true + console.error('Staging directory not found') + } + + if (!versionRe.test(path.basename(stagingDir))) { + bad = true + console.error(`Bad staging directory name: ${stagingDir}`) + } - if (!distDir || !versionRe.test(path.basename(distDir))) { + if (!distDir || !versionRe.test(path.basename(distDir))) { + bad = true + console.error(`Bad dist directory name: ${distDir}`) + } + } else { bad = true - console.error(`Bad dist directory name: ${distDir}`) } if (bad) { @@ -589,7 +594,7 @@ function test () { } else if (passes === tests.length) { console.error('\n\u001b[32mAll tests have passed!\u001b[39m') } else { - console.error(`\n\u001b[31mNot all tests seem to have run, something\'s wrong\u001b[39m`) + console.error(`\n\u001b[31mNot all tests seem to have run, something's wrong\u001b[39m`) } }) } From 79092c539a20257ccb48fa61a06d9de882c1a823 Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Tue, 27 Nov 2018 17:03:04 +1100 Subject: [PATCH 4/5] fixup! release: account for SHASUMS files, lots of refactoring --- setup/www/tools/promote/check_assets.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/www/tools/promote/check_assets.js b/setup/www/tools/promote/check_assets.js index 2d178d20c..4f144297a 100755 --- a/setup/www/tools/promote/check_assets.js +++ b/setup/www/tools/promote/check_assets.js @@ -51,7 +51,7 @@ async function checkArgs () { } if (bad) { - console.error('Usage: check_assets.js ') + console.error('Usage: check_assets.js ') process.exit(1) } } From 8f61e8e775a2e2a1555b97adf9b0fdf2e4a0327c Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Tue, 27 Nov 2018 20:18:14 +1100 Subject: [PATCH 5/5] release: integrate asset checker into promotion script --- setup/www/tools/promote/README.md | 0 setup/www/tools/promote/promote_nightly.sh | 12 ++++++------ setup/www/tools/promote/promote_release.sh | 21 ++++++++++++++++++--- 3 files changed, 24 insertions(+), 9 deletions(-) mode change 100644 => 100755 setup/www/tools/promote/README.md diff --git a/setup/www/tools/promote/README.md b/setup/www/tools/promote/README.md old mode 100644 new mode 100755 diff --git a/setup/www/tools/promote/promote_nightly.sh b/setup/www/tools/promote/promote_nightly.sh index a50c5cd74..a19454ae2 100755 --- a/setup/www/tools/promote/promote_nightly.sh +++ b/setup/www/tools/promote/promote_nightly.sh @@ -37,6 +37,12 @@ dirmatch=$test_dirmatch . ${__dirname}/_promote.sh $site +srcdir=$v8_canary_srcdir +dstdir=$v8_canary_dstdir +dirmatch=$v8_canary_dirmatch + +. ${__dirname}/_promote.sh $site + srcdir=$chakracore_nightly_srcdir dstdir=$chakracore_nightly_dstdir dirmatch=$chakracore_nightly_dirmatch @@ -54,9 +60,3 @@ dstdir=$chakracore_release_dstdir dirmatch=$chakracore_release_dirmatch . ${__dirname}/_promote.sh $site - -srcdir=$v8_canary_srcdir -dstdir=$v8_canary_dstdir -dirmatch=$v8_canary_dirmatch - -. ${__dirname}/_promote.sh $site diff --git a/setup/www/tools/promote/promote_release.sh b/setup/www/tools/promote/promote_release.sh index dca2e8eb0..f720985f9 100755 --- a/setup/www/tools/promote/promote_release.sh +++ b/setup/www/tools/promote/promote_release.sh @@ -22,6 +22,21 @@ srcdir=$release_srcdir dstdir=$release_dstdir dirmatch=$release_dirmatch -. ${__dirname}/_promote.sh $site $2 - -nodejs-latest-linker /home/dist/${site}/release/ /home/dist/${site}/docs/ +node --no-warnings /home/staging/tools/promote/check_assets.js $srcdir/$2 $dstdir/$2 + +while true; do + echo -n "Are you sure you want to promote the $2 assets? [y/n] " + yorn="" + read yorn + + if [ "X${yorn}" == "Xn" ]; then + echo "Bailing out ..." + exit 1 + fi + + if [ "X${yorn}" == "Xy" ]; then + . ${__dirname}/_promote.sh $site $2 + nodejs-latest-linker /home/dist/${site}/release/ /home/dist/${site}/docs/ + break + fi +done