feat: cache and verify downloaded archive (#32)

* feat: cache and verify downloaded archive

- replaces node-fetch with got which simplifies code
  and is more robust on CI (retries twice before failing to download)
- adds caching of downloaded archives which should speed up CI
  of projects using this library (eg. ipfs-desktop)
- adds pre-commit hook to protect against commiting real binary to git
- downloads .sha512 manifest and compares it with sha512sum of
  downloaded archive

* test: ensure fetched version match dep version
* test(ci): run npm test on ci

.github/workflows/test.yml

@@ -0,0 +1,19 @@
+name: test
+
+on:
+  push:
+    branches: [ main, master ]
+  pull_request:
+    branches: [ main, master ]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/setup-node@v1
+      with:
+        node-version: 15.x
+    - run: npm install
+    - run: npm run build --if-present
+    - run: npm test

README.md

@@ -49,9 +49,13 @@ console.info('go-ipfs is installed at', path())
 
 An error will be thrown if the path to the binary cannot be resolved.
 
+### Caching
+
+Downloaded archives are placed in OS-specific cache directory which can be customized by setting `NPM_GO_IPFS_CACHE` in env.
+
 ## Development
 
-**Warning**: the file `bin/ipfs` is a placeholder, when downloading stuff, it gets replaced. so if you run `node install.js` it will then be dirty in the git repo. **Do not commit this file**, as then you would be commiting a big binary and publishing it to npm. (**TODO: add a pre-commit or pre-publish hook that warns about this**)
+**Warning**: the file `bin/ipfs` is a placeholder, when downloading stuff, it gets replaced. so if you run `node install.js` it will then be dirty in the git repo. **Do not commit this file**, as then you would be commiting a big binary and publishing it to npm. A pre-commit hook exists and should protect against this, but better safe than sorry.
 
 ### Publish a new version
 

package.json

@@ -5,9 +5,11 @@
   "main": "src/index.js",
   "scripts": {
     "postinstall": "node src/post-install.js",
+    "restore-bin": "git restore --source=HEAD --staged --worktree -- bin/ipfs",
     "test": "tape test/*.js | tap-spec",
     "lint": "standard"
   },
+  "pre-commit": "restore-bin",
   "bin": {
     "ipfs": "bin/ipfs"
   },
@@ -28,15 +30,18 @@
   "devDependencies": {
     "execa": "^4.0.1",
     "fs-extra": "^9.0.0",
+    "pre-commit": "^1.2.2",
     "standard": "^13.1.0",
     "tap-spec": "^5.0.0",
     "tape": "^4.13.2",
     "tape-promise": "^4.0.0"
   },
   "dependencies": {
+    "cachedir": "^2.3.0",
     "go-platform": "^1.0.0",
+    "got": "^11.7.0",
     "gunzip-maybe": "^1.4.2",
-    "node-fetch": "^2.6.0",
+    "hasha": "^5.2.2",
     "pkg-conf": "^3.1.0",
     "tar-fs": "^2.1.0",
     "unzip-stream": "^0.3.0"

src/download.js

@@ -15,16 +15,60 @@
 */
 const goenv = require('go-platform')
 const gunzip = require('gunzip-maybe')
+const got = require('got')
 const path = require('path')
 const tarFS = require('tar-fs')
 const unzip = require('unzip-stream')
-const fetch = require('node-fetch')
 const pkgConf = require('pkg-conf')
+const cachedir = require('cachedir')
 const pkg = require('../package.json')
 const fs = require('fs')
+const hasha = require('hasha')
 const cproc = require('child_process')
 const isWin = process.platform === 'win32'
 
+// avoid expensive fetch if file is already in cache
+async function cachingFetchAndVerify (url) {
+  const cacheDir = process.env.NPM_GO_IPFS_CACHE || cachedir('npm-go-ipfs')
+  const filename = url.split('/').pop()
+  const cachedFilePath = path.join(cacheDir, filename)
+  const cachedHashPath = `${cachedFilePath}.sha512`
+
+  if (!fs.existsSync(cacheDir)) {
+    fs.mkdirSync(cacheDir, { recursive: true })
+  }
+  if (!fs.existsSync(cachedFilePath)) {
+    console.info(`Downloading ${url} to ${cacheDir}`)
+    // download file
+    fs.writeFileSync(cachedFilePath, await got(url).buffer())
+    console.info(`Downloaded ${url}`)
+
+    // ..and checksum
+    console.info(`Downloading ${filename}.sha512`)
+    fs.writeFileSync(cachedHashPath, await got(`${url}.sha512`).buffer())
+    console.info(`Downloaded ${filename}.sha512`)
+  } else {
+    console.info(`Found ${cachedFilePath}`)
+  }
+
+  console.info(`Verifying ${filename}.sha512`)
+
+  const digest = Buffer.alloc(128)
+  const fd = fs.openSync(cachedHashPath, 'r')
+  fs.readSync(fd, digest, 0, digest.length, 0)
+  fs.closeSync(fd)
+  const expectedSha = digest.toString('utf8')
+  const calculatedSha = await hasha.fromFile(cachedFilePath, { encoding: 'hex', algorithm: 'sha512' })
+  if (calculatedSha !== expectedSha) {
+    console.log(`Expected   SHA512: ${expectedSha}`)
+    console.log(`Calculated SHA512: ${calculatedSha}`)
+    throw new Error(`SHA512 of ${cachedFilePath}' (${calculatedSha}) does not match expected value from ${cachedFilePath}.sha512 (${expectedSha})`)
+  }
+  console.log(`OK (${expectedSha})`)
+
+  return fs.createReadStream(cachedFilePath)
+}
+
 function unpack (url, installPath, stream) {
   return new Promise((resolve, reject) => {
     if (url.endsWith('.zip')) {
@@ -66,13 +110,8 @@ function cleanArguments (version, platform, arch, installPath) {
 }
 
 async function ensureVersion (version, distUrl) {
-  const res = await fetch(`${distUrl}/go-ipfs/versions`)
   console.info(`${distUrl}/go-ipfs/versions`)
-  if (!res.ok) {
-    throw new Error(`Unexpected status: ${res.status}`)
-  }
-
-  const versions = (await res.text()).trim().split('\n')
+  const versions = (await got(`${distUrl}/go-ipfs/versions`).text()).trim().split('\n')
 
   if (versions.indexOf(version) === -1) {
     throw new Error(`Version '${version}' not available`)
@@ -82,9 +121,7 @@ async function ensureVersion (version, distUrl) {
 async function getDownloadURL (version, platform, arch, distUrl) {
   await ensureVersion(version, distUrl)
 
-  const res = await fetch(`${distUrl}/go-ipfs/${version}/dist.json`)
-  if (!res.ok) throw new Error(`Unexpected status: ${res.status}`)
-  const data = await res.json()
+  const data = await got(`${distUrl}/go-ipfs/${version}/dist.json`).json()
 
   if (!data.platforms[platform]) {
     throw new Error(`No binary available for platform '${platform}'`)
@@ -100,19 +137,9 @@ async function getDownloadURL (version, platform, arch, distUrl) {
 
 async function download ({ version, platform, arch, installPath, distUrl }) {
   const url = await getDownloadURL(version, platform, arch, distUrl)
+  const data = await cachingFetchAndVerify(url)
 
-  console.info(`Downloading ${url}`)
-
-  const res = await fetch(url)
-
-  if (!res.ok) {
-    throw new Error(`Unexpected status: ${res.status}`)
-  }
-
-  console.info(`Downloaded ${url}`)
-
-  await unpack(url, installPath, res.body)
-
+  await unpack(url, installPath, data)
   console.info(`Unpacked ${installPath}`)
 
   return path.join(installPath, 'go-ipfs', `ipfs${platform === 'windows' ? '.exe' : ''}`)

test/fixtures/example-project/package.json

@@ -7,8 +7,5 @@
   "license": "ISC",
   "dependencies": {
     "go-ipfs": "file://../../../"
-  },
-  "go-ipfs": {
-    "version": "v0.4.20"
   }
 }

test/install.js

@@ -2,31 +2,43 @@ const fs = require('fs-extra')
 const path = require('path')
 const test = require('tape')
 const execa = require('execa')
+const cachedir = require('cachedir')
 
 /*
-  Test that go-ipfs is downloaded during npm install.
-  - package up the current source code with `npm pack`
-  - install the tarball into the example project
-  - ensure that the "go-ipfs.version" prop in the package.json is used
+  Test that correct go-ipfs is downloaded during npm install.
 */
 
-const testVersion = require('./fixtures/example-project/package.json')['go-ipfs'].version
+const expectedVersion = require('../package.json').version
 
 async function clean () {
   await fs.remove(path.join(__dirname, 'fixtures', 'example-project', 'node_modules'))
   await fs.remove(path.join(__dirname, 'fixtures', 'example-project', 'package-lock.json'))
+  await fs.remove(cachedir('npm-go-ipfs'))
 }
 
 test.onFinish(clean)
 
-test('Ensure go-ipfs.version defined in parent package.json is used', async (t) => {
+test('Ensure go-ipfs defined in package.json is fetched on dependency install', async (t) => {
   await clean()
 
+  const exampleProjectRoot = path.join(__dirname, 'fixtures', 'example-project')
+
   // from `example-project`, install the module
   const res = execa.sync('npm', ['install'], {
-    cwd: path.join(__dirname, 'fixtures', 'example-project')
+    cwd: exampleProjectRoot
+  })
+
+  // confirm package.json is correct
+  const fetchedVersion = require(path.join(exampleProjectRoot, 'node_modules', 'go-ipfs', 'package.json')).version
+  t.ok(expectedVersion === fetchedVersion, `package.json versions match '${expectedVersion}'`)
+
+  // confirm binary is correct
+  const binary = path.join(exampleProjectRoot, 'node_modules', 'go-ipfs', 'bin', 'ipfs')
+  const versionRes = execa.sync(binary, ['--version'], {
+    cwd: exampleProjectRoot
   })
-  const msg = `Downloading https://dist.ipfs.io/go-ipfs/${testVersion}`
-  t.ok(res.stdout.includes(msg), msg)
+
+  t.ok(versionRes.stdout === `ipfs version ${expectedVersion}`, `ipfs --version output match '${expectedVersion}'`)
+
   t.end()
 })