feat: Pull from GitHub instead of npm

Fixes: #36

- fetch from ICU’s GitHub release instead of npm (ICU v50+)
- set env FULL_ICU_PREFER_NPM=1 to prefer npm instead
- add .eslint (as well as standard)
- for ICU 67 and following, fetch from icu4c-___-data-_.zip
- otherwise fetch from icu4c-src.zip for little endian
- otherwise, use npm as before
- work around ICU issue with data file name
  https://unicode-org.atlassian.net/browse/ICU-21764

.eslintrc.js

@@ -0,0 +1,17 @@
+module.exports = {
+  env: {
+    commonjs: true,
+    es6: true,
+    node: true
+  },
+  extends: 'standard',
+  globals: {
+    Atomics: 'readonly',
+    SharedArrayBuffer: 'readonly'
+  },
+  parserOptions: {
+    ecmaVersion: 2018
+  },
+  rules: {
+  }
+}

.github/workflows/lint.yml

@@ -3,7 +3,7 @@ name: Lint
 on: [push, pull_request]
 
 jobs:
-  build:
+  lint:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
@@ -12,3 +12,12 @@ jobs:
         node-version: 16
     - run: npm i
     - run: npm run lint
+  test:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/setup-node@v1
+      with:
+        node-version: 16
+    - run: npm i
+    - run: npm t

.gitignore

@@ -2,3 +2,5 @@ node_modules
 *.dat
 npm-debug.log
 /yarn.lock
+package-lock.json
+/.nyc_output

.npmignore

@@ -2,4 +2,5 @@
 node_modules
 .svn
 .git
-npm-debug.log
+npm-debug.log
+/test

.npmrc

@@ -0,0 +1 @@
+package-lock=false

.travis.yml

@@ -0,0 +1,14 @@
+language: node_js
+node_js:
+- '8'
+- '10'
+- '11'
+- '12'
+script:
+- npm install
+- npm t
+cache:
+  directories:
+  - node_modules
+  - ".nvm"
+# this is a comment

install-gh-data.js

@@ -0,0 +1,69 @@
+// Copyright (C) 2015-2016 IBM Corporation and Others. All Rights Reserved.
+
+// Install by fetching ICU source tarball
+// This will only work for little endian systems, but will work for ancient ICU (back to v50)
+
+const fs = require('fs')
+const { URL } = require('url')
+const process = require('process')
+const myFetch = require('./myFetch')
+const yauzl = require('yauzl')
+
+// var isglobal = process.env.npm_config_global === 'true';
+
+module.exports = async function installFromGithub (fullIcu, advice) {
+//   const icupkg = fullIcu.icupkg
+  const { icudat, icuend } = fullIcu
+  if (fs.existsSync(icudat)) {
+    console.log(` √ ${icudat} (exists)`)
+    return
+  }
+
+  // var cmdPath = nodePath = process.env.npm_node_execpath;
+
+  // var npmPath = process.env.npm_execpath;
+
+  // var args;
+  // https://github.com/unicode-org/icu/releases/download/release-51-3/icu4c-51_3-src.zip
+  const _baseUrl = process.env.FULL_ICU_BASEURL || 'https://github.com/unicode-org/icu/releases/'
+  const baseUrl = new URL(_baseUrl)
+  const versionsAsHyphen = fullIcu.icuver.replace(/\./g, '-')
+  // ICU v67/v68 use "68.1" and "67.1" in the filename instead of 68_1 and 69_1
+  // https://unicode-org.atlassian.net/browse/ICU-21764
+  // Can remove this conditional if the files are updated later.
+  const versionsAsUnderscore = (fullIcu.icumaj >= 69) ? fullIcu.icuver.replace(/\./g, '_') : fullIcu.icuver
+  const tag = `release-${versionsAsHyphen}`
+  const file = `icu4c-${versionsAsUnderscore}-data-bin-${icuend}.zip`
+  const fullUrl = new URL(`./download/${tag}/${file}`, baseUrl)
+  console.log(fullUrl.toString())
+  const [srcZip, tmpd] = await myFetch(fullUrl)
+
+  console.log(srcZip, tmpd)
+  // now, unpack it
+
+  console.log(`Looking for ${icudat}`)
+  return new Promise((resolve, reject) =>
+    yauzl.open(srcZip, { lazyEntries: true }, (err, zipfile) => {
+      if (err) return reject(err)
+      zipfile.readEntry()
+      zipfile.on('end', () => reject(Error(`Not found in zipfile: ${icudat}`)))
+      zipfile.on('entry', (entry) => {
+        if (entry.fileName.endsWith('/')) {
+          zipfile.readEntry()
+        } else if (entry.fileName.endsWith(icudat) || entry.fileName.endsWith('/' + icudat)) {
+          console.log('found ' + entry.fileName)
+          zipfile.openReadStream(entry, (err, readStream) => {
+            if (err) return reject(err)
+            // if entry.file
+            readStream.on('end', () => zipfile.readEntry())
+            const pipeOut = fs.createWriteStream(icudat)
+            readStream.pipe(pipeOut)
+            console.log(` √ ${icudat} (from ICU binary data tarball)`)
+            return resolve()
+          })
+        } else {
+          zipfile.readEntry() // get next
+        }
+      })
+    }))
+}

install-gh.js

@@ -0,0 +1,71 @@
+// Copyright (C) 2015-2016 IBM Corporation and Others. All Rights Reserved.
+
+// Install by fetching ICU source tarball
+// This will only work for little endian systems, but will work for ancient ICU (back to v50)
+
+const fs = require('fs')
+const { URL } = require('url')
+const process = require('process')
+const myFetch = require('./myFetch')
+const yauzl = require('yauzl')
+
+// var isglobal = process.env.npm_config_global === 'true';
+
+module.exports = async function installFromGithub (fullIcu, advice) {
+  const { icudat, icuend } = fullIcu
+
+  if (fs.existsSync(icudat)) {
+    console.log(` √ ${icudat} (exists)`)
+    return
+  }
+
+  if (icuend !== 'l') {
+    // Should not hit this, as versions 67 and prior are already in NPM
+    console.error('Warning: this method probably will fail, because the ICU source tarball only contains little endian data.')
+  }
+
+  // var cmdPath = nodePath = process.env.npm_node_execpath;
+
+  // var npmPath = process.env.npm_execpath;
+
+  // var args;
+  // https://github.com/unicode-org/icu/releases/download/release-51-3/icu4c-51_3-src.zip
+  const _baseUrl = process.env.FULL_ICU_BASEURL || 'https://github.com/unicode-org/icu/releases/'
+  const baseUrl = new URL(_baseUrl)
+  const versionsAsHyphen = fullIcu.icuver.replace(/\./g, '-')
+  const versionsAsUnderscore = fullIcu.icuver.replace(/\./g, '_')
+  const tag = `release-${versionsAsHyphen}`
+  const file = `icu4c-${versionsAsUnderscore}-src.zip`
+  const fullUrl = new URL(`./download/${tag}/${file}`, baseUrl)
+  console.log(fullUrl.toString())
+  const [srcZip, tmpd] = await myFetch(fullUrl)
+
+  console.log(srcZip, tmpd)
+  // now, unpack it
+
+  console.log(`Looking for ${icudat}`)
+  return new Promise((resolve, reject) =>
+    yauzl.open(srcZip, { lazyEntries: true }, (err, zipfile) => {
+      if (err) return reject(err)
+      zipfile.readEntry()
+      zipfile.on('end', () => reject(Error(`Not found in zipfile: ${icudat}`)))
+      zipfile.on('entry', (entry) => {
+        if (entry.fileName.endsWith('/')) {
+          zipfile.readEntry()
+        } else if (entry.fileName.endsWith('/' + icudat)) {
+          console.log('found ' + entry.fileName)
+          zipfile.openReadStream(entry, (err, readStream) => {
+            if (err) return reject(err)
+            // if entry.file
+            readStream.on('end', () => zipfile.readEntry())
+            const pipeOut = fs.createWriteStream(icudat)
+            readStream.pipe(pipeOut)
+            console.log(` √ ${icudat} (from ICU source tarball)`)
+            return resolve()
+          })
+        } else {
+          zipfile.readEntry() // get next
+        }
+      })
+    }))
+}

myFetch.js

@@ -0,0 +1,58 @@
+// Copyright (C) 2015-2016 IBM Corporation and Others. All Rights Reserved.
+
+const os = require('os')
+const path = require('path')
+const fs = require('fs')
+
+function getFetcher (u) {
+  if (u.protocol === 'https:') return require('https')
+  if (u.protocol === 'http:') return require('http')
+  return null
+}
+
+/**
+ * @param {URL} fullUrl url to fetch
+ * @returns {Promse<String[]>} filename, tmpdir
+ */
+function myFetch (fullUrl) {
+  return new Promise((resolve, reject) => {
+    const fetcher = getFetcher(fullUrl)
+    console.log('Fetch:', fullUrl.toString())
+    if (!fetcher) {
+      return reject(Error(`Unknown URL protocol ${fullUrl.protocol} in ${fullUrl.toString()}`))
+    }
+
+    fetcher.get(fullUrl, res => {
+      const length = res.headers['content-length']
+      if (res.statusCode === 302 && res.headers.location) {
+        return resolve(myFetch(new URL(res.headers.location)))
+      } else if (res.statusCode !== 200) {
+        return reject(Error(`Bad status code ${res.statusCode}`))
+      }
+      const tmpd = fs.mkdtempSync(os.tmpdir())
+      const tmpf = path.join(tmpd, 'icu-download.zip')
+      let gotSoFar = 0
+      console.dir(tmpd)
+
+      res.on('data', data => {
+        gotSoFar += data.length
+        fs.appendFileSync(tmpf, data)
+        // console.dir(res.headers);
+        process.stdout.write(`${gotSoFar}/${length}\r`)
+        // console.log(`chunk: ${data.length}`);
+      })
+      res.on('end', () => {
+        resolve([tmpf, tmpd])
+        console.log(`${gotSoFar}/${length}\n`)
+      })
+      res.on('error', error => {
+        fs.unlinkSync(tmpf)
+        fs.rmdirSync(tmpd)
+        console.error(error)
+        return reject(error)
+      })
+    })
+  })
+}
+
+module.exports = myFetch

package.json

@@ -3,8 +3,9 @@
   "version": "1.3.5-0",
   "description": "install 'full-icu' data for your current node",
   "scripts": {
-    "lint": "standard",
-    "postinstall": "node postinstall.js"
+    "lint": "standard && eslint *.js test/*.js",
+    "postinstall": "node postinstall.js",
+    "test": "tap test/*.js"
   },
   "keywords": [
     "icu4c"
@@ -23,7 +24,17 @@
   "bugs": {
     "url": "https://github.com/unicode-org/full-icu-npm/issues"
   },
+  "dependencies": {
+    "yauzl": "^2.10.0"
+  },
   "devDependencies": {
-    "standard": "^16.0.3"
+    "eslint": "^7.7.0",
+    "eslint-config-standard": "^16.0.3",
+    "eslint-plugin-header": "^3.0.0",
+    "eslint-plugin-import": "^2.24.2",
+    "eslint-plugin-node": "^11.1.0",
+    "eslint-plugin-promise": "^4.2.1",
+    "standard": "^16.0.3",
+    "tap": "^15.0.10"
   }
 }

postinstall.js

@@ -60,8 +60,29 @@ function advice () {
   console.log("... will show “enero”. If it shows “January” you don't have full data.")
 }
 
-// install by using spawn
-const npmInstall = require('./install-spawn')
+// Choose install method
+let npmInstall
+
+// GitHub has v50+ as releases
+// Experimentally, pull from GitHub for little endian
+if (!process.env.FULL_ICU_PREFER_NPM) {
+  if (fullIcu.icumaj >= 67) {
+    // Pull from bin data zip, first arrived in ICU v67
+    // https://unicode-org.atlassian.net/browse/ICU-20600
+    npmInstall = require('./install-gh-data')
+  } else {
+    if (fullIcu.icuend === 'l') {
+      // Little Endian can pull from icu4c-src.zip which contains a prebuilt data file
+      npmInstall = require('./install-gh')
+    } else {
+      // Fall back to npm
+      console.log(`ICU data bin zip not available until ICU v${fullIcu.icumaj} for endianness ${fullIcu.icuend}: Falling back to npm`)
+      npmInstall = require('./install-spawn')
+    }
+  }
+} else {
+  npmInstall = require('./install-spawn')
+}
 
 if (fs.existsSync(fullIcu.icudat)) {
   console.log('√ ' + fullIcu.icudat + ' Already there (for Node ' + fullIcu.nodever + ' and small-icu ' + fullIcu.icuver + ')')

test/test-unzipOne.js

@@ -0,0 +1,32 @@
+const tap = require('tap')
+const fs = require('fs')
+const unzipOne = require('../unzipOne')
+
+tap.test('unzipOne', async t => {
+  t.test('setup', t => {
+    try {
+      fs.unlinkSync('test/tmp/needle.txt')
+    } catch (e) { /* ignore */ }
+    t.end()
+  })
+  t.test('no easteregg in haystack.zip', async t => {
+    const ee = await unzipOne('./test/data/haystack.zip', 'easteregg.txt', './test/tmp/')
+    t.notOk(ee, 'Did not expect to find easteregg in haystack: ' + ee)
+    t.end()
+  })
+  t.test('get needle.txt in haystack.zip', async t => {
+    const ee = await unzipOne('./test/data/haystack.zip', 'needle.txt', './test/tmp/')
+    t.ok(ee, 'Did expect to find needle.txt in haystack: ' + ee)
+    const truism = fs.readFileSync('./test/tmp/needle.txt', 'utf-8')
+    t.ok(truism)
+    t.equal(truism.trim(), 'true')
+    t.end()
+  })
+  t.test('cleanup', t => {
+    try {
+      fs.unlinkSync('test/tmp/needle.txt')
+    } catch (e) { /* ignore */ }
+    t.end()
+  })
+  t.end()
+})