Added retry logic for all git operations.

In the complex CI environments, intermittent issues happen. In order to prevent build failures, it is good to give a server an opportunity to recover from high load spikes.

Put in the ~/.gitconfig following section:
[http]
        proxy = http://192.168.1.101:8888

As you don't have a proxy running on the port you will get a timeout error from git.

lib/util/git.js

@@ -12,6 +12,7 @@ const optCheck = require('./opt-check')
 const osenv = require('osenv')
 const path = require('path')
 const pinflight = require('promise-inflight')
+const promiseRetry = require('promise-retry')
 const uniqueFilename = require('unique-filename')
 const which = BB.promisify(require('which'))
 const semver = require('semver')
@@ -26,6 +27,23 @@ const GOOD_ENV_VARS = new Set([
  'GIT_SSL_NO_VERIFY'
 ])
 
+const GIT_TRANSIENT_ERRORS = [
+ 'remote error: Internal Server Error',
+ 'fatal: Couldn\'t find remote ref ',
+ 'The remote end hung up unexpectedly',
+ 'Connection timed out',
+ 'Operation timed out',
+ 'Failed to connect to .* Timed out',
+ 'Connection reset by peer',
+ 'Couldn\'t resolve host'
+].join('|')
+
+const GIT_TRANSIENT_ERROR_RE = new RegExp(GIT_TRANSIENT_ERRORS)
+
+function shouldRetry(error) {
+ return GIT_TRANSIENT_ERROR_RE.test(error);
+}
+
 const GIT_ = 'GIT_'
 let GITENV
 function gitEnv () {
@@ -114,57 +132,51 @@ function revs (repo, opts) {
  return pinflight(`ls-remote:${repo}`, () => {
  return spawnGit(['ls-remote', '-h', '-t', repo], {
  env: gitEnv()
- }, opts).then(child => {
- let stdout = ''
- let stderr = ''
- child.stdout.on('data', d => { stdout += d })
- child.stderr.on('data', d => { stderr += d })
- return finished(child).catch(err => {
- err.message = `Error while executing:\n${GITPATH} ls-remote -h -t ${repo}\n\n${stderr}\n${err.message}`
- throw err
- }).then(() => {
- return stdout.split('\n').reduce((revs, line) => {
- const split = line.split(/\s+/, 2)
- if (split.length < 2) { return revs }
- const sha = split[0].trim()
- const ref = split[1].trim().match(/(?:refs\/[^/]+\/)?(.*)/)[1]
- if (!ref) { return revs } // ???
- if (ref.endsWith(CARET_BRACES)) { return revs } // refs/tags/x^{} crap
- const type = refType(line)
- const doc = {sha, ref, type}
-
- revs.refs[ref] = doc
- // We can check out shallow clones on specific SHAs if we have a ref
- if (revs.shas[sha]) {
- revs.shas[sha].push(ref)
- } else {
- revs.shas[sha] = [ref]
- }
+ }, opts).then((stdout) => {
+ return stdout.split('\n').reduce((revs, line) => {
+ const split = line.split(/\s+/, 2)
+ if (split.length < 2) { return revs }
+ const sha = split[0].trim()
+ const ref = split[1].trim().match(/(?:refs\/[^/]+\/)?(.*)/)[1]
+ if (!ref) { return revs } // ???
+ if (ref.endsWith(CARET_BRACES)) { return revs } // refs/tags/x^{} crap
+ const type = refType(line)
+ const doc = {sha, ref, type}
 
- if (type === 'tag') {
- const match = ref.match(/v?(\d+\.\d+\.\d+(?:[-+].+)?)$/)
- if (match && semver.valid(match[1], true)) {
- revs.versions[semver.clean(match[1], true)] = doc
- }
- }
+ revs.refs[ref] = doc
+ // We can check out shallow clones on specific SHAs if we have a ref
+ if (revs.shas[sha]) {
+ revs.shas[sha].push(ref)
+ } else {
+ revs.shas[sha] = [ref]
+ }
 
- return revs
- }, {versions: {}, 'dist-tags': {}, refs: {}, shas: {}})
- }).then(revs => {
- if (revs.refs.HEAD) {
- const HEAD = revs.refs.HEAD
- Object.keys(revs.versions).forEach(v => {
- if (v.sha === HEAD.sha) {
- revs['dist-tags'].HEAD = v
- if (!revs.refs.latest) {
- revs['dist-tags'].latest = revs.refs.HEAD
- }
- }
- })
+ if (type === 'tag') {
+ const match = ref.match(/v?(\d+\.\d+\.\d+(?:[-+].+)?)$/)
+ if (match && semver.valid(match[1], true)) {
+ revs.versions[semver.clean(match[1], true)] = doc
+ }
  }
- REVS.set(repo, revs)
+
  return revs
- })
+ }, {versions: {}, 'dist-tags': {}, refs: {}, shas: {}})
+ }, err => {
+ err.message = `Error while executing:\n${GITPATH} ls-remote -h -t ${repo}\n\n${err.stderr}\n${err.message}`
+ throw err
+ }).then(revs => {
+ if (revs.refs.HEAD) {
+ const HEAD = revs.refs.HEAD
+ Object.keys(revs.versions).forEach(v => {
+ if (v.sha === HEAD.sha) {
+ revs['dist-tags'].HEAD = v
+ if (!revs.refs.latest) {
+ revs['dist-tags'].latest = revs.refs.HEAD
+ }
+ }
+ })
+ }
+ REVS.set(repo, revs)
+ return revs
  })
  })
 }
@@ -173,15 +185,47 @@ module.exports._exec = execGit
 function execGit (gitArgs, gitOpts, opts) {
  opts = optCheck(opts)
  return checkGit().then(gitPath => {
- return execFileAsync(gitPath, gitArgs, mkOpts(gitOpts, opts))
+ return promiseRetry((retry, number) => {
+ if (number != 1) {
+ opts.log.info('pacote', 'Retrying git command: ' + gitArgs.join(' ') + ' attempt # ' + number);
+ }
+ return execFileAsync(gitPath, gitArgs, mkOpts(gitOpts, opts)).catch((err) => {
+ if (shouldRetry(err)) {
+ retry(err);
+ } else {
+ throw err;
+ }
+ })
+ }, opts.retry)
  })
 }
 
 module.exports._spawn = spawnGit
 function spawnGit (gitArgs, gitOpts, opts) {
  opts = optCheck(opts)
  return checkGit().then(gitPath => {
- return cp.spawn(gitPath, gitArgs, mkOpts(gitOpts, opts))
+ return promiseRetry((retry, number) => {
+ if (number != 1) {
+ opts.log.info('pacote', 'Retrying git command: ' + gitArgs.join(' ') + ' attempt # ' + number);
+ }
+ const child = cp.spawn(gitPath, gitArgs, mkOpts(gitOpts, opts));
+
+ let stdout = ''
+ let stderr = ''
+ child.stdout.on('data', d => { stdout += d })
+ child.stderr.on('data', d => { stderr += d }) 
+
+ return finished(child).catch(err => {
+ if (shouldRetry(stderr)) {
+ retry(err);
+ } else {
+ err.stderr = stderr;
+ throw err;
+ }
+ }).then(() => {
+ return stdout;
+ })
+ }, opts.retry)
  })
 }