fix: remove external urls from addFromURL tests (#834)

Relying on external URLs makes our tests dependent on the availability and response time of these services as well as not being able to run without an internet connection. This PR removes those requests to external URLs and instead spins up test HTTP(s) servers as they are needed to serve the content that `ipfs.util.addFromURL` consumes.

This PR also strengthens some of the tests to make assertions that the result received by `ipfs.util.addFromURL` is the same as the result of adding the data directly using `ipfs.add`.

resolves #803

License: MIT
Signed-off-by: Alan Shaw <alan@tableflip.io>

Author: alanshaw

test/fixtures/ssl/cert.pem

@@ -0,0 +1,20 @@
+-----BEGIN CERTIFICATE-----
+MIIDOzCCAiMCCQCVqVeRIp9pFDANBgkqhkiG9w0BAQUFADBnMQswCQYDVQQGEwJV
+UzENMAsGA1UECAwEVXRhaDEOMAwGA1UEBwwFUHJvdm8xIzAhBgNVBAoMGkFDTUUg
+U2lnbmluZyBBdXRob3JpdHkgSW5jMRQwEgYDVQQDDAtleGFtcGxlLmNvbTAeFw0x
+ODA4MTQyMDEzNTdaFw0xOTEyMjcyMDEzNTdaMFgxCzAJBgNVBAYTAlVTMQ0wCwYD
+VQQIDARVdGFoMQ4wDAYDVQQHDAVQcm92bzEWMBQGA1UECgwNQUNNRSBUZWNoIElu
+YzESMBAGA1UEAwwJMTI3LjAuMC4xMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB
+CgKCAQEA6x6mTXV+rC35QW/sPutT1O1cugtnw+UsJx7EGgzyjh7EoXE3gb7sO96P
+tOI5zknb0vecckbiVkesmLnAs2iNa1u9EiRr6WHdc+1MfUCxyHRfP731vRZyo0kx
+bSXerE0qZ2N3M1XyndZF7VMthKDKIg0ZR0TvdjwLqyLYEHAnRBhJLRS0Oy0fC6Of
+VWCO3gIuk1HkTXH+/ZMA/obqrtlisxY85mMdlRz+1PNdZBMf+NxmrXN59uq+JqUu
+8/v1oQ8jH2iU9IWeqyawHDEvPW3aDorfaWGyats5Xd3cT2Ph4xF9tBLT+3PDGU8c
+oBmTHWDenYn+TCkCseayo1JCO5igJQIDAQABMA0GCSqGSIb3DQEBBQUAA4IBAQCr
+R7eZxicHjJoRcbsPBDQpzx9uSux3uvpN93pyJrXXHiil/5SE7CYeDqv5+nV2p6HA
+6KONUAmpId0iHAEi9u+0/LgPWyYQMzT3sfBhkO8RRaMYI87VuKbk5PFmlZbD843+
+Qmg3Se2F7BDnTf88xA6QWR4DCejy+ZHfDRFrh3xfFl4tX1UNgqiTGfjPCzblhWx9
+ygzlT+flN2j3NkAlhUEV89pnH4EQWILePMTT4wh2XOQj1VFJ+2ATojHFVUTtNWAJ
+xrY/Q9cMYsZ++I8i9bHMZoyc1bSUd5CNFpQdfjVzlgMPT9Jj/fzWIQz+wq0KeRLI
+dLWsa2MZr0GZnTU39YwH
+-----END CERTIFICATE-----

test/fixtures/ssl/privkey.pem

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEogIBAAKCAQEA6x6mTXV+rC35QW/sPutT1O1cugtnw+UsJx7EGgzyjh7EoXE3
+gb7sO96PtOI5zknb0vecckbiVkesmLnAs2iNa1u9EiRr6WHdc+1MfUCxyHRfP731
+vRZyo0kxbSXerE0qZ2N3M1XyndZF7VMthKDKIg0ZR0TvdjwLqyLYEHAnRBhJLRS0
+Oy0fC6OfVWCO3gIuk1HkTXH+/ZMA/obqrtlisxY85mMdlRz+1PNdZBMf+NxmrXN5
+9uq+JqUu8/v1oQ8jH2iU9IWeqyawHDEvPW3aDorfaWGyats5Xd3cT2Ph4xF9tBLT
++3PDGU8coBmTHWDenYn+TCkCseayo1JCO5igJQIDAQABAoIBAH5fbfFqOpie6T8T
+wj4bTGbA4bsZkD9JeU7ZiXubA/ABd5xyduwky2JugH0vrvRC3IVrE0qU8OiBA7Le
+/EUx5/kRSPFsZBf/wwChRiB4WlYsvllLZ76nRxyepZNN7H5dx3Hkk1gjVREi71jd
+ATUtGxfsRG77DV5WbcshIlLLhT9iaohsalmClAFBmwhqnRMvOXHiQyRbvB0fOX08
+uVlObOqo9jLB8N5C/ux+wFEP4wi/AxVqs9ih7Ss7T7+pmOCVWhOnbYcoY2jdaJ11
+iLK4F3rv/jQ82OwUpzrWsPedmZUzlOO8xdV3b8hOcPHs/BKvYed7aHSn6b5eVKKT
+zT8vQoECgYEA+K9pvw9K/7+F810MHG+nZ0gtVWmXJp49jB7zQ6QMIex2sUajY2y9
+bEJX8T6rdu3qd+HYU4zl3qt+MUnsVQEBNkLPAn3od0qIWXxu1SL2GF8SDV1xJWK1
+Fp0YDe9blaz1JsmSgieNcSoSwqE2V97Wfd/m+EUfyhQt9HX55H5UgAUCgYEA8gkW
+0xZKuHhBMYpcES2P5H5q6HN2fcEQycMuS3agAOhrFPYUT1DVNhbfNVmbOvL2NRWI
+hXixo5SkuEuq2fjmEoeLPTmeKO5LM4IVqovWCYomSftKDpzw4HRn2jvKzi2+mg8J
+qktIMqRqHu/O1NUIsszCIu4c5DzUdhr4N7GXOaECgYAEd1oF1Wd6al0kfsJN7G9s
+Om6d/xR43BSs5I1n5JVXMqD7FBKxIW3ReOuNaJu5uhIg7wxsi7ZBJoFQr0wwRqFX
+8SE4oTxAkDUcrlBrQYJ785Embkwu6LPp4Q5iia7yZDXO6YXZEo7GvoOxvSV1tInT
+nubOBKfKgExG/KttQBuSZQKBgAzYOqPdLP35M8yDQTuQJXDE3LuVVRZ7Zn6uowhS
+NU+XBgfIv28uJQKH2DSmmrxYJITQrbwXmaXKv6sgKOMEeIFHPDZ1llUpwEftgWTZ
+ovRCpqGKenWoEoh25QQJ5Eto1hKq9aJZ+GznmNIne9yDqcCDaVIdPN9H8yaJa97Y
+x+PBAoGAOiK6xAbPyJSKDSTGZzdv8+yeOdNeJjRHxKJs+4YsDchmdumrqow83DBP
+7ulIJD9pcmsWj+8fntMcsTX5mvzJd5LsKc7Maa5/LtitsLsynu78QFg4Njj8sAKn
+3991i8J98DZ9zqmkxJJhGwstCHG+c+Q7lA6kZ1UdbWJwYwIHjAs=
+-----END RSA PRIVATE KEY-----

test/util.spec.js

@@ -10,6 +10,11 @@ const isNode = require('detect-node')
 const path = require('path')
 const fs = require('fs')
 const os = require('os')
+const http = require('http')
+const https = require('https')
+const each = require('async/each')
+const waterfall = require('async/waterfall')
+const parallel = require('async/parallel')
 
 const IPFSApi = require('../src')
 const f = require('./utils/factory')
@@ -32,7 +37,8 @@ describe('.util', () => {
     })
   })
 
-  after((done) => {
+  after(function (done) {
+    this.timeout(10 * 1000)
     if (!ipfsd) return done()
     ipfsd.stop(done)
   })
@@ -113,81 +119,257 @@ describe('.util', () => {
   })
 
   describe('.urlAdd', () => {
-    it('http', function (done) {
-      this.timeout(40 * 1000)
-
-      ipfs.util.addFromURL('http://example.com/', (err, result) => {
-        expect(err).to.not.exist()
-        expect(result.length).to.equal(1)
-        done()
+    let testServers = []
+
+    const sslOpts = {
+      key: fs.readFileSync(path.join(__dirname, 'fixtures', 'ssl', 'privkey.pem')),
+      cert: fs.readFileSync(path.join(__dirname, 'fixtures', 'ssl', 'cert.pem'))
+    }
+
+    const startTestServer = (handler, opts, cb) => {
+      if (typeof opts === 'function') {
+        cb = opts
+        opts = {}
+      }
+
+      const server = opts.secure
+        ? https.createServer(sslOpts, handler)
+        : http.createServer(handler)
+
+      server.listen((err) => {
+        if (err) return cb(err)
+        testServers.push(server)
+        cb(null, server)
       })
+    }
+
+    beforeEach(() => {
+      // Instructs node to not reject our snake oil SSL certificate when it
+      // can't verify the certificate authority
+      process.env.NODE_TLS_REJECT_UNAUTHORIZED = 0
     })
 
-    it('https', function (done) {
-      this.timeout(40 * 1000)
+    afterEach((done) => {
+      // Reinstate unauthorised SSL cert rejection
+      process.env.NODE_TLS_REJECT_UNAUTHORIZED = 1
 
-      ipfs.util.addFromURL('https://example.com/', (err, result) => {
-        expect(err).to.not.exist()
-        expect(result.length).to.equal(1)
-        done()
+      each(testServers, (server, cb) => server.close(cb), (err) => {
+        testServers = []
+        done(err)
       })
     })
 
-    it('http with redirection', function (done) {
-      this.timeout(40 * 1000)
-
-      ipfs.util.addFromURL('http://covers.openlibrary.org/book/id/969165.jpg', (err, result) => {
+    it('http', (done) => {
+      const data = Buffer.from(`TEST${Date.now()}`)
+
+      parallel({
+        server: (cb) => {
+          const handler = (req, res) => {
+            res.write(data)
+            res.end()
+          }
+          startTestServer(handler, cb)
+        },
+        expectedResult: (cb) => ipfs.add(data, cb)
+      }, (err, taskResult) => {
         expect(err).to.not.exist()
-        expect(result[0].hash).to.equal('QmaL9zy7YUfvWmtD5ZXp42buP7P4xmZJWFkm78p8FJqgjg')
-        done()
+        const { server, expectedResult } = taskResult
+
+        const url = `http://127.0.0.1:${server.address().port}/`
+        ipfs.util.addFromURL(url, (err, result) => {
+          expect(err).to.not.exist()
+          expect(result).to.deep.equal(expectedResult)
+          done()
+        })
       })
     })
 
-    it('https with redirection', function (done) {
-      this.timeout(40 * 1000)
+    it('https', (done) => {
+      const data = Buffer.from(`TEST${Date.now()}`)
+
+      parallel({
+        server: (cb) => {
+          const handler = (req, res) => {
+            res.write(data)
+            res.end()
+          }
+          startTestServer(handler, { secure: true }, cb)
+        },
+        expectedResult: (cb) => ipfs.add(data, cb)
+      }, (err, taskResult) => {
+        expect(err).to.not.exist()
+        const { server, expectedResult } = taskResult
 
-      ipfs.util.addFromURL('https://coverartarchive.org/release/6e2a1694-d8b9-466a-aa33-b1077b2333c1', (err, result) => {
+        const url = `https://127.0.0.1:${server.address().port}/`
+        ipfs.util.addFromURL(url, (err, result) => {
+          expect(err).to.not.exist()
+          expect(result).to.deep.equal(expectedResult)
+          done()
+        })
+      })
+    })
+
+    it('http with redirection', (done) => {
+      const data = Buffer.from(`TEST${Date.now()}`)
+
+      waterfall([
+        (cb) => {
+          const handler = (req, res) => {
+            res.write(data)
+            res.end()
+          }
+          startTestServer(handler, cb)
+        },
+        (serverA, cb) => {
+          const url = `http://127.0.0.1:${serverA.address().port}`
+          const handler = (req, res) => {
+            res.statusCode = 302
+            res.setHeader('Location', url)
+            res.end()
+          }
+          startTestServer(handler, (err, serverB) => {
+            if (err) return cb(err)
+            cb(null, { a: serverA, b: serverB })
+          })
+        }
+      ], (err, servers) => {
         expect(err).to.not.exist()
-        expect(result[0].hash).to.equal('QmSUdDvmXuq5YGrL4M3SEz7UZh5eT9WMuAsd9K34sambSj')
-        done()
+
+        ipfs.add(data, (err, res) => {
+          expect(err).to.not.exist()
+
+          const expectedHash = res[0].hash
+          const url = `http://127.0.0.1:${servers.b.address().port}`
+
+          ipfs.util.addFromURL(url, (err, result) => {
+            expect(err).to.not.exist()
+            expect(result[0].hash).to.equal(expectedHash)
+            done()
+          })
+        })
       })
     })
 
-    it('with only-hash=true', function () {
-      this.timeout(40 * 1000)
+    it('https with redirection', (done) => {
+      const data = Buffer.from(`TEST${Date.now()}`)
+
+      waterfall([
+        (cb) => {
+          const handler = (req, res) => {
+            res.write(data)
+            res.end()
+          }
+          startTestServer(handler, { secure: true }, cb)
+        },
+        (serverA, cb) => {
+          const url = `https://127.0.0.1:${serverA.address().port}`
+          const handler = (req, res) => {
+            res.statusCode = 302
+            res.setHeader('Location', url)
+            res.end()
+          }
+          startTestServer(handler, { secure: true }, (err, serverB) => {
+            if (err) return cb(err)
+            cb(null, { a: serverA, b: serverB })
+          })
+        }
+      ], (err, servers) => {
+        expect(err).to.not.exist()
+
+        ipfs.add(data, (err, res) => {
+          expect(err).to.not.exist()
+
+          const expectedHash = res[0].hash
+          const url = `https://127.0.0.1:${servers.b.address().port}`
 
-      return ipfs.util.addFromURL('http://www.randomtext.me/#/gibberish', { onlyHash: true })
-        .then(out => expectTimeout(ipfs.object.get(out[0].hash), 4000))
+          ipfs.util.addFromURL(url, (err, result) => {
+            expect(err).to.not.exist()
+            expect(result[0].hash).to.equal(expectedHash)
+            done()
+          })
+        })
+      })
     })
 
-    it('with wrap-with-directory=true', function (done) {
-      this.timeout(40 * 1000)
+    it('with only-hash=true', (done) => {
+      const handler = (req, res) => {
+        res.write(`TEST${Date.now()}`)
+        res.end()
+      }
 
-      ipfs.util.addFromURL('http://ipfs.io/ipfs/QmWjppACLcFLQ2qL38unKQvJBhXH3RUtcGLPk7zmrTwV61/969165.jpg?foo=bar#buzz', {
-        wrapWithDirectory: true
-      }, (err, result) => {
+      startTestServer(handler, (err, server) => {
         expect(err).to.not.exist()
-        expect(result[0].hash).to.equal('QmaL9zy7YUfvWmtD5ZXp42buP7P4xmZJWFkm78p8FJqgjg')
-        expect(result[0].path).to.equal('969165.jpg')
-        expect(result[1].hash).to.equal('QmWjppACLcFLQ2qL38unKQvJBhXH3RUtcGLPk7zmrTwV61')
-        expect(result.length).to.equal(2)
-        done()
+
+        const url = `http://127.0.0.1:${server.address().port}/`
+
+        ipfs.util.addFromURL(url, { onlyHash: true }, (err, res) => {
+          expect(err).to.not.exist()
+
+          // A successful object.get for this size data took my laptop ~14ms
+          let didTimeout = false
+          const timeoutId = setTimeout(() => {
+            didTimeout = true
+            done()
+          }, 500)
+
+          ipfs.object.get(res[0].hash, () => {
+            clearTimeout(timeoutId)
+            if (didTimeout) return
+            expect(new Error('did not timeout')).to.not.exist()
+          })
+        })
       })
     })
 
-    it('with wrap-with-directory=true and URL-escaped file name', function (done) {
-      this.timeout(40 * 1000)
+    it('with wrap-with-directory=true', (done) => {
+      const filename = `TEST${Date.now()}.txt`
+      const data = Buffer.from(`TEST${Date.now()}`)
+
+      parallel({
+        server: (cb) => startTestServer((req, res) => {
+          res.write(data)
+          res.end()
+        }, cb),
+        expectedResult: (cb) => {
+          ipfs.add([{ path: filename, content: data }], { wrapWithDirectory: true }, cb)
+        }
+      }, (err, taskResult) => {
+        expect(err).to.not.exist()
+
+        const { server, expectedResult } = taskResult
+        const url = `http://127.0.0.1:${server.address().port}/${filename}?foo=bar#buzz`
+
+        ipfs.util.addFromURL(url, { wrapWithDirectory: true }, (err, result) => {
+          expect(err).to.not.exist()
+          expect(result).to.deep.equal(expectedResult)
+          done()
+        })
+      })
+    })
 
-      // Sample URL contains URL-escaped ( ) and local diacritics
-      ipfs.util.addFromURL('https://upload.wikimedia.org/wikipedia/commons/thumb/c/cf/Doma%C5%BElice%2C_Jir%C3%A1skova_43_%289102%29.jpg/320px-Doma%C5%BElice%2C_Jir%C3%A1skova_43_%289102%29.jpg?foo=bar#buzz', {
-        wrapWithDirectory: true
-      }, (err, result) => {
+    it('with wrap-with-directory=true and URL-escaped file name', (done) => {
+      const filename = '320px-Domažlice,_Jiráskova_43_(9102).jpg'
+      const data = Buffer.from(`TEST${Date.now()}`)
+
+      parallel({
+        server: (cb) => startTestServer((req, res) => {
+          res.write(data)
+          res.end()
+        }, cb),
+        expectedResult: (cb) => {
+          ipfs.add([{ path: filename, content: data }], { wrapWithDirectory: true }, cb)
+        }
+      }, (err, taskResult) => {
         expect(err).to.not.exist()
-        expect(result[0].hash).to.equal('QmRJ9ExxSMV4BLF9ZJUb2mLngupm6BXZEek755VHGTJo2Y')
-        expect(result[0].path).to.equal('320px-Domažlice,_Jiráskova_43_(9102).jpg')
-        expect(result[1].hash).to.equal('QmbxsHFU3sCfr8wszDHuDLA76C2xCv9HT8L3aC1pBwgaHk')
-        expect(result.length).to.equal(2)
-        done()
+
+        const { server, expectedResult } = taskResult
+        const url = `http://127.0.0.1:${server.address().port}/${encodeURIComponent(filename)}?foo=bar#buzz`
+
+        ipfs.util.addFromURL(url, { wrapWithDirectory: true }, (err, result) => {
+          expect(err).to.not.exist()
+          expect(result).to.deep.equal(expectedResult)
+          done()
+        })
       })
     })