Merge pull request #470 from qtomlinson/qt/fix_pypi_missing

Mark missing for absent pypi package download url

providers/fetch/pypiFetch.js

@@ -26,7 +26,9 @@ class PyPiFetch extends AbstractFetch {
     request.url = spec.toUrl()
     super.handle(request)
     const file = this.createTempFile(request)
-    await this._getPackage(spec, registryData, file.name)
+    const downloaded = await this._getPackage(spec, registryData, file.name)
+    if (!downloaded) return request.markSkip('Missing  ')
+
     const dir = this.createTempDir(request)
     await this.decompress(file.name, dir.name)
     const hashes = await this.computeHashes(file.name)
@@ -83,17 +85,16 @@ class PyPiFetch extends AbstractFetch {
 
   async _getPackage(spec, registryData, destination) {
     const releaseTypes = get(registryData, ['releases', spec.revision])
-    const release = find(releaseTypes, entry => {
-      return entry.url && entry.url.length > 6 && entry.url.slice(-6) === 'tar.gz'
-    })
-    if (!release) return
+    const release = find(releaseTypes, entry => entry.url?.endsWith('tar.gz') || entry.url?.endsWith('zip'))
+    if (!release) return false
+
     return new Promise((resolve, reject) => {
       nodeRequest
         .get(release.url, (error, response) => {
           if (error) return reject(error)
           if (response.statusCode !== 200) reject(new Error(`${response.statusCode} ${response.statusMessage}`))
         })
-        .pipe(fs.createWriteStream(destination).on('finish', () => resolve(null)))
+        .pipe(fs.createWriteStream(destination).on('finish', () => resolve(true)))
     })
   }
 }

test/unit/providers/fetch/pypiFetchTests.js

@@ -6,14 +6,16 @@ const sinon = require('sinon')
 const PypiFetch = require('../../../../providers/fetch/pypiFetch')
 const requestRetryWithDefaults = require('../../../../providers/fetch/requestRetryWithDefaults')
 const Request = require('../../../../ghcrawler/lib/request.js')
-const pypiFetchOptions = { logger: { info: sinon.stub() }, cdFileLocation: 'test/fixtures/debian/fragment' }
+const pypiFetchOptions = { logger: { info: sinon.stub() } }
 
 describe('pypiFetch handle function', () => {
   let sandbox = sinon.createSandbox()
   let requestGetStub
+  let fetch
 
   beforeEach(function () {
     requestGetStub = sandbox.stub(requestRetryWithDefaults, 'get')
+    fetch = PypiFetch(pypiFetchOptions)
   })
 
   afterEach(function () {
@@ -24,9 +26,21 @@ describe('pypiFetch handle function', () => {
     // Setup the stub to return an empty response (e.g. no body)
     requestGetStub.returns({})
 
-    let fetch = PypiFetch(pypiFetchOptions)
     let result = await fetch.handle(new Request('pypi', 'cd:/pypi/pypi/-/reuse/0.8.1'))
 
     expect(result.outcome).to.be.equal('Missing  ')
   })
+
+  it('returns missing when failed to find download url', async () => {
+    // release information in the registry data is empty
+    requestGetStub.returns({
+      body: {
+        'releases': { '1.10.0': [] }
+      },
+      statusCode: 200
+    })
+
+    let result = await fetch.handle(new Request('pypi', 'cd:/pypi/pypi/-/dnspython/1.10.0'))
+    expect(result.outcome).to.be.equal('Missing  ')
+  })
 })