Ivy and PyPi module, file names compatibility

pivy-importer/src/main/groovy/com/linkedin/python/importer/deps/DependencyDownloader.groovy

@@ -18,7 +18,6 @@ package com.linkedin.python.importer.deps
 import com.linkedin.python.importer.pypi.PypiApiCache
 import com.linkedin.python.importer.util.ProxyDetector
 import groovy.util.logging.Slf4j
-import org.apache.commons.io.FilenameUtils
 import org.apache.http.client.fluent.Request
 
 @Slf4j
@@ -58,10 +57,7 @@ abstract class DependencyDownloader {
     abstract downloadDependency(
         String dep, boolean latestVersions, boolean allowPreReleases, boolean fetchExtras, boolean lenient)
 
-    protected static File downloadArtifact(File destDir, String url) {
-
-        def filename = FilenameUtils.getName(new URL(url).getPath())
-        def contents = new File(destDir, filename)
+    protected static File downloadArtifact(File contents, String url) {
 
         if (!contents.exists()) {
             def proxy = ProxyDetector.maybeGetHttpProxy()
@@ -88,12 +84,27 @@ abstract class DependencyDownloader {
     }
 
     /**
-     * Get the actual module name from artifact name, which has the correct letter case.
+     * Get the module name from artifact name, which has the correct letter case.
      * @param filename the filename of artifact
      * @param revision module version
-     * @return actual module name, which is from PyPI
+     * @return module name
      */
-    static String getActualModuleNameFromFilename(String filename, String revision) {
+    static String getModuleNameFromFilename(String filename, String revision) {
         return filename.substring(0, filename.indexOf(revision) - 1)
     }
+
+    /**
+     * Compose filename from artifact file name and module name.
+     * Makes filename that has to be compatible with ivy layout pattern <code>[module]-[revision].[ext]</code>,
+     * where <code>[module]</code> is from PyPI metadata.
+     * @param name module name from PyPI metadata
+     * @param filename the filename of artifact
+     * @param revision module version
+     * @return
+     */
+    static String buildFilenameByModuleName(String moduleName, String filename, String revision) {
+        final String moduleNameFromFilename = getModuleNameFromFilename(filename, revision)
+
+        return filename.replace(moduleNameFromFilename, moduleName)
+    }
 }

pivy-importer/src/main/groovy/com/linkedin/python/importer/deps/SdistDownloader.groovy

@@ -18,6 +18,7 @@ package com.linkedin.python.importer.deps
 import com.linkedin.python.importer.distribution.SourceDistPackage
 import com.linkedin.python.importer.ivy.IvyFileWriter
 import groovy.util.logging.Slf4j
+
 import java.nio.file.Paths
 
 @Slf4j
@@ -58,13 +59,16 @@ class SdistDownloader extends DependencyDownloader {
         }
 
         // make sure the module name has the right letter case and dash or underscore as PyPI
-        name = getActualModuleNameFromFilename(sdistDetails.filename, version)
+        name = projectDetails.name
         log.info("Pulling in $name:$version")
 
         def destDir = Paths.get(ivyRepoRoot.absolutePath, SOURCE_DIST_ORG, name, version).toFile()
         destDir.mkdirs()
 
-        def sdistArtifact = downloadArtifact(destDir, sdistDetails.url)
+        def filename = buildFilenameByModuleName(name, sdistDetails.filename, version)
+        def contents = new File(destDir, filename)
+
+        def sdistArtifact = downloadArtifact(contents, sdistDetails.url)
         def packageDependencies = new SourceDistPackage(name, version, sdistArtifact, cache, dependencySubstitution)
             .getDependencies(latestVersions, allowPreReleases, fetchExtras, lenient)
 

pivy-importer/src/main/groovy/com/linkedin/python/importer/deps/WheelsDownloader.groovy

@@ -73,13 +73,16 @@ class WheelsDownloader extends DependencyDownloader {
         }
 
         // make sure the module name has the same letter case as PyPI
-        name = getActualModuleNameFromFilename(wheelDetails.filename, version)
+        name = projectDetails.name
         log.info("Pulling in $name:$version:$classifier")
 
         def destDir = Paths.get(ivyRepoRoot.absolutePath, BINARY_DIST_ORG, name, version, classifier).toFile()
         destDir.mkdirs()
 
-        def wheelArtifact = downloadArtifact(destDir, wheelDetails.url)
+        def filename = buildFilenameByModuleName(name, wheelDetails.filename, version)
+        def contents = new File(destDir, filename)
+
+        def wheelArtifact = downloadArtifact(contents, wheelDetails.url)
         def packageDependencies = new WheelsPackage(name, version, wheelArtifact, cache, dependencySubstitution)
             .getDependencies(latestVersions, allowPreReleases, fetchExtras, lenient)
 

pivy-importer/src/test/groovy/com/linkedin/python/importer/deps/DependencyDownloaderTest.groovy

@@ -80,39 +80,88 @@ class DependencyDownloaderTest extends Specification {
         assert dependencyDownloader.processedDependencies.contains(testProject)
     }
 
-    def "get actual module name from filename"() {
+    def "get module name from filename"() {
         given:
         String testSdistFilename
         String actualModuleName
         String expectedModuleName
 
         when:
         testSdistFilename = "zc.buildout-2.12.1.tar.gz"
-        actualModuleName = DependencyDownloader.getActualModuleNameFromFilename(testSdistFilename, "2.12.1")
+        actualModuleName = DependencyDownloader.getModuleNameFromFilename(testSdistFilename, "2.12.1")
         expectedModuleName = "zc.buildout"
         then:
         actualModuleName == expectedModuleName
 
         when:
         testSdistFilename = "google-api-python-client-1.7.3.tar.gz"
-        actualModuleName = DependencyDownloader.getActualModuleNameFromFilename(testSdistFilename, "1.7.3")
+        actualModuleName = DependencyDownloader.getModuleNameFromFilename(testSdistFilename, "1.7.3")
         expectedModuleName = "google-api-python-client"
         then:
         actualModuleName == expectedModuleName
 
         when:
         testSdistFilename = "sphinx_rtd_theme-0.4.0-py2.py3-none-any.whl"
-        actualModuleName = DependencyDownloader.getActualModuleNameFromFilename(testSdistFilename, "0.4.0")
+        actualModuleName = DependencyDownloader.getModuleNameFromFilename(testSdistFilename, "0.4.0")
         expectedModuleName = "sphinx_rtd_theme"
         then:
         actualModuleName == expectedModuleName
 
         when:
         testSdistFilename = "psycopg2_binary-2.7.5-cp34-cp34m-macosx_10_6_intel.macosx_10_9_intel.macosx_10_9_x86_64.macosx_10_10_intel.macosx_10_10_x86_64.whl"
-        actualModuleName = DependencyDownloader.getActualModuleNameFromFilename(testSdistFilename, "2.7.5")
+        actualModuleName = DependencyDownloader.getModuleNameFromFilename(testSdistFilename, "2.7.5")
         expectedModuleName = "psycopg2_binary"
 
         then:
         assert actualModuleName == expectedModuleName
     }
+
+    def "build filename by module name"() {
+        given:
+        String testSdistFilename
+        String moduleName
+        String filename
+        String expectedFilename
+
+        when:
+        moduleName = "zc.buildout"
+        testSdistFilename = "zc.buildout-2.12.1.tar.gz"
+        filename = DependencyDownloader.buildFilenameByModuleName(moduleName, testSdistFilename, "2.12.1")
+        expectedFilename = "zc.buildout-2.12.1.tar.gz"
+        then:
+        filename == expectedFilename
+
+        when:
+        moduleName = "google-api-python-client"
+        testSdistFilename = "google-api-python-client-1.7.3.tar.gz"
+        filename = DependencyDownloader.buildFilenameByModuleName(moduleName, testSdistFilename, "1.7.3")
+        expectedFilename = "google-api-python-client-1.7.3.tar.gz"
+        then:
+        filename == expectedFilename
+
+        when:
+        moduleName = "sphinx-rtd-theme"
+        testSdistFilename = "sphinx_rtd_theme-0.4.0-py2.py3-none-any.whl"
+        filename = DependencyDownloader.buildFilenameByModuleName(moduleName, testSdistFilename, "0.4.0")
+        expectedFilename = "sphinx-rtd-theme-0.4.0-py2.py3-none-any.whl"
+        then:
+        filename == expectedFilename
+
+        when:
+        moduleName = "psycopg2-binary"
+        testSdistFilename = "psycopg2_binary-2.7.5-cp34-cp34m-macosx_10_6_intel.macosx_10_9_intel.macosx_10_9_x86_64.macosx_10_10_intel.macosx_10_10_x86_64.whl"
+        filename = DependencyDownloader.buildFilenameByModuleName(moduleName, testSdistFilename, "2.7.5")
+        expectedFilename = "psycopg2-binary-2.7.5-cp34-cp34m-macosx_10_6_intel.macosx_10_9_intel.macosx_10_9_x86_64.macosx_10_10_intel.macosx_10_10_x86_64.whl"
+        then:
+        filename == expectedFilename
+
+        when:
+        moduleName = "typed_ast"
+        testSdistFilename = "typed-ast-1.3.2.tar.gz"
+        filename = DependencyDownloader.buildFilenameByModuleName(moduleName, testSdistFilename, "1.3.2")
+        expectedFilename = "typed_ast-1.3.2.tar.gz"
+
+        then:
+        assert filename == expectedFilename
+    }
 }