Improve support for alternative package repositories (#908)

* Add support for specifying dependency source

* Make installation from forced repositories possible

* Add possibility to declare a custom repository as the default

* Improve handling of sources in pools

* Fallback on the default repository when install via pip

* Add support for declaring repositories as secondary

* Update documentation

docs/docs/repositories.md

@@ -64,7 +64,36 @@ url = "https://foo.bar/simple/"
 
 From now on, Poetry will also look for packages in your private repository.
 
+!!!note
+
+    Any custom repository will have precedence over PyPI.
+
+    If you still want PyPI to be your primary source for your packages
+    you can declare custom repositories as secondary.
+
+    ```toml
+    [[tool.poetry.source]]
+    name = "foo"
+    url = "https://foo.bar/simple/"
+    secondary = true
+    ```
+
 If your private repository requires HTTP Basic Auth be sure to add the username and
-password to your `http-basic` config using the example above (be sure to use the
+password to your `http-basic` configuration using the example above (be sure to use the
 same name that is in the `tool.poetry.source` section). Poetry will use these values
 to authenticate to your private repository when downloading or looking for packages.
+
+
+### Disabling the PyPI repository
+
+If you want your packages to be exclusively looked up from a private
+repository, you can set it as the default one by using the `default` keyword
+
+```toml
+[[tool.poetry.source]]
+name = "foo"
+url = "https://foo.bar/simple/"
+default = true
+```
+
+A default source will also be the fallback source if you add other sources.

poetry/installation/installer.py

@@ -207,7 +207,7 @@ def _do_install(self, local_repo):
             ".".join([str(i) for i in self._env.version_info[:3]])
         ):
             # We resolve again by only using the lock file
-            pool = Pool()
+            pool = Pool(ignore_repository_names=True)
 
             # Making a new repo containing the packages
             # newly resolved and the ones from the current lock file
@@ -512,7 +512,7 @@ def _extra_packages(packages):
         return _extra_packages(extra_packages)
 
     def _get_installer(self):  # type: () -> BaseInstaller
-        return PipInstaller(self._env, self._io)
+        return PipInstaller(self._env, self._io, self._pool)
 
     def _get_installed(self):  # type: () -> InstalledRepository
         return InstalledRepository.load(self._env)

poetry/installation/pip_installer.py

@@ -22,9 +22,10 @@
 
 
 class PipInstaller(BaseInstaller):
-    def __init__(self, env, io):  # type: (Env, ...) -> None
+    def __init__(self, env, io, pool):  # type: (Env, ...) -> None
         self._env = env
         self._io = io
+        self._pool = pool
 
     def install(self, package, update=False):
         if package.source_type == "directory":
@@ -40,6 +41,7 @@ def install(self, package, update=False):
         args = ["install", "--no-deps"]
 
         if package.source_type == "legacy" and package.source_url:
+            repository = self._pool.repository(package.source_reference)
             parsed = urlparse.urlparse(package.source_url)
             if parsed.scheme == "http":
                 self._io.error(
@@ -49,21 +51,12 @@ def install(self, package, update=False):
                 )
                 args += ["--trusted-host", parsed.hostname]
 
-            auth = get_http_basic_auth(
-                Config.create("auth.toml"), package.source_reference
-            )
-            if auth:
-                index_url = "{scheme}://{username}:{password}@{netloc}{path}".format(
-                    scheme=parsed.scheme,
-                    username=auth[0],
-                    password=auth[1],
-                    netloc=parsed.netloc,
-                    path=parsed.path,
-                )
-            else:
-                index_url = package.source_url
+            index_url = repository.authenticated_url
 
             args += ["--index-url", index_url]
+            if self._pool.has_default():
+                if repository.name != self._pool.default.name:
+                    args += ["--extra-index-url", self._pool.default.authenticated_url]
 
         if update:
             args.append("-U")

poetry/json/schemas/poetry-schema.json

@@ -228,6 +228,10 @@
                     "items": {
                         "type": "string"
                     }
+                },
+                "source": {
+                    "type": "string",
+                    "description": "The exclusive source used to search for this dependency."
                 }
             }
         },
@@ -417,6 +421,14 @@
                     "type": "string",
                     "description": "The url of the repository",
                     "format": "uri"
+                },
+                "default": {
+                    "type": "boolean",
+                    "description": "Make this repository the default (disable PyPI)"
+                },
+                "secondary": {
+                    "type": "boolean",
+                    "description": "Declare this repository as secondary, i.e. it will only be looked up last for packages."
                 }
             }
         }

poetry/packages/dependency.py

@@ -1,3 +1,5 @@
+from typing import Optional
+
 import poetry.packages
 
 from poetry.semver import parse_constraint
@@ -23,6 +25,7 @@ def __init__(
         optional=False,  # type: bool
         category="main",  # type: str
         allows_prereleases=False,  # type: bool
+        source_name=None,  # type: Optional[str]
     ):
         self._name = canonicalize_name(name)
         self._pretty_name = name
@@ -45,6 +48,7 @@ def __init__(
             )
 
         self._allows_prereleases = allows_prereleases
+        self._source_name = source_name
 
         self._python_versions = "*"
         self._python_constraint = parse_constraint("*")
@@ -79,6 +83,10 @@ def pretty_name(self):
     def category(self):
         return self._category
 
+    @property
+    def source_name(self):
+        return self._source_name
+
     @property
     def python_versions(self):
         return self._python_versions

poetry/packages/package.py

@@ -54,6 +54,7 @@ def __init__(self, name, version, pretty_version=None):
         self._license = None
         self.readme = None
 
+        self.source_name = ""
         self.source_type = ""
         self.source_reference = ""
         self.source_url = ""
@@ -297,6 +298,7 @@ def add_dependency(
                     optional=optional,
                     category=category,
                     allows_prereleases=allows_prereleases,
+                    source_name=constraint.get("source"),
                 )
 
             marker = AnyMarker()

poetry/poetry.py

@@ -44,10 +44,17 @@ def __init__(
         # Configure sources
         self._pool = Pool()
         for source in self._local_config.get("source", []):
-            self._pool.add_repository(self.create_legacy_repository(source))
+            repository = self.create_legacy_repository(source)
+            self._pool.add_repository(
+                repository,
+                source.get("default", False),
+                secondary=source.get("secondary", False),
+            )
 
         # Always put PyPI last to prefer private repositories
-        self._pool.add_repository(PyPiRepository())
+        # but only if we have no other default source
+        if not self._pool.has_default():
+            self._pool.add_repository(PyPiRepository(), True)
 
     @property
     def file(self):

poetry/puzzle/provider.py

@@ -131,6 +131,7 @@ def search_for(self, dependency):  # type: (Dependency) -> List[Package]
                 constraint,
                 extras=dependency.extras,
                 allow_prereleases=dependency.allows_prereleases(),
+                repository=dependency.source_name,
             )
 
             packages.sort(
@@ -450,7 +451,10 @@ def complete_package(
             package = DependencyPackage(
                 package.dependency,
                 self._pool.package(
-                    package.name, package.version.text, extras=package.requires_extras
+                    package.name,
+                    package.version.text,
+                    extras=package.requires_extras,
+                    repository=package.dependency.source_name,
                 ),
             )
 

poetry/repositories/auth.py

@@ -10,6 +10,14 @@ def __init__(self, url, username, password):  # type: (str, str, str) -> None
         self._hostname = urlparse.urlparse(url).hostname
         self._auth = HTTPBasicAuth(username, password)
 
+    @property
+    def hostname(self):  # type: () -> str
+        return self._hostname
+
+    @property
+    def auth(self):  # type: () -> HTTPBasicAuth
+        return self._auth
+
     def __call__(self, r):  # type: (Request) -> Request
         if urlparse.urlparse(r.url).hostname != self._hostname:
             return r

poetry/repositories/legacy_repository.py

@@ -162,6 +162,7 @@ def __init__(
         self._packages = []
         self._name = name
         self._url = url.rstrip("/")
+        self._auth = auth
         self._cache_dir = Path(CACHE_DIR) / "cache" / "repositories" / name
 
         self._cache = CacheManager(
@@ -181,14 +182,25 @@ def __init__(
         )
 
         url_parts = urlparse.urlparse(self._url)
-        if not url_parts.username and auth:
-            self._session.auth = auth
+        if not url_parts.username and self._auth:
+            self._session.auth = self._auth
 
         self._disable_cache = disable_cache
 
     @property
-    def name(self):
-        return self._name
+    def authenticated_url(self):  # type: () -> str
+        if not self._auth:
+            return self.url
+
+        parsed = urlparse.urlparse(self.url)
+
+        return "{scheme}://{username}:{password}@{netloc}{path}".format(
+            scheme=parsed.scheme,
+            username=self._auth.auth.username,
+            password=self._auth.auth.password,
+            netloc=parsed.netloc,
+            path=parsed.path,
+        )
 
     def find_packages(
         self, name, constraint=None, extras=None, allow_prereleases=False