feat(core) merge index parameters (#1683)

docs/docs/usage/project.md

@@ -134,6 +134,9 @@ pdm config pypi.extra.username "foo"
 pdm config pypi.extra.password "password4foo"
 ```
 
+The index parameters can be put in different levels of configuration files. That is, you can only put the `url` in `pyproject.toml`,
+and store the credentials in `~/.config/pdm/config.toml` under the `[pypi.<name>]` section of which the name must match.
+
 !!! NOTE
     Configured indexes will be tried **after** the sources in `pyproject.toml`, if you want to completely ignore the
     locally configured indexes, including the main index, set the config value `pypi.ignore_stored_index`

news/1667.feature.md

@@ -0,0 +1 @@
+Merge the index parameters from different configuration files.

src/pdm/_types.py

@@ -1,17 +1,50 @@
 from __future__ import annotations
 
+import dataclasses
 from typing import Any, Dict, List, NamedTuple, Tuple, TypeVar, Union
 
-from pdm.compat import Literal, Protocol, TypedDict
+from pdm.compat import Protocol
 
 
-class Source(TypedDict, total=False):
-    url: str
-    verify_ssl: bool
-    name: str
-    type: Literal["index", "find_links"]  # noqa: F821
-    username: str
-    password: str
+@dataclasses.dataclass
+class RepositoryConfig:
+    url: str | None = None
+    username: str | None = None
+    password: str | None = None
+    verify_ssl: bool | None = None
+    type: str | None = None
+    ca_certs: str | None = None
+
+    config_prefix: str | None = None
+    name: str | None = None
+
+    def passive_update(self, other: RepositoryConfig | None = None, **kwargs: Any) -> None:
+        """An update method that prefers the existing value over the new one."""
+        if other is not None:
+            for k in other.__dataclass_fields__:
+                v = getattr(other, k)
+                if getattr(self, k) is None and v is not None:
+                    setattr(self, k, v)
+        for k, v in kwargs.items():
+            if getattr(self, k) is None and v is not None:
+                setattr(self, k, v)
+
+    def __rich__(self) -> str:
+        config_prefix = f"{self.config_prefix}." if self.config_prefix is not None else ""
+        lines: list[str] = []
+        if self.url:
+            lines.append(f"[primary]{config_prefix}url[/] = {self.url}")
+        if self.username:
+            lines.append(f"[primary]{config_prefix}username[/] = {self.username}")
+        if self.password:
+            lines.append(f"[primary]{config_prefix}password[/] = [i]<hidden>[/]")
+        if self.verify_ssl is not None:
+            lines.append(f"[primary]{config_prefix}verify_ssl[/] = {self.verify_ssl}")
+        if self.type:
+            lines.append(f"[primary]{config_prefix}type[/] = {self.type}")
+        if self.ca_certs:
+            lines.append(f"[primary]{config_prefix}ca_certs[/] = {self.ca_certs}")
+        return "\n".join(lines)
 
 
 RequirementDict = Union[str, Dict[str, Union[str, bool]]]

src/pdm/cli/commands/config.py

@@ -2,15 +2,10 @@
 from typing import Any, Mapping
 
 from pdm import termui
+from pdm._types import RepositoryConfig
 from pdm.cli.commands.base import BaseCommand
 from pdm.project import Project
-from pdm.project.config import (
-    DEFAULT_REPOSITORIES,
-    REPOSITORY,
-    Config,
-    RegistryConfig,
-    RepositoryConfig,
-)
+from pdm.project.config import DEFAULT_REPOSITORIES, REPOSITORY, Config
 
 
 class Command(BaseCommand):
@@ -85,7 +80,7 @@ def _show_config(self, config: Mapping[str, Any], supersedes: Mapping[str, Any])
                         style=extra_style,
                         verbosity=termui.Verbosity.DETAIL,
                     )
-                    self.ui.echo(RegistryConfig(**config[key], config_prefix=key))
+                    self.ui.echo(RepositoryConfig(**config[key], config_prefix=key))
                 elif key.startswith(REPOSITORY):
                     for item in config[key]:
                         self.ui.echo(
@@ -95,7 +90,7 @@ def _show_config(self, config: Mapping[str, Any], supersedes: Mapping[str, Any])
                         )
                         repository = dict(config[key][item])
                         if "url" not in repository and item in DEFAULT_REPOSITORIES:
-                            repository["url"] = DEFAULT_REPOSITORIES[item].url
+                            repository["url"] = DEFAULT_REPOSITORIES[item]
                         self.ui.echo(RepositoryConfig(**repository, config_prefix=f"{key}.{item}"))
                 continue
             config_item = Config._config_map[canonical_key]

src/pdm/cli/commands/publish/__init__.py

@@ -110,6 +110,7 @@ def get_repository(project: Project, options: argparse.Namespace) -> Repository:
         config = project.global_config.get_repository_config(repository)
         if config is None:
             raise PdmUsageError(f"Missing repository config of {repository}")
+        assert config.url is not None
         if username is not None:
             config.username = username
         if password is not None:

src/pdm/cli/commands/publish/repository.py

@@ -77,9 +77,9 @@ def _convert_to_list_of_tuples(data: dict[str, Any]) -> list[tuple[str, Any]]:
         return result
 
     def get_release_urls(self, packages: list[PackageFile]) -> Iterable[str]:
-        if self.url.startswith(DEFAULT_REPOSITORIES["pypi"].url.rstrip("/")):
+        if self.url.startswith(DEFAULT_REPOSITORIES["pypi"].rstrip("/")):
             base = "https://pypi.org/"
-        elif self.url.startswith(DEFAULT_REPOSITORIES["testpypi"].url.rstrip("/")):
+        elif self.url.startswith(DEFAULT_REPOSITORIES["testpypi"].rstrip("/")):
             base = "https://test.pypi.org/"
         else:
             return set()

src/pdm/formats/poetry.py

@@ -19,14 +19,12 @@
 from pdm.models.markers import Marker
 from pdm.models.requirements import Requirement
 from pdm.models.specifiers import PySpecSet
+from pdm.utils import cd
 
 if TYPE_CHECKING:
     from argparse import Namespace
 
-    from pdm._types import RequirementDict, Source
-    from pdm.project.core import Project
-
-from pdm.utils import cd
+    from pdm._types import RequirementDict
 
 
 def check_fingerprint(project: Project | None, filename: Path | str) -> bool:
@@ -178,7 +176,7 @@ def build(self, value: str | dict) -> None:
         raise Unset()
 
     @convert_from("source")
-    def sources(self, value: list[Source]) -> None:
+    def sources(self, value: list[dict[str, Any]]) -> None:
         self.settings["source"] = [
             {
                 "name": item.get("name", ""),

src/pdm/formats/requirements.py

@@ -7,17 +7,14 @@
 import urllib.parse
 from argparse import Namespace
 from os import PathLike
-from typing import TYPE_CHECKING, Any, Mapping, cast
+from typing import Any, Mapping
 
 from pdm.formats.base import make_array
 from pdm.models.candidates import Candidate
 from pdm.models.requirements import Requirement, parse_requirement
 from pdm.project import Project
 from pdm.utils import expand_env_vars_in_auth
 
-if TYPE_CHECKING:
-    from pdm._types import Source
-
 
 class RequirementParser:
     """Reference:
@@ -109,7 +106,7 @@ def _is_url_trusted(url: str, trusted_hosts: list[str]) -> bool:
     return False
 
 
-def convert_url_to_source(url: str, name: str | None, trusted_hosts: list[str], type: str = "index") -> Source:
+def convert_url_to_source(url: str, name: str | None, trusted_hosts: list[str], type: str = "index") -> dict[str, Any]:
     if not name:
         name = hashlib.sha1(url.encode("utf-8")).hexdigest()[:6]
     source = {
@@ -119,7 +116,7 @@ def convert_url_to_source(url: str, name: str | None, trusted_hosts: list[str],
     }
     if type != "index":
         source["type"] = type
-    return cast("Source", source)
+    return source
 
 
 def convert(project: Project, filename: PathLike, options: Namespace) -> tuple[Mapping[str, Any], Mapping[str, Any]]:
@@ -146,7 +143,7 @@ def convert(project: Project, filename: PathLike, options: Namespace) -> tuple[M
         data["optional-dependencies"] = {options.group: deps}
     else:
         data["dependencies"] = deps
-    sources: list[Source] = []
+    sources: list[dict[str, Any]] = []
     if parser.index_url and not parser.no_index:
         sources.append(convert_url_to_source(parser.index_url, "pypi", parser.trusted_hosts))
     if not parser.no_index:

src/pdm/models/auth.py

@@ -5,7 +5,7 @@
 from unearth.auth import MaybeAuth, MultiDomainBasicAuth
 from unearth.utils import split_auth_from_netloc
 
-from pdm._types import Source
+from pdm._types import RepositoryConfig
 from pdm.exceptions import PdmException
 from pdm.termui import UI
 
@@ -24,7 +24,7 @@ class PdmBasicAuth(MultiDomainBasicAuth):
         - It shows an error message when credentials are not provided or correct.
     """
 
-    def __init__(self, sources: list[Source], prompting: bool = True) -> None:
+    def __init__(self, sources: list[RepositoryConfig], prompting: bool = True) -> None:
         super().__init__(prompting=True)
         self._real_prompting = prompting
         self.sources = sources
@@ -33,12 +33,13 @@ def _get_auth_from_index_url(self, netloc: str) -> tuple[MaybeAuth, str | None]:
         if not self.sources:
             return None, None
         for source in self.sources:
-            parsed = urllib.parse.urlparse(source["url"])
+            assert source.url
+            parsed = urllib.parse.urlparse(source.url)
             auth, index_netloc = split_auth_from_netloc(parsed.netloc)
             if index_netloc == netloc:
-                if "username" in source:
-                    auth = (source["username"], source.get("password"))
-                return auth, source["url"]
+                if source.username:
+                    auth = (source.username, source.password)
+                return auth, source.url
         return None, None
 
     def _prompt_for_password(self, netloc: str) -> tuple[str | None, str | None, bool]:

src/pdm/models/environment.py

@@ -33,7 +33,7 @@
 )
 
 if TYPE_CHECKING:
-    from pdm._types import Source
+    from pdm._types import RepositoryConfig
     from pdm.project import Project
 
 
@@ -147,7 +147,7 @@ def _build_session(self, index_urls: list[str], trusted_hosts: list[str]) -> PDM
     @contextmanager
     def get_finder(
         self,
-        sources: list[Source] | None = None,
+        sources: list[RepositoryConfig] | None = None,
         ignore_compatibility: bool = False,
     ) -> Generator[unearth.PackageFinder, None, None]:
         """Return the package finder of given index sources.

src/pdm/models/repositories.py

@@ -27,7 +27,7 @@
 )
 
 if TYPE_CHECKING:
-    from pdm._types import CandidateInfo, SearchResult, Source
+    from pdm._types import CandidateInfo, RepositoryConfig, SearchResult
     from pdm.models.environment import Environment
 
 ALLOW_ALL_PYTHON = PySpecSet()
@@ -51,7 +51,7 @@ class BaseRepository:
 
     def __init__(
         self,
-        sources: list[Source],
+        sources: list[RepositoryConfig],
         environment: Environment,
         ignore_compatibility: bool = True,
     ) -> None:
@@ -67,7 +67,7 @@ def __init__(
         self._candidate_info_cache = environment.project.make_candidate_info_cache()
         self._hash_cache = environment.project.make_hash_cache()
 
-    def get_filtered_sources(self, req: Requirement) -> list[Source]:
+    def get_filtered_sources(self, req: Requirement) -> list[RepositoryConfig]:
         """Get matching sources based on the index attribute."""
         return self.sources
 
@@ -274,7 +274,7 @@ def _get_dependencies_from_json(self, candidate: Candidate) -> CandidateInfo:
         sources = self.get_filtered_sources(candidate.req)
         url_prefixes = [
             proc_url[:-7]  # Strip "/simple".
-            for proc_url in (raw_url.rstrip("/") for raw_url in (source.get("url", "") for source in sources))
+            for proc_url in (raw_url.rstrip("/") for raw_url in (source.url for source in sources) if raw_url)
             if proc_url.endswith("/simple")
         ]
         with self.environment.get_finder(sources) as finder:
@@ -323,7 +323,7 @@ def _find_candidates(self, requirement: Requirement) -> Iterable[Candidate]:
         return cans
 
     def search(self, query: str) -> SearchResult:
-        pypi_simple = self.sources[0]["url"].rstrip("/")
+        pypi_simple = self.sources[0].url.rstrip("/")  # type: ignore[union-attr]
 
         if pypi_simple.endswith("/simple"):
             search_url = pypi_simple[:-6] + "search"
@@ -352,7 +352,7 @@ class LockedRepository(BaseRepository):
     def __init__(
         self,
         lockfile: Mapping[str, Any],
-        sources: list[Source],
+        sources: list[RepositoryConfig],
         environment: Environment,
     ) -> None:
         super().__init__(sources, environment, ignore_compatibility=False)