Merge branch 'main' into feature/229-relative-sources-rebased

.github/CODEOWNERS

@@ -0,0 +1,3 @@
+# Syntax for this file at https://help.github.com/articles/about-codeowners/
+
+* @conda/conda-lock

.github/workflows/test.yml

@@ -45,11 +45,14 @@ jobs:
  - name: run-test
  run: |
  conda activate test
+ copy pyproject.toml "%RUNNER_TEMP%"
  Xcopy /E /I tests "%RUNNER_TEMP%\\tests"
  pushd "${RUNNER_TEMP}"
  set TMPDIR="%RUNNER_TEMP%"
  dir
- pytest -n auto -vrsx --cov=conda_lock tests
+ pytest --cov=conda_lock --cov-branch --cov-report=xml --cov-report=term tests
+ copy coverage.xml %GITHUB_WORKSPACE%
+ - uses: codecov/codecov-action@v3
 
  test:
  runs-on: ${{ matrix.os }}
@@ -96,13 +99,16 @@ jobs:
  shell: bash -l {0}
  run: |
  conda activate test
+ cp pyproject.toml "${RUNNER_TEMP}/"
  cp -a tests "${RUNNER_TEMP}/"
  pushd "${RUNNER_TEMP}"
  export TMPDIR="${RUNNER_TEMP}"
  ls -lah
  set -x
  which pytest
- pytest -n auto -vrsx --cov=conda_lock tests
+ pytest --cov=conda_lock --cov-branch --cov-report=xml --cov-report=term tests
+ cp coverage.xml "${GITHUB_WORKSPACE}"
+ - uses: codecov/codecov-action@v3
 
  - name: test-gdal
  shell: bash -l {0}

.pre-commit-config.yaml

@@ -30,7 +30,7 @@ repos:
  args: ["--profile", "black", "--filter-files"]
 
 - repo: https://github.com/pre-commit/mirrors-mypy
- rev: v0.991
+ rev: v1.0.1
  hooks:
  - id: mypy
  additional_dependencies: [types-filelock, types-requests, types-toml, types-PyYAML, types-freezegun, types-setuptools, pydantic]

README.md

@@ -4,6 +4,7 @@
 [![PyPI](https://img.shields.io/pypi/v/conda-lock?style=for-the-badge)](https://pypi.org/project/conda-lock/)
 [![Conda](https://img.shields.io/conda/v/conda-forge/conda-lock?style=for-the-badge)](https://github.com/conda-forge/conda-lock-feedstock)
 [![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?style=for-the-badge)](https://results.pre-commit.ci/latest/github/conda/conda-lock/main)
+[![codecov](https://img.shields.io/codecov/c/github/conda/conda-lock/main?style=for-the-badge)](https://codecov.io/gh/conda/conda-lock)
 
 Conda lock is a lightweight library that can be used to generate fully reproducible lock files for [conda][conda]
 environments.
@@ -152,8 +153,7 @@ The default category is `main`.
 ### pip support
 
 `conda-lock` can also lock the `dependencies.pip` section of
-[environment.yml][envyaml], using [Poetry's][poetry] dependency solver, if
-installed with the `pip_support` extra.
+[environment.yml][envyaml], using a vendored copy of [Poetry's][poetry] dependency solver.
 
 ### private pip repositories
 Right now `conda-lock` only supports [legacy](https://warehouse.pypa.io/api-reference/legacy.html) pypi repos with basic auth. Most self-hosted repositories like Nexus, Artifactory etc. use this. To use this feature, add your private repo into Poetry's config _including_ the basic auth in the url:

codecov.yml

@@ -0,0 +1,8 @@
+coverage:
+ status:
+ patch: false
+ project: false
+
+github_checks: false
+
+comment: false

conda_lock/conda_lock.py

@@ -53,17 +53,7 @@
  determine_conda_executable,
  is_micromamba,
 )
-from conda_lock.models.channel import Channel
-
-
-try:
- from conda_lock.pypi_solver import solve_pypi
-
- PIP_SUPPORT = True
-except ImportError:
- PIP_SUPPORT = False
 from conda_lock.lockfile import (
- Dependency,
  GitMeta,
  InputMeta,
  LockedDependency,
@@ -76,12 +66,11 @@
  write_conda_lock_file,
 )
 from conda_lock.lookup import set_lookup_location
-from conda_lock.src_parser import LockSpecification, aggregate_lock_specs
-from conda_lock.src_parser.environment_yaml import parse_environment_file
-from conda_lock.src_parser.meta_yaml import parse_meta_yaml_file
-from conda_lock.src_parser.pyproject_toml import parse_pyproject_toml
+from conda_lock.models.channel import Channel
+from conda_lock.models.lock_spec import LockSpecification
+from conda_lock.pypi_solver import solve_pypi
+from conda_lock.src_parser import make_lock_spec
 from conda_lock.virtual_package import (
- FakeRepoData,
  default_virtual_package_repodata,
  virtual_package_repo_from_specification,
 )
@@ -90,11 +79,15 @@
 logger = logging.getLogger(__name__)
 DEFAULT_FILES = [pathlib.Path("environment.yml")]
 
-# Captures basic auth credentials, if they exists, in the second capture group.
-AUTH_PATTERN = re.compile(r"^(https?:\/\/)(.*:.*@)?(.*)")
+# Captures basic auth credentials, if they exists, in the third capture group.
+AUTH_PATTERN = re.compile(r"^(# pip .* @ )?(https?:\/\/)(.*:.*@)?(.*)")
 
-# Captures the domain in the second group.
-DOMAIN_PATTERN = re.compile(r"^(https?:\/\/)?([^\/]+)(.*)")
+# Do not substitute in comments, but do substitute in pip installable packages
+# with the pattern: # pip package @ url.
+PKG_PATTERN = re.compile(r"(^[^#@].*|^# pip .*)")
+
+# Captures the domain in the third group.
+DOMAIN_PATTERN = re.compile(r"^(# pip .* @ )?(https?:\/\/)?([^\/]+)(.*)")
 
 # Captures the platform in the first group.
 PLATFORM_PATTERN = re.compile(r"^# platform: (.*)$")
@@ -114,8 +107,6 @@
  sys.exit(1)
 
 
-DEFAULT_PLATFORMS = ["osx-64", "linux-64", "win-64"]
-
 KIND_EXPLICIT: Literal["explicit"] = "explicit"
 KIND_LOCK: Literal["lock"] = "lock"
 KIND_ENV: Literal["env"] = "env"
@@ -242,44 +233,6 @@ def fn_to_dist_name(fn: str) -> str:
  return fn
 
 
-def make_lock_spec(
- *,
- src_files: List[pathlib.Path],
- virtual_package_repo: FakeRepoData,
- channel_overrides: Optional[Sequence[str]] = None,
- platform_overrides: Optional[Sequence[str]] = None,
- required_categories: Optional[AbstractSet[str]] = None,
-) -> LockSpecification:
- """Generate the lockfile specs from a set of input src_files. If required_categories is set filter out specs that do not match those"""
- lock_specs = parse_source_files(
- src_files=src_files, platform_overrides=platform_overrides
- )
-
- lock_spec = aggregate_lock_specs(lock_specs)
- lock_spec.virtual_package_repo = virtual_package_repo
- lock_spec.channels = (
- [Channel.from_string(co) for co in channel_overrides]
- if channel_overrides
- else lock_spec.channels
- )
- lock_spec.platforms = (
- list(platform_overrides) if platform_overrides else lock_spec.platforms
- ) or list(DEFAULT_PLATFORMS)
-
- if required_categories is not None:
-
- def dep_has_category(d: Dependency, categories: AbstractSet[str]) -> bool:
- return d.category in categories
-
- lock_spec.dependencies = [
- d
- for d in lock_spec.dependencies
- if dep_has_category(d, categories=required_categories)
- ]
-
- return lock_spec
-
-
 def make_lock_files(
  *,
  conda: PathLike,
@@ -678,10 +631,10 @@ def sanitize_lockfile_line(line: str) -> str:
 
  if len(pip_deps) > 0:
  logger.warning(
- "WARNING: installation of pip dependencies is only supported "
- "by the 'conda-lock install' command. Other tools may silently "
- "ignore them. For portability, we recommend using the newer "
- "unified lockfile format (i.e. removing the --kind=explicit "
+ "WARNING: installation of pip dependencies is only supported by the "
+ "'conda-lock install' and 'micromamba install' commands. Other tools "
+ "may silently ignore them. For portability, we recommend using the "
+ "newer unified lockfile format (i.e. removing the --kind=explicit "
  "argument."
  )
  else:
@@ -729,8 +682,6 @@ def _solve_for_arch(
  )
 
  if requested_deps_by_name["pip"]:
- if not PIP_SUPPORT:
- raise ValueError("pip support is not enabled")
  if "python" not in conda_deps:
  raise ValueError("Got pip specs without Python")
  pip_deps = solve_pypi(
@@ -866,42 +817,6 @@ def create_lockfile_from_spec(
  )
 
 
-def parse_source_files(
- src_files: List[pathlib.Path],
- platform_overrides: Optional[Sequence[str]],
-) -> List[LockSpecification]:
- """
- Parse a sequence of dependency specifications from source files
-
- Parameters
- ----------
- src_files :
- Files to parse for dependencies
- platform_overrides :
- Target platforms to render environment.yaml and meta.yaml files for
- """
- desired_envs: List[LockSpecification] = []
- for src_file in src_files:
- if src_file.name == "meta.yaml":
- desired_envs.append(
- parse_meta_yaml_file(
- src_file, list(platform_overrides or DEFAULT_PLATFORMS)
- )
- )
- elif src_file.name == "pyproject.toml":
- desired_envs.append(parse_pyproject_toml(src_file))
- else:
- desired_envs.append(
- parse_environment_file(
- src_file,
- platform_overrides,
- default_platforms=DEFAULT_PLATFORMS,
- pip_support=PIP_SUPPORT,
- )
- )
- return desired_envs
-
-
 def _add_auth_to_line(line: str, auth: Dict[str, str]) -> str:
  matching_auths = [a for a in auth if a in line]
  if not matching_auths:
@@ -914,7 +829,7 @@ def _add_auth_to_line(line: str, auth: Dict[str, str]) -> str:
 
 def _add_auth_to_lockfile(lockfile: str, auth: Dict[str, str]) -> str:
  lockfile_with_auth = "\n".join(
- _add_auth_to_line(line, auth) if line[0] not in ("#", "@") else line
+ _add_auth_to_line(line, auth) if PKG_PATTERN.match(line) else line
  for line in lockfile.strip().split("\n")
  )
  if lockfile.endswith("\n"):
@@ -930,17 +845,17 @@ def _add_auth(lockfile: str, auth: Dict[str, str]) -> Iterator[pathlib.Path]:
 
 
 def _strip_auth_from_line(line: str) -> str:
- return AUTH_PATTERN.sub(r"\1\3", line)
+ return AUTH_PATTERN.sub(r"\1\2\4", line)
 
 
 def _extract_domain(line: str) -> str:
- return DOMAIN_PATTERN.sub(r"\2", line)
+ return DOMAIN_PATTERN.sub(r"\3", line)
 
 
 def _strip_auth_from_lockfile(lockfile: str) -> str:
  lockfile_lines = lockfile.strip().split("\n")
  stripped_lockfile_lines = tuple(
- _strip_auth_from_line(line) if line[0] not in ("#", "@") else line
+ _strip_auth_from_line(line) if PKG_PATTERN.match(line) else line
  for line in lockfile_lines
  )
  stripped_domains = sorted(

conda_lock/conda_solver.py

@@ -33,7 +33,7 @@
 )
 from conda_lock.lockfile import HashModel, LockedDependency, _apply_categories
 from conda_lock.models.channel import Channel
-from conda_lock.src_parser import Dependency, VersionedDependency
+from conda_lock.models.lock_spec import Dependency, VersionedDependency
 
 
 logger = logging.getLogger(__name__)

conda_lock/lockfile/__init__.py

@@ -7,7 +7,7 @@
 
 import yaml
 
-from conda_lock.src_parser import Dependency
+from conda_lock.models.lock_spec import Dependency
 
 from .models import DependencySource as DependencySource
 from .models import GitMeta as GitMeta

conda_lock/models/__init__.py

@@ -2,6 +2,8 @@
 
 
 class StrictModel(BaseModel):
+ """A Pydantic BaseModel forbidding extra fields and encoding frozensets as lists"""
+
  class Config:
  extra = "forbid"
  json_encoders = {

conda_lock/models/channel.py

@@ -68,7 +68,7 @@ class CondaUrl(BaseModel):
 
  @classmethod
  def from_string(cls, value: str) -> "CondaUrl":
- return env_var_normalize(value)
+ return _env_var_normalize(value)
 
  def conda_token_replaced_url(self) -> str:
  """This is basically a crazy thing that conda does for the token replacement in the output"""
@@ -134,7 +134,7 @@ def conda_token_replaced_url(self) -> str:
  return expanded_url
 
 
-def detect_used_env_var(
+def _detect_used_env_var(
  value: str, preferred_env_var_suffix: List[str]
 ) -> Optional[str]:
  """Detects if the string exactly matches any current environment variable
@@ -157,7 +157,7 @@ def detect_used_env_var(
  return None
 
 
-def env_var_normalize(url: str) -> CondaUrl:
+def _env_var_normalize(url: str) -> CondaUrl:
  """
  Normalizes url by using env vars
  """
@@ -195,7 +195,7 @@ def get_or_raise(val: Optional[str]) -> str:
  return val
 
  if res.username:
- user_env_var = detect_used_env_var(res.username, ["USERNAME", "USER"])
+ user_env_var = _detect_used_env_var(res.username, ["USERNAME", "USER"])
  if user_env_var:
  res_replaced = res_replaced._replace(
  netloc=make_netloc(
@@ -206,7 +206,7 @@ def get_or_raise(val: Optional[str]) -> str:
  )
  )
  if res.password:
- password_env_var = detect_used_env_var(
+ password_env_var = _detect_used_env_var(
  res.password, ["PASSWORD", "PASS", "TOKEN", "KEY"]
  )
  if password_env_var:
@@ -222,7 +222,7 @@ def get_or_raise(val: Optional[str]) -> str:
  _token_match = token_pattern.search(res.path)
  token = _token_match.groups()[1][3:] if _token_match else None
  if token:
- token_env_var = detect_used_env_var(
+ token_env_var = _detect_used_env_var(
  token, ["TOKEN", "CRED", "PASSWORD", "PASS", "KEY"]
  )
  if not token_env_var: