fix: project.unpack() to include full contracts directory (ApeWorX#…

…2123)

pyproject.toml

@@ -12,7 +12,7 @@ plugins = ["pydantic.mypy"]
 # This version is purposely set to really high minor so that it should always work
 # with newer, stricter plugin releases.
 # NOTE: This should be bumped with every minor release!
-fallback_version = "0.7.999"
+fallback_version = "0.8.999"
 write_to = "src/ape/version.py"
 
 # NOTE: you have to use single-quoted strings in TOML for regular expressions.
@@ -21,7 +21,7 @@ write_to = "src/ape/version.py"
 # character.
 [tool.black]
 line-length = 100
-target-version = ['py38', 'py39', 'py310', 'py311', 'py312']
+target-version = ['py39', 'py310', 'py311', 'py312']
 include = '\.pyi?$'
 
 [tool.pytest.ini_options]

src/ape/cli/arguments.py

@@ -1,5 +1,4 @@
 from collections.abc import Iterable
-from functools import cached_property
 from pathlib import Path
 from typing import TYPE_CHECKING, Optional, Union
 
@@ -64,7 +63,7 @@ def callback(cls, ctx, param, value) -> set[Path]:
  project = ctx.params.get("project")
  return cls(value, project=project).filtered_paths
 
- @cached_property
+ @property
  def filtered_paths(self) -> set[Path]:
  """
  Get the filtered set of paths.
@@ -78,9 +77,10 @@ def filtered_paths(self) -> set[Path]:
  elif not value or value == "*":
  # Get all file paths in the project.
  return {p for p in self.project.sources.paths}
- else:
- # Given a sequence of paths.
+ elif isinstance(value, Iterable):
  contract_paths = value
+ else:
+ raise BadArgumentUsage(f"Not a path or iter[Path]: {value}")
 
  # Convert source IDs or relative paths to absolute paths.
  path_set = self.lookup(contract_paths)
@@ -120,8 +120,9 @@ def compiler_is_unknown(self, path: Union[Path, str]) -> bool:
 
  def lookup(self, path_iter: Iterable, path_set: Optional[set] = None) -> set[Path]:
  path_set = path_set or set()
+ given_paths = [p for p in path_iter] # Handle iterators w/o losing it.
 
- for path_id in path_iter:
+ for path_id in given_paths:
  path = Path(path_id)
  contracts_folder = self.project.contracts_folder
  if (
@@ -148,13 +149,8 @@ def lookup(self, path_iter: Iterable, path_set: Optional[set] = None) -> set[Pat
  # NOTE: ^ Also tracks.
  continue
 
- suffix = get_full_extension(resolved_path)
- if suffix in self.compiler_manager.registered_compilers:
- # File exists and is compile-able.
- path_set.add(resolved_path)
-
- elif suffix:
- raise BadArgumentUsage(f"Source file '{resolved_path.name}' not found.")
+ # We know here that the compiler is known.
+ path_set.add(resolved_path)
 
  else:
  raise BadArgumentUsage(f"Source file '{path.name}' not found.")

src/ape/managers/compilers.py

@@ -122,7 +122,6 @@ def compile(
 
  pm = project or self.local_project
  files_by_ext = defaultdict(list)
-
  if isinstance(contract_filepaths, (str, Path)):
  contract_filepaths = (contract_filepaths,)
 

src/ape/managers/project.py

@@ -77,7 +77,7 @@ def __len__(self) -> int:
  if self._path_cache is not None:
  return len(self._path_cache)
 
- # Will set _path_cache, eliminates need to iterte (perf).
+ # Will set _path_cache, eliminates need to iterate (perf).
  return len(list(self.paths))
 
  def __iter__(self) -> Iterator[str]:
@@ -216,9 +216,11 @@ def is_excluded(self, path: Path) -> bool:
  for excl in self.exclude_globs:
  if isinstance(excl, Pattern):
  for opt in options:
- if excl.match(opt):
- self._exclude_cache[source_id] = True
- return True
+ if not excl.match(opt):
+ continue
+
+ self._exclude_cache[source_id] = True
+ return True
 
  else:
  # perf: Check parent directory first to exclude faster by marking them all.
@@ -280,13 +282,10 @@ def find_in_dir(dir_path: Path, path: Path) -> Optional[Path]:
  for file in full_path.parent.iterdir():
  if not file.is_file():
  continue
- elif not (file_ext := get_full_extension(file)):
- continue
 
  # Check exact match w/o extension.
- prefix = file_ext.join(str(file).split(file_ext)[:-1])
-
- if str(full_path) == prefix:
+ prefix = str(file.with_suffix("")).strip(" /\\")
+ if str(full_path).strip(" /\\") == prefix:
  return file
 
  # Look for stem-only matches (last resort).
@@ -1786,6 +1785,7 @@ def extract_manifest(self) -> PackageManifest:
 
  def clean(self):
  self._manifest.contract_types = None
+ self._config_override = {}
 
 
 class DeploymentManager(ManagerAccessMixin):
@@ -2246,7 +2246,17 @@ def isolate_in_tempdir(self, **config_override) -> Iterator["LocalProject"]:
  yield project
 
  def unpack(self, destination: Path, config_override: Optional[dict] = None) -> "LocalProject":
- project = super().unpack(destination, config_override=config_override)
+ config_override = {**self._config_override, **(config_override or {})}
+
+ # Unpack contracts.
+ if self.contracts_folder.is_dir():
+ contracts_path = get_relative_path(self.contracts_folder, self.path)
+ contracts_destination = destination / contracts_path
+ shutil.copytree(self.contracts_folder, contracts_destination, dirs_exist_ok=True)
+
+ # Unpack config file.
+ if not (destination / "ape-config.yaml").is_file():
+ self.config.write_to_disk(destination / "ape-config.yaml")
 
  # Unpack scripts folder.
  if self.scripts_folder.is_dir():
@@ -2265,7 +2275,7 @@ def unpack(self, destination: Path, config_override: Optional[dict] = None) -> "
  interfaces_destination.parent.mkdir(parents=True, exist_ok=True)
  shutil.copytree(self.interfaces_folder, interfaces_destination, dirs_exist_ok=True)
 
- return project
+ return LocalProject(destination, config_override=config_override)
 
  def load_manifest(self) -> PackageManifest:
  """

src/ape_compile/__init__.py

@@ -2,7 +2,7 @@
 from re import Pattern
 from typing import Union
 
-from pydantic import field_validator
+from pydantic import field_serializer, field_validator
 
 from ape import plugins
 from ape.api.config import ConfigEnum, PluginConfig
@@ -74,6 +74,21 @@ def validate_exclude(cls, value):
  # Include defaults.
  return {*given_values, *SOURCE_EXCLUDE_PATTERNS}
 
+ @field_serializer("exclude", when_used="json")
+ def serialize_exclude(self, exclude, info):
+ """
+ Exclude is put back with the weird r-prefix so we can
+ go to-and-from.
+ """
+ result: list[str] = []
+ for excl in exclude:
+ if isinstance(excl, Pattern):
+ result.append(f'r"{excl.pattern}"')
+ else:
+ result.append(excl)
+
+ return result
+
 
 @plugins.register(plugins.Config)
 def config_class():

src/ape_compile/_cli.py

@@ -6,6 +6,7 @@
 
 from ape.cli.arguments import contract_file_paths_argument
 from ape.cli.options import ape_cli_context, config_override_option, project_option
+from ape.utils.os import clean_path
 
 
 def _include_dependencies_callback(ctx, param, value):
@@ -92,7 +93,8 @@ def cli(
  _display_byte_code_sizes(cli_ctx, contract_types)
 
  if not compiled:
- cli_ctx.logger.warning("Nothing to compile.")
+ folder = clean_path(project.contracts_folder)
+ cli_ctx.logger.warning(f"Nothing to compile ({folder}).")
 
  if errored:
  # Ensure exit code.

tests/conftest.py

@@ -79,8 +79,8 @@ def validate_cwd(start_dir):
  os.chdir(start_dir)
 
 
-@pytest.fixture(scope="session")
-def project(config):
+@pytest.fixture
+def project():
  path = "functional/data/contracts/local"
  with ape.project.temp_config(contracts_folder=path):
  yield ape.project

tests/functional/data/projects/ApeProject/contracts/Path.with.sub.json

@@ -0,0 +1 @@
+{}

tests/functional/test_compilers.py

@@ -1,4 +1,5 @@
 from pathlib import Path
+from re import Pattern
 from typing import cast
 
 import pytest
@@ -7,6 +8,7 @@
 from ape.contracts import ContractContainer
 from ape.exceptions import APINotImplementedError, CompilerError, ContractLogicError, CustomError
 from ape.types import AddressType
+from ape_compile import Config
 
 
 def test_get_imports(project, compilers):
@@ -162,3 +164,21 @@ def test_enrich_error_custom_error_with_inputs(compilers, setup_custom_error):
  assert actual.__class__.__name__ == "AllowanceExpired"
  assert actual.inputs["deadline"] == deadline
  assert repr(actual) == f"AllowanceExpired(deadline={deadline})"
+
+
+def test_config_exclude_regex_serialize():
+ """
+ Show we can to-and-fro with exclude regexes.
+ """
+ raw_value = 'r"FooBar"'
+ cfg = Config(exclude=[raw_value])
+ excl = [x for x in cfg.exclude if isinstance(x, Pattern)]
+ assert len(excl) == 1
+ assert excl[0].pattern == "FooBar"
+ # NOTE: Use json mode to ensure we can go from most minimum value back.
+ model_dump = cfg.model_dump(mode="json", by_alias=True)
+ assert raw_value in model_dump.get("exclude", [])
+ new_cfg = Config.model_validate(cfg.model_dump(mode="json", by_alias=True))
+ excl = [x for x in new_cfg.exclude if isinstance(x, Pattern)]
+ assert len(excl) == 1
+ assert excl[0].pattern == "FooBar"

tests/functional/test_coverage.py

@@ -173,7 +173,7 @@ def src(self):
  def contract_source(self, vyper_contract_type, src):
  return ContractSource(contract_type=vyper_contract_type, source=src)
 
- @pytest.fixture(scope="class")
+ @pytest.fixture
  def coverage_data(self, project, contract_source):
  return CoverageData(project, (contract_source,))
 

tests/functional/test_project.py

@@ -13,6 +13,7 @@
 from ape.contracts import ContractContainer
 from ape.exceptions import ProjectError
 from ape.logging import LogLevel
+from ape.utils import create_tempdir
 from ape_pm import BrownieProject, FoundryProject
 from tests.conftest import skip_if_plugin_installed
 
@@ -415,6 +416,15 @@ def test_clean(tmp_project):
  assert tmp_project.sources._path_cache is None
 
 
+def test_unpack(project_with_source_files_contract):
+ with create_tempdir() as path:
+ project_with_source_files_contract.unpack(path)
+ assert (path / "contracts" / "Contract.json").is_file()
+
+ # Show that even non-sources end up in the unpacked destination.
+ assert (path / "contracts" / "Path.with.sub.json").is_file()
+
+
 def test_add_compiler_data(project_with_dependency_config):
  # NOTE: Using different project than default to lessen
  # chance of race-conditions from multi-process test runners.

tests/functional/utils/test_misc.py

@@ -1,3 +1,5 @@
+import asyncio
+
 import pytest
 from eth_pydantic_types import HexBytes
 from packaging.version import Version
@@ -73,6 +75,8 @@ def test_run_until_complete_coroutine():
  async def foo():
  return 3
 
+ loop = asyncio.new_event_loop()
+ asyncio.set_event_loop(loop)
  actual = run_until_complete(foo())
  assert actual == 3
 
@@ -84,6 +88,8 @@ async def foo():
  async def bar():
  return 4
 
+ loop = asyncio.new_event_loop()
+ asyncio.set_event_loop(loop)
  actual = run_until_complete(foo(), bar())
  assert actual == [3, 4]
 

tests/integration/cli/conftest.py

@@ -85,7 +85,7 @@ def pytest_collection_modifyitems(session, config, items):
  items[:] = modified_items
 
 
-@pytest.fixture(autouse=True)
+@pytest.fixture(autouse=True, scope="session")
 def project_dir_map():
  """
  Ensure only copying projects once to prevent `TooManyOpenFilesError`.
@@ -97,9 +97,12 @@ class ProjectDirCache:
  def load(self, name: str) -> Path:
  base_path = Path(__file__).parent / "projects"
  if name in self.project_map:
- # Already copied.
- return self.project_map[name]
+ res = self.project_map[name]
+ if res.is_dir():
+ # Already copied and still exists!
+ return res
 
+ # Either re-copy or copy for the first time.
  project_source_dir = __projects_directory__ / name
  project_dest_dir = base_path / project_source_dir.name
  project_dest_dir.parent.mkdir(exist_ok=True, parents=True)
@@ -114,11 +117,17 @@ def load(self, name: str) -> Path:
 
 
 @pytest.fixture(autouse=True, params=__project_names__)
-def project(request, config, project_dir_map):
+def integ_project(request, project_dir_map):
  project_dir = project_dir_map.load(request.param)
+ if not project_dir.is_dir():
+ # Should not happen because of logic in fixture,
+ # but just in case!
+ pytest.fail("Setup failed - project dir not exists.")
+
  root_project = Project(project_dir)
  # Using tmp project so no .build folder get kept around.
  with root_project.isolate_in_tempdir(name=request.param) as tmp_project:
+ assert tmp_project.path.is_dir(), "Setup failed - tmp-project dir not exists"
  yield tmp_project