Merge pull request #105 from adfinis/better-state-caching

feat: Replace SystemStateReader.read() with lru_cache

pyaptly/cli.py

@@ -79,7 +79,7 @@ def legacy(passthrough):
 )
 @click.argument("config", type=click.Path(file_okay=True, dir_okay=False, exists=True))
 @click.argument("task", type=click.Choice(["create"]))
-@click.option("--repo-name", "-n", default="all", type=str, help='deafult: "all"')
+@click.option("--repo-name", "-n", default="all", type=str, help='default: "all"')
 def repo(**kwargs):
  """Create aptly repos."""
  from . import main, repo
@@ -102,7 +102,7 @@ def repo(**kwargs):
 )
 @click.argument("config", type=click.Path(file_okay=True, dir_okay=False, exists=True))
 @click.argument("task", type=click.Choice(["create", "update"]))
-@click.option("--mirror-name", "-n", default="all", type=str, help='deafult: "all"')
+@click.option("--mirror-name", "-n", default="all", type=str, help='default: "all"')
 def mirror(**kwargs):
  """Manage aptly mirrors."""
  from . import main, mirror
@@ -125,7 +125,7 @@ def mirror(**kwargs):
 )
 @click.argument("config", type=click.Path(file_okay=True, dir_okay=False, exists=True))
 @click.argument("task", type=click.Choice(["create", "update"]))
-@click.option("--snapshot-name", "-n", default="all", type=str, help='deafult: "all"')
+@click.option("--snapshot-name", "-n", default="all", type=str, help='default: "all"')
 def snapshot(**kwargs):
  """Manage aptly snapshots."""
  from . import main, snapshot
@@ -148,7 +148,7 @@ def snapshot(**kwargs):
 )
 @click.argument("config", type=click.Path(file_okay=True, dir_okay=False, exists=True))
 @click.argument("task", type=click.Choice(["create", "update"]))
-@click.option("--publish-name", "-n", default="all", type=str, help='deafult: "all"')
+@click.option("--publish-name", "-n", default="all", type=str, help='default: "all"')
 def publish(**kwargs):
  """Manage aptly publishs."""
  from . import main, publish

pyaptly/command.py

@@ -3,8 +3,6 @@
 import collections
 import logging
 
-from frozendict import frozendict
-
 from . import state_reader, util
 
 lg = logging.getLogger(__name__)
@@ -27,7 +25,6 @@ def __init__(self, cmd: list[str]):
  self._finished: bool = False
  self._known_dependency_types = (
  "snapshot",
- "mirror",
  "repo",
  "publish",
  "virtual",
@@ -84,6 +81,21 @@ def provide(self, type_, identifier):
  assert type_ in self._known_dependency_types
  self._provides.add((type_, str(identifier)))
 
+ def clear_caches(self):
+ """Clear state_reader caches of functions which have changed"""
+ provides = set(p[0] for p in self.get_provides())
+ for provide in provides:
+ lg.debug("clearing cache for " + provide)
+ match provide:
+ case "snapshot":
+ state_reader.state_reader().snapshots.cache_clear()
+ state_reader.state_reader().snapshot_map.cache_clear()
+ case "repo":
+ state_reader.state_reader().repos.cache_clear()
+ case "publish":
+ state_reader.state_reader().publishes.cache_clear()
+ state_reader.state_reader().publish_map.cache_clear()
+
  def execute(self):
  """Execute the command. Return the return value of the command.
 
@@ -103,6 +115,7 @@ def execute(self):
  # So I decided to change that. For now we fail hard if a `Command` fails.
  # I guess we will see in production what happens.
  util.run_command(self.cmd, check=True)
+ self.clear_caches()
  else:
  lg.info("Pretending to run command: %s", " ".join(self.cmd))
  self._finished = True
@@ -269,8 +282,6 @@ def order_commands(commands, has_dependency_cb=lambda x: False):
  # Break out of the requirements loop, as the
  # command cannot be scheduled anyway.
  break
- # command cannot be scheduled anyway.
- break
 
  if can_schedule:
  lg.debug("%s: all dependencies fulfilled" % cmd)
@@ -297,70 +308,42 @@ def order_commands(commands, has_dependency_cb=lambda x: False):
  return scheduled
 
 
-class FunctionCommand(Command):
- """Repesents a function command.
+class DummyCommand(Command):
+ """Represents a dummy command.
 
- Is used to resolve dependencies between such commands. This command executes
- the given function. *args and **kwargs are passed through.
+ Is used to resolve dependencies between commands, but does nothing itself
 
  :param func: The function to execute
  :type func: callable
  """
 
- def __init__(self, func, *args, **kwargs):
+ def __init__(self, identifier: str):
  super().__init__([])
-
- assert callable(func)
- self.cmd = [str(id(func))]
- self.func = func
- self.args = args
- self.kwargs = kwargs
+ self.identifier = identifier
 
  def freeze(self):
  """Freeze the class to make it hashable."""
  self._freeze_common()
- # manually checking using self.frozen
- self.kwargs = frozendict(self.kwargs) # type: ignore
 
  def __hash__(self):
  """Hash the class."""
  dependencies_hash = self._hash_base()
- return hash((id(self.func), self.args, self.kwargs, dependencies_hash))
+ return hash((self.identifier, dependencies_hash))
 
  def __eq__(self, other):
  """Compare the class."""
- return (
- self._eq_base(other)
- and id(self.func) == id(other.func)
- and self.args == other.args
- and self.kwargs == other.kwargs
- )
+ return self._eq_base(other) and self.identifier == other.identifier
 
  def execute(self):
- """Execute the command.
-
- Call the function.
- """
+ """Mark command as executed"""
  if self._finished: # pragma: no cover
  return self._finished
  if not Command.pretend_mode:
- lg.debug(
- "Running code: %s(args=%s, kwargs=%s)",
- self.func.__name__,
- repr(self.args),
- repr(self.kwargs),
- )
-
- self.func(*self.args, **self.kwargs)
+ lg.debug("Running dummy Command with provides %s", self._provides)
 
  self._finished = True
  else: # pragma: no cover
- lg.info(
- "Pretending to run code: %s(args=%s, kwargs=%s)",
- self.repr_cmd(),
- repr(self.args),
- repr(self.kwargs),
- )
+ lg.info("Pretending to run dummy Command with provides: %s", self._provides)
 
  return self._finished
 
@@ -369,13 +352,11 @@ def repr_cmd(self):
 
  :rtype: str
  """
- # We need to "id" ourselves here so that multiple commands that call a
- # function with the same name won't be shown as being equal.
- return "%s|%s" % (self.func.__name__, id(self))
+ return self.identifier
 
  def __repr__(self):
- """Show repr for FunctionCommand."""
- return "FunctionCommand<%s requires %s, provides %s>\n" % (
+ """Show repr for DummyCommand."""
+ return "DummyCommand<%s requires %s, provides %s>\n" % (
  self.repr_cmd(),
  ", ".join([repr(x) for x in self._requires]),
  ", ".join([repr(x) for x in self._provides]),

pyaptly/conftest.py

@@ -65,6 +65,14 @@ def environment(debug_mode):
  os.environ["GNUPGHOME"] = str(gnupg)
  util._PYTEST_KEYSERVER = "hkp://127.0.0.1:8080"
 
+ # Make sure we start with a clean slate
+ state_reader.state_reader().mirrors.cache_clear()
+ state_reader.state_reader().snapshots.cache_clear()
+ state_reader.state_reader().snapshot_map.cache_clear()
+ state_reader.state_reader().repos.cache_clear()
+ state_reader.state_reader().publishes.cache_clear()
+ state_reader.state_reader().publish_map.cache_clear()
+
  try:
  yield
  finally:
@@ -78,6 +86,7 @@ def test_key_03(environment):
  """Get test gpg-key number 3."""
  util.run_command(["gpg", "--import", setup_base / "test03.key"], check=True)
  util.run_command(["gpg", "--import", setup_base / "test03.pub"], check=True)
+ state_reader.state_reader().gpg_keys.cache_clear()
 
 
 @pytest.fixture()
@@ -120,11 +129,9 @@ def mirror_update(environment, config):
  """Test if updating mirrors works."""
  args = ["-c", config, "mirror", "create"]
  state = state_reader.SystemStateReader()
- state.read()
- assert "fakerepo01" not in state.mirrors
+ assert "fakerepo01" not in state.mirrors()
  main.main(args)
- state.read()
- assert "fakerepo01" in state.mirrors
+ assert "fakerepo01" in state.mirrors()
  args[3] = "update"
  main.main(args)
  args = [
@@ -144,9 +151,8 @@ def snapshot_create(config, mirror_update, freeze):
  args = ["-c", config, "snapshot", "create"]
  main.main(args)
  state = state_reader.SystemStateReader()
- state.read()
  assert set(["fakerepo01-20121010T0000Z", "fakerepo02-20121006T0000Z"]).issubset(
- state.snapshots
+ state.snapshots()
  )
  yield state
 
@@ -162,29 +168,27 @@ def snapshot_update_rotating(config, mirror_update, freeze):
  ]
  main.main(args)
  state = state_reader.SystemStateReader()
- state.read()
  assert set(
  [
  "fake-current",
  "fakerepo01-current",
  "fakerepo02-current",
  ]
- ).issubset(state.snapshots)
+ ).issubset(state.snapshots())
  args = [
  "-c",
  config,
  "snapshot",
  "update",
  ]
  main.main(args)
- state.read()
  assert set(
  [
  "fake-current",
  "fakerepo01-current-rotated-20121010T1010Z",
  "fakerepo02-current-rotated-20121010T1010Z",
  ]
- ).issubset(state.snapshots)
+ ).issubset(state.snapshots())
  expected = {
  "fake-current": set(["fakerepo01-current", "fakerepo02-current"]),
  "fake-current-rotated-20121010T1010Z": set(
@@ -198,7 +202,7 @@ def snapshot_update_rotating(config, mirror_update, freeze):
  "fakerepo02-current": set([]),
  "fakerepo02-current-rotated-20121010T1010Z": set([]),
  }
- assert state.snapshot_map == expected
+ assert state.snapshot_map() == expected
 
 
 @pytest.fixture()
@@ -207,7 +211,6 @@ def repo_create(environment, config, test_key_03):
  args = ["-c", config, "repo", "create"]
  main.main(args)
  state = state_reader.SystemStateReader()
- state.read()
  util.run_command(
  [
  "aptly",
@@ -217,7 +220,8 @@ def repo_create(environment, config, test_key_03):
  "/source/compose/setup/hellome_0.1-1_amd64.deb",
  ]
  )
- assert set(["centrify"]) == state.repos
+ state_reader.state_reader().repos.cache_clear()
+ assert set(["centrify"]) == state.repos()
 
 
 @pytest.fixture()
@@ -226,13 +230,12 @@ def publish_create(config, snapshot_create, test_key_03):
  args = ["-c", config, "publish", "create"]
  main.main(args)
  state = state_reader.SystemStateReader()
- state.read()
- assert set(["fakerepo02 main", "fakerepo01 main"]) == state.publishes
+ assert set(["fakerepo02 main", "fakerepo01 main"]) == state.publishes()
  expect = {
  "fakerepo02 main": set(["fakerepo02-20121006T0000Z"]),
  "fakerepo01 main": set(["fakerepo01-20121010T0000Z"]),
  }
- assert expect == state.publish_map
+ assert expect == state.publish_map()
 
 
 @pytest.fixture()
@@ -241,7 +244,6 @@ def publish_create_rotating(config, snapshot_update_rotating, test_key_03):
  args = ["-c", config, "publish", "create"]
  main.main(args)
  state = state_reader.SystemStateReader()
- state.read()
  assert (
  set(
  [
@@ -250,14 +252,14 @@ def publish_create_rotating(config, snapshot_update_rotating, test_key_03):
  "fakerepo02/current stable",
  ]
  )
- == state.publishes
+ == state.publishes()
  )
  expect = {
  "fake/current stable": set(["fake-current"]),
  "fakerepo01/current stable": set(["fakerepo01-current"]),
  "fakerepo02/current stable": set(["fakerepo02-current"]),
  }
- assert expect == state.publish_map
+ assert expect == state.publish_map()
 
 
 @pytest.fixture()
@@ -277,5 +279,4 @@ def publish_create_republish(config, publish_create, caplog):
  ]
  main.main(args)
  state = state_reader.SystemStateReader()
- state.read()
- assert "fakerepo01-stable main" in state.publishes
+ assert "fakerepo01-stable main" in state.publishes()

pyaptly/main.py

@@ -13,7 +13,6 @@
  publish,
  repo,
  snapshot,
- state_reader,
 )
 
 _logging_setup = False
@@ -48,7 +47,6 @@ def prepare(args):
 
  with open(args.config, "rb") as f:
  cfg = tomli.load(f)
- state_reader.state_reader().read()
  return cfg