feat: Replace SystemStateReader.read() with lru_cache

pyaptly/cli.py

@@ -88,7 +88,7 @@ def legacy(passthrough):
 )
 @click.argument("config", type=click.Path(file_okay=True, dir_okay=False, exists=True))
 @click.argument("task", type=click.Choice(["create"]))
-@click.option("--repo-name", "-n", default="all", type=str, help='deafult: "all"')
+@click.option("--repo-name", "-n", default="all", type=str, help='default: "all"')
 def repo(**kwargs):
     """Create aptly repos."""
     from . import main, repo
@@ -111,7 +111,7 @@ def repo(**kwargs):
 )
 @click.argument("config", type=click.Path(file_okay=True, dir_okay=False, exists=True))
 @click.argument("task", type=click.Choice(["create", "update"]))
-@click.option("--mirror-name", "-n", default="all", type=str, help='deafult: "all"')
+@click.option("--mirror-name", "-n", default="all", type=str, help='default: "all"')
 def mirror(**kwargs):
     """Manage aptly mirrors."""
     from . import main, mirror
@@ -134,7 +134,7 @@ def mirror(**kwargs):
 )
 @click.argument("config", type=click.Path(file_okay=True, dir_okay=False, exists=True))
 @click.argument("task", type=click.Choice(["create", "update"]))
-@click.option("--snapshot-name", "-n", default="all", type=str, help='deafult: "all"')
+@click.option("--snapshot-name", "-n", default="all", type=str, help='default: "all"')
 def snapshot(**kwargs):
     """Manage aptly snapshots."""
     from . import main, snapshot
@@ -157,7 +157,7 @@ def snapshot(**kwargs):
 )
 @click.argument("config", type=click.Path(file_okay=True, dir_okay=False, exists=True))
 @click.argument("task", type=click.Choice(["create", "update"]))
-@click.option("--publish-name", "-n", default="all", type=str, help='deafult: "all"')
+@click.option("--publish-name", "-n", default="all", type=str, help='default: "all"')
 def publish(**kwargs):
     """Manage aptly publishs."""
     from . import main, publish

pyaptly/command.py

@@ -3,8 +3,6 @@
 import collections
 import logging
 
-from frozendict import frozendict
-
 from . import state_reader, util
 
 lg = logging.getLogger(__name__)
@@ -27,7 +25,6 @@ def __init__(self, cmd: list[str]):
         self._finished: bool = False
         self._known_dependency_types = (
             "snapshot",
-            "mirror",
             "repo",
             "publish",
             "virtual",
@@ -84,6 +81,21 @@ def provide(self, type_, identifier):
         assert type_ in self._known_dependency_types
         self._provides.add((type_, str(identifier)))
 
+    def clear_caches(self):
+        """Clear state_reader caches of functions which have changed"""
+        provides = set(p[0] for p in self.get_provides())
+        for provide in provides:
+            lg.debug("clearing cache for " + provide)
+            match provide:
+                case "snapshot":
+                    state_reader.state_reader().snapshots.cache_clear()
+                    state_reader.state_reader().snapshot_map.cache_clear()
+                case "repo":
+                    state_reader.state_reader().repos.cache_clear()
+                case "publish":
+                    state_reader.state_reader().publishes.cache_clear()
+                    state_reader.state_reader().publish_map.cache_clear()
+
     def execute(self):
         """Execute the command. Return the return value of the command.
 
@@ -103,6 +115,7 @@ def execute(self):
             # So I decided to change that. For now we fail hard if a `Command` fails.
             # I guess we will see in production what happens.
             util.run_command(self.cmd, check=True)
+            self.clear_caches()
         else:
             lg.info("Pretending to run command: %s", " ".join(self.cmd))
         self._finished = True
@@ -269,8 +282,6 @@ def order_commands(commands, has_dependency_cb=lambda x: False):
                             # Break out of the requirements loop, as the
                             # command cannot be scheduled anyway.
                             break
-                            # command cannot be scheduled anyway.
-                            break
 
                 if can_schedule:
                     lg.debug("%s: all dependencies fulfilled" % cmd)
@@ -297,70 +308,42 @@ def order_commands(commands, has_dependency_cb=lambda x: False):
         return scheduled
 
 
-class FunctionCommand(Command):
-    """Repesents a function command.
+class DummyCommand(Command):
+    """Represents a dummy command.
 
-    Is used to resolve dependencies between such commands. This command executes
-    the given function. *args and **kwargs are passed through.
+    Is used to resolve dependencies between commands, but does nothing itself
 
     :param func: The function to execute
     :type  func: callable
     """
 
-    def __init__(self, func, *args, **kwargs):
+    def __init__(self, identifier: str):
         super().__init__([])
-
-        assert callable(func)
-        self.cmd = [str(id(func))]
-        self.func = func
-        self.args = args
-        self.kwargs = kwargs
+        self.identifier = identifier
 
     def freeze(self):
         """Freeze the class to make it hashable."""
         self._freeze_common()
-        # manually checking using self.frozen
-        self.kwargs = frozendict(self.kwargs)  # type: ignore
 
     def __hash__(self):
         """Hash the class."""
         dependencies_hash = self._hash_base()
-        return hash((id(self.func), self.args, self.kwargs, dependencies_hash))
+        return hash((self.identifier, dependencies_hash))
 
     def __eq__(self, other):
         """Compare the class."""
-        return (
-            self._eq_base(other)
-            and id(self.func) == id(other.func)
-            and self.args == other.args
-            and self.kwargs == other.kwargs
-        )
+        return self._eq_base(other) and self.identifier == other.identifier
 
     def execute(self):
-        """Execute the command.
-
-        Call the function.
-        """
+        """Mark command as executed"""
         if self._finished:  # pragma: no cover
             return self._finished
         if not Command.pretend_mode:
-            lg.debug(
-                "Running code: %s(args=%s, kwargs=%s)",
-                self.func.__name__,
-                repr(self.args),
-                repr(self.kwargs),
-            )
-
-            self.func(*self.args, **self.kwargs)
+            lg.debug("Running dummy Command with provides %s", self._provides)
 
             self._finished = True
         else:  # pragma: no cover
-            lg.info(
-                "Pretending to run code: %s(args=%s, kwargs=%s)",
-                self.repr_cmd(),
-                repr(self.args),
-                repr(self.kwargs),
-            )
+            lg.info("Pretending to run dummy Command with provides: %s", self._provides)
 
         return self._finished
 
@@ -369,13 +352,11 @@ def repr_cmd(self):
 
         :rtype: str
         """
-        # We need to "id" ourselves here so that multiple commands that call a
-        # function with the same name won't be shown as being equal.
-        return "%s|%s" % (self.func.__name__, id(self))
+        return self.identifier
 
     def __repr__(self):
-        """Show repr for FunctionCommand."""
-        return "FunctionCommand<%s requires %s, provides %s>\n" % (
+        """Show repr for DummyCommand."""
+        return "DummyCommand<%s requires %s, provides %s>\n" % (
             self.repr_cmd(),
             ", ".join([repr(x) for x in self._requires]),
             ", ".join([repr(x) for x in self._provides]),

pyaptly/conftest.py

@@ -65,6 +65,14 @@ def environment(debug_mode):
     os.environ["GNUPGHOME"] = str(gnupg)
     util._PYTEST_KEYSERVER = "hkp://127.0.0.1:8080"
 
+    # Make sure we start with a clean slate
+    state_reader.state_reader().mirrors.cache_clear()
+    state_reader.state_reader().snapshots.cache_clear()
+    state_reader.state_reader().snapshot_map.cache_clear()
+    state_reader.state_reader().repos.cache_clear()
+    state_reader.state_reader().publishes.cache_clear()
+    state_reader.state_reader().publish_map.cache_clear()
+
     try:
         yield
     finally:
@@ -78,6 +86,7 @@ def test_key_03(environment):
     """Get test gpg-key number 3."""
     util.run_command(["gpg", "--import", setup_base / "test03.key"], check=True)
     util.run_command(["gpg", "--import", setup_base / "test03.pub"], check=True)
+    state_reader.state_reader().gpg_keys.cache_clear()
 
 
 @pytest.fixture()
@@ -120,11 +129,9 @@ def mirror_update(environment, config):
     """Test if updating mirrors works."""
     args = ["-c", config, "mirror", "create"]
     state = state_reader.SystemStateReader()
-    state.read()
-    assert "fakerepo01" not in state.mirrors
+    assert "fakerepo01" not in state.mirrors()
     main.main(args)
-    state.read()
-    assert "fakerepo01" in state.mirrors
+    assert "fakerepo01" in state.mirrors()
     args[3] = "update"
     main.main(args)
     args = [
@@ -144,9 +151,8 @@ def snapshot_create(config, mirror_update, freeze):
     args = ["-c", config, "snapshot", "create"]
     main.main(args)
     state = state_reader.SystemStateReader()
-    state.read()
     assert set(["fakerepo01-20121010T0000Z", "fakerepo02-20121006T0000Z"]).issubset(
-        state.snapshots
+        state.snapshots()
     )
     yield state
 
@@ -162,29 +168,27 @@ def snapshot_update_rotating(config, mirror_update, freeze):
     ]
     main.main(args)
     state = state_reader.SystemStateReader()
-    state.read()
     assert set(
         [
             "fake-current",
             "fakerepo01-current",
             "fakerepo02-current",
         ]
-    ).issubset(state.snapshots)
+    ).issubset(state.snapshots())
     args = [
         "-c",
         config,
         "snapshot",
         "update",
     ]
     main.main(args)
-    state.read()
     assert set(
         [
             "fake-current",
             "fakerepo01-current-rotated-20121010T1010Z",
             "fakerepo02-current-rotated-20121010T1010Z",
         ]
-    ).issubset(state.snapshots)
+    ).issubset(state.snapshots())
     expected = {
         "fake-current": set(["fakerepo01-current", "fakerepo02-current"]),
         "fake-current-rotated-20121010T1010Z": set(
@@ -198,7 +202,7 @@ def snapshot_update_rotating(config, mirror_update, freeze):
         "fakerepo02-current": set([]),
         "fakerepo02-current-rotated-20121010T1010Z": set([]),
     }
-    assert state.snapshot_map == expected
+    assert state.snapshot_map() == expected
 
 
 @pytest.fixture()
@@ -207,7 +211,6 @@ def repo_create(environment, config, test_key_03):
     args = ["-c", config, "repo", "create"]
     main.main(args)
     state = state_reader.SystemStateReader()
-    state.read()
     util.run_command(
         [
             "aptly",
@@ -217,7 +220,8 @@ def repo_create(environment, config, test_key_03):
             "/source/compose/setup/hellome_0.1-1_amd64.deb",
         ]
     )
-    assert set(["centrify"]) == state.repos
+    state_reader.state_reader().repos.cache_clear()
+    assert set(["centrify"]) == state.repos()
 
 
 @pytest.fixture()
@@ -226,13 +230,12 @@ def publish_create(config, snapshot_create, test_key_03):
     args = ["-c", config, "publish", "create"]
     main.main(args)
     state = state_reader.SystemStateReader()
-    state.read()
-    assert set(["fakerepo02 main", "fakerepo01 main"]) == state.publishes
+    assert set(["fakerepo02 main", "fakerepo01 main"]) == state.publishes()
     expect = {
         "fakerepo02 main": set(["fakerepo02-20121006T0000Z"]),
         "fakerepo01 main": set(["fakerepo01-20121010T0000Z"]),
     }
-    assert expect == state.publish_map
+    assert expect == state.publish_map()
 
 
 @pytest.fixture()
@@ -241,7 +244,6 @@ def publish_create_rotating(config, snapshot_update_rotating, test_key_03):
     args = ["-c", config, "publish", "create"]
     main.main(args)
     state = state_reader.SystemStateReader()
-    state.read()
     assert (
         set(
             [
@@ -250,14 +252,14 @@ def publish_create_rotating(config, snapshot_update_rotating, test_key_03):
                 "fakerepo02/current stable",
             ]
         )
-        == state.publishes
+        == state.publishes()
     )
     expect = {
         "fake/current stable": set(["fake-current"]),
         "fakerepo01/current stable": set(["fakerepo01-current"]),
         "fakerepo02/current stable": set(["fakerepo02-current"]),
     }
-    assert expect == state.publish_map
+    assert expect == state.publish_map()
 
 
 @pytest.fixture()
@@ -277,5 +279,4 @@ def publish_create_republish(config, publish_create, caplog):
     ]
     main.main(args)
     state = state_reader.SystemStateReader()
-    state.read()
-    assert "fakerepo01-stable main" in state.publishes
+    assert "fakerepo01-stable main" in state.publishes()

pyaptly/main.py

@@ -13,7 +13,6 @@
     publish,
     repo,
     snapshot,
-    state_reader,
 )
 
 _logging_setup = False
@@ -48,7 +47,6 @@ def prepare(args):
 
     with open(args.config, "rb") as f:
         cfg = tomli.load(f)
-    state_reader.state_reader().read()
     return cfg