feat: Replace state_reader.read() with lru_cache

pyaptly/cli.py

@@ -88,7 +88,7 @@ def legacy(passthrough):
 )
 @click.argument("config", type=click.Path(file_okay=True, dir_okay=False, exists=True))
 @click.argument("task", type=click.Choice(["create"]))
-@click.option("--repo-name", "-n", default="all", type=str, help='deafult: "all"')
+@click.option("--repo-name", "-n", default="all", type=str, help='default: "all"')
 def repo(**kwargs):
     """Create aptly repos."""
     from . import main, repo
@@ -111,7 +111,7 @@ def repo(**kwargs):
 )
 @click.argument("config", type=click.Path(file_okay=True, dir_okay=False, exists=True))
 @click.argument("task", type=click.Choice(["create", "update"]))
-@click.option("--mirror-name", "-n", default="all", type=str, help='deafult: "all"')
+@click.option("--mirror-name", "-n", default="all", type=str, help='default: "all"')
 def mirror(**kwargs):
     """Manage aptly mirrors."""
     from . import main, mirror
@@ -134,7 +134,7 @@ def mirror(**kwargs):
 )
 @click.argument("config", type=click.Path(file_okay=True, dir_okay=False, exists=True))
 @click.argument("task", type=click.Choice(["create", "update"]))
-@click.option("--snapshot-name", "-n", default="all", type=str, help='deafult: "all"')
+@click.option("--snapshot-name", "-n", default="all", type=str, help='default: "all"')
 def snapshot(**kwargs):
     """Manage aptly snapshots."""
     from . import main, snapshot
@@ -157,7 +157,7 @@ def snapshot(**kwargs):
 )
 @click.argument("config", type=click.Path(file_okay=True, dir_okay=False, exists=True))
 @click.argument("task", type=click.Choice(["create", "update"]))
-@click.option("--publish-name", "-n", default="all", type=str, help='deafult: "all"')
+@click.option("--publish-name", "-n", default="all", type=str, help='default: "all"')
 def publish(**kwargs):
     """Manage aptly publishs."""
     from . import main, publish

pyaptly/command.py

@@ -84,6 +84,22 @@ def provide(self, type_, identifier):
         assert type_ in self._known_dependency_types
         self._provides.add((type_, str(identifier)))
 
+    def clear_caches(self):
+        """Clear state_reader caches of functions which have changed"""
+        provides = set(p[0] for p in self.get_provides())
+        for provide in provides:
+            lg.debug("clearing cache for " + provide)
+            match provide:
+                case "mirrors":
+                    state_reader.state_reader().mirrors.cache_clear()
+                case "snapshot" | "virtual":
+                    state_reader.state_reader().snapshots.cache_clear()
+                    state_reader.state_reader().snapshot_map.cache_clear()
+                case "repo":
+                    state_reader.state_reader().repos.cache_clear()
+                case "publish":
+                    state_reader.state_reader().publishes.cache_clear()
+
     def execute(self):
         """Execute the command. Return the return value of the command.
 
@@ -103,6 +119,7 @@ def execute(self):
             # So I decided to change that. For now we fail hard if a `Command` fails.
             # I guess we will see in production what happens.
             util.run_command(self.cmd, check=True)
+            self.clear_caches()
         else:
             lg.info("Pretending to run command: %s", " ".join(self.cmd))
         self._finished = True
@@ -269,8 +286,6 @@ def order_commands(commands, has_dependency_cb=lambda x: False):
                             # Break out of the requirements loop, as the
                             # command cannot be scheduled anyway.
                             break
-                            # command cannot be scheduled anyway.
-                            break
 
                 if can_schedule:
                     lg.debug("%s: all dependencies fulfilled" % cmd)
@@ -298,7 +313,7 @@ def order_commands(commands, has_dependency_cb=lambda x: False):
 
 
 class FunctionCommand(Command):
-    """Repesents a function command.
+    """Represents a function command.
 
     Is used to resolve dependencies between such commands. This command executes
     the given function. *args and **kwargs are passed through.

pyaptly/conftest.py

@@ -78,6 +78,7 @@ def test_key_03(environment):
     """Get test gpg-key number 3."""
     util.run_command(["gpg", "--import", setup_base / "test03.key"], check=True)
     util.run_command(["gpg", "--import", setup_base / "test03.pub"], check=True)
+    state_reader.state_reader().gpg_keys.cache_clear()
 
 
 @pytest.fixture()
@@ -120,11 +121,9 @@ def mirror_update(environment, config):
     """Test if updating mirrors works."""
     args = ["-c", config, "mirror", "create"]
     state = state_reader.SystemStateReader()
-    state.read()
-    assert "fakerepo01" not in state.mirrors
+    assert "fakerepo01" not in state.mirrors()
     main.main(args)
-    state.read()
-    assert "fakerepo01" in state.mirrors
+    assert "fakerepo01" in state.mirrors()
     args[3] = "update"
     main.main(args)
     args = [
@@ -144,9 +143,8 @@ def snapshot_create(config, mirror_update, freeze):
     args = ["-c", config, "snapshot", "create"]
     main.main(args)
     state = state_reader.SystemStateReader()
-    state.read()
     assert set(["fakerepo01-20121010T0000Z", "fakerepo02-20121006T0000Z"]).issubset(
-        state.snapshots
+        state.snapshots()
     )
     yield state
 
@@ -162,29 +160,27 @@ def snapshot_update_rotating(config, mirror_update, freeze):
     ]
     main.main(args)
     state = state_reader.SystemStateReader()
-    state.read()
     assert set(
         [
             "fake-current",
             "fakerepo01-current",
             "fakerepo02-current",
         ]
-    ).issubset(state.snapshots)
+    ).issubset(state.snapshots())
     args = [
         "-c",
         config,
         "snapshot",
         "update",
     ]
     main.main(args)
-    state.read()
     assert set(
         [
             "fake-current",
             "fakerepo01-current-rotated-20121010T1010Z",
             "fakerepo02-current-rotated-20121010T1010Z",
         ]
-    ).issubset(state.snapshots)
+    ).issubset(state.snapshots())
     expected = {
         "fake-current": set(["fakerepo01-current", "fakerepo02-current"]),
         "fake-current-rotated-20121010T1010Z": set(
@@ -198,7 +194,7 @@ def snapshot_update_rotating(config, mirror_update, freeze):
         "fakerepo02-current": set([]),
         "fakerepo02-current-rotated-20121010T1010Z": set([]),
     }
-    assert state.snapshot_map == expected
+    assert state.snapshot_map() == expected
 
 
 @pytest.fixture()
@@ -207,7 +203,6 @@ def repo_create(environment, config, test_key_03):
     args = ["-c", config, "repo", "create"]
     main.main(args)
     state = state_reader.SystemStateReader()
-    state.read()
     util.run_command(
         [
             "aptly",
@@ -217,7 +212,8 @@ def repo_create(environment, config, test_key_03):
             "/source/compose/setup/hellome_0.1-1_amd64.deb",
         ]
     )
-    assert set(["centrify"]) == state.repos
+    state_reader.state_reader().repos.cache_clear()
+    assert set(["centrify"]) == state.repos()
 
 
 @pytest.fixture()
@@ -226,13 +222,12 @@ def publish_create(config, snapshot_create, test_key_03):
     args = ["-c", config, "publish", "create"]
     main.main(args)
     state = state_reader.SystemStateReader()
-    state.read()
-    assert set(["fakerepo02 main", "fakerepo01 main"]) == state.publishes
+    assert set(["fakerepo02 main", "fakerepo01 main"]) == state.publishes()
     expect = {
         "fakerepo02 main": set(["fakerepo02-20121006T0000Z"]),
         "fakerepo01 main": set(["fakerepo01-20121010T0000Z"]),
     }
-    assert expect == state.publish_map
+    assert expect == state.publish_map()
 
 
 @pytest.fixture()
@@ -241,7 +236,6 @@ def publish_create_rotating(config, snapshot_update_rotating, test_key_03):
     args = ["-c", config, "publish", "create"]
     main.main(args)
     state = state_reader.SystemStateReader()
-    state.read()
     assert (
         set(
             [
@@ -250,14 +244,14 @@ def publish_create_rotating(config, snapshot_update_rotating, test_key_03):
                 "fakerepo02/current stable",
             ]
         )
-        == state.publishes
+        == state.publishes()
     )
     expect = {
         "fake/current stable": set(["fake-current"]),
         "fakerepo01/current stable": set(["fakerepo01-current"]),
         "fakerepo02/current stable": set(["fakerepo02-current"]),
     }
-    assert expect == state.publish_map
+    assert expect == state.publish_map()
 
 
 @pytest.fixture()
@@ -277,5 +271,4 @@ def publish_create_republish(config, publish_create, caplog):
     ]
     main.main(args)
     state = state_reader.SystemStateReader()
-    state.read()
-    assert "fakerepo01-stable main" in state.publishes
+    assert "fakerepo01-stable main" in state.publishes()

pyaptly/main.py

@@ -48,7 +48,6 @@ def prepare(args):
 
     with open(args.config, "rb") as f:
         cfg = tomli.load(f)
-    state_reader.state_reader().read()
     return cfg
 
 

pyaptly/mirror.py

@@ -33,7 +33,7 @@ def add_gpg_keys(mirror_config):
                 keys_urls[key] = None
 
     for key in keys_urls.keys():
-        if key in state_reader.state_reader().gpg_keys:
+        if key in state_reader.state_reader().gpg_keys():
             continue
         try:
             key_command = [
@@ -59,8 +59,8 @@ def add_gpg_keys(mirror_config):
                 util.run_command(["bash", "-c", key_shell], check=True)
             else:
                 raise
-    state_reader.state_reader().read_gpg()
-
+    # Clear gpg cache right now
+    state_reader.state_reader().gpg_keys.cache_clear()
 
 def mirror(cfg, args):
     """Create mirror commands, orders and executes them.
@@ -102,7 +102,7 @@ def cmd_mirror_create(cfg, mirror_name, mirror_config):
     :param mirror_config: Configuration of the snapshot from the toml file.
     :type  mirror_config: dict
     """
-    if mirror_name in state_reader.state_reader().mirrors:  # pragma: no cover
+    if mirror_name in state_reader.state_reader().mirrors():  # pragma: no cover
         return
 
     add_gpg_keys(mirror_config)
@@ -130,6 +130,7 @@ def cmd_mirror_create(cfg, mirror_name, mirror_config):
 
     lg.debug("Running command: %s", " ".join(aptly_cmd))
     util.run_command(aptly_cmd, check=True)
+    state_reader.state_reader().mirrors.cache_clear()
 
 
 def cmd_mirror_update(cfg, mirror_name, mirror_config):
@@ -142,7 +143,7 @@ def cmd_mirror_update(cfg, mirror_name, mirror_config):
     :param mirror_config: Configuration of the snapshot from the toml file.
     :type  mirror_config: dict
     """
-    if mirror_name not in state_reader.state_reader().mirrors:  # pragma: no cover
+    if mirror_name not in state_reader.state_reader().mirrors():  # pragma: no cover
         raise Exception("Mirror not created yet")
     add_gpg_keys(mirror_config)
     aptly_cmd = ["aptly", "mirror", "update"]
@@ -152,3 +153,4 @@ def cmd_mirror_update(cfg, mirror_name, mirror_config):
     aptly_cmd.append(mirror_name)
     lg.debug("Running command: %s", " ".join(aptly_cmd))
     util.run_command(aptly_cmd, check=True)
+    state_reader.state_reader().mirrors.cache_clear()

pyaptly/publish.py

@@ -80,10 +80,12 @@ def publish_cmd_update(cfg, publish_name, publish_config, ignore_existing=False)
 
     if "repo" in publish_config:
         publish_cmd.append("update")
-        return command.Command(publish_cmd + options + args)
+        cmd = command.Command(publish_cmd + options + args)
+        cmd.provide("publish", publish_name)
+        return cmd
 
     publish_fullname = "%s %s" % (publish_name, publish_config["distribution"])
-    current_snapshots = state_reader.state_reader().publish_map[publish_fullname]
+    current_snapshots = state_reader.state_reader().publish_map()[publish_fullname]
     if "snapshots" in publish_config:
         snapshots_config = publish_config["snapshots"]
         new_snapshots = [
@@ -97,7 +99,7 @@ def publish_cmd_update(cfg, publish_name, publish_config, ignore_existing=False)
             if publish["distribution"] == distribution:
                 snapshots_config.extend(publish["snapshots"])
                 break
-        new_snapshots = list(state_reader.state_reader().publish_map[conf_value])
+        new_snapshots = list(state_reader.state_reader().publish_map()[conf_value])
     else:  # pragma: no cover
         raise ValueError(
             "No snapshot references configured in publish %s" % publish_name
@@ -122,7 +124,7 @@ def publish_cmd_update(cfg, publish_name, publish_config, ignore_existing=False)
                     "%T", date_tools.format_timestamp(datetime.datetime.now())
                 )
                 if (
-                    archive in state_reader.state_reader().snapshots
+                    archive in state_reader.state_reader().snapshots()
                 ):  # pragma: no cover
                     continue
                 prefix_to_search = re.sub("%T$", "", snap["name"])
@@ -141,7 +143,9 @@ def publish_cmd_update(cfg, publish_name, publish_config, ignore_existing=False)
     if "skip-contents" in publish_config and publish_config["skip-contents"]:
         options.append("-skip-contents=true")
 
-    return command.Command(publish_cmd + options + args + new_snapshots)
+    cmd = command.Command(publish_cmd + options + args + new_snapshots)
+    cmd.provide("publish", publish_fullname)
+    return cmd
 
 
 def publish_cmd_create(cfg, publish_name, publish_config, ignore_existing=False):
@@ -158,7 +162,7 @@ def publish_cmd_create(cfg, publish_name, publish_config, ignore_existing=False)
     """
     publish_fullname = "%s %s" % (publish_name, publish_config["distribution"])
     if (
-        publish_fullname in state_reader.state_reader().publishes
+        publish_fullname in state_reader.state_reader().publishes()
         and not ignore_existing
     ):
         # Nothing to do, publish already created
@@ -233,7 +237,7 @@ def publish_cmd_create(cfg, publish_name, publish_config, ignore_existing=False)
             conf_value = " ".join(conf_value.split("/"))
             source_args.append("snapshot")
             try:
-                sources = state_reader.state_reader().publish_map[conf_value]
+                sources = state_reader.state_reader().publish_map()[conf_value]
             except KeyError:
                 lg.critical(
                     (
@@ -256,4 +260,6 @@ def publish_cmd_create(cfg, publish_name, publish_config, ignore_existing=False)
     assert has_source
     assert len(components) == num_sources
 
-    return command.Command(publish_cmd + options + source_args + endpoint_args)
+    cmd = command.Command(publish_cmd + options + source_args + endpoint_args)
+    cmd.provide("publish", publish_fullname)
+    return cmd

pyaptly/repo.py

@@ -57,7 +57,7 @@ def repo_cmd_create(cfg, repo_name, repo_config):
     :param repo_config: Configuration of the repo from the toml file.
     :type  repo_config: dict
     """
-    if repo_name in state_reader.state_reader().repos:  # pragma: no cover
+    if repo_name in state_reader.state_reader().repos():  # pragma: no cover
         # Nothing to do, repo already created
         return
 
@@ -86,4 +86,7 @@ def repo_cmd_create(cfg, repo_name, repo_config):
                 )
             )
 
-    return command.Command(repo_cmd + options + endpoint_args)
+
+    cmd = command.Command(repo_cmd + options + endpoint_args)
+    cmd.provide("repo", repo_name)
+    return cmd