feat: Add LRU cache for state_reader() functions

Add an LRU cache to the state_reader so state is only reread when an
actual changing command has been executed.
After command execution the cache of the relevant types is cleared.

pyaptly/command.py

@@ -84,6 +84,23 @@ def provide(self, type_, identifier):
         assert type_ in self._known_dependency_types
         self._provides.add((type_, str(identifier)))
 
+    def clear_caches(self):
+        """Clear state_reader caches of functions which have changed"""
+        provides = set(p[0] for p in self.get_provides())
+        for provide in provides:
+            lg.debug("clearing cache for " + provide)
+            match provide:
+                case "mirrors":
+                    state_reader.state_reader().mirrors.cache_clear()
+                case "snapshot":
+                    state_reader.state_reader().snapshots.cache_clear()
+                    state_reader.state_reader().snapshot_map.cache_clear()
+                case "repo":
+                    state_reader.state_reader().repos.cache_clear()
+                case "publish":
+                    state_reader.state_reader().publishes.cache_clear()
+                    state_reader.state_reader().publish_map.cache_clear()
+
     def execute(self):
         """Execute the command. Return the return value of the command.
 
@@ -103,6 +120,7 @@ def execute(self):
             # So I decided to change that. For now we fail hard if a `Command` fails.
             # I guess we will see in production what happens.
             util.run_command(self.cmd, check=True)
+            self.clear_caches()
         else:
             lg.info("Pretending to run command: %s", " ".join(self.cmd))
         self._finished = True

pyaptly/conftest.py

@@ -65,6 +65,14 @@ def environment(debug_mode):
     os.environ["GNUPGHOME"] = str(gnupg)
     util._PYTEST_KEYSERVER = "hkp://127.0.0.1:8080"
 
+    # Make sure we start with a clean slate
+    state_reader.state_reader().mirrors.cache_clear()
+    state_reader.state_reader().snapshots.cache_clear()
+    state_reader.state_reader().snapshot_map.cache_clear()
+    state_reader.state_reader().repos.cache_clear()
+    state_reader.state_reader().publishes.cache_clear()
+    state_reader.state_reader().publish_map.cache_clear()
+
     try:
         yield
     finally:
@@ -78,6 +86,7 @@ def test_key_03(environment):
     """Get test gpg-key number 3."""
     util.run_command(["gpg", "--import", setup_base / "test03.key"], check=True)
     util.run_command(["gpg", "--import", setup_base / "test03.pub"], check=True)
+    state_reader.state_reader().gpg_keys.cache_clear()
 
 
 @pytest.fixture()
@@ -211,6 +220,7 @@ def repo_create(environment, config, test_key_03):
             "/source/compose/setup/hellome_0.1-1_amd64.deb",
         ]
     )
+    state_reader.state_reader().repos.cache_clear()
     assert set(["centrify"]) == state.repos()
 
 

pyaptly/mirror.py

@@ -59,8 +59,7 @@ def add_gpg_keys(mirror_config):
                 util.run_command(["bash", "-c", key_shell], check=True)
             else:
                 raise
-    #state_reader.state_reader().read_gpg()
-
+    state_reader.state_reader().gpg_keys.cache_clear()
 
 def mirror(cfg, args):
     """Create mirror commands, orders and executes them.
@@ -130,6 +129,7 @@ def cmd_mirror_create(cfg, mirror_name, mirror_config):
 
     lg.debug("Running command: %s", " ".join(aptly_cmd))
     util.run_command(aptly_cmd, check=True)
+    state_reader.state_reader().mirrors.cache_clear()
 
 
 def cmd_mirror_update(cfg, mirror_name, mirror_config):
@@ -152,3 +152,4 @@ def cmd_mirror_update(cfg, mirror_name, mirror_config):
     aptly_cmd.append(mirror_name)
     lg.debug("Running command: %s", " ".join(aptly_cmd))
     util.run_command(aptly_cmd, check=True)
+    state_reader.state_reader().mirrors.cache_clear()

pyaptly/state_reader.py

@@ -2,6 +2,7 @@
 
 import logging
 import re
+from functools import lru_cache
 
 from . import util
 
@@ -13,6 +14,7 @@ class SystemStateReader(object):
 
     To find out what operations have to be performed to reach the state defined
     in the toml config-file.
+    Functions are cached and execution commands clear the cache of functions which need to be rerun
     """
 
     known_dependency_types = ("repo", "snapshot", "mirror", "gpg_key")
@@ -41,6 +43,7 @@ def _extract_sources(self, data):
 
         return sources
 
+    @lru_cache(None)
     def gpg_keys(self):
         """Read all trusted keys in gp and cache in lru_cache."""
         gpg_keys = set()
@@ -62,6 +65,7 @@ def gpg_keys(self):
                 gpg_keys.add(key_short)
         return gpg_keys
 
+    @lru_cache(None)
     def publish_map(self):
         publish_map = {}
         re_snap = re.compile(r"\s+[\w\d-]+\:\s([\w\d-]+)\s\[snapshot\]")
@@ -77,6 +81,7 @@ def publish_map(self):
         lg.debug("Joined snapshots and publishes: %s", publish_map)
         return publish_map
 
+    @lru_cache(None)
     def snapshot_map(self):
         snapshot_map = {}
         # match example:  test-snapshot [snapshot]
@@ -93,18 +98,22 @@ def snapshot_map(self):
         lg.debug("Joined snapshots with self(snapshots): %s", snapshot_map)
         return snapshot_map
 
+    @lru_cache(None)
     def publishes(self):
         """Read all available publishes and cache in lru_cache"""
         return self.read_aptly_list("publish")
 
+    @lru_cache(None)
     def repos(self):
         """Read all available repo and cache in lru_cache."""
         return self.read_aptly_list("repo")
 
+    @lru_cache(None)
     def mirrors(self):
         """Read all available mirror and cache in lru_cache."""
         return self.read_aptly_list("mirror")
 
+    @lru_cache(None)
     def snapshots(self):
         """Read all available snapshot and cache in lru_cache."""
         return self.read_aptly_list("snapshot")