api: adding tag_pattern to autoprune API (PROJQUAY-7668)

data/database.py

@@ -49,7 +49,7 @@
     ReadReplicaSupportedModel,
     disallow_replica_use,
 )
-from data.text import match_like, match_mysql
+from data.text import match_like, match_mysql, regex_search, regex_sqlite
 from util.metrics.prometheus import (
     db_close_calls,
     db_connect_calls,
@@ -89,6 +89,10 @@
     "postgresql+psycopg2": match_like,
 }
 
+SCHEME_REGEX_FUNCTION = {
+    "sqlite": regex_sqlite,
+}
+
 
 SCHEME_RANDOM_FUNCTION = {
     "mysql": fn.Rand,
@@ -329,6 +333,7 @@ def _build_iterator(self):
 read_only_config = Proxy()
 db_random_func = CallableProxy()
 db_match_func = CallableProxy()
+db_regex_search = CallableProxy()
 db_for_update = CallableProxy()
 db_transaction = CallableProxy()
 db_disallow_replica_use = CallableProxy()
@@ -506,6 +511,7 @@ def configure(config_object, testing=False):
     )
     db_encrypter.initialize(FieldEncrypter(config_object.get("DATABASE_SECRET_KEY")))
     db_count_estimator.initialize(SCHEME_ESTIMATOR_FUNCTION[parsed_write_uri.drivername])
+    db_regex_search.initialize(SCHEME_REGEX_FUNCTION.get(parsed_write_uri.drivername, regex_search))
 
     read_replicas = config_object.get("DB_READ_REPLICAS", None)
     is_read_only = config_object.get("REGISTRY_STATE", "normal") == "readonly"

data/model/autoprune.py

@@ -1,5 +1,6 @@
 import json
 import logging.config
+import re
 from enum import Enum
 
 from data.database import AutoPruneTaskStatus
@@ -56,7 +57,13 @@
         return self._db_row
 
     def get_view(self):
-        return {"uuid": self.uuid, "method": self.method, "value": self.config.get("value")}
+        return {
+            "uuid": self.uuid,
+            "method": self.method,
+            "value": self.config.get("value"),
+            "tagPattern": self.config.get("tag_pattern"),
+            "tagPatternMatches": self.config.get("tag_pattern_matches"),
+        }
 
 
 class RepositoryAutoPrunePolicy:
@@ -79,7 +86,13 @@
         return self._db_row
 
     def get_view(self):
-        return {"uuid": self.uuid, "method": self.method, "value": self.config.get("value")}
+        return {
+            "uuid": self.uuid,
+            "method": self.method,
+            "value": self.config.get("value"),
+            "tagPattern": self.config.get("tag_pattern"),
+            "tagPatternMatches": self.config.get("tag_pattern_matches"),
+        }
 
 
 def valid_value(method, value):
@@ -116,6 +129,18 @@
     if not valid_value(method, policy_config.get("value")):
         raise InvalidNamespaceAutoPrunePolicy("Invalid value given for method type")
 
+    if policy_config.get("tag_pattern") is not None:
+        if not isinstance(policy_config.get("tag_pattern"), str):
+            raise InvalidNamespaceAutoPrunePolicy("tag_pattern must be string")
+
+        if policy_config.get("tag_pattern") == "":
+            raise InvalidNamespaceAutoPrunePolicy("tag_pattern cannot be empty")
+
+    if policy_config.get("tag_pattern_matches") is not None and not isinstance(
+        policy_config.get("tag_pattern_matches"), bool
+    ):
+        raise InvalidNamespaceAutoPrunePolicy("tag_pattern_matches must be bool")
+
 
 def assert_valid_repository_autoprune_policy(policy_config):
     """
@@ -129,6 +154,18 @@
     if not valid_value(method, policy_config.get("value")):
         raise InvalidRepositoryAutoPrunePolicy("Invalid value given for method type")
 
+    if policy_config.get("tag_pattern") is not None:
+        if not isinstance(policy_config.get("tag_pattern"), str):
+            raise InvalidRepositoryAutoPrunePolicy("tag_pattern must be string")
+
+        if policy_config.get("tag_pattern") == "":
+            raise InvalidRepositoryAutoPrunePolicy("tag_pattern cannot be empty")
+
+    if policy_config.get("tag_pattern_matches") is not None and not isinstance(
+        policy_config.get("tag_pattern_matches"), bool
+    ):
+        raise InvalidRepositoryAutoPrunePolicy("tag_pattern_matches must be bool")
+
 
 def get_namespace_autoprune_policies_by_orgname(orgname):
     """
@@ -546,10 +583,16 @@
     page = 1
     while True:
         tags = oci.tag.fetch_paginated_autoprune_repo_tags_by_number(
-            repo_id, int(policy_config["value"]), tag_page_limit, page
+            repo_id,
+            int(policy_config["value"]),
+            tag_page_limit,
+            page,
+            policy_config.get("tag_pattern"),
+            policy_config.get("tag_pattern_matches"),
         )
         if len(tags) == 0:
             break
+
         all_tags.extend(tags)
         page += 1
 
@@ -580,10 +623,16 @@
     page = 1
     while True:
         tags = oci.tag.fetch_paginated_autoprune_repo_tags_older_than_ms(
-            repo_id, time_ms, tag_page_limit, page
+            repo_id,
+            time_ms,
+            tag_page_limit,
+            page,
+            policy_config.get("tag_pattern"),
+            policy_config.get("tag_pattern_matches"),
         )
         if len(tags) == 0:
             break
+
         all_tags.extend(tags)
         page += 1
     return all_tags

data/model/oci/tag.py

@@ -15,6 +15,7 @@
     Tag,
     User,
     db_random_func,
+    db_regex_search,
     db_transaction,
     get_epoch_timestamp_ms,
 )
@@ -780,7 +781,7 @@ def reset_child_manifest_expiration(repository_id, manifest, expiration=None):
 
 
 def fetch_paginated_autoprune_repo_tags_by_number(
-    repo_id, max_tags_allowed: int, items_per_page, page
+    repo_id, max_tags_allowed: int, items_per_page, page, tag_pattern=None, tag_pattern_matches=True
 ):
     """
     Fetch repository's active tags sorted by creation date & are more than max_tags_allowed
@@ -801,6 +802,13 @@ def fetch_paginated_autoprune_repo_tags_by_number(
             .offset(tags_offset)
             .limit(items_per_page)
         )
+        if tag_pattern is not None:
+            query = db_regex_search(
+                Tag.select(query.c.name).from_(query),
+                query.c.name,
+                tag_pattern,
+                matches=tag_pattern_matches,
+            )
         return list(query)
     except Exception as err:
         raise Exception(
@@ -809,7 +817,12 @@ def fetch_paginated_autoprune_repo_tags_by_number(
 
 
 def fetch_paginated_autoprune_repo_tags_older_than_ms(
-    repo_id, tag_lifetime_ms: int, items_per_page=100, page: int = 1
+    repo_id,
+    tag_lifetime_ms: int,
+    items_per_page=100,
+    page: int = 1,
+    tag_pattern=None,
+    tag_pattern_matches=True,
 ):
     """
     Return repository's active tags older than tag_lifetime_ms
@@ -828,6 +841,8 @@ def fetch_paginated_autoprune_repo_tags_older_than_ms(
             .offset(tags_offset)  # type: ignore[func-returns-value]
             .limit(items_per_page)
         )
+        if tag_pattern is not None:
+            query = db_regex_search(query, Tag.name, tag_pattern, matches=tag_pattern_matches)
         return list(query)
     except Exception as err:
         raise Exception(

data/text.py

@@ -1,3 +1,6 @@
+import inspect
+import re
+
 from peewee import SQL, Field, NodeList, fn
 
 
@@ -56,3 +59,16 @@
     escaped_query = _escape_wildcard(search_query)
     clause = NodeList(("%" + escaped_query + "%", SQL("ESCAPE '!'")))
     return Field.__pow__(field, clause)
+
+
+def regex_search(query, field, pattern, matches=True):
+    return query.where(field.regexp(pattern)) if matches else query.where(~field.regexp(pattern))
+
+
+def regex_sqlite(query, field, pattern, matches=True):
+    rows = query.execute()
+    return (
+        [row for row in rows if re.search(pattern, getattr(row, field.name))]
+        if matches
+        else [row for row in rows if not re.search(pattern, getattr(row, field.name))]
+    )