diff --git a/data/database.py b/data/database.py index 8008d163a8..162ceddec8 100644 --- a/data/database.py +++ b/data/database.py @@ -49,7 +49,7 @@ ReadReplicaSupportedModel, disallow_replica_use, ) -from data.text import match_like, match_mysql +from data.text import match_like, match_mysql, regex_search, regex_sqlite from util.metrics.prometheus import ( db_close_calls, db_connect_calls, @@ -89,6 +89,10 @@ "postgresql+psycopg2": match_like, } +SCHEME_REGEX_FUNCTION = { + "sqlite": regex_sqlite, +} + SCHEME_RANDOM_FUNCTION = { "mysql": fn.Rand, @@ -329,6 +333,7 @@ def _build_iterator(self): read_only_config = Proxy() db_random_func = CallableProxy() db_match_func = CallableProxy() +db_regex_search = CallableProxy() db_for_update = CallableProxy() db_transaction = CallableProxy() db_disallow_replica_use = CallableProxy() @@ -506,6 +511,7 @@ def configure(config_object, testing=False): ) db_encrypter.initialize(FieldEncrypter(config_object.get("DATABASE_SECRET_KEY"))) db_count_estimator.initialize(SCHEME_ESTIMATOR_FUNCTION[parsed_write_uri.drivername]) + db_regex_search.initialize(SCHEME_REGEX_FUNCTION.get(parsed_write_uri.drivername, regex_search)) read_replicas = config_object.get("DB_READ_REPLICAS", None) is_read_only = config_object.get("REGISTRY_STATE", "normal") == "readonly" diff --git a/data/model/autoprune.py b/data/model/autoprune.py index ca6fd619b4..ce0588d9e1 100644 --- a/data/model/autoprune.py +++ b/data/model/autoprune.py @@ -1,5 +1,6 @@ import json import logging.config +import re from enum import Enum from data.database import AutoPruneTaskStatus @@ -56,7 +57,13 @@ def get_row(self): return self._db_row def get_view(self): - return {"uuid": self.uuid, "method": self.method, "value": self.config.get("value")} + return { + "uuid": self.uuid, + "method": self.method, + "value": self.config.get("value"), + "tagPattern": self.config.get("tag_pattern"), + "tagPatternMatches": self.config.get("tag_pattern_matches"), + } class RepositoryAutoPrunePolicy: @@ -79,7 +86,13 @@ def get_row(self): return self._db_row def get_view(self): - return {"uuid": self.uuid, "method": self.method, "value": self.config.get("value")} + return { + "uuid": self.uuid, + "method": self.method, + "value": self.config.get("value"), + "tagPattern": self.config.get("tag_pattern"), + "tagPatternMatches": self.config.get("tag_pattern_matches"), + } def valid_value(method, value): @@ -116,6 +129,18 @@ def assert_valid_namespace_autoprune_policy(policy_config): if not valid_value(method, policy_config.get("value")): raise InvalidNamespaceAutoPrunePolicy("Invalid value given for method type") + if policy_config.get("tag_pattern") is not None: + if not isinstance(policy_config.get("tag_pattern"), str): + raise InvalidNamespaceAutoPrunePolicy("tag_pattern must be string") + + if policy_config.get("tag_pattern") == "": + raise InvalidNamespaceAutoPrunePolicy("tag_pattern cannot be empty") + + if policy_config.get("tag_pattern_matches") is not None and not isinstance( + policy_config.get("tag_pattern_matches"), bool + ): + raise InvalidNamespaceAutoPrunePolicy("tag_pattern_matches must be bool") + def assert_valid_repository_autoprune_policy(policy_config): """ @@ -129,6 +154,18 @@ def assert_valid_repository_autoprune_policy(policy_config): if not valid_value(method, policy_config.get("value")): raise InvalidRepositoryAutoPrunePolicy("Invalid value given for method type") + if policy_config.get("tag_pattern") is not None: + if not isinstance(policy_config.get("tag_pattern"), str): + raise InvalidRepositoryAutoPrunePolicy("tag_pattern must be string") + + if policy_config.get("tag_pattern") == "": + raise InvalidRepositoryAutoPrunePolicy("tag_pattern cannot be empty") + + if policy_config.get("tag_pattern_matches") is not None and not isinstance( + policy_config.get("tag_pattern_matches"), bool + ): + raise InvalidRepositoryAutoPrunePolicy("tag_pattern_matches must be bool") + def get_namespace_autoprune_policies_by_orgname(orgname): """ @@ -546,10 +583,16 @@ def fetch_tags_expiring_by_tag_count_policy(repo_id, policy_config, tag_page_lim page = 1 while True: tags = oci.tag.fetch_paginated_autoprune_repo_tags_by_number( - repo_id, int(policy_config["value"]), tag_page_limit, page + repo_id, + int(policy_config["value"]), + tag_page_limit, + page, + policy_config.get("tag_pattern"), + policy_config.get("tag_pattern_matches"), ) if len(tags) == 0: break + all_tags.extend(tags) page += 1 @@ -580,10 +623,16 @@ def fetch_tags_expiring_by_creation_date_policy(repo_id, policy_config, tag_page page = 1 while True: tags = oci.tag.fetch_paginated_autoprune_repo_tags_older_than_ms( - repo_id, time_ms, tag_page_limit, page + repo_id, + time_ms, + tag_page_limit, + page, + policy_config.get("tag_pattern"), + policy_config.get("tag_pattern_matches"), ) if len(tags) == 0: break + all_tags.extend(tags) page += 1 return all_tags diff --git a/data/model/oci/tag.py b/data/model/oci/tag.py index 6e503d4633..24e638cf0f 100644 --- a/data/model/oci/tag.py +++ b/data/model/oci/tag.py @@ -15,6 +15,7 @@ Tag, User, db_random_func, + db_regex_search, db_transaction, get_epoch_timestamp_ms, ) @@ -780,7 +781,7 @@ def reset_child_manifest_expiration(repository_id, manifest, expiration=None): def fetch_paginated_autoprune_repo_tags_by_number( - repo_id, max_tags_allowed: int, items_per_page, page + repo_id, max_tags_allowed: int, items_per_page, page, tag_pattern=None, tag_pattern_matches=True ): """ Fetch repository's active tags sorted by creation date & are more than max_tags_allowed @@ -801,6 +802,13 @@ def fetch_paginated_autoprune_repo_tags_by_number( .offset(tags_offset) .limit(items_per_page) ) + if tag_pattern is not None: + query = db_regex_search( + Tag.select(query.c.name).from_(query), + query.c.name, + tag_pattern, + matches=tag_pattern_matches, + ) return list(query) except Exception as err: raise Exception( @@ -809,7 +817,12 @@ def fetch_paginated_autoprune_repo_tags_by_number( def fetch_paginated_autoprune_repo_tags_older_than_ms( - repo_id, tag_lifetime_ms: int, items_per_page=100, page: int = 1 + repo_id, + tag_lifetime_ms: int, + items_per_page=100, + page: int = 1, + tag_pattern=None, + tag_pattern_matches=True, ): """ Return repository's active tags older than tag_lifetime_ms @@ -828,6 +841,8 @@ def fetch_paginated_autoprune_repo_tags_older_than_ms( .offset(tags_offset) # type: ignore[func-returns-value] .limit(items_per_page) ) + if tag_pattern is not None: + query = db_regex_search(query, Tag.name, tag_pattern, matches=tag_pattern_matches) return list(query) except Exception as err: raise Exception( diff --git a/data/text.py b/data/text.py index 4bae32e94b..de02d5548f 100644 --- a/data/text.py +++ b/data/text.py @@ -1,3 +1,6 @@ +import inspect +import re + from peewee import SQL, Field, NodeList, fn @@ -56,3 +59,16 @@ def match_like(field, search_query): escaped_query = _escape_wildcard(search_query) clause = NodeList(("%" + escaped_query + "%", SQL("ESCAPE '!'"))) return Field.__pow__(field, clause) + + +def regex_search(query, field, pattern, matches=True): + return query.where(field.regexp(pattern)) if matches else query.where(~field.regexp(pattern)) + + +def regex_sqlite(query, field, pattern, matches=True): + rows = query.execute() + return ( + [row for row in rows if re.search(pattern, getattr(row, field.name))] + if matches + else [row for row in rows if not re.search(pattern, getattr(row, field.name))] + ) diff --git a/endpoints/api/policy.py b/endpoints/api/policy.py index 9c2b193b1f..d753d173bd 100644 --- a/endpoints/api/policy.py +++ b/endpoints/api/policy.py @@ -54,6 +54,14 @@ class OrgAutoPrunePolicies(ApiResource): "type": ["integer", "string"], "description": "The value to use for the pruning method (number of tags e.g. 10, time delta e.g. 7d (7 days))", }, + "tagPattern": { + "type": "string", + "description": "Tags only matching this pattern will be pruned", + }, + "tagPatternMatches": { + "type": "boolean", + "description": "Determine whether pruned tags should or should not match the tagPattern", + }, }, }, } @@ -90,6 +98,10 @@ def post(self, orgname): app_data = request.get_json() method = app_data.get("method", None) value = app_data.get("value", None) + tag_pattern = app_data.get("tagPattern", None) + if tag_pattern is not None and isinstance(tag_pattern, str): + tag_pattern = tag_pattern.strip() + tag_pattern_matches = app_data.get("tagPatternMatches", True) if method is None or value is None: request_error(message="Missing the following parameters: method, value") @@ -97,6 +109,8 @@ def post(self, orgname): policy_config = { "method": method, "value": value, + "tag_pattern": tag_pattern, + "tag_pattern_matches": tag_pattern_matches, } try: @@ -116,6 +130,8 @@ def post(self, orgname): { "method": policy_config["method"], "value": policy_config["value"], + "tag_pattern": policy_config.get("tag_pattern"), + "tag_pattern_matches": policy_config.get("tag_pattern_matches"), "namespace": orgname, }, ) @@ -146,6 +162,14 @@ class OrgAutoPrunePolicy(ApiResource): "type": ["integer", "string"], "description": "The value to use for the pruning method (number of tags e.g. 10, time delta e.g. 7d (7 days))", }, + "tagPattern": { + "type": "string", + "description": "Tags only matching this pattern will be pruned", + }, + "tagPatternMatches": { + "type": "boolean", + "description": "Determine whether pruned tags should or should not match the tagPattern", + }, }, }, } @@ -184,6 +208,10 @@ def put(self, orgname, policy_uuid): app_data = request.get_json() method = app_data.get("method", None) value = app_data.get("value", None) + tag_pattern = app_data.get("tagPattern", None) + if tag_pattern is not None and isinstance(tag_pattern, str): + tag_pattern = tag_pattern.strip() + tag_pattern_matches = app_data.get("tagPatternMatches", True) if method is None or value is None: request_error(message="Missing the following parameters: method, value") @@ -191,6 +219,8 @@ def put(self, orgname, policy_uuid): policy_config = { "method": method, "value": value, + "tag_pattern": tag_pattern, + "tag_pattern_matches": tag_pattern_matches, } try: @@ -212,6 +242,8 @@ def put(self, orgname, policy_uuid): { "method": policy_config["method"], "value": policy_config["value"], + "tag_pattern": policy_config.get("tag_pattern"), + "tag_pattern_matches": policy_config.get("tag_pattern_matches"), "namespace": orgname, }, ) @@ -268,6 +300,14 @@ class RepositoryAutoPrunePolicies(RepositoryParamResource): "type": ["integer", "string"], "description": "The value to use for the pruning method (number of tags e.g. 10, time delta e.g. 7d (7 days))", }, + "tagPattern": { + "type": "string", + "description": "Tags only matching this pattern will be pruned", + }, + "tagPatternMatches": { + "type": "boolean", + "description": "Determine whether pruned tags should or should not match the tagPattern", + }, }, }, } @@ -312,6 +352,10 @@ def post(self, namespace, repository): app_data = request.get_json() method = app_data.get("method", None) value = app_data.get("value", None) + tag_pattern = app_data.get("tagPattern", None) + if tag_pattern is not None and isinstance(tag_pattern, str): + tag_pattern = tag_pattern.strip() + tag_pattern_matches = app_data.get("tagPatternMatches", True) if method is None or value is None: request_error(message="Missing the following parameters: method, value") @@ -319,6 +363,8 @@ def post(self, namespace, repository): policy_config = { "method": method, "value": value, + "tag_pattern": tag_pattern, + "tag_pattern_matches": tag_pattern_matches, } try: @@ -340,6 +386,8 @@ def post(self, namespace, repository): { "method": policy_config["method"], "value": policy_config["value"], + "tag_pattern": policy_config.get("tag_pattern"), + "tag_pattern_matches": policy_config.get("tag_pattern_matches"), "namespace": namespace, "repo": repository, }, @@ -372,6 +420,14 @@ class RepositoryAutoPrunePolicy(RepositoryParamResource): "type": ["integer", "string"], "description": "The value to use for the pruning method (number of tags e.g. 10, time delta e.g. 7d (7 days))", }, + "tagPattern": { + "type": "string", + "description": "Tags only matching this pattern will be pruned", + }, + "tagPatternMatches": { + "type": "boolean", + "description": "Determine whether pruned tags should or should not match the tagPattern", + }, }, }, } @@ -410,6 +466,10 @@ def put(self, namespace, repository, policy_uuid): app_data = request.get_json() method = app_data.get("method", None) value = app_data.get("value", None) + tag_pattern = app_data.get("tagPattern", None) + if tag_pattern is not None and isinstance(tag_pattern, str): + tag_pattern = tag_pattern.strip() + tag_pattern_matches = app_data.get("tagPatternMatches", True) if method is None or value is None: request_error(message="Missing the following parameters: method, value") @@ -417,6 +477,8 @@ def put(self, namespace, repository, policy_uuid): policy_config = { "method": method, "value": value, + "tag_pattern": tag_pattern, + "tag_pattern_matches": tag_pattern_matches, } try: @@ -440,6 +502,8 @@ def put(self, namespace, repository, policy_uuid): { "method": policy_config["method"], "value": policy_config["value"], + "tag_pattern": policy_config.get("tag_pattern"), + "tag_pattern_matches": policy_config.get("tag_pattern_matches"), "namespace": namespace, "repo": repository, }, @@ -502,6 +566,14 @@ class UserAutoPrunePolicies(ApiResource): "type": ["integer", "string"], "description": "The value to use for the pruning method (number of tags e.g. 10, time delta e.g. 7d (7 days))", }, + "tagPattern": { + "type": "string", + "description": "Tags only matching this pattern will be pruned", + }, + "tagPatternMatches": { + "type": "boolean", + "description": "Determine whether pruned tags should or should not match the tagPattern", + }, }, }, } @@ -530,6 +602,10 @@ def post(self): app_data = request.get_json() method = app_data.get("method", None) value = app_data.get("value", None) + tag_pattern = app_data.get("tagPattern", None) + if tag_pattern is not None and isinstance(tag_pattern, str): + tag_pattern = tag_pattern.strip() + tag_pattern_matches = app_data.get("tagPatternMatches", True) if method is None or value is None: request_error(message="Missing the following parameters: method, value") @@ -537,7 +613,10 @@ def post(self): policy_config = { "method": method, "value": value, + "tag_pattern": tag_pattern, + "tag_pattern_matches": tag_pattern_matches, } + try: policy = model.autoprune.create_namespace_autoprune_policy( user.username, policy_config, create_task=True @@ -556,6 +635,8 @@ def post(self): "method": policy_config["method"], "value": policy_config["value"], "namespace": user.username, + "tag_pattern": policy_config.get("tag_pattern"), + "tag_pattern_matches": policy_config.get("tag_pattern_matches"), }, ) @@ -584,6 +665,14 @@ class UserAutoPrunePolicy(ApiResource): "type": ["integer", "string"], "description": "The value to use for the pruning method (number of tags e.g. 10, time delta e.g. 7d (7 days))", }, + "tagPattern": { + "type": "string", + "description": "Tags only matching this pattern will be pruned", + }, + "tagPatternMatches": { + "type": "boolean", + "description": "Determine whether pruned tags should or should not match the tagPattern", + }, }, }, } @@ -614,6 +703,10 @@ def put(self, policy_uuid): app_data = request.get_json() method = app_data.get("method", None) value = app_data.get("value", None) + tag_pattern = app_data.get("tagPattern", None) + if tag_pattern is not None and isinstance(tag_pattern, str): + tag_pattern = tag_pattern.strip() + tag_pattern_matches = app_data.get("tagPatternMatches", True) if method is None or value is None: request_error(message="Missing the following parameters: method, value") @@ -621,6 +714,8 @@ def put(self, policy_uuid): policy_config = { "method": method, "value": value, + "tag_pattern": tag_pattern, + "tag_pattern_matches": tag_pattern_matches, } try: @@ -643,6 +738,8 @@ def put(self, policy_uuid): "method": policy_config["method"], "value": policy_config["value"], "namespace": user.username, + "tag_pattern": policy_config.get("tag_pattern"), + "tag_pattern_matches": policy_config.get("tag_pattern_matches"), }, ) diff --git a/endpoints/api/test/test_policy.py b/endpoints/api/test/test_policy.py index 3c298ce8fe..3801080443 100644 --- a/endpoints/api/test/test_policy.py +++ b/endpoints/api/test/test_policy.py @@ -25,14 +25,29 @@ def test_get_org_policies(initialized_db, app): assert response["policies"][0]["value"] == "5d" -def test_create_org_policy(initialized_db, app): +@pytest.mark.parametrize( + "policy", + [ + ({"method": "creation_date", "value": "2w"}), + ({"method": "creation_date", "value": "2w", "tagPattern": "match.*"}), + ( + { + "method": "creation_date", + "value": "2w", + "tagPattern": "match.*", + "tagPatternMatches": False, + } + ), + ], +) +def test_create_org_policy(policy, initialized_db, app): with client_with_identity("devtable", app) as cl: response = conduct_api_call( cl, OrgAutoPrunePolicies, "POST", {"orgname": "sellnsmall"}, - {"method": "creation_date", "value": "2w"}, + policy, 201, ).json assert response["uuid"] is not None @@ -52,9 +67,21 @@ def test_create_org_policy(initialized_db, app): log = l break assert log is not None - assert json.loads(log.metadata_json)["method"] == "creation_date" - assert json.loads(log.metadata_json)["value"] == "2w" + assert json.loads(log.metadata_json)["method"] == policy.get("method") + assert json.loads(log.metadata_json)["value"] == policy.get("value") assert json.loads(log.metadata_json)["namespace"] == "sellnsmall" + assert json.loads(log.metadata_json)["tag_pattern"] == policy.get("tagPattern") + assert json.loads(log.metadata_json)["tag_pattern_matches"] == policy.get( + "tagPatternMatches", True + ) + + # Assert policy information is correct + response = conduct_api_call(cl, OrgAutoPrunePolicies, "GET", {"orgname": "sellnsmall"}).json + assert len(response["policies"]) == 1 + assert response["policies"][0]["method"] == policy.get("method") + assert response["policies"][0]["value"] == policy.get("value") + assert response["policies"][0]["tagPattern"] == policy.get("tagPattern") + assert response["policies"][0]["tagPatternMatches"] == policy.get("tagPatternMatches", True) def test_create_org_policy_already_existing(initialized_db, app): @@ -98,7 +125,22 @@ def test_get_org_policy(initialized_db, app): assert response["value"] == "5d" -def test_update_org_policy(initialized_db, app): +@pytest.mark.parametrize( + "policy", + [ + ({"method": "creation_date", "value": "2w"}), + ({"method": "creation_date", "value": "2w", "tagPattern": "match.*"}), + ( + { + "method": "creation_date", + "value": "2w", + "tagPattern": "match.*", + "tagPatternMatches": False, + } + ), + ], +) +def test_update_org_policy(policy, initialized_db, app): policies = model.autoprune.get_namespace_autoprune_policies_by_orgname("buynlarge") assert len(policies) == 1 policy_uuid = policies[0].uuid @@ -108,7 +150,7 @@ def test_update_org_policy(initialized_db, app): OrgAutoPrunePolicy, "PUT", {"orgname": "buynlarge", "policy_uuid": policy_uuid}, - {"method": "creation_date", "value": "2w"}, + policy, expected_code=204, ) @@ -128,9 +170,22 @@ def test_update_org_policy(initialized_db, app): log = l break assert log is not None - assert json.loads(log.metadata_json)["method"] == "creation_date" - assert json.loads(log.metadata_json)["value"] == "2w" + assert json.loads(log.metadata_json)["method"] == policy.get("method") + assert json.loads(log.metadata_json)["value"] == policy.get("value") assert json.loads(log.metadata_json)["namespace"] == "buynlarge" + assert json.loads(log.metadata_json)["tag_pattern"] == policy.get("tagPattern") + assert json.loads(log.metadata_json)["tag_pattern_matches"] == policy.get( + "tagPatternMatches", True + ) + + # Assert policy information is correct + response = conduct_api_call( + cl, OrgAutoPrunePolicy, "GET", {"orgname": "buynlarge", "policy_uuid": policy_uuid} + ).json + assert response["method"] == policy.get("method") + assert response["value"] == policy.get("value") + assert response["tagPattern"] == policy.get("tagPattern") + assert response["tagPatternMatches"] == policy.get("tagPatternMatches", True) def test_update_org_policy_nonexistent_policy(initialized_db, app): @@ -191,20 +246,35 @@ def test_delete_org_policy_nonexistent_policy(initialized_db, app): def test_get_user_policies(initialized_db, app): with client_with_identity("devtable", app) as cl: - response = conduct_api_call(cl, UserAutoPrunePolicies, "GET", {"orgname": "devtable"}).json + response = conduct_api_call(cl, UserAutoPrunePolicies, "GET", {}).json assert len(response["policies"]) == 1 assert response["policies"][0]["method"] == "number_of_tags" assert response["policies"][0]["value"] == 10 -def test_create_user_policy(initialized_db, app): +@pytest.mark.parametrize( + "policy", + [ + ({"method": "creation_date", "value": "2w"}), + ({"method": "creation_date", "value": "2w", "tagPattern": "match.*"}), + ( + { + "method": "creation_date", + "value": "2w", + "tagPattern": "match.*", + "tagPatternMatches": False, + } + ), + ], +) +def test_create_user_policy(policy, initialized_db, app): with client_with_identity("freshuser", app) as cl: response = conduct_api_call( cl, UserAutoPrunePolicies, "POST", - {"orgname": "freshuser"}, - {"method": "creation_date", "value": "2w"}, + None, + policy, 201, ).json assert response["uuid"] is not None @@ -224,9 +294,21 @@ def test_create_user_policy(initialized_db, app): log = l break assert log is not None - assert json.loads(log.metadata_json)["method"] == "creation_date" - assert json.loads(log.metadata_json)["value"] == "2w" + assert json.loads(log.metadata_json)["method"] == policy.get("method") + assert json.loads(log.metadata_json)["value"] == policy.get("value") assert json.loads(log.metadata_json)["namespace"] == "freshuser" + assert json.loads(log.metadata_json)["tag_pattern"] == policy.get("tagPattern") + assert json.loads(log.metadata_json)["tag_pattern_matches"] == policy.get( + "tagPatternMatches", True + ) + + # Assert policy information is correct + response = conduct_api_call(cl, UserAutoPrunePolicies, "GET", {}).json + assert len(response["policies"]) == 1 + assert response["policies"][0]["method"] == policy.get("method") + assert response["policies"][0]["value"] == policy.get("value") + assert response["policies"][0]["tagPattern"] == policy.get("tagPattern") + assert response["policies"][0]["tagPatternMatches"] == policy.get("tagPatternMatches", True) def test_create_user_policy_already_existing(initialized_db, app): @@ -235,7 +317,7 @@ def test_create_user_policy_already_existing(initialized_db, app): cl, UserAutoPrunePolicies, "POST", - {"orgname": "devtable"}, + None, {"method": "creation_date", "value": "2w"}, expected_code=400, ).json @@ -257,7 +339,22 @@ def test_get_user_policy(initialized_db, app): assert response["value"] == 10 -def test_update_user_policy(initialized_db, app): +@pytest.mark.parametrize( + "policy", + [ + ({"method": "creation_date", "value": "2w"}), + ({"method": "creation_date", "value": "2w", "tagPattern": "match.*"}), + ( + { + "method": "creation_date", + "value": "2w", + "tagPattern": "match.*", + "tagPatternMatches": False, + } + ), + ], +) +def test_update_user_policy(policy, initialized_db, app): policies = model.autoprune.get_namespace_autoprune_policies_by_orgname("devtable") assert len(policies) == 1 policy_uuid = policies[0].uuid @@ -266,15 +363,15 @@ def test_update_user_policy(initialized_db, app): cl, UserAutoPrunePolicy, "PUT", - {"orgname": "devtable", "policy_uuid": policy_uuid}, - {"method": "creation_date", "value": "2w"}, + {"policy_uuid": policy_uuid}, + policy, 204, ) assert response is not None # Make another request asserting it was updated get_response = conduct_api_call( - cl, UserAutoPrunePolicy, "GET", {"orgname": "devtable", "policy_uuid": policy_uuid} + cl, UserAutoPrunePolicy, "GET", {"policy_uuid": policy_uuid} ).json assert get_response["method"] == "creation_date" assert get_response["value"] == "2w" @@ -288,9 +385,22 @@ def test_update_user_policy(initialized_db, app): log = l break assert log is not None - assert json.loads(log.metadata_json)["method"] == "creation_date" - assert json.loads(log.metadata_json)["value"] == "2w" + assert json.loads(log.metadata_json)["method"] == policy.get("method") + assert json.loads(log.metadata_json)["value"] == policy.get("value") assert json.loads(log.metadata_json)["namespace"] == "devtable" + assert json.loads(log.metadata_json)["tag_pattern"] == policy.get("tagPattern") + assert json.loads(log.metadata_json)["tag_pattern_matches"] == policy.get( + "tagPatternMatches", True + ) + + # Assert policy information is correct + response = conduct_api_call( + cl, UserAutoPrunePolicy, "GET", {"policy_uuid": policy_uuid} + ).json + assert response["method"] == policy.get("method") + assert response["value"] == policy.get("value") + assert response["tagPattern"] == policy.get("tagPattern") + assert response["tagPatternMatches"] == policy.get("tagPatternMatches", True) def test_update_user_policy_nonexistent_policy(initialized_db, app): @@ -358,7 +468,22 @@ def test_get_repo_policies(initialized_db, app): assert response["policies"][0]["value"] == 10 -def test_create_repo_policy(initialized_db, app): +@pytest.mark.parametrize( + "policy", + [ + ({"method": "creation_date", "value": "2w"}), + ({"method": "creation_date", "value": "2w", "tagPattern": "match.*"}), + ( + { + "method": "creation_date", + "value": "2w", + "tagPattern": "match.*", + "tagPatternMatches": False, + } + ), + ], +) +def test_create_repo_policy(policy, initialized_db, app): with client_with_identity("devtable", app) as cl: params = {"repository": "testorgforautoprune/autoprunerepo"} response = conduct_api_call( @@ -366,7 +491,7 @@ def test_create_repo_policy(initialized_db, app): RepositoryAutoPrunePolicies, "POST", params, - {"method": "creation_date", "value": "2w"}, + policy, 201, ).json assert response["uuid"] is not None @@ -388,9 +513,21 @@ def test_create_repo_policy(initialized_db, app): log = l break assert log is not None - assert json.loads(log.metadata_json)["method"] == "creation_date" - assert json.loads(log.metadata_json)["value"] == "2w" + assert json.loads(log.metadata_json)["method"] == policy.get("method") + assert json.loads(log.metadata_json)["value"] == policy.get("value") assert json.loads(log.metadata_json)["namespace"] == "testorgforautoprune" + assert json.loads(log.metadata_json)["tag_pattern"] == policy.get("tagPattern") + assert json.loads(log.metadata_json)["tag_pattern_matches"] == policy.get( + "tagPatternMatches", True + ) + + # Assert policy information is correct + response = conduct_api_call(cl, RepositoryAutoPrunePolicies, "GET", params).json + assert len(response["policies"]) == 1 + assert response["policies"][0]["method"] == policy.get("method") + assert response["policies"][0]["value"] == policy.get("value") + assert response["policies"][0]["tagPattern"] == policy.get("tagPattern") + assert response["policies"][0]["tagPatternMatches"] == policy.get("tagPatternMatches", True) def test_create_repo_policy_already_existing(initialized_db, app): @@ -435,7 +572,22 @@ def test_get_repo_policy(initialized_db, app): assert response["value"] == 10 -def test_update_repo_policy(initialized_db, app): +@pytest.mark.parametrize( + "policy", + [ + ({"method": "creation_date", "value": "2w"}), + ({"method": "creation_date", "value": "2w", "tagPattern": "match.*"}), + ( + { + "method": "creation_date", + "value": "2w", + "tagPattern": "match.*", + "tagPatternMatches": False, + } + ), + ], +) +def test_update_repo_policy(policy, initialized_db, app): policies = model.autoprune.get_repository_autoprune_policies_by_repo_name("devtable", "simple") assert len(policies) == 1 policy_uuid = policies[0].uuid @@ -446,7 +598,7 @@ def test_update_repo_policy(initialized_db, app): RepositoryAutoPrunePolicy, "PUT", params_for_update, - {"method": "creation_date", "value": "2w"}, + policy, expected_code=204, ) @@ -465,9 +617,20 @@ def test_update_repo_policy(initialized_db, app): log = l break assert log is not None - assert json.loads(log.metadata_json)["method"] == "creation_date" - assert json.loads(log.metadata_json)["value"] == "2w" + assert json.loads(log.metadata_json)["method"] == policy.get("method") + assert json.loads(log.metadata_json)["value"] == policy.get("value") assert json.loads(log.metadata_json)["namespace"] == "devtable" + assert json.loads(log.metadata_json)["tag_pattern"] == policy.get("tagPattern") + assert json.loads(log.metadata_json)["tag_pattern_matches"] == policy.get( + "tagPatternMatches", True + ) + + # Assert policy information is correct + response = conduct_api_call(cl, RepositoryAutoPrunePolicy, "GET", params).json + assert response["method"] == policy.get("method") + assert response["value"] == policy.get("value") + assert response["tagPattern"] == policy.get("tagPattern") + assert response["tagPatternMatches"] == policy.get("tagPatternMatches", True) def test_update_repo_policy_nonexistent_policy(initialized_db, app): @@ -531,3 +694,40 @@ def test_delete_repo_policy_nonexistent_policy(initialized_db, app): params_for_delete, expected_code=404, ) + + +@pytest.mark.parametrize( + "tag_pattern, expected, class_obj, params", + [ + ("match", 201, OrgAutoPrunePolicies, {"orgname": "sellnsmall"}), + ("", 400, OrgAutoPrunePolicies, {"orgname": "sellnsmall"}), + (123, 400, OrgAutoPrunePolicies, {"orgname": "sellnsmall"}), + ("match", 201, UserAutoPrunePolicies, None), + ("", 400, UserAutoPrunePolicies, None), + (123, 400, UserAutoPrunePolicies, None), + ( + "match", + 201, + RepositoryAutoPrunePolicies, + {"repository": "testorgforautoprune/autoprunerepo"}, + ), + ("", 400, RepositoryAutoPrunePolicies, {"repository": "testorgforautoprune/autoprunerepo"}), + ( + 123, + 400, + RepositoryAutoPrunePolicies, + {"repository": "testorgforautoprune/autoprunerepo"}, + ), + ], +) +def test_valid_tag_patterm(tag_pattern, expected, class_obj, params, initialized_db, app): + user = "freshuser" if class_obj.__name__ == "UserAutoPrunePolicies" else "devtable" + with client_with_identity(user, app) as cl: + conduct_api_call( + cl, + class_obj, + "POST", + params, + {"method": "creation_date", "value": "2w", "tagPattern": tag_pattern}, + expected, + ) diff --git a/web/cypress/e2e/autopruning.cy.ts b/web/cypress/e2e/autopruning.cy.ts index 6fecf9c9bb..944a4fbeb2 100644 --- a/web/cypress/e2e/autopruning.cy.ts +++ b/web/cypress/e2e/autopruning.cy.ts @@ -166,6 +166,28 @@ describe('Namespace settings - autoprune policies', () => { cy.get('[data-testid="registry-autoprune-policy-value"]').contains('10'); }); + it('creates policy with tag filter', () => { + cy.visit('/organization/testorg?tab=Settings'); + cy.contains('Auto-Prune Policies').click(); + cy.get('[data-testid="namespace-auto-prune-method"]').select( + 'By age of tags', + ); + cy.get('input[aria-label="tag creation date value"]').should( + 'have.value', + '7', + ); + cy.get('select[aria-label="tag creation date unit"]').contains('days'); + cy.get('input[aria-label="tag creation date value"]').type( + '2{leftArrow}{backspace}', + ); + cy.get('select[aria-label="tag creation date unit"]').select('weeks'); + + cy.get('input[aria-label="tag pattern"]').type('v1.*'); + cy.get('select[aria-label="tag pattern matches"]').select('does not match'); + cy.contains('Save').click(); + cy.contains('Successfully created auto-prune policy'); + }); + // TODO: Uncomment once user settings is supported // it('updates policy for users', () => { // cy.visit('/organization/user1?tab=Settings'); diff --git a/web/cypress/e2e/repository-autopruning.cy.ts b/web/cypress/e2e/repository-autopruning.cy.ts index 82da5d3375..24efed37e3 100644 --- a/web/cypress/e2e/repository-autopruning.cy.ts +++ b/web/cypress/e2e/repository-autopruning.cy.ts @@ -196,4 +196,24 @@ describe('Repository settings - Repository autoprune policies', () => { ); cy.get('[data-testid="registry-autoprune-policy-value"]').contains('10'); }); + + it('creates policy with tag filter', () => { + cy.visit('/repository/testorg/testrepo?tab=settings'); + cy.contains('Repository Auto-Prune Policies').click(); + cy.get('[data-testid="repository-auto-prune-method"]').select( + 'By age of tags', + ); + cy.get('input[aria-label="tag creation date value"]').should( + 'have.value', + '7', + ); + cy.get('select[aria-label="tag creation date unit"]').contains('days'); + cy.get('input[aria-label="tag creation date value"]').type( + '2{leftArrow}{backspace}', + ); + cy.get('select[aria-label="tag creation date unit"]').select('weeks'); + cy.get('input[aria-label="tag pattern"]').type('v1.*'); + cy.get('select[aria-label="tag pattern matches"]').select('does not match'); + cy.contains('Save').click(); + }); }); diff --git a/web/src/resources/NamespaceAutoPruneResource.ts b/web/src/resources/NamespaceAutoPruneResource.ts index 84f40363cf..67e197801f 100644 --- a/web/src/resources/NamespaceAutoPruneResource.ts +++ b/web/src/resources/NamespaceAutoPruneResource.ts @@ -12,6 +12,8 @@ export interface NamespaceAutoPrunePolicy { method: AutoPruneMethod; uuid?: string; value?: string | number; + tagPattern?: string; + tagPatternMatches?: boolean; } export async function fetchNamespaceAutoPrunePolicies( diff --git a/web/src/resources/RepositoryAutoPruneResource.ts b/web/src/resources/RepositoryAutoPruneResource.ts index 73c703c9a6..3f7661d2c2 100644 --- a/web/src/resources/RepositoryAutoPruneResource.ts +++ b/web/src/resources/RepositoryAutoPruneResource.ts @@ -7,6 +7,8 @@ export interface RepositoryAutoPrunePolicy { method: AutoPruneMethod; uuid?: string; value?: string | number; + tagPattern?: string; + tagPatternMatches?: boolean; } export async function fetchRepositoryAutoPrunePolicies( diff --git a/web/src/routes/OrganizationsList/Organization/Tabs/Settings/AutoPruning.tsx b/web/src/routes/OrganizationsList/Organization/Tabs/Settings/AutoPruning.tsx index f3c628640e..9964208ddd 100644 --- a/web/src/routes/OrganizationsList/Organization/Tabs/Settings/AutoPruning.tsx +++ b/web/src/routes/OrganizationsList/Organization/Tabs/Settings/AutoPruning.tsx @@ -12,6 +12,7 @@ import { FormHelperText, HelperText, HelperTextItem, + TextInput, } from '@patternfly/react-core'; import {useEffect, useState} from 'react'; import {AlertVariant} from 'src/atoms/AlertState'; @@ -47,6 +48,8 @@ export default function AutoPruning(props: AutoPruning) { const [tagCount, setTagCount] = useState(20); const [tagCreationDateUnit, setTagCreationDateUnit] = useState('d'); const [tagCreationDateValue, setTagCreationDateValue] = useState(7); + const [tagPattern, setTagPattern] = useState(null); + const [tagPatternMatches, setTagPatternMatches] = useState(true); const {addAlert} = useAlerts(); const config = useQuayConfig(); const { @@ -83,6 +86,8 @@ export default function AutoPruning(props: AutoPruning) { const policy: NamespaceAutoPrunePolicy = nsPolicies[0]; setMethod(policy.method); setUuid(policy.uuid); + setTagPattern(policy.tagPattern); + setTagPatternMatches(policy.tagPatternMatches); switch (policy.method) { case AutoPruneMethod.TAG_NUMBER: { setTagCount(policy.value as number); @@ -110,6 +115,8 @@ export default function AutoPruning(props: AutoPruning) { setTagCount(20); setTagCreationDateUnit('d'); setTagCreationDateValue(7); + setTagPattern(null); + setTagPatternMatches(true); } } }, [successFetchingPolicies, dataUpdatedAt]); @@ -192,9 +199,23 @@ export default function AutoPruning(props: AutoPruning) { return; } if (isNullOrUndefined(uuid)) { - createPolicy({method: method, value: value}); + const policy: NamespaceAutoPrunePolicy = {method: method, value: value}; + if (tagPattern != null) { + policy.tagPattern = tagPattern; + policy.tagPatternMatches = tagPatternMatches; + } + createPolicy(policy); } else { - updatePolicy({uuid: uuid, method: method, value: value}); + const policy: NamespaceAutoPrunePolicy = { + uuid: uuid, + method: method, + value: value, + }; + if (tagPattern != null) { + policy.tagPattern = tagPattern; + policy.tagPatternMatches = tagPatternMatches; + } + updatePolicy(policy); } }; @@ -350,7 +371,58 @@ export default function AutoPruning(props: AutoPruning) { - + + +
+ { + setTagPatternMatches(val === 'matches'); + }} + aria-label="tag pattern matches" + > + + + +
+ + + + {tagPatternMatches + ? 'Only tags matching the given regex pattern will be pruned' + : 'Only tags not matching the given regex pattern will be pruned'} + + + +
+ +
+ + val !== '' ? setTagPattern(val) : setTagPattern(null) + } + aria-label="tag pattern" + data-testid="tag-pattern" + /> +
+ + + + The regex pattern to match tags against. Defaults to all tags + if left empty. + + + +
+
(20); const [tagCreationDateUnit, setTagCreationDateUnit] = useState('d'); const [tagCreationDateValue, setTagCreationDateValue] = useState(7); + const [tagPattern, setTagPattern] = useState(null); + const [tagPatternMatches, setTagPatternMatches] = useState(true); const {addAlert} = useAlerts(); const {organization} = useOrganization(props.organizationName); const config = useQuayConfig(); @@ -110,6 +107,8 @@ export default function RepositoryAutoPruning(props: RepositoryAutoPruning) { const policy: RepositoryAutoPrunePolicy = repoPolicies[0]; setMethod(policy.method); setUuid(policy.uuid); + setTagPattern(policy.tagPattern); + setTagPatternMatches(policy.tagPatternMatches); switch (policy.method) { case AutoPruneMethod.TAG_NUMBER: { setTagCount(policy.value as number); @@ -136,6 +135,8 @@ export default function RepositoryAutoPruning(props: RepositoryAutoPruning) { setTagCount(20); setTagCreationDateUnit('d'); setTagCreationDateValue(7); + setTagPattern(null); + setTagPatternMatches(true); } } }, [ @@ -222,9 +223,23 @@ export default function RepositoryAutoPruning(props: RepositoryAutoPruning) { return; } if (isNullOrUndefined(uuid)) { - createRepoPolicy({method: method, value: value}); + const policy: RepositoryAutoPrunePolicy = {method: method, value: value}; + if (tagPattern != null) { + policy.tagPattern = tagPattern; + policy.tagPatternMatches = tagPatternMatches; + } + createRepoPolicy(policy); } else { - updateRepoPolicy({uuid: uuid, method: method, value: value}); + const policy: RepositoryAutoPrunePolicy = { + uuid: uuid, + method: method, + value: value, + }; + if (tagPattern != null) { + policy.tagPattern = tagPattern; + policy.tagPatternMatches = tagPatternMatches; + } + updateRepoPolicy(policy); } }; @@ -386,6 +401,55 @@ export default function RepositoryAutoPruning(props: RepositoryAutoPruning) { + + +
+ { + setTagPatternMatches(val === 'matches'); + }} + aria-label="tag pattern matches" + > + + + +
+ + + + {tagPatternMatches + ? 'Only the tags matching the given pattern will be pruned' + : 'Only the tags not matching the given pattern will be pruned'} + + + +
+ +
+ + val !== '' ? setTagPattern(val) : setTagPattern(null) + } + aria-label="tag pattern" + data-testid="tag-pattern" + /> +
+ + + + Only the tags matching the given pattern will be pruned + + + +
+