From 83f00ffad6af829d541d55f70922d208e5462ebc Mon Sep 17 00:00:00 2001 From: etj Date: Wed, 25 Jan 2023 19:33:50 +0100 Subject: [PATCH] [Fixes #10537] Improve rules creation using GeoFence batch - more improvements --- geonode/api/tests.py | 27 +- geonode/geoserver/geofence.py | 182 +++++++++---- geonode/geoserver/helpers.py | 18 +- geonode/geoserver/manager.py | 103 +++++--- geonode/geoserver/security.py | 290 ++++++++++----------- geonode/local_settings.py.geoserver.sample | 1 + geonode/security/tests.py | 234 ++++++++--------- geonode/settings.py | 1 + geonode/upload/tests/integration.py | 4 +- geonode/utils.py | 2 - 10 files changed, 479 insertions(+), 383 deletions(-) diff --git a/geonode/api/tests.py b/geonode/api/tests.py index 56ebc396da8..3a890474011 100644 --- a/geonode/api/tests.py +++ b/geonode/api/tests.py @@ -30,6 +30,7 @@ from guardian.shortcuts import get_anonymous_user from geonode import geoserver +from geonode.geoserver.manager import GeoServerResourceManager from geonode.maps.models import Map from geonode.layers.models import Dataset from geonode.documents.models import Document @@ -166,26 +167,31 @@ def test_dataset_get_detail_unauth_dataset_not_public(self): 'api_name': 'api', 'resource_name': 'datasets'}) + resp = self.client.get(list_url) + self.assertEqual(len(self.deserialize(resp)['objects']), 8) + layer = Dataset.objects.first() layer.set_permissions(self.perm_spec) layer.clear_dirty_state() - self.assertHttpNotFound(self.api_client.get( - f"{list_url + str(layer.id)}/")) - self.api_client.client.login(username=self.user, password=self.passwd) - resp = self.api_client.get(f"{list_url + str(layer.id)}/") - self.assertValidJSONResponse(resp) + resp = self.client.get(list_url) + self.assertEqual(len(self.deserialize(resp)['objects']), 7) + self.assertHttpNotFound(self.client.get(f"{list_url + str(layer.id)}/")) + + self.client.login(username=self.user, password=self.passwd) + self.assertValidJSONResponse(self.client.get(f"{list_url + str(layer.id)}/")) # with delayed security - with self.settings(DELAYED_SECURITY_SIGNALS=True): + with self.settings(DELAYED_SECURITY_SIGNALS=True, GEOFENCE_SECURITY_ENABLED=True): if check_ogc_backend(geoserver.BACKEND_PACKAGE): - from geonode.geoserver.security import sync_geofence_with_guardian - sync_geofence_with_guardian(layer, self.perm_spec) + + gm = GeoServerResourceManager() + gm.set_permissions(layer.uuid, instance=layer, permissions=self.perm_spec) self.assertTrue(layer.dirty_state) self.client.login(username=self.user, password=self.passwd) resp = self.client.get(list_url) - self.assertEqual(len(self.deserialize(resp)['objects']), 7) + self.assertEqual(len(self.deserialize(resp)['objects']), 7) # admin can't see resources in dirty_state self.client.logout() resp = self.client.get(list_url) @@ -194,8 +200,7 @@ def test_dataset_get_detail_unauth_dataset_not_public(self): from django.contrib.auth import get_user_model get_user_model().objects.create( username='imnew', - password='pbkdf2_sha256$12000$UE4gAxckVj4Z$N\ - 6NbOXIQWWblfInIoq/Ta34FdRiPhawCIZ+sOO3YQs=') + password='pbkdf2_sha256$12000$UE4gAxckVj4Z$N6NbOXIQWWblfInIoq/Ta34FdRiPhawCIZ+sOO3YQs=') self.client.login(username='imnew', password='thepwd') resp = self.client.get(list_url) self.assertEqual(len(self.deserialize(resp)['objects']), 7) diff --git a/geonode/geoserver/geofence.py b/geonode/geoserver/geofence.py index d8c385c64e4..8006f01b2e8 100644 --- a/geonode/geoserver/geofence.py +++ b/geonode/geoserver/geofence.py @@ -17,17 +17,15 @@ # ######################################################################### +import itertools import logging -import urllib import requests - -from django.conf import settings from requests.auth import HTTPBasicAuth +import traceback +import urllib logger = logging.getLogger(__name__) -ogc_server_settings = settings.OGC_SERVER['default'] - class GeofenceException(Exception): pass @@ -35,7 +33,8 @@ class GeofenceException(Exception): class Rule: """_summary_ - JSON representation of a GeoFence Rule + A GeoFence Rule. + Provides the object to be rendered as JSON. e.g.: {"Rule": @@ -56,9 +55,11 @@ class Rule: ALLOW = "ALLOW" DENY = "DENY" LIMIT = "LIMIT" + CM_MIXED = "MIXED" - def __init__(self, priority, workspace, layer, access: (str, bool), + def __init__(self, access: (str, bool), + priority=None, workspace=None, layer=None, user=None, group=None, service=None, request=None, subfield=None, geo_limit=None, catalog_mode=None) -> None: @@ -99,14 +100,18 @@ def __init__(self, priority, workspace, layer, access: (str, bool), if limits: self.fields['limits'] = limits - def get_object(self): + def set_priority(self, pri: int): + self.fields['priority'] = pri + + def get_object(self) -> dict: logger.debug(f"Creating Rule object: {self.fields}") return {'Rule': self.fields} class Batch: """_summary_ - Returns a list of Operations that GeoFence can execute in a batch + A GeoFence Batch. + It's a list of operations that will be executed transactionally inside GeoFence. e.g.: { @@ -201,11 +206,10 @@ def add_insert_rule(self, rule: Rule): operation.update(rule.get_object()) self.operations.append(operation) - def get_batch_length(self): + def get_batch_length(self) -> int: return len(self.operations) - def get_object(self): - logger.debug(f"Creating Batch object {self.log_name} with {len(self.operations)} operations") + def get_object(self) -> dict: return { 'Batch': { 'operations': self.operations @@ -213,32 +217,53 @@ def get_object(self): } -class GeofenceClient: +class AutoPriorityBatch(Batch): """_summary_ - Instance of a simple GeoFence REST client allowing to interact with the GeoServer APIs. - Exposes few utility methods to insert or purge the rules and run batches of operations. + A Batch that handles the priority of the inserted rules. + The first rule will have the declared `start_rule_pri`, next Rules will have the priority incremented. + """ - Returns: - _type_: Rule + def __init__(self, start_rule_pri: int, log_name=None) -> None: + super().__init__(log_name) + self.pri = itertools.count(start_rule_pri) + + def add_insert_rule(self, rule: Rule): + rule.set_priority(self.pri.__next__()) + super().add_insert_rule(rule) + + +class GeoFenceClient: + """_summary_ + A GeoFence REST client allowing to interact with the embedded GeoFence API (which is slightly incompatible + with the original standalone GeoFence API.) + The class methods map on GeoFence operations. + Functionalities needing more than one call are implemented within GeoFenceUtils. """ def __init__(self, baseurl: str, username: str, pw: str) -> None: + if not baseurl.endswith('/'): + baseurl += '/' + self.baseurl = baseurl self.username = username self.pw = pw + self.timeout = 60 + + def set_timeout(self, timeout: int): + self.timeout = timeout def invalidate_cache(self): r = requests.put( - f'{self.baseurl.rstrip("/")}/geofence/ruleCache/invalidate', + f'{self.baseurl}ruleCache/invalidate', auth=HTTPBasicAuth(self.username, self.pw)) if r.status_code != 200: logger.debug("Could not invalidate cache") raise GeofenceException("Could not invalidate cache") - def get_rules(self, page=None, entries=None, - workspace=None, workspace_any=None, - layer=None, layer_any=None): + def get_rules(self, page: int = None, entries: int = None, + workspace: str = None, workspace_any: bool = None, + layer: str = None, layer_any: bool = None): if (page is None and entries is not None) or (page is not None and entries is None): raise GeofenceException(f"Bad page/entries combination {page}/{entries}") @@ -261,13 +286,13 @@ def get_rules(self, page=None, entries=None, if value is not None: params[param] = value - url = f'{self.baseurl.rstrip("/")}/geofence/rules.json?{urllib.parse.urlencode(params)}' + url = f'{self.baseurl}rules.json?{urllib.parse.urlencode(params)}' r = requests.get( url, headers={'Content-type': 'application/json'}, auth=HTTPBasicAuth(self.username, self.pw), - timeout=ogc_server_settings.get('TIMEOUT', 10), + timeout=self.timeout, verify=False) if r.status_code != 200: @@ -287,10 +312,10 @@ def get_rules_count(self): http://:/geoserver/rest/geofence/rules/count.json """ r = requests.get( - f'{self.baseurl.rstrip("/")}/geofence/rules/count.json', + f'{self.baseurl}rules/count.json', headers={'Content-type': 'application/json'}, auth=HTTPBasicAuth(self.username, self.pw), - timeout=ogc_server_settings.get('TIMEOUT', 10), + timeout=self.timeout, verify=False) if r.status_code != 200: @@ -312,11 +337,10 @@ def insert_rule(self, rule: Rule): http://:/geoserver/rest/geofence/rules """ r = requests.post( - f'{self.baseurl.rstrip("/")}/geofence/rules', - # headers={'Content-type': 'application/json'}, + f'{self.baseurl}rules', json=rule.get_object(), auth=HTTPBasicAuth(self.username, self.pw), - timeout=ogc_server_settings.get('TIMEOUT', 60), + timeout=self.timeout, verify=False) if r.status_code not in (200, 201): @@ -327,36 +351,44 @@ def insert_rule(self, rule: Rule): logger.debug("Error while inserting rule", exc_info=e) raise GeofenceException(f"Error while inserting rule: {e}") - def run_batch(self, batch: Batch): + def run_batch(self, batch: Batch, timeout: int = None) -> bool: if batch.get_batch_length() == 0: logger.debug(f'Skipping batch execution {batch.log_name}') - return + return False + logger.debug(f"Running batch {batch.log_name} with {batch.get_batch_length()} operations") try: """ curl -X GET -u admin:geoserver \ http://:/geoserver/rest/geofence/rules/count.json """ r = requests.post( - f'{self.baseurl.rstrip("/")}/geofence/batch/exec', + f'{self.baseurl}batch/exec', json=batch.get_object(), auth=HTTPBasicAuth(self.username, self.pw), - timeout=ogc_server_settings.get('TIMEOUT', 60), + timeout=timeout or self.timeout, verify=False) if r.status_code != 200: - logger.debug(f"Error while running batch {batch.log_name}: [{r.status_code}] - {r.content}") + logger.debug(f"Error while running batch {batch.log_name}: [{r.status_code}] - {r.content}" + f"\n {batch.get_object()}") raise GeofenceException(f"Error while running batch {batch.log_name}: [{r.status_code}]") - return + return True except Exception as e: - logger.debug(f"Error while requesting batch execution {batch.log_name}", exc_info=e) + logger.info(f"Error while requesting batch exec {batch.log_name}") + logger.debug(f"Error while requesting batch exec {batch.log_name} --> {batch.get_object()}", exc_info=e) raise GeofenceException(f"Error while requesting batch execution {batch.log_name}: {e}") - def purge_all_rules(self): + +class GeoFenceUtils: + def __init__(self, client: GeoFenceClient): + self.geofence = client + + def delete_all_rules(self): """purge all existing GeoFence Cache Rules""" - rules_objs = self.get_rules() + rules_objs = self.geofence.get_rules() rules = rules_objs['rules'] batch = Batch('Purge All') @@ -364,21 +396,61 @@ def purge_all_rules(self): batch.add_delete_rule(rule['id']) logger.debug(f"Going to remove all {len(rules)} rules in geofence") - self.run_batch(batch) - - def purge_layer_rules(self, layer_name: str, workspace: str = None): - """purge existing GeoFence Cache Rules related to a specific Layer""" - gs_rules = self.get_rules( - workspace=workspace, workspace_any=False, - layer=layer_name, layer_any=False) - - batch = Batch(f'Purge {workspace}:{layer_name}') - - if gs_rules and gs_rules['rules']: - logger.debug(f"Going to remove {len(gs_rules['rules'])} rules for layer '{layer_name}'") - for r in gs_rules['rules']: - if r['layer'] and r['layer'] == layer_name: - batch.add_delete_rule(r['id']) - else: - logger.debug(f"Bad rule retrieved for dataset '{layer_name}': {r}") - self.run_batch(batch) + self.geofence.run_batch(batch) + + def collect_delete_layer_rules(self, workspace_name: str, layer_name: str, batch: Batch = None) -> Batch: + """Collect delete operations in a Batch for all rules related to a layer""" + + try: + # Scan GeoFence Rules associated to the Dataset + gs_rules = self.geofence.get_rules( + workspace=workspace_name, workspace_any=False, + layer=layer_name, layer_any=False) + + if not batch: + batch = Batch(f'Delete {workspace_name}:{layer_name}') + + cnt = 0 + if gs_rules and gs_rules['rules']: + logger.debug(f"Going to collect {len(gs_rules['rules'])} rules for layer '{workspace_name}:{layer_name}'") + for r in gs_rules['rules']: + if r['layer'] and r['layer'] == layer_name: + batch.add_delete_rule(r['id']) + cnt += 1 + else: + logger.warning(f"Bad rule retrieved for dataset '{workspace_name or ''}:{layer_name}': {r}") + + logger.debug(f"Adding {cnt} rule deletion operations for '{workspace_name or ''}:{layer_name}") + return batch + + except Exception as e: + logger.error(f"Error collecting rules for {workspace_name}:{layer_name}", exc_info=e) + tb = traceback.format_exc() + logger.debug(tb) + + def delete_layer_rules(self, workspace_name: str, layer_name: str) -> bool: + """Delete all Rules related to a specific Layer""" + try: + batch = self.collect_delete_layer_rules(workspace_name, layer_name) + return self.geofence.run_batch(batch) + + except Exception as e: + logger.error(f"Error removing rules for {workspace_name}:{layer_name}", exc_info=e) + tb = traceback.format_exc() + logger.debug(tb) + return False + + def get_first_available_priority(self): + """Get the highest Rules priority""" + try: + rules_count = self.geofence.get_rules_count() + rules_objs = self.geofence.get_rules(page=rules_count - 1, entries=1) + if len(rules_objs['rules']) > 0: + highest_priority = rules_objs['rules'][0]['priority'] + else: + highest_priority = 0 + return int(highest_priority) + 1 + except Exception: + tb = traceback.format_exc() + logger.debug(tb) + return -1 diff --git a/geonode/geoserver/helpers.py b/geonode/geoserver/helpers.py index 2500e8ea3f4..e4c7514f171 100755 --- a/geonode/geoserver/helpers.py +++ b/geonode/geoserver/helpers.py @@ -76,7 +76,7 @@ is_monochromatic_image, set_resource_default_links) -from .geofence import GeofenceClient +from .geofence import GeoFenceClient, GeoFenceUtils logger = logging.getLogger(__name__) @@ -1919,7 +1919,21 @@ def get_time_info(layer): retries=ogc_server_settings.MAX_RETRIES, backoff_factor=ogc_server_settings.BACKOFF_FACTOR) gs_uploader = Client(url, _user, _password) -gf_client = GeofenceClient(url, _user, _password) + + +def create_geofence_client(): + gs_url = settings.OGC_SERVER['default']['LOCATION'] + user = settings.OGC_SERVER['default']['USER'] + passwd = settings.OGC_SERVER['default']['PASSWORD'] + + gf_rest_url = f'{gs_url.rstrip("/")}/rest/geofence/' + client = GeoFenceClient(gf_rest_url, user, passwd) + client.set_timeout(settings.OGC_SERVER['default'].get('GEOFENCE_TIMEOUT', 60)) + return client + + +geofence = create_geofence_client() +gf_utils = GeoFenceUtils(geofence) _punc = re.compile(r"[\.:]") # regex for punctuation that confuses restconfig _foregrounds = [ diff --git a/geonode/geoserver/manager.py b/geonode/geoserver/manager.py index ff9aeb9bd5a..2a4abbb4570 100644 --- a/geonode/geoserver/manager.py +++ b/geonode/geoserver/manager.py @@ -47,6 +47,7 @@ ResourceManager, ResourceManagerInterface) from geonode.geoserver.signals import geofence_rule_assign +from .geofence import AutoPriorityBatch from .tasks import ( geoserver_set_style, geoserver_delete_map, @@ -63,13 +64,16 @@ sync_instance_with_geoserver, set_attributes_from_geoserver, create_gs_thumbnail, - create_geoserver_db_featurestore) + create_geoserver_db_featurestore, + geofence, gf_utils, +) from .security import ( _get_gwc_filters_and_formats, toggle_dataset_cache, - purge_geofence_dataset_rules, - set_geofence_invalidate_cache, - sync_permissions_and_disable_cache) + invalidate_geofence_cache, + has_geolimits, + create_geofence_rules, +) logger = logging.getLogger(__name__) @@ -382,8 +386,10 @@ def remove_permissions(self, uuid: str, /, instance: ResourceBase = None) -> boo if instance and isinstance(instance.get_real_instance(), Dataset): if settings.OGC_SERVER['default']['GEOFENCE_SECURITY_ENABLED']: if not getattr(settings, 'DELAYED_SECURITY_SIGNALS', False): - purge_geofence_dataset_rules(instance.get_real_instance()) - set_geofence_invalidate_cache() + workspace = get_dataset_workspace(instance) + removed = gf_utils.delete_layer_rules(workspace, instance.name) + if removed: + invalidate_geofence_cache() else: instance.set_dirty_state() except Exception as e: @@ -391,82 +397,101 @@ def remove_permissions(self, uuid: str, /, instance: ResourceBase = None) -> boo return False return True - def set_permissions(self, uuid: str, /, instance: ResourceBase = None, owner: settings.AUTH_USER_MODEL = None, permissions: dict = {}, created: bool = False, - approval_status_changed: bool = False, group_status_changed: bool = False) -> bool: + def set_permissions(self, uuid: str, /, instance: ResourceBase = None, + owner: settings.AUTH_USER_MODEL = None, + permissions: dict = {}, + created: bool = False, + approval_status_changed: bool = False, + group_status_changed: bool = False) -> bool: + _resource = instance or ResourceManager._get_instance(uuid) try: if _resource: _resource = _resource.get_real_instance() - logger.error(f'Fixup GIS Backend Security Rules Accordingly on resource {instance} {isinstance(_resource, Dataset)}') + logger.info(f'Requesting GeoFence rules on resource "{_resource}" :: {type(_resource).__name__}') if isinstance(_resource, Dataset): - if settings.OGC_SERVER['default'].get("GEOFENCE_SECURITY_ENABLED", False): + if settings.OGC_SERVER['default'].get("GEOFENCE_SECURITY_ENABLED", False) or \ + getattr(settings, "GEOFENCE_SECURITY_ENABLED", False): if not getattr(settings, 'DELAYED_SECURITY_SIGNALS', False): - _disable_cache = [] + batch = AutoPriorityBatch( + gf_utils.get_first_available_priority(), f"Set permission for resource {_resource}") + + workspace = get_dataset_workspace(_resource) + + if not created: + gf_utils.collect_delete_layer_rules(workspace, _resource.name, batch) + + exist_geolimits = None _owner = owner or _resource.owner - if permissions is not None and len(permissions): - if not created: - purge_geofence_dataset_rules(_resource) + if permissions is not None and len(permissions): # Owner perms = OWNER_PERMISSIONS.copy() + DATASET_ADMIN_PERMISSIONS.copy() + DOWNLOAD_PERMISSIONS.copy() - _disable_cache = sync_permissions_and_disable_cache(_disable_cache, _resource, perms, _owner, None, None) + create_geofence_rules(_resource, perms, _owner, None, batch) + exist_geolimits = exist_geolimits or has_geolimits(_resource, _owner, None) # All the other users if 'users' in permissions and len(permissions['users']) > 0: - for user, perms in permissions['users'].items(): + for user, user_perms in permissions['users'].items(): _user = get_user_model().objects.get(username=user) if _user != _owner: - # Set the GeoFence Rules - group_perms = None - if 'groups' in permissions and len(permissions['groups']) > 0: - group_perms = permissions['groups'] if user == "AnonymousUser": _user = None - _group = list(group_perms.keys())[0] if group_perms else None - _disable_cache = sync_permissions_and_disable_cache(_disable_cache, _resource, perms, _user, _group, group_perms) + create_geofence_rules(_resource, user_perms, _user, None, batch) + exist_geolimits = exist_geolimits or has_geolimits(_resource, _user, None) # All the other groups if 'groups' in permissions and len(permissions['groups']) > 0: for group, perms in permissions['groups'].items(): _group = Group.objects.get(name=group) - # Set the GeoFence Rules if _group and _group.name and _group.name == 'anonymous': _group = None - _disable_cache = sync_permissions_and_disable_cache(_disable_cache, _resource, perms, None, _group, None) + create_geofence_rules(_resource, perms, None, _group, batch) + exist_geolimits = exist_geolimits or has_geolimits(_resource, None, _group) else: - anonymous_can_view = settings.DEFAULT_ANONYMOUS_VIEW_PERMISSION - anonymous_can_download = settings.DEFAULT_ANONYMOUS_DOWNLOAD_PERMISSION - - if not created: - purge_geofence_dataset_rules(_resource.get_self_resource()) - # Owner & Managers perms = OWNER_PERMISSIONS.copy() + DATASET_ADMIN_PERMISSIONS.copy() + DOWNLOAD_PERMISSIONS.copy() - _disable_cache = sync_permissions_and_disable_cache(_disable_cache, _resource, perms, _owner, None, None) + create_geofence_rules(_resource, perms, _owner, None, batch) + exist_geolimits = exist_geolimits or has_geolimits(_resource, _owner, None) _resource_groups, _group_managers = _resource.get_group_managers(group=_resource.group) for _group_manager in _group_managers: - _disable_cache = sync_permissions_and_disable_cache(_disable_cache, _resource, perms, _group_manager, None, None) + create_geofence_rules(_resource, perms, _group_manager, None, batch) + exist_geolimits = exist_geolimits or has_geolimits(_resource, _group_manager, None) for user_group in _resource_groups: if not skip_registered_members_common_group(user_group): - _disable_cache = sync_permissions_and_disable_cache(_disable_cache, _resource, perms, None, user_group, None) + create_geofence_rules(_resource, perms, None, user_group, batch) + exist_geolimits = exist_geolimits or has_geolimits(_resource, None, user_group) # Anonymous - if anonymous_can_view: - _disable_cache = sync_permissions_and_disable_cache(_disable_cache, _resource, VIEW_PERMISSIONS, None, None, None) + if settings.DEFAULT_ANONYMOUS_VIEW_PERMISSION: + create_geofence_rules(_resource, VIEW_PERMISSIONS, None, None, batch) + exist_geolimits = exist_geolimits or has_geolimits(_resource, None, None) - if anonymous_can_download: - _disable_cache = sync_permissions_and_disable_cache(_disable_cache, _resource, DOWNLOAD_PERMISSIONS, None, None, None) + if settings.DEFAULT_ANONYMOUS_DOWNLOAD_PERMISSION: + create_geofence_rules(_resource, DOWNLOAD_PERMISSIONS, None, None, batch) + exist_geolimits = exist_geolimits or has_geolimits(_resource, None, None) - if _disable_cache: - filters, formats = _get_gwc_filters_and_formats(_disable_cache) + if exist_geolimits is not None: + filters, formats = _get_gwc_filters_and_formats(exist_geolimits) try: _dataset_workspace = get_dataset_workspace(_resource) toggle_dataset_cache(f'{_dataset_workspace}:{_resource.name}', filters=filters, formats=formats) except Dataset.DoesNotExist: pass + + try: + logger.info(f'Pushing {batch.get_batch_length()} ' + f'changes into GeoFence for resource {_resource.name}') + executed = geofence.run_batch(batch) + if executed: + geofence.invalidate_cache() + except Exception as e: + logger.warning(f'Could not sync GeoFence for resource {_resource}: {e}.' + ' Retrying async.') + _resource.set_dirty_state() else: _resource.set_dirty_state() except Exception as e: diff --git a/geonode/geoserver/security.py b/geonode/geoserver/security.py index 18da209bfa8..5cc4785617b 100644 --- a/geonode/geoserver/security.py +++ b/geonode/geoserver/security.py @@ -16,11 +16,11 @@ # along with this program. If not, see . # ######################################################################### -import itertools + import logging -import typing import requests import traceback +import typing import xml.etree.ElementTree as ET from lxml import etree @@ -31,60 +31,39 @@ from django.conf import settings from django.contrib.auth.models import Group from django.contrib.auth import get_user_model - +from geonode.geoserver.geofence import Batch, Rule, AutoPriorityBatch +from geonode.geoserver.helpers import geofence, gf_utils from geonode.groups.models import GroupProfile from geonode.utils import get_dataset_workspace -from geonode.geoserver.helpers import gf_client -from geonode.geoserver.geofence import Batch, Rule - -logger = logging.getLogger(__name__) -def get_highest_priority(): - """Get the highest Rules priority""" - try: - rules_count = gf_client.get_rules_count() - rules_objs = gf_client.get_rules(page=rules_count - 1, entries=1) - if len(rules_objs['rules']) > 0: - highest_priority = rules_objs['rules'][0]['priority'] - else: - highest_priority = 0 - return int(highest_priority) - except Exception: - tb = traceback.format_exc() - logger.debug(tb) - return -1 +logger = logging.getLogger(__name__) -def purge_geofence_all(): - """purge all existing GeoFence Cache Rules""" +def delete_all_geofence_rules(): + """purge all existing GeoFence Rules""" if settings.OGC_SERVER['default']['GEOFENCE_SECURITY_ENABLED']: - gf_client.purge_all_rules() + gf_utils.delete_all_rules() -def purge_geofence_dataset_rules(resource): - """purge layer existing GeoFence Cache Rules""" - # Scan GeoFence Rules associated to the Dataset - """ - curl -u admin:geoserver - http://:/geoserver/rest/geofence/rules.json?workspace=geonode&layer={layer} +def delete_geofence_rules_for_layer(instance): + """Delete all rules for a given layer + This function wraps the gf_util method, since it doesn't know about Dataset model """ - workspace = get_dataset_workspace(resource.dataset) - dataset_name = resource.dataset.name if resource.dataset and hasattr(resource.dataset, 'name') \ - else resource.dataset.alternate - try: - gf_client.purge_layer_rules(dataset_name, workspace=workspace) - except Exception as e: - logger.error(f"Error removing rules for {workspace}:{dataset_name}", exc_info=e) - tb = traceback.format_exc() - logger.debug(tb) + if settings.OGC_SERVER['default']['GEOFENCE_SECURITY_ENABLED']: + resource = instance.get_self_resource() + workspace_name = get_dataset_workspace(resource.dataset) + layer_name = resource.dataset.name if resource.dataset and hasattr(resource.dataset, 'name') \ + else resource.dataset.alternate + logger.debug(f"Removing rules for layer {workspace_name}:{layer_name}") + gf_utils.delete_layer_rules(workspace_name, layer_name) -def set_geofence_invalidate_cache(): +def invalidate_geofence_cache(): """invalidate GeoFence Cache Rules""" if settings.OGC_SERVER['default']['GEOFENCE_SECURITY_ENABLED']: try: - gf_client.invalidate_cache() + geofence.invalidate_cache() return True except Exception: tb = traceback.format_exc() @@ -237,7 +216,7 @@ def set_geowebcache_invalidate_cache(dataset_alternate, cat=None): logger.debug(tb) -def set_geofence_all(instance): +def allow_layer_to_all(instance): """assign access permissions to all users This method is only relevant to Dataset instances that have their @@ -249,28 +228,30 @@ def set_geofence_all(instance): """ resource = instance.get_self_resource() - logger.debug(f"Inside set_geofence_all for instance {instance}") + logger.debug(f"Inside allow_layer_to_all for instance {instance}") workspace = get_dataset_workspace(resource.dataset) dataset_name = resource.dataset.name if resource.dataset and hasattr(resource.dataset, 'name') \ else resource.dataset.alternate - logger.debug(f"going to work in workspace {workspace}") - if not getattr(settings, 'DELAYED_SECURITY_SIGNALS', False): - try: - priority = get_highest_priority() + 1 - gf_client.insert_rule(Rule(priority, workspace, dataset_name, Rule.ALLOW)) - except Exception as e: - tb = traceback.format_exc() - logger.debug(tb) - raise RuntimeError(f"Could not ADD GeoServer ANONYMOUS Rule for Dataset {dataset_name}: {e}") - finally: - set_geofence_invalidate_cache() - else: - resource.set_dirty_state() + logger.debug(f"Allowing {workspace}:{dataset_name} access to everybody") + try: + priority = gf_utils.get_first_available_priority() + geofence.insert_rule(Rule(Rule.ALLOW, priority=priority, workspace=workspace, layer=dataset_name)) + except Exception as e: + tb = traceback.format_exc() + logger.debug(tb) + raise RuntimeError(f"Could not ADD GeoServer ANONYMOUS Rule for Dataset {dataset_name}: {e}") + finally: + if not getattr(settings, 'DELAYED_SECURITY_SIGNALS', False): + invalidate_geofence_cache() + else: + resource.set_dirty_state() -def sync_geofence_with_guardian(dataset, perms, user=None, group=None, group_perms=None): +# def sync_geofence_with_guardian(dataset, perms, user=None, group=None, group_perms=None): +def create_geofence_rules(dataset, perms, user=None, group=None, batch: Batch = None): """ - Sync Guardian permissions to GeoFence. + Collect GeoFence rules related to passed perms into a Batch. + If the batch does not exist, it is created and returned. """ layer_name = dataset.name if dataset and hasattr(dataset, 'name') else dataset.alternate workspace_name = get_dataset_workspace(dataset) @@ -279,94 +260,78 @@ def sync_geofence_with_guardian(dataset, perms, user=None, group=None, group_per gf_requests = {} if 'change_dataset_data' not in perms: - _skip_perm = False - if user and group_perms: - if isinstance(user, str): - user = get_user_model().objects.get(username=user) - user_groups = list(user.groups.values_list('name', flat=True)) - for _group, _perm in group_perms.items(): - if 'change_dataset_data' in _perm and _group in user_groups: - _skip_perm = True - break - if not _skip_perm: - gf_requests["WFS"] = { - "TRANSACTION": False, - "LOCKFEATURE": False, - "GETFEATUREWITHLOCK": False - } - _user = None - _group = None - - _group, _user, _disable_cache, users_geolimits, groups_geolimits, anonymous_geolimits = get_user_geolimits(dataset, user, group) - - if _disable_cache: - gf_services_limits_first = {"*": gf_services.pop('*')} - gf_services_limits_first.update(gf_services) - gf_services = gf_services_limits_first - - batch = Batch(f'Sync {workspace_name}:{layer_name}') - priority = get_highest_priority() + 1 - pri = itertools.count(priority) - - def resolve_geolimits(geolimits): - return geolimits.last().wkt if geolimits and geolimits.exists() else None + gf_requests["WFS"] = { + "TRANSACTION": False, + "LOCKFEATURE": False, + "GETFEATUREWITHLOCK": False + } + + username = (user if isinstance(user, str) else user.username) if user else None + groupname = (group if isinstance(group, str) else group.name) if group else None + + users_geolimits, groups_geolimits, anonymous_geolimits = get_geolimits(dataset, username, groupname) + # _disable_cache = has_geolimits(dataset, user, group) + # + # if _disable_cache: + # gf_services_limits_first = {"*": gf_services.pop('*')} + # gf_services_limits_first.update(gf_services) + # gf_services = gf_services_limits_first + + if not batch: + batch = AutoPriorityBatch(gf_utils.get_first_available_priority(), f'Sync {workspace_name}:{layer_name}') # Set global geolimits - wkt = resolve_geolimits(users_geolimits) - if wkt: - logger.debug(f"Adding GeoFence USER GeoLimit rule: U:{_user} L:{dataset} ") - batch.add_insert_rule(Rule(pri.__next__(), workspace_name, layer_name, Rule.LIMIT, catalog_mode=Rule.CM_MIXED, - user=_user, - geo_limit=wkt)) - wkt = resolve_geolimits(anonymous_geolimits) - if wkt: - logger.debug(f"Adding GeoFence ANON GeoLimit rule: L:{dataset} ") - batch.add_insert_rule(Rule(pri.__next__(), workspace_name, layer_name, Rule.LIMIT, catalog_mode=Rule.CM_MIXED, - geo_limit=wkt)) - wkt = resolve_geolimits(groups_geolimits) - if wkt: - logger.debug(f"Adding GeoFence GROUP GeoLimit rule: G:{_group} L:{dataset} ") - batch.add_insert_rule(Rule(pri.__next__(), workspace_name, layer_name, Rule.LIMIT, catalog_mode=Rule.CM_MIXED, - group=_group, - geo_limit=wkt)) + # Anon limits should go at the end, but it's responsibility of the caller to create first user/group rules + for limits, scope, u, g in ( + (users_geolimits, 'USER', username, None), + (anonymous_geolimits, 'ANON', None, None), + (groups_geolimits, 'GROUP', None, groupname), + ): + if limits and limits.exists(): + logger.debug(f"Adding GeoFence {scope} GeoLimit rule: U:{u} G:{g} L:{dataset} ") + wkt = limits.last().wkt + batch.add_insert_rule(Rule(Rule.LIMIT, + workspace=workspace_name, layer=layer_name, + user=u, group=g, + geo_limit=wkt, catalog_mode=Rule.CM_MIXED)) + # Set services rules for service, allowed in gf_services.items(): if dataset and layer_name and allowed: - if _user: - logger.debug(f"Adding GeoFence USER rules: U:{_user} S:{service} L:{dataset} ") + if username: + logger.debug(f"Adding GeoFence USER rules: U:{username} S:{service} L:{dataset} ") if service in gf_requests: for request, enabled in gf_requests[service].items(): - batch.add_insert_rule(Rule(pri.__next__(), workspace_name, layer_name, enabled, - service=service, request=request, user=_user)) - batch.add_insert_rule(Rule(pri.__next__(), workspace_name, layer_name, Rule.ALLOW, - service=service, user=_user)) - - elif not _group: + batch.add_insert_rule(Rule(enabled, user=username, + workspace=workspace_name, layer=layer_name, + service=service, request=request)) + batch.add_insert_rule(Rule(Rule.ALLOW, user=username, + workspace=workspace_name, layer=layer_name, + service=service)) + elif not groupname: logger.debug(f"Adding GeoFence ANON rules: S:{service} L:{dataset} ") if service in gf_requests: for request, enabled in gf_requests[service].items(): - batch.add_insert_rule(Rule(pri.__next__(), workspace_name, layer_name, enabled, + batch.add_insert_rule(Rule(enabled, + workspace=workspace_name, layer=layer_name, service=service, request=request)) - batch.add_insert_rule(Rule(pri.__next__(), workspace_name, layer_name, Rule.ALLOW, - service=service)) - if _group: - logger.debug(f"Adding GeoFence GROUP rules: G:{_group} S:{service} L:{dataset} ") + batch.add_insert_rule(Rule(Rule.ALLOW, + workspace=workspace_name, layer=layer_name, + service=service)) + if groupname: + logger.debug(f"Adding GeoFence GROUP rules: G:{groupname} S:{service} L:{dataset} ") if service in gf_requests: for request, enabled in gf_requests[service].items(): - batch.add_insert_rule(Rule(pri.__next__(), workspace_name, layer_name, enabled, - service=service, request=request, group=_group)) - batch.add_insert_rule(Rule(pri.__next__(), workspace_name, layer_name, Rule.ALLOW, - service=service, group=_group)) - - gf_client.run_batch(batch) - - if not getattr(settings, 'DELAYED_SECURITY_SIGNALS', False): - set_geofence_invalidate_cache() - else: - dataset.set_dirty_state() + batch.add_insert_rule(Rule(enabled, group=groupname, + workspace=workspace_name, layer=layer_name, + service=service, request=request)) + batch.add_insert_rule(Rule(Rule.ALLOW, group=groupname, + workspace=workspace_name, layer=layer_name, + service=service)) + return batch def sync_resources_with_guardian(resource=None): @@ -382,12 +347,18 @@ def sync_resources_with_guardian(resource=None): dirty_resources = ResourceBase.objects.filter(dirty_state=True) if dirty_resources and dirty_resources.exists(): logger.debug(" --------------------------- synching with guardian!") + + rules_committed = False + for r in dirty_resources: if r.polymorphic_ctype.name == 'dataset': layer = None try: - purge_geofence_dataset_rules(r) layer = Dataset.objects.get(id=r.id) + batch = AutoPriorityBatch(gf_utils.get_first_available_priority(), f"Sync resources {r}") + + gf_utils.collect_delete_layer_rules(get_dataset_workspace(layer), layer.name, batch) + perm_spec = layer.get_all_level_info() # All the other users if 'users' in perm_spec: @@ -395,43 +366,60 @@ def sync_resources_with_guardian(resource=None): user = get_user_model().objects.get(username=user) # Set the GeoFence User Rules geofence_user = str(user) - if "AnonymousUser" in geofence_user or get_anonymous_user() in geofence_user: + if "AnonymousUser" in geofence_user or str(get_anonymous_user()) in geofence_user: geofence_user = None - sync_geofence_with_guardian(layer, perms, user=geofence_user) + create_geofence_rules(layer, perms, user=geofence_user, batch=batch) # All the other groups if 'groups' in perm_spec: for group, perms in perm_spec['groups'].items(): group = Group.objects.get(name=group) # Set the GeoFence Group Rules - sync_geofence_with_guardian(layer, perms, group=group) + create_geofence_rules(layer, perms, group=group, batch=batch) + + logger.info(f"Going to synch permissions in GeoFence for resource {resource}") + rules_committed = geofence.run_batch(batch) r.clear_dirty_state() except Exception as e: logger.exception(e) - logger.warn(f"!WARNING! - Failure Synching-up Security Rules for Resource [{r}]") + logger.warning(f"!WARNING! - Failure Synching-up Security Rules for Resource [{r}]") + + if rules_committed: + invalidate_geofence_cache() -def get_user_geolimits(layer, user, group): - _user = None - _group = None - _disable_dataset_cache = None +def get_geolimits(layer, username, groupname): users_geolimits = None groups_geolimits = None anonymous_geolimits = None + + if username: + users_geolimits = layer.users_geolimits.filter(user=get_user_model().objects.get(username=username)) + + if groupname: + if GroupProfile.objects.filter(group__name=groupname).exists(): + groups_geolimits = layer.groups_geolimits.filter(group=GroupProfile.objects.get(group__name=groupname)) + + if not username and not groupname: + anonymous_geolimits = layer.users_geolimits.filter(user=get_anonymous_user()) + + return users_geolimits, groups_geolimits, anonymous_geolimits + + +def has_geolimits(layer, user, group): if user: _user = user if isinstance(user, str) else user.username users_geolimits = layer.users_geolimits.filter(user=get_user_model().objects.get(username=_user)) - _disable_dataset_cache = users_geolimits.exists() + return users_geolimits.exists() if group: _group = group if isinstance(group, str) else group.name if GroupProfile.objects.filter(group__name=_group).count() == 1: groups_geolimits = layer.groups_geolimits.filter(group=GroupProfile.objects.get(group__name=_group)) - _disable_dataset_cache = groups_geolimits.exists() + return groups_geolimits.exists() if not user and not group: anonymous_geolimits = layer.users_geolimits.filter(user=get_anonymous_user()) - _disable_dataset_cache = anonymous_geolimits.exists() - return _group, _user, _disable_dataset_cache, users_geolimits, groups_geolimits, anonymous_geolimits + return anonymous_geolimits.exists() def _get_gf_services(layer, perms): @@ -449,7 +437,7 @@ def _get_gf_services(layer, perms): return gf_services -def _get_gwc_filters_and_formats(disable_cache: list = []) -> typing.Tuple[list, list]: +def _get_gwc_filters_and_formats(disable_cache: bool) -> typing.Tuple[list, list]: filters = [{ "styleParameterFilter": { "STYLES": "" @@ -464,17 +452,7 @@ def _get_gwc_filters_and_formats(disable_cache: list = []) -> typing.Tuple[list, 'image/vnd.jpeg-png', 'image/vnd.jpeg-png8' ] - if disable_cache and any(disable_cache): + if disable_cache: filters = None formats = None - return (filters, formats) - - -def sync_permissions_and_disable_cache(cache_rules, resource, perms, user, group, group_perms): - if group_perms: - sync_geofence_with_guardian(dataset=resource, perms=perms, user=user, group_perms=group_perms) - else: - sync_geofence_with_guardian(dataset=resource, perms=perms, user=user, group=group) - _, _, _disable_dataset_cache, _, _, _ = get_user_geolimits(layer=resource, user=user, group=group) - cache_rules.append(_disable_dataset_cache) - return list(set(cache_rules)) + return filters, formats diff --git a/geonode/local_settings.py.geoserver.sample b/geonode/local_settings.py.geoserver.sample index ce76fd213b4..733589d6d36 100644 --- a/geonode/local_settings.py.geoserver.sample +++ b/geonode/local_settings.py.geoserver.sample @@ -132,6 +132,7 @@ OGC_SERVER = { 'PRINT_NG_ENABLED': True, 'GEONODE_SECURITY_ENABLED': True, 'GEOFENCE_SECURITY_ENABLED': True, + 'GEOFENCE_TIMEOUT': int(os.getenv('GEOFENCE_TIMEOUT', os.getenv('OGC_REQUEST_TIMEOUT', '60'))), 'WMST_ENABLED': False, 'BACKEND_WRITE_ENABLED': True, 'WPS_ENABLED': False, diff --git a/geonode/security/tests.py b/geonode/security/tests.py index b677d0217ca..83e0ad72f57 100644 --- a/geonode/security/tests.py +++ b/geonode/security/tests.py @@ -43,6 +43,7 @@ get_anonymous_user) from geonode import geoserver +from geonode.geoserver.helpers import geofence, gf_utils from geonode.maps.models import Map from geonode.layers.models import Dataset from geonode.documents.models import Document @@ -71,17 +72,16 @@ create_single_dataset) from geonode.geoserver.security import ( _get_gf_services, - get_user_geolimits, - # get_geofence_rules, - # get_geofence_rules_count, - get_highest_priority, - set_geofence_all, - purge_geofence_all, - sync_geofence_with_guardian, + allow_layer_to_all, + delete_all_geofence_rules, sync_resources_with_guardian, _get_gwc_filters_and_formats, + has_geolimits, + create_geofence_rules, + delete_geofence_rules_for_layer, ) + from .utils import ( get_users_with_perms, get_visible_resources, @@ -98,16 +98,6 @@ def _log(msg, *args): logger.debug(msg, *args) -def get_geofence_rules_count(): - from geonode.geoserver.helpers import gf_client - return gf_client.get_rules_count() - - -def get_geofence_rules(): - from geonode.geoserver.helpers import gf_client - return gf_client.get_rules() - - class StreamToLogger: """ Fake file-like stream object that redirects writes to a logger instance. @@ -381,12 +371,13 @@ def test_invalidate_tileddataset_cache(self): def test_set_bulk_permissions(self): """Test that after restrict view permissions on two layers bobby is unable to see them""" - geofence_rules_count = 0 + + rules_count = 0 if check_ogc_backend(geoserver.BACKEND_PACKAGE): - purge_geofence_all() + delete_all_geofence_rules() # Reset GeoFence Rules - geofence_rules_count = get_geofence_rules_count() - self.assertEqual(geofence_rules_count, 0) + rules_count = geofence.get_rules_count() + self.assertEqual(rules_count, 0) layers = Dataset.objects.all()[:2].values_list('id', flat=True) layers_id = [str(x) for x in layers] @@ -404,13 +395,13 @@ def test_set_bulk_permissions(self): if check_ogc_backend(geoserver.BACKEND_PACKAGE): # Check GeoFence Rules have been correctly created - geofence_rules_count = get_geofence_rules_count() - _log(f"1. geofence_rules_count: {geofence_rules_count} ") - self.assertGreaterEqual(geofence_rules_count, 10) - set_geofence_all(test_perm_dataset) - geofence_rules_count = get_geofence_rules_count() - _log(f"2. geofence_rules_count: {geofence_rules_count} ") - self.assertGreaterEqual(geofence_rules_count, 11) + rules_count = geofence.get_rules_count() + _log(f"1. rules_count: {rules_count} ") + self.assertGreaterEqual(rules_count, 10) + allow_layer_to_all(test_perm_dataset) + rules_count = geofence.get_rules_count() + _log(f"2. rules_count: {rules_count} ") + self.assertGreaterEqual(rules_count, 11) self.client.logout() @@ -419,34 +410,38 @@ def test_set_bulk_permissions(self): resp = self.client.get(self.list_url) self.assertGreaterEqual(len(self.deserialize(resp)['objects']), 6) - perms = get_users_with_perms(test_perm_dataset) - _log(f"3. perms: {perms} ") - sync_geofence_with_guardian(test_perm_dataset, perms, user='bobby') + # perms = get_users_with_perms(test_perm_dataset) + # _log(f"3. perms: {perms} ") + # batch = AutoPriorityBatch(get_first_available_priority(), f'test batch for {test_perm_dataset}') + # for u, p in perms.items(): + # create_geofence_rules(test_perm_dataset, p, user=u, batch=batch) + # geofence.run_batch(batch) # Check GeoFence Rules have been correctly created - geofence_rules_count = get_geofence_rules_count() - _log(f"4. geofence_rules_count: {geofence_rules_count} ") - self.assertGreaterEqual(geofence_rules_count, 13) + rules_count = geofence.get_rules_count() + _log(f"4. rules_count: {rules_count} ") + self.assertGreaterEqual(rules_count, 13) # Validate maximum priority - geofence_rules_highest_priority = get_highest_priority() - _log(f"5. geofence_rules_highest_priority: {geofence_rules_highest_priority} ") - self.assertTrue(geofence_rules_highest_priority > 0) + available_priority = gf_utils.get_first_available_priority() + _log(f"5. available_priority: {available_priority} ") + self.assertTrue(available_priority > 0) url = settings.OGC_SERVER['default']['LOCATION'] user = settings.OGC_SERVER['default']['USER'] passwd = settings.OGC_SERVER['default']['PASSWORD'] - r = requests.get(f"{url}gwc/rest/seed/{test_perm_dataset.alternate}.json", + test_url = f"{url}gwc/rest/seed/{test_perm_dataset.alternate}.json" + r = requests.get(test_url, auth=HTTPBasicAuth(user, passwd)) - self.assertEqual(r.status_code, 400) + self.assertEqual(r.status_code, 400, f"GWC error for user: {user} URL: {test_url}\n{r.text}") - geofence_rules_count = 0 + rules_count = 0 if check_ogc_backend(geoserver.BACKEND_PACKAGE): - purge_geofence_all() + delete_all_geofence_rules() # Reset GeoFence Rules - geofence_rules_count = get_geofence_rules_count() - self.assertEqual(geofence_rules_count, 0) + rules_count = geofence.get_rules_count() + self.assertEqual(rules_count, 0) def test_bobby_cannot_set_all(self): """Test that Bobby can set the permissions only on the ones @@ -516,27 +511,27 @@ def test_perm_specs_synchronization(self): self.client.login(username='admin', password='admin') # Reset GeoFence Rules - purge_geofence_all() - self.assertEqual(get_geofence_rules_count(), 0) + delete_all_geofence_rules() + self.assertEqual(geofence.get_rules_count(), 0) perm_spec = {'users': {'AnonymousUser': []}, 'groups': []} layer.set_permissions(perm_spec) - geofence_rules_count = get_geofence_rules_count() - _log(f"1. geofence_rules_count: {geofence_rules_count} ") - self.assertEqual(geofence_rules_count, 5) + rules_count = geofence.get_rules_count() + _log(f"1. rules_count: {rules_count} ") + self.assertEqual(rules_count, 5) perm_spec = { "users": {"admin": ["view_resourcebase"]}, "groups": []} layer.set_permissions(perm_spec) - geofence_rules_count = get_geofence_rules_count() - _log(f"2. geofence_rules_count: {geofence_rules_count} ") - self.assertEqual(geofence_rules_count, 7) + rules_count = geofence.get_rules_count() + _log(f"2. rules_count: {rules_count} ") + self.assertEqual(rules_count, 7, f"Bad rules count. Got rules: {geofence.get_rules()}") perm_spec = {'users': {"admin": ['change_dataset_data']}, 'groups': []} layer.set_permissions(perm_spec) - geofence_rules_count = get_geofence_rules_count() - _log(f"3. geofence_rules_count: {geofence_rules_count} ") - self.assertEqual(geofence_rules_count, 7) + rules_count = geofence.get_rules_count() + _log(f"3. rules_count: {rules_count} ") + self.assertEqual(rules_count, 7) # FULL WFS-T perm_spec = { @@ -551,10 +546,10 @@ def test_perm_specs_synchronization(self): 'groups': [] } layer.set_permissions(perm_spec) - geofence_rules_count = get_geofence_rules_count() - self.assertEqual(geofence_rules_count, 10) + rules_count = geofence.get_rules_count() + self.assertEqual(rules_count, 10) - rules_objs = get_geofence_rules() + rules_objs = geofence.get_rules() _deny_wfst_rule_exists = False for rule in rules_objs['rules']: if rule['service'] == "WFS" and \ @@ -576,10 +571,10 @@ def test_perm_specs_synchronization(self): 'groups': [] } layer.set_permissions(perm_spec) - geofence_rules_count = get_geofence_rules_count() - self.assertEqual(geofence_rules_count, 13) + rules_count = geofence.get_rules_count() + self.assertEqual(rules_count, 13) - rules_objs = get_geofence_rules() + rules_objs = geofence.get_rules() _deny_wfst_rule_exists = False _deny_wfst_rule_position = -1 _allow_wfs_rule_position = -1 @@ -606,10 +601,10 @@ def test_perm_specs_synchronization(self): 'groups': [] } layer.set_permissions(perm_spec) - geofence_rules_count = get_geofence_rules_count() - self.assertEqual(geofence_rules_count, 7) + rules_count = geofence.get_rules_count() + self.assertEqual(rules_count, 7) - rules_objs = get_geofence_rules() + rules_objs = geofence.get_rules() _deny_wfst_rule_exists = False for rule in rules_objs['rules']: if rule['service'] == "WFS" and \ @@ -621,26 +616,26 @@ def test_perm_specs_synchronization(self): perm_spec = {'users': {}, 'groups': {'bar': ['view_resourcebase']}} layer.set_permissions(perm_spec) - geofence_rules_count = get_geofence_rules_count() - _log(f"4. geofence_rules_count: {geofence_rules_count} ") - self.assertEqual(geofence_rules_count, 7) + rules_count = geofence.get_rules_count() + _log(f"4. rules_count: {rules_count} ") + self.assertEqual(rules_count, 7, f'Bad rule count, got rules {geofence.get_rules()}') perm_spec = {'users': {}, 'groups': {'bar': ['change_resourcebase']}} layer.set_permissions(perm_spec) - geofence_rules_count = get_geofence_rules_count() - _log(f"5. geofence_rules_count: {geofence_rules_count} ") - self.assertEqual(geofence_rules_count, 5) + rules_count = geofence.get_rules_count() + _log(f"5. rules_count: {rules_count} ") + self.assertEqual(rules_count, 5) # Testing GeoLimits # Reset GeoFence Rules - purge_geofence_all() - geofence_rules_count = get_geofence_rules_count() - self.assertEqual(geofence_rules_count, 0) + delete_all_geofence_rules() + rules_count = geofence.get_rules_count() + self.assertEqual(rules_count, 0) layer = Dataset.objects.first() # grab bobby bobby = get_user_model().objects.get(username="bobby") - _, _, _disable_dataset_cache, _, _, _ = get_user_geolimits(layer, None, None) - filters, formats = _get_gwc_filters_and_formats([_disable_dataset_cache]) + _disable_dataset_cache = has_geolimits(layer, None, None) + filters, formats = _get_gwc_filters_and_formats(_disable_dataset_cache) self.assertListEqual(filters, [{ "styleParameterFilter": { "STYLES": "" @@ -666,7 +661,7 @@ def test_perm_specs_synchronization(self): geo_limit.save() layer.users_geolimits.add(geo_limit) self.assertEqual(layer.users_geolimits.all().count(), 1) - _, _, _disable_dataset_cache, _, _, _ = get_user_geolimits(layer, bobby, None) + _disable_dataset_cache = has_geolimits(layer, bobby, None) filters, formats = _get_gwc_filters_and_formats([_disable_dataset_cache]) self.assertIsNone(filters) self.assertIsNone(formats) @@ -674,10 +669,10 @@ def test_perm_specs_synchronization(self): perm_spec = { "users": {"bobby": ["view_resourcebase"]}, "groups": []} layer.set_permissions(perm_spec) - geofence_rules_count = get_geofence_rules_count() - self.assertEqual(geofence_rules_count, 8) + rules_count = geofence.get_rules_count() + self.assertEqual(rules_count, 8) - rules_objs = get_geofence_rules() + rules_objs = geofence.get_rules() self.assertEqual(len(rules_objs['rules']), 8) # Order is important _limit_rule_position = -1 @@ -713,10 +708,10 @@ def test_perm_specs_synchronization(self): perm_spec = { 'users': {}, 'groups': {'bar': ['change_resourcebase']}} layer.set_permissions(perm_spec) - geofence_rules_count = get_geofence_rules_count() - self.assertEqual(geofence_rules_count, 6) + rules_count = geofence.get_rules_count() + self.assertEqual(rules_count, 6) - rules_objs = get_geofence_rules() + rules_objs = geofence.get_rules() self.assertEqual(len(rules_objs['rules']), 6) # Order is important _limit_rule_position = -1 @@ -747,9 +742,9 @@ def test_perm_specs_synchronization(self): layer.save() layer.set_permissions(perm_spec) - geofence_rules_count = get_geofence_rules_count() + rules_count = geofence.get_rules_count() - rules_objs = get_geofence_rules() + rules_objs = geofence.get_rules() # Order is important _limit_rule_position = -1 for cnt, rule in enumerate(rules_objs['rules']): @@ -778,9 +773,9 @@ def test_perm_specs_synchronization(self): layer.save() # Reset GeoFence Rules - purge_geofence_all() - geofence_rules_count = get_geofence_rules_count() - self.assertEqual(geofence_rules_count, 0) + delete_all_geofence_rules() + rules_count = geofence.get_rules_count() + self.assertEqual(rules_count, 0) @on_ogc_backend(geoserver.BACKEND_PACKAGE) def test_dataset_upload_with_time(self): @@ -1002,9 +997,9 @@ def test_dataset_permissions(self): self.assertEqual(layer.alternate, 'geonode:san_andres_y_providencia_poi') # Reset GeoFence Rules - purge_geofence_all() - geofence_rules_count = get_geofence_rules_count() - self.assertEqual(geofence_rules_count, 0) + delete_all_geofence_rules() + rules_count = geofence.get_rules_count() + self.assertEqual(rules_count, 0) layer = Dataset.objects.get(name='san_andres_y_providencia_poi') # removing duplicates @@ -1013,9 +1008,9 @@ def test_dataset_permissions(self): layer = Dataset.objects.get(alternate=layer.alternate) layer.set_default_permissions(owner=bobby) check_dataset(layer) - geofence_rules_count = get_geofence_rules_count() - _log(f"0. geofence_rules_count: {geofence_rules_count} ") - self.assertGreaterEqual(geofence_rules_count, 4) + rules_count = geofence.get_rules_count() + _log(f"0. rules_count: {rules_count} ") + self.assertGreaterEqual(rules_count, 4) # Set the layer private for not authenticated users perm_spec = {'users': {'AnonymousUser': []}, 'groups': []} @@ -1109,9 +1104,9 @@ def test_dataset_permissions(self): # self.assertEqual(_content_type, 'image/png') # Reset GeoFence Rules - purge_geofence_all() - geofence_rules_count = get_geofence_rules_count() - self.assertTrue(geofence_rules_count == 0) + delete_all_geofence_rules() + rules_count = geofence.get_rules_count() + self.assertTrue(rules_count == 0) def test_maplayers_default_permissions(self): """Verify that Dataset.set_default_permissions is behaving as expected @@ -1337,12 +1332,12 @@ def test_perms_info(self): def test_not_superuser_permissions(self): - geofence_rules_count = 0 + rules_count = 0 if check_ogc_backend(geoserver.BACKEND_PACKAGE): - purge_geofence_all() + delete_all_geofence_rules() # Reset GeoFence Rules - geofence_rules_count = get_geofence_rules_count() - self.assertTrue(geofence_rules_count == 0) + rules_count = geofence.get_rules_count() + self.assertTrue(rules_count == 0) # grab bobby bob = get_user_model().objects.get(username='bobby') @@ -1362,8 +1357,8 @@ def test_not_superuser_permissions(self): if check_ogc_backend(geoserver.BACKEND_PACKAGE): # Check GeoFence Rules have been correctly created - geofence_rules_count = get_geofence_rules_count() - _log(f"1. geofence_rules_count: {geofence_rules_count} ") + rules_count = geofence.get_rules_count() + _log(f"1. rules_count: {rules_count} ") self.assertTrue(self.client.login(username='bobby', password='bob')) @@ -1406,7 +1401,9 @@ def test_not_superuser_permissions(self): self.assertTrue(response.status_code in (401, 403), response.status_code) # 3.2 has delete_resourcebase: verify that bobby can access the layer # delete page - layer.set_permissions({'users': {'bobby': ['change_resourcebase', 'change_resourcebase_metadata', 'delete_resourcebase']}, 'groups': []}) + layer.set_permissions({ + 'users': {'bobby': ['change_resourcebase', 'change_resourcebase_metadata', 'delete_resourcebase']}, + 'groups': []}) self.assertTrue( bob.has_perm( 'change_resourcebase_metadata', @@ -1417,11 +1414,12 @@ def test_not_superuser_permissions(self): if check_ogc_backend(geoserver.BACKEND_PACKAGE): perms = get_users_with_perms(layer) _log(f"2. perms: {perms} ") - sync_geofence_with_guardian(layer, perms, user=bob, group=anonymous_group) + batch = create_geofence_rules(layer, perms, user=bob, group=anonymous_group) + geofence.run_batch(batch) # Check GeoFence Rules have been correctly created - geofence_rules_count = get_geofence_rules_count() - _log(f"3. geofence_rules_count: {geofence_rules_count} ") + rules_count = geofence.get_rules_count() + _log(f"3. rules_count: {rules_count} ") # 4. change_resourcebase_permissions # should be impossible for the user without change_resourcebase_permissions @@ -1442,7 +1440,10 @@ def test_not_superuser_permissions(self): # change layer style page if check_ogc_backend(geoserver.BACKEND_PACKAGE): # Only for geoserver backend - layer.set_permissions({'users': {'bobby': ['change_resourcebase', 'change_resourcebase_metadata', 'delete_resourcebase', 'change_dataset_style']}, 'groups': []}) + layer.set_permissions({ + 'users': + {'bobby': ['change_resourcebase', 'change_resourcebase_metadata', 'delete_resourcebase', 'change_dataset_style']}, + 'groups': []}) self.assertTrue( bob.has_perm( 'change_dataset_style', @@ -1450,12 +1451,12 @@ def test_not_superuser_permissions(self): response = self.client.get(reverse('dataset_style_manage', args=(layer.alternate,))) self.assertEqual(response.status_code, 200, response.status_code) - geofence_rules_count = 0 + rules_count = 0 if check_ogc_backend(geoserver.BACKEND_PACKAGE): - purge_geofence_all() + delete_all_geofence_rules() # Reset GeoFence Rules - geofence_rules_count = get_geofence_rules_count() - self.assertEqual(geofence_rules_count, 0, geofence_rules_count) + rules_count = geofence.get_rules_count() + self.assertEqual(rules_count, 0, rules_count) def test_anonymus_permissions(self): # grab a layer @@ -2154,7 +2155,7 @@ def setUp(self): def test_sync_resources_with_guardian_delay_false(self): with self.settings(DELAYED_SECURITY_SIGNALS=False): # Set geofence (and so the dirty state) - set_geofence_all(self._l) + allow_layer_to_all(self._l) # Retrieve the same layer dirty_dataset = Dataset.objects.get(pk=self._l.id) # Check dirty state (True) @@ -2167,9 +2168,10 @@ def test_sync_resources_with_guardian_delay_false(self): # TODO: DELAYED SECURITY MUST BE REVISED def test_sync_resources_with_guardian_delay_true(self): + delete_geofence_rules_for_layer(self._l) with self.settings(DELAYED_SECURITY_SIGNALS=True): # Set geofence (and so the dirty state) - set_geofence_all(self._l) + allow_layer_to_all(self._l) # Retrieve the same layer dirty_dataset = Dataset.objects.get(pk=self._l.id) # Check dirty state (True) @@ -2192,7 +2194,7 @@ def setUp(self): self.gf_services = _get_gf_services(self.layer, self.perms) def test_should_not_disable_cache_for_user_without_geolimits(self): - _, _, _disable_dataset_cache, _, _, _ = get_user_geolimits(self.layer, self.owner, None) + _disable_dataset_cache = has_geolimits(self.layer, self.owner, None) self.assertFalse(_disable_dataset_cache) def test_should_disable_cache_for_user_with_geolimits(self): @@ -2202,11 +2204,11 @@ def test_should_disable_cache_for_user_with_geolimits(self): ) self.layer.users_geolimits.set([geo_limit]) self.layer.refresh_from_db() - _, _, _disable_dataset_cache, _, _, _ = get_user_geolimits(self.layer, self.owner, None) + _disable_dataset_cache = has_geolimits(self.layer, self.owner, None) self.assertTrue(_disable_dataset_cache) def test_should_not_disable_cache_for_anonymous_without_geolimits(self): - _, _, _disable_dataset_cache, _, _, _ = get_user_geolimits(self.layer, None, None) + _disable_dataset_cache = has_geolimits(self.layer, None, None) self.assertFalse(_disable_dataset_cache) def test_should_disable_cache_for_anonymous_with_geolimits(self): @@ -2216,7 +2218,7 @@ def test_should_disable_cache_for_anonymous_with_geolimits(self): ) self.layer.users_geolimits.set([geo_limit]) self.layer.refresh_from_db() - _, _, _disable_dataset_cache, _, _, _ = get_user_geolimits(self.layer, None, None) + _disable_dataset_cache = has_geolimits(self.layer, None, None) self.assertTrue(_disable_dataset_cache) diff --git a/geonode/settings.py b/geonode/settings.py index 92816eeaa32..40a6bd56787 100644 --- a/geonode/settings.py +++ b/geonode/settings.py @@ -1028,6 +1028,7 @@ 'GEONODE_SECURITY_ENABLED': ast.literal_eval(os.getenv('GEONODE_SECURITY_ENABLED', 'True')), 'GEOFENCE_SECURITY_ENABLED': GEOFENCE_SECURITY_ENABLED, 'GEOFENCE_URL': os.getenv('GEOFENCE_URL', 'internal:/'), + 'GEOFENCE_TIMEOUT': int(os.getenv('GEOFENCE_TIMEOUT', os.getenv('OGC_REQUEST_TIMEOUT', '60'))), 'WMST_ENABLED': ast.literal_eval(os.getenv('WMST_ENABLED', 'False')), 'BACKEND_WRITE_ENABLED': ast.literal_eval(os.getenv('BACKEND_WRITE_ENABLED', 'True')), 'WPS_ENABLED': ast.literal_eval(os.getenv('WPS_ENABLED', 'False')), diff --git a/geonode/upload/tests/integration.py b/geonode/upload/tests/integration.py index c454a723ace..c22c1d0bf1f 100644 --- a/geonode/upload/tests/integration.py +++ b/geonode/upload/tests/integration.py @@ -164,8 +164,8 @@ def tearDown(self): # Cleanup if settings.OGC_SERVER['default'].get( "GEOFENCE_SECURITY_ENABLED", False): - from geonode.geoserver.security import purge_geofence_all - purge_geofence_all() + from geonode.geoserver.security import delete_all_geofence_rules + delete_all_geofence_rules() def check_dataset_geonode_page(self, path): """ Check that the final dataset page render's correctly after diff --git a/geonode/utils.py b/geonode/utils.py index 1316afd35da..e73ab1cfd97 100755 --- a/geonode/utils.py +++ b/geonode/utils.py @@ -346,8 +346,6 @@ def get_dataset_name(dataset): def get_dataset_workspace(dataset): """Get the workspace where the input layer belongs""" - alternate = None - workspace = None try: alternate = dataset.alternate except Exception: