Ref #575 - Add new configuration settings for set validation

irrdnet · Dec 18, 2021 · 62ce979 · 62ce979
1 parent 6db7d15
commit 62ce979
Show file tree

Hide file tree

Showing 8 changed files with 193 additions and 115 deletions.
diff --git a/docs/releases/4.3.0.rst b/docs/releases/4.3.0.rst
@@ -0,0 +1,5 @@
+============================
+Release notes for IRRd 4.3.0
+============================
+
+* removal of compatibility.permit_non_hierarchical_as_set_name
diff --git a/irrd/conf/__init__.py b/irrd/conf/__init__.py
@@ -19,88 +19,7 @@
 PASSWORD_HASH_DUMMY_VALUE = 'DummyValue'
 SOURCE_NAME_RE = re.compile('^[A-Z][A-Z0-9-]*[A-Z0-9]$')
 RPKI_IRR_PSEUDO_SOURCE = 'RPKI'
-
-# Note that sources are checked separately,
-# and 'access_lists' is always permitted
-KNOWN_CONFIG_KEYS = DottedDict({
-    'database_url': {},
-    'database_readonly': {},
-    'redis_url': {},
-    'piddir': {},
-    'user': {},
-    'group': {},
-    'server': {
-        'http': {
-            'interface': {},
-            'port': {},
-            'status_access_list': {},
-            'workers': {},
-            'forwarded_allowed_ips': {},
-        },
-        'whois': {
-            'interface': {},
-            'port': {},
-            'access_list': {},
-            'max_connections': {},
-        },
-    },
-    'email': {
-        'from': {},
-        'footer': {},
-        'smtp': {},
-        'recipient_override': {},
-        'notification_header': {},
-    },
-    'auth': {
-        'override_password': {},
-        'authenticate_related_mntners': {},
-        'gnupg_keyring': {},
-    },
-    'rpki': {
-        'roa_source': {},
-        'roa_import_timer': {},
-        'slurm_source': {},
-        'pseudo_irr_remarks': {},
-        'notify_invalid_enabled': {},
-        'notify_invalid_subject': {},
-        'notify_invalid_header': {},
-    },
-    'scopefilter': {
-        'prefixes': {},
-        'asns': {},
-    },
-    'log': {
-        'logfile_path': {},
-        'level': {},
-        'logging_config_path': {},
-    },
-    'sources_default': {},
-    'compatibility': {
-        'inetnum_search_disabled': {},
-        'irrd42_migration_in_progress': {},
-        'permit_non_hierarchical_as_set_name': {},
-        'ipv4_only_route_set_members': {},
-    }
-})
-
-KNOWN_SOURCES_KEYS = {
-    'authoritative',
-    'keep_journal',
-    'nrtm_host',
-    'nrtm_port',
-    'import_source',
-    'import_serial_source',
-    'import_timer',
-    'object_class_filter',
-    'export_destination',
-    'export_destination_unfiltered',
-    'export_timer',
-    'nrtm_access_list',
-    'nrtm_access_list_unfiltered',
-    'strict_import_keycert_objects',
-    'rpki_excluded',
-    'scopefilter_excluded',
-}
+VALID_SET_AUTNUM_AUTHENTICATION = ['disabled', 'opportunistic', 'required']
 
 LOGGING = {
     'version': 1,
@@ -162,6 +81,9 @@ def __init__(self, user_config_path: Optional[str]=None, commit=True):
         Load the default config and load and check the user provided config.
         If a logfile was specified, direct logs there.
         """
+        from .known_keys import KNOWN_CONFIG_KEYS, KNOWN_SOURCES_KEYS
+        self.known_config_keys = KNOWN_CONFIG_KEYS
+        self.known_sources_keys = KNOWN_SOURCES_KEYS
         self.user_config_path = user_config_path if user_config_path else CONFIG_PATH_DEFAULT
         default_config_path = str(Path(__file__).resolve().parents[0] / 'default_config.yaml')
         default_config_yaml = yaml.safe_load(open(default_config_path))
@@ -211,10 +133,10 @@ def get_setting_live(self, setting_name: str, default: Any=None) -> Any:
         """
         if setting_name.startswith('sources'):
             components = setting_name.split('.')
-            if len(components) == 3 and components[2] not in KNOWN_SOURCES_KEYS:
+            if len(components) == 3 and components[2] not in self.known_sources_keys:
                 raise ValueError(f'Unknown setting {setting_name}')
         elif not setting_name.startswith('access_lists'):
-            if KNOWN_CONFIG_KEYS.get(setting_name) is None:
+            if self.known_config_keys.get(setting_name) is None:
                 raise ValueError(f'Unknown setting {setting_name}')
 
         env_key = 'IRRD_' + setting_name.upper().replace('.', '_')
@@ -289,18 +211,24 @@ def _check_staging_config(self) -> List[str]:
         errors = []
         config = self.user_config_staging
 
-        for key, value in config.items():
-            if key in ['sources', 'access_lists']:
-                continue
-            known = KNOWN_CONFIG_KEYS.get(key)
-            if known is None:
-                errors.append(f'Unknown setting key: {key}')
+        def _validate_subconfig(key, value):
             if hasattr(value, 'items'):
                 for key2, value2 in value.items():
                     subkey = key + '.' + key2
-                    known_sub = KNOWN_CONFIG_KEYS.get(subkey)
+                    known_sub = self.known_config_keys.get(subkey)
+
                     if known_sub is None:
                         errors.append(f'Unknown setting key: {subkey}')
+                    _validate_subconfig(subkey, value2)
+
+        for key, value in config.items():
+            if key in ['sources', 'access_lists']:
+                continue
+            known = self.known_config_keys.get(key)
+
+            if known is None:
+                errors.append(f'Unknown setting key: {key}')
+            _validate_subconfig(key, value)
 
         if not self._check_is_str(config, 'database_url'):
             errors.append('Setting database_url is required.')
@@ -337,6 +265,12 @@ def _check_staging_config(self) -> List[str]:
         if not self._check_is_str(config, 'auth.gnupg_keyring'):
             errors.append('Setting auth.gnupg_keyring is required.')
 
+        for set_name, params in config.get('auth.set_creation', {}).items():
+            if not isinstance(params.get('prefix_required', False), bool):
+                errors.append(f'Setting auth.set_creation.{set_name}.prefix_required must be a bool')
+            if params.get('autnum_authentication') and params['autnum_authentication'].lower() not in VALID_SET_AUTNUM_AUTHENTICATION:
+                errors.append(f'Setting auth.set_creation.{set_name}.autnum_authentication must be one of {VALID_SET_AUTNUM_AUTHENTICATION} if set')
+
         for name, access_list in config.get('access_lists', {}).items():
             for item in access_list:
                 try:
@@ -365,7 +299,7 @@ def _check_staging_config(self) -> List[str]:
 
         has_authoritative_sources = False
         for name, details in config.get('sources', {}).items():
-            unknown_keys = set(details.keys()) - KNOWN_SOURCES_KEYS
+            unknown_keys = set(details.keys()) - self.known_sources_keys
             if unknown_keys:
                 errors.append(f'Unknown key(s) under source {name}: {", ".join(unknown_keys)}')
             if details.get('authoritative'):

diff --git a/irrd/conf/known_keys.py b/irrd/conf/known_keys.py
@@ -0,0 +1,91 @@
+from irrd.vendor.dotted.collection import DottedDict
+from irrd.rpsl.rpsl_objects import OBJECT_CLASS_MAPPING, RPSLSet
+
+# Note that sources are checked separately,
+# and 'access_lists' is always permitted
+KNOWN_CONFIG_KEYS = DottedDict({
+    'database_url': {},
+    'database_readonly': {},
+    'redis_url': {},
+    'piddir': {},
+    'user': {},
+    'group': {},
+    'server': {
+        'http': {
+            'interface': {},
+            'port': {},
+            'status_access_list': {},
+            'workers': {},
+            'forwarded_allowed_ips': {},
+        },
+        'whois': {
+            'interface': {},
+            'port': {},
+            'access_list': {},
+            'max_connections': {},
+        },
+    },
+    'email': {
+        'from': {},
+        'footer': {},
+        'smtp': {},
+        'recipient_override': {},
+        'notification_header': {},
+    },
+    'auth': {
+        'override_password': {},
+        'authenticate_related_mntners': {},
+        'gnupg_keyring': {},
+        'set_creation': {
+            rpsl_object_class: {'prefix_required': {}, 'autnum_authentication': {}}
+            for rpsl_object_class in [
+                set_object.rpsl_object_class
+                for set_object in OBJECT_CLASS_MAPPING.values()
+                if issubclass(set_object, RPSLSet)
+            ] + ['DEFAULT']
+        },
+    },
+    'rpki': {
+        'roa_source': {},
+        'roa_import_timer': {},
+        'slurm_source': {},
+        'pseudo_irr_remarks': {},
+        'notify_invalid_enabled': {},
+        'notify_invalid_subject': {},
+        'notify_invalid_header': {},
+    },
+    'scopefilter': {
+        'prefixes': {},
+        'asns': {},
+    },
+    'log': {
+        'logfile_path': {},
+        'level': {},
+        'logging_config_path': {},
+    },
+    'sources_default': {},
+    'compatibility': {
+        'inetnum_search_disabled': {},
+        'irrd42_migration_in_progress': {},
+        'ipv4_only_route_set_members': {},
+    }
+})
+
+KNOWN_SOURCES_KEYS = {
+    'authoritative',
+    'keep_journal',
+    'nrtm_host',
+    'nrtm_port',
+    'import_source',
+    'import_serial_source',
+    'import_timer',
+    'object_class_filter',
+    'export_destination',
+    'export_destination_unfiltered',
+    'export_timer',
+    'nrtm_access_list',
+    'nrtm_access_list_unfiltered',
+    'strict_import_keycert_objects',
+    'rpki_excluded',
+    'scopefilter_excluded',
+}
diff --git a/irrd/conf/test_conf.py b/irrd/conf/test_conf.py
@@ -73,7 +73,18 @@ def test_load_valid_reload_valid_config(self, monkeypatch, save_yaml_config, tmp
                     }
                 },
                 'auth': {
-                    'gnupg_keyring': str(tmpdir)
+                    'gnupg_keyring': str(tmpdir),
+                    'authenticate_related_mntners': True,
+                    'set_creation': {
+                        'as-set': {
+                            'prefix_required': True,
+                            'autnum_authentication': 'opportunistic',
+                        },
+                        'DEFAULT': {
+                            'prefix_required': True,
+                            'autnum_authentication': 'required',
+                        },
+                    },
                 },
                 'sources_default': ['TESTDB2', 'TESTDB'],
                 'sources': {
@@ -188,7 +199,7 @@ def test_load_valid_reload_invalid_config(self, save_yaml_config, tmpdir, caplog
                     }
                 },
                 'auth': {
-                    'gnupg_keyring': str(tmpdir)
+                    'gnupg_keyring': str(tmpdir),
                 },
                 'rpki': {
                     'roa_source': 'https://example.com/roa.json',
@@ -241,6 +252,17 @@ def test_load_invalid_config(self, save_yaml_config, tmpdir):
                         '192.0.2.2.1'
                     },
                 },
+                'auth': {
+                    'set_creation': {
+                        'as-set': {
+                            'prefix_required': 'not-a-bool',
+                            'autnum_authentication': 'unknown-value',
+                        },
+                        'not-a-real-set': {
+                            'prefix_required': True,
+                        },
+                    },
+                },
                 'rpki': {
                     'roa_source': 'https://example.com/roa.json',
                     'roa_import_timer': 'foo',
@@ -297,6 +319,9 @@ def test_load_invalid_config(self, save_yaml_config, tmpdir):
         assert 'Setting email.recipient_override must be an email address if set.' in str(ce.value)
         assert 'Settings user and group must both be defined, or neither.' in str(ce.value)
         assert 'Setting auth.gnupg_keyring is required.' in str(ce.value)
+        assert 'Unknown setting key: auth.set_creation.not-a-real-set.prefix_required' in str(ce.value)
+        assert 'Setting auth.set_creation.as-set.prefix_required must be a bool' in str(ce.value)
+        assert 'Setting auth.set_creation.as-set.autnum_authentication must be one of' in str(ce.value)
         assert 'Access lists doesnotexist, invalid-list referenced in settings, but not defined.' in str(ce.value)
         assert 'Setting server.http.status_access_list must be a string, if defined.' in str(ce.value)
         assert 'Invalid item in access list bad-list: IPv4 Address with more than 4 bytes.' in str(ce.value)

diff --git a/irrd/integration_tests/run.py b/irrd/integration_tests/run.py
@@ -765,6 +765,21 @@ def _start_irrds(self):
                 'auth': {
                     'gnupg_keyring': None,
                     'override_password': '$1$J6KycItM$MbPaBU6iFSGFV299Rk7Di0',
+                    'set_creation': {
+                        'filter-set': {
+                            'prefix_required': False,
+                        },
+                        'peering-set': {
+                            'prefix_required': False,
+                        },
+                        'route-set': {
+                            'prefix_required': False,
+                        },
+                        'rtr-set': {
+                            'prefix_required': False,
+                        },
+                    },
+
                 },
 
                 'email': {