diff --git a/docs/releases/4.3.0.rst b/docs/releases/4.3.0.rst new file mode 100644 index 000000000..b3dad79d5 --- /dev/null +++ b/docs/releases/4.3.0.rst @@ -0,0 +1,5 @@ +============================ +Release notes for IRRd 4.3.0 +============================ + +* removal of compatibility.permit_non_hierarchical_as_set_name diff --git a/irrd/conf/__init__.py b/irrd/conf/__init__.py index aa1657c47..6721aea09 100644 --- a/irrd/conf/__init__.py +++ b/irrd/conf/__init__.py @@ -19,88 +19,7 @@ PASSWORD_HASH_DUMMY_VALUE = 'DummyValue' SOURCE_NAME_RE = re.compile('^[A-Z][A-Z0-9-]*[A-Z0-9]$') RPKI_IRR_PSEUDO_SOURCE = 'RPKI' - -# Note that sources are checked separately, -# and 'access_lists' is always permitted -KNOWN_CONFIG_KEYS = DottedDict({ - 'database_url': {}, - 'database_readonly': {}, - 'redis_url': {}, - 'piddir': {}, - 'user': {}, - 'group': {}, - 'server': { - 'http': { - 'interface': {}, - 'port': {}, - 'status_access_list': {}, - 'workers': {}, - 'forwarded_allowed_ips': {}, - }, - 'whois': { - 'interface': {}, - 'port': {}, - 'access_list': {}, - 'max_connections': {}, - }, - }, - 'email': { - 'from': {}, - 'footer': {}, - 'smtp': {}, - 'recipient_override': {}, - 'notification_header': {}, - }, - 'auth': { - 'override_password': {}, - 'authenticate_related_mntners': {}, - 'gnupg_keyring': {}, - }, - 'rpki': { - 'roa_source': {}, - 'roa_import_timer': {}, - 'slurm_source': {}, - 'pseudo_irr_remarks': {}, - 'notify_invalid_enabled': {}, - 'notify_invalid_subject': {}, - 'notify_invalid_header': {}, - }, - 'scopefilter': { - 'prefixes': {}, - 'asns': {}, - }, - 'log': { - 'logfile_path': {}, - 'level': {}, - 'logging_config_path': {}, - }, - 'sources_default': {}, - 'compatibility': { - 'inetnum_search_disabled': {}, - 'irrd42_migration_in_progress': {}, - 'permit_non_hierarchical_as_set_name': {}, - 'ipv4_only_route_set_members': {}, - } -}) - -KNOWN_SOURCES_KEYS = { - 'authoritative', - 'keep_journal', - 'nrtm_host', - 'nrtm_port', - 'import_source', - 'import_serial_source', - 'import_timer', - 'object_class_filter', - 'export_destination', - 'export_destination_unfiltered', - 'export_timer', - 'nrtm_access_list', - 'nrtm_access_list_unfiltered', - 'strict_import_keycert_objects', - 'rpki_excluded', - 'scopefilter_excluded', -} +VALID_SET_AUTNUM_AUTHENTICATION = ['disabled', 'opportunistic', 'required'] LOGGING = { 'version': 1, @@ -162,6 +81,9 @@ def __init__(self, user_config_path: Optional[str]=None, commit=True): Load the default config and load and check the user provided config. If a logfile was specified, direct logs there. """ + from .known_keys import KNOWN_CONFIG_KEYS, KNOWN_SOURCES_KEYS + self.known_config_keys = KNOWN_CONFIG_KEYS + self.known_sources_keys = KNOWN_SOURCES_KEYS self.user_config_path = user_config_path if user_config_path else CONFIG_PATH_DEFAULT default_config_path = str(Path(__file__).resolve().parents[0] / 'default_config.yaml') default_config_yaml = yaml.safe_load(open(default_config_path)) @@ -211,10 +133,10 @@ def get_setting_live(self, setting_name: str, default: Any=None) -> Any: """ if setting_name.startswith('sources'): components = setting_name.split('.') - if len(components) == 3 and components[2] not in KNOWN_SOURCES_KEYS: + if len(components) == 3 and components[2] not in self.known_sources_keys: raise ValueError(f'Unknown setting {setting_name}') elif not setting_name.startswith('access_lists'): - if KNOWN_CONFIG_KEYS.get(setting_name) is None: + if self.known_config_keys.get(setting_name) is None: raise ValueError(f'Unknown setting {setting_name}') env_key = 'IRRD_' + setting_name.upper().replace('.', '_') @@ -289,18 +211,24 @@ def _check_staging_config(self) -> List[str]: errors = [] config = self.user_config_staging - for key, value in config.items(): - if key in ['sources', 'access_lists']: - continue - known = KNOWN_CONFIG_KEYS.get(key) - if known is None: - errors.append(f'Unknown setting key: {key}') + def _validate_subconfig(key, value): if hasattr(value, 'items'): for key2, value2 in value.items(): subkey = key + '.' + key2 - known_sub = KNOWN_CONFIG_KEYS.get(subkey) + known_sub = self.known_config_keys.get(subkey) + if known_sub is None: errors.append(f'Unknown setting key: {subkey}') + _validate_subconfig(subkey, value2) + + for key, value in config.items(): + if key in ['sources', 'access_lists']: + continue + known = self.known_config_keys.get(key) + + if known is None: + errors.append(f'Unknown setting key: {key}') + _validate_subconfig(key, value) if not self._check_is_str(config, 'database_url'): errors.append('Setting database_url is required.') @@ -337,6 +265,12 @@ def _check_staging_config(self) -> List[str]: if not self._check_is_str(config, 'auth.gnupg_keyring'): errors.append('Setting auth.gnupg_keyring is required.') + for set_name, params in config.get('auth.set_creation', {}).items(): + if not isinstance(params.get('prefix_required', False), bool): + errors.append(f'Setting auth.set_creation.{set_name}.prefix_required must be a bool') + if params.get('autnum_authentication') and params['autnum_authentication'].lower() not in VALID_SET_AUTNUM_AUTHENTICATION: + errors.append(f'Setting auth.set_creation.{set_name}.autnum_authentication must be one of {VALID_SET_AUTNUM_AUTHENTICATION} if set') + for name, access_list in config.get('access_lists', {}).items(): for item in access_list: try: @@ -365,7 +299,7 @@ def _check_staging_config(self) -> List[str]: has_authoritative_sources = False for name, details in config.get('sources', {}).items(): - unknown_keys = set(details.keys()) - KNOWN_SOURCES_KEYS + unknown_keys = set(details.keys()) - self.known_sources_keys if unknown_keys: errors.append(f'Unknown key(s) under source {name}: {", ".join(unknown_keys)}') if details.get('authoritative'): diff --git a/irrd/conf/known_keys.py b/irrd/conf/known_keys.py new file mode 100644 index 000000000..e3610758e --- /dev/null +++ b/irrd/conf/known_keys.py @@ -0,0 +1,91 @@ +from irrd.vendor.dotted.collection import DottedDict +from irrd.rpsl.rpsl_objects import OBJECT_CLASS_MAPPING, RPSLSet + +# Note that sources are checked separately, +# and 'access_lists' is always permitted +KNOWN_CONFIG_KEYS = DottedDict({ + 'database_url': {}, + 'database_readonly': {}, + 'redis_url': {}, + 'piddir': {}, + 'user': {}, + 'group': {}, + 'server': { + 'http': { + 'interface': {}, + 'port': {}, + 'status_access_list': {}, + 'workers': {}, + 'forwarded_allowed_ips': {}, + }, + 'whois': { + 'interface': {}, + 'port': {}, + 'access_list': {}, + 'max_connections': {}, + }, + }, + 'email': { + 'from': {}, + 'footer': {}, + 'smtp': {}, + 'recipient_override': {}, + 'notification_header': {}, + }, + 'auth': { + 'override_password': {}, + 'authenticate_related_mntners': {}, + 'gnupg_keyring': {}, + 'set_creation': { + rpsl_object_class: {'prefix_required': {}, 'autnum_authentication': {}} + for rpsl_object_class in [ + set_object.rpsl_object_class + for set_object in OBJECT_CLASS_MAPPING.values() + if issubclass(set_object, RPSLSet) + ] + ['DEFAULT'] + }, + }, + 'rpki': { + 'roa_source': {}, + 'roa_import_timer': {}, + 'slurm_source': {}, + 'pseudo_irr_remarks': {}, + 'notify_invalid_enabled': {}, + 'notify_invalid_subject': {}, + 'notify_invalid_header': {}, + }, + 'scopefilter': { + 'prefixes': {}, + 'asns': {}, + }, + 'log': { + 'logfile_path': {}, + 'level': {}, + 'logging_config_path': {}, + }, + 'sources_default': {}, + 'compatibility': { + 'inetnum_search_disabled': {}, + 'irrd42_migration_in_progress': {}, + 'ipv4_only_route_set_members': {}, + } +}) + +KNOWN_SOURCES_KEYS = { + 'authoritative', + 'keep_journal', + 'nrtm_host', + 'nrtm_port', + 'import_source', + 'import_serial_source', + 'import_timer', + 'object_class_filter', + 'export_destination', + 'export_destination_unfiltered', + 'export_timer', + 'nrtm_access_list', + 'nrtm_access_list_unfiltered', + 'strict_import_keycert_objects', + 'rpki_excluded', + 'scopefilter_excluded', +} diff --git a/irrd/conf/test_conf.py b/irrd/conf/test_conf.py index 7a98c9032..79339fdd0 100644 --- a/irrd/conf/test_conf.py +++ b/irrd/conf/test_conf.py @@ -73,7 +73,18 @@ def test_load_valid_reload_valid_config(self, monkeypatch, save_yaml_config, tmp } }, 'auth': { - 'gnupg_keyring': str(tmpdir) + 'gnupg_keyring': str(tmpdir), + 'authenticate_related_mntners': True, + 'set_creation': { + 'as-set': { + 'prefix_required': True, + 'autnum_authentication': 'opportunistic', + }, + 'DEFAULT': { + 'prefix_required': True, + 'autnum_authentication': 'required', + }, + }, }, 'sources_default': ['TESTDB2', 'TESTDB'], 'sources': { @@ -188,7 +199,7 @@ def test_load_valid_reload_invalid_config(self, save_yaml_config, tmpdir, caplog } }, 'auth': { - 'gnupg_keyring': str(tmpdir) + 'gnupg_keyring': str(tmpdir), }, 'rpki': { 'roa_source': 'https://example.com/roa.json', @@ -241,6 +252,17 @@ def test_load_invalid_config(self, save_yaml_config, tmpdir): '192.0.2.2.1' }, }, + 'auth': { + 'set_creation': { + 'as-set': { + 'prefix_required': 'not-a-bool', + 'autnum_authentication': 'unknown-value', + }, + 'not-a-real-set': { + 'prefix_required': True, + }, + }, + }, 'rpki': { 'roa_source': 'https://example.com/roa.json', 'roa_import_timer': 'foo', @@ -297,6 +319,9 @@ def test_load_invalid_config(self, save_yaml_config, tmpdir): assert 'Setting email.recipient_override must be an email address if set.' in str(ce.value) assert 'Settings user and group must both be defined, or neither.' in str(ce.value) assert 'Setting auth.gnupg_keyring is required.' in str(ce.value) + assert 'Unknown setting key: auth.set_creation.not-a-real-set.prefix_required' in str(ce.value) + assert 'Setting auth.set_creation.as-set.prefix_required must be a bool' in str(ce.value) + assert 'Setting auth.set_creation.as-set.autnum_authentication must be one of' in str(ce.value) assert 'Access lists doesnotexist, invalid-list referenced in settings, but not defined.' in str(ce.value) assert 'Setting server.http.status_access_list must be a string, if defined.' in str(ce.value) assert 'Invalid item in access list bad-list: IPv4 Address with more than 4 bytes.' in str(ce.value) diff --git a/irrd/integration_tests/run.py b/irrd/integration_tests/run.py index 9a1513a42..c5025fbac 100644 --- a/irrd/integration_tests/run.py +++ b/irrd/integration_tests/run.py @@ -765,6 +765,21 @@ def _start_irrds(self): 'auth': { 'gnupg_keyring': None, 'override_password': '$1$J6KycItM$MbPaBU6iFSGFV299Rk7Di0', + 'set_creation': { + 'filter-set': { + 'prefix_required': False, + }, + 'peering-set': { + 'prefix_required': False, + }, + 'route-set': { + 'prefix_required': False, + }, + 'rtr-set': { + 'prefix_required': False, + }, + }, + }, 'email': { diff --git a/irrd/rpsl/rpsl_objects.py b/irrd/rpsl/rpsl_objects.py index e97651803..3b58c5439 100644 --- a/irrd/rpsl/rpsl_objects.py +++ b/irrd/rpsl/rpsl_objects.py @@ -31,6 +31,23 @@ def rpsl_object_from_text(text, strict_validation=True, default_source: Optional return klass(from_text=text, strict_validation=strict_validation, default_source=default_source) +class RPSLSet(RPSLObject): + def clean_for_create(self) -> bool: + if get_setting(f'auth.set_creation.{self.rpsl_object_class}.prefix_required') is False: + return True + if get_setting('auth.set_creation.DEFAULT.prefix_required') is False: + return True + + first_segment = self.pk().split(':')[0] + try: + parse_as_number(first_segment) + return True + except ValidationError as ve: + self.messages.error(f'{self.rpsl_object_class} names must be hierarchical and the first ' + f'component must be an AS number, e.g. "AS65537:{first_segment}": {str(ve)}') + return False + + class RPSLAsBlock(RPSLObject): fields = OrderedDict([ ('as-block', RPSLASBlockField(primary_key=True, lookup_key=True)), @@ -45,7 +62,7 @@ class RPSLAsBlock(RPSLObject): ]) -class RPSLAsSet(RPSLObject): +class RPSLAsSet(RPSLSet): fields = OrderedDict([ ('as-set', RPSLSetNameField(primary_key=True, lookup_key=True, prefix='AS')), ('descr', RPSLTextField(multiple=True, optional=True)), @@ -60,19 +77,6 @@ class RPSLAsSet(RPSLObject): ('source', RPSLGenericNameField()), ]) - def clean_for_create(self) -> bool: - if get_setting('compatibility.permit_non_hierarchical_as_set_name'): - return True - - first_segment = self.pk().split(':')[0] - try: - parse_as_number(first_segment) - return True - except ValidationError as ve: - self.messages.error('AS set names must be hierarchical and the first component must ' - f'be an AS number, e.g. "AS65537:AS-EXAMPLE": {str(ve)}') - return False - class RPSLAutNum(RPSLObject): fields = OrderedDict([ @@ -117,7 +121,7 @@ class RPSLDomain(RPSLObject): ]) -class RPSLFilterSet(RPSLObject): +class RPSLFilterSet(RPSLSet): fields = OrderedDict([ ('filter-set', RPSLSetNameField(primary_key=True, lookup_key=True, prefix='FLTR')), ('descr', RPSLTextField(multiple=True, optional=True)), @@ -355,7 +359,7 @@ def _auth_lines(self, password_hashes=True) -> List[Union[str, List[str]]]: return [auth for auth in lines if ' ' not in auth] -class RPSLPeeringSet(RPSLObject): +class RPSLPeeringSet(RPSLSet): fields = OrderedDict([ ('peering-set', RPSLSetNameField(primary_key=True, lookup_key=True, prefix='PRNG')), ('descr', RPSLTextField(multiple=True, optional=True)), @@ -432,7 +436,7 @@ class RPSLRoute(RPSLObject): ]) -class RPSLRouteSet(RPSLObject): +class RPSLRouteSet(RPSLSet): fields = OrderedDict([ ('route-set', RPSLSetNameField(primary_key=True, lookup_key=True, prefix='RS')), ('members', RPSLRouteSetMembersField(ip_version=4, lookup_key=True, optional=True, multiple=True)), @@ -475,7 +479,7 @@ class RPSLRoute6(RPSLObject): ]) -class RPSLRtrSet(RPSLObject): +class RPSLRtrSet(RPSLSet): fields = OrderedDict([ ('rtr-set', RPSLSetNameField(primary_key=True, lookup_key=True, prefix='RTRS')), ('descr', RPSLTextField(multiple=True, optional=True)), diff --git a/irrd/rpsl/tests/test_rpsl_objects.py b/irrd/rpsl/tests/test_rpsl_objects.py index a9f830e8b..f1ebfcc1a 100644 --- a/irrd/rpsl/tests/test_rpsl_objects.py +++ b/irrd/rpsl/tests/test_rpsl_objects.py @@ -154,9 +154,13 @@ def test_clean_for_create(self, config_override): assert obj.__class__ == RPSLAsSet assert not obj.messages.errors() assert not obj.clean_for_create() - assert 'AS set names must be hierarchical and the first ' in obj.messages.errors()[0] + assert 'as-set names must be hierarchical and the first ' in obj.messages.errors()[0] - config_override({'compatibility': {'permit_non_hierarchical_as_set_name': True}}) + config_override({'auth': {'set_creation': {'as-set': {'prefix_required': False}}}}) + obj = rpsl_object_from_text(rpsl_text) + assert obj.clean_for_create() + + config_override({'auth': {'set_creation': {'DEFAULT': {'prefix_required': False}}}}) obj = rpsl_object_from_text(rpsl_text) assert obj.clean_for_create() diff --git a/irrd/updates/tests/test_parser.py b/irrd/updates/tests/test_parser.py index 430a07859..c9e4cb5e0 100644 --- a/irrd/updates/tests/test_parser.py +++ b/irrd/updates/tests/test_parser.py @@ -133,7 +133,7 @@ def test_validates_for_create(self, prepare_mocks): assert not result.validate() assert result.status == UpdateRequestStatus.ERROR_PARSING assert len(result.error_messages) == 1 - assert 'AS set names must be hierarchical and the first' in result.error_messages[0] + assert 'as-set names must be hierarchical and the first' in result.error_messages[0] # Test again with an UPDATE (which then fails on auth to stop) mock_dh.execute_query = lambda query: [{'object_text': SAMPLE_AS_SET}]