Skip to content

Commit

Permalink
♻️ REFACTOR: Profile configuration top-level keys
Browse files Browse the repository at this point in the history
  • Loading branch information
@chrisjsewell
chrisjsewell committed Jan 25, 2022
1 parent 297da50 commit 05aa2d1
Show file tree
Hide file tree
Showing 13 changed files with 306 additions and 253 deletions.
34 changes: 22 additions & 12 deletions aiida/cmdline/params/options/commands/setup.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -253,7 +253,7 @@ def get_quicksetup_password(ctx, param, value): # pylint: disable=unused-argume
SETUP_DATABASE_ENGINE = QUICKSETUP_DATABASE_ENGINE.clone(
prompt='Database engine',
contextual_default=functools.partial(
get_profile_attribute_default, ('storage_config.database_engine', 'postgresql_psycopg2')
get_profile_attribute_default, ('storage.config.database_engine', 'postgresql_psycopg2')
),
cls=options.interactive.InteractiveOption
)
Expand All @@ -267,80 +267,90 @@ def get_quicksetup_password(ctx, param, value): # pylint: disable=unused-argume
SETUP_DATABASE_HOSTNAME = QUICKSETUP_DATABASE_HOSTNAME.clone(
prompt='Database host',
contextual_default=functools.partial(
get_profile_attribute_default, ('storage_config.database_hostname', 'localhost')
get_profile_attribute_default, ('storage.config.database_hostname', 'localhost')
),
cls=options.interactive.InteractiveOption
)

SETUP_DATABASE_PORT = QUICKSETUP_DATABASE_PORT.clone(
prompt='Database port',
contextual_default=functools.partial(
get_profile_attribute_default, ('storage_config.database_port', DEFAULT_DBINFO['port'])
get_profile_attribute_default, ('storage.config.database_port', DEFAULT_DBINFO['port'])
),
cls=options.interactive.InteractiveOption
)

SETUP_DATABASE_NAME = QUICKSETUP_DATABASE_NAME.clone(
prompt='Database name',
required=True,
contextual_default=functools.partial(get_profile_attribute_default, ('storage_config.database_name', None)),
contextual_default=functools.partial(get_profile_attribute_default, ('storage.config.database_name', None)),
cls=options.interactive.InteractiveOption
)

SETUP_DATABASE_USERNAME = QUICKSETUP_DATABASE_USERNAME.clone(
prompt='Database username',
required=True,
contextual_default=functools.partial(get_profile_attribute_default, ('storage_config.database_username', None)),
contextual_default=functools.partial(get_profile_attribute_default, ('storage.config.database_username', None)),
cls=options.interactive.InteractiveOption
)

SETUP_DATABASE_PASSWORD = QUICKSETUP_DATABASE_PASSWORD.clone(
prompt='Database password',
required=True,
contextual_default=functools.partial(get_profile_attribute_default, ('storage_config.database_password', None)),
contextual_default=functools.partial(get_profile_attribute_default, ('storage.config.database_password', None)),
cls=options.interactive.InteractiveOption
)

SETUP_BROKER_PROTOCOL = QUICKSETUP_BROKER_PROTOCOL.clone(
prompt='Broker protocol',
required=True,
contextual_default=functools.partial(get_profile_attribute_default, ('broker_protocol', BROKER_DEFAULTS.protocol)),
contextual_default=functools.partial(
get_profile_attribute_default, ('process_control.config.broker_protocol', BROKER_DEFAULTS.protocol)
),
cls=options.interactive.InteractiveOption
)

SETUP_BROKER_USERNAME = QUICKSETUP_BROKER_USERNAME.clone(
prompt='Broker username',
required=True,
contextual_default=functools.partial(get_profile_attribute_default, ('broker_username', BROKER_DEFAULTS.username)),
contextual_default=functools.partial(
get_profile_attribute_default, ('process_control.config.broker_username', BROKER_DEFAULTS.username)
),
cls=options.interactive.InteractiveOption
)

SETUP_BROKER_PASSWORD = QUICKSETUP_BROKER_PASSWORD.clone(
prompt='Broker password',
required=True,
contextual_default=functools.partial(get_profile_attribute_default, ('broker_password', BROKER_DEFAULTS.password)),
contextual_default=functools.partial(
get_profile_attribute_default, ('process_control.config.broker_password', BROKER_DEFAULTS.password)
),
cls=options.interactive.InteractiveOption
)

SETUP_BROKER_HOST = QUICKSETUP_BROKER_HOST.clone(
prompt='Broker host',
required=True,
contextual_default=functools.partial(get_profile_attribute_default, ('broker_host', BROKER_DEFAULTS.host)),
contextual_default=functools.partial(
get_profile_attribute_default, ('process_control.config.broker_host', BROKER_DEFAULTS.host)
),
cls=options.interactive.InteractiveOption
)

SETUP_BROKER_PORT = QUICKSETUP_BROKER_PORT.clone(
prompt='Broker port',
required=True,
contextual_default=functools.partial(get_profile_attribute_default, ('broker_port', BROKER_DEFAULTS.port)),
contextual_default=functools.partial(
get_profile_attribute_default, ('process_control.config.broker_port', BROKER_DEFAULTS.port)
),
cls=options.interactive.InteractiveOption
)

SETUP_BROKER_VIRTUAL_HOST = QUICKSETUP_BROKER_VIRTUAL_HOST.clone(
prompt='Broker virtual host name',
required=True,
contextual_default=functools.partial(
get_profile_attribute_default, ('broker_virtual_host', BROKER_DEFAULTS.virtual_host)
get_profile_attribute_default, ('process_control.config.broker_virtual_host', BROKER_DEFAULTS.virtual_host)
),
cls=options.interactive.InteractiveOption
)
Expand Down
38 changes: 21 additions & 17 deletions aiida/manage/configuration/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -357,24 +357,28 @@ def load_documentation_profile():
with tempfile.NamedTemporaryFile() as handle:
profile_name = 'readthedocs'
profile_config = {
'storage_backend': 'django',
'storage_config': {
'database_engine': 'postgresql_psycopg2',
'database_port': 5432,
'database_hostname': 'localhost',
'database_name': 'aiidadb',
'database_password': 'aiidadb',
'database_username': 'aiida',
'repository_uri': 'file:///dev/null',
'storage': {
'backend': 'django',
'config': {
'database_engine': 'postgresql_psycopg2',
'database_port': 5432,
'database_hostname': 'localhost',
'database_name': 'aiidadb',
'database_password': 'aiidadb',
'database_username': 'aiida',
'repository_uri': 'file:///dev/null',
}
},
'process_control_backend': 'rabbitmq',
'process_control_config': {
'broker_protocol': 'amqp',
'broker_username': 'guest',
'broker_password': 'guest',
'broker_host': 'localhost',
'broker_port': 5672,
'broker_virtual_host': '',
'process_control': {
'backend': 'rabbitmq',
'config': {
'broker_protocol': 'amqp',
'broker_username': 'guest',
'broker_password': 'guest',
'broker_host': 'localhost',
'broker_port': 5672,
'broker_virtual_host': '',
}
},
}
config = {'default_profile': profile_name, 'profiles': {profile_name: profile_config}}
Expand Down
34 changes: 19 additions & 15 deletions aiida/manage/configuration/migrations/migrations.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -230,35 +230,39 @@ class AbstractStorageAndProcess(SingleMigration):

def upgrade(self, config: ConfigType) -> None:
for profile_name, profile in config.get('profiles', {}).items():
profile.setdefault('storage', {})
if 'AIIDADB_BACKEND' in profile:
profile['storage_backend'] = profile.pop('AIIDADB_BACKEND')
profile.setdefault('storage_config', {})
profile['storage']['backend'] = profile.pop('AIIDADB_BACKEND')
else:
CONFIG_LOGGER.warning(f'profile {profile_name!r} had no expected "AIIDADB_BACKEND" key')
profile['storage']['backend'] = 'sqlalchemy'
profile['storage'].setdefault('config', {})
for old, new in self.storage_conversions:
if old in profile:
profile['storage_config'][new] = profile.pop(old)
profile['storage']['config'][new] = profile.pop(old)
else:
CONFIG_LOGGER.warning(f'profile {profile_name!r} had no expected {old!r} key')
profile['process_control_backend'] = 'rabbitmq'
profile.setdefault('process_control_config', {})
profile.setdefault('process_control', {})
profile['process_control']['backend'] = 'rabbitmq'
profile['process_control'].setdefault('config', {})
for key in self.process_keys:
if key in profile:
profile['process_control_config'][key] = profile.pop(key)
profile['process_control']['config'][key] = profile.pop(key)
elif key not in ('broker_parameters', 'broker_virtual_host'):
CONFIG_LOGGER.warning(f'profile {profile_name!r} had no expected {old!r} key')

def downgrade(self, config: ConfigType) -> None:
for profile in config.get('profiles', {}).values():
if 'backend' in profile.get('storage', {}):
profile['AIIDADB_BACKEND'] = profile['storage']['backend']
for old, new in self.storage_conversions:
if new in profile.get('storage_config', {}):
profile[old] = profile['storage_config'].pop(new)
profile.pop('storage_config', None)
if 'storage_backend' in profile:
profile['AIIDADB_BACKEND'] = profile.pop('storage_backend')
if new in profile.get('storage', {}).get('config', {}):
profile[old] = profile['storage']['config'].pop(new)
profile.pop('storage', None)
for key in self.process_keys:
if key in profile.get('process_control_config', {}):
profile[key] = profile['process_control_config'].pop(key)
profile.pop('process_control_backend', None)
profile.pop('process_control_config', None)
if key in profile.get('process_control', {}).get('config', {}):
profile[key] = profile['process_control']['config'].pop(key)
profile.pop('process_control', None)


MIGRATIONS = (
Expand Down
36 changes: 19 additions & 17 deletions aiida/manage/configuration/profile.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,20 +39,20 @@
class Profile: # pylint: disable=too-many-public-methods
"""Class that models a profile as it is stored in the configuration file of an AiiDA instance."""

KEY_OPTIONS = 'options'
KEY_UUID = 'PROFILE_UUID'
KEY_DEFAULT_USER_EMAIL = 'default_user_email'
KEY_STORAGE_BACKEND = 'storage_backend'
KEY_STORAGE_CONFIG = 'storage_config'
KEY_PROCESS_BACKEND = 'process_control_backend'
KEY_PROCESS_CONFIG = 'process_control_config'
KEY_STORAGE = 'storage'
KEY_PROCESS = 'process_control'
KEY_STORAGE_BACKEND = 'backend'
KEY_STORAGE_CONFIG = 'config'
KEY_PROCESS_BACKEND = 'backend'
KEY_PROCESS_CONFIG = 'config'
KEY_OPTIONS = 'options'

# keys that are expected to be in the parsed configuration
REQUIRED_KEYS = (
KEY_STORAGE_BACKEND,
KEY_STORAGE_CONFIG,
KEY_PROCESS_BACKEND,
KEY_PROCESS_CONFIG,
KEY_STORAGE,
KEY_PROCESS,
)

@classproperty
Expand Down Expand Up @@ -112,12 +112,12 @@ def default_user_email(self, value: Optional[str]) -> None:
@property
def storage_backend(self) -> str:
"""Return the type of the storage backend."""
return self._attributes[self.KEY_STORAGE_BACKEND]
return self._attributes[self.KEY_STORAGE][self.KEY_STORAGE_BACKEND]

@property
def storage_config(self) -> Dict[str, Any]:
"""Return the configuration required by the storage backend."""
return self._attributes[self.KEY_STORAGE_CONFIG]
return self._attributes[self.KEY_STORAGE][self.KEY_STORAGE_CONFIG]

def set_storage(self, name: str, config: Dict[str, Any]) -> None:
"""Set the storage backend and its configuration.
Expand All @@ -126,18 +126,19 @@ def set_storage(self, name: str, config: Dict[str, Any]) -> None:
:param config: the configuration of the storage backend
"""
# to-do validation (by loading the storage backend, and using a classmethod to validate the config)
self._attributes[self.KEY_STORAGE_BACKEND] = name
self._attributes[self.KEY_STORAGE_CONFIG] = config
self._attributes.setdefault(self.KEY_STORAGE, {})
self._attributes[self.KEY_STORAGE][self.KEY_STORAGE_BACKEND] = name
self._attributes[self.KEY_STORAGE][self.KEY_STORAGE_CONFIG] = config

@property
def process_control_backend(self) -> str:
"""Return the type of the process control backend."""
return self._attributes[self.KEY_PROCESS_BACKEND]
return self._attributes[self.KEY_PROCESS][self.KEY_PROCESS_BACKEND]

@property
def process_control_config(self) -> Dict[str, Any]:
"""Return the configuration required by the process control backend."""
return self._attributes[self.KEY_PROCESS_CONFIG]
return self._attributes[self.KEY_PROCESS][self.KEY_PROCESS_CONFIG]

def set_process_controller(self, name: str, config: Dict[str, Any]) -> None:
"""Set the process control backend and its configuration.
Expand All @@ -146,8 +147,9 @@ def set_process_controller(self, name: str, config: Dict[str, Any]) -> None:
:param config: the configuration of the process backend
"""
# to-do validation (by loading the process backend, and using a classmethod to validate the config)
self._attributes[self.KEY_PROCESS_BACKEND] = name
self._attributes[self.KEY_PROCESS_CONFIG] = config
self._attributes.setdefault(self.KEY_PROCESS, {})
self._attributes[self.KEY_PROCESS][self.KEY_PROCESS_BACKEND] = name
self._attributes[self.KEY_PROCESS][self.KEY_PROCESS_CONFIG] = config

@property
def options(self):
Expand Down
Loading

0 comments on commit 05aa2d1

Please sign in to comment.