From 205b9e1d1e6cff0dbefffb0901a3a44cfd6b6464 Mon Sep 17 00:00:00 2001 From: JohannesGaessler Date: Mon, 13 Jan 2020 23:24:12 +0100 Subject: [PATCH 1/4] Implemented Job class --- src/ja/common/docker_context.py | 5 ++ src/ja/common/job.py | 96 +++++++++++++++++++++++++-------- src/ja/common/message/base.py | 49 ++++++++++++++++- 3 files changed, 128 insertions(+), 22 deletions(-) diff --git a/src/ja/common/docker_context.py b/src/ja/common/docker_context.py index 05399974..3b0439c8 100644 --- a/src/ja/common/docker_context.py +++ b/src/ja/common/docker_context.py @@ -54,6 +54,11 @@ def get_mount_points(self) -> List[MountPoint]: @return A list of all mount points for the job. """ + @classmethod + @abstractmethod + def from_dict(cls, property_dict: Dict[str, object]) -> "IDockerContext": + pass + class DockerConstraints(Serializable): """ diff --git a/src/ja/common/job.py b/src/ja/common/job.py index 154ee12c..ac35897d 100644 --- a/src/ja/common/job.py +++ b/src/ja/common/job.py @@ -72,18 +72,25 @@ class Job(Serializable): Represents a job with all of its attributes. @param owner_id The unix user id of the user who owns this job. @param email The email of the user who created the job, or None. - @param ctx The docker context of the job. - @param constraints The docker constraints of the job. + @param scheduling_constraints The docker constraints of the job. + @param docker_context The docker context of the job. @param label The label set by the user for the job. """ def __init__(self, owner_id: int, email: str, - constraints: JobSchedulingConstraints, - ctx: IDockerContext, + scheduling_constraints: JobSchedulingConstraints, + docker_context: IDockerContext, docker_constraints: DockerConstraints, label: str = None): - pass + self._uid: str = None + self._status: JobStatus = JobStatus.NEW + self._owner_id = owner_id + self._email = email + self._scheduling_constraints = scheduling_constraints + self._docker_context = docker_context + self._docker_constraints = docker_constraints + self._label = label @property def uid(self) -> str: @@ -91,6 +98,7 @@ def uid(self) -> str: Get the job unique identifier used to create the job. If the job has not been queued yet, it has no UID. """ + return self._uid @uid.setter def uid(self, value: str) -> None: @@ -100,36 +108,49 @@ def uid(self, value: str) -> None: @param value The new job UID. """ - - @property - def label(self) -> str: - """! - @return: The label set by the user for this job. - """ + self._uid = value @property def owner_id(self) -> int: """! @return The id of the user who owns the job. """ + return self._owner_id @property def email(self) -> str: """! @return The email of the user who owns the job, or None if no email was specified. """ + return self._email @property def scheduling_constraints(self) -> JobSchedulingConstraints: """! @return The scheduling constraints of the job. """ + return self._scheduling_constraints + + @property + def docker_context(self) -> IDockerContext: + """! + @return The docker context of the job. + """ + return self._docker_context + + @property + def docker_constraints(self) -> DockerConstraints: + """! + @return The docker constraints of the job. + """ + return self._docker_constraints @property def status(self) -> JobStatus: """! @return The current job status. """ + return self._status @status.setter def status(self, new_status: JobStatus) -> None: @@ -146,22 +167,55 @@ def status(self, new_status: JobStatus) -> None: @param new_status The new status of the job. """ + self._status = new_status @property - def docker_context(self) -> IDockerContext: - """! - @return The docker context of the job. - """ - - @property - def docker_constraints(self) -> DockerConstraints: + def label(self) -> str: """! - @return The docker constraints of the job. + @return: The label set by the user for this job. """ + return self._label def to_dict(self) -> Dict[str, object]: - pass + _dict: Dict[str, object] = dict() + _dict["uid"] = self.uid + _dict["owner_id"] = self.owner_id + _dict["email"] = self.email + _dict["scheduling_constraints"] = self.scheduling_constraints.to_dict() + _dict["docker_context"] = self.docker_context.to_dict() + _dict["docker_constraints"] = self.docker_constraints.to_dict() + _dict["status"] = self.status + _dict["label"] = self.label + return _dict @classmethod def from_dict(cls, property_dict: Dict[str, object]) -> "Job": - pass + _uid: str = cls._get_str_from_dict(property_dict=property_dict, key="uid", mandatory=False) + _owner_id: int = cls._get_int_from_dict(property_dict=property_dict, key="uid") + _email: str = cls._get_str_from_dict(property_dict=property_dict, key="email") + + _scheduling_constraints = JobSchedulingConstraints.from_dict( + cls._get_dict_from_dict(property_dict=property_dict, key="scheduling_constraints") + ) + + _docker_context = IDockerContext.from_dict( + cls._get_dict_from_dict(property_dict=property_dict, key="docker_context") + ) + + _docker_constraints = DockerConstraints.from_dict( + cls._get_dict_from_dict(property_dict=property_dict, key="docker_constraints") + ) + + _status = JobStatus(cls._get_from_dict(property_dict=property_dict, key="status")) + + _label = cls._get_str_from_dict(property_dict=property_dict, key="label", mandatory=False) + + cls._assert_all_properties_used(property_dict) + + _job = Job( + owner_id=_owner_id, email=_email, scheduling_constraints=_scheduling_constraints, + docker_context=_docker_context, docker_constraints=_docker_constraints, label=_label + ) + _job.uid = _uid + _job.status = _status + return _job diff --git a/src/ja/common/message/base.py b/src/ja/common/message/base.py index 40be8d5a..3c8082da 100644 --- a/src/ja/common/message/base.py +++ b/src/ja/common/message/base.py @@ -4,7 +4,7 @@ of JobAdder. """ from abc import ABC, abstractmethod -from typing import Dict +from typing import Dict, cast import yaml @@ -16,6 +16,53 @@ class Serializable(ABC): also be dumped as/read from a YAML string. """ + @classmethod + def _get_from_dict(cls, property_dict: Dict[str, object], key: str, mandatory: bool = True) -> object: + _property = property_dict.pop(key, None) + if mandatory and _property is None: + raise ValueError( + "Cannot read in object of type %s because the dictionary does not have the mandatory property %s" + % (cls, key) + ) + return _property + + @classmethod + def _raise_error_wrong_type(cls, key: str, expected_type: str, actual_type: str) -> None: + raise ValueError( + "Cannot read in object of type %s. Expected type of property %s to be %s but received %s" + % (cls.__name__, key, expected_type, actual_type) + ) + + @classmethod + def _assert_all_properties_used(cls, property_dict: Dict[str, object]) -> None: + if property_dict: + raise ValueError( + "Received unexpected properties %s when reading in an object of type %s" + % (property_dict.keys(), cls.__name__) + ) + + @classmethod + def _get_dict_from_dict( + cls, property_dict: Dict[str, object], key: str, mandatory: bool = True) -> Dict[str, object]: + _property = cls._get_from_dict(property_dict=property_dict, key=key, mandatory=mandatory) + if not isinstance(_property, dict): + cls._raise_error_wrong_type(key=key, expected_type="dict", actual_type=_property.__class__.__name__) + return cast(Dict[str, object], _property) + + @classmethod + def _get_str_from_dict(cls, property_dict: Dict[str, object], key: str, mandatory: bool = True) -> str: + _property = cls._get_from_dict(property_dict=property_dict, key=key, mandatory=mandatory) + if not isinstance(_property, str): + cls._raise_error_wrong_type(key=key, expected_type="str", actual_type=_property.__class__.__name__) + return cast(str, _property) + + @classmethod + def _get_int_from_dict(cls, property_dict: Dict[str, object], key: str, mandatory: bool = True) -> int: + _property = cls._get_from_dict(property_dict=property_dict, key=key, mandatory=mandatory) + if not isinstance(_property, int): + cls._raise_error_wrong_type(key=key, expected_type="int", actual_type=_property.__class__.__name__) + return cast(int, _property) + @abstractmethod def to_dict(self) -> Dict[str, object]: """! From 75e9a89a98c37e8292c2badecc4103a4507f7701 Mon Sep 17 00:00:00 2001 From: JohannesGaessler Date: Wed, 15 Jan 2020 21:16:37 +0100 Subject: [PATCH 2/4] Implemented JobSchedulingConstraints, DockerContext, DockerConstraints --- src/ja/common/docker_context.py | 90 ++++++++++++++++++++++++++++++--- src/ja/common/job.py | 34 +++++++++---- src/ja/common/message/base.py | 20 +++++++- 3 files changed, 128 insertions(+), 16 deletions(-) diff --git a/src/ja/common/docker_context.py b/src/ja/common/docker_context.py index 3b0439c8..bf3c69e2 100644 --- a/src/ja/common/docker_context.py +++ b/src/ja/common/docker_context.py @@ -4,11 +4,11 @@ and hardware constraints. """ from abc import ABC, abstractmethod -from typing import List, Dict +from typing import List, Dict, cast from ja.common.message.base import Serializable -class MountPoint: +class MountPoint(Serializable): """! A mount point consists of: 1. A directory to be mounted(@source_path). @@ -34,6 +34,13 @@ def mount_path(self) -> str: @return The path where the directory should be mounted. """ + def to_dict(self) -> Dict[str, object]: + pass + + @classmethod + def from_dict(cls, property_dict: Dict[str, object]) -> "MountPoint": + pass + class IDockerContext(Serializable, ABC): """ @@ -41,15 +48,17 @@ class IDockerContext(Serializable, ABC): run a job in. """ + @property @abstractmethod - def get_dockerfile_source(self) -> str: + def dockerfile_source(self) -> str: """! @return The string contents of a Dockerfile which can be used to build the docker image. """ + @property @abstractmethod - def get_mount_points(self) -> List[MountPoint]: + def mount_points(self) -> List[MountPoint]: """! @return A list of all mount points for the job. """ @@ -60,6 +69,57 @@ def from_dict(cls, property_dict: Dict[str, object]) -> "IDockerContext": pass +class DockerContext(IDockerContext): + """ + An implementation of the IDockerContext interface + """ + def __init__(self, dockerfile_source: str, mount_points: List[MountPoint]): + """ + @param dockerfile_source: The string contents of a Dockerfile which can be used to build the docker image. + @param mount_points: A list of all mount points for the job. + """ + self._dockerfile_source = dockerfile_source + self._mount_points = mount_points + + @property + def dockerfile_source(self) -> str: + return self._dockerfile_source + + @property + def mount_points(self) -> List[MountPoint]: + return self._mount_points + + def to_dict(self) -> Dict[str, object]: + _dict: Dict[str, object] = dict() + _dict["dockerfile_source"] = self.dockerfile_source + _dict["mount_points"] = [_mount_point.to_dict() for _mount_point in self.mount_points] + return _dict + + @classmethod + def from_dict(cls, property_dict: Dict[str, object]) -> "IDockerContext": + _dockerfile_source = cls._get_str_from_dict(property_dict=property_dict, key="dockerfile_source") + + _property: object = cls._get_from_dict(property_dict=property_dict, key="mount_points") + if not isinstance(_property, list): + cls._raise_error_wrong_type( + key="mount_points", expected_type="List[MountPoint]", + actual_type=_property.__class__.__name__ + ) + _object_list = cast(List[object], _property) + _mount_point_list: List[MountPoint] = [] + for _object in _object_list: + if not isinstance(_object, dict): + cls._raise_error_wrong_type( + key="mount_points", expected_type="List[MountPoint]", + actual_type="List[object]" + ) + _mount_point_list.append(MountPoint.from_dict( + cast(Dict[str, object], _object) + )) + cls._assert_all_properties_used(property_dict) + return DockerContext(dockerfile_source=_dockerfile_source, mount_points=_mount_point_list) + + class DockerConstraints(Serializable): """ A list of constraints of the docker container. @@ -70,6 +130,11 @@ def __init__(self, cpu_threads: int = -1, memory: int = 1): @param cpu_threads Initial value of @cpu_threads. @param memory The value of @memory. """ + self._cpu_threads = -1 + self.cpu_threads = cpu_threads + if memory < 1: + raise ValueError("Cannot set memory to %s because this value is < 1." % memory) + self._memory = memory @property def cpu_threads(self) -> int: @@ -78,6 +143,7 @@ def cpu_threads(self) -> int: -1 means that the amount of threads is not set, in which case the number of threads will be determined when scheduling the job. """ + return self._cpu_threads @cpu_threads.setter def cpu_threads(self, count_threads: int) -> None: @@ -86,16 +152,28 @@ def cpu_threads(self, count_threads: int) -> None: If the number of cpu threads is already set(i.e it is not equal to -1), this will raise a RuntimeError. """ + if self._cpu_threads != -1: + raise RuntimeError("cpu_threads can only be set once.") + if count_threads < 1: + raise ValueError("Cannot set cpu_threads to %s because this value is < 1." % count_threads) + self._cpu_threads = count_threads @property def memory(self) -> int: """! @return The maximum amount of RAM in MB to allocate for this container. """ + return self._memory def to_dict(self) -> Dict[str, object]: - pass + _dict: Dict[str, object] = dict() + _dict["cpu_threads"] = self.cpu_threads + _dict["memory"] = self.memory + return _dict @classmethod def from_dict(cls, property_dict: Dict[str, object]) -> "DockerConstraints": - pass + _cpu_threads = cls._get_int_from_dict(property_dict=property_dict, key="cpu_threads") + _memory = cls._get_int_from_dict(property_dict=property_dict, key="memory") + cls._assert_all_properties_used(property_dict) + return DockerConstraints(cpu_threads=_cpu_threads, memory=_memory) diff --git a/src/ja/common/job.py b/src/ja/common/job.py index ac35897d..deef3053 100644 --- a/src/ja/common/job.py +++ b/src/ja/common/job.py @@ -3,7 +3,7 @@ """ from typing import List, Dict from enum import Enum -from ja.common.docker_context import DockerConstraints, IDockerContext +from ja.common.docker_context import DockerConstraints, IDockerContext, DockerContext from ja.common.message.base import Serializable @@ -39,32 +39,48 @@ def __init__(self, priority: JobPriority, is_preemtible: bool, special_resources: List[str]): - pass + self._priority = priority + self._is_preemptible = is_preemtible + self._special_resources = special_resources @property def priority(self) -> JobPriority: """! @return The priority of the job. """ + return self._priority @property def is_preemptible(self) -> bool: """! @return Whether the job can be preempted. """ + return self._is_preemptible @property - def get_special_resources(self) -> List[str]: + def special_resources(self) -> List[str]: """! @return A list of special resources used by the job. """ + return self._special_resources def to_dict(self) -> Dict[str, object]: - pass + _dict: Dict[str, object] = dict() + _dict["priority"] = self.priority + _dict["is_preemptible"] = self.is_preemptible + _dict["special_resources"] = self.special_resources + return _dict @classmethod def from_dict(cls, property_dict: Dict[str, object]) -> "JobSchedulingConstraints": - pass + _priority = JobPriority(cls._get_from_dict(property_dict=property_dict, key="priority")) + _is_preemtible = cls._get_bool_from_dict(property_dict=property_dict, key="is_preemtible") + _special_resources = cls._get_str_list_from_dict(property_dict=property_dict, key="special_resources") + + cls._assert_all_properties_used(property_dict) + + return JobSchedulingConstraints( + priority=_priority, is_preemtible=_is_preemtible, special_resources=_special_resources) class Job(Serializable): @@ -190,15 +206,15 @@ def to_dict(self) -> Dict[str, object]: @classmethod def from_dict(cls, property_dict: Dict[str, object]) -> "Job": - _uid: str = cls._get_str_from_dict(property_dict=property_dict, key="uid", mandatory=False) - _owner_id: int = cls._get_int_from_dict(property_dict=property_dict, key="uid") - _email: str = cls._get_str_from_dict(property_dict=property_dict, key="email") + _uid = cls._get_str_from_dict(property_dict=property_dict, key="uid", mandatory=False) + _owner_id = cls._get_int_from_dict(property_dict=property_dict, key="uid") + _email = cls._get_str_from_dict(property_dict=property_dict, key="email") _scheduling_constraints = JobSchedulingConstraints.from_dict( cls._get_dict_from_dict(property_dict=property_dict, key="scheduling_constraints") ) - _docker_context = IDockerContext.from_dict( + _docker_context = DockerContext.from_dict( cls._get_dict_from_dict(property_dict=property_dict, key="docker_context") ) diff --git a/src/ja/common/message/base.py b/src/ja/common/message/base.py index 3c8082da..6db71b3a 100644 --- a/src/ja/common/message/base.py +++ b/src/ja/common/message/base.py @@ -4,7 +4,7 @@ of JobAdder. """ from abc import ABC, abstractmethod -from typing import Dict, cast +from typing import List, Dict, cast import yaml @@ -56,6 +56,17 @@ def _get_str_from_dict(cls, property_dict: Dict[str, object], key: str, mandator cls._raise_error_wrong_type(key=key, expected_type="str", actual_type=_property.__class__.__name__) return cast(str, _property) + @classmethod + def _get_str_list_from_dict(cls, property_dict: Dict[str, object], key: str, mandatory: bool = True) -> List[str]: + _property = cls._get_from_dict(property_dict=property_dict, key=key, mandatory=mandatory) + if not isinstance(_property, list): + cls._raise_error_wrong_type(key=key, expected_type="List[str]", actual_type=_property.__class__.__name__) + _list = cast(List[object], _property) + for _element in _list: + if not isinstance(_element, str): + cls._raise_error_wrong_type(key=key, expected_type="List[str]", actual_type="List[object]") + return cast(List[str], _list) + @classmethod def _get_int_from_dict(cls, property_dict: Dict[str, object], key: str, mandatory: bool = True) -> int: _property = cls._get_from_dict(property_dict=property_dict, key=key, mandatory=mandatory) @@ -63,6 +74,13 @@ def _get_int_from_dict(cls, property_dict: Dict[str, object], key: str, mandator cls._raise_error_wrong_type(key=key, expected_type="int", actual_type=_property.__class__.__name__) return cast(int, _property) + @classmethod + def _get_bool_from_dict(cls, property_dict: Dict[str, object], key: str, mandatory: bool = True) -> bool: + _property = cls._get_from_dict(property_dict=property_dict, key=key, mandatory=mandatory) + if not isinstance(_property, bool): + cls._raise_error_wrong_type(key=key, expected_type="int", actual_type=_property.__class__.__name__) + return cast(bool, _property) + @abstractmethod def to_dict(self) -> Dict[str, object]: """! From 2ccbfa3a9ce9096bf194c575ef8ad7e242857c4a Mon Sep 17 00:00:00 2001 From: JohannesGaessler Date: Wed, 15 Jan 2020 21:36:17 +0100 Subject: [PATCH 3/4] Implemented MountPoint --- src/ja/common/docker_context.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/ja/common/docker_context.py b/src/ja/common/docker_context.py index bf3c69e2..9c6e2b2e 100644 --- a/src/ja/common/docker_context.py +++ b/src/ja/common/docker_context.py @@ -21,25 +21,32 @@ def __init__(self, source_path: str, mount_path: str): @param source_path The host directory to be mounted. @param mount_path The target path in the container to mount at. """ + self._source_path = source_path + self._mount_path = mount_path @property def source_path(self) -> str: """! @return The directory to be mounted. """ + return self._source_path @property def mount_path(self) -> str: """! @return The path where the directory should be mounted. """ + return self._mount_path def to_dict(self) -> Dict[str, object]: - pass + return {"source_path": self.source_path, "mount_path": self.mount_path} @classmethod def from_dict(cls, property_dict: Dict[str, object]) -> "MountPoint": - pass + _source_path = cls._get_str_from_dict(property_dict=property_dict, key="source_path") + _mount_path = cls._get_str_from_dict(property_dict=property_dict, key="mount_path") + cls._assert_all_properties_used(property_dict) + return MountPoint(source_path=_source_path, mount_path=_mount_path) class IDockerContext(Serializable, ABC): From 6a39243164298c4e5756c649f43d7d30795ffa23 Mon Sep 17 00:00:00 2001 From: JohannesGaessler Date: Thu, 16 Jan 2020 00:15:34 +0100 Subject: [PATCH 4/4] Implemented unittests for Job and its components, fixed code style --- src/ja/common/docker_context.py | 82 ++++++++++------- src/ja/common/job.py | 87 ++++++++++++------- src/ja/common/message/base.py | 52 +++++------ src/test/abstract.py | 11 +++ src/test/serializable/base.py | 62 ++++++++++++- src/test/serializable/job/__init__.py | 0 .../job/test_docker_constraints.py | 13 +++ .../serializable/job/test_docker_context.py | 30 +++++++ src/test/serializable/job/test_job.py | 48 ++++++++++ src/test/serializable/job/test_mount_point.py | 13 +++ .../job/test_scheduling_constraints.py | 19 ++++ 11 files changed, 325 insertions(+), 92 deletions(-) create mode 100644 src/test/abstract.py create mode 100644 src/test/serializable/job/__init__.py create mode 100644 src/test/serializable/job/test_docker_constraints.py create mode 100644 src/test/serializable/job/test_docker_context.py create mode 100644 src/test/serializable/job/test_job.py create mode 100644 src/test/serializable/job/test_mount_point.py create mode 100644 src/test/serializable/job/test_scheduling_constraints.py diff --git a/src/ja/common/docker_context.py b/src/ja/common/docker_context.py index 9c6e2b2e..71156ff5 100644 --- a/src/ja/common/docker_context.py +++ b/src/ja/common/docker_context.py @@ -24,6 +24,12 @@ def __init__(self, source_path: str, mount_path: str): self._source_path = source_path self._mount_path = mount_path + def __eq__(self, other: object) -> bool: + if isinstance(other, MountPoint): + return self.source_path == other.source_path and self.mount_path == other.mount_path + else: + return False + @property def source_path(self) -> str: """! @@ -43,10 +49,10 @@ def to_dict(self) -> Dict[str, object]: @classmethod def from_dict(cls, property_dict: Dict[str, object]) -> "MountPoint": - _source_path = cls._get_str_from_dict(property_dict=property_dict, key="source_path") - _mount_path = cls._get_str_from_dict(property_dict=property_dict, key="mount_path") + source_path = cls._get_str_from_dict(property_dict=property_dict, key="source_path") + mount_path = cls._get_str_from_dict(property_dict=property_dict, key="mount_path") cls._assert_all_properties_used(property_dict) - return MountPoint(source_path=_source_path, mount_path=_mount_path) + return MountPoint(source_path=source_path, mount_path=mount_path) class IDockerContext(Serializable, ABC): @@ -55,6 +61,12 @@ class IDockerContext(Serializable, ABC): run a job in. """ + def __eq__(self, other: object) -> bool: + if isinstance(other, IDockerContext): + return self.dockerfile_source == other.dockerfile_source and self.mount_points == other.mount_points + else: + return False + @property @abstractmethod def dockerfile_source(self) -> str: @@ -97,34 +109,34 @@ def mount_points(self) -> List[MountPoint]: return self._mount_points def to_dict(self) -> Dict[str, object]: - _dict: Dict[str, object] = dict() - _dict["dockerfile_source"] = self.dockerfile_source - _dict["mount_points"] = [_mount_point.to_dict() for _mount_point in self.mount_points] - return _dict + return_dict: Dict[str, object] = dict() + return_dict["dockerfile_source"] = self.dockerfile_source + return_dict["mount_points"] = [_mount_point.to_dict() for _mount_point in self.mount_points] + return return_dict @classmethod - def from_dict(cls, property_dict: Dict[str, object]) -> "IDockerContext": - _dockerfile_source = cls._get_str_from_dict(property_dict=property_dict, key="dockerfile_source") + def from_dict(cls, property_dict: Dict[str, object]) -> IDockerContext: + dockerfile_source = cls._get_str_from_dict(property_dict=property_dict, key="dockerfile_source") - _property: object = cls._get_from_dict(property_dict=property_dict, key="mount_points") - if not isinstance(_property, list): + prop: object = cls._get_from_dict(property_dict=property_dict, key="mount_points") + if not isinstance(prop, list): cls._raise_error_wrong_type( key="mount_points", expected_type="List[MountPoint]", - actual_type=_property.__class__.__name__ + actual_type=prop.__class__.__name__ ) - _object_list = cast(List[object], _property) - _mount_point_list: List[MountPoint] = [] - for _object in _object_list: + object_list = cast(List[object], prop) + mount_point_list: List[MountPoint] = [] + for _object in object_list: if not isinstance(_object, dict): cls._raise_error_wrong_type( key="mount_points", expected_type="List[MountPoint]", actual_type="List[object]" ) - _mount_point_list.append(MountPoint.from_dict( - cast(Dict[str, object], _object) - )) + mount_point_list.append(MountPoint.from_dict( + cast(Dict[str, object], _object) + )) cls._assert_all_properties_used(property_dict) - return DockerContext(dockerfile_source=_dockerfile_source, mount_points=_mount_point_list) + return DockerContext(dockerfile_source=dockerfile_source, mount_points=mount_point_list) class DockerConstraints(Serializable): @@ -134,15 +146,22 @@ class DockerConstraints(Serializable): def __init__(self, cpu_threads: int = -1, memory: int = 1): """! Create a new set of Docker constraints. - @param cpu_threads Initial value of @cpu_threads. - @param memory The value of @memory. + @param cpu_threads Initial value of @cpu_threads. -1 if unknown. Must be > 0 if set to exact number. + @param memory The value of @memory, must be > 0. """ self._cpu_threads = -1 - self.cpu_threads = cpu_threads + if cpu_threads != -1: + self.cpu_threads = cpu_threads if memory < 1: raise ValueError("Cannot set memory to %s because this value is < 1." % memory) self._memory = memory + def __eq__(self, other: object) -> bool: + if isinstance(other, DockerConstraints): + return self.cpu_threads == other.cpu_threads and self.memory == other.memory + else: + return False + @property def cpu_threads(self) -> int: """! @@ -161,9 +180,10 @@ def cpu_threads(self, count_threads: int) -> None: """ if self._cpu_threads != -1: raise RuntimeError("cpu_threads can only be set once.") - if count_threads < 1: + elif count_threads < 1: raise ValueError("Cannot set cpu_threads to %s because this value is < 1." % count_threads) - self._cpu_threads = count_threads + else: + self._cpu_threads = count_threads @property def memory(self) -> int: @@ -173,14 +193,14 @@ def memory(self) -> int: return self._memory def to_dict(self) -> Dict[str, object]: - _dict: Dict[str, object] = dict() - _dict["cpu_threads"] = self.cpu_threads - _dict["memory"] = self.memory - return _dict + return_dict: Dict[str, object] = dict() + return_dict["cpu_threads"] = self.cpu_threads + return_dict["memory"] = self.memory + return return_dict @classmethod def from_dict(cls, property_dict: Dict[str, object]) -> "DockerConstraints": - _cpu_threads = cls._get_int_from_dict(property_dict=property_dict, key="cpu_threads") - _memory = cls._get_int_from_dict(property_dict=property_dict, key="memory") + cpu_threads = cls._get_int_from_dict(property_dict=property_dict, key="cpu_threads") + memory = cls._get_int_from_dict(property_dict=property_dict, key="memory") cls._assert_all_properties_used(property_dict) - return DockerConstraints(cpu_threads=_cpu_threads, memory=_memory) + return DockerConstraints(cpu_threads=cpu_threads, memory=memory) diff --git a/src/ja/common/job.py b/src/ja/common/job.py index deef3053..607a3b2f 100644 --- a/src/ja/common/job.py +++ b/src/ja/common/job.py @@ -43,6 +43,14 @@ def __init__(self, self._is_preemptible = is_preemtible self._special_resources = special_resources + def __eq__(self, other: object) -> bool: + if isinstance(other, JobSchedulingConstraints): + return self.priority == other.priority \ + and self.is_preemptible == other.is_preemptible \ + and self.special_resources == other.special_resources + else: + return False + @property def priority(self) -> JobPriority: """! @@ -65,22 +73,22 @@ def special_resources(self) -> List[str]: return self._special_resources def to_dict(self) -> Dict[str, object]: - _dict: Dict[str, object] = dict() - _dict["priority"] = self.priority - _dict["is_preemptible"] = self.is_preemptible - _dict["special_resources"] = self.special_resources - return _dict + return_dict: Dict[str, object] = dict() + return_dict["priority"] = self.priority + return_dict["is_preemptible"] = self.is_preemptible + return_dict["special_resources"] = self.special_resources + return return_dict @classmethod def from_dict(cls, property_dict: Dict[str, object]) -> "JobSchedulingConstraints": - _priority = JobPriority(cls._get_from_dict(property_dict=property_dict, key="priority")) - _is_preemtible = cls._get_bool_from_dict(property_dict=property_dict, key="is_preemtible") - _special_resources = cls._get_str_list_from_dict(property_dict=property_dict, key="special_resources") + priority = JobPriority(cls._get_from_dict(property_dict=property_dict, key="priority")) + is_preemtible = cls._get_bool_from_dict(property_dict=property_dict, key="is_preemptible") + special_resources = cls._get_str_list_from_dict(property_dict=property_dict, key="special_resources") cls._assert_all_properties_used(property_dict) return JobSchedulingConstraints( - priority=_priority, is_preemtible=_is_preemtible, special_resources=_special_resources) + priority=priority, is_preemtible=is_preemtible, special_resources=special_resources) class Job(Serializable): @@ -108,6 +116,19 @@ def __init__(self, self._docker_constraints = docker_constraints self._label = label + def __eq__(self, other: object) -> bool: + if isinstance(other, Job): + return self.uid == other.uid \ + and self.status == other.status \ + and self.owner_id == other.owner_id \ + and self.email == other.email \ + and self.scheduling_constraints == other.scheduling_constraints \ + and self.docker_context == other.docker_context \ + and self.docker_constraints == other.docker_constraints \ + and self.label == other.label + else: + return False + @property def uid(self) -> str: """! @@ -193,45 +214,45 @@ def label(self) -> str: return self._label def to_dict(self) -> Dict[str, object]: - _dict: Dict[str, object] = dict() - _dict["uid"] = self.uid - _dict["owner_id"] = self.owner_id - _dict["email"] = self.email - _dict["scheduling_constraints"] = self.scheduling_constraints.to_dict() - _dict["docker_context"] = self.docker_context.to_dict() - _dict["docker_constraints"] = self.docker_constraints.to_dict() - _dict["status"] = self.status - _dict["label"] = self.label - return _dict + return_dict: Dict[str, object] = dict() + return_dict["uid"] = self.uid + return_dict["owner_id"] = self.owner_id + return_dict["email"] = self.email + return_dict["scheduling_constraints"] = self.scheduling_constraints.to_dict() + return_dict["docker_context"] = self.docker_context.to_dict() + return_dict["docker_constraints"] = self.docker_constraints.to_dict() + return_dict["status"] = self.status + return_dict["label"] = self.label + return return_dict @classmethod def from_dict(cls, property_dict: Dict[str, object]) -> "Job": - _uid = cls._get_str_from_dict(property_dict=property_dict, key="uid", mandatory=False) - _owner_id = cls._get_int_from_dict(property_dict=property_dict, key="uid") - _email = cls._get_str_from_dict(property_dict=property_dict, key="email") + uid = cls._get_str_from_dict(property_dict=property_dict, key="uid", mandatory=False) + owner_id = cls._get_int_from_dict(property_dict=property_dict, key="owner_id") + email = cls._get_str_from_dict(property_dict=property_dict, key="email") - _scheduling_constraints = JobSchedulingConstraints.from_dict( + scheduling_constraints = JobSchedulingConstraints.from_dict( cls._get_dict_from_dict(property_dict=property_dict, key="scheduling_constraints") ) - _docker_context = DockerContext.from_dict( + docker_context = DockerContext.from_dict( cls._get_dict_from_dict(property_dict=property_dict, key="docker_context") ) - _docker_constraints = DockerConstraints.from_dict( + docker_constraints = DockerConstraints.from_dict( cls._get_dict_from_dict(property_dict=property_dict, key="docker_constraints") ) - _status = JobStatus(cls._get_from_dict(property_dict=property_dict, key="status")) + status = JobStatus(cls._get_from_dict(property_dict=property_dict, key="status")) - _label = cls._get_str_from_dict(property_dict=property_dict, key="label", mandatory=False) + label = cls._get_str_from_dict(property_dict=property_dict, key="label", mandatory=False) cls._assert_all_properties_used(property_dict) - _job = Job( - owner_id=_owner_id, email=_email, scheduling_constraints=_scheduling_constraints, - docker_context=_docker_context, docker_constraints=_docker_constraints, label=_label + job = Job( + owner_id=owner_id, email=email, scheduling_constraints=scheduling_constraints, + docker_context=docker_context, docker_constraints=docker_constraints, label=label ) - _job.uid = _uid - _job.status = _status - return _job + job.uid = uid + job.status = status + return job diff --git a/src/ja/common/message/base.py b/src/ja/common/message/base.py index 6db71b3a..a33fbf01 100644 --- a/src/ja/common/message/base.py +++ b/src/ja/common/message/base.py @@ -18,13 +18,13 @@ class Serializable(ABC): @classmethod def _get_from_dict(cls, property_dict: Dict[str, object], key: str, mandatory: bool = True) -> object: - _property = property_dict.pop(key, None) - if mandatory and _property is None: + prop = property_dict.pop(key, None) + if mandatory and prop is None: raise ValueError( "Cannot read in object of type %s because the dictionary does not have the mandatory property %s" % (cls, key) ) - return _property + return prop @classmethod def _raise_error_wrong_type(cls, key: str, expected_type: str, actual_type: str) -> None: @@ -44,42 +44,42 @@ def _assert_all_properties_used(cls, property_dict: Dict[str, object]) -> None: @classmethod def _get_dict_from_dict( cls, property_dict: Dict[str, object], key: str, mandatory: bool = True) -> Dict[str, object]: - _property = cls._get_from_dict(property_dict=property_dict, key=key, mandatory=mandatory) - if not isinstance(_property, dict): - cls._raise_error_wrong_type(key=key, expected_type="dict", actual_type=_property.__class__.__name__) - return cast(Dict[str, object], _property) + prop = cls._get_from_dict(property_dict=property_dict, key=key, mandatory=mandatory) + if mandatory and not isinstance(prop, dict): + cls._raise_error_wrong_type(key=key, expected_type="dict", actual_type=prop.__class__.__name__) + return cast(Dict[str, object], prop) @classmethod def _get_str_from_dict(cls, property_dict: Dict[str, object], key: str, mandatory: bool = True) -> str: - _property = cls._get_from_dict(property_dict=property_dict, key=key, mandatory=mandatory) - if not isinstance(_property, str): - cls._raise_error_wrong_type(key=key, expected_type="str", actual_type=_property.__class__.__name__) - return cast(str, _property) + prop = cls._get_from_dict(property_dict=property_dict, key=key, mandatory=mandatory) + if mandatory and not isinstance(prop, str): + cls._raise_error_wrong_type(key=key, expected_type="str", actual_type=prop.__class__.__name__) + return cast(str, prop) @classmethod def _get_str_list_from_dict(cls, property_dict: Dict[str, object], key: str, mandatory: bool = True) -> List[str]: - _property = cls._get_from_dict(property_dict=property_dict, key=key, mandatory=mandatory) - if not isinstance(_property, list): - cls._raise_error_wrong_type(key=key, expected_type="List[str]", actual_type=_property.__class__.__name__) - _list = cast(List[object], _property) - for _element in _list: - if not isinstance(_element, str): + prop = cls._get_from_dict(property_dict=property_dict, key=key, mandatory=mandatory) + if mandatory and not isinstance(prop, list): + cls._raise_error_wrong_type(key=key, expected_type="List[str]", actual_type=prop.__class__.__name__) + return_list = cast(List[object], prop) + for element in return_list: + if not isinstance(element, str): cls._raise_error_wrong_type(key=key, expected_type="List[str]", actual_type="List[object]") - return cast(List[str], _list) + return cast(List[str], return_list) @classmethod def _get_int_from_dict(cls, property_dict: Dict[str, object], key: str, mandatory: bool = True) -> int: - _property = cls._get_from_dict(property_dict=property_dict, key=key, mandatory=mandatory) - if not isinstance(_property, int): - cls._raise_error_wrong_type(key=key, expected_type="int", actual_type=_property.__class__.__name__) - return cast(int, _property) + prop = cls._get_from_dict(property_dict=property_dict, key=key, mandatory=mandatory) + if mandatory and not isinstance(prop, int): + cls._raise_error_wrong_type(key=key, expected_type="int", actual_type=prop.__class__.__name__) + return cast(int, prop) @classmethod def _get_bool_from_dict(cls, property_dict: Dict[str, object], key: str, mandatory: bool = True) -> bool: - _property = cls._get_from_dict(property_dict=property_dict, key=key, mandatory=mandatory) - if not isinstance(_property, bool): - cls._raise_error_wrong_type(key=key, expected_type="int", actual_type=_property.__class__.__name__) - return cast(bool, _property) + prop = cls._get_from_dict(property_dict=property_dict, key=key, mandatory=mandatory) + if mandatory and not isinstance(prop, bool): + cls._raise_error_wrong_type(key=key, expected_type="int", actual_type=prop.__class__.__name__) + return cast(bool, prop) @abstractmethod def to_dict(self) -> Dict[str, object]: diff --git a/src/test/abstract.py b/src/test/abstract.py new file mode 100644 index 00000000..ecc11be1 --- /dev/null +++ b/src/test/abstract.py @@ -0,0 +1,11 @@ +from typing import Callable, Any + + +def skipIfAbstract(f: Callable[..., None]) -> Callable[..., None]: + def wrapper(*args: Any) -> None: + self = args[0] + if "Abstract" not in type(args[0]).__name__: + f(*args) + else: + self.skipTest('Test is in abstract base class') + return wrapper diff --git a/src/test/serializable/base.py b/src/test/serializable/base.py index b2341db7..ad09a7f0 100644 --- a/src/test/serializable/base.py +++ b/src/test/serializable/base.py @@ -1,6 +1,64 @@ -from ja.common.message.base import Serializable -from typing import Dict +from copy import deepcopy from unittest import TestCase +from typing import Dict, List + +from ja.common.message.base import Serializable +from test.abstract import skipIfAbstract + + +class AbstractSerializableTest(TestCase): + """ + Abstract base class for testing sub-classes of Serializable. + """ + def setUp(self) -> None: + self._object: Serializable = None + self._object_dict: Dict[str, object] = None + self._other_object_dict: Dict[str, object] = None + self._optional_properties: List[str] = None + + @skipIfAbstract + def test_roundtrip(self) -> None: + property_dict = self._object.to_dict() + recreated_object = self._object.__class__.from_dict(property_dict) + self.assertTrue( + self._object == recreated_object, + msg="Roundtrip failed for %s: recreated object not equal." % self._object.__class__.__name__ + ) + + @skipIfAbstract + def test_from_dict(self) -> None: + read_object = self._object.__class__.from_dict(self._object_dict) + self.assertTrue( + self._object == read_object, + msg="Reading in object failed for %s: read in object not equal." % self._object.__class__.__name__ + ) + + @skipIfAbstract + def test_from_other_dict(self) -> None: + read_object = self._object.__class__.from_dict(self._object_dict) + other_read_object = self._object.__class__.from_dict(self._other_object_dict) + self.assertFalse( + read_object == other_read_object, + msg="Reading in different objects failed for %s: read in objects are the same." + % self._object.__class__.__name__ + ) + + @skipIfAbstract + def test_removing_property_raises_error(self) -> None: + for key in list(self._object_dict.keys()): + object_dict_copy = deepcopy(self._object_dict) + object_dict_copy.pop(key) + if key in self._optional_properties: + self._object.__class__.from_dict(object_dict_copy) + else: + with self.assertRaises(ValueError): + self._object.__class__.from_dict(object_dict_copy) + + @skipIfAbstract + def test_adding_property_raises_error(self) -> None: + self._object_dict["AYY"] = "LAMO" + with self.assertRaises(ValueError): + self._object.__class__.from_dict(self._object_dict) class SimpleSerializable(Serializable): diff --git a/src/test/serializable/job/__init__.py b/src/test/serializable/job/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/test/serializable/job/test_docker_constraints.py b/src/test/serializable/job/test_docker_constraints.py new file mode 100644 index 00000000..97462467 --- /dev/null +++ b/src/test/serializable/job/test_docker_constraints.py @@ -0,0 +1,13 @@ +from ja.common.docker_context import DockerConstraints +from test.serializable.base import AbstractSerializableTest + + +class DockerConstraintsTest(AbstractSerializableTest): + """ + Class for testing DockerConstraints. + """ + def setUp(self) -> None: + self._optional_properties = [] + self._object = DockerConstraints(cpu_threads=-1, memory=1024) + self._object_dict = {"cpu_threads": -1, "memory": 1024} + self._other_object_dict = {"cpu_threads": 4, "memory": 1024} diff --git a/src/test/serializable/job/test_docker_context.py b/src/test/serializable/job/test_docker_context.py new file mode 100644 index 00000000..98c5cd4b --- /dev/null +++ b/src/test/serializable/job/test_docker_context.py @@ -0,0 +1,30 @@ +from ja.common.docker_context import DockerContext, MountPoint +from test.serializable.base import AbstractSerializableTest + + +class DockerContextTest(AbstractSerializableTest): + """ + Class for testing DockerContext. + """ + def setUp(self) -> None: + self._optional_properties = [] + self._object = DockerContext( + dockerfile_source="sudo apt install docker", + mount_points=[ + MountPoint(source_path="/home/user", mount_path="/home/user"), + MountPoint(source_path="/opt/thing", mount_path="/opt/THING") + ] + ) + self._object_dict = { + "dockerfile_source": "sudo apt install docker", + "mount_points": [ + {"source_path": "/home/user", "mount_path": "/home/user"}, + {"source_path": "/opt/thing", "mount_path": "/opt/THING"} + ] + } + self._other_object_dict = { + "dockerfile_source": "sudo apt install docker", + "mount_points": [ + {"source_path": "/home/user", "mount_path": "/home/user"}, + ] + } diff --git a/src/test/serializable/job/test_job.py b/src/test/serializable/job/test_job.py new file mode 100644 index 00000000..48ef1528 --- /dev/null +++ b/src/test/serializable/job/test_job.py @@ -0,0 +1,48 @@ +from ja.common.job import Job, JobSchedulingConstraints, JobPriority +from ja.common.docker_context import DockerContext, MountPoint, DockerConstraints +from test.serializable.base import AbstractSerializableTest + + +class JobTest(AbstractSerializableTest): + """ + Class for testing Job. + """ + def setUp(self) -> None: + self._optional_properties = ["uid", "label"] + self._object = Job( + owner_id=1008, + email="user@website.com", + scheduling_constraints=JobSchedulingConstraints( + priority=JobPriority.MEDIUM, is_preemtible=False, special_resources=["THING"] + ), + docker_context=DockerContext( + dockerfile_source="ssh localhost", + mount_points=[MountPoint(source_path="/home/user", mount_path="/home/user")] + ), + docker_constraints=DockerConstraints(cpu_threads=4, memory=4096), + label="thing" + ) + self._object_dict = { + "status": 0, + "owner_id": 1008, + "email": "user@website.com", + "scheduling_constraints": {"priority": 1, "is_preemptible": False, "special_resources": ["THING"]}, + "docker_context": { + "dockerfile_source": "ssh localhost", + "mount_points": [{"source_path": "/home/user", "mount_path": "/home/user"}] + }, + "docker_constraints": {"cpu_threads": 4, "memory": 4096}, + "label": "thing" + } + self._other_object_dict = { + "status": 0, + "owner_id": 1008, + "email": "user@website.com", + "scheduling_constraints": {"priority": 1, "is_preemptible": False, "special_resources": ["THING"]}, + "docker_context": { + "dockerfile_source": "ssh localhost", + "mount_points": [] + }, + "docker_constraints": {"cpu_threads": 4, "memory": 4096}, + "label": "thing" + } diff --git a/src/test/serializable/job/test_mount_point.py b/src/test/serializable/job/test_mount_point.py new file mode 100644 index 00000000..8a01df07 --- /dev/null +++ b/src/test/serializable/job/test_mount_point.py @@ -0,0 +1,13 @@ +from ja.common.docker_context import MountPoint +from test.serializable.base import AbstractSerializableTest + + +class MountPointTest(AbstractSerializableTest): + """ + Class for testing MountPoint. + """ + def setUp(self) -> None: + self._optional_properties = [] + self._object = MountPoint(source_path="/my/home/directory", mount_path="/unix/system/resources") + self._object_dict = {"source_path": "/my/home/directory", "mount_path": "/unix/system/resources"} + self._other_object_dict = {"source_path": "/opt/some/thing", "mount_path": "/opt/some/thing"} diff --git a/src/test/serializable/job/test_scheduling_constraints.py b/src/test/serializable/job/test_scheduling_constraints.py new file mode 100644 index 00000000..c37bec1f --- /dev/null +++ b/src/test/serializable/job/test_scheduling_constraints.py @@ -0,0 +1,19 @@ +from ja.common.job import JobSchedulingConstraints, JobPriority +from test.serializable.base import AbstractSerializableTest + + +class SchedulingConstraintsTest(AbstractSerializableTest): + """ + Class for testing JobSchedulingConstraints. + """ + def setUp(self) -> None: + self._optional_properties = [] + self._object = JobSchedulingConstraints( + priority=JobPriority.HIGH, is_preemtible=True, special_resources=["SPESHUL", "Windows", "RPI4"] + ) + self._object_dict = { + "priority": 2, "is_preemptible": True, "special_resources": ["SPESHUL", "Windows", "RPI4"] + } + self._other_object_dict = { + "priority": 3, "is_preemptible": True, "special_resources": ["SPESHUL", "Windows", "RPI4"] + }