diff --git a/apps/blender/benchmark/benchmark.py b/apps/blender/benchmark/benchmark.py index 7d528f6b6a..638a08b41c 100644 --- a/apps/blender/benchmark/benchmark.py +++ b/apps/blender/benchmark/benchmark.py @@ -37,7 +37,6 @@ def __init__(self): main_scene_file = pathlib.Path(self.blender_task_path) main_scene_file /= self.SCENE_FILE_NAME task_def.main_scene_file = str(main_scene_file) - task_def.main_program_file = self.ENVIRONMENT_CLASS().main_program_file task_def.resources.add(str(main_scene_file.resolve())) diff --git a/apps/blender/blenderenvironment.py b/apps/blender/blenderenvironment.py index d648f10e6c..53b4423525 100644 --- a/apps/blender/blenderenvironment.py +++ b/apps/blender/blenderenvironment.py @@ -1,9 +1,7 @@ -from os import path from typing import Dict from apps.core import nvgpu from apps.core.nvgpu import get_devices -from golem.core.common import get_golem_path from golem.docker.environment import DockerEnvironment from golem.docker.image import DockerImage from golem.environments.environment import SupportStatus, UnsupportReason @@ -11,10 +9,8 @@ class BlenderEnvironment(DockerEnvironment): DOCKER_IMAGE = "golemfactory/blender" - DOCKER_TAG = "1.5" + DOCKER_TAG = "1.7" ENV_ID = "BLENDER" - APP_DIR = path.join(get_golem_path(), 'apps', 'blender') - SCRIPT_NAME = "docker_blendertask.py" SHORT_DESCRIPTION = "Blender (www.blender.org)" diff --git a/apps/blender/resources/scripts/docker_blendertask.py b/apps/blender/resources/images/scripts/job.py similarity index 100% rename from apps/blender/resources/scripts/docker_blendertask.py rename to apps/blender/resources/images/scripts/job.py diff --git a/apps/blender/task/blenderrendertask.py b/apps/blender/task/blenderrendertask.py index 894f4e58d1..5694ee666b 100644 --- a/apps/blender/task/blenderrendertask.py +++ b/apps/blender/task/blenderrendertask.py @@ -44,16 +44,13 @@ def __init__(self): RendererDefaults.__init__(self) self.output_format = "EXR" - self.main_program_file = BlenderEnvironment().main_program_file self.min_subtasks = 1 self.max_subtasks = 100 self.default_subtasks = 6 class BlenderNVGPUDefaults(BlenderDefaults): - def __init__(self): - super().__init__() - self.main_program_file = BlenderNVGPUEnvironment().main_program_file + pass class PreviewUpdater(object): @@ -458,7 +455,8 @@ def query_extra_data(self, perf_index: float, num_cores: int = 0, "path_root": self.main_scene_dir, "start_task": start_task, "total_tasks": self.total_tasks, - "crops": crops + "crops": crops, + "script_filepath": "/golem/scripts/job.py", } subtask_id = self.create_subtask_id() @@ -540,7 +538,8 @@ def query_extra_data_for_test_task(self): "path_root": self.main_scene_dir, "start_task": 1, "total_tasks": 1, - "crops": crops + "crops": crops, + "script_filepath": "/golem/scripts/job.py", } hash = "{}".format(random.getrandbits(128)) diff --git a/apps/core/task/coretask.py b/apps/core/task/coretask.py index 30244aba84..0c4427a16f 100644 --- a/apps/core/task/coretask.py +++ b/apps/core/task/coretask.py @@ -120,15 +120,6 @@ def __init__(self, # pylint: disable=not-callable self.environment = self.ENVIRONMENT_CLASS() - # src_code stuff - self.main_program_file = self.environment.main_program_file - try: - with open(self.main_program_file, "r") as src_file: - src_code = src_file.read() - except OSError as err: - logger.warning("Wrong main program file: %s", err) - src_code = "" - # docker_images stuff if task_definition.docker_images: self.docker_images = task_definition.docker_images @@ -151,7 +142,7 @@ def __init__(self, concent_enabled=task_definition.concent_enabled, ) - Task.__init__(self, th, src_code, task_definition) + Task.__init__(self, th, task_definition) self.total_tasks = total_tasks self.last_task = 0 @@ -343,7 +334,6 @@ def _new_compute_task_def(self, subtask_id, extra_data, ctd['task_id'] = self.header.task_id ctd['subtask_id'] = subtask_id ctd['extra_data'] = extra_data - ctd['src_code'] = self.src_code ctd['performance'] = perf_index if self.docker_images: ctd['docker_images'] = [di.to_dict() for di in self.docker_images] @@ -507,7 +497,6 @@ def __init__(self, self.root_path = dir_manager.root_path self.dir_manager = dir_manager self.owner = owner - self.src_code = "" self.environment = None def build(self): @@ -532,7 +521,6 @@ def build_minimal_definition(cls, task_type: CoreTaskTypeInfo, dictionary): definition.compute_on = dictionary.get('compute_on', 'cpu') definition.resources = set(dictionary['resources']) definition.subtasks_count = int(dictionary['subtasks_count']) - definition.main_program_file = task_type.defaults.main_program_file return definition @classmethod diff --git a/apps/core/task/coretaskstate.py b/apps/core/task/coretaskstate.py index 2c109c934d..5acee7168b 100644 --- a/apps/core/task/coretaskstate.py +++ b/apps/core/task/coretaskstate.py @@ -17,7 +17,6 @@ class TaskDefaults(object): def __init__(self): self.output_format = "" - self.main_program_file = "" self.min_subtasks = 1 self.max_subtasks = 50 self.default_subtasks = 20 @@ -45,7 +44,6 @@ def __init__(self): self.subtasks_count = 0 self.optimize_total = False - self.main_program_file = "" self.output_file = "" self.task_type = None self.name = "" @@ -94,9 +92,6 @@ def __setstate__(self, state): setattr(self, key, attributes[key]) def is_valid(self): - if not path.exists(self.main_program_file): - return False, "Main program file does not exist: {}".format( - self.main_program_file) return self._check_output_file(self.output_file) @staticmethod diff --git a/apps/dummy/benchmark/benchmark.py b/apps/dummy/benchmark/benchmark.py index 8febc7d74f..9e1c6e51cc 100644 --- a/apps/dummy/benchmark/benchmark.py +++ b/apps/dummy/benchmark/benchmark.py @@ -3,7 +3,6 @@ from pathlib import Path from apps.core.benchmark.benchmarkrunner import CoreBenchmark -from apps.dummy.dummyenvironment import DummyTaskEnvironment from apps.dummy.task.dummytask import DummyTask from apps.dummy.task.dummytaskstate import DummyTaskDefinition, \ DummyTaskDefaults @@ -11,8 +10,6 @@ from golem.core.common import get_golem_path from golem.verificator.verifier import SubtaskVerificationState -APP_DIR = join(get_golem_path(), 'apps', 'dummy') - class DummyTaskBenchmark(CoreBenchmark): def __init__(self): @@ -27,7 +24,6 @@ def __init__(self): td.out_file_basename = td.out_file_basename td.task_id = str(uuid.uuid4()) - td.main_program_file = DummyTaskEnvironment().main_program_file td.resources = {join(self.dummy_task_path, "in.data")} td.add_to_resources() diff --git a/apps/dummy/dummyenvironment.py b/apps/dummy/dummyenvironment.py index f21370b463..91cda095df 100644 --- a/apps/dummy/dummyenvironment.py +++ b/apps/dummy/dummyenvironment.py @@ -1,14 +1,9 @@ -from os import path - -from golem.core.common import get_golem_path from golem.docker.environment import DockerEnvironment class DummyTaskEnvironment(DockerEnvironment): - DOCKER_IMAGE = "golemfactory/base" - DOCKER_TAG = "1.3" + DOCKER_IMAGE = "golemfactory/dummy" + DOCKER_TAG = "1.0" ENV_ID = "DUMMYPOW" - APP_DIR = path.join(get_golem_path(), 'apps', 'dummy') - SCRIPT_NAME = "docker_dummytask.py" SHORT_DESCRIPTION = "Dummy task (example app calculating proof-of-work " \ "hash)" diff --git a/apps/dummy/resources/images/Dockerfile b/apps/dummy/resources/images/Dockerfile new file mode 100644 index 0000000000..49c71c5074 --- /dev/null +++ b/apps/dummy/resources/images/Dockerfile @@ -0,0 +1,5 @@ +FROM golemfactory/base:1.3 + +MAINTAINER Golem Tech + +COPY scripts/ /golem/scripts/ diff --git a/apps/dummy/resources/scripts/docker_dummytask.py b/apps/dummy/resources/images/scripts/job.py similarity index 100% rename from apps/dummy/resources/scripts/docker_dummytask.py rename to apps/dummy/resources/images/scripts/job.py diff --git a/apps/dummy/task/dummytask.py b/apps/dummy/task/dummytask.py index b2106e5dfd..7319928049 100644 --- a/apps/dummy/task/dummytask.py +++ b/apps/dummy/task/dummytask.py @@ -68,6 +68,7 @@ def _extra_data(self, perf_index=0.0) -> ComputeTaskDef: "result_size": self.task_definition.result_size, "result_file": self.__get_result_file_name(subtask_id), "subtask_data_size": sbs, + "script_filepath": "/golem/scripts/job.py", } return self._new_compute_task_def(subtask_id, diff --git a/apps/images.ini b/apps/images.ini index 51613fbf3b..19a962e79c 100644 --- a/apps/images.ini +++ b/apps/images.ini @@ -1,5 +1,6 @@ golemfactory/base core/resources/images/base.Dockerfile 1.3 core/resources/images/ golemfactory/nvgpu core/resources/images/nvgpu.Dockerfile 1.1 . apps.core.nvgpu.is_supported -golemfactory/blender blender/resources/images/blender.Dockerfile 1.5 blender/resources/images/ +golemfactory/blender blender/resources/images/blender.Dockerfile 1.7 blender/resources/images/ golemfactory/blender_nvgpu blender/resources/images/blender_nvgpu.Dockerfile 1.1 . apps.core.nvgpu.is_supported -golemfactory/image_metrics rendering/resources/images/Dockerfile 1.9 rendering/resources/images/ +golemfactory/dummy dummy/resources/images/Dockerfile 1.0 dummy/resources/images +golemfactory/image_metrics rendering/resources/images/Dockerfile 1.11 rendering/resources/images/ diff --git a/golem/docker/environment.py b/golem/docker/environment.py index 7280762cb3..dad4d58ae3 100644 --- a/golem/docker/environment.py +++ b/golem/docker/environment.py @@ -6,7 +6,6 @@ from golem.docker.image import DockerImage from golem.environments.environment import (Environment, SupportStatus, UnsupportReason) -from golem.resource.dirmanager import find_task_script @enforce.runtime_validation() @@ -22,9 +21,6 @@ def __init__(self, tag=None, image_id=None, additional_images: List[DockerImage] else DockerImage(self.DOCKER_IMAGE, tag=tag) Environment.__init__(self) - self.main_program_file = find_task_script(self.APP_DIR, - self.SCRIPT_NAME) - self.docker_images = [image] if additional_images: self.docker_images += additional_images @@ -82,16 +78,6 @@ def DOCKER_TAG(cls): def ENV_ID(cls): pass - @property - @abc.abstractmethod - def APP_DIR(cls): - pass - - @property - @abc.abstractmethod - def SCRIPT_NAME(cls): - pass - @property @abc.abstractmethod def SHORT_DESCRIPTION(cls): diff --git a/golem/docker/job.py b/golem/docker/job.py index f6aff2ad93..d04c1661c4 100644 --- a/golem/docker/job.py +++ b/golem/docker/job.py @@ -22,7 +22,7 @@ # pylint:disable=too-many-instance-attributes -class DockerJob(object): +class DockerJob: STATE_NEW = "new" STATE_CREATED = "created" # container created by docker STATE_RUNNING = "running" # docker container running @@ -52,16 +52,13 @@ class DockerJob(object): "OUTPUT_DIR": OUTPUT_DIR } - # Name of the script file, relative to WORK_DIR - TASK_SCRIPT = "job.py" - # Name of the parameters file, relative to WORK_DIR PARAMS_FILE = "params.json" # pylint:disable=too-many-arguments def __init__(self, image: DockerImage, - script_src: str, + script_filepath: str, parameters: Dict, resources_dir: str, work_dir: str, @@ -82,7 +79,7 @@ def __init__(self, raise TypeError('Incorrect image type: {}. ' 'Should be: DockerImage'.format(type(image))) self.image = image - self.script_src = script_src + self.script_filepath = script_filepath self.parameters = parameters if parameters else {} self.parameters.update(self.PATH_PARAMS) @@ -124,23 +121,16 @@ def _prepare(self): with open(params_file_path, "w") as params_file: json.dump(self.parameters, params_file) - # Save the script in work_dir/TASK_SCRIPT - task_script_path = self._get_host_script_path() - with open(task_script_path, "wb") as script_file: - script_file.write(bytearray(self.script_src, "utf-8")) - # Setup volumes for the container client = local_client() host_cfg = client.create_host_config(**self.host_config) - # The location of the task script when mounted in the container - container_script_path = self._get_container_script_path() self.container = client.create_container( image=self.image.name, volumes=self.volumes, host_config=host_cfg, - command=[container_script_path], + command=[self.script_filepath], working_dir=self.WORK_DIR, environment=self.environment, ) @@ -182,9 +172,6 @@ def __enter__(self): def __exit__(self, exc_type, exc_value, traceback): self._cleanup() - def _get_host_script_path(self): - return os.path.join(self.work_dir, self.TASK_SCRIPT) - def _get_host_params_path(self): return os.path.join(self.work_dir, self.PARAMS_FILE) @@ -210,10 +197,6 @@ def _host_dir_chmod(dst_dir, mod): return prev_mod - @staticmethod - def _get_container_script_path(): - return posixpath.join(DockerJob.WORK_DIR, DockerJob.TASK_SCRIPT) - @staticmethod def get_absolute_resource_path(relative_path): return posixpath.join(DockerJob.RESOURCES_DIR, diff --git a/golem/docker/task_thread.py b/golem/docker/task_thread.py index d6ce985cdb..159a42d6d8 100644 --- a/golem/docker/task_thread.py +++ b/golem/docker/task_thread.py @@ -75,7 +75,6 @@ class DockerTaskThread(TaskThread): def __init__(self, # pylint: disable=too-many-arguments docker_images: List[Union[DockerImage, Dict, Tuple]], - src_code: str, extra_data: Dict, dir_mapping: DockerDirMapping, timeout: int, @@ -84,7 +83,6 @@ def __init__(self, # pylint: disable=too-many-arguments if not docker_images: raise AttributeError("docker images is None") super().__init__( - src_code=src_code, extra_data=extra_data, res_path=str(dir_mapping.resources), tmp_path=str(dir_mapping.temporary), @@ -190,7 +188,7 @@ def _run_docker_job(self) -> Optional[int]: params = dict( image=self.image, - script_src=self.src_code, + script_filepath=self.extra_data['script_filepath'], parameters=self.extra_data, resources_dir=str(self.dir_mapping.resources), work_dir=str(self.dir_mapping.work), diff --git a/golem/environments/environment.py b/golem/environments/environment.py index 0f2e292406..42e3b5f5ce 100644 --- a/golem/environments/environment.py +++ b/golem/environments/environment.py @@ -69,9 +69,6 @@ def __init__(self): " without any additional requirements." self.accept_tasks = False - # Check if tasks can define the source code - self.allow_custom_main_program_file = False - self.main_program_file = None def check_support(self) -> SupportStatus: """ Check if this environment is supported on this machine @@ -111,11 +108,6 @@ def get_min_accepted_performance(cls) -> float: return step * MinPerformanceMultiplier.get() - def get_source_code(self): - if self.main_program_file and path.isfile(self.main_program_file): - with open(self.main_program_file) as f: - return f.read() - @classmethod def run_default_benchmark(cls, save=False): logger = logging.getLogger('golem.task.benchmarkmanager') diff --git a/golem/resource/dirmanager.py b/golem/resource/dirmanager.py index f71f5183bb..fcbb8f56b6 100644 --- a/golem/resource/dirmanager.py +++ b/golem/resource/dirmanager.py @@ -47,15 +47,6 @@ def split_path(path): return split_path(head) + [tail] -def find_task_script(task_dir, script_name): - scripts_path = os.path.abspath(os.path.join(task_dir, "resources", "scripts")) - script_file = os.path.join(scripts_path, script_name) - if os.path.isfile(script_file): - return script_file - - logger.error("Script file {} does not exist!".format(script_file)) - - def list_dir_recursive(dir: str) -> Iterator[str]: for dirpath, dirnames, filenames in os.walk(dir, followlinks=True): for name in filenames: diff --git a/golem/task/benchmarkmanager.py b/golem/task/benchmarkmanager.py index 6def7a81d7..9e2eba84b3 100644 --- a/golem/task/benchmarkmanager.py +++ b/golem/task/benchmarkmanager.py @@ -98,11 +98,6 @@ def on_success(performance): @staticmethod def _validate_task_state(task_state): - td = task_state.definition - if not os.path.exists(td.main_program_file): - logger.error("Main program file does not exist: {}".format( - td.main_program_file)) - return False return True def run_benchmark_for_env_id(self, env_id, callback, errback): diff --git a/golem/task/localcomputer.py b/golem/task/localcomputer.py index 07071b7404..dc859f4d70 100644 --- a/golem/task/localcomputer.py +++ b/golem/task/localcomputer.py @@ -63,7 +63,6 @@ def run(self) -> None: self.start_time = time.time() self._prepare_tmp_dir() self._prepare_resources(self.resources) # makes a copy - if not self.compute_task_def: ctd = self.get_compute_task_def() else: @@ -186,7 +185,6 @@ def _get_task_thread(self, ctd: ComputeTaskDef) -> DockerTaskThread: ) return DockerTaskThread( ctd['docker_images'], - ctd['src_code'], ctd['extra_data'], dir_mapping, 0, diff --git a/golem/task/taskbase.py b/golem/task/taskbase.py index 089970afa0..27b40e3001 100644 --- a/golem/task/taskbase.py +++ b/golem/task/taskbase.py @@ -95,9 +95,7 @@ def __init__(self, ctd=None, **kwargs): def __init__(self, header: dt_tasks.TaskHeader, - src_code: str, task_definition: TaskDefinition) -> None: - self.src_code = src_code self.header = header self.task_definition = task_definition diff --git a/golem/task/taskcomputer.py b/golem/task/taskcomputer.py index e63fad5673..565718a904 100644 --- a/golem/task/taskcomputer.py +++ b/golem/task/taskcomputer.py @@ -121,7 +121,6 @@ def task_resource_collected(self, task_id, unpack_delta=True): self.__compute_task( subtask['subtask_id'], subtask['docker_images'], - subtask['src_code'], subtask['extra_data'], subtask['deadline']) return True @@ -342,7 +341,7 @@ def __request_resource(self, task_id, subtask_id): self.task_server.request_resource(task_id, subtask_id) def __compute_task(self, subtask_id, docker_images, - src_code, extra_data, subtask_deadline): + extra_data, subtask_deadline): task_id = self.assigned_subtask['task_id'] task_header = self.task_server.task_keeper.task_headers.get(task_id) @@ -374,12 +373,10 @@ def __compute_task(self, subtask_id, docker_images, docker_images = [DockerImage(**did) for did in docker_images] dir_mapping = DockerTaskThread.generate_dir_mapping(resource_dir, temp_dir) - tt = DockerTaskThread(docker_images, - src_code, extra_data, + tt = DockerTaskThread(docker_images, extra_data, dir_mapping, task_timeout) elif self.support_direct_computation: - tt = PyTaskThread(src_code, - extra_data, resource_dir, temp_dir, + tt = PyTaskThread(extra_data, resource_dir, temp_dir, task_timeout) else: logger.error("Cannot run PyTaskThread in this version") @@ -429,17 +426,15 @@ def __init__(self, src_code, extra_data, owner_address, owner_port): class PyTaskThread(TaskThread): # pylint: disable=too-many-arguments - def __init__(self, src_code, - extra_data, res_path, tmp_path, timeout): + def __init__(self, extra_data, res_path, tmp_path, timeout): super(PyTaskThread, self).__init__( - src_code, extra_data, res_path, tmp_path, timeout) + extra_data, res_path, tmp_path, timeout) self.vm = PythonProcVM() class PyTestTaskThread(PyTaskThread): # pylint: disable=too-many-arguments - def __init__(self, src_code, - extra_data, res_path, tmp_path, timeout): + def __init__(self, extra_data, res_path, tmp_path, timeout): super(PyTestTaskThread, self).__init__( - src_code, extra_data, res_path, tmp_path, timeout) + extra_data, res_path, tmp_path, timeout) self.vm = PythonTestVM() diff --git a/golem/task/tasksession.py b/golem/task/tasksession.py index 6efcdf06b5..fe58a97823 100644 --- a/golem/task/tasksession.py +++ b/golem/task/tasksession.py @@ -1090,13 +1090,6 @@ def _set_env_params(self, ctd): if not self.__check_docker_images(ctd, env): return False - if not env.allow_custom_main_program_file: - ctd['src_code'] = env.get_source_code() - - if not ctd['src_code']: - self.err_msg = reasons.NoSourceCode - return False - return True def __check_docker_images(self, diff --git a/golem/task/taskthread.py b/golem/task/taskthread.py index 9efd052273..b04eb087b3 100644 --- a/golem/task/taskthread.py +++ b/golem/task/taskthread.py @@ -24,7 +24,6 @@ class TaskThread(threading.Thread): # pylint:disable=too-many-arguments def __init__(self, - src_code: str, extra_data: Dict, res_path: str, tmp_path: str, @@ -32,7 +31,6 @@ def __init__(self, super(TaskThread, self).__init__() self.vm = None - self.src_code = src_code self.extra_data = extra_data self.result = None self.done = False @@ -118,7 +116,7 @@ def __do_work(self): extra_data["resourcePath"] = abs_res_path extra_data["tmp_path"] = abs_tmp_path self.result, self.error_msg = self.vm.run_task( - self.src_code, + extra_data['src_code'], extra_data ) finally: diff --git a/golem/verificator/blender_verifier.py b/golem/verificator/blender_verifier.py index 9ba7e09569..da2435dd80 100644 --- a/golem/verificator/blender_verifier.py +++ b/golem/verificator/blender_verifier.py @@ -9,7 +9,6 @@ from threading import Lock from shutil import copy -from golem.core.common import get_golem_path from golem.verificator.verifier import SubtaskVerificationState from .rendering_verifier import FrameRenderingVerifier @@ -22,7 +21,7 @@ # pylint: disable=R0902 class BlenderVerifier(FrameRenderingVerifier): DOCKER_NAME = "golemfactory/image_metrics" - DOCKER_TAG = '1.8' + DOCKER_TAG = '1.11' def __init__(self, verification_data, cropper_cls: Type, docker_task_cls: Type) -> None: @@ -31,14 +30,6 @@ def __init__(self, verification_data, cropper_cls: Type, self.verified_crops_counter = 0 self.finished = Deferred() self.current_results_files = None - self.program_file = os.path.join(get_golem_path(), - 'golem', - 'verificator', - 'docker', - 'blender', - 'images', - 'scripts', - 'runner.py') self.already_called = False self.cropper = cropper_cls() self.docker_task_cls = docker_task_cls @@ -141,9 +132,6 @@ def _crop_rendered(self, result): logger.info("Crop no [%r] rendered for verification. Time spent: %r.", crop_number, time_spend) - with open(self.program_file, "r") as src_file: - src_code = src_file.read() - work_dir = verification_context.get_crop_path( str(crop_number)) if not work_dir: @@ -162,9 +150,7 @@ def _crop_rendered(self, result): crop_number, dir_mapping) self.docker_task = self.docker_task_cls( - subtask_id=self.subtask_info['subtask_id'], docker_images=[(self.DOCKER_NAME, self.DOCKER_TAG)], - src_code=src_code, extra_data=extra_data, dir_mapping=dir_mapping, timeout=self.timeout) @@ -175,7 +161,6 @@ def error(e): self.docker_task.run() self.docker_task._deferred.addErrback(error) - was_failure = self.docker_task.error self.metrics[crop_number] = dict() for root, _, files in os.walk(str(dir_mapping.output)): @@ -225,6 +210,7 @@ def create_extra_data(self, results, verification_context, crop_number, verification_files=verification_pairs, xres=x, yres=y, + script_filepath="/golem/scripts/runner.py", ) def make_verdict(self, result): diff --git a/golem/verificator/docker/blender/images/scripts/runner.py b/golem/verificator/docker/blender/images/scripts/runner.py index 0b6f70ec45..e257ef2057 100644 --- a/golem/verificator/docker/blender/images/scripts/runner.py +++ b/golem/verificator/docker/blender/images/scripts/runner.py @@ -45,12 +45,8 @@ def run_img_compare_task(verification_files, xres, yres): file=sys.stderr) sys.exit(1) - dir_path = os.path.dirname(os.path.realpath(__file__)) - results_path = os.path.join(dir_path, params['OUTPUT_DIR'][1:]) file_path = os.path.join( params['OUTPUT_DIR'], 'result_' + str(counter) + '.txt') - if not os.path.exists(results_path): - os.makedirs(results_path) results_path = img_metrics_calculator.\ calculate_metrics(cropped_img_path, diff --git a/scripts/pyinstaller/hooks/hook-golem.py b/scripts/pyinstaller/hooks/hook-golem.py index d1515cead1..44ae45eed9 100644 --- a/scripts/pyinstaller/hooks/hook-golem.py +++ b/scripts/pyinstaller/hooks/hook-golem.py @@ -21,10 +21,6 @@ 'apps/blender/resources/images/'), ('apps/blender/resources/images/scripts/templates/blendercrop.py.template', 'apps/blender/resources/scripts/'), - ('apps/blender/resources/scripts/docker_blendertask.py', - 'apps/blender/resources/scripts/'), - ('apps/dummy/resources/scripts/docker_dummytask.py', - 'apps/dummy/resources/scripts/'), ('apps/dummy/resources/code_dir/computing.py', 'apps/dummy/resources/code_dir/'), ('apps/dummy/test_data/in.data', diff --git a/setup.py b/setup.py index 487b04331f..3ef209d872 100755 --- a/setup.py +++ b/setup.py @@ -83,17 +83,11 @@ (path.normpath('../../golem/apps/rendering/benchmark/minilight'), [ path.normpath('apps/rendering/benchmark/minilight/cornellbox.ml.txt'), ]), - (path.normpath('../../golem/apps/blender/resources/scripts'), [ - path.normpath('apps/blender/resources/scripts/docker_blendertask.py') - ]), (path.normpath( '../../golem/apps/blender/resources/images/scripts/templates'), [ path.normpath('apps/blender/resources/images/scripts/' 'templates/blendercrop.py.template')] ), - (path.normpath('../../golem/apps/dummy/resources/scripts'), [ - path.normpath('apps/dummy/resources/scripts/docker_dummytask.py') - ]), (path.normpath('../../golem/apps/dummy/resources/code_dir'), [ path.normpath('apps/dummy/resources/code_dir/computing.py') ]), diff --git a/tests/apps/blender/benchmark/test_blenderbenchmark.py b/tests/apps/blender/benchmark/test_blenderbenchmark.py index e2d17f121e..4568bd6d8e 100644 --- a/tests/apps/blender/benchmark/test_blenderbenchmark.py +++ b/tests/apps/blender/benchmark/test_blenderbenchmark.py @@ -39,9 +39,6 @@ def test_task_settings(self): self.assertTrue( os.path.isfile(self.bb.task_definition.main_scene_file) ) - self.assertTrue( - os.path.isfile(self.bb.task_definition.main_program_file) - ) @ci_skip diff --git a/tests/apps/blender/task/test_blenderrendertask.py b/tests/apps/blender/task/test_blenderrendertask.py index 95eb5f0bae..10e7b56815 100644 --- a/tests/apps/blender/task/test_blenderrendertask.py +++ b/tests/apps/blender/task/test_blenderrendertask.py @@ -16,8 +16,7 @@ import OpenEXR -from apps.blender.task.blenderrendertask import (BlenderDefaults, - BlenderRenderTask, +from apps.blender.task.blenderrendertask import (BlenderRenderTask, BlenderRenderTaskBuilder, BlenderRendererOptions, generate_expected_offsets, @@ -35,13 +34,6 @@ from golem.tools.assertlogs import LogTestCase -class TestBlenderDefaults(unittest.TestCase): - - def test_init(self): - bd = BlenderDefaults() - self.assertTrue(path.isfile(bd.main_program_file)) - - class BlenderTaskInitTest(TempDirFixture, LogTestCase): def test_compositing(self): diff --git a/tests/apps/blender/test_blenderenvironment.py b/tests/apps/blender/test_blenderenvironment.py index cd4f3cca16..7a716246f0 100644 --- a/tests/apps/blender/test_blenderenvironment.py +++ b/tests/apps/blender/test_blenderenvironment.py @@ -1,5 +1,3 @@ -from os import path - from apps.blender.blenderenvironment import BlenderEnvironment from golem.environments.minperformancemultiplier import MinPerformanceMultiplier @@ -47,6 +45,3 @@ def test_get_min_accepted_performance(self): # then self.assertEqual(MinPerformanceMultiplier.get(), 3.141) self.assertEqual(self.env.get_min_accepted_performance(), 314.1) - - def test_main_program_file(self): - assert path.isfile(BlenderEnvironment().main_program_file) diff --git a/tests/apps/blender/verification/test_verificator_integration.py b/tests/apps/blender/verification/test_verificator_integration.py index d8c99e9c3f..a6ff15fc58 100644 --- a/tests/apps/blender/verification/test_verificator_integration.py +++ b/tests/apps/blender/verification/test_verificator_integration.py @@ -60,6 +60,7 @@ def setUp(self): self.subtask_info['all_frames'] = [1] self.subtask_info['tmp_dir'] = self.tempdir self.subtask_info['subtask_timeout'] = 600 + self.subtask_info['script_filepath'] = '/golem/scripts/job.py' self.subtask_info['path_root'] = os.path.dirname(self.resources[0]) self.subtask_info['parts'] = 1 @@ -67,7 +68,7 @@ def setUp(self): self.subtask_info['ctd'] = dict() self.subtask_info['ctd']['deadline'] = time.time() + 3600 self.subtask_info['ctd']['docker_images'] = [DockerImage( - 'golemfactory/blender', tag='1.5').to_dict()] + 'golemfactory/blender', tag='1.7').to_dict()] self.subtask_info['ctd']['extra_data'] = dict() self.subtask_info['ctd']['extra_data']['scene_file'] = \ self.subtask_info['scene_file'] @@ -89,12 +90,9 @@ def setUp(self): self.subtask_info['crops'] self.subtask_info['ctd']['extra_data']['path_root'] = \ self.subtask_info['path_root'] + self.subtask_info['ctd']['extra_data']['script_filepath'] = \ + self.subtask_info['script_filepath'] self.subtask_info['ctd']['short_description'] = '' - self.subtask_info['ctd']['src_code'] = open( - os.path.join( - self.golem_dir, - 'apps/blender/resources/scripts/docker_blendertask.py'), - 'r').read() self.subtask_info['ctd']['subtask_id'] = self.subtask_info['subtask_id'] def test_bad_image(self): diff --git a/tests/apps/core/benchmark/test_benchmarkrunner.py b/tests/apps/core/benchmark/test_benchmarkrunner.py index 505a64e3f2..6244289694 100644 --- a/tests/apps/core/benchmark/test_benchmarkrunner.py +++ b/tests/apps/core/benchmark/test_benchmarkrunner.py @@ -91,7 +91,7 @@ def setUp(self): super().setUp() self.benchmark = mock.MagicMock() self.instance = benchmarkrunner.BenchmarkRunner( - task=DummyTask(None, None, None), + task=DummyTask(None, None), root_path=self.tempdir, success_callback=self._success, error_callback=self._error, @@ -269,7 +269,7 @@ class BenchmarkRunnerWrongTaskTest(TempDirFixture): def test_run_with_error(self): benchmark = mock.MagicMock() instance = benchmarkrunner.BenchmarkRunner( - task=WrongTask(None, None, None), + task=WrongTask(None, None), root_path=self.tempdir, success_callback=mock.Mock(), error_callback=mock.Mock(), diff --git a/tests/apps/core/task/test_coretask.py b/tests/apps/core/task/test_coretask.py index 970a053aef..dd6b22974c 100644 --- a/tests/apps/core/task/test_coretask.py +++ b/tests/apps/core/task/test_coretask.py @@ -80,22 +80,6 @@ def query_extra_data_for_test_task(self): task = CoreTaskDeabstractedEnv(task_def, node) self.assertIsInstance(task, CoreTask) - def test_init(self): - task_def = TestCoreTask._get_core_task_definition() - - class CoreTaskWrongFile(self.CoreTaskDeabstracted): - ENVIRONMENT_CLASS = env_with_file - - with patch("logging.Logger.warning") as log_mock: - task = CoreTaskWrongFile( - task_definition=task_def, - owner=dt_p2p_factory.Node(), - resource_size=1024 - ) - log_mock.assert_called_once() - self.assertIn("Wrong main program file", log_mock.call_args[0][0]) - self.assertEqual(task.src_code, "") - def _get_core_task(self): task_def = TestCoreTask._get_core_task_definition() task = self.CoreTaskDeabstracted( @@ -476,7 +460,6 @@ def test_new_compute_task_def(self): assert ctd['task_id'] == c.header.task_id assert ctd['subtask_id'] == hash assert ctd['extra_data'] == extra_data - assert ctd['src_code'] == c.src_code assert ctd['performance'] == perf_index assert ctd['docker_images'] == c.docker_images diff --git a/tests/apps/core/task/test_coretaskstate.py b/tests/apps/core/task/test_coretaskstate.py index 7e75449780..6a185ac6e6 100644 --- a/tests/apps/core/task/test_coretaskstate.py +++ b/tests/apps/core/task/test_coretaskstate.py @@ -23,7 +23,6 @@ class TestCoreTaskDefautls(TestCase): def test_init(self): defaults = TaskDefaults() assert defaults.output_format == "" - assert defaults.main_program_file == "" assert defaults.timeout == 4 * 3600 assert defaults.subtask_timeout == 20 * 60 assert defaults.min_subtasks == 1 diff --git a/tests/apps/dummy/benchmark/test_dummybenchmark.py b/tests/apps/dummy/benchmark/test_dummybenchmark.py index 5319e1e7fa..32585bbe81 100644 --- a/tests/apps/dummy/benchmark/test_dummybenchmark.py +++ b/tests/apps/dummy/benchmark/test_dummybenchmark.py @@ -24,8 +24,6 @@ def test_task_settings(self): self.assertTrue(all(os.path.isfile(x) for x in self.db.task_definition.shared_data_files)) - self.assertTrue(os.path.isfile( - self.db.task_definition.main_program_file)) self.assertEquals(self.db.task_definition.options.difficulty, 0xffff0000) diff --git a/tests/apps/dummy/test_dummyenvironment.py b/tests/apps/dummy/test_dummyenvironment.py index 350a8c6ee6..4efe21e4b4 100644 --- a/tests/apps/dummy/test_dummyenvironment.py +++ b/tests/apps/dummy/test_dummyenvironment.py @@ -1,5 +1,3 @@ -from os import path - from apps.dummy.dummyenvironment import DummyTaskEnvironment from golem.environments.minperformancemultiplier import MinPerformanceMultiplier from golem.model import Performance @@ -39,6 +37,3 @@ def test_get_min_accepted_performance(self): # then self.assertEqual(MinPerformanceMultiplier.get(), 3.141) self.assertEqual(self.env.get_min_accepted_performance(), 314.1) - - def test_main_program_file(self): - assert path.isfile(DummyTaskEnvironment().main_program_file) diff --git a/tests/apps/rendering/task/test_renderingtask.py b/tests/apps/rendering/task/test_renderingtask.py index 7770760ab4..f6f08190f7 100644 --- a/tests/apps/rendering/task/test_renderingtask.py +++ b/tests/apps/rendering/task/test_renderingtask.py @@ -54,19 +54,6 @@ def query_extra_data_for_test_task(self): pass -class TestInitRenderingTask(TestDirFixture, LogTestCase): - def test_init(self): - with self.assertLogs(logger_core, level="WARNING"): - rt = RenderingTaskMock(main_program_file="notexisting", - task_definition=RenderingTaskDefinition(), - owner=dt_p2p_factory.Node(), - total_tasks=10, - root_path=self.path - ) - assert isinstance(rt, RenderingTask) - assert rt.src_code == "" - - class TestRenderingTask(TestDirFixture, LogTestCase): def setUp(self): super(TestRenderingTask, self).setUp() diff --git a/tests/golem/docker/docker-blender-cycles-task.json b/tests/golem/docker/docker-blender-cycles-task.json index a843dddad4..adcc2fd606 100644 --- a/tests/golem/docker/docker-blender-cycles-task.json +++ b/tests/golem/docker/docker-blender-cycles-task.json @@ -11,8 +11,8 @@ { "py/object": "golem.docker.image.DockerImage", "repository": "golemfactory/blender", - "tag": "1.5", - "name": "golemfactory/blender:1.5", + "tag": "1.7", + "name": "golemfactory/blender:1.7", "id": null } ], @@ -49,7 +49,6 @@ "estimated_memory":0, "timeout":14400, "task_type":"Blender", - "main_program_file":"$GOLEM_DIR/apps/blender/resources/scripts/docker_blendertask.py", "subtask_timeout":1200, "subtasks_count":6, "max_price": 12, @@ -57,8 +56,8 @@ { "py/object": "golem.docker.image.DockerImage", "repository": "golemfactory/blender", - "tag": "1.5", - "name": "golemfactory/blender:1.5", + "tag": "1.7", + "name": "golemfactory/blender:1.7", "id": null } ], diff --git a/tests/golem/docker/docker-blender-render-task.json b/tests/golem/docker/docker-blender-render-task.json index b487cc153d..3fc2e2a4f3 100644 --- a/tests/golem/docker/docker-blender-render-task.json +++ b/tests/golem/docker/docker-blender-render-task.json @@ -11,10 +11,10 @@ "docker_images":[ { "py/object":"golem.docker.image.DockerImage", - "tag":"1.5", + "tag":"1.7", "id":null, "repository":"golemfactory/blender", - "name":"golemfactory/blender:1.5" + "name":"golemfactory/blender:1.7" } ], "caps":[], @@ -44,10 +44,10 @@ "docker_images":[ { "py/object":"golem.docker.image.DockerImage", - "tag":"1.5", + "tag":"1.7", "id":null, "repository":"golemfactory/blender", - "name":"golemfactory/blender:1.5" + "name":"golemfactory/blender:1.7" } ], "resolution":[ @@ -63,7 +63,6 @@ "timeout":14400, "max_price": 13, "task_type":"Blender", - "main_program_file":"$GOLEM_DIR/apps/blender/resources/scripts/docker_blendertask.py", "subtask_timeout":1200, "subtasks_count":6, "concent_enabled": false diff --git a/tests/golem/docker/docker-dummy-test-task.json b/tests/golem/docker/docker-dummy-test-task.json index e2d634f455..0011d7f736 100644 --- a/tests/golem/docker/docker-dummy-test-task.json +++ b/tests/golem/docker/docker-dummy-test-task.json @@ -7,9 +7,9 @@ "docker_images":[ { "py/object": "golem.docker.image.DockerImage", - "repository": "golemfactory/base", - "tag": "1.3", - "name": "golemfactory/base:1.3", + "repository": "golemfactory/dummy", + "tag": "1.0", + "name": "golemfactory/dummy:1.0", "id": null } ], @@ -43,16 +43,15 @@ "estimated_memory":0, "timeout":14400, "task_type":"Dummy", - "main_program_file":"$GOLEM_DIR/apps/dummy/resources/scripts/docker_dummytask.py", "subtask_timeout":1200, "subtasks_count":5, "max_price": 11, "docker_images":[ { "py/object": "golem.docker.image.DockerImage", - "repository": "golemfactory/base", - "tag": "1.3", - "name": "golemfactory/base:1.3", + "repository": "golemfactory/dummy", + "tag": "1.0", + "name": "golemfactory/dummy:1.0", "id": null } ], diff --git a/tests/golem/docker/test_blender_job.py b/tests/golem/docker/test_blender_job.py index a907c95ed0..d32976f5d1 100644 --- a/tests/golem/docker/test_blender_job.py +++ b/tests/golem/docker/test_blender_job.py @@ -4,7 +4,6 @@ from golem.core.common import get_golem_path from golem.docker.job import DockerJob -from golem.resource.dirmanager import find_task_script from .test_docker_job import TestDockerJob @@ -15,14 +14,9 @@ def _get_test_repository(self): return "golemfactory/blender" def _get_test_tag(self): - return "1.5" + return "1.7" def test_blender_job(self): - app_dir = os.path.join(get_golem_path(), "apps", "blender") - task_script = find_task_script(app_dir, "docker_blendertask.py") - with open(task_script) as f: - task_script_src = f.read() - # copy the scene file to the resources dir scene_file = pathlib.Path(get_golem_path()) scene_file /= "apps/blender/benchmark/test_task/cube.blend" @@ -49,8 +43,8 @@ def test_blender_job(self): "crops": crops } - with self._create_test_job(script=task_script_src, params=params) \ - as job: + with self._create_test_job( + script="/golem/scripts/job.py", params=params) as job: job.start() exit_code = job.wait(timeout=300) self.assertEqual(exit_code, 0) diff --git a/tests/golem/docker/test_docker_blender_task.py b/tests/golem/docker/test_docker_blender_task.py index 1c0f80af6b..167e770663 100644 --- a/tests/golem/docker/test_docker_blender_task.py +++ b/tests/golem/docker/test_docker_blender_task.py @@ -98,7 +98,6 @@ def test_build(self): assert isinstance(task.preview_file_path, str) assert not task.preview_updaters assert task.scale_factor == 0.8 - assert task.src_code assert isinstance(task.header, dt_tasks.TaskHeader) assert task.header.task_id == '7220aa01-ad45-4fb4-b199-ba72b37a1f0c' assert task.header.task_owner.key == 'dd72b37a1f0c' @@ -111,7 +110,7 @@ def test_build(self): assert task.header.environment == 'BLENDER' assert task.header.estimated_memory == 0 assert task.docker_images[0].repository == 'golemfactory/blender' - assert task.docker_images[0].tag == '1.5' + assert task.docker_images[0].tag == '1.7' assert task.header.max_price == 12 assert not task.header.signature assert task.listeners == [] @@ -161,38 +160,11 @@ def test_wrong_image_id_specified(self): self.assertIsInstance(task_thread.error_msg, str) self.assertTrue(task_thread.error_msg) - def test_blender_subtask_script_error(self): - task = self._get_test_task() - # Replace the main script source with another script that will - # produce errors when run in the task environment: - task.src_code = 'main :: IO()\nmain = putStrLn "Hello, Haskell World"\n' - task.main_program_file = path.join( - path.join(get_golem_path(), "golem"), "node.py") - task.task_resources = {task.main_program_file, task.main_scene_file} - task_thread = self._run_task(task) - self.assertIsInstance(task_thread, DockerTaskThread) - self.assertIsInstance(task_thread.error_msg, str) - self.assertTrue( - task_thread.error_msg.startswith("Subtask computation failed")) - - def test_subtask_killed(self): - task = self._get_test_task() - # Replace the main script source with another script that will - # kill itself - task.src_code = \ - 'import os; import signal; os.kill(os.getpid(), signal.SIGKILL)' - task.main_program_file = path.join( - path.join(get_golem_path(), "golem"), "node.py") - task.task_resources = {task.main_program_file, task.main_scene_file} - task_thread = self._run_task(task) - self.assertIsInstance(task_thread, DockerTaskThread) - self.assertIsInstance(task_thread.error_msg, str) - self.assertIn("out-of-memory", task_thread.error_msg) - def test_blender_scene_file_error(self): task = self._get_test_task() # Replace scene file with some other, non-blender file: - task.main_scene_file = task.main_program_file + task.main_scene_file = path.join( + path.join(get_golem_path(), "golem"), "node.py") task_thread = self._run_task(task) self.assertIsInstance(task_thread, DockerTaskThread) self.assertIsInstance(task_thread.error_msg, str) diff --git a/tests/golem/docker/test_docker_environment.py b/tests/golem/docker/test_docker_environment.py index 9b7ef0a758..da539b4e26 100644 --- a/tests/golem/docker/test_docker_environment.py +++ b/tests/golem/docker/test_docker_environment.py @@ -7,12 +7,11 @@ from .test_docker_image import DockerTestCase + class DockerEnvironmentMock(DockerEnvironment): DOCKER_IMAGE = "" DOCKER_TAG = "" ENV_ID = "" - APP_DIR = "" - SCRIPT_NAME = "" SHORT_DESCRIPTION = "" @@ -26,7 +25,7 @@ def test_docker_environment(self): DockerEnvironmentMock(additional_images=["aaa"]) de = DockerEnvironmentMock(additional_images=[ - DockerImage("golemfactory/blender", tag="1.5")]) + DockerImage("golemfactory/blender", tag="1.7")]) self.assertTrue(de.check_support()) self.assertTrue(de.check_docker_images()) diff --git a/tests/golem/docker/test_docker_job.py b/tests/golem/docker/test_docker_job.py index e8626cb9ae..d77262b29e 100644 --- a/tests/golem/docker/test_docker_job.py +++ b/tests/golem/docker/test_docker_job.py @@ -62,7 +62,6 @@ def testDockerJobInit(self): job = DockerJob(self.image, self.TEST_SCRIPT, None, self.resources_dir, self.work_dir, self.output_dir) self.assertEqual(job.image, self.image) - self.assertEqual(job.script_src, self.TEST_SCRIPT) parameters = {'OUTPUT_DIR': '/golem/output', 'RESOURCES_DIR': '/golem/resources', @@ -96,7 +95,7 @@ def tearDown(self): def _create_test_job(self, script=TEST_SCRIPT, params=None): self.test_job = DockerJob( image=self.image, - script_src=script, + script_filepath=script, parameters=params, resources_dir=self.resources_dir, work_dir=self.work_dir, @@ -140,7 +139,6 @@ def test_create(self): self.assertIsNotNone(job.resources_dir) self.assertIsNotNone(job.output_dir) self.assertTrue(job._get_host_params_path().startswith(job.work_dir)) - self.assertTrue(job._get_host_script_path().startswith(job.work_dir)) def _load_dict(self, path): with open(path, 'r') as f: @@ -161,20 +159,6 @@ def test_params_saved_nonascii(self): # only in param values: self._test_params_saved({"length": "pięćdziesiąt łokci"}) - def _test_script_saved(self, task_script): - with self._create_test_job(script=task_script) as job: - script_path = job._get_host_script_path() - self.assertTrue(path.isfile(script_path)) - with open(script_path, 'rb') as f: - script = f.read().decode('utf-8') - self.assertEqual(task_script, script) - - def test_script_saved(self): - self._test_script_saved(TestDockerJob.TEST_SCRIPT) - - def test_script_saved_nonascii(self): - self._test_script_saved("print u'Halo? Świeci!'\n") - def test_container_created(self): with self._create_test_job() as job: self.assertIsNotNone(job.container_id) @@ -253,12 +237,13 @@ def test_start(self): self.assertIn("Path", info) self.assertEqual(info["Path"], "/usr/local/bin/entrypoint.sh") self.assertIn("Args", info) - self.assertEqual(info["Args"], [job._get_container_script_path()]) def test_logs_stdout(self): text = "Adventure Time!" src = "print('{}')\n".format(text) - with self._create_test_job(script=src) as job: + with open(path.join(self.resources_dir, "custom.py"), "w") as f: + f.write(src) + with self._create_test_job(script='/golem/resources/custom.py') as job: job.start() out_file = path.join(self.output_dir, "stdout.log") err_file = path.join(self.output_dir, "stderr.log") @@ -270,7 +255,7 @@ def test_logs_stdout(self): self.assertEqual(line, text) def test_logs_stderr(self): - with self._create_test_job(script="syntax error!@#$%!") as job: + with self._create_test_job(script="/non/existing") as job: job.start() err_file = path.join(self.output_dir, "stderr.log") job.dump_logs(stderr_file=err_file) @@ -278,22 +263,15 @@ def test_logs_stderr(self): self.assertEqual(out_files, ["stderr.log"]) with open(err_file, "r") as out: line = out.readline().strip() - text = 'File "{}"'.format(job._get_container_script_path()) - self.assertTrue(line.startswith(text)) - - def test_wait(self): - src = "import time\ntime.sleep(5)\n" - with self._create_test_job(script=src) as job: - job.start() - self.assertEqual(job.get_status(), DockerJob.STATE_RUNNING) - exit_code = job.wait() - self.assertEqual(exit_code, 0) - self.assertEqual(job.get_status(), DockerJob.STATE_EXITED) + self.assertTrue(line.startswith("python3: can't open file")) def test_wait_timeout(self): src = "import time\ntime.sleep(10)\n" + with open(path.join(self.resources_dir, "custom.py"), "w") as f: + f.write(src) with self.assertRaises(requests.exceptions.ConnectionError): - with self._create_test_job(script=src) as job: + with self._create_test_job(script='/golem/resources/custom.py') \ + as job: job.start() self.assertEqual(job.get_status(), DockerJob.STATE_RUNNING) job.wait(1) @@ -341,7 +319,9 @@ def test_logger_thread(self): def test_working_dir_set(self): script = "import os\nprint(os.getcwd())\n" - with self._create_test_job(script=script) as job: + with open(path.join(self.resources_dir, "custom.py"), "w") as f: + f.write(script) + with self._create_test_job(script='/golem/resources/custom.py') as job: job.start() job.wait() out_file = path.join(self.output_dir, "stdout.log") @@ -364,10 +344,12 @@ def test_copy_job(self): """ sample_text = "Adventure Time!\n" - with open(path.join(self.resources_dir, "in.txt"), "w") as input: - input.write(sample_text) + with open(path.join(self.resources_dir, "in.txt"), "w") as f: + f.write(sample_text) + with open(path.join(self.resources_dir, "copy.py"), "w") as f: + f.write(copy_script) - with self._create_test_job(script=copy_script) as job: + with self._create_test_job(script='/golem/resources/copy.py') as job: job.start() job.wait() diff --git a/tests/golem/docker/test_docker_manager.py b/tests/golem/docker/test_docker_manager.py index e7fbd4edaa..9e5e10cd5b 100644 --- a/tests/golem/docker/test_docker_manager.py +++ b/tests/golem/docker/test_docker_manager.py @@ -303,9 +303,9 @@ def command(key, *args, **kwargs): from apps.core import nvgpu if nvgpu.is_supported(): - expected = 5 + expected = 6 else: - expected = 3 + expected = 4 assert pulls[0] == expected @@ -332,9 +332,9 @@ def command(key, *args, **kwargs): from apps.core import nvgpu if nvgpu.is_supported(): - expected = 5 + expected = 6 else: - expected = 3 + expected = 4 assert builds[0] == expected assert tags[0] == expected diff --git a/tests/golem/docker/test_docker_task_thread.py b/tests/golem/docker/test_docker_task_thread.py index 6d033b3f7c..251a1c6046 100644 --- a/tests/golem/docker/test_docker_task_thread.py +++ b/tests/golem/docker/test_docker_task_thread.py @@ -23,8 +23,6 @@ def tearDown(self): TestWithDatabase.tearDown(self) def test_termination(self): - script = "import time\ntime.sleep(20)" - task_server = Mock() task_server.config_desc = ClientConfigDescriptor() task_server.client.datadir = self.test_dir @@ -40,16 +38,13 @@ def test_termination(self): with self.assertRaises(AttributeError): dir_mapping = DockerTaskThread.generate_dir_mapping( self.resources_dir, self.output_dir) - DockerTaskThread(None, - script, None, + DockerTaskThread(None, None, dir_mapping, timeout=30) def test(): dir_mapping = DockerTaskThread.generate_dir_mapping( self.resources_dir, self.output_dir) - tt = DockerTaskThread([image], - script, None, - "test task thread", dir_mapping, timeout=30) + tt = DockerTaskThread([image], None, dir_mapping, timeout=30) task_computer.counting_thread = tt task_computer.counting_task = True tt.setDaemon(True) diff --git a/tests/golem/docker/test_dummy_job.py b/tests/golem/docker/test_dummy_job.py index f8e7d1c200..ee0863433a 100644 --- a/tests/golem/docker/test_dummy_job.py +++ b/tests/golem/docker/test_dummy_job.py @@ -3,7 +3,6 @@ from os import path from golem.core.common import get_golem_path -from golem.resource.dirmanager import find_task_script from golem.tools.ci import ci_skip from .test_docker_job import TestDockerJob @@ -13,18 +12,12 @@ class TestDummyTaskDockerJob(TestDockerJob): """Tests for Docker image golem/base""" def _get_test_repository(self): - return "golemfactory/base" + return "golemfactory/dummy" def _get_test_tag(self): - return "1.3" + return "1.0" def test_dummytask_job(self): - app_dir = path.join(get_golem_path(), "apps", "dummy") - task_script = find_task_script(app_dir, "docker_dummytask.py") - - with open(task_script) as f: - task_script_src = f.read() - os.mkdir(os.path.join(self.resources_dir, "data")) os.mkdir(os.path.join(self.resources_dir, "code")) @@ -59,7 +52,8 @@ def test_dummytask_job(self): "result_file": "out.result", } - with self._create_test_job(script=task_script_src, params=params) as job: + with self._create_test_job( + script='/golem/scripts/job.py', params=params) as job: job.start() exit_code = job.wait() self.assertEqual(exit_code, 0) diff --git a/tests/golem/environments/test_environment.py b/tests/golem/environments/test_environment.py index efb6d1223b..4d613a260e 100644 --- a/tests/golem/environments/test_environment.py +++ b/tests/golem/environments/test_environment.py @@ -1,5 +1,3 @@ -from os import path - from golem.environments.minperformancemultiplier import MinPerformanceMultiplier from golem.testutils import DatabaseFixture @@ -25,25 +23,6 @@ def test_get_performance(self): # then self.assertEqual(self.env.get_performance(), perf_value) - def test_get_source_code(self): - # check defaults - assert self.env.get_source_code() is None - - # given - file_name = path.join(self.path, "mainprogramfile") - self.env.main_program_file = file_name - - # then - assert self.env.get_source_code() is None - - # re-given - with open(file_name, 'w') as f: - f.write("PROGRAM CODE") - - # then - self.env.main_program_file = file_name - assert self.env.get_source_code() == "PROGRAM CODE" - def test_run_default_benchmark(self): assert Environment.get_performance() == 0.0 assert Environment.run_default_benchmark(save=True) > 0.0 diff --git a/tests/golem/resource/test_dirmanager.py b/tests/golem/resource/test_dirmanager.py index 261241f4b0..71d611224e 100644 --- a/tests/golem/resource/test_dirmanager.py +++ b/tests/golem/resource/test_dirmanager.py @@ -5,8 +5,7 @@ from golem.core.common import is_linux, is_osx from golem.resource.dirmanager import symlink_or_copy, DirManager, \ - find_task_script, logger, list_dir_recursive -from golem.tools.assertlogs import LogTestCase + list_dir_recursive from golem.testutils import TempDirFixture @@ -58,7 +57,6 @@ class TestDirManager(TempDirFixture): def testInit(self): self.assertIsNotNone(DirManager(self.path)) - def test_getFileExtension(self): dm = DirManager(self.path) path = 'some/long/path/to/somefile.abc' @@ -264,20 +262,6 @@ def testClearOutput(self): self.assertFalse(os.path.isdir(dir1)) -class TestFindTaskScript(TempDirFixture, LogTestCase): - def test_find_task_script(self): - script_path = os.path.join(self.path, "resources", "scripts") - os.makedirs(script_path) - script = os.path.join(script_path, "bla") - open(script, "w").close() - task_file = os.path.join(self.path, "task", "testtask.py") - path = find_task_script(self.path, "bla") - self.assertTrue(os.path.isdir(os.path.dirname(path))) - self.assertEqual(os.path.basename(path), "bla") - with self.assertLogs(logger, level="ERROR"): - find_task_script(self.path, "notexisting") - - class TestUtilityFunction(TempDirFixture): def test_ls_r(self): os.makedirs(os.path.join(self.tempdir, "aa", "bb", "cc")) diff --git a/tests/golem/task/dummy/runner.py b/tests/golem/task/dummy/runner.py index f84beba96f..1ac7d24fb3 100644 --- a/tests/golem/task/dummy/runner.py +++ b/tests/golem/task/dummy/runner.py @@ -43,10 +43,6 @@ class DummyEnvironment(Environment): def get_id(cls): return DummyTask.ENVIRONMENT_NAME - def __init__(self): - super(DummyEnvironment, self).__init__() - self.allow_custom_main_program_file = True - def format_msg(kind, pid, msg): return "[{} {:>5}] {}".format(kind, pid, msg) diff --git a/tests/golem/task/dummy/task.py b/tests/golem/task/dummy/task.py index cb7fc05d2f..cdbcf13ccf 100644 --- a/tests/golem/task/dummy/task.py +++ b/tests/golem/task/dummy/task.py @@ -90,14 +90,14 @@ def __init__(self, client_id, params, num_subtasks, public_key): # load the script to be run remotely from the file in the current dir script_path = path.join(path.dirname(__file__), 'computation.py') with open(script_path, 'r') as f: - src_code = f.read() - src_code += '\noutput = run_dummy_task(' \ + self.src_code = f.read() + self.src_code += '\noutput = run_dummy_task(' \ 'data_file, subtask_data, difficulty, result_size, tmp_path)' from apps.dummy.task.dummytaskstate import DummyTaskDefinition from apps.dummy.task.dummytaskstate import DummyTaskDefaults task_definition = DummyTaskDefinition(DummyTaskDefaults()) - Task.__init__(self, header, src_code, task_definition) + Task.__init__(self, header, task_definition) self.task_id = task_id self.task_params = params @@ -182,14 +182,14 @@ def query_extra_data(self, perf_index: float, num_cores: int = 1, subtask_def = ComputeTaskDef() subtask_def['task_id'] = self.task_id subtask_def['subtask_id'] = subtask_id - subtask_def['src_code'] = self.src_code subtask_def['deadline'] = timeout_to_deadline(5 * 60) subtask_def['extra_data'] = { 'data_file': self.shared_data_file, 'subtask_data': self.subtask_data[subtask_id], 'difficulty': self.task_params.difficulty, 'result_size': self.task_params.result_size, - 'result_file': 'result.' + subtask_id[0:6] + 'result_file': 'result.' + subtask_id[0:6], + 'src_code': self.src_code, } return self.ExtraData(ctd=subtask_def) diff --git a/tests/golem/task/test_localcomputer.py b/tests/golem/task/test_localcomputer.py index dad1d93caf..184d529750 100644 --- a/tests/golem/task/test_localcomputer.py +++ b/tests/golem/task/test_localcomputer.py @@ -136,6 +136,8 @@ def _get_better_task_def(self): ctd['docker_images'] = [ di.to_dict() for di in BlenderEnvironment().docker_images ] + # Hack, we don't really need to execute any script here + ctd['extra_data']['script_filepath'] = '--version' return ctd def _success_callback(self, result, time_spent): diff --git a/tests/golem/task/test_taskcomputer.py b/tests/golem/task/test_taskcomputer.py index 0d41c9e538..d4a8d4eb06 100644 --- a/tests/golem/task/test_taskcomputer.py +++ b/tests/golem/task/test_taskcomputer.py @@ -96,12 +96,12 @@ def test_computation(self): # pylint: disable=too-many-statements ctd = ComputeTaskDef() ctd['task_id'] = "xyz" ctd['subtask_id'] = "xxyyzz" - ctd['src_code'] = \ + ctd['extra_data'] = {} + ctd['extra_data']['src_code'] = \ "cnt=0\n" \ "for i in range(10000):\n" \ "\tcnt += 1\n" \ "output={'data': cnt, 'result_type': 0}" - ctd['extra_data'] = {} ctd['deadline'] = timeout_to_deadline(10) task_server = self.task_server @@ -140,7 +140,7 @@ def test_computation(self): # pylint: disable=too-many-statements tc.task_given(ctd) assert tc.task_resource_collected("xyz") assert tc.counting_thread is not None - self.assertGreater(tc.counting_thread.time_to_compute, 9) + self.assertGreater(tc.counting_thread.time_to_compute, 8) self.assertLessEqual(tc.counting_thread.time_to_compute, 10) mock_finished.assert_called_once_with() mock_finished.reset_mock() @@ -159,7 +159,7 @@ def test_computation(self): # pylint: disable=too-many-statements mock_finished.reset_mock() ctd['subtask_id'] = "aabbcc" - ctd['src_code'] = "raise Exception('some exception')" + ctd['extra_data']['src_code'] = "raise Exception('some exception')" ctd['deadline'] = timeout_to_deadline(5) tc.task_given(ctd) self.assertEqual(tc.assigned_subtask, ctd) @@ -178,7 +178,7 @@ def test_computation(self): # pylint: disable=too-many-statements mock_finished.reset_mock() ctd['subtask_id'] = "aabbcc2" - ctd['src_code'] = "print('Hello world')" + ctd['extra_data']['src_code'] = "print('Hello world')" ctd['deadline'] = timeout_to_deadline(5) tc.task_given(ctd) self.assertTrue(tc.task_resource_collected("xyz")) @@ -192,7 +192,7 @@ def test_computation(self): # pylint: disable=too-many-statements task_server.task_keeper.task_headers["xyz"].deadline = \ timeout_to_deadline(20) ctd['subtask_id'] = "aabbcc3" - ctd['src_code'] = "output={'data': 0, 'result_type': 0}" + ctd['extra_data']['src_code'] = "output={'data': 0, 'result_type': 0}" ctd['deadline'] = timeout_to_deadline(40) tc.task_given(ctd) self.assertTrue(tc.task_resource_collected("xyz")) @@ -297,7 +297,6 @@ def test_compute_task(self, start): args = (task_computer, subtask_id) kwargs = dict( docker_images=[], - src_code='print("test")', extra_data=mock.Mock(), subtask_deadline=time.time() + 3600 ) @@ -369,8 +368,7 @@ def _new_task_thread(self, task_computer): output = cnt """ - return PyTaskThread(src_code=src_code, - extra_data={}, + return PyTaskThread(extra_data={'src_code': src_code}, res_path=os.path.dirname(files[0]), tmp_path=os.path.dirname(files[1]), timeout=20) diff --git a/tests/golem/task/test_taskmanager.py b/tests/golem/task/test_taskmanager.py index b77a268776..aa9efedcee 100644 --- a/tests/golem/task/test_taskmanager.py +++ b/tests/golem/task/test_taskmanager.py @@ -124,7 +124,7 @@ def _get_task_header(self, task_id, timeout, subtask_timeout): def _get_task_mock(self, task_id="xyz", subtask_id="xxyyzz", timeout=120, subtask_timeout=120): header = self._get_task_header(task_id, timeout, subtask_timeout) - task_mock = TaskMock(header, src_code='', task_definition=Mock()) + task_mock = TaskMock(header, task_definition=Mock()) task_mock.tmp_dir = self.path ctd = ComputeTaskDef() @@ -417,8 +417,8 @@ def test_computed_task_received(self, _): th.subtask_timeout = 1 class TestTask(Task): - def __init__(self, header, src_code, subtasks_id, verify_subtasks): - super(TestTask, self).__init__(header, src_code, Mock()) + def __init__(self, header, subtasks_id, verify_subtasks): + super(TestTask, self).__init__(header, Mock()) self.finished = {k: False for k in subtasks_id} self.restarted = {k: False for k in subtasks_id} self.verify_subtasks = verify_subtasks @@ -463,7 +463,7 @@ def should_accept_client(self, node_id): def accept_client(self, node_id): return AcceptClientVerdict.ACCEPTED - t = TestTask(th, "print 'Hello world'", ["xxyyzz"], + t = TestTask(th, ["xxyyzz"], verify_subtasks={"xxyyzz": True}) self.tm.add_new_task(t) self.tm.start_task(t.header.task_id) @@ -493,7 +493,7 @@ def accept_client(self, node_id): del handler th.task_id = "abc" - t2 = TestTask(th, "print 'Hello world'", ["aabbcc"], + t2 = TestTask(th, ["aabbcc"], verify_subtasks={"aabbcc": True}) self.tm.add_new_task(t2) self.tm.start_task(t2.header.task_id) @@ -521,7 +521,7 @@ def accept_client(self, node_id): del handler th.task_id = "qwe" - t3 = TestTask(th, "print 'Hello world!", ["qqwwee", "rrttyy"], + t3 = TestTask(th, ["qqwwee", "rrttyy"], {"qqwwee": True, "rrttyy": True}) self.tm.add_new_task(t3) self.tm.start_task(t3.header.task_id) @@ -549,7 +549,7 @@ def accept_client(self, node_id): del handler assert self.tm.verification_finished.call_count == 3 th.task_id = "task4" - t2 = TestTask(th, "print 'Hello world!", ["ttt4", "sss4"], + t2 = TestTask(th, ["ttt4", "sss4"], {'ttt4': False, 'sss4': True}) self.tm.add_new_task(t2) self.tm.start_task(t2.header.task_id) @@ -729,7 +729,6 @@ def test_resource_send(self): subtask_timeout=1, max_price=1, ), - src_code="print 'hello world'", task_definition=None, ) listener_mock = Mock() @@ -944,7 +943,6 @@ def test_update_signatures(self): subtask_timeout=1, max_price=1, ), - src_code='', task_definition=TaskDefinition()) self.tm.keys_auth = KeysAuth(self.path, 'priv_key', 'password') diff --git a/tests/golem/task/test_tasksession.py b/tests/golem/task/test_tasksession.py index 1e62e597cc..4c32e11063 100644 --- a/tests/golem/task/test_tasksession.py +++ b/tests/golem/task/test_tasksession.py @@ -51,8 +51,6 @@ class DockerEnvironmentMock(DockerEnvironment): DOCKER_IMAGE = "" DOCKER_TAG = "" ENV_ID = "" - APP_DIR = "" - SCRIPT_NAME = "" SHORT_DESCRIPTION = "" @@ -592,8 +590,6 @@ def test_react_to_task_to_compute(self): env = Mock() env.docker_images = [DockerImage("dockerix/xii", tag="323")] - env.allow_custom_main_program_file = False - env.get_source_code.return_value = None ts.task_server.get_environment_by_id.return_value = env keys = cryptography.ECCx(None) @@ -648,11 +644,6 @@ def _prepare_and_react(compute_task_def): ts._react_to_task_to_compute(msg) return msg - _prepare_and_react(ctd) - ts.task_manager.comp_task_keeper.receive_subtask.assert_not_called() - ts.task_computer.session_closed.assert_called_with() - assert conn.close.called - # Source code from local environment -> proper execution __reset_mocks() env.get_source_code.return_value = "print 'Hello world'" @@ -702,18 +693,9 @@ def _prepare_and_react(compute_task_def): _prepare_and_react(ctd) conn.close.assert_not_called() - # Allow custom code / no code in ComputeTaskDef -> failure - __reset_mocks() - env.allow_custom_main_program_file = True - ctd['src_code'] = "" - _prepare_and_react(ctd) - ts.task_manager.comp_task_keeper.receive_subtask.assert_not_called() - ts.task_computer.session_closed.assert_called_with() - assert conn.close.called - # Allow custom code / code in ComputerTaskDef -> proper execution __reset_mocks() - ctd['src_code'] = "print 'Hello world!'" + ctd['extra_data']['src_code'] = "print 'Hello world!'" msg = _prepare_and_react(ctd) ts.task_computer.session_closed.assert_not_called() ts.task_server.add_task_session.assert_called_with(msg.subtask_id, ts) @@ -747,36 +729,6 @@ def _prepare_and_react(compute_task_def): ts.task_computer.session_closed.assert_called_with() assert conn.close.called - # Envrionment is Docker environment with proper images, - # but no srouce code -> failure - __reset_mocks() - de = DockerEnvironmentMock(additional_images=[ - DockerImage("dockerix/xii", tag="323"), - DockerImage("dockerix/xiii", tag="325"), - DockerImage("dockerix/xiii", tag="323") - ]) - ts.task_server.get_environment_by_id.return_value = de - _prepare_and_react(ctd) - assert ts.err_msg == reasons.NoSourceCode - ts.task_manager.comp_task_keeper.receive_subtask.assert_not_called() - ts.task_computer.session_closed.assert_called_with() - assert conn.close.called - - # Proper Docker environment with source code - __reset_mocks() - file_name = os.path.join(self.path, "main_program_file") - with open(file_name, 'w') as f: - f.write("Hello world!") - de.main_program_file = file_name - msg = _prepare_and_react(ctd) - ts.task_server.add_task_session.assert_called_with(msg.subtask_id, ts) - ts.task_server.task_given.assert_called_with( - header.task_owner.key, - ctd, - msg.price, - ) - conn.close.assert_not_called() - # pylint: enable=too-many-statements def test_get_resource(self): diff --git a/tests/golem/task/test_tasktester.py b/tests/golem/task/test_tasktester.py index 54f7d90fdb..a8631a09f5 100644 --- a/tests/golem/task/test_tasktester.py +++ b/tests/golem/task/test_tasktester.py @@ -31,12 +31,12 @@ class TestTaskTester(TestDirFixture, LogTestCase): name = 'task1' def test_init(self): - task = Task(Mock(), Mock(), Mock()) + task = Task(Mock(), Mock()) task.query_extra_data_for_test_task = Mock() self.assertIsNotNone(TaskTester(task, self.path, None, None)) def test_task_computed(self): - task = Task(Mock(), Mock(), Mock()) + task = Task(Mock(), Mock()) result = [{"data": True}, 123] @@ -84,7 +84,7 @@ def success_callback(res, est_mem, time_spent, after_test_data): self.assertTrue("ble" in self.message) def test_is_success(self): - task = Task(Mock(), Mock(), Mock()) + task = Task(Mock(), Mock()) task.query_extra_data_for_test_task = Mock() tt = TaskTester(task, self.path, Mock(), Mock())