Move jobpy to Docker images

apps/blender/benchmark/benchmark.py

@@ -37,7 +37,6 @@ def __init__(self):
         main_scene_file = pathlib.Path(self.blender_task_path)
         main_scene_file /= self.SCENE_FILE_NAME
         task_def.main_scene_file = str(main_scene_file)
-        task_def.main_program_file = self.ENVIRONMENT_CLASS().main_program_file
         task_def.resources.add(str(main_scene_file.resolve()))
 
 

apps/blender/blenderenvironment.py

@@ -1,20 +1,16 @@
-from os import path
 from typing import Dict
 
 from apps.core import nvgpu
 from apps.core.nvgpu import get_devices
-from golem.core.common import get_golem_path
 from golem.docker.environment import DockerEnvironment
 from golem.docker.image import DockerImage
 from golem.environments.environment import SupportStatus, UnsupportReason
 
 
 class BlenderEnvironment(DockerEnvironment):
     DOCKER_IMAGE = "golemfactory/blender"
-    DOCKER_TAG = "1.5"
+    DOCKER_TAG = "1.7"
     ENV_ID = "BLENDER"
-    APP_DIR = path.join(get_golem_path(), 'apps', 'blender')
-    SCRIPT_NAME = "docker_blendertask.py"
     SHORT_DESCRIPTION = "Blender (www.blender.org)"
 
 

apps/blender/task/blenderrendertask.py

@@ -44,16 +44,13 @@ def __init__(self):
         RendererDefaults.__init__(self)
         self.output_format = "EXR"
 
-        self.main_program_file = BlenderEnvironment().main_program_file
         self.min_subtasks = 1
         self.max_subtasks = 100
         self.default_subtasks = 6
 
 
 class BlenderNVGPUDefaults(BlenderDefaults):
-    def __init__(self):
-        super().__init__()
-        self.main_program_file = BlenderNVGPUEnvironment().main_program_file
+    pass
 
 
 class PreviewUpdater(object):
@@ -458,7 +455,8 @@ def query_extra_data(self, perf_index: float, num_cores: int = 0,
                       "path_root": self.main_scene_dir,
                       "start_task": start_task,
                       "total_tasks": self.total_tasks,
-                      "crops": crops
+                      "crops": crops,
+                      "script_filepath": "/golem/scripts/job.py",
                       }
 
         subtask_id = self.create_subtask_id()
@@ -540,7 +538,8 @@ def query_extra_data_for_test_task(self):
                       "path_root": self.main_scene_dir,
                       "start_task": 1,
                       "total_tasks": 1,
-                      "crops": crops
+                      "crops": crops,
+                      "script_filepath": "/golem/scripts/job.py",
                       }
 
         hash = "{}".format(random.getrandbits(128))

apps/core/task/coretask.py

@@ -120,15 +120,6 @@ def __init__(self,
         # pylint: disable=not-callable
         self.environment = self.ENVIRONMENT_CLASS()
 
-        # src_code stuff
-        self.main_program_file = self.environment.main_program_file
-        try:
-            with open(self.main_program_file, "r") as src_file:
-                src_code = src_file.read()
-        except OSError as err:
-            logger.warning("Wrong main program file: %s", err)
-            src_code = ""
-
         # docker_images stuff
         if task_definition.docker_images:
             self.docker_images = task_definition.docker_images
@@ -151,7 +142,7 @@ def __init__(self,
             concent_enabled=task_definition.concent_enabled,
         )
 
-        Task.__init__(self, th, src_code, task_definition)
+        Task.__init__(self, th, task_definition)
 
         self.total_tasks = total_tasks
         self.last_task = 0
@@ -343,7 +334,6 @@ def _new_compute_task_def(self, subtask_id, extra_data,
         ctd['task_id'] = self.header.task_id
         ctd['subtask_id'] = subtask_id
         ctd['extra_data'] = extra_data
-        ctd['src_code'] = self.src_code
         ctd['performance'] = perf_index
         if self.docker_images:
             ctd['docker_images'] = [di.to_dict() for di in self.docker_images]
@@ -507,7 +497,6 @@ def __init__(self,
         self.root_path = dir_manager.root_path
         self.dir_manager = dir_manager
         self.owner = owner
-        self.src_code = ""
         self.environment = None
 
     def build(self):
@@ -532,7 +521,6 @@ def build_minimal_definition(cls, task_type: CoreTaskTypeInfo, dictionary):
         definition.compute_on = dictionary.get('compute_on', 'cpu')
         definition.resources = set(dictionary['resources'])
         definition.subtasks_count = int(dictionary['subtasks_count'])
-        definition.main_program_file = task_type.defaults.main_program_file
         return definition
 
     @classmethod

apps/core/task/coretaskstate.py

@@ -17,7 +17,6 @@ class TaskDefaults(object):
 
     def __init__(self):
         self.output_format = ""
-        self.main_program_file = ""
         self.min_subtasks = 1
         self.max_subtasks = 50
         self.default_subtasks = 20
@@ -45,7 +44,6 @@ def __init__(self):
 
         self.subtasks_count = 0
         self.optimize_total = False
-        self.main_program_file = ""
         self.output_file = ""
         self.task_type = None
         self.name = ""
@@ -94,9 +92,6 @@ def __setstate__(self, state):
             setattr(self, key, attributes[key])
 
     def is_valid(self):
-        if not path.exists(self.main_program_file):
-            return False, "Main program file does not exist: {}".format(
-                self.main_program_file)
         return self._check_output_file(self.output_file)
 
     @staticmethod

apps/dummy/benchmark/benchmark.py

@@ -3,16 +3,13 @@
 from pathlib import Path
 
 from apps.core.benchmark.benchmarkrunner import CoreBenchmark
-from apps.dummy.dummyenvironment import DummyTaskEnvironment
 from apps.dummy.task.dummytask import DummyTask
 from apps.dummy.task.dummytaskstate import DummyTaskDefinition, \
     DummyTaskDefaults
 from apps.dummy.task.verifier import DummyTaskVerifier
 from golem.core.common import get_golem_path
 from golem.verificator.verifier import SubtaskVerificationState
 
-APP_DIR = join(get_golem_path(), 'apps', 'dummy')
-
 
 class DummyTaskBenchmark(CoreBenchmark):
     def __init__(self):
@@ -27,7 +24,6 @@ def __init__(self):
         td.out_file_basename = td.out_file_basename
 
         td.task_id = str(uuid.uuid4())
-        td.main_program_file = DummyTaskEnvironment().main_program_file
         td.resources = {join(self.dummy_task_path, "in.data")}
         td.add_to_resources()
 

apps/dummy/dummyenvironment.py

@@ -1,14 +1,9 @@
-from os import path
-
-from golem.core.common import get_golem_path
 from golem.docker.environment import DockerEnvironment
 
 
 class DummyTaskEnvironment(DockerEnvironment):
-    DOCKER_IMAGE = "golemfactory/base"
-    DOCKER_TAG = "1.3"
+    DOCKER_IMAGE = "golemfactory/dummy"
+    DOCKER_TAG = "1.0"
     ENV_ID = "DUMMYPOW"
-    APP_DIR = path.join(get_golem_path(), 'apps', 'dummy')
-    SCRIPT_NAME = "docker_dummytask.py"
     SHORT_DESCRIPTION = "Dummy task (example app calculating proof-of-work " \
                         "hash)"

apps/dummy/resources/images/Dockerfile

@@ -0,0 +1,5 @@
+FROM golemfactory/base:1.3
+
+MAINTAINER Golem Tech <tech@golem.network>
+
+COPY scripts/ /golem/scripts/

apps/dummy/task/dummytask.py

@@ -68,6 +68,7 @@ def _extra_data(self, perf_index=0.0) -> ComputeTaskDef:
             "result_size": self.task_definition.result_size,
             "result_file": self.__get_result_file_name(subtask_id),
             "subtask_data_size": sbs,
+            "script_filepath": "/golem/scripts/job.py",
         }
 
         return self._new_compute_task_def(subtask_id,

apps/images.ini

@@ -1,5 +1,6 @@
 golemfactory/base core/resources/images/base.Dockerfile 1.3 core/resources/images/
 golemfactory/nvgpu core/resources/images/nvgpu.Dockerfile 1.1 . apps.core.nvgpu.is_supported
-golemfactory/blender blender/resources/images/blender.Dockerfile 1.5 blender/resources/images/
+golemfactory/blender blender/resources/images/blender.Dockerfile 1.7 blender/resources/images/
 golemfactory/blender_nvgpu blender/resources/images/blender_nvgpu.Dockerfile 1.1 . apps.core.nvgpu.is_supported
-golemfactory/image_metrics rendering/resources/images/Dockerfile 1.9 rendering/resources/images/
+golemfactory/dummy dummy/resources/images/Dockerfile 1.0 dummy/resources/images
+golemfactory/image_metrics rendering/resources/images/Dockerfile 1.11 rendering/resources/images/

golem/docker/environment.py

@@ -6,7 +6,6 @@
 from golem.docker.image import DockerImage
 from golem.environments.environment import (Environment, SupportStatus,
                                             UnsupportReason)
-from golem.resource.dirmanager import find_task_script
 
 
 @enforce.runtime_validation()
@@ -22,9 +21,6 @@ def __init__(self, tag=None, image_id=None, additional_images: List[DockerImage]
             else DockerImage(self.DOCKER_IMAGE, tag=tag)
         Environment.__init__(self)
 
-        self.main_program_file = find_task_script(self.APP_DIR,
-                                                  self.SCRIPT_NAME)
-
         self.docker_images = [image]
         if additional_images:
             self.docker_images += additional_images
@@ -82,16 +78,6 @@ def DOCKER_TAG(cls):
     def ENV_ID(cls):
         pass
 
-    @property
-    @abc.abstractmethod
-    def APP_DIR(cls):
-        pass
-
-    @property
-    @abc.abstractmethod
-    def SCRIPT_NAME(cls):
-        pass
-
     @property
     @abc.abstractmethod
     def SHORT_DESCRIPTION(cls):

golem/docker/job.py

@@ -22,7 +22,7 @@
 
 
 # pylint:disable=too-many-instance-attributes
-class DockerJob(object):
+class DockerJob:
     STATE_NEW = "new"
     STATE_CREATED = "created"  # container created by docker
     STATE_RUNNING = "running"  # docker container running
@@ -52,16 +52,13 @@ class DockerJob(object):
         "OUTPUT_DIR": OUTPUT_DIR
     }
 
-    # Name of the script file, relative to WORK_DIR
-    TASK_SCRIPT = "job.py"
-
     # Name of the parameters file, relative to WORK_DIR
     PARAMS_FILE = "params.json"
 
     # pylint:disable=too-many-arguments
     def __init__(self,
                  image: DockerImage,
-                 script_src: str,
+                 script_filepath: str,
                  parameters: Dict,
                  resources_dir: str,
                  work_dir: str,
@@ -82,7 +79,7 @@ def __init__(self,
             raise TypeError('Incorrect image type: {}. '
                             'Should be: DockerImage'.format(type(image)))
         self.image = image
-        self.script_src = script_src
+        self.script_filepath = script_filepath
         self.parameters = parameters if parameters else {}
 
         self.parameters.update(self.PATH_PARAMS)
@@ -124,23 +121,16 @@ def _prepare(self):
         with open(params_file_path, "w") as params_file:
             json.dump(self.parameters, params_file)
 
-        # Save the script in work_dir/TASK_SCRIPT
-        task_script_path = self._get_host_script_path()
-        with open(task_script_path, "wb") as script_file:
-            script_file.write(bytearray(self.script_src, "utf-8"))
-
         # Setup volumes for the container
         client = local_client()
 
         host_cfg = client.create_host_config(**self.host_config)
 
-        # The location of the task script when mounted in the container
-        container_script_path = self._get_container_script_path()
         self.container = client.create_container(
             image=self.image.name,
             volumes=self.volumes,
             host_config=host_cfg,
-            command=[container_script_path],
+            command=[self.script_filepath],
             working_dir=self.WORK_DIR,
             environment=self.environment,
         )
@@ -182,9 +172,6 @@ def __enter__(self):
     def __exit__(self, exc_type, exc_value, traceback):
         self._cleanup()
 
-    def _get_host_script_path(self):
-        return os.path.join(self.work_dir, self.TASK_SCRIPT)
-
     def _get_host_params_path(self):
         return os.path.join(self.work_dir, self.PARAMS_FILE)
 
@@ -210,10 +197,6 @@ def _host_dir_chmod(dst_dir, mod):
 
         return prev_mod
 
-    @staticmethod
-    def _get_container_script_path():
-        return posixpath.join(DockerJob.WORK_DIR, DockerJob.TASK_SCRIPT)
-
     @staticmethod
     def get_absolute_resource_path(relative_path):
         return posixpath.join(DockerJob.RESOURCES_DIR,

golem/docker/task_thread.py

@@ -75,7 +75,6 @@ class DockerTaskThread(TaskThread):
 
     def __init__(self,  # pylint: disable=too-many-arguments
                  docker_images: List[Union[DockerImage, Dict, Tuple]],
-                 src_code: str,
                  extra_data: Dict,
                  dir_mapping: DockerDirMapping,
                  timeout: int,
@@ -84,7 +83,6 @@ def __init__(self,  # pylint: disable=too-many-arguments
         if not docker_images:
             raise AttributeError("docker images is None")
         super().__init__(
-            src_code=src_code,
             extra_data=extra_data,
             res_path=str(dir_mapping.resources),
             tmp_path=str(dir_mapping.temporary),
@@ -190,7 +188,7 @@ def _run_docker_job(self) -> Optional[int]:
 
         params = dict(
             image=self.image,
-            script_src=self.src_code,
+            script_filepath=self.extra_data['script_filepath'],
             parameters=self.extra_data,
             resources_dir=str(self.dir_mapping.resources),
             work_dir=str(self.dir_mapping.work),

golem/environments/environment.py

@@ -69,9 +69,6 @@ def __init__(self):
                                  " without any additional requirements."
 
         self.accept_tasks = False
-        # Check if tasks can define the source code
-        self.allow_custom_main_program_file = False
-        self.main_program_file = None
 
     def check_support(self) -> SupportStatus:
         """ Check if this environment is supported on this machine
@@ -111,11 +108,6 @@ def get_min_accepted_performance(cls) -> float:
 
         return step * MinPerformanceMultiplier.get()
 
-    def get_source_code(self):
-        if self.main_program_file and path.isfile(self.main_program_file):
-            with open(self.main_program_file) as f:
-                return f.read()
-
     @classmethod
     def run_default_benchmark(cls, save=False):
         logger = logging.getLogger('golem.task.benchmarkmanager')

golem/resource/dirmanager.py

@@ -47,15 +47,6 @@ def split_path(path):
     return split_path(head) + [tail]
 
 
-def find_task_script(task_dir, script_name):
-    scripts_path = os.path.abspath(os.path.join(task_dir, "resources", "scripts"))
-    script_file = os.path.join(scripts_path, script_name)
-    if os.path.isfile(script_file):
-        return script_file
-
-    logger.error("Script file {} does not exist!".format(script_file))
-
-
 def list_dir_recursive(dir: str) -> Iterator[str]:
     for dirpath, dirnames, filenames in os.walk(dir, followlinks=True):
         for name in filenames:

golem/task/benchmarkmanager.py

@@ -98,11 +98,6 @@ def on_success(performance):
 
     @staticmethod
     def _validate_task_state(task_state):
-        td = task_state.definition
-        if not os.path.exists(td.main_program_file):
-            logger.error("Main program file does not exist: {}".format(
-                td.main_program_file))
-            return False
         return True
 
     def run_benchmark_for_env_id(self, env_id, callback, errback):