task-api - Added rpc calls for managing apps

golem/apps/__init__.py

@@ -6,6 +6,7 @@
 from dataclasses import dataclass, field
 from dataclasses_json import dataclass_json, config
 from marshmallow import fields as mm_fields
+from pathvalidate import sanitize_filename
 
 from golem.marketplace import (
     RequestorMarketStrategy,
@@ -89,3 +90,19 @@ def load_apps_from_dir(app_dir: Path) -> Iterator[AppDefinition]:
             yield load_app_from_json_file(json_file)
         except ValueError:
             continue
+
+
+def delete_app_from_dir(app_dir, app_def: AppDefinition) -> bool:
+    filename = app_json_file_name(app_def)
+    file = app_dir / filename
+    if not file.exists():
+        logger.warning('Can not delete app, file not found. file=%r', file)
+        return False
+    file.unlink()
+    return True
+
+
+def app_json_file_name(app_def: AppDefinition) -> str:
+    filename = f"{app_def.name}_{app_def.version}_{app_def.id}.json"
+    filename = sanitize_filename(filename, replacement_text="_")
+    return filename

golem/apps/default.py

@@ -2,9 +2,10 @@
 from typing import List
 
 from golem_task_api.envs import DOCKER_CPU_ENV_ID
-from pathvalidate import sanitize_filename
 
-from golem.apps import AppId, AppDefinition, save_app_to_json_file
+from golem.apps import (
+    AppId, AppDefinition, save_app_to_json_file, app_json_file_name,
+)
 from golem.marketplace import RequestorBrassMarketStrategy
 
 BlenderAppDefinition = AppDefinition(
@@ -31,16 +32,13 @@
 
 
 def save_built_in_app_definitions(path: Path) -> List[AppId]:
-    app_ids = []
-
+    new_app_ids = []
     for app_id, app in APPS.items():
-
-        filename = f"{app.name}_{app.version}_{app_id}.json"
-        filename = sanitize_filename(filename, replacement_text="_")
+        filename = app_json_file_name(app)
         json_file = path / filename
 
         if not json_file.exists():
             save_app_to_json_file(app, json_file)
-            app_ids.append(app_id)
+            new_app_ids.append(app_id)
 
-    return app_ids
+    return new_app_ids

golem/apps/manager.py

@@ -1,7 +1,11 @@
 import logging
 from typing import Dict, List, Tuple
+from pathlib import Path
 
-from golem.apps import AppId, AppDefinition
+from golem.apps import (
+    AppId, AppDefinition, load_apps_from_dir, delete_app_from_dir,
+)
+from golem.apps.default import save_built_in_app_definitions
 from golem.model import AppConfiguration
 
 logger = logging.getLogger(__name__)
@@ -10,9 +14,22 @@
 class AppManager:
     """ Manager class for applications using Task API. """
 
-    def __init__(self) -> None:
+    def __init__(self, app_dir: Path, save_apps=True) -> None:
         self._apps: Dict[AppId, AppDefinition] = {}
         self._state = AppStates()
+        self._app_dir = app_dir
+
+        # Save build in apps, then load apps from path
+        built_in_apps = []
+        if save_apps:
+            built_in_apps = save_built_in_app_definitions(app_dir)
+        for app_def in load_apps_from_dir(app_dir):
+            self.register_app(app_def)
+        for app_id in built_in_apps:
+            self.set_enabled(app_id, True)
+
+    def registered(self, app_id) -> bool:
+        return app_id in self._apps
 
     def register_app(self, app: AppDefinition) -> None:
         """ Register an application in the manager. """
@@ -58,6 +75,18 @@ def app(self, app_id: AppId) -> AppDefinition:
         """ Get an app with given ID (assuming it is registered). """
         return self._apps[app_id]
 
+    def delete(self, app_id: AppId) -> bool:
+        # Delete self._state from the database first
+        try:
+            AppConfiguration.delete() \
+                .where(AppConfiguration.app_id == app_id).execute()
+        except AppConfiguration.DoesNotExist:
+            logger.warning('Can not delete app, not found. id=%e', app_id)
+            return False
+        delete_app_from_dir(self._app_dir, self._apps[app_id])
+        del self._apps[app_id]
+        return True
+
 
 class AppStates:
 

golem/apps/rpc.py

@@ -0,0 +1,54 @@
+import logging
+import typing
+
+from golem.rpc import utils as rpc_utils
+
+if typing.TYPE_CHECKING:
+    # pylint:disable=unused-import, ungrouped-imports
+    from golem.client import Client
+
+logger = logging.getLogger(__name__)
+
+
+class ClientAppProvider:
+    def __init__(self, client: 'Client'):
+        self.app_manager = client.task_server.app_manager
+
+    @rpc_utils.expose('apps.list')
+    def apps_list(self):
+        logger.debug('apps.list called from rpc')
+        result = []
+        for app_id, app_def in self.app_manager.apps():
+            logger.debug('app_id=%r, app_def=%r', app_id, app_def)
+            app_result = {
+                'id': app_id,
+                'name': app_def.name,
+                'version': app_def.version,
+                'enabled': self.app_manager.enabled(app_id),
+            }
+            # TODO: Add full argument for more values
+            result.append(app_result)
+        logger.info('Listing apps. count=%r', len(result))
+        return result
+
+    @rpc_utils.expose('apps.update')
+    def apps_update(self, app_id, enabled):
+        logger.debug(
+            'apps.update called from rpc. app_id=%r, enabled=%r',
+            app_id,
+            enabled,
+        )
+        if not self.app_manager.registered(app_id):
+            logger.warning('App not found, not updated. app_id=%r', app_id)
+            return f"App not found, please check the app_id={app_id}"
+        self.app_manager.set_enabled(app_id, bool(enabled))
+        logger.info('Updated app. app_id=%r, enabled=%r', app_id, enabled)
+        return "App state updated."
+
+    @rpc_utils.expose('apps.delete')
+    def apps_delete(self, app_id):
+        logger.debug('apps.delete called from rpc. app_id=%r', app_id)
+        if not self.app_manager.delete(app_id):
+            return "Failed to delete app."
+        logger.info('Deleted app. app_id=%r', app_id)
+        return "App deleted with success."

golem/client.py

@@ -36,6 +36,7 @@
 from golem import model
 from golem.appconfig import TASKARCHIVE_MAINTENANCE_INTERVAL, AppConfig
 from golem.apps.default import APPS
+import golem.apps.rpc as apps_rpc
 from golem.clientconfigdescriptor import ConfigApprover, ClientConfigDescriptor
 from golem.core import variables
 from golem.core.common import (
@@ -266,6 +267,7 @@ def get_wamp_rpc_mapping(self):
         from golem.rpc.api import ethereum_ as api_ethereum
         from golem.task import rpc as task_rpc
         task_rpc_provider = task_rpc.ClientProvider(self)
+        app_rpc_provider = apps_rpc.ClientAppProvider(self)
         providers = (
             self,
             concent_soft_switch,
@@ -276,6 +278,7 @@ def get_wamp_rpc_mapping(self):
             self.environments_manager,
             self.transaction_system,
             task_rpc_provider,
+            app_rpc_provider,
             api_ethereum.ETSProvider(self.transaction_system),
         )
         mapping = {}

golem/task/taskserver.py

@@ -32,12 +32,10 @@
 from twisted.internet.defer import inlineCallbacks, Deferred, \
     TimeoutError as DeferredTimeoutError
 
-import golem.apps
 from apps.appsmanager import AppsManager
 from apps.core.task.coretask import CoreTask
 from golem import constants as gconst
 from golem.apps import manager as app_manager
-from golem.apps.default import save_built_in_app_definitions
 from golem.clientconfigdescriptor import ClientConfigDescriptor
 from golem.core.common import (
     short_node_id,
@@ -154,14 +152,7 @@ def __init__(
             dev_mode=task_api_dev_mode,
         )
 
-        app_dir = self.get_app_dir()
-        built_in_apps = save_built_in_app_definitions(app_dir)
-
-        self.app_manager = app_mgr = app_manager.AppManager()
-        for app_def in golem.apps.load_apps_from_dir(app_dir):
-            app_mgr.register_app(app_def)
-        for app_id in built_in_apps:
-            app_mgr.set_enabled(app_id, True)
+        self.app_manager = app_manager.AppManager(self.get_app_dir())
 
         self.node = node
         self.task_archiver = task_archiver
@@ -182,7 +173,7 @@ def __init__(
         )
 
         self.requested_task_manager = RequestedTaskManager(
-            app_manager=app_mgr,
+            app_manager=self.app_manager,
             env_manager=new_env_manager,
             public_key=self.keys_auth.public_key,
             root_path=Path(TaskServer.__get_task_manager_root(client.datadir)),

tests/golem/apps/test_app_manager.py

@@ -1,3 +1,5 @@
+from pathlib import Path
+
 from golem.apps.manager import AppManager
 from golem.apps import (
     AppDefinition,
@@ -22,7 +24,9 @@ class AppManagerTestBase(DatabaseFixture):
 
     def setUp(self):
         super().setUp()
-        self.app_manager = AppManager()
+        app_path = self.new_path / 'apps'
+        app_path.mkdir(exist_ok=True)
+        self.app_manager = AppManager(app_path, False)
 
 
 class TestRegisterApp(AppManagerTestBase):
@@ -38,6 +42,12 @@ def test_re_register(self):
         with self.assertRaises(ValueError):
             self.app_manager.register_app(APP_DEF)
 
+    def test_delete_app(self):
+        self.app_manager.register_app(APP_DEF)
+        self.assertEqual(self.app_manager.apps(), [(APP_ID, APP_DEF)])
+        self.app_manager.delete(APP_ID)
+        self.assertEqual(self.app_manager.apps(), [])
+
 
 class TestSetEnabled(AppManagerTestBase):
 

tests/golem/apps/test_app_rpc.py

@@ -0,0 +1,86 @@
+import pytest
+from mock import Mock
+
+from golem.apps.rpc import ClientAppProvider
+from golem.apps.manager import AppManager
+from golem.apps.default import BlenderAppDefinition
+from golem.client import Client
+from golem.testutils import pytest_database_fixture  # noqa pylint: disable=unused-import
+from golem.task.taskserver import TaskServer
+
+
+class TestClientAppProvider:
+    @pytest.fixture(autouse=True)
+    def setup_method(self):
+        self._client = Mock(spec=Client)
+        self._client.task_server = Mock(spec=TaskServer)
+        self._client.task_server.app_manager = Mock(spec=AppManager)
+        self._app_manger = self._client.task_server.app_manager
+        self._handler = ClientAppProvider(self._client)
+
+    def test_list(self):
+        # given
+        mocked_apps = [(BlenderAppDefinition.id, BlenderAppDefinition)]
+        self._app_manger.apps = Mock(
+            return_value=mocked_apps
+        )
+
+        # when
+        result = self._handler.apps_list()
+
+        # then
+        assert len(result) == len(mocked_apps), \
+            'count of result does not match input count'
+        assert result[0]['id'] == mocked_apps[0][0], \
+            'the first returned app id does not match input'
+        assert self._app_manger.apps.called_once_with()
+
+    def test_update(self):
+        # given
+        app_id = 'a'
+        enabled = True
+
+        # when
+        result = self._handler.apps_update(app_id, enabled)
+
+        # then
+        self._app_manger.registered.called_once_with(app_id)
+        self._app_manger.set_enabled.called_once_with(app_id, enabled)
+        assert result == 'App state updated.'
+
+    def test_update_not_registered(self):
+        # given
+        app_id = 'a'
+        enabled = True
+        self._app_manger.registered.return_value = False
+
+        # when
+        result = self._handler.apps_update(app_id, enabled)
+
+        # then
+        self._app_manger.registered.called_once_with(app_id)
+        self._app_manger.set_enabled.assert_not_called()
+        assert result == f"App not found, please check the app_id={app_id}"
+
+    def test_delete(self):
+        # given
+        app_id = 'a'
+
+        # when
+        result = self._handler.apps_delete(app_id)
+
+        # then
+        self._app_manger.delete.called_once_with(app_id)
+        assert result == 'App deleted with success.'
+
+    def test_delete_failed(self):
+        # given
+        app_id = 'a'
+        self._app_manger.delete.return_value = False
+
+        # when
+        result = self._handler.apps_delete(app_id)
+
+        # then
+        self._app_manger.delete.called_once_with(app_id)
+        assert result == 'Failed to delete app.'

tests/golem/core/test_fileshelper.py

@@ -210,6 +210,8 @@ def tearDown(self):
         if os.path.isdir(self.testdir):
             shutil.rmtree(self.testdir)
 
+        super().tearDown()
+
 
 class TestDu(TestDirFixture):
 
@@ -290,6 +292,7 @@ def tearDown(self):
 
         os.rmdir(self.test_dir2)
         os.rmdir(self.test_dir1)
+        super().tearDown()
 
     def test_find_file_with_ext(self):
         """ Test find_file_with_ext method """

tests/golem/network/concent/test_received_handler.py

@@ -187,6 +187,8 @@ def tearDown(self):
         # Remove registered handlers
         del self.task_server
         gc.collect()
+        testutils.DatabaseFixture.tearDown(self)
+        testutils.TestWithClient.tearDown(self)
 
 
 class IsOursTest(TaskServerMessageHandlerTestBase):

tests/golem/verifier/test_blenderverifier.py

@@ -29,6 +29,7 @@
 
 logger = logging.getLogger(__name__)
 
+
 @pytest.mark.slow
 @pytest.mark.skipif(
     not is_linux(),
@@ -71,6 +72,7 @@ def tearDown(self):
             # Try again after 3 seconds
             sleep(3)
             self.remove_files()
+        super().tearDown()
 
     def remove_files(self):
         above_tmp_dir = os.path.dirname(self.tempdir)