Client: Separate plugin file system

.github/workflows/qiita-ci.yml

@@ -93,7 +93,7 @@ jobs:
           conda activate qiita
           export QIITA_ROOTCA_CERT=`pwd`/qiita-dev/qiita_core/support_files/ci_rootca.crt
           export QIITA_CONFIG_FP=`pwd`/qiita-dev/qiita_core/support_files/config_test_local.cfg
-          sed "s#/home/runner/work/qiita/qiita#${PWD}/qiita-dev/#g" `pwd`/qiita-dev/qiita_core/support_files/config_test.cfg > ${QIITA_CONFIG_FP}
+          sed "s#/home/runner/work/qiita/qiita#${PWD}/qiita-dev#g" `pwd`/qiita-dev/qiita_core/support_files/config_test.cfg > ${QIITA_CONFIG_FP}
 
           export REDBIOM_HOST="http://localhost:7379"
 
@@ -104,7 +104,8 @@ jobs:
           mkdir -p ${CONDA_PREFIX}/var/run/nginx/
           export NGINX_FILE=`pwd`/qiita-dev/qiita_pet/nginx_example.conf
           export NGINX_FILE_NEW=`pwd`/qiita-dev/qiita_pet/nginx_example_local.conf
-          sed "s#/home/runner/work/qiita/qiita#${PWD}/qiita-dev/#g" ${NGINX_FILE} > ${NGINX_FILE_NEW}
+          sed "s#/home/runner/work/qiita/qiita#${PWD}/qiita-dev#g" ${NGINX_FILE} > ${NGINX_FILE_NEW}
+          sed -i "s#/Users/username/qiita#${PWD}/qiita-dev#g" ${NGINX_FILE_NEW}
           nginx -c ${NGINX_FILE_NEW}
 
           echo "3. Setting up qiita"

qiita_client/plugin.py

@@ -136,13 +136,46 @@ def __init__(self, name, description, can_be_submitted_to_ebi,
 
 
 class BaseQiitaPlugin(object):
-    def __init__(self, name, version, description, publications=None):
+    # default must be first element
+    _ALLOWED_PLUGIN_COUPLINGS = ['filesystem', 'https']
+
+    def __init__(self, name, version, description, publications=None,
+                 plugincoupling=_ALLOWED_PLUGIN_COUPLINGS[0]):
         logger.debug('Entered BaseQiitaPlugin.__init__()')
         self.name = name
         self.version = version
         self.description = description
         self.publications = dumps(publications) if publications else ""
 
+        # Depending on your compute architecture, there are multiple options
+        # available how "thight" plugins are coupled to the central
+        # Qiita master/workers
+        # --- filesystem ---
+        # The default scenario is "filesystem", i.e. plugins as well as
+        # master/worker have unrestricted direct access to a shared filesystem,
+        # e.g. a larger volume / directory, defined in the server configuration
+        # as base_data_dir
+        # --- https ---
+        # A second scenario is that your plugins execute as independent jobs on
+        # another machine, e.g. as docker containers or other cloud techniques.
+        # Intentionally, you don't want to use a shared filesystem, but you
+        # have to make sure necessary input files are provided to the
+        # containerized plugin before execution and resulting files are
+        # transfered back to the central Qiita master/worker. In this case,
+        # files are pulled / pushed through functions
+        # qiita_client.fetch_file_from_central and
+        # qiita_client.push_file_to_central, respectivey.
+        # Actually, all files need to be decorated with this function.
+        # The decision how data are transferred is then made within these two
+        # functions according to the "plugincoupling" setting.
+        if plugincoupling not in self._ALLOWED_PLUGIN_COUPLINGS:
+            raise ValueError(
+                ("valid plugincoupling values are ['%s'], but you "
+                 "provided %s") % (
+                     "', '".join(self._ALLOWED_PLUGIN_COUPLINGS),
+                     plugincoupling))
+        self.plugincoupling = plugincoupling
+
         # Will hold the different commands
         self.task_dict = {}
 
@@ -151,7 +184,8 @@ def __init__(self, name, version, description, publications=None):
             'QIITA_PLUGINS_DIR', join(expanduser('~'), '.qiita_plugins'))
         self.conf_fp = join(conf_dir, "%s_%s.conf" % (self.name, self.version))
 
-    def generate_config(self, env_script, start_script, server_cert=None):
+    def generate_config(self, env_script, start_script, server_cert=None,
+                        plugin_coupling=_ALLOWED_PLUGIN_COUPLINGS[0]):
         """Generates the plugin configuration file
 
         Parameters
@@ -165,6 +199,9 @@ def generate_config(self, env_script, start_script, server_cert=None):
             If the Qiita server used does not have a valid certificate, the
             path to the Qiita certificate so the plugin can connect over
             HTTPS to it
+        plugin_coupling : str
+            Type of coupling of plugin to central for file exchange.
+            Valid values are 'filesystem' and 'https'.
         """
         logger.debug('Entered BaseQiitaPlugin.generate_config()')
         sr = SystemRandom()
@@ -178,7 +215,8 @@ def generate_config(self, env_script, start_script, server_cert=None):
             f.write(CONF_TEMPLATE % (self.name, self.version, self.description,
                                      env_script, start_script,
                                      self._plugin_type, self.publications,
-                                     server_cert, client_id, client_secret))
+                                     server_cert, client_id, client_secret,
+                                     plugin_coupling))
 
     def _register_command(self, command):
         """Registers a command in the plugin
@@ -188,8 +226,8 @@ def _register_command(self, command):
         command: QiitaCommand
             The command to be added to the plugin
         """
-        logger.debug(
-            f'Entered BaseQiitaPlugin._register_command({command.name})')
+        logger.debug('Entered BaseQiitaPlugin._register_command(%s)' %
+                     command.name)
         self.task_dict[command.name] = command
 
     def _register(self, qclient):
@@ -244,14 +282,33 @@ def __call__(self, server_url, job_id, output_dir):
         with open(self.conf_fp, 'U') as conf_file:
             config.readfp(conf_file)
 
+        # the plugin coupling protocoll can be set in three ways
+        # 1. default is always "filesystem", i.e. first value in
+        #    _ALLOWED_PLUGIN_COUPLINGS. This is to be downward compatible.
+        # 2. the plugin configuration can hold a section 'network' with an
+        #    option 'PLUGINCOUPLING'. For old config files, this might not
+        #    (yet) be the case. Therefore, we are double checking existance
+        #    of this section and parameter here.
+        # 3. you can set the environment variable QIITA_PLUGINCOUPLING
+        # Precedence is 3, 2, 1, i.e. the environment variable overrides the
+        # other two ways.
+        plugincoupling = self._ALLOWED_PLUGIN_COUPLINGS[0]
+        if config.has_section('network') and \
+           config.has_option('network', 'PLUGINCOUPLING'):
+            plugincoupling = config.get('network', 'PLUGINCOUPLING')
+        if 'QIITA_PLUGINCOUPLING' in environ.keys() and \
+           environ['QIITA_PLUGINCOUPLING'] is not None:
+            plugincoupling = environ['QIITA_PLUGINCOUPLING']
+
         qclient = QiitaClient(server_url, config.get('oauth2', 'CLIENT_ID'),
                               config.get('oauth2', 'CLIENT_SECRET'),
                               # for this group of tests, confirm optional
                               # ca_cert parameter works as intended. Setting
                               # this value will prevent underlying libraries
                               # from validating the server's cert using
                               # certifi's pem cache.
-                              ca_cert=config.get('oauth2', 'SERVER_CERT'))
+                              ca_cert=config.get('oauth2', 'SERVER_CERT'),
+                              plugincoupling=plugincoupling)
 
         if job_id == 'register':
             self._register(qclient)
@@ -314,9 +371,11 @@ class QiitaTypePlugin(BaseQiitaPlugin):
     _plugin_type = "artifact definition"
 
     def __init__(self, name, version, description, validate_func,
-                 html_generator_func, artifact_types, publications=None):
+                 html_generator_func, artifact_types, publications=None,
+                 plugincoupling=BaseQiitaPlugin._ALLOWED_PLUGIN_COUPLINGS[0]):
         super(QiitaTypePlugin, self).__init__(name, version, description,
-                                              publications=publications)
+                                              publications=publications,
+                                              plugincoupling=plugincoupling)
 
         logger.debug('Entered QiitaTypePlugin.__init__()')
         self.artifact_types = artifact_types
@@ -382,4 +441,8 @@ def register_command(self, command):
 [oauth2]
 SERVER_CERT = %s
 CLIENT_ID = %s
-CLIENT_SECRET = %s"""
+CLIENT_SECRET = %s
+
+[network]
+PLUGINCOUPLING = %s
+"""