Merge pull request #2263 from antgonza/rm-moi-ipython

rm moi and ipython

Author: josenavas

.travis.yml

@@ -32,7 +32,7 @@ install:
     'pandas>=0.18' 'matplotlib>=1.1.0' 'scipy>0.13.0' 'numpy>=1.7' 'h5py>=2.3.1'
   - source activate qiita
   - pip install -U pip
-  - pip install sphinx sphinx-bootstrap-theme 'ipython[all]==2.4.1' nose-timer codecov
+  - pip install sphinx sphinx-bootstrap-theme nose-timer codecov
   - travis_retry pip install . --process-dependency-links
   - 'echo "backend: Agg" > matplotlibrc'
   # Install the biom plugin so we can run the analysis tests
@@ -50,8 +50,6 @@ before_script:
   #   export QIITA_CONFIG_FP=`pwd`/qiita_core/support_files/config_test_travis.cfg;
   #   export MOI_CONFIG_FP=`pwd`/qiita_core/support_files/config_test_travis.cfg;
   #   fi
-  - ipython profile create qiita-general --parallel
-  - qiita-env start_cluster qiita-general
   - qiita-env make --no-load-ontologies
   - |
     if [ ${TEST_ADD_STUDIES} == "False" ]; then

CONTRIBUTING.md

@@ -56,7 +56,7 @@ Since Qiita is a package that is continuously growing, we found ourselves in a p
 
 ### Configuration file
 
-The Qiita configuration file determines how the package interacts with your system’s resources (redis, postgres and the IPython cluster). Thus you should review the documentation detailed [here](https://docs.google.com/document/d/1u7kwLP31NM513-8xwpwvLbSQxYu0ehI6Jau1APR13e0/edit#), but especially bear in mind the following points:
+The Qiita configuration file determines how the package interacts with your system’s resources (redis and postgres). Thus you should review the documentation detailed [here](https://docs.google.com/document/d/1u7kwLP31NM513-8xwpwvLbSQxYu0ehI6Jau1APR13e0/edit#), but especially bear in mind the following points:
 
 * An example version of this file can be found here `qiita_core/support_files/qiita_config.txt` and if you don’t set a `QIITA_CONFIG_FP` environment variable, that’s the file that Qiita will use.
 * The `[main]` section sets a `TEST_ENVIRONMENT` variable, which determines whether your system will be running unit tests or if it a demo/production system. You will want to set the value to TRUE if you are running the unit tests.
@@ -83,8 +83,6 @@ Scripts in Qiita are located inside the scripts directory, their actions will re
 
 * `qiita-env make` will create a new environment (as specified by the Qiita config file).
 * `qiita-env drop` will delete the environment (as specified by the Qiita config file).
-* `qiita-env start_cluster qiita-general`, starts an IPython cluster named ‘qiita-general’. Normally you’ll want to wait a few seconds for the engines to start and become responsive (30-40 seconds depending on your system).
-* `qiita-env stop_cluster qiita-general`, terminates a cluster named ‘qiita-general’.
 * `qiita pet webserver start`, will start the Qiita web-application running on port 21174, you can change this using the `--port` flag, for example `--port=7532`.
 
 ## Making Database Changes

INSTALL.md

@@ -110,11 +110,6 @@ Install Qiita (this occurs through setuptools' `setup.py` file in the qiita dire
 pip install -e . --process-dependency-links
 ```
 
-Install the development version of moi:
-```bash
-pip install https://github.com/biocore/mustached-octo-ironman/archive/master.zip --no-deps
-```
-
 At this point, Qiita will be installed and the system will start. However,
 you will need to install plugins in order to process any kind of data. For a list
 of available plugins, visit the [Qiita Spots](https://github.com/qiita-spots)
@@ -151,12 +146,6 @@ Set your `QIITA_CONFIG_FP` environment variable to point to that file (into `.ba
   source activate qiita
 ```
 
-Setup ipython profile for qiita:
-
-```bash
-ipython profile create qiita-general --parallel
-```
-
 Next, make a test environment:
 
 ```bash
@@ -173,12 +162,6 @@ Next, start redis server (the command may differ depending on your operating sys
 redis-server
 ```
 
-Start the IPython cluster:
-
-```bash
-qiita-env start_cluster qiita-general && sleep 30
-```
-
 Start the qiita server:
 
 ```bash

qiita_core/configuration_manager.py

@@ -58,18 +58,6 @@ class ConfigurationManager(object):
         The host where the database lives
     port : int
         The port used to connect to the postgres database in the previous host
-    ipyc_demo : str
-        The IPython demo cluster profile
-    ipyc_demo_n : int
-        The size of the demo cluster
-    ipyc_reserved : str
-        The IPython reserved cluster profile
-    ipyc_reserved_n : int
-        The size of the reserved cluster
-    ipyc_general : str
-        The IPython general cluster profile
-    ipyc_general_n : int
-        The size of the general cluster
     smtp_host : str
         The SMTP host from which mail will be sent
     smtp_port : int
@@ -145,7 +133,7 @@ def __init__(self):
             config.readfp(conf_file)
 
         _required_sections = {'main', 'redis', 'postgres', 'smtp', 'ebi',
-                              'ipython', 'portal'}
+                              'portal'}
         if not _required_sections.issubset(set(config.sections())):
             missing = _required_sections - set(config.sections())
             raise MissingConfigSection(', '.join(missing))
@@ -155,7 +143,6 @@ def __init__(self):
         self._get_postgres(config)
         self._get_redis(config)
         self._get_ebi(config)
-        self._get_ipython(config)
         self._get_vamps(config)
         self._get_portal(config)
 
@@ -283,10 +270,6 @@ def _get_ebi(self, config):
         self.ebi_center_name = sec_get('EBI_CENTER_NAME')
         self.ebi_organization_prefix = sec_get('EBI_ORGANIZATION_PREFIX')
 
-    def _get_ipython(self, config):
-        self.ipython_contexts = config.get('ipython', 'context').split(',')
-        self.ipython_default = config.get('ipython', 'default')
-
     def _get_vamps(self, config):
         self.vamps_user = config.get('vamps', 'USER')
         self.vamps_pass = config.get('vamps', 'PASSWORD')

qiita_core/environment_manager.py

@@ -6,34 +6,12 @@
 # The full license is in the file LICENSE, distributed with this software.
 # -----------------------------------------------------------------------------
 
-from os import fork
-from sys import exit, exc_info, stderr
+from sys import exc_info, stderr
 import traceback
 
-from IPython.parallel.apps.ipclusterapp import IPClusterStart, IPClusterStop
-
 
 MAX_TEST_WAIT = 5
-TEST_RUNNERS = ('local', 'remote', 'moi', 'all')
-
-
-def start_cluster(profile):
-    """Start a cluster"""
-    me = fork()
-    if me == 0:
-        c = IPClusterStart(profile=profile, log_level=0, daemonize=True)
-        c.initialize(argv=[])
-        c.start()
-
-
-def stop_cluster(profile):
-    """Stop a cluster"""
-    me = fork()
-    if me == 0:
-        c = IPClusterStop(profile=profile, log_level=0)
-        c.initialize(argv=[])
-        c.start()
-        exit(0)
+TEST_RUNNERS = ('local', 'remote', 'all')
 
 
 def _test_wrapper_local(func):
@@ -44,39 +22,6 @@ def _test_wrapper_local(func):
         return ('FAIL', traceback.format_exception(*exc_info()))
 
 
-def _test_wrapper_moi(func):
-    """Submit a function through moi"""
-    try:
-        from moi.job import submit_nouser
-        _, _, ar = submit_nouser(func)
-    except:
-        return ('FAIL', traceback.format_exception(*exc_info()))
-
-    return _ipy_wait(ar)
-
-
-def _test_wrapper_remote(func):
-    """Execute a function on a remote ipengine"""
-    from IPython.parallel import Client
-    from qiita_core.configuration_manager import ConfigurationManager
-    config = ConfigurationManager()
-    c = Client(profile=config.ipython_default)
-    bv = c.load_balanced_view()
-    return _ipy_wait(bv.apply_async(func))
-
-
-def _ipy_wait(ar):
-    """Wait on a IPython AsyncResult"""
-    ar.wait(timeout=MAX_TEST_WAIT)
-
-    if ar.ready():
-        result = _test_wrapper_local(ar.get)
-    else:
-        result = ('FAIL', 'No result after %d seconds' % MAX_TEST_WAIT)
-
-    return result
-
-
 def _test_result(test_type, name, state, result, expected):
     """Write out the results of the test"""
     correct_result = result == expected
@@ -104,26 +49,11 @@ def _test_result(test_type, name, state, result, expected):
     stderr.write('\n')
 
 
-def _test_runner(test_type, name, func, expected):
-    """Dispatch to the corresponding runner"""
-    if test_type == 'local':
-        state, result = _test_wrapper_local(func)
-    elif test_type == 'moi':
-        state, result = _test_wrapper_moi(func)
-    elif test_type == 'remote':
-        state, result = _test_wrapper_remote(func)
-    else:
-        raise ValueError("Unknown test type: %s" % test_type)
-
-    _test_result(test_type, name, state, result, expected)
-
-
 def test(runner):
     """Test the environment
 
-    * Verify redis connectivity indepedent of moi
+    * Verify redis connectivity
     * Verify database connectivity
-    * Verify submission via moi
 
     Tests are performed both on the server and ipengines.
     """
@@ -148,23 +78,21 @@ def postgres_test(**kwargs):
         c = SQLConnectionHandler()
         return c.execute_fetchone("SELECT 42")[0]
 
-    def moi_test(**kwargs):
-        """Submit a function via moi"""
-        from moi.job import submit_nouser
-
-        def inner(a, b, **kwargs):
-            return a + b
-
-        _, _, ar = submit_nouser(inner, 7, 35)
-        state, result = _ipy_wait(ar)
-        return result
-
     if runner == 'all':
-        runner = ('local', 'remote', 'moi')
+        runner = ('local', )
     else:
         runner = [runner]
 
     for name in runner:
         _test_runner(name, "redis", redis_test, 42)
         _test_runner(name, "postgres", postgres_test, 42)
-        _test_runner(name, "submit via moi", moi_test, 42)
+
+
+def _test_runner(test_type, name, func, expected):
+    """Dispatch to the corresponding runner"""
+    if test_type == 'local':
+        state, result = _test_wrapper_local(func)
+    else:
+        raise ValueError("Unknown test type: %s" % test_type)
+
+    _test_result(test_type, name, state, result, expected)

qiita_core/support_files/config_test.cfg

@@ -144,13 +144,6 @@ EBI_CENTER_NAME = qiita-test
 # study aliases
 EBI_ORGANIZATION_PREFIX = example_organization
 
-
-# ----------------------------- IPython settings -----------------------------
-[ipython]
-# context is expected to be a comma separated list
-context = qiita-general
-default = qiita-general
-
 # ----------------------------- VAMPS settings -----------------------------
 [vamps]
 # general info to submit to vamps

qiita_core/tests/test_configuration_manager.py

@@ -94,10 +94,6 @@ def test_init(self):
         self.assertEqual(obs.ebi_center_name, "qiita-test")
         self.assertEqual(obs.ebi_organization_prefix, "example_organization")
 
-        # IPython section
-        self.assertEqual(obs.ipython_contexts, ["qiita-general"])
-        self.assertEqual(obs.ipython_default, "qiita-general")
-
         # VAMPS section
         self.assertEqual(obs.vamps_user, "user")
         self.assertEqual(obs.vamps_pass, "password")
@@ -333,13 +329,6 @@ def test_get_portal(self):
 # study aliases
 EBI_ORGANIZATION_PREFIX = example_organization
 
-
-# ----------------------------- IPython settings -----------------------------
-[ipython]
-# context is expected to be a comma separated list
-context = qiita-general
-default = qiita-general
-
 # ----------------------------- VAMPS settings -----------------------------
 [vamps]
 # general info to submit to vamps

qiita_pet/webserver.py

@@ -11,12 +11,9 @@
 import tornado.escape
 import tornado.web
 import tornado.websocket
-from os.path import dirname, join, exists
-from shutil import copy
+from os.path import dirname, join
 from base64 import b64encode
 from uuid import uuid4
-from moi import moi_js, moi_list_js
-from moi.websocket import MOIMessageHandler
 
 from qiita_core.qiita_settings import qiita_config
 from qiita_core.util import is_test_environment
@@ -85,9 +82,6 @@
 
 
 _vendor_js = join(STATIC_PATH, 'vendor', 'js')
-if not exists(join(_vendor_js, 'moi.js')):
-    copy(moi_js(), _vendor_js)
-    copy(moi_list_js(), _vendor_js)
 
 
 class Application(tornado.web.Application):
@@ -117,7 +111,6 @@ def __init__(self):
             (r"/analysis/sharing/", ShareAnalysisAJAX),
             (r"/artifact/samples/", ArtifactGetSamples),
             (r"/artifact/info/", ArtifactGetInfo),
-            (r"/moi-ws/", MOIMessageHandler),
             (r"/consumer/", MessageHandler),
             (r"/admin/error/", LogEntryViewerHandler),
             (r"/admin/approval/", StudyApprovalList),

qiita_ware/commands.py

@@ -12,10 +12,10 @@
 from tempfile import mkdtemp
 from os import environ
 from traceback import format_exc
-from moi.job import system_call
 
 from qiita_db.artifact import Artifact
 from qiita_db.logger import LogEntry
+from qiita_db.processing_job import _system_call as system_call
 from qiita_core.qiita_settings import qiita_config
 from qiita_ware.ebi import EBISubmission
 from qiita_ware.exceptions import ComputeError, EBISubmissionError