Rewrite backend to be more modular, maintainable

.travis.yml

@@ -26,15 +26,15 @@ matrix:
       script:
         - export PATH="$HOME/miniconda/bin:$PATH"
         - yapf -dr modin/pandas
-        - flake8 .
+        - flake8 --max-line-length=88 .
 
 install:
   - ./.travis/install-dependencies.sh
 
 script:
   - export PATH="$HOME/miniconda/bin:$PATH"
-
-  - python -m pytest modin/pandas/test/test_dataframe.py
-  - python -m pytest modin/pandas/test/test_concat.py
-  - python -m pytest modin/pandas/test/test_io.py
-  - python -m pytest modin/pandas/test/test_groupby.py
+  - if [[ "$PYTHON" == "2.7" ]]; then python .travis/strip-type-hints.py; fi
+  - python -m pytest --disable-pytest-warnings modin/pandas/test/test_dataframe.py
+  - python -m pytest --disable-pytest-warnings modin/pandas/test/test_concat.py
+  - python -m pytest --disable-pytest-warnings modin/pandas/test/test_io.py
+  - python -m pytest --disable-pytest-warnings modin/pandas/test/test_groupby.py

.travis/install-dependencies.sh

@@ -1,16 +1,57 @@
 #!/usr/bin/env bash
+set -e
+set -x
 
-if [[ "$PYTHON" == "2.7" ]]; then
+
+ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
+echo "PYTHON is $PYTHON"
+
+platform="unknown"
+unamestr="$(uname)"
+if [[ "$unamestr" == "Linux" ]]; then
+  echo "Platform is linux."
+  platform="linux"
+elif [[ "$unamestr" == "Darwin" ]]; then
+  echo "Platform is macosx."
+  platform="macosx"
+else
+  echo "Unrecognized platform."
+  exit 1
+fi
+
+if [[ "$PYTHON" == "2.7" ]] && [[ "$platform" == "linux" ]]; then
   wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -O miniconda.sh -nv
   bash miniconda.sh -b -p $HOME/miniconda
   export PATH="$HOME/miniconda/bin:$PATH"
 
-elif [[ "$PYTHON" == "3.6" ]] || [[ "$LINT" == "1" ]]; then
+elif [[ "$PYTHON" == "3.6" ]] && [[ "$platform" == "linux" ]]; then
   wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh -nv
   bash miniconda.sh -b -p $HOME/miniconda
   export PATH="$HOME/miniconda/bin:$PATH"
   conda install -y python==3.6.5
+
+elif [[ "$PYTHON" == "2.7" ]] && [[ "$platform" == "macosx" ]]; then
+  wget https://repo.continuum.io/miniconda/Miniconda2-latest-MacOSX-x86_64.sh -O miniconda.sh -nv
+  bash miniconda.sh -b -p $HOME/miniconda
+  export PATH="$HOME/miniconda/bin:$PATH"
+
+elif [[ "$PYTHON" == "3.6" ]] && [[ "$platform" == "macosx" ]]; then
+  wget https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -O miniconda.sh -nv
+  bash miniconda.sh -b -p $HOME/miniconda
+  export PATH="$HOME/miniconda/bin:$PATH"
+  conda install -y python==3.6.5
+
+elif [[ "$LINT" == "1" ]]; then
+  wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh -nv
+  bash miniconda.sh -b -p $HOME/miniconda
+  export PATH="$HOME/miniconda/bin:$PATH"
+  conda install -y python==3.6.5
+
+else
+  echo "Unrecognized environment."
+  exit 1
 fi
 
 pip install -r requirements.txt
 pip install -q pytest flake8 flake8-comprehensions yapf feather-format lxml openpyxl xlrd numpy
+

.travis/strip-type-hints.py

@@ -0,0 +1,20 @@
+from strip_hints import strip_file_to_string
+import os
+import fnmatch
+
+# From https://stackoverflow.com/questions/2186525/use-a-glob-to-find-files-recursively-in-python
+def find_files(directory, pattern):
+    for root, dirs, files in os.walk(directory):
+        for basename in files:
+            if fnmatch.fnmatch(basename, pattern):
+                filename = os.path.join(root, basename)
+                yield filename
+
+# Go up to modin root
+modin_path = os.path.relpath("../")
+
+for path in find_files(modin_path, '*.py'):
+    string = strip_file_to_string(path, to_empty=False, no_ast=False,
+                                  no_colon_move=False, only_assigns_and_defs=False)
+    with open(path, 'w') as f:
+        f.write(string)

modin/__init__.py

@@ -19,10 +19,30 @@ def _execute_cmd_in_temp_env(cmd):
 
     try:
         git_revision = _execute_cmd_in_temp_env(['git', 'rev-parse', 'HEAD'])
-        return git_revision.strip().decode('ascii')
+        return git_revision.strip().decode()
     except OSError:
         return "Unknown"
 
 
+def get_execution_engine():
+    # In the future, when there are multiple engines and different ways of
+    # backing the DataFrame, there will have to be some changed logic here to
+    # decide these things. In the meantime, we will use the currently supported
+    # execution engine + backing (Pandas + Ray).
+    return "Ray"
+
+
+def get_partition_format():
+    # See note above about engine + backing.
+    return "Pandas"
+
+
 __git_revision__ = git_version()
 __version__ = "0.1.2"
+__execution_engine__ = get_execution_engine()
+__partition_format__ = get_partition_format()
+
+# We don't want these used outside of this file.
+del git_version
+del get_execution_engine
+del get_partition_format