Combat batch effects

.travis.yml

@@ -25,8 +25,6 @@ install:
   # Specific notebook version for documentaiton
   # Install the rest via pip (brewer2mpl, gspread, seaborn, etc)
   - pip install -r requirements.txt
-  - pip install pep8
-  - pip install https://github.com/dcramer/pyflakes/tarball/master
   - pip install coveralls
 
 before_script:
@@ -41,7 +39,6 @@ before_script:
 script:
   - make coverage
   - make lint
-  - make pep8
   # For building the docs
   - MASTER=master
   - if [[ $TRAVIS_BRANCH == v*.*.* ]] ; then export DEPLOY_HTML_DIR=docs ; fi

CONTRIBUTING.md

@@ -1,11 +1,10 @@
+Contributing to `flotilla`
+==========================
 
-For developers
-==============
-
-Please put ALL import statements at the top of the `*.py` file (potentially underneath docstrings, of course).
-The only exception is if a package is not listed in `requirements.txt`,then a "function-only" import may be allowed.
-If this doesn't make sense to you, just put the import at the top of the file.
-
+You are an awesome human being for wanting to contribute to the `flotilla` 
+large-scale RNA-seq open-source software! Check out our guidelines below for
+how to best format your code and features so your sweet new idea can be added
+as soon as possible.
 
 Install in development mode
 ---------------------------
@@ -110,7 +109,7 @@ before the pull request is merged.
   `examples/`, and is that notebook added to `doc/tutorial.rst`?
 - [ ] If it adds a new plot, is it documented in the gallery, as a `.py` file 
   in `examples/`?
-- [ ] Is it well formatted? Look at `make pep8` and `make lint` output
+- [ ] Is it well formatted? Look at the output of `make lint`.
 - [ ] Is it documented in the doc/releases/?
 - [ ] Was a spellchecker run on the source code and documentation after
   changes were made?

Makefile

@@ -11,7 +11,4 @@ coverage:
 	rm matplotlibrc
 
 lint:
-	pyflakes flotilla
-
-pep8:
-	pep8 flotilla
+	flake8 --exclude flotilla/external,doc flotilla

conda_requirements.txt

@@ -26,4 +26,5 @@ jinja2
 joblib >= 0.8.4
 ipython
 jsonschema
-pyyaml
+pyyaml
+flake8

flotilla/data_model/metadata.py

@@ -24,6 +24,7 @@ def __init__(self, data, phenotype_order=None, phenotype_to_color=None,
                  pooled_col=POOLED_COL,
                  outlier_col=OUTLIER_COL,
                  predictor_config_manager=None,
+                 ignore_subset_cols=None,
                  minimum_sample_subset=MINIMUM_SAMPLE_SUBSET):
         super(MetaData, self).__init__(
             data, outliers=None,
@@ -37,6 +38,7 @@ def __init__(self, data, phenotype_order=None, phenotype_to_color=None,
         self.pooled_col = pooled_col
         self.minimum_sample_subset = minimum_sample_subset
         self.outlier_col = outlier_col
+        self.ignore_subset_cols = ignore_subset_cols
 
         phenotypes_not_in_order = set(self.unique_phenotypes).difference(
             set(self.phenotype_order))
@@ -196,7 +198,7 @@ def sample_id_to_color(self):
     @property
     def sample_subsets(self):
         return subsets_from_metadata(self.data, self.minimum_sample_subset,
-                                     'samples')
+                                     'samples', ignore=self.ignore_subset_cols)
 
     @property
     def phenotype_series(self):

flotilla/data_model/study.py

@@ -73,13 +73,25 @@ class Study(object):
 
     _default_plot_kwargs = {'marker': 'o', 'color': blue}
 
-    def __init__(self, sample_metadata, version='0.1.0', expression_data=None,
+    def __init__(self, sample_metadata, version='0.1.0',
+                 metadata_pooled_col=POOLED_COL,
+                 metadata_minimum_samples=0,
+                 metadata_phenotype_col=PHENOTYPE_COL,
+                 metadata_phenotype_order=None,
+                 metadata_phenotype_to_color=None,
+                 metadata_phenotype_to_marker=None,
+                 metadata_outlier_col=OUTLIER_COL,
+                 metadata_ignore_subset_cols=None,
+                 metadata_batch_cols='batch',
+                 metadata_batch_min_samples=3,
+                 expression_data=None,
                  expression_feature_data=None,
                  expression_feature_rename_col=None,
                  expression_feature_ignore_subset_cols=None,
                  expression_log_base=None,
                  expression_thresh=-np.inf,
                  expression_plus_one=False,
+                 expression_correct_batch_effects=False,
                  splicing_data=None,
                  splicing_feature_data=None,
                  splicing_feature_rename_col=None,
@@ -93,13 +105,6 @@ def __init__(self, sample_metadata, version='0.1.0', expression_data=None,
                  drop_outliers=True, species=None,
                  gene_ontology_data=None,
                  predictor_config_manager=None,
-                 metadata_pooled_col=POOLED_COL,
-                 metadata_minimum_samples=0,
-                 metadata_phenotype_col=PHENOTYPE_COL,
-                 metadata_phenotype_order=None,
-                 metadata_phenotype_to_color=None,
-                 metadata_phenotype_to_marker=None,
-                 metadata_outlier_col=OUTLIER_COL,
                  license=None, title=None, sources=None,
                  default_sample_subset="all_samples",
                  default_feature_subset="variant",
@@ -222,6 +227,7 @@ def __init__(self, sample_metadata, version='0.1.0', expression_data=None,
             sample_metadata, metadata_phenotype_order,
             metadata_phenotype_to_color,
             metadata_phenotype_to_marker, pooled_col=metadata_pooled_col,
+            ignore_subset_cols=metadata_ignore_subset_cols,
             outlier_col=metadata_outlier_col,
             phenotype_col=metadata_phenotype_col,
             predictor_config_manager=self.predictor_config_manager,
@@ -611,7 +617,7 @@ def load_species_data(
                 compression = None if 'compression' not in resource else \
                     resource['compression']
                 name = resource['name']
-                dfs[name] = reader(filename,
+                dfs[name] = reader(filename, index_col=0,
                                    compression=compression)
                 other_keys = set(resource.keys()).difference(
                     DATAPACKAGE_RESOURCE_COMMON_KWS)