Merge pull request #442 from dengzq1234/master

improves compatibility with python3; enable tcoffee aligner, iqtree; fixes #421 and #342

.gitignore

@@ -22,3 +22,4 @@ test_tmp/
 # Coverage files
 .coverage
 htmlcov/
+ete

VERSION

@@ -1 +1 @@
-3.1.1
+3.1.2

ete3/evol/evoltree.py

@@ -126,7 +126,7 @@ def __init__(self, newick=None, alignment=None, alg_format="fasta",
         self.workdir = '/tmp/ete3-tmp/'
         if not binpath:
             ete3_path = which("ete3")
-            binpath = os.path.join(os.path.split(ete3_path)[0], "ete3_apps", "bin")
+            binpath = os.path.split(ete3_path)[0]
 
         self.execpath = binpath
         self._models = {}

ete3/parser/fasta.py

@@ -63,7 +63,7 @@ def read_fasta(source, obj=None, header_delimiter="\t", fix_duplicates=True):
             import gzip
             _source = gzip.open(source)
         else:
-            _source = open(source, "rU")
+            _source = open(source, "r")
     else:
         _source = iter(source.split("\n"))
 

ete3/parser/newick.py

@@ -248,7 +248,7 @@ class as the root(This allows to work with custom TreeNode
                 import gzip
                 nw = gzip.open(newick).read()
             else:
-                nw = open(newick, 'rU').read()
+                nw = open(newick, 'r').read()
         else:
             nw = newick
 

ete3/parser/paml.py

@@ -90,7 +90,7 @@ def read_paml (source, obj=None, header_delimiter="\t", fix_duplicates=True):
 
     # Prepares handle from which read sequences
     if os.path.isfile(source):
-        _source = open(source, "rU")
+        _source = open(source, "r")
     else:
         _source = iter(source.split("\n"))
 

ete3/parser/phylip.py

@@ -59,7 +59,7 @@ def read_phylip(source, interleaved=True, obj=None,
             import gzip
             _source = gzip.open(source)
         else:
-            _source = open(source, "rU")
+            _source = open(source, "r")
     else:
         _source = iter(source.split("\n"))
 

ete3/tools/ete.py

@@ -135,14 +135,15 @@ def _main(arguments):
             status = ete_upgrade_tools._main()
             sys.exit(status)
 
-        elif subcommand == "build":
+        elif subcommand == "build": 
             from . import ete_build
             del arguments[1]
 
             builtin_apps_path = None
             ete3_path = which("ete3")
-            if ete3_path: 
-                builtin_apps_path = os.path.join(os.path.split(ete3_path)[0], "ete3_apps")
+
+            if ete3_path:
+                builtin_apps_path = os.path.join(os.path.split(ete3_path)[0], "ete3_apps/bin")
             ete_build._main(arguments, builtin_apps_path)
 
             return

ete3/tools/ete_annotate.py

@@ -92,7 +92,7 @@ def run(args):
                            % annotation)
 
             features.add(aname)
-            for line in open(asource, 'rU'):
+            for line in open(asource, 'r'):
                 line = line.strip()
                 if not line or line.startswith('#'):
                     continue

ete3/tools/ete_build.cfg

@@ -152,12 +152,14 @@ _aligners = @muscle_default, @mafft_default, @clustalo_default
 [tcoffee_default]
 _desc = '(EXPERIMENTAL) tcoffee alignment with default paramerters'
 _app = tcoffee
+-output = "fasta_aln"
 
 [mcoffee_ensembl]
 _desc = '(EXPERIMENTAL) mcoffee alignment as used in the Ensembl database'
 _app = tcoffee
--method = "mafftgins_msa,muscle_msa,kalign_msa,t_coffee_msa"
--mode = mcoffee
+-method = "mafftgins_msa,muscle_msa,kalign_msa,t_coffee_msa" 
+-output = "fasta_aln"
+#-mode = mcoffee # choose mode or customize your methods combo
 
 [trimal001]
 _desc = 'trimal alignment cleaning removing columns with >1% gaps'
@@ -405,7 +407,7 @@ _inherits = iqtree_default
 _desc = 'IQTree '
 -m = TESTNEWONLY
 
-[iqtree_codon_default]
+[iqtree_codon_default] 
 _inherits = iqtree_default
 _desc = 'IQTree'
 -st = CODON
@@ -526,3 +528,4 @@ _min_outgroup_support = 0.90
 _outgroup_topology_dist = False
 _first_split = midpoint
 
+

ete3/tools/ete_build.py

@@ -652,9 +652,10 @@ def parse_workflows(names, target_wtype, parse_filters=False):
 def _main(arguments, builtin_apps_path=None):
     global BASEPATH, APPSPATH, args
 
+    # read path of ete_toolchain
     if builtin_apps_path:
         APPSPATH = builtin_apps_path
-
+        
     ETEHOMEDIR = os.path.expanduser("~/.etetoolkit/")
 
     if len(arguments) == 1:
@@ -669,7 +670,6 @@ def _main(arguments, builtin_apps_path=None):
 
     if len(arguments) > 1:
         _config_path = pjoin(BASEPATH, 'ete_build.cfg')
-
         if arguments[1] == "check":
             if not pexist(APPSPATH):
                 print(colorify('\nWARNING: external applications not found', "yellow"), file=sys.stderr)
@@ -998,7 +998,7 @@ def _main(arguments, builtin_apps_path=None):
     exec_group.add_argument("--nochecks", dest="nochecks",
                             action="store_true",
                             help="Skip basic checks (i.e. tools available) everytime the application starts.")
-
+    
     # Interface related flags
     ui_group = parser.add_argument_group("==== Program Interface Options ====")
     # ui_group.add_argument("-u", dest="enable_ui",

ete3/tools/ete_build_lib/apps.py

@@ -63,6 +63,7 @@
 builtin_apps = {
     'muscle'             : "%BIN%/muscle",
     'mafft'              : "MAFFT_BINARIES=%BIN%  %BIN%/mafft --thread %CORES%",
+    #'mafft'              : "%BIN%/mafft --thread %CORES%",
     'clustalo'           : "%BIN%/clustalo --threads %CORES%",
     'trimal'             : "%BIN%/trimal",
     'readal'             : "%BIN%/readal",
@@ -135,7 +136,7 @@ def get_call(appname, apps_path, exec_path, cores):
     except KeyError:
         return None
 
-    bin_path = os.path.join(apps_path, "bin")
+    bin_path = apps_path 
     tmp_path = os.path.join(exec_path, "tmp")
     #apps_base = apps_path.rstrip("/x86-64").rstrip("/x86-32")
     cmd = re.sub("%BIN%", bin_path, cmd)

ete3/tools/ete_build_lib/master_job.py

@@ -139,7 +139,7 @@ def write_pid(self, host, pid):
     def read_pid(self):
         try:
            host, pid = [_f.strip() for _f in 
-                        open(self.pid_file,"rU").readline().split("\t")]
+                        open(self.pid_file,"r").readline().split("\t")]
         except IOError:
             host, pid = "", ""
         else:

ete3/tools/ete_build_lib/master_task.py

@@ -262,7 +262,7 @@ def init(self):
 
     def get_saved_status(self):
         try:
-            return open(self.status_file, "rU").read(1)
+            return open(self.status_file, "r").read(1)
         except IOError:
             return "?"
 

ete3/tools/ete_build_lib/seqio.py

@@ -20,7 +20,7 @@ def iter_fasta_seqs(source):
             import gzip
             _source = gzip.open(source)
         else:
-            _source = open(source, "rU")
+            _source = open(source, "r")
     else:
         _source = iter(source.split("\n"))
 

ete3/tools/ete_build_lib/task/cog_selector.py

@@ -46,6 +46,7 @@
 import os
 import six
 from six.moves import map
+from functools import cmp_to_key
 log = logging.getLogger("main")
 
 from ..master_task import CogSelectorTask
@@ -54,6 +55,7 @@
                      md5, pjoin, _min, _max, _mean, _median, _std, iter_cog_seqs, cmp)
 from .. import db
 
+
 __all__ = ["CogSelector"]
 
 class CogSelector(CogSelectorTask):
@@ -143,13 +145,13 @@ def sort_cogs_by_sp_repr(c1, c2):
         cognumber += 1 # sets the ammount of cogs in file
         for sp, ncogs in sorted(list(sp2cogs.items()), key=lambda x: x[1], reverse=True):
             log.log(28, "% 20s  found in single copy in  % 6d (%0.1f%%) COGs " %(sp, ncogs, 100 * ncogs/float(cognumber)))
-
+        
         valid_cogs = sorted([sing for sing in all_singletons if len(sing) >= min_species],
-                            sort_cogs_by_size)
+                            key=cmp_to_key(sort_cogs_by_size))
 
         log.log(28, "Largest cog size: %s. Smallest cog size: %s" %(
                 largest_cog, smallest_cog))
-        self.cog_analysis = ""
+        self.cog_analysis = ""  
 
         # save original cog names hitting the hard limit
         if len(valid_cogs) > self.cog_hard_limit:

ete3/tools/ete_build_lib/task/concat_alg.py

@@ -42,6 +42,7 @@
 from collections import defaultdict
 import six
 from six.moves import zip
+from functools import cmp_to_key
 log = logging.getLogger("main")
 
 from . import Msf
@@ -252,7 +253,7 @@ def sort_single_algs(alg1, alg2):
         else:
             return r
 
-    sorted_algs = sorted(alg_objects, sort_single_algs)
+    sorted_algs = sorted(alg_objects,  key=cmp_to_key(sort_single_algs))
     concat_alg_lengths = [alg.seqlength for alg in sorted_algs]
     model2win = {}
     model2size = {}

ete3/tools/ete_build_lib/task/iqtree.py

@@ -29,8 +29,8 @@ def __init__(self, nodeid, alg_phylip_file, constrain_id, model,
             base_args["-st"] = "AA" if seqtype == "aa" else "DNA"            
         else:
             if conf[confname]["-st"].startswith("CODON"):
-                if seqtype == "aa" or "aa" not in GLOBALS["seqtypes"]:
-                    raise ConfigError("IQTREE CODON models require a codon alignmen.\nProvide nucleotide sequences with '-n' and set '--nt-switch-thr 0.0' to ensure codon alignments.")
+                if seqtype == "aa" or "aa" not in GLOBALS["seqtypes"]: 
+                    raise ConfigError("IQTREE CODON models require a codon alignment.\nProvide nucleotide sequences with '-n' and set '--nt-switch-thr 0.0' to ensure codon alignments.")
 
         if model:
             raise TaskError('External model selection not yet supported for IQTree')
@@ -53,7 +53,8 @@ def __init__(self, nodeid, alg_phylip_file, constrain_id, model,
         self.lk = None
 
         TreeTask.__init__(self, nodeid, "tree", "IQTree",
-                          base_args, conf[confname])            
+                          base_args, conf[confname])  
+
         self.init()
 
     def load_jobs(self):

ete3/tools/ete_build_lib/task/jmodeltest.py

@@ -87,7 +87,7 @@ def finish(self):
         best_model = None
         best_model_in_next_line = False
         t = None
-        for line in open(self.jobs[-1].stdout_file, "rU"):
+        for line in open(self.jobs[-1].stdout_file, "r"):
             line = line.strip()
             if best_model_in_next_line and line.startswith("Model"):
                 pass#best_model = line.split("=")[1].strip()

ete3/tools/ete_build_lib/task/tcoffee.py

@@ -29,7 +29,7 @@ def load_jobs(self):
         args[""] = pjoin(GLOBALS["input_dir"], self.multiseq_file)
         for k, v in self.args.items():
             args[k] = v
-        args["-outfile"] = "mcoffee.fasta"
+        args["-outfile"] = "mcoffee.fasta" 
         job = Job(self.conf["app"]["tcoffee"], args, parent_ids=[self.nodeid])
         job.add_input_file(self.multiseq_file)
         job.cores = self.conf["threading"]["tcoffee"]

ete3/tools/ete_build_lib/utils.py

@@ -651,4 +651,4 @@ def cmp(x, y):
     """cmp() exists in Python 2 but was removed in Python 3.
     This implements the same behavior on both versions.
     """
-    return (x > y) - (x < y)
+    return bool(x > y) - bool(x < y) 

ete3/version.py

@@ -1,2 +1,2 @@
 #autogenerated during release process. Do not modify
-__version__='3.1.1'
+__version__='3.1.2'

utils/conda_build/meta.yaml.template

@@ -8,7 +8,7 @@ source:
     #git_url: https://github.com/jhcepas/ete.git
 
 build:
-   noarch_python: True
+    noarch: python
 
 requirements:
   build:

utils/conda_build/release_conda.sh

@@ -2,4 +2,4 @@ rm meta.yml build.sh
 git push &&
 python -c 'new = open("meta.yaml.template").read().replace("%VERSION%", open("../../VERSION").readline().strip()); open("meta.yaml", "w").write(new)' &&
 python -c 'new = open("build.sh.template").read().replace("%VERSION%", open("../../VERSION").readline().strip()); open("build.sh", "w").write(new)' &&
-conda build --python 2.7 ./
+conda build --python 3.7 ./

utils/release.py

@@ -1,5 +1,5 @@
 import re
-import commands
+import subprocess
 import os
 import sys
 import readline
@@ -18,7 +18,7 @@
 
 def _ex(cmd, interrupt=True):
     if options.verbose or options.simulate:
-        print "***", cmd
+        print("***"), cmd
     if not options.simulate:
         s = os.system(cmd)
         if s != 0 and interrupt:
@@ -37,7 +37,7 @@ def ask(string, valid_values, default=-1, case_sensitive=False):
     while v not in valid_values:
         readline.set_startup_hook(lambda: readline.insert_text(default))
         try:
-            v = raw_input("%s [%s] " % (string, ', '.join(valid_values))).strip()
+            v = input("%s [%s] " % (string, ', '.join(valid_values))).strip()
             if v == '' and default>=0:
                 v = valid_values[default]
             if not case_sensitive:
@@ -49,7 +49,7 @@ def ask(string, valid_values, default=-1, case_sensitive=False):
 def ask_path(string, default_path):
     v = None
     while v is None:
-        v = raw_input("%s [%s] " % (string, default_path)).strip()
+        v = input("%s [%s] " % (string, default_path)).strip()
         if v == '':
             v = default_path
         if not os.path.exists(v):
@@ -65,11 +65,11 @@ def ask_path(string, default_path):
 a, b, c, tag, ncom, hcom  = re.search("(\d+)\.(\d+)\.(\d+)(-?\w+\d+)?-?(\d+)?-?(\w+)?", CURRENT_VERSION).groups()
 a, b, c = map(int, (a, b, c))
 SERIES_VERSION = "%s.%s" %(a, b)
-print '===================================================='
-print 'CURRENT VERSION:', a, b, c, tag, ncom, hcom
-print '===================================================='
+print('====================================================')
+print('CURRENT VERSION:', a, b, c, tag, ncom, hcom)
+print('====================================================')
 # test examples
-raw_input('continue?')
+input('continue?')
 
 if not options.doconly:
     # commit changes in VERSION
@@ -81,7 +81,7 @@ def ask_path(string, default_path):
         NEW_VERSION = "%s.%s.%s" %(a, b, c+1)
 
     if ask('Increase version to "%s" ?' %NEW_VERSION, ['y', 'n']) == 'n':
-        NEW_VERSION = raw_input('new version string:').strip()
+        NEW_VERSION = input('new version string:').strip()
 
     if ask('Write "%s" and commit changes?' %NEW_VERSION, ['y', 'n']) == 'y':
         open('../VERSION', 'w').write(NEW_VERSION)        
@@ -94,13 +94,14 @@ def ask_path(string, default_path):
     _ex('rm -rf release/  && git clone .. release/')
 
     # build docs
-    _ex('cd release/sdoc/ && make html && make latex')
-    _ex('cd release/sdoc/_build/latex && make all-pdf')
+    ## _ex('cd release/sdoc/ && make html && make latex')
+    ## _ex('cd release/sdoc/_build/latex && make all-pdf')
     # Generates HTML doc (it includes a link to the PDF doc, so it
     # must be executed after PDF commands)
-    _ex('cp -a release/sdoc/_build/html/ release/doc/')
-    _ex('cp -a release/sdoc/_build/latex/*.pdf release/doc/')
+    ## _ex('cp -a release/sdoc/_build/html/ release/doc/')
+    ## _ex('cp -a release/sdoc/_build/latex/*.pdf release/doc/')
     # Build dist
+
     _ex('cd release/ && python setup.py sdist')
 
     # test distribution