diff --git a/.gitignore b/.gitignore index 110dfd05d..1c8b888ac 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,4 @@ test_tmp/ # Coverage files .coverage htmlcov/ +ete diff --git a/VERSION b/VERSION index 50e47c89c..ef538c281 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.1.1 \ No newline at end of file +3.1.2 diff --git a/ete3/evol/evoltree.py b/ete3/evol/evoltree.py index 54369a571..d37f5de94 100644 --- a/ete3/evol/evoltree.py +++ b/ete3/evol/evoltree.py @@ -126,7 +126,7 @@ def __init__(self, newick=None, alignment=None, alg_format="fasta", self.workdir = '/tmp/ete3-tmp/' if not binpath: ete3_path = which("ete3") - binpath = os.path.join(os.path.split(ete3_path)[0], "ete3_apps", "bin") + binpath = os.path.split(ete3_path)[0] self.execpath = binpath self._models = {} diff --git a/ete3/parser/fasta.py b/ete3/parser/fasta.py index a3e111ba4..84b285e1a 100644 --- a/ete3/parser/fasta.py +++ b/ete3/parser/fasta.py @@ -63,7 +63,7 @@ def read_fasta(source, obj=None, header_delimiter="\t", fix_duplicates=True): import gzip _source = gzip.open(source) else: - _source = open(source, "rU") + _source = open(source, "r") else: _source = iter(source.split("\n")) diff --git a/ete3/parser/newick.py b/ete3/parser/newick.py index 72d0fd197..42608cbee 100644 --- a/ete3/parser/newick.py +++ b/ete3/parser/newick.py @@ -248,7 +248,7 @@ class as the root(This allows to work with custom TreeNode import gzip nw = gzip.open(newick).read() else: - nw = open(newick, 'rU').read() + nw = open(newick, 'r').read() else: nw = newick diff --git a/ete3/parser/paml.py b/ete3/parser/paml.py index 21f83a0f5..6675d8a54 100644 --- a/ete3/parser/paml.py +++ b/ete3/parser/paml.py @@ -90,7 +90,7 @@ def read_paml (source, obj=None, header_delimiter="\t", fix_duplicates=True): # Prepares handle from which read sequences if os.path.isfile(source): - _source = open(source, "rU") + _source = open(source, "r") else: _source = iter(source.split("\n")) diff --git a/ete3/parser/phylip.py b/ete3/parser/phylip.py index c029d9c9b..ba62b13e7 100644 --- a/ete3/parser/phylip.py +++ b/ete3/parser/phylip.py @@ -59,7 +59,7 @@ def read_phylip(source, interleaved=True, obj=None, import gzip _source = gzip.open(source) else: - _source = open(source, "rU") + _source = open(source, "r") else: _source = iter(source.split("\n")) diff --git a/ete3/tools/ete.py b/ete3/tools/ete.py index f3c560e3f..dfbf712a4 100755 --- a/ete3/tools/ete.py +++ b/ete3/tools/ete.py @@ -135,14 +135,15 @@ def _main(arguments): status = ete_upgrade_tools._main() sys.exit(status) - elif subcommand == "build": + elif subcommand == "build": from . import ete_build del arguments[1] builtin_apps_path = None ete3_path = which("ete3") - if ete3_path: - builtin_apps_path = os.path.join(os.path.split(ete3_path)[0], "ete3_apps") + + if ete3_path: + builtin_apps_path = os.path.join(os.path.split(ete3_path)[0], "ete3_apps/bin") ete_build._main(arguments, builtin_apps_path) return diff --git a/ete3/tools/ete_annotate.py b/ete3/tools/ete_annotate.py index d28991a17..3d614ab90 100644 --- a/ete3/tools/ete_annotate.py +++ b/ete3/tools/ete_annotate.py @@ -92,7 +92,7 @@ def run(args): % annotation) features.add(aname) - for line in open(asource, 'rU'): + for line in open(asource, 'r'): line = line.strip() if not line or line.startswith('#'): continue diff --git a/ete3/tools/ete_build.cfg b/ete3/tools/ete_build.cfg index 01a87a0c5..9e290021a 100644 --- a/ete3/tools/ete_build.cfg +++ b/ete3/tools/ete_build.cfg @@ -152,12 +152,14 @@ _aligners = @muscle_default, @mafft_default, @clustalo_default [tcoffee_default] _desc = '(EXPERIMENTAL) tcoffee alignment with default paramerters' _app = tcoffee +-output = "fasta_aln" [mcoffee_ensembl] _desc = '(EXPERIMENTAL) mcoffee alignment as used in the Ensembl database' _app = tcoffee --method = "mafftgins_msa,muscle_msa,kalign_msa,t_coffee_msa" --mode = mcoffee +-method = "mafftgins_msa,muscle_msa,kalign_msa,t_coffee_msa" +-output = "fasta_aln" +#-mode = mcoffee # choose mode or customize your methods combo [trimal001] _desc = 'trimal alignment cleaning removing columns with >1% gaps' @@ -405,7 +407,7 @@ _inherits = iqtree_default _desc = 'IQTree ' -m = TESTNEWONLY -[iqtree_codon_default] +[iqtree_codon_default] _inherits = iqtree_default _desc = 'IQTree' -st = CODON @@ -526,3 +528,4 @@ _min_outgroup_support = 0.90 _outgroup_topology_dist = False _first_split = midpoint + diff --git a/ete3/tools/ete_build.py b/ete3/tools/ete_build.py index e523b7e57..3ac666b8e 100755 --- a/ete3/tools/ete_build.py +++ b/ete3/tools/ete_build.py @@ -652,9 +652,10 @@ def parse_workflows(names, target_wtype, parse_filters=False): def _main(arguments, builtin_apps_path=None): global BASEPATH, APPSPATH, args + # read path of ete_toolchain if builtin_apps_path: APPSPATH = builtin_apps_path - + ETEHOMEDIR = os.path.expanduser("~/.etetoolkit/") if len(arguments) == 1: @@ -669,7 +670,6 @@ def _main(arguments, builtin_apps_path=None): if len(arguments) > 1: _config_path = pjoin(BASEPATH, 'ete_build.cfg') - if arguments[1] == "check": if not pexist(APPSPATH): print(colorify('\nWARNING: external applications not found', "yellow"), file=sys.stderr) @@ -998,7 +998,7 @@ def _main(arguments, builtin_apps_path=None): exec_group.add_argument("--nochecks", dest="nochecks", action="store_true", help="Skip basic checks (i.e. tools available) everytime the application starts.") - + # Interface related flags ui_group = parser.add_argument_group("==== Program Interface Options ====") # ui_group.add_argument("-u", dest="enable_ui", diff --git a/ete3/tools/ete_build_lib/apps.py b/ete3/tools/ete_build_lib/apps.py index 450dbd4ba..99bd63271 100644 --- a/ete3/tools/ete_build_lib/apps.py +++ b/ete3/tools/ete_build_lib/apps.py @@ -63,6 +63,7 @@ builtin_apps = { 'muscle' : "%BIN%/muscle", 'mafft' : "MAFFT_BINARIES=%BIN% %BIN%/mafft --thread %CORES%", + #'mafft' : "%BIN%/mafft --thread %CORES%", 'clustalo' : "%BIN%/clustalo --threads %CORES%", 'trimal' : "%BIN%/trimal", 'readal' : "%BIN%/readal", @@ -135,7 +136,7 @@ def get_call(appname, apps_path, exec_path, cores): except KeyError: return None - bin_path = os.path.join(apps_path, "bin") + bin_path = apps_path tmp_path = os.path.join(exec_path, "tmp") #apps_base = apps_path.rstrip("/x86-64").rstrip("/x86-32") cmd = re.sub("%BIN%", bin_path, cmd) diff --git a/ete3/tools/ete_build_lib/master_job.py b/ete3/tools/ete_build_lib/master_job.py index 61e73fbae..148f90595 100644 --- a/ete3/tools/ete_build_lib/master_job.py +++ b/ete3/tools/ete_build_lib/master_job.py @@ -139,7 +139,7 @@ def write_pid(self, host, pid): def read_pid(self): try: host, pid = [_f.strip() for _f in - open(self.pid_file,"rU").readline().split("\t")] + open(self.pid_file,"r").readline().split("\t")] except IOError: host, pid = "", "" else: diff --git a/ete3/tools/ete_build_lib/master_task.py b/ete3/tools/ete_build_lib/master_task.py index fcceb108f..6986ab6b7 100644 --- a/ete3/tools/ete_build_lib/master_task.py +++ b/ete3/tools/ete_build_lib/master_task.py @@ -262,7 +262,7 @@ def init(self): def get_saved_status(self): try: - return open(self.status_file, "rU").read(1) + return open(self.status_file, "r").read(1) except IOError: return "?" diff --git a/ete3/tools/ete_build_lib/seqio.py b/ete3/tools/ete_build_lib/seqio.py index 2e8c78c67..b0d8527f2 100644 --- a/ete3/tools/ete_build_lib/seqio.py +++ b/ete3/tools/ete_build_lib/seqio.py @@ -20,7 +20,7 @@ def iter_fasta_seqs(source): import gzip _source = gzip.open(source) else: - _source = open(source, "rU") + _source = open(source, "r") else: _source = iter(source.split("\n")) diff --git a/ete3/tools/ete_build_lib/task/cog_selector.py b/ete3/tools/ete_build_lib/task/cog_selector.py index 967b1e2cd..35681ed47 100644 --- a/ete3/tools/ete_build_lib/task/cog_selector.py +++ b/ete3/tools/ete_build_lib/task/cog_selector.py @@ -46,6 +46,7 @@ import os import six from six.moves import map +from functools import cmp_to_key log = logging.getLogger("main") from ..master_task import CogSelectorTask @@ -54,6 +55,7 @@ md5, pjoin, _min, _max, _mean, _median, _std, iter_cog_seqs, cmp) from .. import db + __all__ = ["CogSelector"] class CogSelector(CogSelectorTask): @@ -143,13 +145,13 @@ def sort_cogs_by_sp_repr(c1, c2): cognumber += 1 # sets the ammount of cogs in file for sp, ncogs in sorted(list(sp2cogs.items()), key=lambda x: x[1], reverse=True): log.log(28, "% 20s found in single copy in % 6d (%0.1f%%) COGs " %(sp, ncogs, 100 * ncogs/float(cognumber))) - + valid_cogs = sorted([sing for sing in all_singletons if len(sing) >= min_species], - sort_cogs_by_size) + key=cmp_to_key(sort_cogs_by_size)) log.log(28, "Largest cog size: %s. Smallest cog size: %s" %( largest_cog, smallest_cog)) - self.cog_analysis = "" + self.cog_analysis = "" # save original cog names hitting the hard limit if len(valid_cogs) > self.cog_hard_limit: diff --git a/ete3/tools/ete_build_lib/task/concat_alg.py b/ete3/tools/ete_build_lib/task/concat_alg.py index 47a27507a..2826e27bb 100644 --- a/ete3/tools/ete_build_lib/task/concat_alg.py +++ b/ete3/tools/ete_build_lib/task/concat_alg.py @@ -42,6 +42,7 @@ from collections import defaultdict import six from six.moves import zip +from functools import cmp_to_key log = logging.getLogger("main") from . import Msf @@ -252,7 +253,7 @@ def sort_single_algs(alg1, alg2): else: return r - sorted_algs = sorted(alg_objects, sort_single_algs) + sorted_algs = sorted(alg_objects, key=cmp_to_key(sort_single_algs)) concat_alg_lengths = [alg.seqlength for alg in sorted_algs] model2win = {} model2size = {} diff --git a/ete3/tools/ete_build_lib/task/iqtree.py b/ete3/tools/ete_build_lib/task/iqtree.py index cd3baa654..2a0c4707c 100644 --- a/ete3/tools/ete_build_lib/task/iqtree.py +++ b/ete3/tools/ete_build_lib/task/iqtree.py @@ -29,8 +29,8 @@ def __init__(self, nodeid, alg_phylip_file, constrain_id, model, base_args["-st"] = "AA" if seqtype == "aa" else "DNA" else: if conf[confname]["-st"].startswith("CODON"): - if seqtype == "aa" or "aa" not in GLOBALS["seqtypes"]: - raise ConfigError("IQTREE CODON models require a codon alignmen.\nProvide nucleotide sequences with '-n' and set '--nt-switch-thr 0.0' to ensure codon alignments.") + if seqtype == "aa" or "aa" not in GLOBALS["seqtypes"]: + raise ConfigError("IQTREE CODON models require a codon alignment.\nProvide nucleotide sequences with '-n' and set '--nt-switch-thr 0.0' to ensure codon alignments.") if model: raise TaskError('External model selection not yet supported for IQTree') @@ -53,7 +53,8 @@ def __init__(self, nodeid, alg_phylip_file, constrain_id, model, self.lk = None TreeTask.__init__(self, nodeid, "tree", "IQTree", - base_args, conf[confname]) + base_args, conf[confname]) + self.init() def load_jobs(self): diff --git a/ete3/tools/ete_build_lib/task/jmodeltest.py b/ete3/tools/ete_build_lib/task/jmodeltest.py index 091d574eb..d996f1064 100644 --- a/ete3/tools/ete_build_lib/task/jmodeltest.py +++ b/ete3/tools/ete_build_lib/task/jmodeltest.py @@ -87,7 +87,7 @@ def finish(self): best_model = None best_model_in_next_line = False t = None - for line in open(self.jobs[-1].stdout_file, "rU"): + for line in open(self.jobs[-1].stdout_file, "r"): line = line.strip() if best_model_in_next_line and line.startswith("Model"): pass#best_model = line.split("=")[1].strip() diff --git a/ete3/tools/ete_build_lib/task/tcoffee.py b/ete3/tools/ete_build_lib/task/tcoffee.py index a03a2a768..9b963f9ad 100644 --- a/ete3/tools/ete_build_lib/task/tcoffee.py +++ b/ete3/tools/ete_build_lib/task/tcoffee.py @@ -29,7 +29,7 @@ def load_jobs(self): args[""] = pjoin(GLOBALS["input_dir"], self.multiseq_file) for k, v in self.args.items(): args[k] = v - args["-outfile"] = "mcoffee.fasta" + args["-outfile"] = "mcoffee.fasta" job = Job(self.conf["app"]["tcoffee"], args, parent_ids=[self.nodeid]) job.add_input_file(self.multiseq_file) job.cores = self.conf["threading"]["tcoffee"] diff --git a/ete3/tools/ete_build_lib/utils.py b/ete3/tools/ete_build_lib/utils.py index 259737762..d84697156 100644 --- a/ete3/tools/ete_build_lib/utils.py +++ b/ete3/tools/ete_build_lib/utils.py @@ -651,4 +651,4 @@ def cmp(x, y): """cmp() exists in Python 2 but was removed in Python 3. This implements the same behavior on both versions. """ - return (x > y) - (x < y) + return bool(x > y) - bool(x < y) diff --git a/ete3/version.py b/ete3/version.py index 2288bbe98..9552785ea 100644 --- a/ete3/version.py +++ b/ete3/version.py @@ -1,2 +1,2 @@ #autogenerated during release process. Do not modify -__version__='3.1.1' +__version__='3.1.2' diff --git a/utils/conda_build/meta.yaml.template b/utils/conda_build/meta.yaml.template index 245cfdf18..8cfce8c4a 100644 --- a/utils/conda_build/meta.yaml.template +++ b/utils/conda_build/meta.yaml.template @@ -8,7 +8,7 @@ source: #git_url: https://github.com/jhcepas/ete.git build: - noarch_python: True + noarch: python requirements: build: diff --git a/utils/conda_build/release_conda.sh b/utils/conda_build/release_conda.sh index ad253be44..3103265f4 100755 --- a/utils/conda_build/release_conda.sh +++ b/utils/conda_build/release_conda.sh @@ -2,4 +2,4 @@ rm meta.yml build.sh git push && python -c 'new = open("meta.yaml.template").read().replace("%VERSION%", open("../../VERSION").readline().strip()); open("meta.yaml", "w").write(new)' && python -c 'new = open("build.sh.template").read().replace("%VERSION%", open("../../VERSION").readline().strip()); open("build.sh", "w").write(new)' && -conda build --python 2.7 ./ +conda build --python 3.7 ./ diff --git a/utils/release.py b/utils/release.py index de8e7eefd..7b56d5eeb 100755 --- a/utils/release.py +++ b/utils/release.py @@ -1,5 +1,5 @@ import re -import commands +import subprocess import os import sys import readline @@ -18,7 +18,7 @@ def _ex(cmd, interrupt=True): if options.verbose or options.simulate: - print "***", cmd + print("***"), cmd if not options.simulate: s = os.system(cmd) if s != 0 and interrupt: @@ -37,7 +37,7 @@ def ask(string, valid_values, default=-1, case_sensitive=False): while v not in valid_values: readline.set_startup_hook(lambda: readline.insert_text(default)) try: - v = raw_input("%s [%s] " % (string, ', '.join(valid_values))).strip() + v = input("%s [%s] " % (string, ', '.join(valid_values))).strip() if v == '' and default>=0: v = valid_values[default] if not case_sensitive: @@ -49,7 +49,7 @@ def ask(string, valid_values, default=-1, case_sensitive=False): def ask_path(string, default_path): v = None while v is None: - v = raw_input("%s [%s] " % (string, default_path)).strip() + v = input("%s [%s] " % (string, default_path)).strip() if v == '': v = default_path if not os.path.exists(v): @@ -65,11 +65,11 @@ def ask_path(string, default_path): a, b, c, tag, ncom, hcom = re.search("(\d+)\.(\d+)\.(\d+)(-?\w+\d+)?-?(\d+)?-?(\w+)?", CURRENT_VERSION).groups() a, b, c = map(int, (a, b, c)) SERIES_VERSION = "%s.%s" %(a, b) -print '====================================================' -print 'CURRENT VERSION:', a, b, c, tag, ncom, hcom -print '====================================================' +print('====================================================') +print('CURRENT VERSION:', a, b, c, tag, ncom, hcom) +print('====================================================') # test examples -raw_input('continue?') +input('continue?') if not options.doconly: # commit changes in VERSION @@ -81,7 +81,7 @@ def ask_path(string, default_path): NEW_VERSION = "%s.%s.%s" %(a, b, c+1) if ask('Increase version to "%s" ?' %NEW_VERSION, ['y', 'n']) == 'n': - NEW_VERSION = raw_input('new version string:').strip() + NEW_VERSION = input('new version string:').strip() if ask('Write "%s" and commit changes?' %NEW_VERSION, ['y', 'n']) == 'y': open('../VERSION', 'w').write(NEW_VERSION) @@ -94,13 +94,14 @@ def ask_path(string, default_path): _ex('rm -rf release/ && git clone .. release/') # build docs - _ex('cd release/sdoc/ && make html && make latex') - _ex('cd release/sdoc/_build/latex && make all-pdf') + ## _ex('cd release/sdoc/ && make html && make latex') + ## _ex('cd release/sdoc/_build/latex && make all-pdf') # Generates HTML doc (it includes a link to the PDF doc, so it # must be executed after PDF commands) - _ex('cp -a release/sdoc/_build/html/ release/doc/') - _ex('cp -a release/sdoc/_build/latex/*.pdf release/doc/') + ## _ex('cp -a release/sdoc/_build/html/ release/doc/') + ## _ex('cp -a release/sdoc/_build/latex/*.pdf release/doc/') # Build dist + _ex('cd release/ && python setup.py sdist') # test distribution