From 9ea6be8e9cb0cee4c2239788ac222d40f188024c Mon Sep 17 00:00:00 2001 From: Ryan Dale Date: Mon, 25 May 2020 08:27:37 -0400 Subject: [PATCH 01/11] test on py38 --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 9ca25dbe..c6752af6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,6 +13,7 @@ matrix: - python: 3.5 - python: 3.6 - python: 3.7 + - python: 3.8 dist: xenial sudo: true From 53cebeca6881421c0d8900b1707c50f67a166744 Mon Sep 17 00:00:00 2001 From: Ryan Dale Date: Mon, 25 May 2020 12:07:35 -0400 Subject: [PATCH 02/11] filter out pkgs problematic via conda in py3.8 --- condatest.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/condatest.sh b/condatest.sh index 643e5781..00c38603 100755 --- a/condatest.sh +++ b/condatest.sh @@ -60,13 +60,16 @@ if ! conda env list | grep -q $no_cy; then # pysam not available from bioconda for py37 so remove it from # requirements. TMPREQS=$(tempfile) - grep -v pysam requirements.txt > $TMPREQS + REQS=requirements.txt if [[ "$PY_VERSION" == "3.7" ]]; then + grep -v pysam requirements.txt > $TMPREQS REQS=$TMPREQS - else - REQS=requirements.txt fi + if [[ "$PY_VERSION" == "3.8" ]]; then + grep -v -E "matplotlib|pysam" requirements.txt > $TMPREQS + REQS=$TMPREQS + fi conda create -n $no_cy -y \ --channel conda-forge \ --channel bioconda \ From bc19c09f5983f2586987b112e3a80c21979db0fd Mon Sep 17 00:00:00 2001 From: Ryan Dale Date: Mon, 1 Jun 2020 10:39:15 -0400 Subject: [PATCH 03/11] rename genomepy integration test --- .../{test_genomepy_integration.py => genomepy_integration.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename pybedtools/test/{test_genomepy_integration.py => genomepy_integration.py} (100%) diff --git a/pybedtools/test/test_genomepy_integration.py b/pybedtools/test/genomepy_integration.py similarity index 100% rename from pybedtools/test/test_genomepy_integration.py rename to pybedtools/test/genomepy_integration.py From eb69cadf734a45bafe744a95cd73a1c6cfc6a351 Mon Sep 17 00:00:00 2001 From: Ryan Dale Date: Mon, 1 Jun 2020 10:39:44 -0400 Subject: [PATCH 04/11] use renamed test --- condatest.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/condatest.sh b/condatest.sh index b7455ff0..50115623 100755 --- a/condatest.sh +++ b/condatest.sh @@ -98,8 +98,8 @@ pip install -e . # manipulation in test_helpers and test_issues. So run in its own separate # pytests process. log "Unit tests" -pytest -v --doctest-modules --ignore pybedtools/test/test_genomepy_integration.py -pytest -v pybedtools/test/test_genomepy_integration.py +pytest -v --doctest-modules +pytest -v pybedtools/test/genomepy_integration.py # ---------------------------------------------------------------------------- # sphinx doctests From db104ffaca7959b0d6b029a4314ba413853741c4 Mon Sep 17 00:00:00 2001 From: Ryan Dale Date: Mon, 1 Jun 2020 10:40:03 -0400 Subject: [PATCH 05/11] fix warning from numpy --- pybedtools/bedtool.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pybedtools/bedtool.py b/pybedtools/bedtool.py index 3a6e3620..82017368 100644 --- a/pybedtools/bedtool.py +++ b/pybedtools/bedtool.py @@ -2794,7 +2794,7 @@ def percentileofscore(a, score): a_len = np.array(list(range(len(a)))) + 1.0 a = np.sort(a) - idx = [a == score] + idx = tuple([a == score]) pct = (np.mean(a_len[idx]) / n) * 100.0 return pct From 2da70f68c668dd7be42fe2672023c9c766335af2 Mon Sep 17 00:00:00 2001 From: Ryan Dale Date: Mon, 1 Jun 2020 10:40:33 -0400 Subject: [PATCH 06/11] fix warning from pandas --- pybedtools/test/test_issues.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pybedtools/test/test_issues.py b/pybedtools/test/test_issues.py index 5936b2f7..9b93ccab 100644 --- a/pybedtools/test/test_issues.py +++ b/pybedtools/test/test_issues.py @@ -690,10 +690,11 @@ def test_issue_251(): def test_issue_257(): try: import pandas + import numpy as np except ImportError: pytest.mark.skip("Pandas not installed; skipping") df = pybedtools.example_bedtool("a.bed").to_dataframe() - df.iloc[-1, -3:] = pandas.np.nan + df.iloc[-1, -3:] = np.nan b = pybedtools.BedTool.from_dataframe(df) assert str(b) == fix( """ From 118d8868353002059ca80dfeb6c5c798bd4d75e9 Mon Sep 17 00:00:00 2001 From: Ryan Dale Date: Mon, 1 Jun 2020 10:40:45 -0400 Subject: [PATCH 07/11] force tabix indexing (make local tests more convenient) --- pybedtools/test/test_1.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pybedtools/test/test_1.py b/pybedtools/test/test_1.py index fff107fd..ba61283e 100644 --- a/pybedtools/test/test_1.py +++ b/pybedtools/test/test_1.py @@ -144,7 +144,7 @@ def test_tuple_creation(): def test_tabix(): try: a = pybedtools.example_bedtool("a.bed") - t = a.tabix() + t = a.tabix(force=True) assert t._tabixed() results = t.tabix_intervals("chr1:99-200") results = str(results) From 35764f571cae5534eb0bff383c2c078882e5fcd9 Mon Sep 17 00:00:00 2001 From: Ryan Dale Date: Mon, 1 Jun 2020 10:41:24 -0400 Subject: [PATCH 08/11] try BUFSIZE=-1 to avoid py3.8 warnings --- pybedtools/helpers.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pybedtools/helpers.py b/pybedtools/helpers.py index d787c689..7194c8bf 100644 --- a/pybedtools/helpers.py +++ b/pybedtools/helpers.py @@ -25,7 +25,7 @@ from .logger import logger from .cbedtools import create_interval_from_list -BUFSIZE = 1 +BUFSIZE = -1 _tags = {} @@ -744,7 +744,7 @@ def get_chromsizes_from_ucsc( d = {} try: p = subprocess.Popen( - cmds, stdout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize=1 + cmds, stdout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize=BUFSIZE ) stdout, stderr = p.communicate() if stderr: @@ -770,7 +770,7 @@ def get_chromsizes_from_ucsc( try: cmds = [fetchchromsizes, genome] p = subprocess.Popen( - cmds, stdout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize=1 + cmds, stdout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize=BUFSIZE ) stdout, stderr = p.communicate() if stderr: From 43728120454ea16e978461caac2c52ea3d36c223 Mon Sep 17 00:00:00 2001 From: Ryan Dale Date: Mon, 1 Jun 2020 10:51:52 -0400 Subject: [PATCH 09/11] handle pysam for py38 --- condatest.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/condatest.sh b/condatest.sh index 50115623..2e2001a7 100755 --- a/condatest.sh +++ b/condatest.sh @@ -57,8 +57,8 @@ no_cy="pbtpy${PY_VERSION}_conda_no_cython" if ! conda env list | grep -q $no_cy; then log "creating environment" - # pysam not available from bioconda for py37 so remove it from - # requirements. + # conflicts with pysam in bioconda for py37 or py38 so remove it from + # conda requirements; allow to install from pip TMPREQS=$(tempfile) REQS=requirements.txt if [[ "$PY_VERSION" == "3.7" ]]; then @@ -66,6 +66,11 @@ if ! conda env list | grep -q $no_cy; then REQS=$TMPREQS fi + if [[ "$PY_VERSION" == "3.8" ]]; then + grep -v pysam requirements.txt > $TMPREQS + REQS=$TMPREQS + fi + # genomepy>=0.8 not available for py27 TMPOPTREQS=$(tempfile) grep -v genomepy optional-requirements.txt > $TMPOPTREQS From 564c9aaf1fa05ee23aee535ad0eebb6aba97276a Mon Sep 17 00:00:00 2001 From: Ryan Dale Date: Mon, 1 Jun 2020 22:09:05 -0400 Subject: [PATCH 10/11] use context manager for multiprocessing pool to avoid hanging --- pybedtools/scripts/intron_exon_reads.py | 38 ++++++++++++------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/pybedtools/scripts/intron_exon_reads.py b/pybedtools/scripts/intron_exon_reads.py index 82d4a9d4..5955430c 100644 --- a/pybedtools/scripts/intron_exon_reads.py +++ b/pybedtools/scripts/intron_exon_reads.py @@ -80,31 +80,31 @@ def count_reads_in_features(features): g = pybedtools.BedTool(gff).remove_invalid().saveas() # Set up pool of workers - pool = multiprocessing.Pool(processes=args.processes) + with multiprocessing.Pool(processes=args.processes) as pool: - # Get separate files for introns and exons in parallel - featuretypes = ["intron", "exon"] - introns, exons = pool.map(subset_featuretypes, featuretypes) + # Get separate files for introns and exons in parallel + featuretypes = ["intron", "exon"] + introns, exons = pool.map(subset_featuretypes, featuretypes) - # Since `subset_featuretypes` returns filenames, we convert to BedTool objects - # to do intersections below. - introns = pybedtools.BedTool(introns) - exons = pybedtools.BedTool(exons) + # Since `subset_featuretypes` returns filenames, we convert to BedTool objects + # to do intersections below. + introns = pybedtools.BedTool(introns) + exons = pybedtools.BedTool(exons) - # Identify unique and shared regions using bedtools commands subtract, merge, - # and intersect. - exon_only = exons.subtract(introns).merge() - intron_only = introns.subtract(exons).merge() - intron_and_exon = exons.intersect(introns).merge() + # Identify unique and shared regions using bedtools commands subtract, merge, + # and intersect. + exon_only = exons.subtract(introns).merge() + intron_only = introns.subtract(exons).merge() + intron_and_exon = exons.intersect(introns).merge() - # Do intersections with BAM file in parallel. Note that we're passing filenames - # to multiprocessing.Pool rather than BedTool objects. - features = (exon_only.fn, intron_only.fn, intron_and_exon.fn) + # Do intersections with BAM file in parallel. Note that we're passing filenames + # to multiprocessing.Pool rather than BedTool objects. + features = (exon_only.fn, intron_only.fn, intron_and_exon.fn) - # Run count_reads_in_features in parallel over features - results = pool.map(count_reads_in_features, features) + # Run count_reads_in_features in parallel over features + results = pool.map(count_reads_in_features, features) - labels = ("exon_only", "intron_only", "intron_and_exon") + labels = ("exon_only", "intron_only", "intron_and_exon") for label, reads in zip(labels, results): print("{0}\t{1}".format(label, reads)) From 8138b4f938284812a4266514af2c5957ab9fd0db Mon Sep 17 00:00:00 2001 From: Ryan Dale Date: Mon, 1 Jun 2020 22:21:37 -0400 Subject: [PATCH 11/11] skip multiprocessing test in py2 --- pybedtools/test/test_scripts.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pybedtools/test/test_scripts.py b/pybedtools/test/test_scripts.py index 69339c8e..27b34ad6 100644 --- a/pybedtools/test/test_scripts.py +++ b/pybedtools/test/test_scripts.py @@ -1,4 +1,5 @@ import pybedtools +import six from textwrap import dedent from .tfuncs import setup_module, teardown_module from pybedtools.scripts import annotate, venn_mpl, venn_gchart @@ -188,7 +189,7 @@ def test_venn_gchart_main(): sys.stderr = orig_stderr os.unlink("gcharttmp") - +@pytest.mark.skipif(six.PY2, reason="multiprocessing pool context manager does not work for py2") def test_intron_exon_reads(): gff = pybedtools.example_filename("gdc.gff") bam = pybedtools.example_filename("gdc.bam")