Merge pull request #451 from ged-lab/feature/package-tests

Feature/package tests

.ycm_extra_conf.py

@@ -31,6 +31,14 @@
     # For a C project, you would set this to 'c' instead of 'c++'.
     '-x',
     'c++',
+    '-I/usr/include/c++/4.8',
+    '-I/usr/include/x86_64-linux-gnu/c++/4.8',
+    '-I/usr/include/c++/4.8/backward',
+    '-I/usr/lib/gcc/x86_64-linux-gnu/4.8/include',
+    '-I/usr/local/include',
+    '-I/usr/lib/gcc/x86_64-linux-gnu/4.8/include-fixed',
+    '-I/usr/include/x86_64-linux-gnu',
+    '-I/usr/include'
 ]
 
 

ChangeLog

@@ -1,3 +1,8 @@
+2014-06-13  Michael R. Crusoe  <mcrusoe@msu.edu>
+
+    * Makefile: split coverity target in two: -build and -upload. Added
+    configuration target
+
 2014-06-13  Titus Brown  <t@Titus-MacBook-Air-2.local>
 
     * doc/install.txt: updated virtuaenv command to use python2 explicitly,
@@ -18,6 +23,17 @@
     * scripts/load-into-counting: Fixed docstring misnomer to 
     load-into-counting.py
 
+2014-06-10  Michael R. Crusoe  <mcrusoe@msu.edu>
+
+    * setup.py,tests/{__init__,khmer_tst_utils,test_scripts,
+    khmer_test_counting_single}.py: made tests runnable after installation.
+    * lib/{khmer.hh,hashtable.hh,read_parsers.cc,read_parsers.hh}: restructure
+    exception hierarchy.
+    * khmer/_khmermodule.cc: Nicer error checking for hash_consume_fasta,
+    hash_abundance_distribution, hashbits_consume_{fasta,fasta_and_tag
+    {,with_stoptags},partitioned_fasta}, hashbits_output_partitions, and
+    labelhash_consume_{,partitioned_}fasta_and_tag_with_labels.
+
 2014-06-10  Titus Brown  <t@idyll.org>
 
     * Makefile: remove SHELL setting so that 'make doc' works in virtualenvs.
@@ -33,21 +49,29 @@
     * scripts/fastq-to-fasta: Fixed outdated argparse oversight.
     * tests/test_scripts.py: Added tests for extract-long-sequences.py
 
-2014-06-04  Michael R. Crusoe <mcrusoe@msu.edu>
-
-    * lib/Makefile: do a distclean as the CFLAGS may have changed. Fixes #442
-
 2014-06-08  Titus Brown  <t@idyll.org>
 
     * doc/conf.py: set google_analytics_id and disqus_shortname properly;
 	disable "editme" popup.
     * doc/_templates/page.html: take google_analytics_id and disqus_shortname
 	from doc/conf.py.
 
+2014-06-04  Michael R. Crusoe <mcrusoe@msu.edu>
+
+    * lib/Makefile: do a distclean as the CFLAGS may have changed. Fixes #442
+
 2014-06-03 Chuck Pepe-Ranney <chuck.peperanney@gmail.com>
 
     * scripts/abundance-dist.py: removed call to check_space on infiles.  
 
+2014-05-31  Michael R. Crusoe  <mcrusoe@msu.edu>
+
+    * khmer/_khmermodule.cc,lib/counting.{cc,hh},
+    sandbox/{stoptag-abundance-ham1-hist.py,off-by-one.py,filter-ham1.py}:
+    Remove CountingHash get_kmer_abund_mean, get_kmer_abund_abs_deviation, and
+    max_hamming1_count along with Python glue code and sandbox scripts. They
+    are no longer useful.
+
 2014-05-30  Titus Brown  <t@idyll.org>
 
     * khmer/_khmermodule.cc: remove merge2* functions: unused, untested.
@@ -77,6 +101,11 @@
     * tests/test-data: Added test-fastq-n-to-fasta.py file with N's in 
     sequence for testing
 
+2014-05-19  Michael R. Crusoe  <mcrusoe@msu.edu>
+
+    * Makefile: add target for python test coverage plain-text report;
+    clarified where the HTML report is
+
 2014-05-16  Michael R. Crusoe  <mcrusoe@msu.edu>
 
     * docs/scripts.txt: include sweep-reads-buffered.py
@@ -93,13 +122,6 @@
     * Makefile: add sloccount target for humans and the sloccount.sc target for
    Jenkins
 
-2014-05-14  Michael R. Crusoe  <mcrusoe@msu.edu>
-
-    * lib/{zlib,bzip2} -> third-party/
-    * setup.{cfg,py}: Move third party libraries to their own directory
-    * Makefile: add sloccount target for humans and the sloccount.sc target for
-   Jenkins
-
 2014-05-13  Michael Wright  <wrigh517@msu.edu>
 
     * sandbox/fastq-to-fasta.py: now reports number of reads dropped due to

Makefile

@@ -2,20 +2,32 @@
 # make autopep8 to fix most pep8 errors
 # make pylint to check Python code for enhanced compliance including naming
 #  and documentation
+# make coverage-report to check coverage of the python scripts by the tests
+
+CPPSOURCES=$(wildcard {lib,khmer}/*.{cc,hh})
+PYSOURCES=$(wildcard {khmer,scripts}/*.py)
+SOURCES=$(PYSOURCES) $(CPPSOURCES) setup.py
+
+SHELL=/bin/bash
 GCOVRURL=git+https://github.com/nschum/gcovr.git@never-executed-branches
-VERSION=`git describe --tags --dirty | sed s/v//`
+VERSION=$(shell git describe --tags --dirty | sed s/v//)
 
-all: FORCE
+all: khmer/_khmermodule.so
+
+khmer/_khmermodule.so: $(SOURCES)
 	./setup.py build_ext --inplace
 
-coverage-debug: FORCE
-	export CFLAGS="-pg -fprofile-arcs -ftest-coverage"; ./setup.py \
+coverage-debug: $(SOURCES)
+	export CFLAGS="-pg -fprofile-arcs -ftest-coverage -O0"; ./setup.py \
 		build_ext --debug --inplace --libraries gcov
+	touch coverage-debug
 
 install: FORCE
 	./setup.py build install
 
-dist: FORCE
+dist: dist/khmer-$(VERSION).tar.gz
+
+dist/khmer-$(VERSION).tar.gz: $(SOURCES)
 	./setup.py sdist
 
 clean: FORCE
@@ -24,43 +36,48 @@ clean: FORCE
 	rm -f khmer/_khmermodule.so || true
 	rm khmer/*.pyc lib/*.pyc || true
 	./setup.py clean --all || true
+	rm coverage-debug || true
 
-debug:
+debug: FORCE
 	export CFLAGS="-pg -fprofile-arcs"; python setup.py build_ext --debug \
 		--inplace
 
-doc: all
+doc: build/sphinx/html/index.html
+
+build/sphinx/html/index.html: $(SOURCES) $(wildcard doc/*.txt) doc/conf.py
 	pip2 install --user sphinx sphinxcontrib-autoprogram || pip2 install \
 		sphinx sphinxcontrib-autoprogram
 	./setup.py build_sphinx --fresh-env
 	@echo ''
 	@echo '--> docs in build/sphinx/html <--'
 	@echo ''
 
-pdf: FORCE
+pdf: build/sphinx/latex/khmer.pdf
+
+build/sphinx/latex/khmer.pdf: $(SOURCES) doc/conf.py $(wildcard doc/*.txt)
 	pip2 install --user sphinx sphinxcontrib-autoprogram || pip2 install \
 		sphinx sphinxcontrib-autoprogram
 	./setup.py build_sphinx --fresh-env --builder latex
 	cd build/sphinx/latex && ${MAKE} all-pdf
 	@echo ''
 	@echo '--> pdf in build/sphinx/latex/khmer.pdf'
 
-cppcheck-result.xml: FORCE
+cppcheck-result.xml: $(CPPSOURCES)
 	ls lib/*.cc khmer/_khmermodule.cc | grep -v test | cppcheck -DNDEBUG \
 		-DVERSION=0.0.cppcheck -UNO_UNIQUE_RC --enable=all \
 		--file-list=- -j8 --platform=unix64 --std=posix --xml \
 		--xml-version=2 2> cppcheck-result.xml
 
-cppcheck: FORCE
+cppcheck: $(CPPSOURCES)
 	ls lib/*.cc khmer/_khmermodule.cc | grep -v test | cppcheck -DNDEBUG \
 		-DVERSION=0.0.cppcheck -UNO_UNIQUE_RC --enable=all \
 		--file-list=- -j8 --platform=unix64 --std=posix --quiet
 
-pep8: FORCE
+pep8: $(PYSOURCES) $(wildcard tests/*.py)
 	pip2 install --user --quiet pep8==1.5 || pip2 install --quiet pep8==1.5
 	pep8 --exclude=_version.py setup.py khmer/ scripts/ tests/ || true
 
-pep8_report.txt: FORCE
+pep8_report.txt: $(PYSOURCES) $(wildcard tests/*.py)
 	pip2 install --user --quiet pep8==1.5 || pip2 install --quiet pep8==1.5
 	pep8 --exclude=_version.py setup.py khmer/ scripts/ tests/ \
 		> pep8_report.txt || true
@@ -69,18 +86,18 @@ diff_pep8_report: pep8_report.txt
 	pip2 install --user diff_cover || pip2 install diff_cover
 	diff-quality --violations=pep8 pep8_report.txt
 
-autopep8: FORCE
+autopep8: $(PYSOURCES) $(wildcard tests/*.py)
 	pip2 install --user autopep8 || pip2 install autopep8
 	autopep8 --recursive --in-place --exclude _version.py --ignore E309 \
 		setup.py khmer/ scripts/ tests/
 
-pylint: FORCE
+pylint: $(PYSOURCES) $(wildcard tests/*.py)
 	pip2 install --user pylint || pip2 install pylint
 	pylint --msg-template="{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}" \
 		setup.py khmer/[!_]*.py khmer/__init__.py scripts/*.py tests \
 		|| true
 
-pylint_report.txt: FORCE
+pylint_report.txt: ${PYSOURCES} $(wildcard tests/*.py)
 	pip2 install --user pylint || pip2 install pylint
 	pylint --msg-template="{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}" \
 		setup.py khmer/[!_]*.py khmer/__init__.py scripts/*.py tests \
@@ -93,37 +110,52 @@ diff_pylint_report: pylint_report.txt
 # We need to get coverage to look at our scripts. Since they aren't in a
 # python module we can't tell nosetests to look for them (via an import
 # statement). So we run nose inside of coverage.
-.coverage: coverage-debug
-	pip2 install --user coverage || pip2 install coverage
+.coverage: $(PYSOURCES) $(wildcard tests/*.py)
+	pip2 install --user --upgrade coverage || pip2 install --upgrade \
+		coverage
 	coverage run --branch --source=scripts,khmer --omit=khmer/_version.py \
 		-m nose --with-xunit --attr=\!known_failing --processes=0
 
 coverage.xml: .coverage
+	pip2 install --user --upgrade coverage || pip2 install --upgrade \
+		coverage
 	coverage xml
 
-coverage.html: .coverage
+coverage.html: htmlcov/index.html
+
+htmlcov/index.html: .coverage
+	pip2 install --user --upgrade coverage || pip2 install --upgrade \
+		coverage
 	coverage html
+	@echo Test coverage of the Python code is now in htmlcov/index.html
 
-coverage-gcovr.xml: coverage-debug test
+coverage-report: .coverage
+	pip2 install --user --upgrade coverage || pip2 install --upgrade \
+		coverage
+	coverage report
+
+coverage-gcovr.xml: coverage-debug .coverage
 	pip2 install --user --upgrade ${GCOVRURL}'#gcovr' || pip2 install \
 		--upgrade ${GCOVRURL}'#gcovr'
 	gcovr --root=. --branches --gcov-exclude='.*zlib.*|.*bzip2.*' --xml \
 		--output=coverage-gcovr.xml
 
-diff-cover: clean coverage-gcovr.xml coverage.xml
+diff-cover: coverage-gcovr.xml coverage.xml
 	pip2 install --user diff_cover || pip2 install diff_cover
 	diff-cover coverage-gcovr.xml coverage.xml
 
-diff-cover.html: clean coverage-gcovr.xml coverage.xml
+diff-cover.html: coverage-gcovr.xml coverage.xml
 	pip2 install --user diff_cover || pip2 install diff_cover
 	diff-cover coverage-gcovr.xml coverage.xml \
 		--html-report diff-cover.html
 
-nosetests.xml: all
+nosetests.xml: FORCE
 	pip2 install --user nose || pip2 install nose
 	./setup.py nosetests --with-xunit
 
-doxygen: FORCE
+doxygen: doc/doxygen/html/index.html
+
+doc/doxygen/html/index.html: ${CPPSOURCES} ${PYSOURCES}
 	mkdir -p doc/doxygen
 	sed "s/\$${VERSION}/`python ./lib/get_version.py`/" Doxyfile.in > \
 		Doxyfile
@@ -137,32 +169,49 @@ test:
 	pip2 install --user nose || pip2 install nose
 	./setup.py nosetests
 
-sloccount.sc: FORCE 
+sloccount.sc: ${CPPSOURCES} ${PYSOURCES} $(wildcard tests/*.py) Makefile
 	sloccount --duplicates --wide --details lib khmer scripts tests \
 		setup.py Makefile > sloccount.sc
 
-sloccount: FORCE
+sloccount: 
 	sloccount lib khmer scripts tests setup.py Makefile
 
-coverity:
-	if [[ -x ${cov_analysis_dir}/cov-build ]]; \
-	then if [[ -n "${COVERITY_TOKEN}" ]]; \
-		then \
-			export PATH=${PATH}:${cov_analysis_dir}; \
-			cov-build --dir cov-int python setup.py build_ext \
-				--build-temp ${PWD}; \
-			tar czf khmer-cov.tgz cov-int; \
-			curl --form project=ged-lab/khmer \
-				--form token=${COVERITY_TOKEN} --form \
-				email=mcrusoe@msu.edu --form \
-				file=@khmer-cov.tgz --form \
-				version=${VERSION} \
-				http://scan5.coverity.com/cgi-bin/upload.py; \
-		else echo 'Missing coverity credentials in $$COVERITY_TOKEN,'\
-			'skipping scan'; \
-		fi; \
-	else echo 'cov-build does not exist in $$cov_analysis_dir: '\
+coverity-build:
+	if [[ -x ${cov_analysis_dir}/bin/cov-build ]]; \
+	then \
+		export PATH=${PATH}:${cov_analysis_dir}/bin; \
+		cov-build --dir cov-int --c-coverage gcov --disable-gcov-arg-injection make coverage-debug; \
+		cov-capture --dir cov-int --c-coverage gcov python -m nose --attr '!known_failing' ; \
+		cov-import-scm --dir cov-int --scm git 2>/dev/null; \
+	else echo 'bin/cov-build does not exist in $$cov_analysis_dir: '\
 		'${cov_analysis_dir}. Skipping coverity scan.'; \
 	fi
 
+coverity-upload: cov-int
+	if [[ -n "${COVERITY_TOKEN}" ]]; \
+	then \
+		tar czf khmer-cov.tgz cov-int; \
+		curl --form project=ged-lab/khmer \
+			--form token=${COVERITY_TOKEN} --form \
+			email=mcrusoe@msu.edu --form file=@khmer-cov.tgz \
+			--form version=${VERSION} \
+			http://scan5.coverity.com/cgi-bin/upload.py; \
+	else echo 'Missing coverity credentials in $$COVERITY_TOKEN,'\
+		'skipping scan'; \
+	fi
+
+coverity-clean-configuration:
+	rm -f ${cov_analysis_dir}/config/coverity_config.xml
+
+coverity-configure:
+	if [[ -x ${cov_analysis_dir}/bin/cov-configure ]]; \
+	then \
+		export PATH=${PATH}:${cov_analysis_dir}/bin; \
+		for compiler in /usr/bin/gcc-4.8 /usr/bin/x86_64-linux-gnu-gcc; do \
+       			cov-configure --comptype gcc --compiler $${compiler}; \
+		done; \
+	else echo 'bin/cov-configure does not exist in $$cov_analysis_dir: '\
+		'${cov_analysis_dir}. Skipping coverity configuration.'; \
+	fi
+
 FORCE:

jenkins-build.sh

@@ -1,14 +1,14 @@
 #!/bin/bash
 
-rm -Rf .env build dist khmer/_khmermodule.so cov-int lib/zlib/Makefile
+make clean
+
+rm -Rf .env dist cov-int
 
 virtualenv .env
 
 . .env/bin/activate
 pip install --quiet nose coverage pylint pep8==1.5 screed
 
-make clean
-
 if type ccache >/dev/null 2>&1
 then
         echo Enabling ccache
@@ -45,7 +45,8 @@ fi
 if type hg >/dev/null 2>&1
 then
 	rm -Rf sphinx-contrib
-	hg clone http://bitbucket.org/mcrusoe/sphinx-contrib
+	#hg clone http://bitbucket.org/mcrusoe/sphinx-contrib
+	hg clone http://athyra.ged.msu.edu/~mcrusoe/sphinx-contrib
 	pip install --upgrade sphinx-contrib/autoprogram/
 	make doc
 fi