V2

.gitignore

@@ -4,3 +4,9 @@ syntax: glob
 *.*.swp
 *.tar.gz
 image/Dockerfile
+
+*/*/Dockerfile
+*/*/build.sh
+
+*/build.log
+*.gz

README.md

@@ -5,36 +5,57 @@ Things you can do with docker-build:
 - Build [Galaxy Tool Shed](https://toolshed.g2.bx.psu.edu) dependencies
 - Rebuild Debian or Ubuntu source packages (for modifications)
 
+To use, install Docker. Then for building packages there are two scripts that
+can be used, depending on the package recipes available:
+
+```console
+$ ./build.sh <package>
+$ python build.py <package> --version 1.0
+```
+
+The `build.sh` is the older format, and simply uses a single
+`<package>build.sh` file, like [Atlas](Atlas/). The `build.py` is the newer
+format, and uses yaml metadata in `<package>/<version>/build.yml`.
+
 The base image for Galaxy packages is Debian Squeeze. This will hopefully
 produce binaries usable on Galaxy's targeted platforms (at time of writing:
 CentOS 6+, Debian 6.0+, Ubuntu 12.04+).
 
-To use, install Docker. Then, for Tool Shed dependencies:
+To use, install Docker. Then, for Tool Shed dependencies, e.g.
 
 ```console
 $ ./build galaxy <package>
-```
-e.g.
-
-```console
-$ ./build galaxy samtools
+$ python build.py <package>
 ```
 
-For building dpkgs, use:
+To build packages against a different OS, you can use the `--image` flag, e.g.:
 
 ```console
 $ ./build <dist>[:tag] <package>
+$ python build.py <package> --image <dist>[:tag]
 ```
 
-e.g.:
+e.g.
 
 ```console
 $ ./build ubuntu:trusty nginx
+$ python build.py nginx --image debian:squeeze
 ```
 
+# Building all the things
+
+There's a separate `build-all.sh` which allows you to build all of the packages using their preferred build mechanism
+
+# Notes on the two build scripts
+
+## `build.py`
+
+The `<version>` option is optional, and defaults to the string 'default', which
+is useful for recipes that don't have version specific changes (E.g. bcftools 1.0 builds
+identically to 1.2)
+
+
 ## TODO
 
 - The build scripts themselves are targeted at whatever specific dist/release I
   am building them for at the moment and are not generalized to work on others.
-- Some things in the build scripts probably belong in some sort of build script
-  library.

atlas/default/build.yml

@@ -0,0 +1,40 @@
+meta:
+    image: 'debian:squeeze'
+    version: 3.10.2
+prebuild:
+    packages: |
+        libgfortran3
+build:
+    urls:
+        - http://downloads.sourceforge.net/project/math-atlas/Stable/${version}/atlas${version}.tar.bz2
+        - http://www.netlib.org/lapack/lapack-3.5.0.tgz
+    commands:
+        - tar jxf atlas${version}.tar.bz2
+        - cd ATLAS
+        - patch -p1 </host/static_full_blas_lapack.diff
+        - patch -p1 </host/shared_libraries.diff
+        - patch -p1 </host/cpu-throttling-check.diff
+        - cd /build/
+        - |
+            ATLAS/configure \
+                --prefix="/build/dest" \
+                -D c -DWALL \
+                -b 64 \
+                -Fa alg '-fPIC' \
+                -Ss f77lib "-L$(gcc -print-search-dirs|grep ^install:|awk '{print $2}') -lgfortran -lgcc_s -lpthread"  \
+                --with-netlib-lapack-tarfile=/build/lapack-3.5.0.tgz \
+                -A 14 \
+                -V 384 \
+                -v 2 \
+                -t 0 \
+                -Ss ADdir ../../archdefs/amd64 \
+                -Si cputhrchk 0
+        - make
+        - make install
+        - libgfortran=$(readlink -f $(gcc -print-search-dirs|grep ^install:|awk '{print $2}')/libgfortran.so)
+        - basename_libgfortran=$(basename $libgfortran)
+        - libgfortran_version=${basename_libgfortran#"libgfortran.so."}
+        - cp $libgfortran /build/dest/lib
+        - ln -s $(basename $libgfortran) /build/dest/lib/libgfortran.so.${libgfortran_version:0:1}
+        - ln -s libgfortran.so.${libgfortran_version:0:1} /build/dest/lib/libgfortran.so
+        - tar zcf /host/atlas-${version}-Linux-x86_64.tar.gz -C /build/dest .

atlas/default/cpu-throttling-check.diff

@@ -0,0 +1,30 @@
+Description: Reenable the configuration flag for disable CPU throttling check
+ For the generic package, we need the ability to compile on machines with CPU
+ throttling enabled (since some buildds have that feature).
+Author: Sébastien Villemot <sebastien@debian.org>
+Forwarded: not-needed
+Last-Update: 2013-06-11
+---
+This patch header follows DEP-3: http://dep.debian.net/deps/dep3/
+--- a/CONFIG/src/config.c
++++ b/CONFIG/src/config.c
+@@ -928,10 +928,8 @@
+         "      -Si nocygwin <0/1> : Do/don't depend on GPL cygwin library\n");
+       fprintf(stderr,
+         "                           (Windows compiler/cygwin install only)\n");
+-/* Disabled due to abuse
+       fprintf(stderr,
+         "      -Si cputhrchk <0/1> : Ignore/heed CPU throttle probe\n");
+- */
+       fprintf(stderr,
+            "   -tl <#> <list> : set # of threads, use list of affinity IDs\n");
+       fprintf(stderr,
+@@ -1129,6 +1127,8 @@
+             *NoF77 = k;
+          else if (!strcmp(sp0, "nocygwin"))
+             *NoCygwin = k;
++         else if (!strcmp(sp0, "cputhrchk"))
++            *ThrChk = k;
+          else if (!strcmp(sp0, "kern"))
+             gcc3 = sp;
+          else if (!strcmp(sp0, "ADdir") || !strcmp(sp0, "addir"))

atlas/default/shared_libraries.diff

@@ -0,0 +1,130 @@
+Description: Create shared versions of the libraries
+Author: David Evans
+        Sébastien Villemot <sebastien@debian.org>
+Last-Update: 2014-07-16
+---
+This patch header follows DEP-3: http://dep.debian.net/deps/dep3/
+--- a/makes/Make.lib
++++ b/makes/Make.lib
+@@ -33,6 +33,79 @@ $(tarnam).tar.bz2 :
+ 	rm -rf $(ARCH)
+ 	$(BZIP) --best $(tarnam).tar
+
++fullshared: libatlas.so libcblas.so libf77blas.so liblapack.so atlas/libblas.so atlas/liblapack.so
++
++libatlas.so: libatlas.so.3
++	ln -sf $< $@ 
++
++libatlas.so.3: libatlas.so.3.0
++	ln -sf $< $@ 
++
++libatlas.so.3.0 : libatlas.a
++	ld $(LDFLAGS) -shared -soname libatlas.so.3 -o $@ \
++        --whole-archive libatlas.a --no-whole-archive -lc $(LIBS) $(F77SYSLIB)
++
++libcblas.so: libcblas.so.3
++	ln -sf $< $@ 
++
++libcblas.so.3: libcblas.so.3.0
++	ln -sf $< $@ 
++
++libcblas.so.3.0 : libcblas.a libatlas.so
++	ld $(LDFLAGS) -shared -soname libcblas.so.3 -o $@ \
++	  --whole-archive libcblas.a \
++          --no-whole-archive -L. -latlas $(F77SYSLIB) -lm -lc
++
++libf77blas.so: libf77blas.so.3
++	ln -sf $< $@ 
++
++libf77blas.so.3: libf77blas.so.3.0
++	ln -sf $< $@ 
++
++libf77blas.so.3.0 : libf77blas.a libcblas.so libatlas.so
++	ld $(LDFLAGS) -shared -soname libf77blas.so.3 -o $@ \
++	  --whole-archive libf77blas.a \
++          --no-whole-archive -L. -lcblas -latlas $(F77SYSLIB) -lm -lc
++
++liblapack.so: liblapack.so.3
++	ln -sf $< $@ 
++
++liblapack.so.3: liblapack.so.3.0
++	ln -sf $< $@ 
++
++liblapack.so.3.0 : liblapack.a libatlas.so libcblas.so libf77blas.so
++	ld $(LDFLAGS) -shared -soname liblapack.so.3 -o $@ \
++	  --whole-archive liblapack.a  \
++	  --no-whole-archive -L. -lf77blas -lcblas -latlas -lm -lc $(F77SYSLIB)
++
++atlas/libblas.so: atlas/libblas.so.3
++	(cd atlas && ln -sf libblas.so.3 libblas.so)
++
++atlas/libblas.so.3: atlas/libblas.so.3.0
++	(cd atlas && ln -sf libblas.so.3.0 libblas.so.3)
++
++atlas/libblas.so.3.0: libf77blas.a libcblas.a libatlas.so.3.0
++	if test -f libptf77blas.a -a -f libptcblas.a; then \
++		ld $(LDFLAGS) -shared -soname libblas.so.3 -o $@ \
++		   --whole-archive libptf77blas.a libptcblas.a \
++	           --no-whole-archive -L. -latlas $(F77SYSLIB) -lm -lc; \
++	else \
++		ld $(LDFLAGS) -shared -soname libblas.so.3 -o $@ \
++		   --whole-archive libf77blas.a libcblas.a \
++	           --no-whole-archive -L. -latlas $(F77SYSLIB) -lm -lc; \
++	fi
++
++atlas/liblapack.so: atlas/liblapack.so.3
++	(cd atlas && ln -sf liblapack.so.3 liblapack.so)
++
++atlas/liblapack.so.3: atlas/liblapack.so.3.0
++	(cd atlas && ln -sf liblapack.so.3.0 liblapack.so.3)
++
++atlas/liblapack.so.3.0: atlas/liblapack.a atlas/libblas.so libatlas.so.3.0
++	ld $(LDFLAGS) -shared -soname liblapack.so.3 -o $@ \
++	   --whole-archive atlas/liblapack.a \
++           --no-whole-archive -L . -L atlas -lblas -latlas $(F77SYSLIB) -lm -lc
++
+ # ===================================================================
+ # The following commands are to build dynamic/shared objects on Linux
+ # using the gnu gcc or ld
+--- a/Make.top
++++ b/Make.top
+@@ -16,6 +16,7 @@ build:
+ 	cd bin/ ; ./xatlas_build $(INSTFLAGS)
+ 	cd lib/ ; $(MAKE) atlas/libblas.a
+ 	cd lib/ ; $(MAKE) atlas/liblapack.a
++	cd lib/ ; $(MAKE) fullshared
+
+ time: 
+ 	./xatlbench -dc $(BLDdir)/bin/INSTALL_LOG -dp $(BLDdir)/ARCHS/$(ARCH)
+@@ -683,3 +684,21 @@ install_static_lib:
+ 	- cp $(LIBdir)/libsatlas.so $(INSTdir)/.
+ 	- cp $(LIBdir)/libtatlas.so $(INSTdir)/.
+
++install_shared_lib:
++	for lib in atlas cblas f77blas lapack; do \
++		cp $(LIBdir)/lib$$lib.so.3.0 $(INSTdir)/. ; \
++		( cd $(INSTdir) && ln -s lib$$lib.so.3.0 lib$$lib.so.3); \
++		( cd $(INSTdir) && ln -s lib$$lib.so.3 lib$$lib.so); \
++		chmod 0644 $(INSTdir)/lib$$lib.so ;\
++		chmod 0644 $(INSTdir)/lib$$lib.so.3 ;\
++		chmod 0644 $(INSTdir)/lib$$lib.so.3.0 ;\
++	done;
++	mkdir -p $(INSTdir)/atlas
++	for lib in blas lapack; do \
++		cp $(LIBdir)/atlas/lib$$lib.so.3.0 $(INSTdir)/atlas/. ; \
++		( cd $(INSTdir)/atlas/ && ln -s lib$$lib.so.3.0 lib$$lib.so.3); \
++		( cd $(INSTdir)/atlas/ && ln -s lib$$lib.so.3 lib$$lib.so); \
++		chmod 0644 $(INSTdir)/atlas/lib$$lib.so ;\
++		chmod 0644 $(INSTdir)/atlas/lib$$lib.so.3 ;\
++		chmod 0644 $(INSTdir)/atlas/lib$$lib.so.3.0 ;\
++	done;
+--- a/CONFIG/src/Makefile
++++ b/CONFIG/src/Makefile
+@@ -576,6 +576,7 @@ $(LIBINSTdir) :
+ install : $(INCINSTdir) $(LIBINSTdir) $(INCINSTdir)/atlas
+ 	$(MAKE) -f Make.top install_inc INSTdir=$(INCINSTdir) 
+ 	$(MAKE) -f Make.top install_static_lib INSTdir=$(LIBINSTdir)
++	$(MAKE) -f Make.top install_shared_lib INSTdir=$(LIBINSTdir)
+
+ confclean: $(CLEANdep)
+ 	rm -f *core* *.o config?.out

atlas/default/static_full_blas_lapack.diff

@@ -0,0 +1,116 @@
+Description: Create full featured static libblas.a and liblapack.a
+ The resulting BLAS library will use the threaded flavour if the build has been
+ configured to create them. Otherwise, the serial library is used.
+Author: David Evans
+Last-Update: 2013-06-05
+---
+This patch header follows DEP-3: http://dep.debian.net/deps/dep3/
+--- a/Make.top
++++ b/Make.top
+@@ -14,6 +14,9 @@
+ build:
+ 	cd bin/ ; $(MAKE) xatlas_build
+ 	cd bin/ ; ./xatlas_build $(INSTFLAGS)
++	cd lib/ ; $(MAKE) atlas/libblas.a
++	cd lib/ ; $(MAKE) atlas/liblapack.a
++
+ time: 
+ 	./xatlbench -dc $(BLDdir)/bin/INSTALL_LOG -dp $(BLDdir)/ARCHS/$(ARCH)
+ C_sanity_test:
+@@ -401,6 +404,7 @@
+ 	$(MAKE) -f Make.top leafstart leaf=src/blas/reference/level1
+ 	$(MAKE) -f Make.top leafstart leaf=src/auxil
+ 	$(MAKE) -f Make.top leafstart leaf=src/testing
++	$(MAKE) -f Make.top leafstart leaf=lib/atlas
+ 	$(MAKE) -f Make.top leafstart leaf=lib
+ 	$(MAKE) -f Make.top leafstart leaf=bin
+ 	cd src/threads ; touch atomic.inc
+@@ -591,6 +595,7 @@
+
+ lib :
+ 	mkdir lib
++	mkdir lib/atlas
+
+ include:
+ 	mkdir include
+@@ -654,12 +659,17 @@
+ 	chmod 0644 $(INSTdir)/cblas.h $(INSTdir)/clapack.h
+ 	cp $(INCAdir)/* $(INSTdir)/atlas/.
+ 	- chmod 0644 $(INSTdir)/atlas/*
+-install_lib:
++
++install_static_lib:
+ 	cp $(LIBdir)/libatlas.a $(INSTdir)/.
+ 	cp $(LIBdir)/libcblas.a $(INSTdir)/.
+ 	cp $(LIBdir)/liblapack.a $(INSTdir)/.
++	mkdir -p $(INSTdir)/atlas
++	cp $(LIBdir)/atlas/libblas.a $(INSTdir)/atlas/.
++	cp $(LIBdir)/atlas/liblapack.a $(INSTdir)/atlas/.
+ 	chmod 0644 $(INSTdir)/libatlas.a  $(INSTdir)/liblapack.a \
+-                   $(INSTdir)/libcblas.a 
++                   $(INSTdir)/libcblas.a $(INSTdir)/atlas/libblas.a \
++		   $(INSTdir)/atlas/liblapack.a 
+ 	- cp $(LIBdir)/libf77blas.a $(INSTdir)/.
+ 	- chmod 0644 $(INSTdir)/libf77blas.a
+ 	- cp $(LIBdir)/libptcblas.a $(INSTdir)/.
+--- a/makes/Make.lib
++++ b/makes/Make.lib
+@@ -175,6 +175,35 @@
+            LIBS="$(LIBS)" LIBINSTdir="$(LIBINSTdir)" ; \
+         fi
+
++
++# Build full netlib blas/lapack libraries:
++atlas/libblas.a: libatlas.a
++	mkdir tmp
++	cd tmp && \
++		ar x ../libatlas.a && \
++		if test -f ../libptf77blas.a -a -f ../libptcblas.a; then \
++			ar x ../libptf77blas.a && \
++			ar x ../libptcblas.a; \
++		else \
++			ar x ../libf77blas.a && \
++			ar x ../libcblas.a; \
++		fi
++	ar r $@ tmp/*.o
++	rm -rf tmp
++
++atlas/liblapack.a: liblapack.a libatlas.a
++	mkdir tmp
++	cd tmp && \
++		ar x ../liblapack.a && \
++		if test -f ../libptcblas.a; then \
++			ar x ../libptcblas.a; \
++		else \
++			ar x ../libcblas.a; \
++		fi
++	ar r $@ tmp/*.o
++	rm -rf tmp
++
++
+ #
+ # Builds one shared lib from all ATLAS files
+ #
+--- a/CONFIG/src/Makefile
++++ b/CONFIG/src/Makefile
+@@ -575,7 +575,7 @@
+ 	chmod 0755 $(LIBINSTdir)
+ install : $(INCINSTdir) $(LIBINSTdir) $(INCINSTdir)/atlas
+ 	$(MAKE) -f Make.top install_inc INSTdir=$(INCINSTdir) 
+-	$(MAKE) -f Make.top install_lib INSTdir=$(LIBINSTdir)
++	$(MAKE) -f Make.top install_static_lib INSTdir=$(LIBINSTdir)
+
+ confclean: $(CLEANdep)
+ 	rm -f *core* *.o config?.out
+--- a/CONFIG/src/SpewMakeInc.c
++++ b/CONFIG/src/SpewMakeInc.c
+@@ -604,6 +604,9 @@
+    }
+    fprintf(fpout, "   TESTlib = $(LIBdir)/libtstatlas.a\n\n");
+
++   fprintf(fpout, "   FULLBLASlib = $(LIBdir)/atlas/libblas.a\n");
++   fprintf(fpout, "   FULLLAPACKlib = $(LIBdir)/atlas/liblapack.a\n");
++
+    fprintf(fpout, "#  -------------------------------------------\n");
+    fprintf(fpout, "#  Upper bound on largest cache size, in bytes\n");
+    fprintf(fpout, "#  -------------------------------------------\n");