diff --git a/.gitignore b/.gitignore index 938d2aee..100ca39e 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,9 @@ syntax: glob *.*.swp *.tar.gz image/Dockerfile + +*/*/Dockerfile +*/*/build.sh + +*/build.log +*.gz diff --git a/README.md b/README.md index 1193c47f..75978e3a 100644 --- a/README.md +++ b/README.md @@ -5,36 +5,57 @@ Things you can do with docker-build: - Build [Galaxy Tool Shed](https://toolshed.g2.bx.psu.edu) dependencies - Rebuild Debian or Ubuntu source packages (for modifications) +To use, install Docker. Then for building packages there are two scripts that +can be used, depending on the package recipes available: + +```console +$ ./build.sh +$ python build.py --version 1.0 +``` + +The `build.sh` is the older format, and simply uses a single +`build.sh` file, like [Atlas](Atlas/). The `build.py` is the newer +format, and uses yaml metadata in `//build.yml`. + The base image for Galaxy packages is Debian Squeeze. This will hopefully produce binaries usable on Galaxy's targeted platforms (at time of writing: CentOS 6+, Debian 6.0+, Ubuntu 12.04+). -To use, install Docker. Then, for Tool Shed dependencies: +To use, install Docker. Then, for Tool Shed dependencies, e.g. ```console $ ./build galaxy -``` -e.g. - -```console -$ ./build galaxy samtools +$ python build.py ``` -For building dpkgs, use: +To build packages against a different OS, you can use the `--image` flag, e.g.: ```console $ ./build [:tag] +$ python build.py --image [:tag] ``` -e.g.: +e.g. ```console $ ./build ubuntu:trusty nginx +$ python build.py nginx --image debian:squeeze ``` +# Building all the things + +There's a separate `build-all.sh` which allows you to build all of the packages using their preferred build mechanism + +# Notes on the two build scripts + +## `build.py` + +The `` option is optional, and defaults to the string 'default', which +is useful for recipes that don't have version specific changes (E.g. bcftools 1.0 builds +identically to 1.2) + + ## TODO - The build scripts themselves are targeted at whatever specific dist/release I am building them for at the moment and are not generalized to work on others. -- Some things in the build scripts probably belong in some sort of build script - library. diff --git a/atlas/default/build.yml b/atlas/default/build.yml new file mode 100644 index 00000000..d6f87519 --- /dev/null +++ b/atlas/default/build.yml @@ -0,0 +1,40 @@ +meta: + image: 'debian:squeeze' + version: 3.10.2 +prebuild: + packages: | + libgfortran3 +build: + urls: + - http://downloads.sourceforge.net/project/math-atlas/Stable/${version}/atlas${version}.tar.bz2 + - http://www.netlib.org/lapack/lapack-3.5.0.tgz + commands: + - tar jxf atlas${version}.tar.bz2 + - cd ATLAS + - patch -p1 +Forwarded: not-needed +Last-Update: 2013-06-11 +--- +This patch header follows DEP-3: http://dep.debian.net/deps/dep3/ +--- a/CONFIG/src/config.c ++++ b/CONFIG/src/config.c +@@ -928,10 +928,8 @@ + " -Si nocygwin <0/1> : Do/don't depend on GPL cygwin library\n"); + fprintf(stderr, + " (Windows compiler/cygwin install only)\n"); +-/* Disabled due to abuse + fprintf(stderr, + " -Si cputhrchk <0/1> : Ignore/heed CPU throttle probe\n"); +- */ + fprintf(stderr, + " -tl <#> : set # of threads, use list of affinity IDs\n"); + fprintf(stderr, +@@ -1129,6 +1127,8 @@ + *NoF77 = k; + else if (!strcmp(sp0, "nocygwin")) + *NoCygwin = k; ++ else if (!strcmp(sp0, "cputhrchk")) ++ *ThrChk = k; + else if (!strcmp(sp0, "kern")) + gcc3 = sp; + else if (!strcmp(sp0, "ADdir") || !strcmp(sp0, "addir")) diff --git a/atlas/default/shared_libraries.diff b/atlas/default/shared_libraries.diff new file mode 100644 index 00000000..f4b6e1c3 --- /dev/null +++ b/atlas/default/shared_libraries.diff @@ -0,0 +1,130 @@ +Description: Create shared versions of the libraries +Author: David Evans + Sébastien Villemot +Last-Update: 2014-07-16 +--- +This patch header follows DEP-3: http://dep.debian.net/deps/dep3/ +--- a/makes/Make.lib ++++ b/makes/Make.lib +@@ -33,6 +33,79 @@ $(tarnam).tar.bz2 : + rm -rf $(ARCH) + $(BZIP) --best $(tarnam).tar + ++fullshared: libatlas.so libcblas.so libf77blas.so liblapack.so atlas/libblas.so atlas/liblapack.so ++ ++libatlas.so: libatlas.so.3 ++ ln -sf $< $@ ++ ++libatlas.so.3: libatlas.so.3.0 ++ ln -sf $< $@ ++ ++libatlas.so.3.0 : libatlas.a ++ ld $(LDFLAGS) -shared -soname libatlas.so.3 -o $@ \ ++ --whole-archive libatlas.a --no-whole-archive -lc $(LIBS) $(F77SYSLIB) ++ ++libcblas.so: libcblas.so.3 ++ ln -sf $< $@ ++ ++libcblas.so.3: libcblas.so.3.0 ++ ln -sf $< $@ ++ ++libcblas.so.3.0 : libcblas.a libatlas.so ++ ld $(LDFLAGS) -shared -soname libcblas.so.3 -o $@ \ ++ --whole-archive libcblas.a \ ++ --no-whole-archive -L. -latlas $(F77SYSLIB) -lm -lc ++ ++libf77blas.so: libf77blas.so.3 ++ ln -sf $< $@ ++ ++libf77blas.so.3: libf77blas.so.3.0 ++ ln -sf $< $@ ++ ++libf77blas.so.3.0 : libf77blas.a libcblas.so libatlas.so ++ ld $(LDFLAGS) -shared -soname libf77blas.so.3 -o $@ \ ++ --whole-archive libf77blas.a \ ++ --no-whole-archive -L. -lcblas -latlas $(F77SYSLIB) -lm -lc ++ ++liblapack.so: liblapack.so.3 ++ ln -sf $< $@ ++ ++liblapack.so.3: liblapack.so.3.0 ++ ln -sf $< $@ ++ ++liblapack.so.3.0 : liblapack.a libatlas.so libcblas.so libf77blas.so ++ ld $(LDFLAGS) -shared -soname liblapack.so.3 -o $@ \ ++ --whole-archive liblapack.a \ ++ --no-whole-archive -L. -lf77blas -lcblas -latlas -lm -lc $(F77SYSLIB) ++ ++atlas/libblas.so: atlas/libblas.so.3 ++ (cd atlas && ln -sf libblas.so.3 libblas.so) ++ ++atlas/libblas.so.3: atlas/libblas.so.3.0 ++ (cd atlas && ln -sf libblas.so.3.0 libblas.so.3) ++ ++atlas/libblas.so.3.0: libf77blas.a libcblas.a libatlas.so.3.0 ++ if test -f libptf77blas.a -a -f libptcblas.a; then \ ++ ld $(LDFLAGS) -shared -soname libblas.so.3 -o $@ \ ++ --whole-archive libptf77blas.a libptcblas.a \ ++ --no-whole-archive -L. -latlas $(F77SYSLIB) -lm -lc; \ ++ else \ ++ ld $(LDFLAGS) -shared -soname libblas.so.3 -o $@ \ ++ --whole-archive libf77blas.a libcblas.a \ ++ --no-whole-archive -L. -latlas $(F77SYSLIB) -lm -lc; \ ++ fi ++ ++atlas/liblapack.so: atlas/liblapack.so.3 ++ (cd atlas && ln -sf liblapack.so.3 liblapack.so) ++ ++atlas/liblapack.so.3: atlas/liblapack.so.3.0 ++ (cd atlas && ln -sf liblapack.so.3.0 liblapack.so.3) ++ ++atlas/liblapack.so.3.0: atlas/liblapack.a atlas/libblas.so libatlas.so.3.0 ++ ld $(LDFLAGS) -shared -soname liblapack.so.3 -o $@ \ ++ --whole-archive atlas/liblapack.a \ ++ --no-whole-archive -L . -L atlas -lblas -latlas $(F77SYSLIB) -lm -lc ++ + # =================================================================== + # The following commands are to build dynamic/shared objects on Linux + # using the gnu gcc or ld +--- a/Make.top ++++ b/Make.top +@@ -16,6 +16,7 @@ build: + cd bin/ ; ./xatlas_build $(INSTFLAGS) + cd lib/ ; $(MAKE) atlas/libblas.a + cd lib/ ; $(MAKE) atlas/liblapack.a ++ cd lib/ ; $(MAKE) fullshared + + time: + ./xatlbench -dc $(BLDdir)/bin/INSTALL_LOG -dp $(BLDdir)/ARCHS/$(ARCH) +@@ -683,3 +684,21 @@ install_static_lib: + - cp $(LIBdir)/libsatlas.so $(INSTdir)/. + - cp $(LIBdir)/libtatlas.so $(INSTdir)/. + ++install_shared_lib: ++ for lib in atlas cblas f77blas lapack; do \ ++ cp $(LIBdir)/lib$$lib.so.3.0 $(INSTdir)/. ; \ ++ ( cd $(INSTdir) && ln -s lib$$lib.so.3.0 lib$$lib.so.3); \ ++ ( cd $(INSTdir) && ln -s lib$$lib.so.3 lib$$lib.so); \ ++ chmod 0644 $(INSTdir)/lib$$lib.so ;\ ++ chmod 0644 $(INSTdir)/lib$$lib.so.3 ;\ ++ chmod 0644 $(INSTdir)/lib$$lib.so.3.0 ;\ ++ done; ++ mkdir -p $(INSTdir)/atlas ++ for lib in blas lapack; do \ ++ cp $(LIBdir)/atlas/lib$$lib.so.3.0 $(INSTdir)/atlas/. ; \ ++ ( cd $(INSTdir)/atlas/ && ln -s lib$$lib.so.3.0 lib$$lib.so.3); \ ++ ( cd $(INSTdir)/atlas/ && ln -s lib$$lib.so.3 lib$$lib.so); \ ++ chmod 0644 $(INSTdir)/atlas/lib$$lib.so ;\ ++ chmod 0644 $(INSTdir)/atlas/lib$$lib.so.3 ;\ ++ chmod 0644 $(INSTdir)/atlas/lib$$lib.so.3.0 ;\ ++ done; +--- a/CONFIG/src/Makefile ++++ b/CONFIG/src/Makefile +@@ -576,6 +576,7 @@ $(LIBINSTdir) : + install : $(INCINSTdir) $(LIBINSTdir) $(INCINSTdir)/atlas + $(MAKE) -f Make.top install_inc INSTdir=$(INCINSTdir) + $(MAKE) -f Make.top install_static_lib INSTdir=$(LIBINSTdir) ++ $(MAKE) -f Make.top install_shared_lib INSTdir=$(LIBINSTdir) + + confclean: $(CLEANdep) + rm -f *core* *.o config?.out diff --git a/atlas/default/static_full_blas_lapack.diff b/atlas/default/static_full_blas_lapack.diff new file mode 100644 index 00000000..788be9bb --- /dev/null +++ b/atlas/default/static_full_blas_lapack.diff @@ -0,0 +1,116 @@ +Description: Create full featured static libblas.a and liblapack.a + The resulting BLAS library will use the threaded flavour if the build has been + configured to create them. Otherwise, the serial library is used. +Author: David Evans +Last-Update: 2013-06-05 +--- +This patch header follows DEP-3: http://dep.debian.net/deps/dep3/ +--- a/Make.top ++++ b/Make.top +@@ -14,6 +14,9 @@ + build: + cd bin/ ; $(MAKE) xatlas_build + cd bin/ ; ./xatlas_build $(INSTFLAGS) ++ cd lib/ ; $(MAKE) atlas/libblas.a ++ cd lib/ ; $(MAKE) atlas/liblapack.a ++ + time: + ./xatlbench -dc $(BLDdir)/bin/INSTALL_LOG -dp $(BLDdir)/ARCHS/$(ARCH) + C_sanity_test: +@@ -401,6 +404,7 @@ + $(MAKE) -f Make.top leafstart leaf=src/blas/reference/level1 + $(MAKE) -f Make.top leafstart leaf=src/auxil + $(MAKE) -f Make.top leafstart leaf=src/testing ++ $(MAKE) -f Make.top leafstart leaf=lib/atlas + $(MAKE) -f Make.top leafstart leaf=lib + $(MAKE) -f Make.top leafstart leaf=bin + cd src/threads ; touch atomic.inc +@@ -591,6 +595,7 @@ + + lib : + mkdir lib ++ mkdir lib/atlas + + include: + mkdir include +@@ -654,12 +659,17 @@ + chmod 0644 $(INSTdir)/cblas.h $(INSTdir)/clapack.h + cp $(INCAdir)/* $(INSTdir)/atlas/. + - chmod 0644 $(INSTdir)/atlas/* +-install_lib: ++ ++install_static_lib: + cp $(LIBdir)/libatlas.a $(INSTdir)/. + cp $(LIBdir)/libcblas.a $(INSTdir)/. + cp $(LIBdir)/liblapack.a $(INSTdir)/. ++ mkdir -p $(INSTdir)/atlas ++ cp $(LIBdir)/atlas/libblas.a $(INSTdir)/atlas/. ++ cp $(LIBdir)/atlas/liblapack.a $(INSTdir)/atlas/. + chmod 0644 $(INSTdir)/libatlas.a $(INSTdir)/liblapack.a \ +- $(INSTdir)/libcblas.a ++ $(INSTdir)/libcblas.a $(INSTdir)/atlas/libblas.a \ ++ $(INSTdir)/atlas/liblapack.a + - cp $(LIBdir)/libf77blas.a $(INSTdir)/. + - chmod 0644 $(INSTdir)/libf77blas.a + - cp $(LIBdir)/libptcblas.a $(INSTdir)/. +--- a/makes/Make.lib ++++ b/makes/Make.lib +@@ -175,6 +175,35 @@ + LIBS="$(LIBS)" LIBINSTdir="$(LIBINSTdir)" ; \ + fi + ++ ++# Build full netlib blas/lapack libraries: ++atlas/libblas.a: libatlas.a ++ mkdir tmp ++ cd tmp && \ ++ ar x ../libatlas.a && \ ++ if test -f ../libptf77blas.a -a -f ../libptcblas.a; then \ ++ ar x ../libptf77blas.a && \ ++ ar x ../libptcblas.a; \ ++ else \ ++ ar x ../libf77blas.a && \ ++ ar x ../libcblas.a; \ ++ fi ++ ar r $@ tmp/*.o ++ rm -rf tmp ++ ++atlas/liblapack.a: liblapack.a libatlas.a ++ mkdir tmp ++ cd tmp && \ ++ ar x ../liblapack.a && \ ++ if test -f ../libptcblas.a; then \ ++ ar x ../libptcblas.a; \ ++ else \ ++ ar x ../libcblas.a; \ ++ fi ++ ar r $@ tmp/*.o ++ rm -rf tmp ++ ++ + # + # Builds one shared lib from all ATLAS files + # +--- a/CONFIG/src/Makefile ++++ b/CONFIG/src/Makefile +@@ -575,7 +575,7 @@ + chmod 0755 $(LIBINSTdir) + install : $(INCINSTdir) $(LIBINSTdir) $(INCINSTdir)/atlas + $(MAKE) -f Make.top install_inc INSTdir=$(INCINSTdir) +- $(MAKE) -f Make.top install_lib INSTdir=$(LIBINSTdir) ++ $(MAKE) -f Make.top install_static_lib INSTdir=$(LIBINSTdir) + + confclean: $(CLEANdep) + rm -f *core* *.o config?.out +--- a/CONFIG/src/SpewMakeInc.c ++++ b/CONFIG/src/SpewMakeInc.c +@@ -604,6 +604,9 @@ + } + fprintf(fpout, " TESTlib = $(LIBdir)/libtstatlas.a\n\n"); + ++ fprintf(fpout, " FULLBLASlib = $(LIBdir)/atlas/libblas.a\n"); ++ fprintf(fpout, " FULLLAPACKlib = $(LIBdir)/atlas/liblapack.a\n"); ++ + fprintf(fpout, "# -------------------------------------------\n"); + fprintf(fpout, "# Upper bound on largest cache size, in bytes\n"); + fprintf(fpout, "# -------------------------------------------\n"); diff --git a/bcftools/1.2/build.yml b/bcftools/1.2/build.yml new file mode 100644 index 00000000..8d514767 --- /dev/null +++ b/bcftools/1.2/build.yml @@ -0,0 +1,17 @@ +build: + commands: + - tar jxf bcftools-${version}.tar.bz2 + - cd bcftools-${version} + - sed -i.bak 's#/usr/local#/build/dest#' Makefile + - make + - make install + - tar zcf /host/bcftools-${version}-Linux-x86_64.tar.gz -C /build/dest . + urls: + - https://github.com/samtools/bcftools/releases/download/${version}/bcftools-${version}.tar.bz2 +meta: + image: debian:squeeze + version: !!python/unicode '1.2' +prebuild: + packages: 'libc6-dev zlib1g-dev + + ' diff --git a/bcftools/default/build.yml b/bcftools/default/build.yml new file mode 100644 index 00000000..cf81515d --- /dev/null +++ b/bcftools/default/build.yml @@ -0,0 +1,16 @@ +meta: + image: 'debian:squeeze' + version: 1.2 +prebuild: + packages: | + libc6-dev zlib1g-dev +build: + urls: + - https://github.com/samtools/bcftools/releases/download/${version}/bcftools-${version}.tar.bz2 + commands: + - tar jxf bcftools-${version}.tar.bz2 + - cd bcftools-${version} + - sed -i.bak 's#/usr/local#/build/dest#' Makefile + - make + - make install + - tar zcf /host/bcftools-${version}-Linux-x86_64.tar.gz -C /build/dest . diff --git a/build-all.sh b/build-all.sh new file mode 100755 index 00000000..8c87ccbf --- /dev/null +++ b/build-all.sh @@ -0,0 +1,14 @@ +./build ubuntu atlas +./build ubuntu bcftools +./build ubuntu emboos +./build ubuntu jellyfish +./build ubuntu kraken +./build ubuntu ngninx +./build ubuntu openms +./build ubuntu pkiss +./build ubuntu rnashapes +./build ubuntu samtools +./build ubuntu tpp +./build ubuntu ucsc + +python build.py --quiet tbl2asn diff --git a/build.py b/build.py new file mode 100644 index 00000000..33fce4f4 --- /dev/null +++ b/build.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python +import subprocess +import shutil +import re +import yaml +import sys +import os +import argparse + + +def main(package, version='default', image=None, dryrun=False, clone=False, quiet=False): + parent_dir = os.path.dirname(os.path.abspath(__file__)) + build_dir = os.path.join(parent_dir, package, version) + log_file = os.path.join(parent_dir, package, 'build.log') + + _store_new_version = False + if not os.path.exists(build_dir): + if os.path.exists(os.path.join(parent_dir, package, 'default')): + build_dir = os.path.join(parent_dir, package, 'default') + # In this case it's an "unknown" version, so we'll just be friendly + # and serialize their yaml into a new versioned directory for them. + _store_new_version = True + else: + if version == 'default': + print "Default recipe for %s not found" % image + else: + print "Neither a default recipe, nor a recipe at version %s were found" % version + sys.exit(4) + + if _store_new_version and clone: + target = os.path.join(parent_dir, package, version) + # Clone the default + shutil.copytree(build_dir, target) + # Update other variables. + build_dir = target + + tentative_yaml_path = os.path.join(build_dir, 'build.yml') + + if not os.path.exists(tentative_yaml_path) and not os.path.isdir(tentative_yaml_path): + print "No yaml file found at %s" % tentative_yaml_path + sys.exit(3) + + with open(tentative_yaml_path, 'r') as handle: + image_data = yaml.load(handle) + + DOCKER_TEMPLATE = """ +FROM %(image)s +MAINTAINER Nare Coraor + +ENV DEBIAN_FRONTEND noninteractive +%(meta_env_string)s +%(env_string)s + +VOLUME ["/host"] +# Pre-build packages +%(prebuild_packages)s + +# Pre-build commands +%(prebuild_commands)s + +ENTRYPOINT ["/bin/bash", "/host/build.sh"] + """ + + SCRIPT_TEMPLATE = """#!/bin/sh +urls=" +{url_list} +" + +mkdir -p /build/ && cd /build/; + +( for url in $urls; do + wget --quiet "$url" || false || exit +done) + +{commands}""" + + template_values = {'image': 'ubuntu', 'prebuild_commands': "", 'env_string': ""} + # ENVIRONMENT + meta_env = {} + docker_env = {} + # If there's a 'meta' section, copy that into the environment, as we've + # stored things like OS/package name/version there historically. Useful now + # for the 'default' images which have a version hardcoded. + if 'meta' in image_data: + meta_env = image_data['meta'] + template_values['image'] = image_data['meta'].get('image', 'ubuntu') + + # If an image is sepecified, we overwrite the Dockerfile's FROM with that + # image + if image is not None: + template_values['image'] = image + # Additionally we store the actual image that the package is built with in + # the environment, so it's accessible to build scripts. + docker_env['image'] = template_values['image'] + + # The docker environment (in the Dockerfile) is built from the image_data's + # meta, and 'env' sections. + if 'env' in image_data: + docker_env.update(image_data['env']) + template_values['env_string'] = '\n'.join(['ENV %s %s' % (key, docker_env[key]) + for key in docker_env]) + template_values['meta_env_string'] = '\n'.join(['ENV %s %s' % (key, meta_env[key]) + for key in meta_env]) + + # Version is often used to construct downloads, if it's set to "default", + # then re-set it to the value in the default image's metadat + if version == 'default': + version = image_data['meta'].get('version', 'default') + + prebuild_packages = ['wget', 'openssl', 'ca-certificates', 'build-essential'] + if 'prebuild' in image_data and 'packages' in image_data['prebuild']: + pkgs = image_data['prebuild']['packages'] + if isinstance(pkgs, str): + prebuild_packages.extend(pkgs.strip().split()) + elif isinstance(pkgs, list): + prebuild_packages.extend(pkgs) + else: + print "Unknown data type for /prebuild/packages, please use a space delimited string of package names, or a list of strings" + sys.exit(5) + + template_values['prebuild_packages'] = "RUN apt-get -qq update && apt-get install --no-install-recommends -y %s" % ' '.join(prebuild_packages) + + if 'prebuild' in image_data and 'commands' in image_data['prebuild']: + template_values['prebuild_commands'] = '\n'.join([ + 'RUN %s' % command.strip() + for command in image_data['prebuild']['commands'] + ]) + + with open(os.path.join(build_dir, 'Dockerfile'), 'w') as dockerfile: + dockerfile.write(DOCKER_TEMPLATE % template_values) + + with open(os.path.join(build_dir, 'build.sh'), 'w') as script: + urls = image_data['build'].get('urls', []) + commands = image_data['build'].get('commands', []) + + script.write(SCRIPT_TEMPLATE.format(url_list=' '.join(urls), + commands='\n'.join(commands))) + + image_name = re.sub('[^A-Za-z0-9_]', '_', 'docker_build_%s_%s' % (package, template_values['image'])) + command = ['docker', 'build', '-t', image_name, '.'] + runcmd = ['docker', 'run', '--volume=%s/:/host/' % build_dir, '--env=pkg=%s' % package, + '--env=version=%s' % version, image_name] + if not dryrun: + with open(log_file, 'w') as handle: + execute(command, cwd=build_dir, log=handle, quiet=quiet) + if not quiet: + print ' '.join(runcmd) + execute(runcmd, cwd=build_dir, log=handle, quiet=quiet) + else: + # I am *lazy* during debugging phase + runcmd = runcmd[0:3] + ['-it', '--entrypoint=/bin/bash'] + runcmd[3:] + print ' '.join(command) + print ' '.join(runcmd) + + +def execute(command, cwd=None, log=None, quiet=False): + popen = subprocess.Popen(command, stdout=subprocess.PIPE, cwd=cwd) + for line in iter(popen.stdout.readline, b""): + try: + # Log output on demand, rather than just stdout + + # I don't know why I write these features when I only use them one + # way, and it's ugly, non-reusable code in the first place + if log is not None: + log.write(line) + + if not quiet: + print line, + except KeyboardInterrupt: + sys.exit() + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Build things inside of docker') + parser.add_argument('package', help='Name of the package, should be a folder') + parser.add_argument('--version', help='Version of the package, should be a folder inside package, or "default" for versionless recipes', + default='default') + parser.add_argument('--dryrun', action='store_true', help='Only generate files, does not build and run the image') + parser.add_argument('--image', help='Build image against a different target OS, e.g. "debian:squeeze"') + parser.add_argument('--clone', action='store_true', help='When building a default image with a different version, clone the build.yml and hardcode the version') + parser.add_argument('--quiet', action='store_true', help='Be a bit quieter') + args = parser.parse_args() + main(**vars(args)) diff --git a/emboss/6.6.0/build.yml b/emboss/6.6.0/build.yml new file mode 100644 index 00000000..5456140e --- /dev/null +++ b/emboss/6.6.0/build.yml @@ -0,0 +1,22 @@ +build: + commands: + - mkdir -p $HOME/bin/${arch}/ /build/dest/bin /build/dest/lib/ + - tar xfz EMBOSS-${version}.tar.gz + - cd EMBOSS-${version} + - export ORIGIN='$ORIGIN' + - export LDFLAGS='-Wl,-rpath,$${ORIGIN}/../lib' + - ./configure --prefix /build/dest --without-x + - make + - make install + - tar zcf /host/${pkg}-${version}-Linux-${arch}.tar.gz -C /build/dest . + urls: + - ftp://emboss.open-bio.org/pub/EMBOSS/EMBOSS-${version}.tar.gz +env: + arch: x86_64 +meta: + image: debian:squeeze + version: 6.6.0 +prebuild: + packages: 'libc6-dev zlib1g-dev libncurses5-dev + + ' diff --git a/emboss/default/build.yml b/emboss/default/build.yml new file mode 100644 index 00000000..d089c748 --- /dev/null +++ b/emboss/default/build.yml @@ -0,0 +1,21 @@ +meta: + image: 'debian:squeeze' + version: 6.6.0 +env: + arch: x86_64 +prebuild: + packages: | + libc6-dev zlib1g-dev libncurses5-dev +build: + urls: + - ftp://emboss.open-bio.org/pub/EMBOSS/EMBOSS-${version}.tar.gz + commands: + - mkdir -p $HOME/bin/${arch}/ /build/dest/bin /build/dest/lib/ + - tar xfz EMBOSS-${version}.tar.gz + - cd EMBOSS-${version} + - export ORIGIN='$ORIGIN' + - export LDFLAGS='-Wl,-rpath,$${ORIGIN}/../lib' + - ./configure --prefix /build/dest --without-x + - make + - make install + - tar zcf /host/${pkg}-${version}-Linux-${arch}.tar.gz -C /build/dest . diff --git a/jellyfish/default/build.yml b/jellyfish/default/build.yml new file mode 100644 index 00000000..6ae91c5c --- /dev/null +++ b/jellyfish/default/build.yml @@ -0,0 +1,16 @@ +meta: + image: 'debian:squeeze' + version: 1.1.11 +prebuild: + packages: | + libc6-dev zlib1g-dev +build: + urls: + - http://www.cbcb.umd.edu/software/jellyfish/jellyfish-${version}.tar.gz + commands: + - tar xfz jellyfish-${version}.tar.gz + - cd jellyfish-${version} + - ./configure --prefix=/build/dest + - make + - make install + - tar zcf /host/jellyfish-${version}-Linux-x86_64.tar.gz -C /build/dest . diff --git a/kraken/default/build.yml b/kraken/default/build.yml new file mode 100644 index 00000000..ad20ed7c --- /dev/null +++ b/kraken/default/build.yml @@ -0,0 +1,15 @@ +meta: + image: 'ubuntu' + version: 0.10.5-beta +prebuild: + packages: | + libc6-dev zlib1g-dev +build: + urls: + - https://ccb.jhu.edu/software/kraken/dl/kraken-${version}.tgz + commands: + - tar xfz kraken-${version}.tar.gz + - cd kraken-${version} + - mkdir -p /build/dest/bin + - sh install_kraken.sh /build/dest/bin/ + - tar zcf /host/kraken-${version}-Linux-x86_64.tar.gz -C /build/dest . diff --git a/nginx/default/build.yml b/nginx/default/build.yml new file mode 100644 index 00000000..baea8327 --- /dev/null +++ b/nginx/default/build.yml @@ -0,0 +1,39 @@ +meta: + image: 'ubuntu' + version: 1.4.6 +prebuild: + packages: | + git +build: + commands: + - ubuntu_version=${nginx_version}-1ubuntu3.2 + - ppa_version=${ubuntu_version}ppa1 + - dch_message="Restore the nginx-upload module from 2.2 branch in Github, now compatible with nginx 1.4.x." + - DEBFULLNAME="Nathan Coraor" + - DEBEMAIL="nate@bx.psu.edu" + - export ubuntu_version ppa_version dch_message DEBEMAIL DEBFULLNAME + - build=/host/build.$(hostname) + - echo 'America/New_York' > /etc/timezone + - dpkg-reconfigure tzdata + - apt-get source $pkg + - | + git clone -b 2.2 \ + --single-branch https://github.com/vkholodkov/nginx-upload-module.git/ \ + nginx-${nginx_version}/debian/modules/nginx-upload + - upload_module_shortrev=$(git --git-dir=nginx-${nginx_version}/debian/modules/nginx-upload/.git rev-parse --short HEAD) + - rm -rf nginx/debian/modules/nginx-upload/.git + - | + sed -e '/^ #Removed as it no longer works with 1.3.x and above.$/d' \ + -e 's/^ #\(nginx-upload\)$/ \1/' \ + -e 's%^ #\( Homepage: https://github.com/vkholodkov/nginx-upload-module\)$% \1/tree/2.2%' \ + -e "s/^ # Version: 2.2.0.*$/ Version: 2.2.1-${upload_module_shortrev}/" \ + -i nginx-${nginx_version}/debian/modules/README.Modules-versions + - | + sed -e 's#\(^\t --add-module=$(MODULESDIR)/nginx-upload-progress \\\)#\t --add-module=$(MODULESDIR)/nginx-upload \\\n\1#' \ + -i nginx-${nginx_version}/debian/rules + - cd nginx-${nginx_version} + - dch -v ${ppa_version} -D ${dch_dist} "${dch_message}" + - debuild -S -sd -us -uc + - echo "To sign: debsign -S ${pkg}_${ppa_version}_source.changes" && + - echo "To push: dput ppa:natefoo/nginx ${pkg}_${ppa_version}_source.changes" && + - echo "To push: dput ppa:galaxyproject/nginx ${pkg}_${ppa_version}_source.changes" diff --git a/nginx/default/control.patch b/nginx/default/control.patch new file mode 100644 index 00000000..d7271d7d --- /dev/null +++ b/nginx/default/control.patch @@ -0,0 +1,316 @@ +--- debian/control.orig 2015-02-26 20:39:55.592321023 -0500 ++++ debian/control 2015-02-26 21:48:02.044147424 -0500 +@@ -32,16 +32,24 @@ + + Package: nginx + Architecture: all +-Depends: nginx-core (>= ${source:Version}) | +- nginx-full (>= ${source:Version}) | +- nginx-light (>= ${source:Version}) | +- nginx-extras (>= ${source:Version}) | +- nginx-naxsi (>= ${source:Version}) , +- nginx-core (<< ${source:Version}.1~) | +- nginx-full (<< ${source:Version}.1~) | +- nginx-light (<< ${source:Version}.1~) | +- nginx-extras (<< ${source:Version}.1~) | +- nginx-naxsi (<< ${source:Version}.1~) , ++Depends: nginx-core (>= ${nginxSourceDebVersion}) | ++ nginx-full (>= ${nginxSourceDebVersion}) | ++ nginx-light (>= ${nginxSourceDebVersion}) | ++ nginx-extras (>= ${nginxSourceDebVersion}) | ++ nginx-naxsi (>= ${nginxSourceDebVersion}) | ++ nginx-core-upload (>= ${source:Version}) | ++ nginx-full-upload (>= ${source:Version}) | ++ nginx-light-upload (>= ${source:Version}) | ++ nginx-extras-upload (>= ${source:Version}) , ++ nginx-core (<< ${nginxSourceDebVersion}.1~) | ++ nginx-full (<< ${nginxSourceDebVersion}.1~) | ++ nginx-light (<< ${nginxSourceDebVersion}.1~) | ++ nginx-extras (<< ${nginxSourceDebVersion}.1~) | ++ nginx-naxsi (<< ${nginxSourceDebVersion}.1~) | ++ nginx-core-upload (<< ${source:Version}.1~) | ++ nginx-full-upload (<< ${source:Version}.1~) | ++ nginx-light-upload (<< ${source:Version}.1~) | ++ nginx-extras-upload (<< ${source:Version}.1~) , + ${misc:Depends} + Description: small, powerful, scalable web/proxy server + Nginx ("engine X") is a high-performance web and reverse proxy server +@@ -51,45 +59,16 @@ + This is a dependency package to install either nginx-core (by default), + nginx-full, nginx-light, nginx-extras, or nginx-naxsi. + +-Package: nginx-doc +-Architecture: all +-Section: doc +-Depends: lsb-base (>= 3.2-14), ${misc:Depends} +-Description: small, powerful, scalable web/proxy server - documentation +- Nginx ("engine X") is a high-performance web and reverse proxy server +- created by Igor Sysoev. It can be used both as a standalone web server +- and as a proxy to reduce the load on back-end HTTP or mail servers. +- . +- This package provides extra documentation to help unleash the power of Nginx. +- +-Package: nginx-common +-Architecture: all +-Depends: lsb-base (>= 3.2-14), ${misc:Depends} +-Replaces: nginx (<< 0.8.54-4), +- nginx-extras (<< 0.8.54-4), +- nginx-full (<< 0.8.54-4), +- nginx-light (<< 0.8.54-4) +-Breaks: nginx (<< 0.8.54-4), +- nginx-extras (<< 0.8.54-4), +- nginx-full (<< 0.8.54-4), +- nginx-light (<< 0.8.54-4) +-Suggests: fcgiwrap, nginx-doc +-Description: small, powerful, scalable web/proxy server - common files +- Nginx ("engine X") is a high-performance web and reverse proxy server +- created by Igor Sysoev. It can be used both as a standalone web server +- and as a proxy to reduce the load on back-end HTTP or mail servers. +- . +- This package contains base configuration files used by all versions of +- nginx. +- +-Package: nginx-core ++Package: nginx-core-upload + Architecture: any +-Depends: nginx-common (= ${source:Version}), ${misc:Depends}, ${shlibs:Depends} ++Depends: nginx-common (= ${nginxSourceDebVersion}), ${misc:Depends}, ${shlibs:Depends} + Breaks: nginx (<< 1.4.5-1) + Provides: httpd, httpd-cgi, nginx +-Conflicts: nginx-extras, nginx-light, nginx-naxsi, nginx-full +-Suggests: nginx-doc (= ${source:Version}) +-Description: nginx web/proxy server (core version) ++Conflicts: nginx-extras, nginx-light, nginx-naxsi, nginx-full, ++ nginx-extras-upload, nginx-light-upload, ++ nginx-full-upload ++Suggests: nginx-doc (= ${nginxSourceDebVersion}) ++Description: nginx web/proxy server (core+upload version) + Nginx ("engine X") is a high-performance web and reverse proxy server + created by Igor Sysoev. It can be used both as a standalone web server + and as a proxy to reduce the load on back-end HTTP or mail servers. +@@ -109,16 +88,20 @@ + . + MAIL MODULES: Mail Core, IMAP, POP3, SMTP, SSL. + . +- NOTE: This is identical to the -full build, but without any third party +- modules built in. ++ THIRD PARTY MODULES: Upload. ++ . ++ NOTE: This is identical to the -full build, but with only the upload third ++ party module built in. + +-Package: nginx-core-dbg ++Package: nginx-core-upload-dbg + Architecture: any + Section: debug + Priority: extra +-Depends: nginx-core (= ${binary:Version}), ${misc:Depends} +-Conflicts: nginx-extras-dbg, nginx-light-dbg, nginx-naxsi-dbg, nginx-full-dbg +-Description: nginx web/proxy server (core version) - debugging symbols ++Depends: nginx-core-upload (= ${binary:Version}), ${misc:Depends} ++Conflicts: nginx-extras-dbg, nginx-light-dbg, nginx-naxsi-dbg, nginx-full-dbg, ++ nginx-extras-upload-dbg, nginx-light-upload-dbg, ++ nginx-full-upload-dbg ++Description: nginx web/proxy server (core+upload version) - debugging symbols + Nginx ("engine X") is a high-performance web and reverse proxy server + created by Igor Sysoev. It can be used both as a standalone web server + and as a proxy to reduce the load on back-end HTTP or mail servers. +@@ -126,14 +109,15 @@ + This package provides debugging symbols for nginx-core, to assist in debugging + issues that you may find. It should not be required for normal operation. + +-Package: nginx-full ++Package: nginx-full-upload + Architecture: any +-Depends: nginx-common (= ${source:Version}), ${misc:Depends}, ${shlibs:Depends} ++Depends: nginx-common (= ${nginxSourceDebVersion}), ${misc:Depends}, ${shlibs:Depends} + Breaks: nginx (<< 1.4.5-1) + Provides: httpd, httpd-cgi, nginx +-Conflicts: nginx-extras, nginx-light, nginx-naxsi, nginx-core +-Suggests: nginx-doc (= ${source:Version}) +-Description: nginx web/proxy server (standard version) ++Conflicts: nginx-extras, nginx-light, nginx-naxsi, nginx-core, ++ nginx-extras-upload, nginx-light-upload, nginx-core-upload ++Suggests: nginx-doc (= ${nginxSourceDebVersion}) ++Description: nginx web/proxy server (standard+upload version) + Nginx ("engine X") is a high-performance web and reverse proxy server + created by Igor Sysoev. It can be used both as a standalone web server + and as a proxy to reduce the load on back-end HTTP or mail servers. +@@ -154,15 +138,17 @@ + MAIL MODULES: Mail Core, IMAP, POP3, SMTP, SSL. + . + THIRD PARTY MODULES: Auth PAM, DAV Ext, Echo, HTTP Substitution Filter, +- Upstream Fair Queue. ++ Upstream Fair Queue, Upload. + +-Package: nginx-full-dbg ++Package: nginx-full-upload-dbg + Architecture: any + Section: debug + Priority: extra +-Depends: nginx-full (= ${binary:Version}), ${misc:Depends} +-Conflicts: nginx-extras-dbg, nginx-light-dbg, nginx-naxsi-dbg, nginx-core-dbg +-Description: nginx web/proxy server (standard version) - debugging symbols ++Depends: nginx-full-upload (= ${binary:Version}), ${misc:Depends} ++Conflicts: nginx-extras-dbg, nginx-light-dbg, nginx-naxsi-dbg, nginx-core-dbg, ++ nginx-extras-upload-dbg, nginx-light-upload-dbg, ++ nginx-core-upload-dbg ++Description: nginx web/proxy server (standard+upload version) - debugging symbols + Nginx ("engine X") is a high-performance web and reverse proxy server + created by Igor Sysoev. It can be used both as a standalone web server + and as a proxy to reduce the load on back-end HTTP or mail servers. +@@ -170,15 +156,16 @@ + This package provides debugging symbols for nginx-full, to assist in debugging + issues that you may find. It should not be required for normal operation. + +-Package: nginx-light ++Package: nginx-light-upload + Architecture: any + Priority: extra +-Depends: nginx-common (= ${source:Version}), ${misc:Depends}, ${shlibs:Depends} ++Depends: nginx-common (= ${nginxSourceDebVersion}), ${misc:Depends}, ${shlibs:Depends} + Breaks: nginx (<< 1.4.5-1) + Provides: httpd, httpd-cgi, nginx +-Conflicts: nginx-extras, nginx-full, nginx-naxsi, nginx-core +-Suggests: nginx-doc (= ${source:Version}) +-Description: nginx web/proxy server (basic version) ++Conflicts: nginx-extras, nginx-full, nginx-naxsi, nginx-core, ++ nginx-extras-upload, nginx-full-upload, nginx-core-upload ++Suggests: nginx-doc (= ${nginxSourceDebVersion}) ++Description: nginx web/proxy server (basic+upload version) + Nginx ("engine X") is a high-performance web and reverse proxy server + created by Igor Sysoev. It can be used both as a standalone web server + and as a proxy to reduce the load on back-end HTTP or mail servers. +@@ -192,15 +179,17 @@ + OPTIONAL HTTP MODULES: Debug, Gzip Precompression, IPv6, Real Ip, SSL, + Stub Status. + . +- THIRD PARTY MODULES: Echo. ++ THIRD PARTY MODULES: Echo, Upload. + +-Package: nginx-light-dbg ++Package: nginx-light-upload-dbg + Architecture: any + Section: debug + Priority: extra +-Depends: nginx-light (= ${binary:Version}), ${misc:Depends} +-Conflicts: nginx-extras-dbg, nginx-full-dbg, nginx-naxsi-dbg, nginx-core-dbg +-Description: nginx web/proxy server (basic version) - debugging symbols ++Depends: nginx-light-upload (= ${binary:Version}), ${misc:Depends} ++Conflicts: nginx-extras-dbg, nginx-full-dbg, nginx-naxsi-dbg, nginx-core-dbg, ++ nginx-extras-upload-dbg, nginx-full-upload-dbg, ++ nginx-core-upload-dbg ++Description: nginx web/proxy server (basic+upload version) - debugging symbols + Nginx ("engine X") is a high-performance web and reverse proxy server + created by Igor Sysoev. It can be used both as a standalone web server + and as a proxy to reduce the load on back-end HTTP or mail servers. +@@ -209,18 +198,19 @@ + debugging issues that you may find. It should not be required for normal + operation. + +-Package: nginx-extras ++Package: nginx-extras-upload + Architecture: any + Priority: extra +-Depends: nginx-common (= ${source:Version}), ++Depends: nginx-common (= ${nginxSourceDebVersion}), + ${misc:Depends}, + ${perl:Depends}, + ${shlibs:Depends} + Breaks: nginx (<< 1.4.5-1) + Provides: httpd, httpd-cgi, nginx +-Conflicts: nginx-full, nginx-light, nginx-naxsi, nginx-core +-Suggests: nginx-doc (= ${source:Version}) +-Description: nginx web/proxy server (extended version) ++Conflicts: nginx-full, nginx-light, nginx-naxsi, nginx-core, ++ nginx-full-upload, nginx-light-upload, nginx-core-upload ++Suggests: nginx-doc (= ${nginxSourceDebVersion}) ++Description: nginx web/proxy server (extended+upload version) + Nginx ("engine X") is a high-performance web and reverse proxy server + created by Igor Sysoev. It can be used both as a standalone web server + and as a proxy to reduce the load on back-end HTTP or mail servers. +@@ -242,15 +232,15 @@ + . + THIRD PARTY MODULES: Auth PAM, Chunkin, DAV Ext, Echo, Embedded Lua, + Fancy Index, HttpHeadersMore, HTTP Substitution Filter, http push, +- Nginx Development Kit, Upload Progress, Upstream Fair Queue. ++ Nginx Development Kit, Upload Progress, Upstream Fair Queue, Upload. + +-Package: nginx-extras-dbg ++Package: nginx-extras-upload-dbg + Architecture: any + Section: debug + Priority: extra +-Depends: nginx-extras (= ${binary:Version}), ${misc:Depends} ++Depends: nginx-extras-upload (= ${binary:Version}), ${misc:Depends} + Conflicts: nginx-full-dbg, nginx-light-dbg, nginx-naxsi-dbg, nginx-core-dbg +-Description: nginx web/proxy server (extended version) - debugging symbols ++Description: nginx web/proxy server (extended+upload version) - debugging symbols + Nginx ("engine X") is a high-performance web and reverse proxy server + created by Igor Sysoev. It can be used both as a standalone web server + and as a proxy to reduce the load on back-end HTTP or mail servers. +@@ -258,64 +248,3 @@ + This package provides debugging symbols for nginx-extras, to assist in + debugging issues that you may find. It should not be required for normal + operation. +- +-Package: nginx-naxsi +-Architecture: any +-Priority: extra +-Depends: nginx-common (= ${source:Version}), ${misc:Depends}, ${shlibs:Depends} +-Breaks: nginx (<< 1.4.5-1), +- nginx-naxsi-ui (<< 1.4.5-1) +-Provides: httpd, httpd-cgi, nginx +-Conflicts: nginx-extras, nginx-full, nginx-light, nginx-core +-Description: nginx web/proxy server (version with naxsi) +- Nginx ("engine X") is a high-performance web and reverse proxy server +- created by Igor Sysoev. It can be used both as a standalone web server +- and as a proxy to reduce the load on back-end HTTP or mail servers. +- . +- This package provides a version of nginx with the basic modules, plus +- the naxsi Web Application Firewall module. +- . +- STANDARD HTTP MODULES: Core, Access, Auth Basic, Auto Index, Browser, +- Charset, Core, Empty GIF, FastCGI, Geo, Gzip, Headers, Index, +- Limit Requests, Limit Zone, Log, Map, Memcached, Proxy, Referer, +- Rewrite, Split Clients, SSI, Upstream, User ID. +- . +- OPTIONAL HTTP MODULES: Debug, IPv6, Real IP, SSL, Stub Status. +- . +- THIRD PARTY MODULES: Naxsi, Cache Purge, Upstream Fair. +- +-Package: nginx-naxsi-dbg +-Architecture: any +-Section: debug +-Priority: extra +-Depends: nginx-naxsi (= ${binary:Version}), ${misc:Depends} +-Conflicts: nginx-extras-dbg, nginx-full-dbg, nginx-light-dbg, nginx-core-dbg +-Description: nginx web/proxy server (version with naxsi) - debugging symbols +- Nginx ("engine X") is a high-performance web and reverse proxy server +- created by Igor Sysoev. It can be used both as a standalone web server +- and as a proxy to reduce the load on back-end HTTP or mail servers. +- . +- This package provides debugging symbols for nginx-naxsi, to assist in +- debugging issues that you may find. It should not be required for normal +- operation. +- +-Package: nginx-naxsi-ui +-Architecture: all +-Priority: extra +-Depends: daemon, +- dbconfig-common, +- nginx-naxsi (>= ${source:Version}), +- nginx-naxsi (<< ${source:Version}.1~), +- python-twisted-web, +- ${misc:Depends} +-Description: nginx web/proxy server - naxsi configuration front-end +- Nginx ("engine X") is a high-performance web and reverse proxy server +- created by Igor Sysoev. It can be used both as a standalone web server +- and as a proxy to reduce the load on back-end HTTP or mail servers. +- . +- This package provides the autolearning daemon and web user interface for +- nginx's naxsi module. +- . +- It includes an interceptor (listening on TCP port 8080), which monitors +- HTTP requests from naxsi, and an extractor (running on TCP port 8081), +- which reads the database and prints reports about blocked requests. diff --git a/nginx/default/rules.patch b/nginx/default/rules.patch new file mode 100644 index 00000000..eb5ac8f2 --- /dev/null +++ b/nginx/default/rules.patch @@ -0,0 +1,119 @@ +--- debian/rules.orig 2015-02-26 19:58:46.218909084 -0500 ++++ debian/rules 2015-02-26 22:13:48.193023932 -0500 +@@ -2,19 +2,19 @@ + + debian_cflags:=$(shell dpkg-buildflags --get CFLAGS | sed 's/-O3/-O2/') $(shell dpkg-buildflags --get CPPFLAGS) + debian_ldflags:=$(shell dpkg-buildflags --get LDFLAGS) ++nginxSourceDebVersion:=$(shell cat debian/nginxSourceDebVersion) + + # export necessary for (hardening) flags for perl + # (src/http/modules/perl/Makefile.PL). + DEBIAN_NGINX_PERL_LDFLAGS:= $(debian_ldflags) + export DEBIAN_NGINX_PERL_LDFLAGS + +-FLAVOURS:=core full light extras naxsi ++FLAVOURS:=core-upload full-upload light-upload extras-upload + +-BUILDDIR_core = $(CURDIR)/debian/build-core +-BUILDDIR_full = $(CURDIR)/debian/build-full +-BUILDDIR_light = $(CURDIR)/debian/build-light +-BUILDDIR_extras = $(CURDIR)/debian/build-extras +-BUILDDIR_naxsi = $(CURDIR)/debian/build-naxsi ++BUILDDIR_core-upload = $(CURDIR)/debian/build-core-upload ++BUILDDIR_full-upload = $(CURDIR)/debian/build-full-upload ++BUILDDIR_light-upload = $(CURDIR)/debian/build-light-upload ++BUILDDIR_extras-upload = $(CURDIR)/debian/build-extras-upload + MODULESDIR = $(CURDIR)/debian/modules + BASEDIR = $(CURDIR) + +@@ -53,6 +53,7 @@ + --with-http_ssl_module \ + --with-http_stub_status_module \ + --with-http_realip_module \ ++ --add-module=$(MODULESDIR)/nginx-upload \ + + config.env.%: + dh_testdir +@@ -64,8 +65,8 @@ + cp -Pa $(CURDIR)/src $(BUILDDIR_$*)/ + cp -Pa $(CURDIR)/man $(BUILDDIR_$*)/ + +-config.status.core: config.env.core +- cd $(BUILDDIR_core) && ./configure \ ++config.status.core-upload: config.env.core-upload ++ cd $(BUILDDIR_core-upload) && ./configure \ + $(common_configure_flags) \ + --with-http_addition_module \ + --with-http_dav_module \ +@@ -80,8 +81,8 @@ + >$@ + touch $@ + +-config.status.full: config.env.full +- cd $(BUILDDIR_full) && ./configure \ ++config.status.full-upload: config.env.full-upload ++ cd $(BUILDDIR_full-upload) && ./configure \ + $(common_configure_flags) \ + --with-http_addition_module \ + --with-http_dav_module \ +@@ -101,8 +102,8 @@ + >$@ + touch $@ + +-config.status.light: config.env.light +- cd $(BUILDDIR_light) && ./configure \ ++config.status.light-upload: config.env.light-upload ++ cd $(BUILDDIR_light-upload) && ./configure \ + $(common_configure_flags) \ + --with-http_gzip_static_module \ + --without-http_browser_module \ +@@ -120,8 +121,8 @@ + >$@ + touch $@ + +-config.status.extras: config.env.extras +- cd $(BUILDDIR_extras) && ./configure \ ++config.status.extras-upload: config.env.extras-upload ++ cd $(BUILDDIR_extras-upload) && ./configure \ + $(common_configure_flags) \ + --with-http_addition_module \ + --with-http_dav_module \ +@@ -153,20 +154,6 @@ + >$@ + touch $@ + +-config.status.naxsi: config.env.naxsi +- cd $(BUILDDIR_naxsi) && ./configure \ +- $(common_configure_flags) \ +- --without-mail_pop3_module \ +- --without-mail_smtp_module \ +- --without-mail_imap_module \ +- --without-http_uwsgi_module \ +- --without-http_scgi_module \ +- --add-module=$(MODULESDIR)/naxsi/naxsi_src \ +- --add-module=$(MODULESDIR)/nginx-cache-purge \ +- --add-module=$(MODULESDIR)/nginx-upstream-fair \ +- >$@ +- touch $@ +- + config.status.%: + echo "configuration for flavour $* not yet defined." + +@@ -223,7 +210,7 @@ + dh_compress -i + dh_fixperms -i + dh_installdeb -i +- dh_gencontrol -i ++ dh_gencontrol -i -- -VnginxSourceDebVersion=$(nginxSourceDebVersion) + dh_md5sums -i + dh_builddeb -i + +@@ -239,7 +226,7 @@ + dh_fixperms -a + dh_installdeb -a + dh_shlibdeps -a +- dh_gencontrol -a ++ dh_gencontrol -a -- -VnginxSourceDebVersion=$(nginxSourceDebVersion) + dh_md5sums -a + dh_builddeb -a + diff --git a/openms/default/build.yml b/openms/default/build.yml new file mode 100644 index 00000000..efc2bda5 --- /dev/null +++ b/openms/default/build.yml @@ -0,0 +1,49 @@ +meta: + image: 'debian:squeeze' + version: master +prebuild: + packages: | + libc6-dev zlib1g-dev libncurses5-dev libtool libqt4-dev libbz2-dev + cmake autoconf automake + commands: + # Unfortunately this executes after packges :| + # TODO: refactor prebuild a list of dicts with a + # 'type=package|command' + - echo 'deb http://http.debian.net/debian-backports squeeze-backports main' >> /etc/apt/sources.list +build: + urls: + - https://github.com/OpenMS/OpenMS/archive/Release2.0.0.tar.gz + - https://github.com/OpenMS/contrib/archive/master.tar.gz + commands: + # Since a newer version of cmake is required...we update it now. + - apt-get -qq update && apt-get dist-upgrade -y + - mkdir -p /build/dest/bin /build/dest/lib /usr/lib/x86_64-linux-gnu/libgthread-2.0.so + - tar xfz Release2.0.0.tar.gz + - tar xfz master.tar.gz + - mv contrib-master OpenMS-Release2.0.0/contrib + - cd OpenMS-Release2.0.0/contrib + - cmake . -DBUILD_TYPE=SEQAN + - cmake . -DBUILD_TYPE=LIBSVM + - cmake . -DBUILD_TYPE=XERCESC + - cmake . -DBUILD_TYPE=GSL + - cmake . -DBUILD_TYPE=BOOST -DNUMBER_OF_JOBS=4 + - cmake . -DBUILD_TYPE=COINOR + - cmake . -DBUILD_TYPE=BZIP2 + - cmake . -DBUILD_TYPE=GLPK + - cmake . -DBUILD_TYPE=EIGEN + - cmake . -DBUILD_TYPE=WILDMAGIC + - cd .. + - mkdir build + - cd build + - ORIGIN='$ORIGIN' + - export ORIGIN + - LDFLAGS='-Wl,-rpath,$${ORIGIN}/../lib' cmake .. -DCMAKE_INSTALL_PREFIX=/build/dest -DHAS_XSERVER=OFF -DENABLE_TUTORIALS=OFF -DENABLE_STYLE_TESTING=OFF -DENABLE_UNITYBUILD=OFF -DWITH_GUI=OFF + - make OpenMS TOPP UTILS + - make install + - cp -d /usr/lib/libQtCore* /build/dest/lib/ + - cp -d /usr/lib/libQtNetwork* /build/dest/lib/ + - cp -d /usr/lib/libgomp.* /build/dest/lib/ + - cp -d /lib/libglib-2.0* /build/dest/lib/ + - cp -d /usr/lib/libgthread-2.0.so* /build/dest/lib/ + - cp -d /lib/libpcre.so* /build/dest/lib/ + - tar zcf /host/${pkg}-${version}-Linux-${arch}.tar.gz -C /build/dest . diff --git a/perlgd/default/build.yml b/perlgd/default/build.yml new file mode 100644 index 00000000..038093e4 --- /dev/null +++ b/perlgd/default/build.yml @@ -0,0 +1,10 @@ +meta: + image: 'debian:latest' + version: 2.56 +build: + urls: + - https://cpan.metacpan.org/authors/id/L/LD/LDS/GD-${version}.tar.gz + commands: + - tar xfz GD-${version}.tar.gz + - chmod ugo+w GD-${version}/bdf_scripts/* + - tar zcf /host/GD-${version}.tar.gz GD-${version}/ diff --git a/pkiss/default/build.yml b/pkiss/default/build.yml new file mode 100644 index 00000000..9571c536 --- /dev/null +++ b/pkiss/default/build.yml @@ -0,0 +1,33 @@ +meta: + image: 'debian:squeeze' + version: 2.2.12 +prebuild: + packages: | + libboost-dev libboost-test-dev libboost-program-options-dev bison flex + mercurial ksh +build: + urls: + - https://launchpad.net/~bibi-help/+archive/ubuntu/bibitools/+files/bellmansgapc_2015.03.17.orig.tar.gz + - https://launchpad.net/~bibi-help/+archive/ubuntu/bibitools/+files/pkiss_${version}.orig.tar.gz + commands: + - mkdir -p /build/dest/bin + - mkdir -p /build/dest/lib + - tar -xf bellmansgapc_2015.03.17.orig.tar.gz + - tar -xf pkiss_${version}.orig.tar.gz + - cd bellmansgapc-2015.03.17 + - cp config-templates/generic.mf config.mf + - sed -i 's|#PREFIX ?=|PREFIX = /build/gapc|g' config.mf + - make + - make install + - cp -r /build/gapc/lib /build/dest + - export PATH=$PATH:/build/gapc/bin + - cd ../pkiss_2.2.12/pkiss + - make -C Misc/Applications/pKiss all + - cp -r Misc/Applications/lib/* /build/dest/lib + - cp Misc/Applications/pKiss/x86_64-linux-gnu/* /build/dest/bin + - cp Misc/Applications/pKiss/pKiss /build/dest/bin + # for the librna.so and librnafast.so + # export LD_LIBRARY_PATH=installdir/lib:$LD_LIBRARY_PATH + # for the binaries + # export PATH=installdir/bin:$PATH + - tar zcf /host/${pkg}-${version}-Linux-${arch}.tar.gz -C /build/dest . diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..c83b3ed2 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +pyyaml +beautifulsoup4 +filecache diff --git a/samtools/default/build.yml b/samtools/default/build.yml new file mode 100644 index 00000000..0697c506 --- /dev/null +++ b/samtools/default/build.yml @@ -0,0 +1,19 @@ +meta: + image: 'debian:squeeze' + version: 0.1.19 +prebuild: + packages: | + libc6-dev zlib1g-dev libncurses5-dev +build: + urls: + - http://downloads.sourceforge.net/project/samtools/samtools/${version}/samtools-${version}.tar.bz2 + commands: + - tar jxf samtools-${version}.tar.bz2 + - cd samtools-${version} + - make + - mkdir -p /build/dest/bin /build/dest/lib /build/dest/include/bam /build/dest/include/bcf + - cp samtools bcftools/bcftools bcftools/vcfutils.pl /build/dest/bin + - cp libbam.a bcftools/libbcf.a /build/dest/lib + - cp *.h /build/dest/include/bam + - cp bcftools/*.h /build/dest/include/bcf + - tar zcf /host/samtools-${version}-Linux-x86_64.tar.gz -C /build/dest . diff --git a/tbl2asn/default/build.yml b/tbl2asn/default/build.yml new file mode 100644 index 00000000..854994f4 --- /dev/null +++ b/tbl2asn/default/build.yml @@ -0,0 +1,21 @@ +meta: + image: 'ubuntu' + pkg: tbl2asn +prebuild: + packages: wget +build: + urls: + # I'm not convinced that these will all be at the same version # but....well...there's not a lot to do. + - ftp://ftp.ncbi.nih.gov/toolbox/ncbi_tools/converters/by_program/tbl2asn/linux.tbl2asn.gz + - ftp://ftp.ncbi.nih.gov/toolbox/ncbi_tools/converters/by_program/tbl2asn/linux64.tbl2asn.gz + - ftp://ftp.ncbi.nih.gov/toolbox/ncbi_tools/converters/by_program/tbl2asn/mac.tbl2asn.gz + - ftp://ftp.ncbi.nih.gov/toolbox/ncbi_tools/converters/by_program/tbl2asn/win.tbl2asn.zip + commands: + - gunzip linux64.tbl2asn.gz + - chmod +x linux64.tbl2asn + - version=$(./linux64.tbl2asn p | egrep 'tbl2asn ([^ ]*)' -o | sed 's/tbl2asn //g') + - gzip linux64.tbl2asn + - mv linux.tbl2asn.gz /host/tbl2asn-${version}-Linux-x86.gz + - mv linux64.tbl2asn.gz /host/tbl2asn-${version}-Linux-x64.gz + - mv mac.tbl2asn.gz /host/tbl2asn-${version}-Darwin-x64.gz + - mv win.tbl2asn.zip /host/tbl2asn-${version}-Windows-x64.zip diff --git a/tpp/default/build.yml b/tpp/default/build.yml new file mode 100644 index 00000000..d6c48ab4 --- /dev/null +++ b/tpp/default/build.yml @@ -0,0 +1,29 @@ +meta: + image: 'debian:squeeze' + version: 4.8.0 +env: + arch: x86_64 +prebuild: + packages: | + libc6-dev zlib1g-dev libncurses5-dev libbz2-dev time libxml-parser-perl + libgd2-xpm-dev +build: + urls: + - http://downloads.sourceforge.net/project/sashimi/Trans-Proteomic%20Pipeline%20%28TPP%29/TPP%20v4.8%20%28philae%29%20rev%200/TPP_${version}-src.tgz + commands: + - export build_dir=/build/dest/ + - mkdir -p ${build_dir}/bin ${build_dir}/lib + - tar xfz TPP_${version}-src.tgz + - cd TPP-${version}/trans_proteomic_pipeline/src/ + - echo "TPP_ROOT=${build_dir}" > Makefile.config.incl + - echo "TPP_WEB=${build_dir}/web/" >> Makefile.config.incl + - echo "CGI_USER_DIR=${build_dir}/cgi-bin/" >> Makefile.config.incl + - echo "HTMLDOC_BIN=" >> Makefile.config.incl + - echo "LINK=shared" >> Makefile.config.incl + - echo "LIBEXT=a" >> Makefile.config.incl + - make ARCH=linux-${arch} && make install ARCH=linux-${arch} + - cp /lib/libbz2.so.1.0.4 /build/dest/lib + - cd /build/dest/lib/ + - ln -s ./libbz2.so.1.0.4 ./libbz2.so + - ln -s ./libbz2.so.1.0.4 ./libbz2.so.1.0 + - tar zcf /host/${pkg}-${version}-Linux-${arch}.tar.gz -C /build/dest . diff --git a/ucsc/314/build.yml b/ucsc/314/build.yml new file mode 100644 index 00000000..9e3255e0 --- /dev/null +++ b/ucsc/314/build.yml @@ -0,0 +1,31 @@ +build: + commands: + - mkdir -p $HOME/bin/${arch}/ /build/dest/bin /build/dest/lib + - unzip jksrc.v${version}.zip + - cd kent/src/lib/ + - make + - cd ../utils/ + - COPT='-O -g -Wl,-rpath,$${ORIGIN}/../lib' + - ORIGIN='$ORIGIN' + - export COPT ORIGIN + - find . -type d -maxdepth 1 -mindepth 1 -exec make -C '{}' \; + - mv $HOME/bin/${arch}/* /build/dest/bin/ + - "for lib in $(ldd /build/dest/bin/faToTwoBit | grep -o '=> /[^ ]*' | sed 's/=>\ + \ //g'); \ndo\n case $lib in\n */libcrypto.so*|*/libgcc_s.so*|*/libmysqlclient.so*|*/libssl.so*|*/libstdc++.so*)\n\ + \ echo \"including $lib\";\n cp $lib /build/dest/lib/;\n\ + \ ;;\n esac\ndone\n" + - tar zcf /host/${pkg}-${version}-Linux-${arch}.tar.gz -C /build/dest . + urls: + - http://hgdownload.soe.ucsc.edu/admin/jksrc.v${version}.zip +env: + arch: x86_64 +meta: + image: ubuntu + pkg: ucsc + version: '314' +prebuild: + packages: 'libc6-dev zlib1g-dev libncurses5-dev libmysqlclient-dev unzip + + libpng-dev libssl-dev + + ' diff --git a/ucsc/default/build.yml b/ucsc/default/build.yml new file mode 100644 index 00000000..f1bc44ce --- /dev/null +++ b/ucsc/default/build.yml @@ -0,0 +1,35 @@ +meta: + image: 'ubuntu' + pkg: ucsc + version: 312 +env: + arch: x86_64 +prebuild: + packages: | + libc6-dev zlib1g-dev libncurses5-dev libmysqlclient-dev unzip + libpng-dev libssl-dev +build: + urls: + - http://hgdownload.soe.ucsc.edu/admin/jksrc.v${version}.zip + commands: + - mkdir -p $HOME/bin/${arch}/ /build/dest/bin /build/dest/lib + - unzip jksrc.v${version}.zip + - cd kent/src/lib/ + - make + - cd ../utils/ + - COPT='-O -g -Wl,-rpath,$${ORIGIN}/../lib' + - ORIGIN='$ORIGIN' + - export COPT ORIGIN + - find . -type d -maxdepth 1 -mindepth 1 -exec make -C '{}' \; + - mv $HOME/bin/${arch}/* /build/dest/bin/ + - | + for lib in $(ldd /build/dest/bin/faToTwoBit | grep -o '=> /[^ ]*' | sed 's/=> //g'); + do + case $lib in + */libcrypto.so*|*/libgcc_s.so*|*/libmysqlclient.so*|*/libssl.so*|*/libstdc++.so*) + echo "including $lib"; + cp $lib /build/dest/lib/; + ;; + esac + done + - tar zcf /host/${pkg}-${version}-Linux-${arch}.tar.gz -C /build/dest . diff --git a/util/check-for-updates.py b/util/check-for-updates.py new file mode 100644 index 00000000..f9fd80e8 --- /dev/null +++ b/util/check-for-updates.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python +import fnmatch +import shutil +import filecache +import os +import yaml +import urllib2 +import json +import re + + +class UpdateFetcher(object): + + def __init__(self): + self.packages = {} + + def register_package(self, package, version): + if package not in self.packages: + self.packages[package] = [] + self.packages[package].append(version) + + def check_for_updates(self): + flu_kwattrs = {} + for package in self.packages: + if 'default' in self.packages[package]: + version = 'default' + else: + version = sorted(self.packages[package])[0] + + if package in ('bcftools', 'samtools'): + flu_kwattrs['version_scheme'] = 'numbered' + + urls = self.get_urls(package, version) + versioned_urls = [url for url in urls if '${' in url] + if len(versioned_urls) > 0: + for url in versioned_urls: + latest_version = self.find_latest_url(url, **flu_kwattrs) + + # If that latest version doesn't have a folder all to itself... + if latest_version is not None: + if latest_version not in self.packages[package]: + self.create_new_version(package, version, latest_version) + else: + print "Already have %s@%s" % (package, latest_version) + + def create_new_version(self, package, version, latest): + """Given a package name, a version to copy from, and the version to + create...create a new folder for that version + """ + print 'Creating %s %s %s' % (package, version, latest) + + src_yml = os.path.join(package, version, 'build.yml') + dest = os.path.join(package, latest) + dest_yml = os.path.join(package, latest, 'build.yml') + os.mkdir(dest) + + with open(dest_yml, 'w') as out_handle: + with open(src_yml, 'r') as in_handle: + data = yaml.load(in_handle) + data['meta']['version'] = latest + out_handle.write(yaml.dump(data, default_flow_style=False)) + + def get_urls(self, package, version): + with open(os.path.join(package, version, 'build.yml'), 'r') as handle: + data = yaml.load(handle) + return data.get('build', {}).get('urls', []) + + @classmethod + def filter_version(cls, version_list, scheme): + if scheme == 'numbered': + return [v for v in version_list if re.match('^[0-9.]*$', v)] + + @filecache.filecache(60 * 60 * 24) + def find_latest_url(self, url, version_scheme=None): + """Given a URL, find the most recent version of that available on the + website. + + This is obviously not an error-free process, and will involve lots of + hardcoded data, however, if it replaces a human doing it, that's all we + need. + """ + + if 'github' in url: + url_parts = url.split('/') + username = url_parts[3] + reponame = url_parts[4] + if '/archive/' in url: + # Latest is tough here as we're downloading a specific revision + # number. + gh_api_url = 'https://api.github.com/repos/%s/%s/git/refs/heads/master' % (username, reponame) + gh_api_response = urllib2.urlopen(gh_api_url) + return json.load(gh_api_response).get('object', {}).get('sha', None) + elif 'releases/download' in url: + gh_api_url = 'https://api.github.com/repos/%s/%s/tags' % (username, reponame) + gh_api_response = urllib2.urlopen(gh_api_url) + try: + versions = json.load(gh_api_response) + good_versions = self.filter_version([x['name'] for x in versions], version_scheme) + return good_versions[0] + # Have to catch the case when it isn't a list + except Exception: + return None + else: + print "Unknown github URL type %s" % url + elif 'hgdownload.soe.ucsc.edu' in url: + ucsc_url = 'http://hgdownload.soe.ucsc.edu/admin/' + ucsc_response = urllib2.urlopen(ucsc_url).read() + from bs4 import BeautifulSoup + soup = BeautifulSoup(ucsc_response) + versions = [x.text.replace('jksrc.v', '').replace('.zip', '') + for x in soup.find_all('a') if 'jksrc.v' in x.text] + return str(sorted(map(int, versions))[-1]) + elif 'emboss.open-bio' in url: + emboss_url = 'ftp://emboss.open-bio.org/pub/EMBOSS/' + emboss_response = urllib2.urlopen(emboss_url).read() + emboss_tar = [line for line in emboss_response.split('\n') if + 'EMBOSS-' in line and 'latest' not in line] + return emboss_tar[0].strip().split()[-1].replace('EMBOSS-', '').replace('.tar.gz', '') + else: + print "Updated unimplemented for %s" % url + return None + # http://downloads.sourceforge.net/project/math-atlas/Stable/${version}/atlas${version}.tar.bz2 + + +def main(): + uf = UpdateFetcher() + for root, dirs, files in os.walk('.'): + for filename in files: + if fnmatch.fnmatch(filename, 'build.yml'): + package, version = os.path.split(root) + package = package[2:] + uf.register_package(package, version) + + uf.check_for_updates() + + +if __name__ == '__main__': + main()