From b515bc504d229d4a6a2ae9602265f839a950ed6e Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Mon, 18 Sep 2017 17:43:59 -0400 Subject: [PATCH 1/4] Allow building docker images with dev versions of Starforge, bump version to a dev version. --- wheels/image/update-for-jenkins.sh | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/wheels/image/update-for-jenkins.sh b/wheels/image/update-for-jenkins.sh index 27f8fe79..7467af93 100755 --- a/wheels/image/update-for-jenkins.sh +++ b/wheels/image/update-for-jenkins.sh @@ -17,6 +17,7 @@ function print_usage() { echo ' -g: Append git short rev to version (automatic if version ends with `.dev*`)' echo ' -l: Build manylinux1 Docker images' echo ' -m: Build macOS QEMU images' + echo " -n: Don't update the 'latest' tag" } function get_tag() { @@ -51,9 +52,10 @@ function remove_new_images() { linux=0 macos=0 +no_latest=0 shortrev=0 cmds=(uuidgen) -while getopts "lmg" opt; do +while getopts "lmgn" opt; do case "$opt" in l) linux=1 @@ -63,6 +65,9 @@ while getopts "lmg" opt; do macos=1 cmds+=(ansible-playbook) ;; + n) + no_latest=1 + ;; g) shortrev=1 ;; @@ -127,8 +132,10 @@ if [ $linux -eq 1 ]; then cd "${dir}" for base in 'starforge/manylinux1' 'starforge/manylinux1-32'; do - if docker inspect --type=image ${base}:${new_ver} >/dev/null 2>&1; then + if ! docker inspect --type=image ${base}:${new_ver} >/dev/null 2>&1; then echo "Image exists: '${base}:${new_ver}', skipped" + elif [ $no_latest -ne 1 ]; then + continue # nothing to do elif ! docker inspect --type=image ${base}:latest >/dev/null 2>&1; then echo "Image '${base}:latest' does not exist, will create" else @@ -170,7 +177,9 @@ if [ $linux -eq 1 ]; then new_image="$(docker images -q ${base}:${new_ver}-${tmp_tag})" || { remove_new_images "Failed to get image ID for ${base}:${new_ver}-${tmp_tag}"; exit 1; } new_images+=("${new_image}") echo "New image for ${base} is ${new_image}" - for tag in "${new_ver}" 'latest'; do + tags=("${new_ver}") + [ $no_latest -ne 1 ] && tags+=('latest') + for tag in $tags; do echo "Tagging image ${new_image} with ${base}:${tag}" docker tag -f "${new_image}" "${base}:${tag}" || { remove_new_images "Failed to tag ${new_image} with tag ${base}:${tag}"; exit 1; } done From f48ab33755ae34cc2128f25b280fab2c0c5afd42 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Mon, 18 Sep 2017 17:45:25 -0400 Subject: [PATCH 2/4] Add lzma support for wheel building --- setup.py | 1 + starforge/util.py | 34 +++++++++++++++++++++++++++ wheels/image/manylinux1-32/Dockerfile | 4 ++-- wheels/image/manylinux1/Dockerfile | 4 ++-- 4 files changed, 39 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 9de4598d..1446bb77 100644 --- a/setup.py +++ b/setup.py @@ -41,6 +41,7 @@ install_requires = requirements, extras_require = { ':python_version=="2.6"': ["ordereddict"], + 'lzma:python_version<="3.3"': ['backports.lzma'], 'platform_specific': ["lionshead"], }, entry_points = { diff --git a/starforge/util.py b/starforge/util.py index ce6661f1..11600a83 100644 --- a/starforge/util.py +++ b/starforge/util.py @@ -8,6 +8,13 @@ import zipfile from os.path import join, abspath, expanduser from subprocess import Popen, CalledProcessError, PIPE +try: + import lzma +except ImportError: + try: + import backports.lzma as lzma + except ImportError: + lzma = None try: from configparser import ConfigParser, NoSectionError, NoOptionError except ImportError: @@ -15,6 +22,8 @@ from six import iteritems, string_types +UNSUPPORTED_ARCHIVE_MESSAGE = "Missing support for '{arctype}' archives, use `pip install starforge[{extra}]` to install" + def dict_merge(old, new): """Recursive dictionary merge, values in `new` will replace values of @@ -82,6 +91,9 @@ def __init__(self, arcfile): elif zipfile.is_zipfile(arcfile): self.arctype = 'zip' self.arc = zipfile.ZipFile(arcfile) + elif arcfile.endswith('.tar.xz'): + self.arctype = 'tar' + self.arc = tarfile.open(fileobj=lzma.open(arcfile)) else: raise Exception('Unknown archive type: %s' % arcfile) @@ -131,3 +143,25 @@ def universal(self): return asbool(universal) except (KeyError, NoSectionError, NoOptionError): return False + + +class UnsupportedArchiveModule(object): + def __init__(self, arctype, extra): + self.arctype = arctype + self.extra = extra + + def open(self, *args, **kwargs): + raise UnsupportedArchiveType( + UNSUPPORTED_ARCHIVE_MESSAGE.format( + arctype=self.arctype, + extra=self.extra, + ) + ) + + +class UnsupportedArchiveType(Exception): + pass + + +if lzma is None: + lzma = UnsupportedArchiveModule('xz', 'lzma') diff --git a/wheels/image/manylinux1-32/Dockerfile b/wheels/image/manylinux1-32/Dockerfile index 5d5d39f6..c53300c8 100644 --- a/wheels/image/manylinux1-32/Dockerfile +++ b/wheels/image/manylinux1-32/Dockerfile @@ -3,7 +3,7 @@ MAINTAINER Nate Coraor VOLUME ["/host"] -RUN /usr/bin/linux32 yum install -y PyYAML +RUN /usr/bin/linux32 yum install -y PyYAML xz-devel WORKDIR /work @@ -17,7 +17,7 @@ RUN /opt/wheelenv/bin/pip install --upgrade pip setuptools WORKDIR / COPY starforge.tar.gz /work/starforge.tar.gz -RUN /usr/bin/linux32 /opt/wheelenv/bin/pip install /work/starforge.tar.gz +RUN /usr/bin/linux32 /opt/wheelenv/bin/pip install /work/starforge.tar.gz[lzma] RUN rm -rf /work /root/.cache && \ yum clean all diff --git a/wheels/image/manylinux1/Dockerfile b/wheels/image/manylinux1/Dockerfile index 53222588..a1696b1e 100644 --- a/wheels/image/manylinux1/Dockerfile +++ b/wheels/image/manylinux1/Dockerfile @@ -3,7 +3,7 @@ MAINTAINER Nate Coraor VOLUME ["/host"] -RUN yum install -y PyYAML +RUN yum install -y PyYAML xz-devel WORKDIR /work @@ -17,7 +17,7 @@ RUN /opt/wheelenv/bin/pip install --upgrade pip setuptools WORKDIR / COPY starforge.tar.gz /work/starforge.tar.gz -RUN /opt/wheelenv/bin/pip install /work/starforge.tar.gz +RUN /opt/wheelenv/bin/pip install /work/starforge.tar.gz[lzma] RUN rm -rf /work /root/.cache && \ yum clean all From 57dd460abf1b459d0923317b2b6c4300ce384c96 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Sun, 1 Oct 2017 01:45:37 -0400 Subject: [PATCH 3/4] Changes for LZMA image on OS X --- wheels/image/osx-playbook.yml | 2 +- wheels/image/update-for-jenkins.sh | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/wheels/image/osx-playbook.yml b/wheels/image/osx-playbook.yml index 681f61b4..91799b9a 100644 --- a/wheels/image/osx-playbook.yml +++ b/wheels/image/osx-playbook.yml @@ -125,7 +125,7 @@ - 2.7 - name: Install Starforge - command: /python/{{ item }}/bin/pip install {{ work_dir }}/starforge.tar.gz + command: /python/{{ item }}/bin/pip install {{ work_dir }}/starforge.tar.gz[lzma] args: creates: /python/{{ item }}/bin/starforge with_items: diff --git a/wheels/image/update-for-jenkins.sh b/wheels/image/update-for-jenkins.sh index 7467af93..3df4edb3 100755 --- a/wheels/image/update-for-jenkins.sh +++ b/wheels/image/update-for-jenkins.sh @@ -17,7 +17,7 @@ function print_usage() { echo ' -g: Append git short rev to version (automatic if version ends with `.dev*`)' echo ' -l: Build manylinux1 Docker images' echo ' -m: Build macOS QEMU images' - echo " -n: Don't update the 'latest' tag" + echo " -n: Don't update the 'latest' tag (automatic if version ends with \`.dev*\`)" } function get_tag() { @@ -118,6 +118,11 @@ if [ $shortrev -ne 0 -o "${last:0:3}" == "dev" ]; then echo "Adding git shortrev '${shortrev}' to version, version is: $new_ver" fi +if [ "${last:0:3}" == "dev" ]; then + no_latest=1 + macos_snap_users="$USER" +fi + tmp_tag="$(uuidgen)" || { "Failed to generate UUID for temporary build tag"; exit 1; } if [ $linux -eq 1 ]; then @@ -240,6 +245,8 @@ if [ $macos -eq 1 ]; then echo "Waiting for guest shutdown..." sleep 5 done + echo "Setting version in snapshot version file" + echo "${new_ver}" | sudo tee ${tmp_snap_path}/version echo "Creating RO snapshot" sudo btrfs subvolume snapshot -r ${tmp_snap_path} ${new_snap_path} echo "Removing RW snapshot" From 55f2ed7da6a48c8da7df60a44213ff64027e39b0 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Sun, 1 Oct 2017 02:46:14 -0400 Subject: [PATCH 4/4] xz/lzma support osx fixes --- wheels/image/osx-playbook.yml | 21 +++++++++++++-------- wheels/image/update-for-jenkins.sh | 16 ++++++++++------ 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/wheels/image/osx-playbook.yml b/wheels/image/osx-playbook.yml index 91799b9a..98cd56a8 100644 --- a/wheels/image/osx-playbook.yml +++ b/wheels/image/osx-playbook.yml @@ -124,14 +124,6 @@ with_items: - 2.7 - - name: Install Starforge - command: /python/{{ item }}/bin/pip install {{ work_dir }}/starforge.tar.gz[lzma] - args: - creates: /python/{{ item }}/bin/starforge - with_items: - - wheelenv - - cp27m-x86_64 - - name: Allow admin(s) to sudo without a password lineinfile: dest: /etc/sudoers @@ -143,3 +135,16 @@ shell: ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"