Merge pull request #166 from natefoo/wheel-lzma

Support xz/lzma tarballs for wheel builds

setup.py

@@ -41,6 +41,7 @@
     install_requires = requirements,
     extras_require = {
         ':python_version=="2.6"': ["ordereddict"],
+        'lzma:python_version<="3.3"': ['backports.lzma'],
         'platform_specific': ["lionshead"],
     },
     entry_points = {

starforge/util.py

@@ -8,13 +8,22 @@
 import zipfile
 from os.path import join, abspath, expanduser
 from subprocess import Popen, CalledProcessError, PIPE
+try:
+    import lzma
+except ImportError:
+    try:
+        import backports.lzma as lzma
+    except ImportError:
+        lzma = None
 try:
     from configparser import ConfigParser, NoSectionError, NoOptionError
 except ImportError:
     from ConfigParser import ConfigParser, NoSectionError, NoOptionError
 
 from six import iteritems, string_types
 
+UNSUPPORTED_ARCHIVE_MESSAGE = "Missing support for '{arctype}' archives, use `pip install starforge[{extra}]` to install"
+
 
 def dict_merge(old, new):
     """Recursive dictionary merge, values in `new` will replace values of
@@ -82,6 +91,9 @@ def __init__(self, arcfile):
         elif zipfile.is_zipfile(arcfile):
             self.arctype = 'zip'
             self.arc = zipfile.ZipFile(arcfile)
+        elif arcfile.endswith('.tar.xz'):
+            self.arctype = 'tar'
+            self.arc = tarfile.open(fileobj=lzma.open(arcfile))
         else:
             raise Exception('Unknown archive type: %s' % arcfile)
 
@@ -131,3 +143,25 @@ def universal(self):
             return asbool(universal)
         except (KeyError, NoSectionError, NoOptionError):
             return False
+
+
+class UnsupportedArchiveModule(object):
+    def __init__(self, arctype, extra):
+        self.arctype = arctype
+        self.extra = extra
+
+    def open(self, *args, **kwargs):
+        raise UnsupportedArchiveType(
+            UNSUPPORTED_ARCHIVE_MESSAGE.format(
+                arctype=self.arctype,
+                extra=self.extra,
+            )
+        )
+
+
+class UnsupportedArchiveType(Exception):
+    pass
+
+
+if lzma is None:
+    lzma = UnsupportedArchiveModule('xz', 'lzma')

wheels/image/manylinux1-32/Dockerfile

@@ -3,7 +3,7 @@ MAINTAINER Nate Coraor <nate@bx.psu.edu>
 
 VOLUME ["/host"]
 
-RUN /usr/bin/linux32 yum install -y PyYAML
+RUN /usr/bin/linux32 yum install -y PyYAML xz-devel
 
 WORKDIR /work
 
@@ -17,7 +17,7 @@ RUN /opt/wheelenv/bin/pip install --upgrade pip setuptools
 WORKDIR /
 
 COPY starforge.tar.gz /work/starforge.tar.gz
-RUN /usr/bin/linux32 /opt/wheelenv/bin/pip install /work/starforge.tar.gz
+RUN /usr/bin/linux32 /opt/wheelenv/bin/pip install /work/starforge.tar.gz[lzma]
 
 RUN rm -rf /work /root/.cache && \
     yum clean all

wheels/image/manylinux1/Dockerfile

@@ -3,7 +3,7 @@ MAINTAINER Nate Coraor <nate@bx.psu.edu>
 
 VOLUME ["/host"]
 
-RUN yum install -y PyYAML
+RUN yum install -y PyYAML xz-devel
 
 WORKDIR /work
 
@@ -17,7 +17,7 @@ RUN /opt/wheelenv/bin/pip install --upgrade pip setuptools
 WORKDIR /
 
 COPY starforge.tar.gz /work/starforge.tar.gz
-RUN /opt/wheelenv/bin/pip install /work/starforge.tar.gz
+RUN /opt/wheelenv/bin/pip install /work/starforge.tar.gz[lzma]
 
 RUN rm -rf /work /root/.cache && \
     yum clean all

wheels/image/osx-playbook.yml

@@ -124,14 +124,6 @@
     with_items:
       - 2.7
 
-  - name: Install Starforge
-    command: /python/{{ item }}/bin/pip install {{ work_dir }}/starforge.tar.gz
-    args:
-      creates: /python/{{ item }}/bin/starforge
-    with_items:
-      - wheelenv
-      - cp27m-x86_64
-
   - name: Allow admin(s) to sudo without a password
     lineinfile:
       dest: /etc/sudoers
@@ -143,3 +135,16 @@
     shell: ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" </dev/null
     args:
       creates: /usr/local/bin/brew
+
+  - name: Install xz
+    remote_user: admin
+    homebrew:
+      name: xz
+
+  - name: Install Starforge
+    command: /python/{{ item }}/bin/pip install {{ work_dir }}/starforge.tar.gz[lzma]
+    args:
+      creates: /python/{{ item }}/bin/starforge
+    with_items:
+      - wheelenv
+      - cp27m-x86_64

wheels/image/update-for-jenkins.sh

@@ -17,6 +17,7 @@ function print_usage() {
     echo '  -g: Append git short rev to version (automatic if version ends with `.dev*`)'
     echo '  -l: Build manylinux1 Docker images'
     echo '  -m: Build macOS QEMU images'
+    echo "  -n: Don't update the 'latest' tag (automatic if version ends with \`.dev*\`)"
 }
 
 function get_tag() {
@@ -51,9 +52,10 @@ function remove_new_images() {
 
 linux=0
 macos=0
+no_latest=0
 shortrev=0
 cmds=(uuidgen)
-while getopts "lmg" opt; do
+while getopts "lmgn" opt; do
     case "$opt" in
         l)
             linux=1
@@ -63,6 +65,9 @@ while getopts "lmg" opt; do
             macos=1
             cmds+=(ansible-playbook)
             ;;
+        n)
+            no_latest=1
+            ;;
         g)
             shortrev=1
             ;;
@@ -113,6 +118,11 @@ if [ $shortrev -ne 0 -o "${last:0:3}" == "dev" ]; then
     echo "Adding git shortrev '${shortrev}' to version, version is: $new_ver"
 fi
 
+if [ "${last:0:3}" == "dev" ]; then
+    no_latest=1
+    macos_snap_users="$USER"
+fi
+
 tmp_tag="$(uuidgen)" || { "Failed to generate UUID for temporary build tag"; exit 1; }
 
 if [ $linux -eq 1 ]; then
@@ -127,8 +137,10 @@ if [ $linux -eq 1 ]; then
     cd "${dir}"
 
     for base in 'starforge/manylinux1' 'starforge/manylinux1-32'; do
-        if docker inspect --type=image ${base}:${new_ver} >/dev/null 2>&1; then
+        if ! docker inspect --type=image ${base}:${new_ver} >/dev/null 2>&1; then
             echo "Image exists: '${base}:${new_ver}', skipped"
+        elif [ $no_latest -ne 1 ]; then
+            continue # nothing to do
         elif ! docker inspect --type=image ${base}:latest >/dev/null 2>&1; then
             echo "Image '${base}:latest' does not exist, will create"
         else
@@ -170,7 +182,9 @@ if [ $linux -eq 1 ]; then
         new_image="$(docker images -q ${base}:${new_ver}-${tmp_tag})" || { remove_new_images "Failed to get image ID for ${base}:${new_ver}-${tmp_tag}"; exit 1; }
         new_images+=("${new_image}")
         echo "New image for ${base} is ${new_image}"
-        for tag in "${new_ver}" 'latest'; do
+        tags=("${new_ver}")
+        [ $no_latest -ne 1 ] && tags+=('latest')
+        for tag in $tags; do
             echo "Tagging image ${new_image} with ${base}:${tag}"
             docker tag -f "${new_image}" "${base}:${tag}" || { remove_new_images "Failed to tag ${new_image} with tag ${base}:${tag}"; exit 1; }
         done
@@ -215,7 +229,8 @@ if [ $macos -eq 1 ]; then
             sleep 5
         done
         echo "Running Playbook"
-        ansible-playbook -i localhost, -e "ansible_ssh_port=${macos_ssh_port}" -e "ansible_ssh_private_key_file=${macos_ssh_identity_file}" -e "ssh_args='-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'" osx-playbook.yml
+        failed=0
+        ansible-playbook -i localhost, -e "ansible_ssh_port=${macos_ssh_port}" -e "ansible_ssh_private_key_file=${macos_ssh_identity_file}" -e "ssh_args='-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'" osx-playbook.yml || failed=1
         echo "Shutting down"
         ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p ${macos_ssh_port} -i ${macos_ssh_identity_file} root@localhost shutdown -h now || true  # returns 255
         count=0
@@ -231,13 +246,18 @@ if [ $macos -eq 1 ]; then
             echo "Waiting for guest shutdown..."
             sleep 5
         done
-        echo "Creating RO snapshot"
-        sudo btrfs subvolume snapshot -r ${tmp_snap_path} ${new_snap_path}
+        if [ $failed -eq 0 ]; then
+            echo "Setting version in snapshot version file"
+            echo "${new_ver}" | sudo tee ${tmp_snap_path}/version
+            echo "Creating RO snapshot"
+            sudo btrfs subvolume snapshot -r ${tmp_snap_path} ${new_snap_path}
+        fi
         echo "Removing RW snapshot"
         sudo btrfs subvolume delete ${tmp_snap_path}
+        [ $failed -eq 1 ] && exit 1
         for user in ${macos_snap_users}; do
             echo "Snapshotting for $user"
-            sudo bash -c "cd ${new_snap_path}; ./snap_for $user"
+            sudo bash -c "cd ${new_snap_path}; ./snap_for $user $no_latest"
         done
     fi