diff --git a/setup.py b/setup.py index 9de4598d..1446bb77 100644 --- a/setup.py +++ b/setup.py @@ -41,6 +41,7 @@ install_requires = requirements, extras_require = { ':python_version=="2.6"': ["ordereddict"], + 'lzma:python_version<="3.3"': ['backports.lzma'], 'platform_specific': ["lionshead"], }, entry_points = { diff --git a/starforge/util.py b/starforge/util.py index ce6661f1..11600a83 100644 --- a/starforge/util.py +++ b/starforge/util.py @@ -8,6 +8,13 @@ import zipfile from os.path import join, abspath, expanduser from subprocess import Popen, CalledProcessError, PIPE +try: + import lzma +except ImportError: + try: + import backports.lzma as lzma + except ImportError: + lzma = None try: from configparser import ConfigParser, NoSectionError, NoOptionError except ImportError: @@ -15,6 +22,8 @@ from six import iteritems, string_types +UNSUPPORTED_ARCHIVE_MESSAGE = "Missing support for '{arctype}' archives, use `pip install starforge[{extra}]` to install" + def dict_merge(old, new): """Recursive dictionary merge, values in `new` will replace values of @@ -82,6 +91,9 @@ def __init__(self, arcfile): elif zipfile.is_zipfile(arcfile): self.arctype = 'zip' self.arc = zipfile.ZipFile(arcfile) + elif arcfile.endswith('.tar.xz'): + self.arctype = 'tar' + self.arc = tarfile.open(fileobj=lzma.open(arcfile)) else: raise Exception('Unknown archive type: %s' % arcfile) @@ -131,3 +143,25 @@ def universal(self): return asbool(universal) except (KeyError, NoSectionError, NoOptionError): return False + + +class UnsupportedArchiveModule(object): + def __init__(self, arctype, extra): + self.arctype = arctype + self.extra = extra + + def open(self, *args, **kwargs): + raise UnsupportedArchiveType( + UNSUPPORTED_ARCHIVE_MESSAGE.format( + arctype=self.arctype, + extra=self.extra, + ) + ) + + +class UnsupportedArchiveType(Exception): + pass + + +if lzma is None: + lzma = UnsupportedArchiveModule('xz', 'lzma') diff --git a/wheels/image/manylinux1-32/Dockerfile b/wheels/image/manylinux1-32/Dockerfile index 5d5d39f6..c53300c8 100644 --- a/wheels/image/manylinux1-32/Dockerfile +++ b/wheels/image/manylinux1-32/Dockerfile @@ -3,7 +3,7 @@ MAINTAINER Nate Coraor VOLUME ["/host"] -RUN /usr/bin/linux32 yum install -y PyYAML +RUN /usr/bin/linux32 yum install -y PyYAML xz-devel WORKDIR /work @@ -17,7 +17,7 @@ RUN /opt/wheelenv/bin/pip install --upgrade pip setuptools WORKDIR / COPY starforge.tar.gz /work/starforge.tar.gz -RUN /usr/bin/linux32 /opt/wheelenv/bin/pip install /work/starforge.tar.gz +RUN /usr/bin/linux32 /opt/wheelenv/bin/pip install /work/starforge.tar.gz[lzma] RUN rm -rf /work /root/.cache && \ yum clean all diff --git a/wheels/image/manylinux1/Dockerfile b/wheels/image/manylinux1/Dockerfile index 53222588..a1696b1e 100644 --- a/wheels/image/manylinux1/Dockerfile +++ b/wheels/image/manylinux1/Dockerfile @@ -3,7 +3,7 @@ MAINTAINER Nate Coraor VOLUME ["/host"] -RUN yum install -y PyYAML +RUN yum install -y PyYAML xz-devel WORKDIR /work @@ -17,7 +17,7 @@ RUN /opt/wheelenv/bin/pip install --upgrade pip setuptools WORKDIR / COPY starforge.tar.gz /work/starforge.tar.gz -RUN /opt/wheelenv/bin/pip install /work/starforge.tar.gz +RUN /opt/wheelenv/bin/pip install /work/starforge.tar.gz[lzma] RUN rm -rf /work /root/.cache && \ yum clean all diff --git a/wheels/image/osx-playbook.yml b/wheels/image/osx-playbook.yml index 681f61b4..98cd56a8 100644 --- a/wheels/image/osx-playbook.yml +++ b/wheels/image/osx-playbook.yml @@ -124,14 +124,6 @@ with_items: - 2.7 - - name: Install Starforge - command: /python/{{ item }}/bin/pip install {{ work_dir }}/starforge.tar.gz - args: - creates: /python/{{ item }}/bin/starforge - with_items: - - wheelenv - - cp27m-x86_64 - - name: Allow admin(s) to sudo without a password lineinfile: dest: /etc/sudoers @@ -143,3 +135,16 @@ shell: ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" /dev/null 2>&1; then + if ! docker inspect --type=image ${base}:${new_ver} >/dev/null 2>&1; then echo "Image exists: '${base}:${new_ver}', skipped" + elif [ $no_latest -ne 1 ]; then + continue # nothing to do elif ! docker inspect --type=image ${base}:latest >/dev/null 2>&1; then echo "Image '${base}:latest' does not exist, will create" else @@ -170,7 +182,9 @@ if [ $linux -eq 1 ]; then new_image="$(docker images -q ${base}:${new_ver}-${tmp_tag})" || { remove_new_images "Failed to get image ID for ${base}:${new_ver}-${tmp_tag}"; exit 1; } new_images+=("${new_image}") echo "New image for ${base} is ${new_image}" - for tag in "${new_ver}" 'latest'; do + tags=("${new_ver}") + [ $no_latest -ne 1 ] && tags+=('latest') + for tag in $tags; do echo "Tagging image ${new_image} with ${base}:${tag}" docker tag -f "${new_image}" "${base}:${tag}" || { remove_new_images "Failed to tag ${new_image} with tag ${base}:${tag}"; exit 1; } done @@ -215,7 +229,8 @@ if [ $macos -eq 1 ]; then sleep 5 done echo "Running Playbook" - ansible-playbook -i localhost, -e "ansible_ssh_port=${macos_ssh_port}" -e "ansible_ssh_private_key_file=${macos_ssh_identity_file}" -e "ssh_args='-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'" osx-playbook.yml + failed=0 + ansible-playbook -i localhost, -e "ansible_ssh_port=${macos_ssh_port}" -e "ansible_ssh_private_key_file=${macos_ssh_identity_file}" -e "ssh_args='-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'" osx-playbook.yml || failed=1 echo "Shutting down" ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p ${macos_ssh_port} -i ${macos_ssh_identity_file} root@localhost shutdown -h now || true # returns 255 count=0 @@ -231,13 +246,18 @@ if [ $macos -eq 1 ]; then echo "Waiting for guest shutdown..." sleep 5 done - echo "Creating RO snapshot" - sudo btrfs subvolume snapshot -r ${tmp_snap_path} ${new_snap_path} + if [ $failed -eq 0 ]; then + echo "Setting version in snapshot version file" + echo "${new_ver}" | sudo tee ${tmp_snap_path}/version + echo "Creating RO snapshot" + sudo btrfs subvolume snapshot -r ${tmp_snap_path} ${new_snap_path} + fi echo "Removing RW snapshot" sudo btrfs subvolume delete ${tmp_snap_path} + [ $failed -eq 1 ] && exit 1 for user in ${macos_snap_users}; do echo "Snapshotting for $user" - sudo bash -c "cd ${new_snap_path}; ./snap_for $user" + sudo bash -c "cd ${new_snap_path}; ./snap_for $user $no_latest" done fi