diff --git a/.architectures-lib b/.architectures-lib new file mode 100644 index 0000000..05e126a --- /dev/null +++ b/.architectures-lib @@ -0,0 +1,57 @@ +#!/usr/bin/env bash + +_awkArch() { + local awkExpr="$1"; shift + awk "$@" "/^#|^\$/ { next } $awkExpr" release-architectures +} + +dpkgArches() { + _awkArch '{ print $2 }' +} + +dpkgToBashbrewArch() { + local dpkgArch="$1"; shift + _awkArch '$2 == dpkgArch { print $1; exit }' -v dpkgArch="$dpkgArch" +} + +dpkgToPyPyArch() { + local dpkgArch="$1"; shift + _awkArch '$2 == dpkgArch { print $3; exit }' -v dpkgArch="$dpkgArch" +} + +_generateParentRepoToArches() { + local repo="$1"; shift + local officialImagesUrl='https://github.com/docker-library/official-images/raw/master/library/' + + eval "declare -g -A parentRepoToArches=( $( + find -name 'Dockerfile' -exec awk ' + toupper($1) == "FROM" && $2 !~ /^('"$repo"'|scratch|microsoft\/[^:]+)(:|$)/ { + print "'"$officialImagesUrl"'" $2 + } + ' '{}' + \ + | sort -u \ + | xargs bashbrew cat --format '[{{ .RepoName }}:{{ .TagName }}]="{{ join " " .TagEntry.Architectures }}"' + ) )" +} +_generateParentRepoToArches 'pypy' + +hasBashbrewArch() { + local dir="$1"; shift + local bashbrewArch="$1"; shift + grep -qE "^# ${bashbrewArch}\$" "$dir/Dockerfile" +} + +parentArches() { + local dir="$1"; shift # "2", "3/slim", etc + + local parent="$(awk 'toupper($1) == "FROM" { print $2 }' "$dir/Dockerfile")" + local parentArches="${parentRepoToArches[$parent]:-}" + + local arches=() + for arch in $parentArches; do + if hasBashbrewArch "$dir" "$arch"; then + arches+=( "$arch" ) + fi + done + echo "${arches[*]:-}" +} diff --git a/2/Dockerfile b/2/Dockerfile index 54779d1..5f19570 100644 --- a/2/Dockerfile +++ b/2/Dockerfile @@ -14,16 +14,32 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ && rm -rf /var/lib/apt/lists/* ENV PYPY_VERSION 5.8.0 -ENV PYPY_SHA256SUM 6274292d0e954a2609b15978cde6efa30942ba20aa5d2acbbf1c70c0a54e9b1e # if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value ''" ENV PYTHON_PIP_VERSION 9.0.1 RUN set -ex; \ - wget -O pypy.tar.bz2 "https://bitbucket.org/pypy/pypy/downloads/pypy2-v${PYPY_VERSION}-linux64.tar.bz2"; \ - echo "$PYPY_SHA256SUM *pypy.tar.bz2" | sha256sum -c; \ + \ +# this "case" statement is generated via "update.sh" + dpkgArch="$(dpkg --print-architecture)"; \ + case "${dpkgArch##*-}" in \ +# amd64 + amd64) pypyArch='linux64'; sha256='6274292d0e954a2609b15978cde6efa30942ba20aa5d2acbbf1c70c0a54e9b1e' ;; \ +# arm32v5 + armel) pypyArch='linux-armel'; sha256='28b7fd0cc7418ffc66c71520728e87941be40ebf4b82675c57e25598a2a702b0' ;; \ +# arm32v7 + armhf) pypyArch='linux-armhf-raring'; sha256='ddceca9c5c9a456d4bf1beab177660adffbbdf255a922244e1cc05f20318be46' ;; \ +# i386 + i386) pypyArch='linux32'; sha256='a0b125a5781f7e5ddfc3baca46503b14f4ee6a0e234e8d72bfcf3afdf4120bef' ;; \ + *) echo >&2 "error: current architecture ($dpkgArch) does not have a corresponding PyPy $PYPY_VERSION binary release"; exit 1 ;; \ + esac; \ + \ + wget -O pypy.tar.bz2 "https://bitbucket.org/pypy/pypy/downloads/pypy2-v${PYPY_VERSION}-${pypyArch}.tar.bz2"; \ + echo "$sha256 *pypy.tar.bz2" | sha256sum -c; \ tar -xjC /usr/local --strip-components=1 -f pypy.tar.bz2; \ - rm pypy.tar.bz2 + rm pypy.tar.bz2; \ + \ + pypy --version RUN set -ex; \ \ diff --git a/2/slim/Dockerfile b/2/slim/Dockerfile index b1cd204..0167936 100644 --- a/2/slim/Dockerfile +++ b/2/slim/Dockerfile @@ -16,24 +16,39 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ && rm -rf /var/lib/apt/lists/* ENV PYPY_VERSION 5.8.0 -ENV PYPY_SHA256SUM 6274292d0e954a2609b15978cde6efa30942ba20aa5d2acbbf1c70c0a54e9b1e # if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value ''" ENV PYTHON_PIP_VERSION 9.0.1 RUN set -ex; \ \ +# this "case" statement is generated via "update.sh" + dpkgArch="$(dpkg --print-architecture)"; \ + case "${dpkgArch##*-}" in \ +# amd64 + amd64) pypyArch='linux64'; sha256='6274292d0e954a2609b15978cde6efa30942ba20aa5d2acbbf1c70c0a54e9b1e' ;; \ +# arm32v5 + armel) pypyArch='linux-armel'; sha256='28b7fd0cc7418ffc66c71520728e87941be40ebf4b82675c57e25598a2a702b0' ;; \ +# arm32v7 + armhf) pypyArch='linux-armhf-raring'; sha256='ddceca9c5c9a456d4bf1beab177660adffbbdf255a922244e1cc05f20318be46' ;; \ +# i386 + i386) pypyArch='linux32'; sha256='a0b125a5781f7e5ddfc3baca46503b14f4ee6a0e234e8d72bfcf3afdf4120bef' ;; \ + *) echo >&2 "error: current architecture ($dpkgArch) does not have a corresponding PyPy $PYPY_VERSION binary release"; exit 1 ;; \ + esac; \ + \ fetchDeps=' \ bzip2 \ wget \ '; \ apt-get update && apt-get install -y $fetchDeps --no-install-recommends && rm -rf /var/lib/apt/lists/*; \ \ - wget -O pypy.tar.bz2 "https://bitbucket.org/pypy/pypy/downloads/pypy2-v${PYPY_VERSION}-linux64.tar.bz2"; \ - echo "$PYPY_SHA256SUM *pypy.tar.bz2" | sha256sum -c; \ + wget -O pypy.tar.bz2 "https://bitbucket.org/pypy/pypy/downloads/pypy2-v${PYPY_VERSION}-${pypyArch}.tar.bz2"; \ + echo "$sha256 *pypy.tar.bz2" | sha256sum -c; \ tar -xjC /usr/local --strip-components=1 -f pypy.tar.bz2; \ rm pypy.tar.bz2; \ \ + pypy --version; \ + \ wget -O get-pip.py 'https://bootstrap.pypa.io/get-pip.py'; \ \ pypy get-pip.py \ diff --git a/3/Dockerfile b/3/Dockerfile index 1e500d3..875070e 100644 --- a/3/Dockerfile +++ b/3/Dockerfile @@ -14,16 +14,26 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ && rm -rf /var/lib/apt/lists/* ENV PYPY_VERSION 5.8.0 -ENV PYPY_SHA256SUM 57d871a7f1135719c138cee4e3533c3275d682a76a40ff668e95150c65923035 # if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value ''" ENV PYTHON_PIP_VERSION 9.0.1 RUN set -ex; \ - wget -O pypy.tar.bz2 "https://bitbucket.org/pypy/pypy/downloads/pypy3-v${PYPY_VERSION}-linux64.tar.bz2"; \ - echo "$PYPY_SHA256SUM *pypy.tar.bz2" | sha256sum -c; \ + \ +# this "case" statement is generated via "update.sh" + dpkgArch="$(dpkg --print-architecture)"; \ + case "${dpkgArch##*-}" in \ +# amd64 + amd64) pypyArch='linux64'; sha256='57d871a7f1135719c138cee4e3533c3275d682a76a40ff668e95150c65923035' ;; \ + *) echo >&2 "error: current architecture ($dpkgArch) does not have a corresponding PyPy $PYPY_VERSION binary release"; exit 1 ;; \ + esac; \ + \ + wget -O pypy.tar.bz2 "https://bitbucket.org/pypy/pypy/downloads/pypy3-v${PYPY_VERSION}-${pypyArch}.tar.bz2"; \ + echo "$sha256 *pypy.tar.bz2" | sha256sum -c; \ tar -xjC /usr/local --strip-components=1 -f pypy.tar.bz2; \ - rm pypy.tar.bz2 + rm pypy.tar.bz2; \ + \ + pypy3 --version RUN set -ex; \ \ diff --git a/3/slim/Dockerfile b/3/slim/Dockerfile index aa61671..503fefa 100644 --- a/3/slim/Dockerfile +++ b/3/slim/Dockerfile @@ -16,24 +16,33 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ && rm -rf /var/lib/apt/lists/* ENV PYPY_VERSION 5.8.0 -ENV PYPY_SHA256SUM 57d871a7f1135719c138cee4e3533c3275d682a76a40ff668e95150c65923035 # if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value ''" ENV PYTHON_PIP_VERSION 9.0.1 RUN set -ex; \ \ +# this "case" statement is generated via "update.sh" + dpkgArch="$(dpkg --print-architecture)"; \ + case "${dpkgArch##*-}" in \ +# amd64 + amd64) pypyArch='linux64'; sha256='57d871a7f1135719c138cee4e3533c3275d682a76a40ff668e95150c65923035' ;; \ + *) echo >&2 "error: current architecture ($dpkgArch) does not have a corresponding PyPy $PYPY_VERSION binary release"; exit 1 ;; \ + esac; \ + \ fetchDeps=' \ bzip2 \ wget \ '; \ apt-get update && apt-get install -y $fetchDeps --no-install-recommends && rm -rf /var/lib/apt/lists/*; \ \ - wget -O pypy.tar.bz2 "https://bitbucket.org/pypy/pypy/downloads/pypy3-v${PYPY_VERSION}-linux64.tar.bz2"; \ - echo "$PYPY_SHA256SUM *pypy.tar.bz2" | sha256sum -c; \ + wget -O pypy.tar.bz2 "https://bitbucket.org/pypy/pypy/downloads/pypy3-v${PYPY_VERSION}-${pypyArch}.tar.bz2"; \ + echo "$sha256 *pypy.tar.bz2" | sha256sum -c; \ tar -xjC /usr/local --strip-components=1 -f pypy.tar.bz2; \ rm pypy.tar.bz2; \ \ + pypy3 --version; \ + \ wget -O get-pip.py 'https://bootstrap.pypa.io/get-pip.py'; \ \ pypy3 get-pip.py \ diff --git a/Dockerfile-slim.template b/Dockerfile-slim.template new file mode 100644 index 0000000..e806b38 --- /dev/null +++ b/Dockerfile-slim.template @@ -0,0 +1,54 @@ +FROM debian:jessie-slim + +# ensure local pypy is preferred over distribution python +ENV PATH /usr/local/bin:$PATH + +# http://bugs.python.org/issue19846 +# > At the moment, setting "LANG=C" on a Linux system *fundamentally breaks Python 3*, and that's not OK. +ENV LANG C.UTF-8 + +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates \ + libexpat1 \ + libffi6 \ + libgdbm3 \ + libsqlite3-0 \ + && rm -rf /var/lib/apt/lists/* + +ENV PYPY_VERSION %%PYPY_VERSION%% + +# if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value ''" +ENV PYTHON_PIP_VERSION 9.0.1 + +RUN set -ex; \ + \ +# this "case" statement is generated via "update.sh" + %%ARCH-CASE%%; \ + \ + fetchDeps=' \ + bzip2 \ + wget \ + '; \ + apt-get update && apt-get install -y $fetchDeps --no-install-recommends && rm -rf /var/lib/apt/lists/*; \ + \ + wget -O pypy.tar.bz2 "https://bitbucket.org/pypy/pypy/downloads/%%TAR%%-v${PYPY_VERSION}-${pypyArch}.tar.bz2"; \ + echo "$sha256 *pypy.tar.bz2" | sha256sum -c; \ + tar -xjC /usr/local --strip-components=1 -f pypy.tar.bz2; \ + rm pypy.tar.bz2; \ + \ + %%CMD%% --version; \ + \ + wget -O get-pip.py 'https://bootstrap.pypa.io/get-pip.py'; \ + \ + %%CMD%% get-pip.py \ + --disable-pip-version-check \ + --no-cache-dir \ + "pip==$PYTHON_PIP_VERSION" \ + ; \ + pip --version; \ + \ + rm -f get-pip.py; \ + \ + apt-get purge -y --auto-remove $fetchDeps + +CMD ["%%CMD%%"] diff --git a/Dockerfile.template b/Dockerfile.template new file mode 100644 index 0000000..2663f0b --- /dev/null +++ b/Dockerfile.template @@ -0,0 +1,46 @@ +FROM buildpack-deps:jessie + +# ensure local pypy is preferred over distribution python +ENV PATH /usr/local/bin:$PATH + +# http://bugs.python.org/issue19846 +# > At the moment, setting "LANG=C" on a Linux system *fundamentally breaks Python 3*, and that's not OK. +ENV LANG C.UTF-8 + +# runtime dependencies +RUN apt-get update && apt-get install -y --no-install-recommends \ + tcl \ + tk \ + && rm -rf /var/lib/apt/lists/* + +ENV PYPY_VERSION %%PYPY_VERSION%% + +# if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value ''" +ENV PYTHON_PIP_VERSION 9.0.1 + +RUN set -ex; \ + \ +# this "case" statement is generated via "update.sh" + %%ARCH-CASE%%; \ + \ + wget -O pypy.tar.bz2 "https://bitbucket.org/pypy/pypy/downloads/%%TAR%%-v${PYPY_VERSION}-${pypyArch}.tar.bz2"; \ + echo "$sha256 *pypy.tar.bz2" | sha256sum -c; \ + tar -xjC /usr/local --strip-components=1 -f pypy.tar.bz2; \ + rm pypy.tar.bz2; \ + \ + %%CMD%% --version + +RUN set -ex; \ + \ + wget -O get-pip.py 'https://bootstrap.pypa.io/get-pip.py'; \ + \ + %%CMD%% get-pip.py \ + --disable-pip-version-check \ + --no-cache-dir \ + "pip==$PYTHON_PIP_VERSION" \ + ; \ + pip --version; \ + \ + rm -f get-pip.py + +CMD ["%%CMD%%"] diff --git a/generate-stackbrew-library.sh b/generate-stackbrew-library.sh index cf54e7d..cbfd047 100755 --- a/generate-stackbrew-library.sh +++ b/generate-stackbrew-library.sh @@ -1,5 +1,5 @@ -#!/bin/bash -set -eu +#!/usr/bin/env bash +set -Eeuo pipefail declare -A aliases=( [3]='latest' @@ -8,6 +8,8 @@ declare -A aliases=( self="$(basename "$BASH_SOURCE")" cd "$(dirname "$(readlink -f "$BASH_SOURCE")")" +source '.architectures-lib' + versions=( */ ) versions=( "${versions[@]%/}" ) @@ -61,26 +63,29 @@ for version in "${versions[@]}"; do done versionAliases+=( $version ${aliases[$version]:-} ) - echo - cat <<-EOE - Tags: $(join ', ' "${versionAliases[@]}") - GitCommit: $commit - Directory: $version - EOE + for variant in '' slim onbuild; do + dir="$version${variant:+/$variant}" + [ -f "$dir/Dockerfile" ] || continue - for variant in slim onbuild; do - [ -f "$version/$variant/Dockerfile" ] || continue + commit="$(dirCommit "$dir")" - commit="$(dirCommit "$version/$variant")" + variantAliases=( "${versionAliases[@]}" ) + if [ -n "$variant" ]; then + variantAliases=( "${variantAliases[@]/%/-$variant}" ) + variantAliases=( "${variantAliases[@]//latest-/}" ) + fi - variantAliases=( "${versionAliases[@]/%/-$variant}" ) - variantAliases=( "${variantAliases[@]//latest-/}" ) + case "$variant" in + onbuild) variantArches="$(parentArches "$(dirname "$dir")" )" ;; + *) variantArches="$(parentArches "$dir")" ;; + esac echo cat <<-EOE Tags: $(join ', ' "${variantAliases[@]}") + Architectures: $(join ', ' $variantArches) GitCommit: $commit - Directory: $version/$variant + Directory: $dir EOE done done diff --git a/release-architectures b/release-architectures new file mode 100644 index 0000000..0778766 --- /dev/null +++ b/release-architectures @@ -0,0 +1,14 @@ +# see https://bitbucket.org/pypy/pypy/downloads/ +# and https://pypy.org/download.html#checksums + +# bashbrew-arch dpkg-arch pypy-release-arch +amd64 amd64 linux64 +arm32v5 armel linux-armel +arm32v7 armhf linux-armhf-raring +i386 i386 linux32 + +# pypy: error while loading shared libraries: libssl.so.10: cannot open shared object file: No such file or directory +#ppc64le ppc64el ppc64le + +# pypy: error while loading shared libraries: libssl.so.10: cannot open shared object file: No such file or directory +#s390x s390x s390x diff --git a/update.sh b/update.sh index 61d8d93..fc7493a 100755 --- a/update.sh +++ b/update.sh @@ -3,6 +3,8 @@ set -Eeuo pipefail cd "$(dirname "$(readlink -f "$BASH_SOURCE")")" +source '.architectures-lib' + versions=( "$@" ) if [ ${#versions[@]} -eq 0 ]; then versions=( */ ) @@ -11,10 +13,32 @@ versions=( "${versions[@]%/}" ) pipVersion="$(curl -fsSL 'https://pypi.org/pypi/pip/json' | jq -r .info.version)" +sha256s="$(curl -fsSL 'http://pypy.org/download.html')" +scrapeSha256() { + local pypy="$1"; shift + local fullVersion="$1"; shift + local arch="$1"; shift + + #

pypy2.7-5.4.0 sha256:

+ #
+	# ...
+	# bdfea513d59dcd580970cb6f79f3a250d00191fd46b68133d5327e924ca845f8  pypy2-v5.4.0-linux64.tar.bz2
+	# ...
+	# 
+ echo "$sha256s" \ + | grep -m1 -E '[a-f0-9]{64} '"$pypy-v$tryVersion"'-'"$arch"'.tar.bz2' \ + | cut -d' ' -f1 +} + +# see http://stackoverflow.com/a/2705678/433558 +sed_escape_rhs() { + echo "$@" | sed -e 's/[\/&]/\\&/g' | sed -e ':a;N;$!ba;s/\n/\\n/g' +} + for version in "${versions[@]}"; do case "$version" in - 3) pypy='pypy3' ;; - 2) pypy='pypy2' ;; + 3) pypy='pypy3'; cmd='pypy3' ;; + 2) pypy='pypy2'; cmd='pypy' ;; *) echo >&2 "error: unknown pypy variant $version"; exit 1 ;; esac @@ -23,8 +47,7 @@ for version in "${versions[@]}"; do IFS=$'\n' tryVersions=( $( curl -fsSL 'https://bitbucket.org/pypy/pypy/downloads/' \ - | grep -E "$pypy"'-v([0-9.]+(-alpha[0-9]*)?)-linux64.tar.bz2' \ - | sed -r 's/^.*'"$pypy"'-v([0-9.]+(-alpha[0-9]*)?)-linux64.tar.bz2.*$/\1/' \ + | sed -rn 's/^.*'"$pypy"'-v([0-9.]+(-alpha[0-9]*)?)-linux64.tar.bz2.*$/\1/gp' \ | sort -rV ) ) unset IFS @@ -32,18 +55,8 @@ for version in "${versions[@]}"; do fullVersion= sha256sum= for tryVersion in "${tryVersions[@]}"; do - #

pypy2.7-5.4.0 sha256:

- #
-		# ...
-		# bdfea513d59dcd580970cb6f79f3a250d00191fd46b68133d5327e924ca845f8  pypy2-v5.4.0-linux64.tar.bz2
-		# ...
-		# 
if \ - sha256sum="$( - curl -fsSL 'http://pypy.org/download.html' \ - | grep -m1 -E '[a-f0-9]{64} '"$pypy-v$tryVersion"'-linux64.tar.bz2' \ - | cut -d' ' -f1 - )" \ + sha256sum="$(scrapeSha256 "$pypy" "$tryVersion" 'linux64')" \ && [ -n "$sha256sum" ] \ ; then fullVersion="$tryVersion" @@ -55,13 +68,32 @@ for version in "${versions[@]}"; do exit 1 fi - ( - set -x - sed -ri ' - s/^(ENV PYPY_VERSION) .*/\1 '"$fullVersion"'/; - s/^(ENV PYPY_SHA256SUM) .*/\1 '"$sha256sum"'/; - s/^(ENV PYTHON_PIP_VERSION) .*/\1 '"$pipVersion"'/; - ' "$version"{,/slim}/Dockerfile - sed -ri 's/^(FROM pypy):.*/\1:'"$version"'/' "$version/onbuild/Dockerfile" - ) + echo "$version: $fullVersion" + + linuxArchCase='dpkgArch="$(dpkg --print-architecture)"; '$'\\\n' + linuxArchCase+=$'\t''case "${dpkgArch##*-}" in '$'\\\n' + for dpkgArch in $(dpkgArches); do + pypyArch="$(dpkgToPyPyArch "$dpkgArch")" + sha256="$(scrapeSha256 "$pypy" "$fullVersion" "$pypyArch")" || : + if [ -z "$sha256" ]; then + echo >&2 "warning: cannot find sha256 for $pypy-$fullVersion on arch $pypyArch ($dpkgArch); skipping it" + continue + fi + bashbrewArch="$(dpkgToBashbrewArch "$dpkgArch")" + linuxArchCase+="# $bashbrewArch"$'\n' + linuxArchCase+=$'\t\t'"$dpkgArch) pypyArch='$pypyArch'; sha256='$sha256' ;; "$'\\\n' + done + linuxArchCase+=$'\t\t''*) echo >&2 "error: current architecture ($dpkgArch) does not have a corresponding PyPy $PYPY_VERSION binary release"; exit 1 ;; '$'\\\n' + linuxArchCase+=$'\t''esac' + + for variant in '' slim; do + sed -r \ + -e 's!%%PYPY_VERSION%%!'"$fullVersion"'!g' \ + -e 's!%%TAR%%!'"$pypy"'!g' \ + -e 's!%%CMD%%!'"$cmd"'!g' \ + -e 's!%%ARCH-CASE%%!'"$(sed_escape_rhs "$linuxArchCase")"'!g' \ + "Dockerfile${variant:+-$variant}.template" > "$version/$variant/Dockerfile" + done + + sed -ri 's/^(FROM pypy):.*/\1:'"$version"'/' "$version/onbuild/Dockerfile" done