diff --git a/.github/workflows/.bashbrew/action.yml b/.github/workflows/.bashbrew/action.yml index 9548d07f2feac..aba6e2c86e8e7 100644 --- a/.github/workflows/.bashbrew/action.yml +++ b/.github/workflows/.bashbrew/action.yml @@ -10,8 +10,8 @@ runs: # these two version numbers are intentionally as close together as I could possibly get them because no matter what I tried, GitHub will not allow me to DRY them (can't have any useful variables in `uses:` and can't even have YAML references to steal it in `env:` or something) - shell: 'bash -Eeuo pipefail -x {0}' - run: echo BASHBREW_VERSION=v0.1.7 >> "$GITHUB_ENV" - - uses: docker-library/bashbrew@v0.1.7 + run: echo BASHBREW_VERSION=v0.1.8 >> "$GITHUB_ENV" + - uses: docker-library/bashbrew@v0.1.8 if: inputs.build == 'host' - run: docker build --pull --tag oisupport/bashbrew:base "https://github.com/docker-library/bashbrew.git#$BASHBREW_VERSION" diff --git a/diff-pr.sh b/diff-pr.sh index 074c825655af1..c03e5c53527dc 100755 --- a/diff-pr.sh +++ b/diff-pr.sh @@ -91,7 +91,6 @@ if [ "$#" -eq 0 ]; then set -- $images fi -export BASHBREW_CACHE="${BASHBREW_CACHE:-${XDG_CACHE_HOME:-$HOME/.cache}/bashbrew}" export BASHBREW_LIBRARY="$PWD/oi/library" : "${BASHBREW_ARCH:=amd64}" # TODO something smarter with arches @@ -103,9 +102,8 @@ template=' {{- "\n" -}} {{- range $.Entries -}} {{- $arch := .HasArchitecture arch | ternary arch (.Architectures | first) -}} - {{- $froms := $.ArchDockerFroms $arch . -}} {{- $outDir := join "_" $.RepoName (.Tags | last) -}} - git -C "$BASHBREW_CACHE/git" archive --format=tar + git -C "{{ gitCache }}" archive --format=tar {{- " " -}} {{- "--prefix=" -}} {{- $outDir -}} @@ -116,12 +114,28 @@ template=' {{- $dir := .ArchDirectory $arch -}} {{- (eq $dir ".") | ternary "" $dir -}} {{- "\n" -}} - mkdir -p "$tempDir/{{- $outDir -}}" && echo "{{- .ArchFile $arch -}}" > "$tempDir/{{- $outDir -}}/.bashbrew-dockerfile-name" + mkdir -p "$tempDir/{{- $outDir -}}" && echo "{{- .ArchBuilder $arch -}}" > "$tempDir/{{- $outDir -}}/.bashbrew-builder" && echo "{{- .ArchFile $arch -}}" > "$tempDir/{{- $outDir -}}/.bashbrew-file" {{- "\n" -}} {{- end -}} tar -cC "$tempDir" . && rm -rf "$tempDir" ' +_tar-t() { + tar -t "$@" \ + | grep -vE "$uninterestingTarballGrep" \ + | sed -e 's!^[.]/!!' \ + -r \ + -e 's!([/.-]|^)((lib)?(c?python|py)-?)[0-9]+([.][0-9]+)?([/.-]|$)!\1\2XXX\6!g' \ + | sort +} + +_jq() { + if [ "$#" -eq 0 ]; then + set -- '.' + fi + jq --tab -S "$@" +} + copy-tar() { local src="$1"; shift local dst="$1"; shift @@ -132,19 +146,73 @@ copy-tar() { return fi - local d dockerfiles=() - for d in "$src"/*/.bashbrew-dockerfile-name; do + local d indexes=() dockerfiles=() + for d in "$src"/*/.bashbrew-file; do [ -f "$d" ] || continue local bf; bf="$(< "$d")" local dDir; dDir="$(dirname "$d")" - dockerfiles+=( "$dDir/$bf" ) - if [ "$bf" = 'Dockerfile' ]; then - # if "Dockerfile.builder" exists, let's check that too (busybox, hello-world) - if [ -f "$dDir/$bf.builder" ]; then - dockerfiles+=( "$dDir/$bf.builder" ) + local builder; builder="$(< "$dDir/.bashbrew-builder")" + if [ "$builder" = 'oci-import' ]; then + indexes+=( "$dDir/$bf" ) + else + dockerfiles+=( "$dDir/$bf" ) + if [ "$bf" = 'Dockerfile' ]; then + # if "Dockerfile.builder" exists, let's check that too (busybox, hello-world) + if [ -f "$dDir/$bf.builder" ]; then + dockerfiles+=( "$dDir/$bf.builder" ) + fi fi fi - rm "$d" # remove the ".bashbrew-dockerfile-name" file we created + rm "$d" "$dDir/.bashbrew-builder" # remove the ".bashbrew-*" files we created + done + + # now that we're done with our globbing needs, let's disable globbing so it doesn't give us wrong answers + local - + set -o noglob + + for i in "${indexes[@]}"; do + local iName; iName="$(basename "$i")" + local iDir; iDir="$(dirname "$i")" + local iDirName; iDirName="$(basename "$iDir")" + local iDst="$dst/$iDirName" + + mkdir -p "$iDst" + + _jq . "$i" > "$iDst/$iName" + + local digest + digest="$(jq -r --arg name "$iName" ' + if $name == "index.json" then + .manifests[0].digest + else + .digest + end + ' "$i")" + + local blob="blobs/${digest//://}" + local blobDir; blobDir="$(dirname "$blob")" + local manifest="$iDir/$blob" + mkdir -p "$iDst/$blobDir" + _jq . "$manifest" > "$iDst/$blob" + + local configDigest; configDigest="$(jq -r '.config.digest' "$manifest")" + local blob="blobs/${configDigest//://}" + local blobDir; blobDir="$(dirname "$blob")" + local config="$iDir/$blob" + mkdir -p "$iDst/$blobDir" + _jq . "$config" > "$iDst/$blob" + + local layers + layers="$(jq -r '[ .layers[].digest | @sh ] | join(" ")' "$manifest")" + eval "layers=( $layers )" + local layerDigest + for layerDigest in "${layers[@]}"; do + local blob="blobs/${layerDigest//://}" + local blobDir; blobDir="$(dirname "$blob")" + local layer="$iDir/$blob" + mkdir -p "$iDst/$blobDir" + _tar-t -f "$layer" > "$iDst/$blob 'tar -t'" + done done for d in "${dockerfiles[@]}"; do @@ -238,13 +306,7 @@ copy-tar() { case "$g" in *.tar.* | *.tgz) if [ -s "$dstG" ]; then - tar -tf "$dstG" \ - | grep -vE "$uninterestingTarballGrep" \ - | sed -e 's!^[.]/!!' \ - -r \ - -e 's!([/.-]|^)((lib)?(c?python|py)-?)[0-9]+([.][0-9]+)?([/.-]|$)!\1\2XXX\6!g' \ - | sort \ - > "$dstG 'tar -t'" + _tar-t -f "$dstG" > "$dstG 'tar -t'" fi ;; esac @@ -269,6 +331,8 @@ _metadata-files() { if [ "$#" -gt 0 ]; then bashbrew list "$@" 2>>temp/_bashbrew.err | sort -uV > temp/_bashbrew-list || : + bashbrew cat --format '{{ range .Entries }}{{ range .Architectures }}{{ . }}{{ "\n" }}{{ end }}{{ end }}' "$@" 2>>temp/_bashbrew.err | sort -u > temp/_bashbrew-arches || : + "$diffDir/_bashbrew-cat-sorted.sh" "$@" 2>>temp/_bashbrew.err > temp/_bashbrew-cat || : bashbrew list --uniq "$@" \ @@ -277,6 +341,7 @@ _metadata-files() { | xargs -r bashbrew cat --format "$templateLastTags" 2>>temp/_bashbrew.err \ > temp/_bashbrew-list-build-order || : + bashbrew fetch --arch-filter "$@" script="$(bashbrew cat --format "$template" "$@")" mkdir tar ( eval "$script" | tar -xiC tar ) @@ -284,7 +349,7 @@ _metadata-files() { rm -rf tar fi - if [ -n "$externalPins" ] && command -v crane &> /dev/null && command -v jq &> /dev/null; then + if [ -n "$externalPins" ] && command -v crane &> /dev/null; then local file for file in $externalPins; do [ -e "oi/$file" ] || continue @@ -293,7 +358,7 @@ _metadata-files() { digest="$(< "oi/$file")" dir="temp/$file" mkdir -p "$dir" - bashbrew remote arches --json "$pin@$digest" | jq -S . > "$dir/bashbrew.json" + bashbrew remote arches --json "$pin@$digest" | _jq > "$dir/bashbrew.json" local manifests manifest manifests="$(jq -r ' [ ( @@ -308,10 +373,10 @@ _metadata-files() { ' "$dir/bashbrew.json")" eval "manifests=( $manifests )" for manifest in "${manifests[@]}"; do - crane manifest "$pin@$manifest" | jq -S . > "$dir/manifest-${manifest//:/_}.json" + crane manifest "$pin@$manifest" | _jq > "$dir/manifest-${manifest//:/_}.json" local config config="$(jq -r '.config.digest' "$dir/manifest-${manifest//:/_}.json")" - crane blob "$pin@$config" | jq -S . > "$dir/manifest-${manifest//:/_}-config.json" + crane blob "$pin@$config" | _jq > "$dir/manifest-${manifest//:/_}-config.json" done done fi diff --git a/naughty-commits.sh b/naughty-commits.sh index c71c89cc4efeb..8943159bc2c1b 100755 --- a/naughty-commits.sh +++ b/naughty-commits.sh @@ -3,16 +3,11 @@ set -Eeuo pipefail fileSizeThresholdMB='2' -: "${BASHBREW_CACHE:=$HOME/.cache/bashbrew}" -export BASHBREW_CACHE BASHBREW_ARCH= - -if [ ! -d "$BASHBREW_CACHE/git" ]; then - # initialize the "bashbrew cache" - bashbrew --arch amd64 from --uniq --apply-constraints hello-world:linux > /dev/null -fi +export BASHBREW_ARCH= +gitCache="$(bashbrew cat --format '{{ gitCache }}' <(echo 'Maintainers: empty hack (@example)'))" _git() { - git -C "$BASHBREW_CACHE/git" "$@" + git -C "$gitCache" "$@" } if [ "$#" -eq 0 ]; then @@ -21,12 +16,11 @@ fi imgs="$(bashbrew list --repos "$@" | sort -u)" for img in $imgs; do + bashbrew fetch --all "$img" # force `git fetch` commits="$( bashbrew cat --format ' {{- range $e := .Entries -}} {{- range $a := .Architectures -}} - {{- /* force `git fetch` */ -}} - {{- $froms := $.ArchDockerFroms $a $e -}} { {{- json "GitRepo" }}:{{ json ($e.ArchGitRepo $a) -}}, {{- json "GitFetch" }}:{{ json ($e.ArchGitFetch $a) -}}, diff --git a/naughty-constraints.sh b/naughty-constraints.sh index c3d0b8b304f41..5bcc41a3e6ba3 100755 --- a/naughty-constraints.sh +++ b/naughty-constraints.sh @@ -1,13 +1,7 @@ #!/usr/bin/env bash set -Eeuo pipefail -: "${BASHBREW_CACHE:=$HOME/.cache/bashbrew}" -export BASHBREW_CACHE BASHBREW_ARCH= - -if [ ! -d "$BASHBREW_CACHE/git" ]; then - # initialize the "bashbrew cache" - bashbrew --arch amd64 from --uniq --apply-constraints hello-world:linux > /dev/null -fi +export BASHBREW_ARCH= if [ "$#" -eq 0 ]; then set -- '--all'