From a820f3420b714805456bc6383004b6cc53016dbb Mon Sep 17 00:00:00 2001 From: Peter Baumgartner Date: Mon, 6 May 2024 08:31:05 -0700 Subject: [PATCH] Use CNB builder for heroku-22 stack (#6) * Use CNB builder for heroku-22 stack refs #5 * Switch default builder to heroku/builder:22 * Add new buildpack * Test app.json too * Procfile buildpack now uses command and args * Test against legacy Heroku builder * Adjust tests for bash wrapper New procfile buildpacks add it --- .github/workflows/build.yml | 122 +++++++++++++++++++++++++++++++++- builder/build/appjson.go | 28 +++++--- builder/build/appjson_test.go | 4 +- builder/build/build.go | 5 ++ builder/build/metadatatoml.go | 3 +- 5 files changed, 148 insertions(+), 14 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8b56bd9..f97e264 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -93,12 +93,132 @@ jobs: - uses: actions/setup-python@v4 with: python-version: "3.11" + - name: Verify apppack.toml + working-directory: ./apppack-demo-python + run: | + set -ex + cat apppack.toml + test "$(python -c 'import tomllib; print(tomllib.load(open("apppack.toml", "rb"))["services"]["web"]["command"])')" = "bash -c 'gunicorn --access-logfile - --bind 0.0.0.0:\$PORT --forwarded-allow-ips '\"'\"'*'\"'\"' app:app'" + + integration-appjson: + runs-on: ubuntu-latest + needs: [test, build-image] + permissions: + id-token: write + contents: read + packages: read + steps: + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Pull image + run: docker pull ${{ needs.build-image.outputs.image }} + - name: configure aws credentials + uses: aws-actions/configure-aws-credentials@v1-node16 + with: + role-to-assume: arn:aws:iam::891426818781:role/github-actions-integration-tests + aws-region: us-east-1 + - name: Checkout sample repo + run: git clone --branch app.json https://github.com/apppackio/apppack-demo-python.git + - name: Run integration tests + working-directory: ./apppack-demo-python + run: | + cat < .envfile + APPNAME=gh-integration + CODEBUILD_BUILD_ID=demo-python:${{ github.run_id }} + CODEBUILD_SOURCE_VERSION=${{ github.sha }} + DOCKERHUB_USERNAME=${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_ACCESS_TOKEN=${{ secrets.DOCKERHUB_ACCESS_TOKEN }} + DOCKER_REPO=891426818781.dkr.ecr.us-east-1.amazonaws.com/github-integration-test + ARTIFACT_BUCKET=integration-test-buildartifacts + AWS_REGION + AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY + AWS_SESSION_TOKEN + EOF + + docker run \ + --rm \ + --privileged \ + --env-file .envfile \ + --volume /var/run/docker.sock:/var/run/docker.sock \ + --volume "$(pwd):/app" \ + --workdir /app \ + ${{ needs.build-image.outputs.image }} \ + /bin/sh -c "set -x; git config --global --add safe.directory /app && apppack-builder prebuild; apppack-builder build; apppack-builder postbuild" + - uses: actions/setup-python@v4 + with: + python-version: "3.11" + - name: Verify apppack.toml + working-directory: ./apppack-demo-python + run: | + set -ex + cat apppack.toml + test "$(python -c 'import tomllib; print(tomllib.load(open("apppack.toml", "rb"))["services"]["web"]["command"])')" = "bash -c 'gunicorn --access-logfile - --bind 0.0.0.0:\$PORT --forwarded-allow-ips '\"'\"'*'\"'\"' app:app'" + + integration-heroku20: + runs-on: ubuntu-latest + needs: [test, build-image] + permissions: + id-token: write + contents: read + packages: read + steps: + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Pull image + run: docker pull ${{ needs.build-image.outputs.image }} + - name: configure aws credentials + uses: aws-actions/configure-aws-credentials@v1-node16 + with: + role-to-assume: arn:aws:iam::891426818781:role/github-actions-integration-tests + aws-region: us-east-1 + - name: Checkout sample repo + run: git clone --branch buildpacks-20 https://github.com/apppackio/apppack-demo-python.git + - name: Run integration tests + working-directory: ./apppack-demo-python + run: | + cat < .envfile + APPNAME=gh-integration + CODEBUILD_BUILD_ID=demo-python:${{ github.run_id }} + CODEBUILD_SOURCE_VERSION=${{ github.sha }} + DOCKERHUB_USERNAME=${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_ACCESS_TOKEN=${{ secrets.DOCKERHUB_ACCESS_TOKEN }} + DOCKER_REPO=891426818781.dkr.ecr.us-east-1.amazonaws.com/github-integration-test + ARTIFACT_BUCKET=integration-test-buildartifacts + ALLOW_EOL_SHIMMED_BUILDER=1 + AWS_REGION + AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY + AWS_SESSION_TOKEN + EOF + + docker run \ + --rm \ + --privileged \ + --env-file .envfile \ + --volume /var/run/docker.sock:/var/run/docker.sock \ + --volume "$(pwd):/app" \ + --workdir /app \ + ${{ needs.build-image.outputs.image }} \ + /bin/sh -c "set -x; git config --global --add safe.directory /app && apppack-builder prebuild; apppack-builder build; apppack-builder postbuild" + - uses: actions/setup-python@v4 + with: + python-version: "3.11" - name: Verify apppack.toml working-directory: ./apppack-demo-python run: | set -ex cat apppack.toml test "$(python -c 'import tomllib; print(tomllib.load(open("apppack.toml", "rb"))["services"]["web"]["command"])')" = 'gunicorn --access-logfile - --bind 0.0.0.0:$PORT --forwarded-allow-ips '"'"'*'"' app:app" + integration-docker: runs-on: ubuntu-latest needs: [test, build-image] @@ -161,7 +281,7 @@ jobs: runs-on: ubuntu-latest # Only run this job if the current commit is tagged with a version if: startswith(github.ref, 'refs/tags/v') - needs: [test, build-image, integration, integration-docker] + needs: [test, build-image, integration, integration-docker, integration-appjson, integration-heroku20] permissions: id-token: write contents: read diff --git a/builder/build/appjson.go b/builder/build/appjson.go index 212a49b..3c32a0b 100644 --- a/builder/build/appjson.go +++ b/builder/build/appjson.go @@ -27,24 +27,24 @@ type AppJSON struct { ctx context.Context } -const DefaultStack = "heroku-20" +const DefaultStack = "heroku-22" // buildpacks included in builder var IncludedBuildpacks = map[string][]string{ "heroku-20": { // $ pack builder inspect heroku/buildpacks:20 -o json | jq '.remote_info.buildpacks[].id' + "heroku/builder-eol-warning", "heroku/go", "heroku/gradle", "heroku/java", - "heroku/java-function", "heroku/jvm", - "heroku/jvm-function-invoker", "heroku/maven", "heroku/nodejs", + "heroku/nodejs-corepack", "heroku/nodejs-engine", - "heroku/nodejs-function", - "heroku/nodejs-function-invoker", - "heroku/nodejs-npm", + "heroku/nodejs-npm-engine", + "heroku/nodejs-npm-install", + "heroku/nodejs-pnpm-install", "heroku/nodejs-yarn", "heroku/php", "heroku/procfile", @@ -53,16 +53,25 @@ var IncludedBuildpacks = map[string][]string{ "heroku/scala", }, "heroku-22": { - // $ pack builder inspect heroku/builder-classic:22 -o json | jq '.remote_info.buildpacks[].id' - "heroku/clojure", + // $ pack builder inspect heroku/builder:22 -o json | jq '.remote_info.buildpacks[].id' "heroku/go", "heroku/gradle", "heroku/java", + "heroku/jvm", + "heroku/maven", "heroku/nodejs", + "heroku/nodejs-corepack", + "heroku/nodejs-engine", + "heroku/nodejs-npm-engine", + "heroku/nodejs-npm-install", + "heroku/nodejs-pnpm-engine", + "heroku/nodejs-pnpm-install", + "heroku/nodejs-yarn", "heroku/php", "heroku/procfile", "heroku/python", "heroku/ruby", + "heroku/sbt", "heroku/scala", }, } @@ -131,9 +140,8 @@ func (a *AppJSON) GetBuilders() []string { if a.Stack == "heroku-20" { return []string{"heroku/buildpacks:20", "heroku/heroku:20-cnb"} } - // TODO: use heroku/builder:22 can be used when all buildpacks are CNB if a.Stack == "heroku-22" { - return []string{"heroku/builder-classic:22", "heroku/heroku:22-cnb"} + return []string{"heroku/builder:22", "heroku/heroku:22-cnb"} } return []string{a.Stack} } diff --git a/builder/build/appjson_test.go b/builder/build/appjson_test.go index 7c8de36..416c6f6 100644 --- a/builder/build/appjson_test.go +++ b/builder/build/appjson_test.go @@ -76,7 +76,7 @@ func TestAppJsonBuilders(t *testing.T) { Stack: "heroku-22", ctx: testContext, } - expected := []string{"heroku/builder-classic:22", "heroku/heroku:22-cnb"} + expected := []string{"heroku/builder:22", "heroku/heroku:22-cnb"} if !stringSliceEqual(a.GetBuilders(), expected) { t.Errorf("expected %s, got %s", expected, a.GetBuilders()) } @@ -159,7 +159,7 @@ func TestAppJsonToApppackToml(t *testing.T) { Build: AppPackTomlBuild{ System: "buildpack", Buildpacks: []string{"urn:cnb:builder:heroku/nodejs", "urn:cnb:builder:heroku/python"}, - Builder: "heroku/builder-classic:22", + Builder: "heroku/builder:22", }, Test: AppPackTomlTest{ Command: "echo test", diff --git a/builder/build/build.go b/builder/build/build.go index 61f7621..ccc16f9 100644 --- a/builder/build/build.go +++ b/builder/build/build.go @@ -32,6 +32,11 @@ func (b *Build) LoadBuildEnv() (map[string]string, error) { env := map[string]string{ "CI": "true", } + // pass ALLOW_EOL_SHIMMED_BUILDER to pack if it is in the environment + // this facilitates testing of the eol shimmed builder + if val, ok := os.LookupEnv("ALLOW_EOL_SHIMMED_BUILDER"); ok { + env["ALLOW_EOL_SHIMMED_BUILDER"] = val + } params, err := b.aws.GetParametersByPath(paths[0]) stripParamPrefix(params, paths[0], &env) if err != nil { diff --git a/builder/build/metadatatoml.go b/builder/build/metadatatoml.go index b2e6d4a..b01f4a8 100644 --- a/builder/build/metadatatoml.go +++ b/builder/build/metadatatoml.go @@ -11,6 +11,7 @@ import ( type BuildpackMetadataTomlProcess struct { Command []string `toml:"command"` + Args []string `toml:"args"` Type string `toml:"type"` BuildpackID string `toml:"buildpack_id"` } @@ -37,7 +38,7 @@ func (m *BuildpackMetadataToml) UpdateAppPackToml(a *AppPackToml) { if process.BuildpackID == "heroku/ruby" && (process.Type == "rake" || process.Type == "console") { continue } - a.Services[process.Type] = AppPackTomlService{Command: commandSliceToString(process.Command)} + a.Services[process.Type] = AppPackTomlService{Command: commandSliceToString(append(process.Command, process.Args...))} } }