Docker release CT-3

.bumpversion.cfg

@@ -35,3 +35,5 @@ first_value = 1
 [bumpversion:file:plugins/postgres/setup.py]
 
 [bumpversion:file:plugins/postgres/dbt/adapters/postgres/__version__.py]
+
+[bumpversion:file:docker/Dockerfile]

.github/actions/latest-wrangler/Dockerfile

@@ -0,0 +1,14 @@
+FROM python:3-slim AS builder
+ADD . /app
+WORKDIR /app
+
+# We are installing a dependency here directly into our app source dir
+RUN pip install --target=/app requests packaging
+
+# A distroless container image with Python and some basics like SSL certificates
+# https://github.com/GoogleContainerTools/distroless
+FROM gcr.io/distroless/python3-debian10
+COPY --from=builder /app /app
+WORKDIR /app
+ENV PYTHONPATH /app
+CMD ["/app/main.py"]

.github/actions/latest-wrangler/README.md

@@ -0,0 +1,50 @@
+# Github package 'latest' tag wrangler for containers
+## Usage
+
+Plug in the necessary inputs to determine if the container being built should be tagged 'latest; at the package level, for example `dbt-redshift:latest`.
+
+## Inputs
+| Input | Description |
+| - | - |
+| `package` | Name of the GH package to check against |
+| `new_version` | Semver of new container |
+| `gh_token` | GH token with package read scope|
+| `halt_on_missing` | Return non-zero exit code if requested package does not exist. (defaults to false)| 
+
+
+## Outputs
+| Output | Description |
+| - | - |
+| `latest` | Wether or not the new container should be tagged 'latest'|
+| `minor_latest` | Wether or not the new container should be tagged major.minor.latest |
+
+## Example workflow
+```yaml
+name: Ship it!
+on: 
+  workflow_dispatch:
+    inputs:
+      package:
+       description: The package to publish
+       required: true
+      version_number:
+       description: The version number
+       required: true
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v1
+    - name: Wrangle latest tag
+      id: is_latest
+      uses: ./.github/actions/latest-wrangler
+      with:
+        package: ${{ github.event.inputs.package }}
+        new_version: ${{ github.event.inputs.new_version }}
+        gh_token: ${{ secrets.GITHUB_TOKEN }}
+    - name: Print the results
+      run: |
+        echo "Is it latest?  Survey says: ${{ steps.is_latest.outputs.latest }} !"
+        echo "Is it minor.latest?  Survey says: ${{ steps.is_latest.outputs.minor_latest }} !"
+```

.github/actions/latest-wrangler/action.yml

@@ -0,0 +1,20 @@
+name: "Github package 'latest' tag wrangler for containers"
+description: "Determines wether or not a given dbt container should be given a bare 'latest' tag (I.E. dbt-core:latest)"
+inputs:
+  package_name:
+    description: "Package to check (I.E. dbt-core, dbt-redshift, etc)"
+    required: true
+  new_version:
+    description: "Semver of the container being built (I.E. 1.0.4)"
+    required: true
+  gh_token: 
+    description: "Auth token for github (must have view packages scope)"
+    required: true
+outputs:
+  latest:
+    description: "Wether or not built container should be tagged latest (bool)"
+  minor_latest:
+    description: "Wether or not built container should be tagged minor.latest (bool)"
+runs:
+  using: "docker"
+  image: "Dockerfile"

.github/actions/latest-wrangler/examples/example_workflow.yml

@@ -0,0 +1,26 @@
+name: Ship it!
+on: 
+  workflow_dispatch:
+    inputs:
+      package:
+       description: The package to publish
+       required: true
+      version_number:
+       description: The version number
+       required: true
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v1
+    - name: Wrangle latest tag
+      id: is_latest
+      uses: ./.github/actions/latest-wrangler
+      with:
+        package: ${{ github.event.inputs.package }}
+        new_version: ${{ github.event.inputs.new_version }}
+        gh_token: ${{ secrets.GITHUB_TOKEN }}
+    - name: Print the results
+      run: |
+        echo "Is it latest?  Survey says: ${{ steps.is_latest.outputs.latest }} !"

.github/actions/latest-wrangler/examples/example_workflow_dispatch.json

@@ -0,0 +1,6 @@
+{
+    "inputs": {
+      "version_number": "1.0.1",
+      "package": "dbt-redshift"
+    }
+}

.github/actions/latest-wrangler/main.py

@@ -0,0 +1,98 @@
+import os
+import sys
+import requests
+from distutils.util import strtobool
+from typing import Union
+from packaging.version import parse, Version
+
+if __name__ == "__main__":
+
+    # get inputs
+    package = os.environ["INPUT_PACKAGE"]
+    new_version = parse(os.environ["INPUT_NEW_VERSION"])
+    gh_token = os.environ["INPUT_GH_TOKEN"]
+    halt_on_missing = strtobool(os.environ.get("INPUT_HALT_ON_MISSING", "False"))
+
+    # get package metadata from github
+    package_request = requests.get(
+        f"https://api.github.com/orgs/dbt-labs/packages/container/{package}/versions",
+        auth=("", gh_token),
+    )
+    package_meta = package_request.json()
+
+    # Log info if we don't get a 200
+    if package_request.status_code != 200:
+        print(f"Call to GH API failed: {package_request.status_code} {package_meta['message']}")
+
+    # Make an early exit if there is no matching package in github
+    if package_request.status_code == 404:
+        if halt_on_missing:
+            sys.exit(1)
+        else:
+            # everything is the latest if the package doesn't exist
+            print(f"::set-output name=latest::{True}")
+            print(f"::set-output name=minor_latest::{True}")
+            sys.exit(0)
+
+    # TODO: verify package meta is "correct"
+    # https://github.com/dbt-labs/dbt-core/issues/4640 
+
+    # map versions and tags
+    version_tag_map = {
+        version["id"]: version["metadata"]["container"]["tags"]
+        for version in package_meta
+    }
+
+    # is pre-release
+    pre_rel = True if any(x in str(new_version) for x in ["a", "b", "rc"]) else False
+
+    # semver of current latest
+    for version, tags in version_tag_map.items():
+        if "latest" in tags:
+            # N.B. This seems counterintuitive, but we expect any version tagged
+            # 'latest' to have exactly three associated tags:
+            # latest, major.minor.latest, and major.minor.patch.
+            # Subtracting everything that contains the string 'latest' gets us
+            # the major.minor.patch which is what's needed for comparison.
+            current_latest = parse([tag for tag in tags if "latest" not in tag][0])
+        else:
+            current_latest = False
+
+    # semver of current_minor_latest
+    for version, tags in version_tag_map.items():
+        if f"{new_version.major}.{new_version.minor}.latest" in tags:
+            # Similar to above, only now we expect exactly two tags:
+            # major.minor.patch and major.minor.latest
+            current_minor_latest = parse(
+                [tag for tag in tags if "latest" not in tag][0]
+            )
+        else:
+            current_minor_latest = False
+
+    def is_latest(
+        pre_rel: bool, new_version: Version, remote_latest: Union[bool, Version]
+    ) -> bool:
+        """Determine if a given contaier should be tagged 'latest' based on:
+         - it's pre-release status
+         - it's version
+         - the version of a previously identified container tagged 'latest'
+
+        :param pre_rel: Wether or not the version of the new container is a pre-release
+        :param new_version: The version of the new container
+        :param remote_latest: The version of the previously identified container that's already tagged latest or False
+        """
+        # is a pre-release = not latest
+        if pre_rel:
+            return False
+        # + no latest tag found = is latest
+        if not remote_latest:
+            return True
+        # + if remote version is lower than current = is latest, else not latest
+        return True if remote_latest <= new_version else False
+
+    latest = is_latest(pre_rel, new_version, current_latest)
+    minor_latest = is_latest(pre_rel, new_version, current_minor_latest)
+
+    print(f"::set-output name=latest::{latest}")
+    print(f"::set-output name=minor_latest::{minor_latest}")
+

.github/workflows/release_docker.yml

@@ -0,0 +1,113 @@
+# **what?**
+# This workflow will generate a series of docker images for dbt and push them to the github container registry
+
+# **why?**
+# Docker images for dbt are used in a number of important places throughout the dbt ecosystem.  This is how we keep those images up-to-date.
+
+# **when?**
+# This is triggered manually 
+
+# **next steps**
+# - build this into the release workflow (or conversly, break out the different release methods into their own workflow files)
+
+name: Docker release
+
+on:
+  workflow_dispatch:
+    inputs:
+      package:
+       description: The package to release. _One_ of [dbt-core, dbt-redshift, dbt-bigquery, dbt-snowflake, dbt-spark, dbt-postgres]
+       required: true
+      version_number:
+       description: The release version number (i.e. 1.0.0b1). Do not include `latest` tags or a leading `v`!
+       required: true
+
+jobs:
+  get_version_meta:
+    name: Get version meta
+    runs-on: ubuntu-latest
+    outputs:
+      major: ${{ steps.version.outputs.major }}
+      minor: ${{ steps.version.outputs.minor }}
+      patch: ${{ steps.version.outputs.patch }}
+      latest: ${{ steps.latest.outputs.latest }}
+      minor_latest: ${{ steps.latest.outputs.minor_latest }}
+    steps:
+      - uses: actions/checkout@v1
+      - name: Split version
+        id: version
+        run: |
+          IFS="." read -r MAJOR MINOR PATCH <<< ${{ github.event.inputs.version_number }}
+          echo "::set-output name=major::$MAJOR"
+          echo "::set-output name=minor::$MINOR"
+          echo "::set-output name=patch::$PATCH"
+
+      - name: Is pkg 'latest'
+        id: latest
+        uses: ./.github/actions/latest-wrangler
+        with:
+          package: ${{ github.event.inputs.package }}
+          new_version: ${{ github.event.inputs.version_number }}
+          gh_token: ${{ secrets.GITHUB_TOKEN }}
+          halt_on_missing: False
+
+  setup_image_builder:
+    name: Set up docker image builder
+    runs-on: ubuntu-latest
+    needs: [get_version_meta]
+    steps:      
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+  build_and_push:
+    name: Build images and push to GHCR
+    runs-on: ubuntu-latest
+    needs: [setup_image_builder, get_version_meta]
+    steps:
+      - name: Get docker build arg
+        id: build_arg
+        run: |
+          echo "::set-output name=build_arg_name::"$(echo ${{ github.event.inputs.package }} | sed 's/\-/_/g')
+          echo "::set-output name=build_arg_value::"$(echo ${{ github.event.inputs.package }} | sed 's/postgres/core/g')
+
+      - name: Log in to the GHCR
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push MAJOR.MINOR.PATCH tag
+        uses: docker/build-push-action@v2
+        with:
+          file: docker/Dockerfile
+          push: True
+          target: ${{ github.event.inputs.package }}
+          build-args: |
+            ${{ steps.build_arg.outputs.build_arg_name }}_ref=${{ steps.build_arg.outputs.build_arg_value }}@v${{ github.event.inputs.version_number }}
+          tags: |
+            ghcr.io/dbt-labs/${{ github.event.inputs.package }}:${{ github.event.inputs.version_number }}
+
+      - name: Build and push MINOR.latest tag
+        uses: docker/build-push-action@v2
+        if: ${{ needs.get_version_meta.outputs.minor_latest == 'True' }}
+        with:
+          file: docker/Dockerfile
+          push: True
+          target: ${{ github.event.inputs.package }}
+          build-args: |
+            ${{ steps.build_arg.outputs.build_arg_name }}_ref=${{ steps.build_arg.outputs.build_arg_value }}@v${{ github.event.inputs.version_number }}
+          tags: |
+            ghcr.io/dbt-labs/${{ github.event.inputs.package }}:${{ needs.get_version_meta.outputs.major }}.${{ needs.get_version_meta.outputs.minor }}.latest
+
+      - name: Build and push latest tag
+        uses: docker/build-push-action@v2
+        if: ${{ needs.get_version_meta.outputs.latest == 'True' }}
+        with:
+          file: docker/Dockerfile
+          push: True
+          target: ${{ github.event.inputs.package }}
+          build-args: |
+            ${{ steps.build_arg.outputs.build_arg_name }}_ref=${{ steps.build_arg.outputs.build_arg_value }}@v${{ github.event.inputs.version_number }}
+          tags: |
+            ghcr.io/dbt-labs/${{ github.event.inputs.package }}:latest

.github/workflows/test/.actrc

@@ -0,0 +1 @@
+-P ubuntu-latest=ghcr.io/catthehacker/ubuntu:act-latest

.github/workflows/test/.gitignore

@@ -0,0 +1 @@
+.secrets

.github/workflows/test/.secrets.EXAMPLE

@@ -0,0 +1 @@
+GITHUB_TOKEN=GH_PERSONAL_ACCESS_TOKEN_GOES_HERE

.github/workflows/test/inputs/release_docker.json

@@ -0,0 +1,6 @@
+{
+    "inputs": {
+      "version_number": "1.0.1",
+      "package": "dbt-postgres"
+    }
+}

docker/Dockerfile

@@ -14,8 +14,8 @@ FROM --platform=$build_for python:3.9.9-slim-bullseye as base
 # N.B. The refs updated automagically every release via bumpversion
 # N.B. dbt-postgres is currently found in the core codebase so a value of dbt-core@<some_version> is correct
 
-ARG dbt_core_ref=dbt-core@v1.0.0
-ARG dbt_postgres_ref=dbt-core@v1.0.0
+ARG dbt_core_ref=dbt-core@v1.0.1
+ARG dbt_postgres_ref=dbt-core@v1.0.1
 ARG dbt_redshift_ref=dbt-redshift@v1.0.0
 ARG dbt_bigquery_ref=dbt-bigquery@v1.0.0
 ARG dbt_snowflake_ref=dbt-snowflake@v1.0.0