MINOR: Small refactor to the Make update_all scripts (#16204)

* Small refactor to the Make update_all scripts * Extract regex update method * decouple release from workflows * Fix issue with docker-openmetadata-db workflow referencing non existent action
open-metadata · May 10, 2024 · b67de74 · b67de74
1 parent 7fe0935
commit b67de74
Show file tree

Hide file tree

Showing 9 changed files with 261 additions and 416 deletions.
diff --git a/.github/actions/prepare-for-docker-build-and-push/action.yml b/.github/actions/prepare-for-docker-build-and-push/action.yml
@@ -0,0 +1,64 @@
+name: Prepare for Docker Build&Push
+description: Set up Docker Build&Push dependencies and generate Docker Tags.
+
+inputs:
+  image:
+    description: image name
+    required: true
+  tag:
+    description: Docker tag to use
+    required: true
+  push_latest:
+    description: true will push the 'latest' tag.
+    required: true
+    default: "false"
+  is_ingestion:
+    description: true if we are building an Ingestion image, false otherwise
+    required: true
+    default: "false"
+
+outputs:
+  tags:
+    description: Generated Docker Tags
+    value: ${{ steps.generate-tags.outputs.tags }}
+
+runs:
+  using: composite
+  steps:
+    - name: Set up QEMU
+      uses: docker/setup-qemu-action@v2
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v2
+
+    - name: Login to DockerHub
+      uses: docker/login-action@v2
+      with:
+        username: ${{ secrets.DOCKERHUB_OPENMETADATA_USERNAME }}
+        password: ${{ secrets.DOCKERHUB_OPENMETADATA_TOKEN }}
+
+    - name: Install Ubuntu dependencies
+      if: ${{ inputs.is_ingestion == "true" }}
+      shell: bash
+      run: |
+        sudo apt-get install -y python3-venv
+
+    - name: Install open-metadata dependencies
+      if: ${{ inputs.is_ingestion == "true" }}
+      shell: bash
+      run: |
+        python3 -m venv env
+        source env/bin/activate
+        pip install --upgrade pip
+        sudo make install_antlr_cli
+        make install_dev generate
+
+    - name: Process Docker Tags
+      id:  generate-tags
+      shell: bash
+      run: |
+          if ${{ inputs.push_latest == "true" }}; then
+              echo "tags=${{ inputs.image }}:${{ inputs.tag }},${{ inputs.image }}:latest" >> $GITHUB_OUTPUT
+          else
+              echo "tags=${{ inputs.image }}:${{ inputs.tag }}" >> $GITHUB_OUTPUT
+          fi
diff --git a/.github/workflows/docker-openmetadata-db.yml b/.github/workflows/docker-openmetadata-db.yml
@@ -19,51 +19,29 @@ on:
       push_latest_tag_to_release:
         description: "Do you want to update docker image latest tag as well ?"
         type: boolean
-  release:
-    types: [published]
 
 jobs:
   push_to_docker_hub:
     runs-on: ubuntu-latest
-    env:
-      input: ${{ github.event.inputs.tag }}
-
+
     steps:
-      - name: Check trigger type
-        if: ${{ env.input == '' }}
-        run: echo "input=1.4.0-SNAPSHOT" >> $GITHUB_ENV
-
       - name: Check out the Repo
         uses: actions/checkout@v3
 
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-
-      - name: Login to DockerHub
-        uses: docker/login-action@v2
+      - name: Prepare for Docker Build&Push
+        id: prepare
+        uses: ./.github/actions/prepare-for-docker-build-and-push
         with:
-          username: ${{ secrets.DOCKERHUB_OPENMETADATA_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_OPENMETADATA_TOKEN }}
+          image: openmetadata/db
+          tag: ${{ inputs.tag }}
+          push_latest: ${{ inputs.push_latest_tag_to_release }}
 
-      - name: Process Docker Tags
-        id:  input_check
-        run: |
-            if ${{ github.event_name == 'release' }}; then
-                echo "tags=openmetadata/db:${{ env.input }},openmetadata/db:latest" >> $GITHUB_OUTPUT
-            elif ${{ github.event_name == 'workflow_dispatch' && inputs.push_latest_tag_to_release }}; then
-                echo "tags=openmetadata/db:${{inputs.tag}},openmetadata/db:latest" >> $GITHUB_OUTPUT
-            else
-                echo "tags=openmetadata/db:${{ inputs.tag }}" >> $GITHUB_OUTPUT
-            fi
 
       - name: Build and push if event is workflow_dispatch and input is checked
         uses: docker/build-push-action@v3
         with:
           context: .
           platforms: linux/amd64,linux/arm64
-          push: ${{ github.event_name == 'release' || github.event_name == 'workflow_dispatch' }}
-          tags: ${{ steps.input_check.outputs.tags }}
-          file: ./docker/mysql/Dockerfile_mysql
+          push: true
+          tags: ${{ steps.prepare.outputs.tags }}
+          file: ./docker/mysql/Dockerfile_mysql
diff --git a/.github/workflows/docker-openmetadata-ingestion-base-slim.yml b/.github/workflows/docker-openmetadata-ingestion-base-slim.yml
@@ -19,65 +19,31 @@ on:
       push_latest_tag_to_release:
         description: "Do you want to update docker image latest tag as well ?"
         type: boolean
-  release:
-    types: [published]
 
 jobs:
   push_to_docker_hub:
     runs-on: ubuntu-latest
-    env:
-      input: ${{ github.event.inputs.tag }}
-
+
     steps:
-      - name: Check trigger type
-        if: ${{ env.input == '' }}
-        run: echo "input=1.4.0-SNAPSHOT" >> $GITHUB_ENV
-
       - name: Check out the Repo
         uses: actions/checkout@v3
 
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-
-      - name: Login to DockerHub
-        uses: docker/login-action@v2
+      - name: Prepare for Docker Build&Push
+        id: prepare
+        uses: ./.github/actions/prepare-for-docker-build-and-push
         with:
-          username: ${{ secrets.DOCKERHUB_OPENMETADATA_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_OPENMETADATA_TOKEN }}
-
-      - name: Install Ubuntu dependencies
-        run: |
-          sudo apt-get install -y python3-venv
-
-      - name: Install open-metadata dependencies
-        run: |
-          python3 -m venv env
-          source env/bin/activate
-          pip install --upgrade pip
-          sudo make install_antlr_cli
-          make install_dev generate
-
-      - name: Process Docker Tags
-        id:  input_check
-        run: |
-            if ${{ github.event_name == 'release' }}; then
-                echo "tags=openmetadata/ingestion-base-slim:${{ env.input }},openmetadata/ingestion-base-slim:latest" >> $GITHUB_OUTPUT
-            elif ${{ github.event_name == 'workflow_dispatch' && inputs.push_latest_tag_to_release }}; then
-                echo "tags=openmetadata/ingestion-base-slim:${{inputs.tag}},openmetadata/ingestion-base-slim:latest" >> $GITHUB_OUTPUT
-            else
-                echo "tags=openmetadata/ingestion-base-slim:${{ inputs.tag }}" >> $GITHUB_OUTPUT
-            fi
+          image: openmetadata/ingestion-base-slim
+          tag: ${{ inputs.tag }}
+          push_latest: ${{ inputs.push_latest_tag_to_release }}
+          is_ingestion: true
 
       - name: Build and push if event is workflow_dispatch and input is checked
         uses: docker/build-push-action@v3
         with:
           context: .
           platforms: linux/amd64,linux/arm64
-          push: ${{ github.event_name == 'release' || github.event_name == 'workflow_dispatch' }}
-          tags: ${{ steps.input_check.outputs.tags }}
+          push: true
+          tags: ${{ steps.prepare.outputs.tags }}
           file: ./ingestion/operators/docker/Dockerfile
           build-args: |
-            INGESTION_DEPENDENCY=slim
+            INGESTION_DEPENDENCY=slim
diff --git a/.github/workflows/docker-openmetadata-ingestion-base.yml b/.github/workflows/docker-openmetadata-ingestion-base.yml
@@ -19,63 +19,29 @@ on:
       push_latest_tag_to_release:
         description: "Do you want to update docker image latest tag as well ?"
         type: boolean
-  release:
-    types: [published]
 
 jobs:
   push_to_docker_hub:
     runs-on: ubuntu-latest
-    env:
-      input: ${{ github.event.inputs.tag }}
-
+
     steps:
-      - name: Check trigger type
-        if: ${{ env.input == '' }}
-        run: echo "input=1.4.0-SNAPSHOT" >> $GITHUB_ENV
-
       - name: Check out the Repo
         uses: actions/checkout@v3
 
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-
-      - name: Login to DockerHub
-        uses: docker/login-action@v2
+      - name: Prepare for Docker Build&Push
+        id: prepare
+        uses: ./.github/actions/prepare-for-docker-build-and-push
         with:
-          username: ${{ secrets.DOCKERHUB_OPENMETADATA_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_OPENMETADATA_TOKEN }}
-
-      - name: Install Ubuntu dependencies
-        run: |
-          sudo apt-get install -y python3-venv
-
-      - name: Install open-metadata dependencies
-        run: |
-          python3 -m venv env
-          source env/bin/activate
-          pip install --upgrade pip
-          sudo make install_antlr_cli
-          make install_dev generate
-
-      - name: Process Docker Tags
-        id:  input_check
-        run: |
-            if ${{ github.event_name == 'release' }}; then
-                echo "tags=openmetadata/ingestion-base:${{ env.input }},openmetadata/ingestion-base:latest" >> $GITHUB_OUTPUT
-            elif ${{ github.event_name == 'workflow_dispatch' && inputs.push_latest_tag_to_release }}; then
-                echo "tags=openmetadata/ingestion-base:${{inputs.tag}},openmetadata/ingestion-base:latest" >> $GITHUB_OUTPUT
-            else
-                echo "tags=openmetadata/ingestion-base:${{ inputs.tag }}" >> $GITHUB_OUTPUT
-            fi
+          image: openmetadata/ingestion-base
+          tag: ${{ inputs.tag }}
+          push_latest: ${{ inputs.push_latest_tag_to_release }}
+          is_ingestion: true
 
       - name: Build and push if event is workflow_dispatch and input is checked
         uses: docker/build-push-action@v3
         with:
           context: .
           platforms: linux/amd64,linux/arm64
-          push: ${{ github.event_name == 'release' || github.event_name == 'workflow_dispatch' }}
-          tags: ${{ steps.input_check.outputs.tags }}
-          file: ./ingestion/operators/docker/Dockerfile
+          push: true
+          tags: ${{ steps.prepare.outputs.tags }}
+          file: ./ingestion/operators/docker/Dockerfile
diff --git a/.github/workflows/docker-openmetadata-ingestion.yml b/.github/workflows/docker-openmetadata-ingestion.yml
@@ -19,63 +19,29 @@ on:
       push_latest_tag_to_release:
         description: "Do you want to update docker image latest tag as well ?"
         type: boolean
-  release:
-    types: [published]
 
 jobs:
   push_to_docker_hub:
     runs-on: ubuntu-latest
-    env:
-      input: ${{ github.event.inputs.tag }}
-
+
     steps:
-      - name: Check trigger type
-        if: ${{ env.input == '' }}
-        run: echo "input=1.4.0-SNAPSHOT" >> $GITHUB_ENV
-
       - name: Check out the Repo
         uses: actions/checkout@v3
 
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-
-      - name: Login to DockerHub
-        uses: docker/login-action@v2
+      - name: Prepare for Docker Build&Push
+        id: prepare
+        uses: ./.github/actions/prepare-for-docker-build-and-push
         with:
-          username: ${{ secrets.DOCKERHUB_OPENMETADATA_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_OPENMETADATA_TOKEN }}
-
-      - name: Install Ubuntu dependencies
-        run: |
-          sudo apt-get install -y python3-venv
-
-      - name: Install open-metadata dependencies
-        run: |
-          python3 -m venv env
-          source env/bin/activate
-          pip install --upgrade pip
-          sudo make install_antlr_cli
-          make install_dev generate
-
-      - name: Process Docker Tags
-        id:  input_check
-        run: |
-            if ${{ github.event_name == 'release' }}; then
-                echo "tags=openmetadata/ingestion:${{ env.input }},openmetadata/ingestion:latest" >> $GITHUB_OUTPUT
-            elif ${{ github.event_name == 'workflow_dispatch' && inputs.push_latest_tag_to_release }}; then
-                echo "tags=openmetadata/ingestion:${{inputs.tag}},openmetadata/ingestion:latest" >> $GITHUB_OUTPUT
-            else
-                echo "tags=openmetadata/ingestion:${{ inputs.tag }}" >> $GITHUB_OUTPUT
-            fi
+          image: openmetadata/ingestion
+          tag: ${{ inputs.tag }}
+          push_latest: ${{ inputs.push_latest_tag_to_release }}
+          is_ingestion: true
 
       - name: Build and push if event is workflow_dispatch and input is checked
         uses: docker/build-push-action@v3
         with:
           context: .
           platforms: linux/amd64,linux/arm64
-          push: ${{ github.event_name == 'release' || github.event_name == 'workflow_dispatch' }}
-          tags: ${{ steps.input_check.outputs.tags }}
-          file: ./ingestion/Dockerfile
+          push: true
+          tags: ${{ steps.prepare.outputs.tags }}
+          file: ./ingestion/Dockerfile