Add Python release.

AbsaOSS · Jan 25, 2024 · 20a319d · 20a319d
1 parent 1375e7e
commit 20a319d
Show file tree

Hide file tree

Showing 5 changed files with 142 additions and 26 deletions.
diff --git a/.github/workflows/python_publish.yml b/.github/workflows/python_publish.yml
@@ -37,20 +37,9 @@ jobs:
       - uses: actions/checkout@v3
       - name: import absa oss ci/cd bot key
         run: |
-          sudo sh -c "echo 'deb https://gitsecret.jfrog.io/artifactory/git-secret-deb git-secret main' >> /etc/apt/sources.list"
-          wget -qO - 'https://gitsecret.jfrog.io/artifactory/api/gpg/key/public' | sudo apt-key add -
-          sudo apt install -y --no-install-recommends \
-            git-secret
           echo "${{ secrets.ABSA_OSS_CI_CD_BOT_GPG_KEY }}" | tr ',' '\n' > ./private_key.gpg
           gpg --batch --yes --pinentry-mode loopback --import private_key.gpg
           rm private_key.gpg
-      - name: reveal secrets
-        working-directory: "."
-        run: |
-          git config --global --add safe.directory /__w/pramen/pramen
-          git init
-          git add .
-          git secret reveal
       - name: install project dependencies
         run: |
           sudo apt install -y --no-install-recommends \
@@ -67,4 +56,5 @@ jobs:
       - name: build and publish the wheel to jfrog
         env:
           ENV: pypi
+          PRAMENPY_PYPI_TOKEN: ${{ secrets.PRAMENPY_PYPI_TOKEN }}
         run: make --silent publish
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -7,11 +7,94 @@ on:
   #pull_request:
   #  branches: [ main ]
 
+defaults:
+  run:
+    shell: bash
+
 jobs:
+  test-python:
+    strategy:
+      matrix:
+        python-version: [ "3.10" ]
+    runs-on: ubuntu-latest
+    name: Test Pramen-Py
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          ref: main
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: setup poetry
+        uses: abatilo/actions-poetry@v2.1.6
+        with:
+          poetry-version: 1.4.2
+      - name: install dependencies
+        working-directory: "./pramen-py"
+        run: make --silent install
+      - name: test
+        working-directory: "./pramen-py"
+        env:
+          ENV: ci
+        run: make --silent test
+  release-oython:
+    needs: [ "test-python" ]
+    runs-on: ubuntu-latest
+    name: Release Python artifact
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: main
+      - name: Prepare the release branch
+        id: release_branch1
+        working-directory: "./pramen"
+        run: |
+          VERSION=$(grep "ThisBuild / version" version.sbt | cut -d\" -f2 | sed 's/-SNAPSHOT//')
+          echo "VERSION=$VERSION" >> $GITHUB_OUTPUT
+          git config --global user.email "absa_ci_cd_bot@absa.africa"
+          git config --global user.name "CI/CD bot"
+          git checkout -b release/$VERSION
+          git push --set-upstream origin release/$VERSION
+      - name: Update version number
+        id: release_branch
+        working-directory: "./pramen-py"
+        env:
+          VERSION: ${{ steps.release_branch1.outputs.VERSION }}
+        run: |
+          PY_VERSION=$(grep -m 1 ^version pyproject.toml | tr -s ' ' | tr -d '"' | tr -d "'" | cut -d' ' -f3)
+          if [[ "$PY_VERSION" != "$VERSION" ]]; then
+            sed -i "s/version = \"$PY_VERSION\"/version = \"$VERSION\"/g" pyproject.toml
+            git add pyproject.toml
+            git commit -m "Update version number to $VERSION"
+            git push
+            cat pyproject.toml 
+          fi
+      - name: install project dependencies
+        run: |
+          sudo apt install -y --no-install-recommends \
+            libssl-dev \
+            make
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+      - uses: abatilo/actions-poetry@v2.1.6
+        with:
+          poetry-version: 1.4.2
+      - name: Install dependencies
+        working-directory: "./pramen-py"
+        run: poetry install --no-interaction --no-root
+      - name: build and publish the wheel to jfrog
+        working-directory: "./pramen-py"
+        env:
+          ENV: pypi
+          PRAMENPY_PYPI_TOKEN: ${{ secrets.PRAMENPY_PYPI_TOKEN }}
+        run: make --silent publish
+
   release-sbt:
+    needs: [ "release-oython" ]
     runs-on: ubuntu-latest
-    name: Release Scala Artifacts
-
+    name: Release Scala artifacts
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -24,38 +107,54 @@ jobs:
           java-version: "adopt@1.8"
 
       - name: Import GPG Key
-        shell: bash
         run: |
           echo "${{ secrets.ABSA_OSS_CI_CD_BOT_GPG_KEY }}" > gpg-secret-key.asc
           gpg --import --batch gpg-secret-key.asc && rm -rf gpg-secret-key.asc
           mkdir -p ~/.gnupg
-          git config --global user.email "absa_ci_cd_bot@absa.africa"
-          git config --global user.name "CI/CD bot"
 
       - name: Setup SonaType config
-        shell: bash
         run: |
           mkdir -p ~/.sbt/1.0
           echo "${{ secrets.SONATYPE_CONFIG }}" | base64 --decode > ~/.sbt/1.0/sonatype.sbt
 
-      - name: Prepare the release branch
-        id: release_branch
-        shell: bash
+      - name: Checkout the release branch
         working-directory: ./pramen
         run: |
           VERSION=$(grep "ThisBuild / version" version.sbt | cut -d\" -f2 | sed 's/-SNAPSHOT//')
           echo "VERSION=$VERSION" >> $GITHUB_OUTPUT
-          git checkout -b release/$VERSION
-          git push --set-upstream origin release/$VERSION
+          git config --global user.email "absa_ci_cd_bot@absa.africa"
+          git config --global user.name "CI/CD bot"
+          git fetch origin release/$VERSION 
+          git checkout release/$VERSION
 
       - name: Run the release plugin
-        shell: bash
         working-directory: ./pramen
         run: sbt releaseNow
 
+  create-pr:
+    needs: [ "release-sbt" ]
+    runs-on: ubuntu-latest
+    name: Create Pull Request
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: main
+
+      - name: Checkout the release branch
+        id: release_branch3
+        working-directory: ./pramen
+        run: |
+          VERSION=$(grep "ThisBuild / version" version.sbt | cut -d\" -f2 | sed 's/-SNAPSHOT//')
+          echo "VERSION=$VERSION" >> $GITHUB_OUTPUT
+          git config --global user.email "absa_ci_cd_bot@absa.africa"
+          git config --global user.name "CI/CD bot"
+          git fetch origin release/$VERSION
+          git checkout release/$VERSION
+
       - name: Create Pull Request
-        shell: bash
         run: gh pr create -B main -H "release/$VERSION" --title "Release Pramen v$VERSION" --body 'Created by Github action'
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          VERSION: ${{ steps.release_branch.outputs.VERSION }}
+          VERSION: ${{ steps.release_branch3.outputs.VERSION }}
diff --git a/pramen-py/.env.pypi b/pramen-py/.env.pypi
@@ -0,0 +1,14 @@
+# enable debug
+export PRAMENPY_DEBUG=True
+# app settings (needed to run the app)
+    ## default metastore filesystem (available options are: hdfs, s3, local)
+    ## default = "hdfs"
+export PRAMENPY_DEFAULT_FS="local"
+    ## in case of Py4JJavaError execution will be retried this number of times
+    ## default is 1
+export PRAMENPY_MAX_RETRIES_EXECUTE_TRANSFORMATION=1
+    ## sparks related
+        ### if not present, will be skipped
+# export PRAMENPY_SPARK_JAVA_HOME=
+        ### if not present, will be skipped
+export PRAMENPY_SPARK_CONFIG=spark.driver.host=127.0.0.1
diff --git a/pramen-py/.env.test b/pramen-py/.env.test
@@ -0,0 +1,14 @@
+# enable debug
+export PRAMENPY_DEBUG=False
+# app settings (needed to run the app)
+    ## default metastore filesystem (available options are: hdfs, s3, local)
+    ## default = "hdfs"
+export PRAMENPY_DEFAULT_FS="local"
+    ## in case of Py4JJavaError execution will be retried this number of times
+    ## default is 1
+export PRAMENPY_MAX_RETRIES_EXECUTE_TRANSFORMATION=1
+    ## sparks related
+        ### if not present, will be skipped
+#export PRAMENPY_SPARK_JAVA_HOME=/usr
+        ### if not present, will be skipped
+export PRAMENPY_SPARK_CONFIG=spark.master=local,spark.driver.host=127.0.0.1,spark.driver.bindAddress=127.0.0.1,spark.executor.instances=1,spark.executor.cores=1,spark.ui.enabled=false
diff --git a/pramen-py/.gitignore b/pramen-py/.gitignore
@@ -144,7 +144,6 @@ dmypy.json
 /cdk.context.json
 
 # DotENV files
-.env*
 !.env.local
 !.env.example
 !.env.ci