Merge pull request #4919 from tonhuisman/feature/GH-up-download-delet…

…e-actions-fix [Build] Fix buildscripts for v4 Actions `upload-artifact`, `download-artifact` and `delete-artifact`
letscontrolit · Jan 26, 2024 · 8e7c162 · 8e7c162
2 parents 0c8a0d1 + c7608cf
commit 8e7c162
Show file tree

Hide file tree

Showing 3 changed files with 68 additions and 117 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -24,10 +24,10 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: '3.x'
-      - uses: actions/cache@v3
+      - uses: actions/cache@v4
         with:
           path: ~/.cache/pip
           key: ${{ runner.os }}-docs-${{ hashFiles('docs/requirements.txt') }}
@@ -39,7 +39,7 @@ jobs:
           make html
           cd ..
           zip -r -qq ESPEasy_docs.zip docs/build/*
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
           name: Documentation
           path: ESPEasy_docs.zip
@@ -49,7 +49,7 @@ jobs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: '3.x'
       - id: set-matrix
@@ -64,19 +64,19 @@ jobs:
       matrix: ${{ fromJson(needs.generate-matrix.outputs.matrix) }}
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
-          python-version: '3.11'
-      - uses: actions/cache@v3
+          python-version: '3.x'
+      - uses: actions/cache@v4
         with:
           path: ~/.cache/pip
           key: ${{ runner.os }}-${{ hashFiles('requirements.txt') }}
-      - uses: actions/cache@v3
+      - uses: actions/cache@v4
         if: ${{ contains(matrix.env, 'esp32') }}
         with:
           path: ~/.platformio
           key: ${{ runner.os }}-esp32-${{ hashFiles('platformio*.ini') }}
-      - uses: actions/cache@v3
+      - uses: actions/cache@v4
         if: ${{ contains(matrix.env, 'esp8266') }}
         with:
           path: ~/.platformio
@@ -89,123 +89,60 @@ jobs:
           pip install wheel
           pip install -r requirements.txt
           platformio update
+      - name: Get current date
+        id: date
+        run: |
+          echo "builddate=$(date +'%Y%m%d')" >> $GITHUB_OUTPUT
       - name: Build and archive
         id: build-and-archive
         env:
           CHIP: ${{ matrix.chip }}
           ENV: ${{ matrix.env }}
         run: |
           python tools/ci/build-and-archive.py
-      - uses: actions/upload-artifact@v3
+      - id: string
+        uses: Entepotenz/change-string-case-action-min-dependencies@v1
+        with:
+          string: ${{ matrix.chip }}
+      - uses: actions/upload-artifact@v4
         with:
-          name: Binaries
+          # FIXME Workaround to (temporarily) not use # in the artifact name, see https://github.com/actions/upload-artifact/issues/473
+          name: Bin-${{ steps.string.outputs.uppercase }}-${{ matrix.env }}-${{ steps.date.outputs.builddate }}_PR_${{ github.event.number }}_${{ github.run_id }} # Sort by ESP type
           path: |
             bin
           if-no-files-found: ignore
 
-  # Repackage separately for ESP82xx and ESP32
-  repackage:
+  # Package all ESP82xx and ESP32 into a single ESPEasy-all-Binaries-<date>_PR#<pr_nr>_<runnr>.zip
+  combine_package:
     needs: build
     runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: '3.x'
-      - uses: actions/cache@v3
+      - uses: actions/cache@v4
         with:
           path: ~/.cache/pip
           key: ${{ runner.os }}-docs-${{ hashFiles('docs/requirements.txt') }}
       - name: Get current date
         id: date
         run: |
           echo "builddate=$(date +'%Y%m%d')" >> $GITHUB_OUTPUT
-      - uses: actions/download-artifact@v3
+      - name: Download all artifacts
+        uses: actions/download-artifact@v4
         with:
-          path: artifacts/
-      - name: Prepare artifacts
+          path: artifacts/Binaries/bin/
+          pattern: Bin-*
+          merge-multiple: true
+      - name: List all files in the package for single-archive upload
         run: |
-          # ESP82xx
-          mkdir ESPEasy_dist_ESP82xx
-          cd dist
-          find . -exec cp -r --parents {} ../ESPEasy_dist_ESP82xx/ \;
-          rm ../ESPEasy_dist_ESP82xx/bin/blank_8MB.bin
-          cd ../artifacts/Binaries
-          find . -not -name '*ESP32*' -exec mv {} ../../ESPEasy_dist_ESP82xx/bin/ \;
-          cd ../..
-          # ESP32 and derived cpus, ESP32 (classic) _MUST_ be last in this list!
-          for cpu in ESP32s2 ESP32c3 ESP32s3 ESP32c2 ESP32c6 ESP32h2 ESP32solo1 ESP32
-          do
-            mkdir ESPEasy_dist_${cpu}
-            spec="*${cpu}*"
-            _files=$(find ./artifacts/Binaries -name "$spec"|wc -l)
-            if [[ $_files > 0 ]]; then
-              cd dist
-              find . -exec cp -r --parents {} ../ESPEasy_dist_${cpu}/ \;
-              cd ../ESPEasy_dist_${cpu}
-              rm bin/blank_1MB.bin bin/blank_2MB.bin bin/ESPEasy_2step_UploaderMega_1024.bin
-              cd ../artifacts/Binaries
-              find . -name "$spec" -exec mv {} ../../ESPEasy_dist_${cpu}/bin/ \;
-              cd ../..
-            fi
-          done
-      # Each supported cpu has to be listed separately, but empty folders are ignored
-      - uses: actions/upload-artifact@v3
-        with:
-          name: ESPEasy_mega_${{ steps.date.outputs.builddate }}_ESP32_PR#${{ github.event.number }}_${{ github.run_id }}
-          path: |
-            ESPEasy_dist_ESP32/*
-          if-no-files-found: ignore
-      - uses: actions/upload-artifact@v3
-        with:
-          name: ESPEasy_mega_${{ steps.date.outputs.builddate }}_ESP32solo1_PR#${{ github.event.number }}_${{ github.run_id }}
-          path: |
-            ESPEasy_dist_ESP32solo1/*
-          if-no-files-found: ignore
-      - uses: actions/upload-artifact@v3
+          cd artifacts/Binaries/
+          ls -R
+      - uses: actions/upload-artifact@v4
         with:
-          name: ESPEasy_mega_${{ steps.date.outputs.builddate }}_ESP32s2_PR#${{ github.event.number }}_${{ github.run_id }}
+          # FIXME Workaround to (temporarily) not use # in the artifact name, see https://github.com/actions/upload-artifact/issues/473
+          name: ESPEasy-all-Binaries-${{ steps.date.outputs.builddate }}_PR_${{ github.event.number }}_${{ github.run_id }}
           path: |
-            ESPEasy_dist_ESP32s2/*
+            artifacts/Binaries/
           if-no-files-found: ignore
-      - uses: actions/upload-artifact@v3
-        with:
-          name: ESPEasy_mega_${{ steps.date.outputs.builddate }}_ESP32c3_PR#${{ github.event.number }}_${{ github.run_id }}
-          path: |
-            ESPEasy_dist_ESP32c3/*
-          if-no-files-found: ignore
-      - uses: actions/upload-artifact@v3
-        with:
-          name: ESPEasy_mega_${{ steps.date.outputs.builddate }}_ESP32s3_PR#${{ github.event.number }}_${{ github.run_id }}
-          path: |
-            ESPEasy_dist_ESP32s3/*
-          if-no-files-found: ignore
-      - uses: actions/upload-artifact@v3
-        with:
-          name: ESPEasy_mega_${{ steps.date.outputs.builddate }}_ESP32c2_PR#${{ github.event.number }}_${{ github.run_id }}
-          path: |
-            ESPEasy_dist_ESP32c2/*
-          if-no-files-found: ignore
-      - uses: actions/upload-artifact@v3
-        with:
-          name: ESPEasy_mega_${{ steps.date.outputs.builddate }}_ESP32c6_PR#${{ github.event.number }}_${{ github.run_id }}
-          path: |
-            ESPEasy_dist_ESP32c6/*
-          if-no-files-found: ignore
-      - uses: actions/upload-artifact@v3
-        with:
-          name: ESPEasy_mega_${{ steps.date.outputs.builddate }}_ESP32h2_PR#${{ github.event.number }}_${{ github.run_id }}
-          path: |
-            ESPEasy_dist_ESP32h2/*
-          if-no-files-found: ignore
-      - uses: actions/upload-artifact@v3
-        with:
-          name: ESPEasy_mega_${{ steps.date.outputs.builddate }}_ESP82xx_PR#${{ github.event.number }}_${{ github.run_id }}
-          path: |
-            ESPEasy_dist_ESP82xx/*
-          if-no-files-found: ignore
-      # When successfully re-packaged, the original Binaries can be removed
-      # comment below 3 lines to not remove the Binaries artifact after repackaging
-      - uses: geekyeggo/delete-artifact@v2
-        with:
-            name: Binaries
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -22,7 +22,7 @@ jobs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: '3.x'
       - id: set-matrix
@@ -37,19 +37,19 @@ jobs:
       matrix: ${{ fromJson(needs.generate-matrix.outputs.matrix) }}
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: '3.x'
-      - uses: actions/cache@v3
+      - uses: actions/cache@v4
         with:
           path: ~/.cache/pip
           key: ${{ runner.os }}-${{ hashFiles('requirements.txt') }}
-      - uses: actions/cache@v3
+      - uses: actions/cache@v4
         if: ${{ contains(matrix.env, 'esp32') }}
         with:
           path: ~/.platformio
           key: ${{ runner.os }}-esp32-${{ hashFiles('platformio*.ini') }}
-      - uses: actions/cache@v3
+      - uses: actions/cache@v4
         if: ${{ contains(matrix.env, 'esp8266') }}
         with:
           path: ~/.platformio
@@ -66,9 +66,13 @@ jobs:
           ENV: ${{ matrix.env }}
         run: |
           python tools/ci/build-and-archive.py
-      - uses: actions/upload-artifact@v3
+      - id: string
+        uses: Entepotenz/change-string-case-action-min-dependencies@v1
         with:
-          name: Binaries
+          string: ${{ matrix.chip }}
+      - uses: actions/upload-artifact@v4
+        with:
+          name: Bin-${{ steps.string.outputs.uppercase }}-${{ matrix.env }}
           path: |
             bin
           if-no-files-found: ignore
@@ -78,10 +82,10 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: '3.x'
-      - uses: actions/cache@v3
+      - uses: actions/cache@v4
         with:
           path: ~/.cache/pip
           key: ${{ runner.os }}-docs-${{ hashFiles('docs/requirements.txt') }}
@@ -98,7 +102,7 @@ jobs:
           cd dist
           zip -r -qq ../ESPEasy_dist.zip *
           cd ..
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
           name: Distribution
           path: |
@@ -112,7 +116,7 @@ jobs:
       notes: ${{ steps.release-notes.outputs.result }}
     steps:
       - id: release-notes
-        uses: actions/github-script@v6
+        uses: actions/github-script@v7
         with:
           result-encoding: string
           script: |
@@ -134,23 +138,24 @@ jobs:
     needs: [build, prepare-dist, prepare-notes]
     runs-on: ubuntu-22.04
     steps:
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: '3.x'
       - name: Get current date
         id: date
         run: |
           echo "builddate=$(date +'%Y%m%d')" >> $GITHUB_OUTPUT
-      - uses: actions/download-artifact@v3
+      - name: Download all artifacts
+        uses: actions/download-artifact@v4
         with:
-          path: artifacts/
+          path: artifacts/Binaries/bin/
+          pattern: Bin-*
+          merge-multiple: true
       - name: Repackage for release upload
         run: |
           ls -R
           sudo apt install zipmerge zip
           cd artifacts/Binaries
-          mkdir bin
-          mv *.* bin
           find . -not -name '*ESP32*' -print | zip -@ ../../ESPEasy_ESP82xx.zip
           # ESP32 and derived chips
           # TODO if/when available: ESP32h2

diff --git a/tools/ci/generate-matrix.py b/tools/ci/generate-matrix.py
@@ -7,16 +7,25 @@
 
 import json
 import os
+import re
 from platformio.project.config import ProjectConfig
 
 
 def get_jobs(cfg):
+    regex = re.compile(r".*(ESP[^_]*).*")
     for env in cfg.envs():
         platform = cfg.get("env:{}".format(env), "platform")
+        match = regex.match(env)
+        if type(match) == re.Match:
+            typ = match.group(1)
+            if "ESP8285" == typ:
+                typ = "ESP8266" # Generalize ESP8285 into ESP8266
+        else:
+            typ = "ESP8266" # Catch WROOM02 and some other 'hard_' builds
         if "espressif8266" in platform:
-            yield {"chip": "esp8266", "env": env}
+            yield {"chip": typ.lower(), "env": env}
         elif "espressif32" in platform:
-            yield {"chip": "esp32", "env": env}
+            yield {"chip": typ.lower(), "env": env}
         else:
             raise ValueError(
                 "Unknown `platform = {}` for `[env:{}]`".format(platform, env)