[dl] Download STM32CubeProgrammer database for memory sizes

.github/workflows/deploy-docs.yml

@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Configure Git
         run: |
@@ -29,7 +29,7 @@ jobs:
           pip3 install -r tools/requirements.txt ".[docs]"
 
       - name: Clone modm-ext/data.modm.io repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           repository: modm-ext/data.modm.io
           ssh-key: ${{secrets.SSH_KEY_DATA_MODM_IO}}

.github/workflows/pypi.yml

@@ -16,7 +16,7 @@ jobs:
       id-token: write
     steps:
     - name: Clone repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Set up Python 3.12
       uses: actions/setup-python@v4

.github/workflows/test.yml

@@ -10,10 +10,10 @@ jobs:
     steps:
 
       - name: Check out Repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Check out Test Docs Repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           repository: modm-ext/modm-data-test-docs
           path: ext/test/regression

.github/workflows/update-archives.yml

@@ -11,7 +11,7 @@ jobs:
     steps:
 
       - name: Check out Repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Configure Git
         run: |
@@ -32,7 +32,7 @@ jobs:
       # This cache repository is private since copyright prevents republishing!
       # Nicely ask @salkinium for access to this repository.
       - name: Check out STMicro PDF Archive
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           repository: modm-ext/archive-stmicro-pdf
           path: ext/stmicro/pdf
@@ -56,7 +56,7 @@ jobs:
       # This cache repository is private since copyright prevents republishing!
       # Nicely ask @salkinium for access to this repository.
       - name: Check out STMicro HTML Archive
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           repository: modm-ext/archive-stmicro-html
           path: ext/stmicro/html-archive
@@ -85,12 +85,12 @@ jobs:
 
 
 
-  update-cubemx:
+  update-cube:
     runs-on: ubuntu-latest
     steps:
 
       - name: Check out Repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Configure Git
         run: |
@@ -108,17 +108,40 @@ jobs:
           pip3 install -r tools/requirements.txt .
 
 
+      # This cache repository is private since copyright prevents republishing!
+      # Nicely ask @salkinium for access to this repository.
+      - name: Check out STMicro CubeProg Archive
+        uses: actions/checkout@v4
+        with:
+          repository: modm-ext/archive-stmicro-cubeprog
+          path: ext/stmicro/cubeprog
+          ssh-key: ${{ secrets.SSH_KEY_STMICRO_CUBEPROG }}
+          persist-credentials: true
+
+      - name: Update CubeProg Archive
+        run: |
+          make download-stmicro-cubeprog
+
+      - name: Git push CubeProg Archive
+        working-directory: ext/stmicro/cubeprog
+        run: |
+          git add .
+          git commit -m "Update CubeProg" || true
+          git gc
+          git count-objects -vH
+          git push origin main
+
+
       # This cache repository is private since copyright prevents republishing!
       # Nicely ask @salkinium for access to this repository.
       - name: Check out STMicro CubeMX Archive
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           repository: modm-ext/archive-stmicro-cubemx
           path: ext/stmicro/cubemx
           ssh-key: ${{ secrets.SSH_KEY_STMICRO_CUBEMX }}
           persist-credentials: true
 
-
       - name: Update CubeMX Archive
         run: |
           make download-stmicro-cubemx

src/modm_data/dl/stmicro/__init__.py

@@ -3,6 +3,7 @@
 
 from .document import Document, load_remote_info, load_local_info, store_remote_info, store_local_info
 from .cubemx import download_cubemx
+from .cubeprog import download_cubeprog
 
 __all__ = [
     "Document",
@@ -11,4 +12,5 @@
     "store_remote_info",
     "store_local_info",
     "download_cubemx",
+    "download_cubeprog",
 ]

src/modm_data/dl/stmicro/__main__.py

@@ -7,7 +7,7 @@
 from pathlib import Path
 
 import modm_data
-from modm_data.dl.stmicro import download_cubemx
+from modm_data.dl.stmicro import download_cubemx, download_cubeprog
 from modm_data.dl.stmicro import load_remote_info, store_remote_info
 from modm_data.dl.stmicro import load_local_info, store_local_info, Document
 
@@ -83,12 +83,16 @@ def download_pdfs(base_dir, with_download, new_pdfs=None):
     cubemx_parser = subparsers.add_parser("cubemx", help="CubeMX database.")
     cubemx_parser.add_argument("--patch", action="store_true", help="Apply the patch to the database.")
 
+    subparsers.add_parser("cubeprog", help="CubeProg database.")
+
     args = parser.parse_args()
 
     match args.command:
         case "pdf":
             result = download_pdfs(args.directory, args.download, args.new)
         case "cubemx":
             result = download_cubemx(args.directory, args.download, args.patch)
+        case "cubeprog":
+            result = download_cubeprog(args.directory, args.download)
 
     exit(0 if result else 1)

src/modm_data/dl/stmicro/cubeprog.py

@@ -0,0 +1,62 @@
+# Copyright 2025, Niklas Hauser
+# SPDX-License-Identifier: MPL-2.0
+
+import zipfile
+import shutil
+import re
+import io
+import random
+import time
+import lzma
+import logging
+from pathlib import Path
+from .cubemx import _dl
+
+LOGGER = logging.getLogger(__name__)
+
+
+def download_cubeprog(extraction_path: Path, with_download: bool = True) -> bool:
+    # com.st.stm32cube.ide.mcu.productdb.debug contains all the SVD files
+    # com.st.stm32cube.ide.mcu.externaltools.cubeprogrammer.linux64 contains the CubeProgrammer database
+    # com.st.stm32cube.common.mx contains the CubeMX database
+    base_url = "http://sw-center.st.com/stm32cubeide/updatesite1/"
+    cubeprog_id = "com.st.stm32cube.ide.mcu.externaltools.cubeprogrammer.linux64"
+
+    if with_download:
+        LOGGER.info("Downloading Update Info...")
+        with io.TextIOWrapper(
+            io.BytesIO(_dl(f"{base_url}/compositeContent.xml")), encoding="utf-8"
+        ) as compositeContent:
+            update_version = re.findall(r"child location='(\d+\.\d+\.\d+)'", compositeContent.read())
+            update_version = sorted(update_version, key=lambda v: tuple(map(int, v.split("."))), reverse=True)[0]
+
+        artifacts = lzma.LZMADecompressor().decompress(_dl(f"{base_url}/{update_version}/artifacts.xml.xz"))
+        version = re.search(f"id='{cubeprog_id}' version='(.*?)'", artifacts.decode("utf-8")).group(1)
+
+        download_url = f"{base_url}/{update_version}/plugins/{cubeprog_id}_{version}.jar"
+        shutil.rmtree(extraction_path / "Data_Base", ignore_errors=True)
+        extraction_path.mkdir(exist_ok=True, parents=True)
+
+        LOGGER.info("Downloading Database...")
+        LOGGER.debug(download_url)
+        time.sleep(random.randrange(1, 6))
+
+        z = zipfile.ZipFile(io.BytesIO(_dl(download_url)))
+        LOGGER.info("Extracting Database...")
+        for file in z.namelist():
+            if file.startswith("tools/Data_Base"):
+                z.extract(file, extraction_path)
+
+        LOGGER.info("Moving Database...")
+        shutil.move(extraction_path / "tools/Data_Base", extraction_path / "Data_Base")
+        shutil.rmtree(extraction_path / "tools", ignore_errors=True)
+
+        LOGGER.info("Normalizing file endings...")
+        for file in Path(extraction_path).glob("**/*"):
+            if str(file).endswith(".xml"):
+                with file.open("r", newline=None, encoding="utf-8", errors="replace") as rfile:
+                    content = [line.rstrip() + "\n" for line in rfile.readlines()]
+                with file.open("w", encoding="utf-8") as wfile:
+                    wfile.writelines(content)
+
+    return True

tools/make/stmicro.mk

@@ -46,6 +46,11 @@ download-stmicro-pdfs:
 download-stmicro-cubemx:
 	@python3 -m modm_data.dl.stmicro --directory ext/stmicro/cubemx --download cubemx --patch
 
+.PHONY: download-stmicro-cubeprog
+## Download the CubeProg database from ST's update server.
+download-stmicro-cubeprog:
+	@python3 -m modm_data.dl.stmicro --directory ext/stmicro/cubeprog --download cubeprog
+
 
 # ========================== Private Input Sources  ===========================
 # Please ping @salkinium for access.
@@ -55,6 +60,11 @@ download-stmicro-cubemx:
 ext/stmicro/cubemx/:
 	@git clone --depth=1 git@github.com:modm-ext/archive-stmicro-cubemx.git $@
 
+# Note: The STMicro CubeProg database archive repo is private and used for our CI.
+#       Please download the database via 'make download-stmicro-cubeprog'!
+ext/stmicro/cubeprog/:
+	@git clone --depth=1 git@github.com:modm-ext/archive-stmicro-cubeprog.git $@
+
 # Note: The STMicro HTML archive repo is private and used for our CI.
 #       Please convert the HTMLs via 'make convert-stmicro-html'.
 ext/stmicro/html-archive/:
@@ -66,12 +76,13 @@ ext/stmicro/pdf/:
 	@git clone --depth=1 git@github.com:modm-ext/archive-stmicro-pdf.git $@
 
 .PHONY: clone-sources-stmicro-private
-clone-sources-stmicro-private: clone-sources-stmicro ext/stmicro/cubemx/ \
+clone-sources-stmicro-private: clone-sources-stmicro ext/stmicro/cubemx/ ext/stmicro/cubeprog/ \
 							   ext/stmicro/pdf/ ext/stmicro/html-archive/
 
 .PHONY: update-sources-stmicro-private
 update-sources-stmicro-private: update-sources-stmicro
 	@(cd ext/stmicro/cubemx && git fetch && git reset --hard origin/main) &
+	@(cd ext/stmicro/cubeprog && git fetch && git reset --hard origin/main) &
 	@(cd ext/stmicro/html-archive && git fetch && git reset --hard origin/main) &
 	@(cd ext/stmicro/pdf && git fetch && git reset --hard origin/main) &
 	@wait