Merge pull request #4 from stm32duino/master

Update 13 octuber 2020

Author: sabas1080

.github/ISSUE_TEMPLATE/new-variant-request.md

@@ -18,3 +18,5 @@ assignees: ''
 **Note that user can add a STM32 based board following this [wiki](https://github.com/stm32duino/wiki/wiki/Add-a-new-variant-(board))**
 
 Any contribution is welcome, so do not hesitate to submit a PR.
+
+All new variant request will be moved to [New Variant] request list #722 and the related issue will be closed.

.github/PULL_REQUEST_TEMPLATE.md

@@ -28,14 +28,14 @@ Explain the **motivation** for making this change. What existing problem does th
 
 **Validation**
 
-* Ensure Travis CI build is passed.
+* Ensure CI build is passed.
 * Demonstrate the code is solid. [e.g. Provide a sketch]
 
-<!-- Make sure tests pass on both Travis CI. -->
+<!-- Make sure tests pass on both CI. -->
 
 **Code formatting**
 
-* Ensure AStyle check is passed thanks Travis CI (soon)
+* Ensure AStyle check is passed thanks CI
 
 <!-- See the simple style guide. -->
 

.github/actions/pio-build/entrypoint.sh

@@ -8,7 +8,7 @@ platformio platform install "https://github.com/platformio/platform-ststm32.git"
   exit 1
 }
 # Prepare framework for CI
-python3 -c "import json; import os; fp=open(os.path.expanduser('~/.platformio/platforms/ststm32/platform.json'), 'r+'); data=json.load(fp); data['packages']['framework-arduinoststm32']['version'] = '*'; fp.seek(0); fp.truncate(); json.dump(data, fp); fp.close()" || {
+python3 -c "import json; import os; fp=open(os.path.expanduser('~/.platformio/platforms/ststm32/platform.json'), 'r+'); data=json.load(fp); data['packages']['framework-arduinoststm32']['version'] = '*'; del data['packages']['framework-arduinoststm32']['owner']; fp.seek(0); fp.truncate(); json.dump(data, fp); fp.close()" || {
   exit 1
 }
 ln --symbolic "$GITHUB_WORKSPACE" "$HOME/.platformio/packages/framework-arduinoststm32" || {

.github/workflows/Continuous-Integration.yml

@@ -11,6 +11,7 @@ on:
       - '!CI/build/arduino-cli.py'
       - '!CI/build/platformio-builder.py'
       - tools/**
+      - '!tools/platformio-build.py'
   pull_request:
     paths-ignore:
       - '*.json'
@@ -20,6 +21,7 @@ on:
       - '!CI/build/arduino-cli.py'
       - '!CI/build/platformio-builder.py'
       - tools/**
+      - '!tools/platformio-build.py'
 jobs:
   astyle_check:
     runs-on: ubuntu-latest
@@ -53,6 +55,8 @@ jobs:
     - name: Compilation
       id: Compile
       uses: stm32duino/actions/compile-examples@master
+      with:
+        additional-url: 'https://github.com/stm32duino/BoardManagerFiles/raw/dev/STM32/package_stm_index.json'
 
     # Use the output from the `Compile` step
     - name: Compilation Errors

CI/build/arduino-cli.py

@@ -4,6 +4,7 @@
 from datetime import timedelta
 import json
 import os
+from packaging import version
 import re
 import shutil
 import subprocess
@@ -52,6 +53,8 @@
 arch = arch_default
 arduino_platform = arduino_platform_default
 arduino_cli = ""
+arduino_cli_default_version = "0.10.0"
+arduino_cli_version = arduino_cli_default_version
 
 # List
 sketch_list = []
@@ -117,9 +120,8 @@ def create_output_log_tree():
         file.write(build_separator + "\n")
     # Folders
     for board in board_fqbn:
+        createFolder(os.path.join(output_dir, board, bin_dir))
         createFolder(os.path.join(output_dir, board))
-        if args.bin:
-            createFolder(os.path.join(output_dir, board, bin_dir))
         createFolder(os.path.join(build_output_dir, board))
 
 
@@ -152,12 +154,15 @@ def create_config():
 
 def check_config():
     global arduino_cli
+    global arduino_cli_version
     global arduino_cli_path
     global sketches_path_list
     global build_output_dir
     global root_output_dir
     global output_dir
     global log_file
+    global stm32_url
+
     if args.ci is False:
         if os.path.isfile(path_config_filename):
             try:
@@ -193,37 +198,69 @@ def check_config():
     else:
         arduino_cli = "arduino-cli"
 
+    try:
+        output = subprocess.check_output(
+            [arduino_cli, "version"], stderr=subprocess.STDOUT,
+        )
+    except subprocess.CalledProcessError as e:
+        print('"' + " ".join(e.cmd) + '" failed with code: {}!'.format(e.returncode))
+        print(e.stdout.decode("utf-8"))
+        quit(e.returncode)
+    else:
+        res = re.match(r".*Version:\s+(\d+\.\d+\.\d+).*", output.decode("utf-8"))
+        if res:
+            arduino_cli_version = res.group(1)
+            print("Arduino CLI version used: " + arduino_cli_version)
+        else:
+            print(
+                "Unable to define Arduino CLI version, use default: "
+                + arduino_cli_default_version
+            )
+
+    if args.url:
+        stm32_url = args.url
+
     try:
         output = subprocess.check_output(
             [arduino_cli, "core", "search", "stm32", "--additional-urls", stm32_url],
-            stderr=subprocess.DEVNULL,
+            stderr=subprocess.STDOUT,
         )
+    except subprocess.CalledProcessError as e:
+        print('"' + " ".join(e.cmd) + '" failed with code: {}!'.format(e.returncode))
+        print(e.stdout.decode("utf-8"))
+        quit(e.returncode)
+    else:
         if arduino_platform not in output.decode("utf-8"):
-            raise subprocess.CalledProcessError(1, "re")
+            print(arduino_platform + " is not installed!")
+            quit(1)
         # Add core and library path to sketches_path_list
         try:
             output = subprocess.check_output(
                 [arduino_cli, "config", "dump", "--format", "json"],
-                stderr=subprocess.DEVNULL,
+                stderr=subprocess.STDOUT,
             ).decode("utf-8")
+        except subprocess.CalledProcessError as e:
+            print(
+                '"' + " ".join(e.cmd) + '" failed with code: {}!'.format(e.returncode)
+            )
+            print(e.stdout.decode("utf-8"))
+            quit(e.returncode)
+        else:
             cli_config = json.loads(output)
             if cli_config is not None:
                 if cli_config["directories"]["data"] is not None:
                     sketches_path_list.append(cli_config["directories"]["data"])
                 else:
-                    raise subprocess.CalledProcessError(3, "No data directory")
+                    print("No data directory")
+                    quit(1)
                 if cli_config["directories"]["user"] is not None:
                     sketches_path_list.append(cli_config["directories"]["user"])
                 else:
-                    raise subprocess.CalledProcessError(2, "No user directory")
+                    print("No user directory!")
+                    quit(1)
             else:
-                raise subprocess.CalledProcessError(1, "No fqbn")
-        except subprocess.CalledProcessError:
-            print("No arduino-cli config!")
-            quit()
-    except subprocess.CalledProcessError:
-        print(arduino_platform + " is not installed!")
-        quit()
+                print("No arduino-cli config!")
+                quit(1)
 
 
 def load_core_config():
@@ -366,7 +403,7 @@ def manage_inos():
                     break
             else:
                 print("Sketch {} path does not exist!".format(args.ino))
-                quit()
+                quit(1)
     # Sketches listed in a file
     elif args.file:
         assert os.path.exists(args.file), "Sketches list file does not exist"
@@ -397,7 +434,7 @@ def manage_inos():
         sketch_list.append(sketch_default)
     if len(sketch_list) == 0:
         print("No sketch to build for " + arduino_platform + "!")
-        quit()
+        quit(1)
 
 
 # Find all .ino files and save directory
@@ -429,32 +466,41 @@ def find_board():
     try:
         output = subprocess.check_output(
             [arduino_cli, "board", "listall", "--format", "json"],
-            stderr=subprocess.DEVNULL,
+            stderr=subprocess.STDOUT,
         ).decode("utf-8")
+    except subprocess.CalledProcessError as e:
+        print('"' + " ".join(e.cmd) + '" failed with code: {}!'.format(e.returncode))
+        print(e.stdout.decode("utf-8"))
+        quit(e.returncode)
+    else:
         boards_list = json.loads(output)
         if boards_list is not None:
             for board in boards_list["boards"]:
                 if arduino_platform in board["FQBN"]:
                     fqbn_list_tmp.append(board["FQBN"])
-            if not len(fqbn_list_tmp):
-                raise subprocess.CalledProcessError(2, "No fqbn")
-        else:
-            raise subprocess.CalledProcessError(1, "No fqbn")
-    except subprocess.CalledProcessError:
-        print("No fqbn found for " + arduino_platform + "!")
-        quit()
+        if not len(fqbn_list_tmp):
+            print("No boards found for " + arduino_platform)
+            quit(1)
 
     # For STM32 core, pnum is requested
     for fqbn in fqbn_list_tmp:
         try:
             output = subprocess.check_output(
                 [arduino_cli, "board", "details", "--format", "json", fqbn],
-                stderr=subprocess.DEVNULL,
+                stderr=subprocess.STDOUT,
             ).decode("utf-8")
+        except subprocess.CalledProcessError as e:
+            print(
+                '"' + " ".join(e.cmd) + '" failed with code: {}!'.format(e.returncode)
+            )
+            print(e.stdout.decode("utf-8"))
+            quit(e.returncode)
+        else:
             board_detail = json.loads(output)
             if board_detail is not None:
                 if "config_options" not in board_detail:
-                    raise subprocess.CalledProcessError(3, "No config_options")
+                    print("No config_options found for " + fqbn)
+                    quit(1)
                 for option in board_detail["config_options"]:
                     if option["option"] == "pnum":
                         for value in option["values"]:
@@ -466,14 +512,12 @@ def find_board():
                             )
                     break
             else:
-                raise subprocess.CalledProcessError(1, "No fqbn")
-        except subprocess.CalledProcessError as e:
-            print("No fqbn detail found for " + e.cmd + "!")
+                print('No detail found for:"' + fqbn + '"!')
     if board_found:
         board_fqbn = collections.OrderedDict(sorted(board_found.items()))
     else:
         print("No board found for " + arduino_platform + "!")
-        quit()
+        quit(1)
 
 
 # Check the status
@@ -484,14 +528,13 @@ def check_status(status, build_conf, boardKo):
 
     if status[1] == 0:
         result = "\033[32msucceeded\033[0m"
-        if args.bin:
-            bin_copy(build_conf[0], sketch_name)
         nb_build_passed += 1
     elif status[1] == 1:
         # Check if failed due to a region overflowed
         logFile = os.path.join(build_conf[3], sketch_name + ".log")
         # error or fatal error
-        error_pattern = re.compile(r":\d+:\d+:\s.*error:\s")
+        fork_pattern = re.compile(r"^Error during build: fork/exec")
+        error_pattern = re.compile(r":\d+:\d+:\s.*error:\s|^Error:")
         ld_pattern = re.compile("arm-none-eabi/bin/ld:")
         overflow_pattern = re.compile(
             r"(will not fit in |section .+ is not within )?region( .+ overflowed by [\d]+ bytes)?"
@@ -500,6 +543,8 @@ def check_status(status, build_conf, boardKo):
         for i, line in enumerate(open(logFile)):
             if error_pattern.search(line):
                 error_found = True
+            elif fork_pattern.search(line):
+                error_found = True
             elif ld_pattern.search(line):
                 # If one ld line is not for region overflowed --> failed
                 if overflow_pattern.search(line) is None:
@@ -514,8 +559,6 @@ def check_status(status, build_conf, boardKo):
         else:
             # else consider it succeeded
             result = "\033[32msucceeded*\033[0m"
-            if args.bin:
-                empty_bin(build_conf[0], sketch_name)
             nb_build_passed += 1
     else:
         result = "\033[31merror\033[0m"
@@ -645,34 +688,6 @@ def log_final_result():
     print(output_dir)
 
 
-# Create an empty binary
-def empty_bin(board_name, sketch_name):
-    empty_path = os.path.abspath(os.path.join(output_dir, board_name, bin_dir))
-    createFolder(empty_path)
-    empty_file = os.path.join(
-        empty_path, sketch_name + "_COULD_NOT_FIT_IN_THIS_BOARD.bin"
-    )
-    try:
-        f = open(empty_file, "w")
-    except IOError:
-        print("Cannot create empty binary: ", empty_file)
-    else:
-        f.close()
-
-
-# Create a "bin" directory for each board and copy all binary files
-# from the builder output directory into it
-def bin_copy(board_name, sketch_name):
-    try:
-        shutil.copy(
-            os.path.join(build_output_dir, board_name, sketch_name + ".bin"),
-            os.path.abspath(os.path.join(output_dir, board_name, bin_dir)),
-        )
-    except OSError as e:
-        print("Cannot copy the binary from the arduino-cli output: " + e.strerror)
-        raise
-
-
 # Set up specific options to customise arduino builder command
 def get_fqbn(b_name):
     if b_name in board_custom_fqbn and board_custom_fqbn[b_name]:
@@ -696,8 +711,12 @@ def genBasicCommand(b_name):
     cmd.append(build_output_cache_dir)
     if args.verbose:
         cmd.append("--verbose")
-    cmd.append("-o")
-    cmd.append(os.path.join(build_output_dir, b_name, "sketch"))
+    if version.parse(arduino_cli_version) <= version.parse(arduino_cli_default_version):
+        cmd.append("--output")
+        cmd.append(os.path.join(output_dir, b_name, bin_dir, "dummy_sketch"))
+    else:
+        cmd.append("--output-dir")
+        cmd.append(os.path.join(output_dir, b_name, bin_dir))
     cmd.append("--fqbn")
     cmd.append(get_fqbn(b_name))
     cmd.append("dummy_sketch")
@@ -727,6 +746,9 @@ def build_config(sketch, boardSkipped):
 
     for idx in reversed(range(len(build_conf_list))):
         build_conf_list[idx][4][-1] = sketch
+        build_conf_list[idx][4][-4] = build_conf_list[idx][4][-4].replace(
+            "dummy_sketch", os.path.basename(sketch)
+        )
         if na_sketch_pattern:
             if build_conf_list[idx][0] in na_sketch_pattern:
                 for pattern in na_sketch_pattern[build_conf_list[idx][0]]:
@@ -860,12 +882,17 @@ def build(build_conf):
     + cores_config_file_default,
 )
 
+parser.add_argument(
+    "-u", "--url", metavar="<string>", help="additional URL for the board manager\
+    Default url : "
+    + stm32_url,
+)
+
 parser.add_argument(
     "-v", "--verbose", help="enable arduino-cli verbose mode", action="store_true"
 )
 
 g1 = parser.add_mutually_exclusive_group()
-g1.add_argument("--bin", help="save binaries", action="store_true")
 g1.add_argument("--ci", help="custom configuration for CI build", action="store_true")
 
 # Sketch options