Merge pull request #1107 from fpistm/CLI_output

[CI] Update after arduino-cli change

Author: fpistm

.github/workflows/Continuous-Integration.yml

@@ -53,8 +53,6 @@ jobs:
     - name: Compilation
       id: Compile
       uses: stm32duino/actions/compile-examples@master
-      with:
-        cli-version: '0.10.0'
 
     # Use the output from the `Compile` step
     - name: Compilation Errors

CI/build/arduino-cli.py

@@ -4,6 +4,7 @@
 from datetime import timedelta
 import json
 import os
+from packaging import version
 import re
 import shutil
 import subprocess
@@ -52,6 +53,8 @@
 arch = arch_default
 arduino_platform = arduino_platform_default
 arduino_cli = ""
+arduino_cli_default_version = "0.10.0"
+arduino_cli_version = arduino_cli_default_version
 
 # List
 sketch_list = []
@@ -117,9 +120,8 @@ def create_output_log_tree():
         file.write(build_separator + "\n")
     # Folders
     for board in board_fqbn:
+        createFolder(os.path.join(output_dir, board, bin_dir))
         createFolder(os.path.join(output_dir, board))
-        if args.bin:
-            createFolder(os.path.join(output_dir, board, bin_dir))
         createFolder(os.path.join(build_output_dir, board))
 
 
@@ -152,6 +154,7 @@ def create_config():
 
 def check_config():
     global arduino_cli
+    global arduino_cli_version
     global arduino_cli_path
     global sketches_path_list
     global build_output_dir
@@ -193,37 +196,66 @@ def check_config():
     else:
         arduino_cli = "arduino-cli"
 
+    try:
+        output = subprocess.check_output(
+            [arduino_cli, "version"], stderr=subprocess.DEVNULL,
+        )
+    except subprocess.CalledProcessError as e:
+        print('"' + " ".join(e.cmd) + '" failed with code: {}!'.format(e.returncode))
+        print(e.stdout)
+        quit(e.returncode)
+    else:
+        res = re.match(r".*Version:\s+(\d+\.\d+\.\d+).*", output.decode("utf-8"))
+        if res:
+            arduino_cli_version = res.group(1)
+            print("Arduino CLI version used: " + arduino_cli_version)
+        else:
+            print(
+                "Unable to define Arduino CLI version, use default: "
+                + arduino_cli_default_version
+            )
+
     try:
         output = subprocess.check_output(
             [arduino_cli, "core", "search", "stm32", "--additional-urls", stm32_url],
             stderr=subprocess.DEVNULL,
         )
+    except subprocess.CalledProcessError as e:
+        print('"' + " ".join(e.cmd) + '" failed with code: {}!'.format(e.returncode))
+        print(e.stdout)
+        quit(e.returncode)
+    else:
         if arduino_platform not in output.decode("utf-8"):
-            raise subprocess.CalledProcessError(1, "re")
+            print(arduino_platform + " is not installed!")
+            quit(1)
         # Add core and library path to sketches_path_list
         try:
             output = subprocess.check_output(
                 [arduino_cli, "config", "dump", "--format", "json"],
                 stderr=subprocess.DEVNULL,
             ).decode("utf-8")
+        except subprocess.CalledProcessError as e:
+            print(
+                '"' + " ".join(e.cmd) + '" failed with code: {}!'.format(e.returncode)
+            )
+            print(e.stdout)
+            quit(e.returncode)
+        else:
             cli_config = json.loads(output)
             if cli_config is not None:
                 if cli_config["directories"]["data"] is not None:
                     sketches_path_list.append(cli_config["directories"]["data"])
                 else:
-                    raise subprocess.CalledProcessError(3, "No data directory")
+                    print("No data directory")
+                    quit(1)
                 if cli_config["directories"]["user"] is not None:
                     sketches_path_list.append(cli_config["directories"]["user"])
                 else:
-                    raise subprocess.CalledProcessError(2, "No user directory")
+                    print("No user directory!")
+                    quit(1)
             else:
-                raise subprocess.CalledProcessError(1, "No fqbn")
-        except subprocess.CalledProcessError:
-            print("No arduino-cli config!")
-            quit()
-    except subprocess.CalledProcessError:
-        print(arduino_platform + " is not installed!")
-        quit()
+                print("No arduino-cli config!")
+                quit(1)
 
 
 def load_core_config():
@@ -366,7 +398,7 @@ def manage_inos():
                     break
             else:
                 print("Sketch {} path does not exist!".format(args.ino))
-                quit()
+                quit(1)
     # Sketches listed in a file
     elif args.file:
         assert os.path.exists(args.file), "Sketches list file does not exist"
@@ -397,7 +429,7 @@ def manage_inos():
         sketch_list.append(sketch_default)
     if len(sketch_list) == 0:
         print("No sketch to build for " + arduino_platform + "!")
-        quit()
+        quit(1)
 
 
 # Find all .ino files and save directory
@@ -429,32 +461,41 @@ def find_board():
     try:
         output = subprocess.check_output(
             [arduino_cli, "board", "listall", "--format", "json"],
-            stderr=subprocess.DEVNULL,
+            stderr=subprocess.STDOUT,
         ).decode("utf-8")
+    except subprocess.CalledProcessError as e:
+        print('"' + " ".join(e.cmd) + '" failed with code: {}!'.format(e.returncode))
+        print(e.stdout)
+        quit(e.returncode)
+    else:
         boards_list = json.loads(output)
         if boards_list is not None:
             for board in boards_list["boards"]:
                 if arduino_platform in board["FQBN"]:
                     fqbn_list_tmp.append(board["FQBN"])
-            if not len(fqbn_list_tmp):
-                raise subprocess.CalledProcessError(2, "No fqbn")
-        else:
-            raise subprocess.CalledProcessError(1, "No fqbn")
-    except subprocess.CalledProcessError:
-        print("No fqbn found for " + arduino_platform + "!")
-        quit()
+        if not len(fqbn_list_tmp):
+            print("No boards found for " + arduino_platform)
+            quit(1)
 
     # For STM32 core, pnum is requested
     for fqbn in fqbn_list_tmp:
         try:
             output = subprocess.check_output(
                 [arduino_cli, "board", "details", "--format", "json", fqbn],
-                stderr=subprocess.DEVNULL,
+                stderr=subprocess.STDOUT,
             ).decode("utf-8")
+        except subprocess.CalledProcessError as e:
+            print(
+                '"' + " ".join(e.cmd) + '" failed with code: {}!'.format(e.returncode)
+            )
+            print(e.stdout)
+            quit(e.returncode)
+        else:
             board_detail = json.loads(output)
             if board_detail is not None:
                 if "config_options" not in board_detail:
-                    raise subprocess.CalledProcessError(3, "No config_options")
+                    print("No config_options found for " + fqbn)
+                    quit(1)
                 for option in board_detail["config_options"]:
                     if option["option"] == "pnum":
                         for value in option["values"]:
@@ -466,14 +507,12 @@ def find_board():
                             )
                     break
             else:
-                raise subprocess.CalledProcessError(1, "No fqbn")
-        except subprocess.CalledProcessError as e:
-            print("No fqbn detail found for " + e.cmd + "!")
+                print('No detail found for:"' + fqbn + '"!')
     if board_found:
         board_fqbn = collections.OrderedDict(sorted(board_found.items()))
     else:
         print("No board found for " + arduino_platform + "!")
-        quit()
+        quit(1)
 
 
 # Check the status
@@ -484,14 +523,12 @@ def check_status(status, build_conf, boardKo):
 
     if status[1] == 0:
         result = "\033[32msucceeded\033[0m"
-        if args.bin:
-            bin_copy(build_conf[0], sketch_name)
         nb_build_passed += 1
     elif status[1] == 1:
         # Check if failed due to a region overflowed
         logFile = os.path.join(build_conf[3], sketch_name + ".log")
         # error or fatal error
-        error_pattern = re.compile(r":\d+:\d+:\s.*error:\s")
+        error_pattern = re.compile(r":\d+:\d+:\s.*error:\s|^Error:")
         ld_pattern = re.compile("arm-none-eabi/bin/ld:")
         overflow_pattern = re.compile(
             r"(will not fit in |section .+ is not within )?region( .+ overflowed by [\d]+ bytes)?"
@@ -514,8 +551,6 @@ def check_status(status, build_conf, boardKo):
         else:
             # else consider it succeeded
             result = "\033[32msucceeded*\033[0m"
-            if args.bin:
-                empty_bin(build_conf[0], sketch_name)
             nb_build_passed += 1
     else:
         result = "\033[31merror\033[0m"
@@ -645,34 +680,6 @@ def log_final_result():
     print(output_dir)
 
 
-# Create an empty binary
-def empty_bin(board_name, sketch_name):
-    empty_path = os.path.abspath(os.path.join(output_dir, board_name, bin_dir))
-    createFolder(empty_path)
-    empty_file = os.path.join(
-        empty_path, sketch_name + "_COULD_NOT_FIT_IN_THIS_BOARD.bin"
-    )
-    try:
-        f = open(empty_file, "w")
-    except IOError:
-        print("Cannot create empty binary: ", empty_file)
-    else:
-        f.close()
-
-
-# Create a "bin" directory for each board and copy all binary files
-# from the builder output directory into it
-def bin_copy(board_name, sketch_name):
-    try:
-        shutil.copy(
-            os.path.join(build_output_dir, board_name, sketch_name + ".bin"),
-            os.path.abspath(os.path.join(output_dir, board_name, bin_dir)),
-        )
-    except OSError as e:
-        print("Cannot copy the binary from the arduino-cli output: " + e.strerror)
-        raise
-
-
 # Set up specific options to customise arduino builder command
 def get_fqbn(b_name):
     if b_name in board_custom_fqbn and board_custom_fqbn[b_name]:
@@ -696,8 +703,12 @@ def genBasicCommand(b_name):
     cmd.append(build_output_cache_dir)
     if args.verbose:
         cmd.append("--verbose")
-    cmd.append("-o")
-    cmd.append(os.path.join(build_output_dir, b_name, "sketch"))
+    if version.parse(arduino_cli_version) <= version.parse(arduino_cli_default_version):
+        cmd.append("--output")
+        cmd.append(os.path.join(output_dir, b_name, bin_dir, "dummy_sketch"))
+    else:
+        cmd.append("--output-dir")
+        cmd.append(os.path.join(output_dir, b_name, bin_dir))
     cmd.append("--fqbn")
     cmd.append(get_fqbn(b_name))
     cmd.append("dummy_sketch")
@@ -727,6 +738,9 @@ def build_config(sketch, boardSkipped):
 
     for idx in reversed(range(len(build_conf_list))):
         build_conf_list[idx][4][-1] = sketch
+        build_conf_list[idx][4][-4] = build_conf_list[idx][4][-4].replace(
+            "dummy_sketch", os.path.basename(sketch)
+        )
         if na_sketch_pattern:
             if build_conf_list[idx][0] in na_sketch_pattern:
                 for pattern in na_sketch_pattern[build_conf_list[idx][0]]:
@@ -865,7 +879,6 @@ def build(build_conf):
 )
 
 g1 = parser.add_mutually_exclusive_group()
-g1.add_argument("--bin", help="save binaries", action="store_true")
 g1.add_argument("--ci", help="custom configuration for CI build", action="store_true")
 
 # Sketch options