Check json logs entries in compile and upload python integration tests (#393)

* add whitebox testing to combo compile upload command integ test

* add log lines to compile and upload in case of success and related tests

* check log traces for exact sequences

* add virtualenv folder linting exclude and licence header cosmetics

Author: Roberto Sora

commands/compile/compile.go

@@ -47,7 +47,7 @@ func Compile(ctx context.Context, req *rpc.CompileReq, outStream, errStream io.W
 		return nil, errors.New("invalid instance")
 	}
 
-	logrus.Info("Executing `arduino compile`")
+	logrus.Tracef("Compile %s for %s started", req.GetSketchPath(), req.GetFqbn())
 	if req.GetSketchPath() == "" {
 		return nil, fmt.Errorf("missing sketchPath")
 	}
@@ -202,18 +202,20 @@ func Compile(ctx context.Context, req *rpc.CompileReq, outStream, errStream io.W
 	// Copy .hex file to sketch directory
 	srcHex := paths.New(outputPath)
 	dstHex := exportPath.Join(exportFile + ext)
-	logrus.WithField("from", srcHex).WithField("to", dstHex).Print("copying sketch build output")
+	logrus.WithField("from", srcHex).WithField("to", dstHex).Debug("copying sketch build output")
 	if err = srcHex.CopyTo(dstHex); err != nil {
 		return nil, fmt.Errorf("copying output file: %s", err)
 	}
 
 	// Copy .elf file to sketch directory
 	srcElf := paths.New(outputPath[:len(outputPath)-3] + "elf")
 	dstElf := exportPath.Join(exportFile + ".elf")
-	logrus.WithField("from", srcElf).WithField("to", dstElf).Print("copying sketch build output")
+	logrus.WithField("from", srcElf).WithField("to", dstElf).Debug("copying sketch build output")
 	if err = srcElf.CopyTo(dstElf); err != nil {
 		return nil, fmt.Errorf("copying elf file: %s", err)
 	}
 
+	logrus.Tracef("Compile %s for %s successful", sketch.Name, fqbnIn)
+
 	return &rpc.CompileResp{}, nil
 }

commands/upload/upload.go

@@ -40,7 +40,7 @@ import (
 
 // Upload FIXMEDOC
 func Upload(ctx context.Context, req *rpc.UploadReq, outStream io.Writer, errStream io.Writer) (*rpc.UploadResp, error) {
-	logrus.Info("Executing `arduino upload`")
+	logrus.Tracef("Upload %s on %s started", req.GetSketchPath(), req.GetFqbn())
 
 	// TODO: make a generic function to extract sketch from request
 	// and remove duplication in commands/compile.go
@@ -248,6 +248,9 @@ func Upload(ctx context.Context, req *rpc.UploadReq, outStream io.Writer, errStr
 	if err := cmd.Wait(); err != nil {
 		return nil, fmt.Errorf("uploading error: %s", err)
 	}
+
+	logrus.Tracef("Upload %s on %s successful", sketch.Name, fqbnIn)
+
 	return &rpc.UploadResp{}, nil
 }
 

test/.flake8

@@ -1,2 +1,3 @@
 [flake8]
-max-line-length = 120
+exclude = venv
+max-line-length = 120

test/__init__.py

@@ -0,0 +1,14 @@
+# This file is part of arduino-cli.
+#
+# Copyright 2019 ARDUINO SA (http://www.arduino.cc/)
+#
+# This software is released under the GNU General Public License version 3,
+# which covers the main part of arduino-cli.
+# The terms of this license can be found at:
+# https://www.gnu.org/licenses/gpl-3.0.en.html
+#
+# You can be released from the requirements of the above licenses by purchasing
+# a commercial license. Buying such a license is mandatory if you want to modify or
+# otherwise use the software for commercial activities involving the Arduino
+# software without disclosing the source code of your own applications. To purchase
+# a commercial license, send an email to license@arduino.cc.

test/common.py

@@ -1,12 +1,12 @@
 # This file is part of arduino-cli.
-
+#
 # Copyright 2019 ARDUINO SA (http://www.arduino.cc/)
-
+#
 # This software is released under the GNU General Public License version 3,
 # which covers the main part of arduino-cli.
 # The terms of this license can be found at:
 # https://www.gnu.org/licenses/gpl-3.0.en.html
-
+#
 # You can be released from the requirements of the above licenses by purchasing
 # a commercial license. Buying such a license is mandatory if you want to modify or
 # otherwise use the software for commercial activities involving the Arduino

test/conftest.py

@@ -1,12 +1,12 @@
 # This file is part of arduino-cli.
-
+#
 # Copyright 2019 ARDUINO SA (http://www.arduino.cc/)
-
+#
 # This software is released under the GNU General Public License version 3,
 # which covers the main part of arduino-cli.
 # The terms of this license can be found at:
 # https://www.gnu.org/licenses/gpl-3.0.en.html
-
+#
 # You can be released from the requirements of the above licenses by purchasing
 # a commercial license. Buying such a license is mandatory if you want to modify or
 # otherwise use the software for commercial activities involving the Arduino

test/test_board.py

@@ -1,12 +1,12 @@
 # This file is part of arduino-cli.
-
+#
 # Copyright 2019 ARDUINO SA (http://www.arduino.cc/)
-
+#
 # This software is released under the GNU General Public License version 3,
 # which covers the main part of arduino-cli.
 # The terms of this license can be found at:
 # https://www.gnu.org/licenses/gpl-3.0.en.html
-
+#
 # You can be released from the requirements of the above licenses by purchasing
 # a commercial license. Buying such a license is mandatory if you want to modify or
 # otherwise use the software for commercial activities involving the Arduino

test/test_compile.py

@@ -1,20 +1,20 @@
 # This file is part of arduino-cli.
-
+#
 # Copyright 2019 ARDUINO SA (http://www.arduino.cc/)
-
+#
 # This software is released under the GNU General Public License version 3,
 # which covers the main part of arduino-cli.
 # The terms of this license can be found at:
 # https://www.gnu.org/licenses/gpl-3.0.en.html
-
+#
 # You can be released from the requirements of the above licenses by purchasing
 # a commercial license. Buying such a license is mandatory if you want to modify or
 # otherwise use the software for commercial activities involving the Arduino
 # software without disclosing the source code of your own applications. To purchase
 # a commercial license, send an email to license@arduino.cc.
-import pytest
 import json
 import os
+import pytest
 
 from .common import running_on_ci
 
@@ -42,33 +42,47 @@ def test_compile_with_simple_sketch(run_command, data_dir):
     result = run_command("core install arduino:avr")
     assert result.ok
 
-    sketch_path = os.path.join(data_dir, "CompileIntegrationTest")
+    sketch_name = "CompileIntegrationTest"
+    sketch_path = os.path.join(data_dir, sketch_name)
+    fqbn = "arduino:avr:uno"
 
     # Create a test sketch
-    result = run_command("sketch new CompileIntegrationTest")
+    result = run_command("sketch new {}".format(sketch_name))
     assert result.ok
     assert "Sketch created in: {}".format(sketch_path) in result.stdout
 
     # Build sketch for arduino:avr:uno
-    result = run_command("compile -b arduino:avr:uno {}".format(sketch_path))
+    log_file_name = "compile.log"
+    log_file_path = os.path.join(data_dir, log_file_name)
+    result = run_command(
+        "compile -b {fqbn} {sketch_path} --log-format json --log-file {log_file} --log-level trace".format(
+            fqbn=fqbn, sketch_path=sketch_path, log_file=log_file_path))
     assert result.ok
-    assert "Sketch uses" in result.stdout
+
+    # let's test from the logs if the hex file produced by successful compile is moved to our sketch folder
+    log_json = open(log_file_path, 'r')
+    json_log_lines = log_json.readlines()
+    expected_trace_sequence = [
+        "Compile {sketch} for {fqbn} started".format(sketch=sketch_path, fqbn=fqbn),
+        "Compile {sketch} for {fqbn} successful".format(sketch=sketch_name, fqbn=fqbn)
+    ]
+    assert is_message_sequence_in_json_log_traces(expected_trace_sequence, json_log_lines)
 
 
 @pytest.mark.skipif(running_on_ci(), reason="VMs have no serial ports")
 def test_compile_and_compile_combo(run_command, data_dir):
-
     # Init the environment explicitly
     result = run_command("core update-index")
     assert result.ok
 
     # Install required core(s)
     result = run_command("core install arduino:avr")
-    # result = run_command("core install arduino:samd")
+    result = run_command("core install arduino:samd")
     assert result.ok
 
     # Create a test sketch
-    sketch_path = os.path.join(data_dir, "CompileAndUploadIntegrationTest")
+    sketch_name = "CompileAndUploadIntegrationTest"
+    sketch_path = os.path.join(data_dir, sketch_name)
     result = run_command("sketch new CompileAndUploadIntegrationTest")
     assert result.ok
     assert "Sketch created in: {}".format(sketch_path) in result.stdout
@@ -96,7 +110,7 @@ def test_compile_and_compile_combo(run_command, data_dir):
     #     }
     #   ]
 
-    detected_boards = []    
+    detected_boards = []
 
     ports = json.loads(result.stdout)
     assert isinstance(ports, list)
@@ -109,12 +123,34 @@ def test_compile_and_compile_combo(run_command, data_dir):
     assert len(detected_boards) >= 1, "There are no boards available for testing"
 
     # Build sketch for each detected board
-    for board in detected_boards:  
-        result = run_command(
-            "compile -b {fqbn} --upload -p {address} {sketch_path}".format(
-                fqbn=board.get('fqbn'),
-                address=board.get('address'),
-                sketch_path=sketch_path)
-        )
+    for board in detected_boards:
+        log_file_name = "{fqbn}-compile.log".format(fqbn=board.get('fqbn'))
+        log_file_path = os.path.join(data_dir, log_file_name)
+        command_log_flags = "--log-format json --log-file {} --log-level trace".format(log_file_path)
+        result = run_command("compile -b {fqbn} --upload -p {address} {sketch_path} {log_flags}".format(
+            fqbn=board.get('fqbn'),
+            address=board.get('address'),
+            sketch_path=sketch_path,
+            log_flags=command_log_flags
+        ))
         assert result.ok
-        assert "Verify successful" in result.stdout
+        # check from the logs if the bin file were uploaded on the current board
+        log_json = open(log_file_path, 'r')
+        json_log_lines = log_json.readlines()
+        expected_trace_sequence = [
+            "Compile {sketch} for {fqbn} started".format(sketch=sketch_path, fqbn=board.get('fqbn')),
+            "Compile {sketch} for {fqbn} successful".format(sketch=sketch_name, fqbn=board.get('fqbn')),
+            "Upload {sketch} on {fqbn} started".format(sketch=sketch_path, fqbn=board.get('fqbn')),
+            "Upload {sketch} on {fqbn} successful".format(sketch=sketch_name, fqbn=board.get('fqbn'))
+        ]
+        assert is_message_sequence_in_json_log_traces(expected_trace_sequence, json_log_lines)
+
+
+def is_message_sequence_in_json_log_traces(message_sequence, log_json_lines):
+    trace_entries = []
+    for entry in log_json_lines:
+        entry = json.loads(entry)
+        if entry.get("level") == "trace":
+            if entry.get("msg") in message_sequence:
+                trace_entries.append(entry.get("msg"))
+    return message_sequence == trace_entries

test/test_lib.py

@@ -1,12 +1,12 @@
 # This file is part of arduino-cli.
-
+#
 # Copyright 2019 ARDUINO SA (http://www.arduino.cc/)
-
+#
 # This software is released under the GNU General Public License version 3,
 # which covers the main part of arduino-cli.
 # The terms of this license can be found at:
 # https://www.gnu.org/licenses/gpl-3.0.en.html
-
+#
 # You can be released from the requirements of the above licenses by purchasing
 # a commercial license. Buying such a license is mandatory if you want to modify or
 # otherwise use the software for commercial activities involving the Arduino

test/test_main.py

@@ -1,12 +1,12 @@
 # This file is part of arduino-cli.
-
+#
 # Copyright 2019 ARDUINO SA (http://www.arduino.cc/)
-
+#
 # This software is released under the GNU General Public License version 3,
 # which covers the main part of arduino-cli.
 # The terms of this license can be found at:
 # https://www.gnu.org/licenses/gpl-3.0.en.html
-
+#
 # You can be released from the requirements of the above licenses by purchasing
 # a commercial license. Buying such a license is mandatory if you want to modify or
 # otherwise use the software for commercial activities involving the Arduino