Add testing for telemetry deaemon and repertory

Author: Roberto Sora

cli/daemon/daemon.go

@@ -18,6 +18,7 @@ package daemon
 import (
 	"errors"
 	"fmt"
+	"github.com/arduino/arduino-cli/telemetry"
 	"github.com/segmentio/stats/v4"
 	"io"
 	"io/ioutil"
@@ -35,7 +36,6 @@ import (
 	srv_debug "github.com/arduino/arduino-cli/rpc/debug"
 	srv_monitor "github.com/arduino/arduino-cli/rpc/monitor"
 	srv_settings "github.com/arduino/arduino-cli/rpc/settings"
-	"github.com/arduino/arduino-cli/telemetry"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
@@ -64,6 +64,7 @@ func runDaemonCommand(cmd *cobra.Command, args []string) {
 
 	if viper.GetBool("telemetry.enabled") {
 		telemetry.Activate("daemon")
+		stats.Incr("daemon", stats.T("success", "true"))
 		defer stats.Flush()
 	}
 

configuration/defaults.go

@@ -38,7 +38,7 @@ func setDefaults(dataDir, userDir string) {
 
 	//telemetry settings
 	viper.SetDefault("telemetry.enabled", true)
-	viper.SetDefault("telemetry.addr", ":2112")
+	viper.SetDefault("telemetry.addr", ":9090")
 	viper.SetDefault("telemetry.pattern", "/metrics")
 
 }

test/requirements.in

@@ -3,5 +3,7 @@ simplejson==3.17.0
 semver==2.9.0
 pyserial==3.4
 pyyaml==5.3
+prometheus-client==0.7.1
+requests==2.22.0
 # temporary, replaces invoke==1.3.0 in favour of https://github.com/pyinvoke/invoke/pull/661
 git+https://github.com/flazzarini/invoke.git

test/requirements.txt

@@ -5,18 +5,24 @@
 #    pip-compile test/requirements.in
 #
 attrs==19.1.0             # via pytest
+certifi==2019.11.28       # via requests
+chardet==3.0.4            # via requests
+idna==2.8                 # via requests
 importlib-metadata==1.5.0  # via pluggy, pytest
 git+https://github.com/flazzarini/invoke.git
 more-itertools==7.1.0     # via pytest
 packaging==19.0           # via pytest
 pluggy==0.13.1            # via pytest
+prometheus-client==0.7.1
 py==1.8.0                 # via pytest
 pyparsing==2.4.0          # via packaging
 pyserial==3.4
 pytest==5.3.4
 pyyaml==5.3
+requests==2.22.0
 semver==2.9.0
 simplejson==3.17.0
 six==1.12.0               # via packaging
+urllib3==1.25.8           # via requests
 wcwidth==0.1.7            # via pytest
 zipp==2.1.0               # via importlib-metadata

test/test_daemon.py

@@ -12,265 +12,36 @@
 # otherwise use the software for commercial activities involving the Arduino
 # software without disclosing the source code of your own applications. To purchase
 # a commercial license, send an email to license@arduino.cc.
-import json
-import os
-import platform
-
-import pytest
-
-from .common import running_on_ci
-
-
-def test_compile_without_fqbn(run_command):
-    # Init the environment explicitly
-    result = run_command("core update-index")
-    assert result.ok
-
-    # Download latest AVR
-    result = run_command("core install arduino:avr")
-    assert result.ok
-
-    # Build sketch without FQBN
-    result = run_command("compile")
-    assert result.failed
-
-
-def test_compile_with_simple_sketch(run_command, data_dir, working_dir):
-    # Init the environment explicitly
-    result = run_command("core update-index")
-    assert result.ok
-
-    # Download latest AVR
-    result = run_command("core install arduino:avr")
-    assert result.ok
-
-    sketch_name = "CompileIntegrationTest"
-    sketch_path = os.path.join(data_dir, sketch_name)
-    fqbn = "arduino:avr:uno"
-
-    # Create a test sketch
-    result = run_command("sketch new {}".format(sketch_path))
-    assert result.ok
-    assert "Sketch created in: {}".format(sketch_path) in result.stdout
-
-    # Build sketch for arduino:avr:uno
-    log_file_name = "compile.log"
-    log_file_path = os.path.join(data_dir, log_file_name)
-    result = run_command(
-        "compile -b {fqbn} {sketch_path} --log-format json --log-file {log_file} --log-level trace".format(
-            fqbn=fqbn, sketch_path=sketch_path, log_file=log_file_path
-        )
-    )
-    assert result.ok
-
-    # let's test from the logs if the hex file produced by successful compile is moved to our sketch folder
-    log_json = open(log_file_path, "r")
-    json_log_lines = log_json.readlines()
-    expected_trace_sequence = [
-        "Compile {sketch} for {fqbn} started".format(sketch=sketch_path, fqbn=fqbn),
-        "Compile {sketch} for {fqbn} successful".format(sketch=sketch_name, fqbn=fqbn),
-    ]
-    assert is_message_sequence_in_json_log_traces(
-        expected_trace_sequence, json_log_lines
-    )
-
-    # Test the --output flag with absolute path
-    target = os.path.join(data_dir, "test.hex")
-    result = run_command(
-        "compile -b {fqbn} {sketch_path} -o {target}".format(
-            fqbn=fqbn, sketch_path=sketch_path, target=target
-        )
-    )
-    assert result.ok
-    assert os.path.exists(target)
-
-
-@pytest.mark.skipif(
-    running_on_ci() and platform.system() == "Windows",
-    reason="Test disabled on Github Actions Win VM until tmpdir inconsistent behavior bug is fixed",
-)
-def test_output_flag_default_path(run_command, data_dir, working_dir):
-    # Init the environment explicitly
-    result = run_command("core update-index")
-    assert result.ok
-
-    # Download latest AVR
-    result = run_command("core install arduino:avr")
-    assert result.ok
-
-    # Create a test sketch
-    sketch_path = os.path.join(data_dir, "test_output_flag_default_path")
-    fqbn = "arduino:avr:uno"
-    result = run_command("sketch new {}".format(sketch_path))
-    assert result.ok
-
-    # Test the --output flag defaulting to current working dir
-    result = run_command(
-        "compile -b {fqbn} {sketch_path} -o test".format(
-            fqbn=fqbn, sketch_path=sketch_path
-        )
-    )
-    assert result.ok
-    assert os.path.exists(os.path.join(working_dir, "test.hex"))
-
-    # Test extension won't be added if already present
-    result = run_command(
-        "compile -b {fqbn} {sketch_path} -o test2.hex".format(
-            fqbn=fqbn, sketch_path=sketch_path
-        )
-    )
-    assert result.ok
-    assert os.path.exists(os.path.join(working_dir, "test2.hex"))
-
-
-def test_compile_with_sketch_with_symlink_selfloop(run_command, data_dir):
-    # Init the environment explicitly
-    result = run_command("core update-index")
-    assert result.ok
-
-    # Download latest AVR
-    result = run_command("core install arduino:avr")
-    assert result.ok
-
-    sketch_name = "CompileIntegrationTestSymlinkSelfLoop"
-    sketch_path = os.path.join(data_dir, sketch_name)
-    fqbn = "arduino:avr:uno"
-
-    # Create a test sketch
-    result = run_command("sketch new {}".format(sketch_path))
-    assert result.ok
-    assert "Sketch created in: {}".format(sketch_path) in result.stdout
-
-    # create a symlink that loops on himself
-    loop_file_path = os.path.join(sketch_path, "loop")
-    os.symlink(loop_file_path, loop_file_path)
-
-    # Build sketch for arduino:avr:uno
-    result = run_command(
-        "compile -b {fqbn} {sketch_path}".format(fqbn=fqbn, sketch_path=sketch_path)
-    )
-    # The assertion is a bit relaxed in this case because win behaves differently from macOs and linux
-    # returning a different error detailed message
-    assert "Error during sketch processing" in result.stderr
-    assert not result.ok
-
-    sketch_name = "CompileIntegrationTestSymlinkDirLoop"
-    sketch_path = os.path.join(data_dir, sketch_name)
-    fqbn = "arduino:avr:uno"
-
-    # Create a test sketch
-    result = run_command("sketch new {}".format(sketch_path))
-    assert result.ok
-    assert "Sketch created in: {}".format(sketch_path) in result.stdout
-
-    # create a symlink that loops on the upper level
-    loop_dir_path = os.path.join(sketch_path, "loop_dir")
-    os.mkdir(loop_dir_path)
-    loop_dir_symlink_path = os.path.join(loop_dir_path, "loop_dir_symlink")
-    os.symlink(loop_dir_path, loop_dir_symlink_path)
-
-    # Build sketch for arduino:avr:uno
-    result = run_command(
-        "compile -b {fqbn} {sketch_path}".format(fqbn=fqbn, sketch_path=sketch_path)
-    )
-    # The assertion is a bit relaxed also in this case because macOS behaves differently from win and linux:
-    # the cli does not follow recursively the symlink til breaking
-    assert "Error during sketch processing" in result.stderr
-    assert not result.ok
-
-
-@pytest.mark.skipif(running_on_ci(), reason="VMs have no serial ports")
-def test_compile_and_compile_combo(run_command, data_dir, detected_boards):
-    # Init the environment explicitly
-    result = run_command("core update-index")
-    assert result.ok
-
-    # Install required core(s)
-    result = run_command("core install arduino:avr")
-    result = run_command("core install arduino:samd")
-    assert result.ok
-
-    # Create a test sketch
-    sketch_name = "CompileAndUploadIntegrationTest"
-    sketch_path = os.path.join(data_dir, sketch_name)
-    result = run_command("sketch new {}".format(sketch_path))
-    assert result.ok
-    assert "Sketch created in: {}".format(sketch_path) in result.stdout
-
-    # Build sketch for each detected board
-    for board in detected_boards:
-        log_file_name = "{fqbn}-compile.log".format(fqbn=board.fqbn.replace(":", "-"))
-        log_file_path = os.path.join(data_dir, log_file_name)
-        command_log_flags = "--log-format json --log-file {} --log-level trace".format(
-            log_file_path
-        )
-        result = run_command(
-            "compile -b {fqbn} --upload -p {address} {sketch_path} {log_flags}".format(
-                fqbn=board.fqbn,
-                address=board.address,
-                sketch_path=sketch_path,
-                log_flags=command_log_flags,
-            )
-        )
-        assert result.ok
-        # check from the logs if the bin file were uploaded on the current board
-        log_json = open(log_file_path, "r")
-        json_log_lines = log_json.readlines()
-        expected_trace_sequence = [
-            "Compile {sketch} for {fqbn} started".format(
-                sketch=sketch_path, fqbn=board.fqbn
-            ),
-            "Compile {sketch} for {fqbn} successful".format(
-                sketch=sketch_name, fqbn=board.fqbn
-            ),
-            "Upload {sketch} on {fqbn} started".format(
-                sketch=sketch_path, fqbn=board.fqbn
-            ),
-            "Upload {sketch} on {fqbn} successful".format(
-                sketch=sketch_name, fqbn=board.fqbn
-            ),
-        ]
-        assert is_message_sequence_in_json_log_traces(
-            expected_trace_sequence, json_log_lines
-        )
 
+import os
+import subprocess
+import time
 
-def is_message_sequence_in_json_log_traces(message_sequence, log_json_lines):
-    trace_entries = []
-    for entry in log_json_lines:
-        entry = json.loads(entry)
-        if entry.get("level") == "trace":
-            if entry.get("msg") in message_sequence:
-                trace_entries.append(entry.get("msg"))
-    return message_sequence == trace_entries
+import requests
+import yaml
+from prometheus_client.parser import text_string_to_metric_families
 
 
-def test_compile_blacklisted_sketchname(run_command, data_dir):
-    """
-    Compile should ignore folders named `RCS`, `.git` and the likes, but
-    it should be ok for a sketch to be named like RCS.ino
-    """
+def test_telemetry_prometheus_endpoint(pytestconfig, data_dir, downloads_dir):
     # Init the environment explicitly
-    result = run_command("core update-index")
-    assert result.ok
-
-    # Download latest AVR
-    result = run_command("core install arduino:avr")
-    assert result.ok
-
-    sketch_name = "RCS"
-    sketch_path = os.path.join(data_dir, sketch_name)
-    fqbn = "arduino:avr:uno"
-
-    # Create a test sketch
-    result = run_command("sketch new {}".format(sketch_path))
-    assert result.ok
-    assert "Sketch created in: {}".format(sketch_path) in result.stdout
-
-    # Build sketch for arduino:avr:uno
-    log_file_name = "compile.log"
-    log_file_path = os.path.join(data_dir, log_file_name)
-    result = run_command(
-        "compile -b {fqbn} {sketch_path}".format(fqbn=fqbn, sketch_path=sketch_path)
-    )
-    assert result.ok
+    cli_path = os.path.join(str(pytestconfig.rootdir), "..", "arduino-cli")
+    env = os.environ.copy()
+    env["ARDUINO_DATA_DIR"] = data_dir
+    env["ARDUINO_DOWNLOADS_DIR"] = downloads_dir
+    env["ARDUINO_SKETCHBOOK_DIR"] = data_dir
+
+    subprocess.Popen([cli_path, "daemon"], env=env)
+
+    time.sleep(5)
+
+    # parse repertory file
+    repertory_file = os.path.join(data_dir, "repertory.yaml")
+    with open(repertory_file, 'r') as stream:
+        repertory = yaml.safe_load(stream)
+
+        # Check if :9090/metrics endpoint is alive,
+        # telemetry is enabled by default in daemon mode
+        metrics = requests.get("http://localhost:9090/metrics").text
+        family = next(text_string_to_metric_families(metrics))
+        sample = family.samples[0]
+        assert repertory["installation"]["id"] == sample.labels["installationID"]

test/test_main.py

@@ -84,7 +84,7 @@ def test_repertory_creation(run_command, data_dir):
     out_lines = run_command("version").stdout.strip().split("\n")
     assert len(out_lines) == 1
 
-    # json on file
+    # parse repertory file
     repertory_file = os.path.join(data_dir, "repertory.yaml")
     with open(repertory_file, 'r') as stream:
         repertory = yaml.safe_load(stream)