Device tests updates (#9246)

Stop pyexpect from closing serial port when running with WSL2
Remove makefile flags & opts making it too silent. At some point it makes a lot of useful info just disappear. There are external means of silencing make, so probably best to leave useful data always available

Replace SPIFFS with LittleFS in FS test
Re-use certs-from-mozilla.py in FS test

Fix libc tests w/o -fno-builtin not actually testing anything, also add the flag via .globals.h in case it breaks
Fix libc tests generating warnings when using invalid (but safe) size arguments

Author: mcspr

libraries/ESP8266WiFi/examples/BearSSL_CertStore/certs-from-mozilla.py

@@ -13,15 +13,9 @@
 import sys
 from shutil import which
 
-from subprocess import Popen, PIPE, call
-try:
-    from urllib.request import urlopen
-except Exception:
-    from urllib2 import urlopen
-try:
-    from StringIO import StringIO
-except Exception:
-    from io import StringIO
+from io import StringIO
+from subprocess import Popen, PIPE, call, CalledProcessError
+from urllib.request import urlopen
 
 # check if ar and openssl are available
 if which('ar') is None and not os.path.isfile('./ar') and not os.path.isfile('./ar.exe'):
@@ -62,10 +56,10 @@
     thisPem = pems[i].replace("'", "")
     print(names[i] + " -> " + certName)
     ssl = Popen(['openssl','x509','-inform','PEM','-outform','DER','-out', certName], shell = False, stdin = PIPE)
-    pipe = ssl.stdin
-    pipe.write(thisPem.encode('utf-8'))
-    pipe.close()
-    ssl.wait()
+    ssl.communicate(thisPem.encode('utf-8'))
+    ret = ssl.wait()
+    if ret != 0:
+        raise CalledProcessError(ret, certName)
     if os.path.exists(certName):
         derFiles.append(certName)
         idx = idx + 1

tests/device/Makefile

@@ -1,45 +1,65 @@
 SHELL := /bin/bash
-V ?= 0
-ESP8266_CORE_PATH ?= $(realpath ../..)
-BUILD_DIR ?= $(PWD)/.build
-HARDWARE_DIR ?= $(PWD)/.hardware
+
+ESP8266_CORE_PATH ?= $(shell git rev-parse --show-toplevel)
+
+BUILD_DIR ?= $(PWD)/build
+BS_DIR ?= $(PWD)/libraries/BSTest
+
 PYTHON ?= python3
 ESPTOOL ?= $(PYTHON) $(ESP8266_CORE_PATH)/tools/esptool/esptool.py
-MKSPIFFS ?= $(ESP8266_CORE_PATH)/tools/mkspiffs/mkspiffs
+
+VENV_PYTHON ?= $(BS_DIR)/virtualenv/bin/python
+VENV_JUNIT2HTML ?= $(BS_DIR)/virtualenv/bin/junit2html
+
+MKFS ?= $(ESP8266_CORE_PATH)/tools/mklittlefs/mklittlefs
+
 UPLOAD_PORT ?= $(shell ls /dev/tty* | grep -m 1 -i USB)
 UPLOAD_BAUD ?= 460800
-UPLOAD_BOARD ?= nodemcu
-BS_DIR ?= libraries/BSTest
-DEBUG_LEVEL ?= lvl=None____
-FQBN ?= esp8266com:esp8266:generic:xtal=160,FlashFreq=40,FlashMode=dio,baud=115200,eesz=4M1M,ip=lm2f,ResetMethod=nodemcu,dbg=Serial,$(DEBUG_LEVEL)
-BUILD_TOOL := $(ARDUINO_IDE_PATH)/arduino-builder
+
+TEST_BAUD ?= 115200
+BUILD_TOOL ?= arduino-cli
+
+BUILD_BOARD ?= generic
+BUILD_CPU ?= 160
+BUILD_SIZE ?= 4M1M
+BUILD_LWIP ?= lm2f
+BUILD_EXTRA ?= ,dbg=Serial,lvl=CORE
+
+FQBN ?= esp8266com:esp8266:$(BUILD_BOARD):xtal=$(BUILD_CPU),baud=$(TEST_BAUD),eesz=$(BUILD_SIZE),ip=$(BUILD_LWIP)$(BUILD_EXTRA)
+
 TEST_CONFIG := test_env.cfg
+TEST_RESULT_XML := test_result.xml
 TEST_REPORT_XML := test_report.xml
 TEST_REPORT_HTML := test_report.html
 
-ifeq ("$(MOCK)", "1")
-# To enable a test for mock testing, just rename dir+files to '*_sw_*'
-TEST_LIST ?= $(wildcard test_sw_*/*.ino)
-else
 TEST_LIST ?= $(wildcard test_*/*.ino)
+BUILD_FLAGS ?=
+BS_FLAGS ?=
+
+# To enable a test for mock testing, just rename dir+files to '*_sw_*'
+ifeq ("$(MOCK)", "1")
+	TEST_LIST := $(filter test_sw_%.ino,$(TEST_LIST))
+	NO_UPLOAD := 1
+	NO_RUN := 1
 endif
 
-ifneq ("$(V)","1")
-	SILENT = @
-	REDIR = >& /dev/null
-else
-	BUILDER_DEBUG_FLAG = -verbose
-	RUNNER_DEBUG_FLAG = -d
-	#UPLOAD_VERBOSE_FLAG = -v
+# To enable verbose mode, call `make V=1` ...
+V ?= 0
+ifeq ("$(V)", "1")
+	BUILD_FLAGS += --verbose
+	BS_FLAGS += --debug
 endif
 
+# ${sketch}.py helper script when building locally
+mock_script = \
+	`test -f $(addsuffix .py, $(basename $(1))) && echo "--mock $(addsuffix .py, $(basename $(1)))" || echo ""`
+
 help:
 	@echo
 	@echo 'make list                    - show list of tests'
 	@echo 'make sometest/sometest.ino   - run one test'
 	@echo 'make all                     - run all tests'
 	@echo 'make MOCK=1 all              - run all emulation-on-host compatible tests'
-	@echo 'variables needed: $$ARDUINO_IDE_PATH $$ESP8266_CORE_PATH'
 	@echo 'make options: V=1 NO_BUILD=1 NO_UPLOAD=1 NO_RUN=1 MOCK=1'
 	@echo
 
@@ -49,110 +69,132 @@ all: count tests test_report
 
 $(TEST_LIST): | virtualenv $(TEST_CONFIG) $(BUILD_DIR) $(HARDWARE_DIR)
 
+.NOTPARALLEL: $(TEST_LIST)
+
 tests: showtestlist $(TEST_LIST)
 
 showtestlist:
 	@echo "-------------------------------- test list:"
-	@echo $(TEST_LIST)
+	@printf '%s\n' $(TEST_LIST)
 	@echo "--------------------------------"
 
 $(TEST_LIST): LOCAL_BUILD_DIR=$(BUILD_DIR)/$(notdir $@)
-
-$(TEST_LIST):
-	@echo "--------------------------------"
-	@echo "Running test '$@' of $(words $(TEST_LIST)) tests"
-	$(SILENT)mkdir -p $(LOCAL_BUILD_DIR)
-ifeq ("$(MOCK)", "1")
-	@echo Compiling $(notdir $@)
-	(cd ../host; make D=$(V) ULIBDIRS=../device/libraries/BSTest ../device/$(@:%.ino=%))
-	$(SILENT)$(BS_DIR)/virtualenv/bin/python \
-		$(BS_DIR)/runner.py \
-			$(RUNNER_DEBUG_FLAG) \
-			-e "$(ESP8266_CORE_PATH)/tests/host/bin/$(@:%.ino=%)" \
-			-n $(basename $(notdir $@)) \
-			-o $(LOCAL_BUILD_DIR)/test_result.xml \
-			--env-file $(TEST_CONFIG) \
-			`test -f $(addsuffix .py, $(basename $@)) && echo "-m $(addsuffix .py, $(basename $@))" || echo ""`
-else
-ifneq ("$(NO_BUILD)","1")
-	@test -n "$(ARDUINO_IDE_PATH)" || (echo "Please export ARDUINO_IDE_PATH" && exit 1)
-	@echo Compiling $(notdir $@)
-	@rm -f $(LOCAL_BUILD_DIR)/build.options.json
-	$(SILENT)$(BUILD_TOOL) -compile -logger=human \
-		-libraries "$(PWD)/libraries" \
-		-core-api-version="10608" \
-		-warnings=all \
-		$(BUILDER_DEBUG_FLAG) \
-		-build-path $(LOCAL_BUILD_DIR) \
-		-tools $(ARDUINO_IDE_PATH)/tools-builder \
-		-hardware $(HARDWARE_DIR)\
-		-hardware $(ARDUINO_IDE_PATH)/hardware \
-		-fqbn=$(FQBN) \
+$(TEST_LIST): LOCAL_DATA_IMG=data.img
+
+define build-arduino
+	rm -f $(LOCAL_BUILD_DIR)/build.options.json
+	$(BUILD_TOOL) compile \
+		$(BUILD_FLAGS) \
+		--libraries "$(PWD)/libraries" \
+		--warnings=all \
+		--build-path $(LOCAL_BUILD_DIR) \
+		--fqbn=$(FQBN) \
 		$@
-endif
-ifneq ("$(NO_UPLOAD)","1")
-	@test -n "$(UPLOAD_PORT)" || (echo "Failed to detect upload port, please export UPLOAD_PORT manually" && exit 1)
-	@test -e $(dir $@)/make_spiffs.py && ( \
-		echo "Generating and uploading SPIFFS" && \
-		(cd $(dir $@) && $(PYTHON) ./make_spiffs.py $(REDIR) ) && \
-		$(MKSPIFFS) --create $(dir $@)data/ --size 0xFB000 \
-			--block 8192 --page 256 $(LOCAL_BUILD_DIR)/spiffs.img $(REDIR) && \
-		$(ESPTOOL) $(UPLOAD_VERBOSE_FLAG) \
+endef
+
+define build-mock
+	(cd $(ESP8266_CORE_PATH)/test/host; \
+		$(MAKE) D=$(V) ULIBDIRS=$(PWD)/libraries/BSTest $(PWD)/$(@:%.ino=%))
+	$(VENV_PYTHON) $(BS_DIR)/runner.py \
+		$(BS_FLAGS) \
+		--name $(basename $(notdir $@)) \
+		--output $(LOCAL_BUILD_DIR)/$(TEST_RESULT_XML) \
+		--env-file $(TEST_CONFIG) \
+		$(call mock_script,$@) \
+		executable "$(ESP8266_CORE_PATH)/tests/host/bin/$(@:%.ino=%)" || echo ""`
+endef
+
+define upload-data
+	@test -n "$(UPLOAD_PORT)" \
+		|| (echo "Failed to detect upload port, please export UPLOAD_PORT manually" && exit 1)
+	@test ! \( -d $(dir $@)/data/ \) -a \( -e $(dir $@)/make_data.py \) && \
+		(cd $(dir $@) && ./make_data.py ) || echo "Filesystem creation skipped"
+	@test -d $(dir $@)/data/ && ( \
+		$(MKFS) \
+			--create $(dir $@)/data/ \
+			--size 0xFB000 \
+			--block 8192 \
+			--page 256 \
+			$(LOCAL_BUILD_DIR)/$(LOCAL_DATA_IMG) && \
+		$(ESPTOOL) \
 			--chip esp8266 \
 			--port $(UPLOAD_PORT) \
 			--baud $(UPLOAD_BAUD) \
 			--after no_reset \
-			write_flash 0x300000 $(LOCAL_BUILD_DIR)/spiffs.img $(REDIR) ) \
-			|| (echo "No SPIFFS to upload")
-	@echo Uploading binary
-	$(SILENT)$(ESPTOOL) $(UPLOAD_VERBOSE_FLAG)  \
+			write_flash 0x300000 $(LOCAL_BUILD_DIR)/$(LOCAL_DATA_IMG) ) \
+		&& (echo "Uploaded filesystem") \
+		|| (echo "Filesystem upload skipped")
+endef
+
+define upload-binary
+	@test -n "$(UPLOAD_PORT)" \
+		|| (echo "Failed to detect upload port, please export UPLOAD_PORT manually" && exit 1)
+	$(ESPTOOL) \
 		--chip esp8266 \
 		--port $(UPLOAD_PORT) \
 		--baud $(UPLOAD_BAUD) \
 		--after no_reset \
-		write_flash 0x0 $(LOCAL_BUILD_DIR)/$(notdir $@).bin $(REDIR) # no reset
-endif
-ifneq ("$(NO_RUN)","1")
-	@test -n "$(UPLOAD_PORT)" || (echo "Failed to detect upload port, please export UPLOAD_PORT manually" && exit 1)
+		write_flash 0x0 $(LOCAL_BUILD_DIR)/$(notdir $@).bin
+endef
+
+define run-test
+	@test -n "$(UPLOAD_PORT)" \
+		|| (echo "Failed to detect upload port, please export UPLOAD_PORT manually" && exit 1)
 	@echo Running tests
-	$(SILENT)$(ESPTOOL) $(UPLOAD_VERBOSE_FLAG)  \
+	$(ESPTOOL) \
 		--chip esp8266 \
 		--port $(UPLOAD_PORT) \
 		--baud $(UPLOAD_BAUD) \
-		read_flash_status $(REDIR) # reset
-	$(SILENT)$(BS_DIR)/virtualenv/bin/python \
-		$(BS_DIR)/runner.py \
-			$(RUNNER_DEBUG_FLAG) \
-			-p $(UPLOAD_PORT) \
-			-n $(basename $(notdir $@)) \
-			-o $(LOCAL_BUILD_DIR)/test_result.xml \
-			--env-file $(TEST_CONFIG) \
-			`test -f $(addsuffix .py, $(basename $@)) && echo "-m $(addsuffix .py, $(basename $@))" || echo ""`
+		read_flash_status # reset via implicit stub reboot
+	$(VENV_PYTHON) $(BS_DIR)/runner.py \
+		$(BS_FLAGS) \
+		--name $(basename $(notdir $@)) \
+		--output $(LOCAL_BUILD_DIR)/$(TEST_RESULT_XML) \
+		--env-file $(TEST_CONFIG) \
+		$(call mock_script,$@) \
+		port $(UPLOAD_PORT) \
+		--baudrate $(TEST_BAUD)
+endef
+
+$(TEST_LIST):
+	@echo "--------------------------------"
+	@echo "Running test '$@' of $(words $(TEST_LIST)) tests"
+	mkdir -p $(LOCAL_BUILD_DIR)
+ifneq ("$(NO_BUILD)","1")
+	@echo Building $(notdir $@)
+ifeq ("$(MOCK)", "1")
+	$(build-mock)
+else
+	$(build-arduino)
 endif
 endif
+ifneq ("$(NO_UPLOAD)","1")
+	$(upload-filesystem)
+	$(upload-binary)
+endif
+ifneq ("$(NO_RUN)","1")
+	$(run-test)
+endif
 
-$(TEST_REPORT_XML): $(HARDWARE_DIR) virtualenv
-	$(SILENT)$(BS_DIR)/xunitmerge $(shell find $(BUILD_DIR) -name 'test_result.xml' | xargs echo) $(TEST_REPORT_XML)
+$(TEST_REPORT_XML): virtualenv
+	$(BS_DIR)/xunitmerge \
+		$(shell find $(BUILD_DIR) -name '$(TEST_RESULT_XML)' | xargs echo) \
+		$(TEST_REPORT_XML)
 
 $(TEST_REPORT_HTML): $(TEST_REPORT_XML) | virtualenv
-	$(SILENT)$(BS_DIR)/virtualenv/bin/junit2html $< $@
+	$(VENV_JUNIT2HTML) $< $@
 
 test_report: $(TEST_REPORT_HTML)
 	@echo "Test report generated in $(TEST_REPORT_HTML)"
 
 $(BUILD_DIR):
-	mkdir -p $(BUILD_DIR)
-
-$(HARDWARE_DIR):
-	mkdir -p $(HARDWARE_DIR)/esp8266com
-	cd $(HARDWARE_DIR)/esp8266com && ln -s $(realpath $(ESP8266_CORE_PATH)) esp8266
+	@mkdir -p $(BUILD_DIR)
 
 virtualenv:
 	@make -C $(BS_DIR) PYTHON=$(PYTHON) virtualenv
 
 clean:
 	rm -rf $(BUILD_DIR)
-	rm -rf $(HARDWARE_DIR)
 	rm -rf $(BS_DIR)/virtualenv
 	rm -f $(TEST_REPORT_HTML) $(TEST_REPORT_XML)