Tests update

Author: igrr

tests/device/Makefile

@@ -71,7 +71,8 @@ ifneq ("$(NO_RUN)","1")
 			$(RUNNER_DEBUG_FLAG) \
 			-p $(UPLOAD_PORT) \
 			-n $(basename $(notdir $@)) \
-			-o $(LOCAL_BUILD_DIR)/test_result.xml
+			-o $(LOCAL_BUILD_DIR)/test_result.xml \
+			`test -f $(addsuffix .py, $(basename $@)) && echo "-m $(addsuffix .py, $(basename $@))" || echo ""`
 endif
 
 $(BUILD_DIR):

tests/device/libraries/BSTest/mock_decorators.py

@@ -0,0 +1,28 @@
+
+env = dict()
+
+def setup(test_name):
+	global env
+	if not test_name in env:
+		env[test_name] = dict()
+	func_env = env[test_name]
+	def decorator(func):
+		def func_wrapper():
+			return func(env[test_name])
+		func_env['setup'] = func_wrapper
+		return func_wrapper
+	return decorator
+
+
+
+def teardown(test_name):
+	global env
+	if not test_name in env:
+		env[test_name] = dict()
+	func_env = env[test_name]
+	def decorator(func):
+		def func_wrapper():
+			return func(env[test_name])
+		func_env['teardown'] = func_wrapper
+		return func_wrapper
+	return decorator

tests/device/libraries/BSTest/requirements.txt

@@ -1,6 +1,12 @@
+Flask==0.10.1
+itsdangerous==0.24
+Jinja2==2.8
 junit-xml==1.6
+MarkupSafe==0.23
 pexpect==4.0.1
 ptyprocess==0.5.1
 pyserial==3.0.1
+PyYAML==3.11
 six==1.10.0
-wheel==0.24.0
+Werkzeug==0.11.9
+wheel==0.24.0

tests/device/libraries/BSTest/runner.py

@@ -8,15 +8,19 @@
 import argparse
 import serial
 import subprocess
+import imp
 from urlparse import urlparse
 from junit_xml import TestSuite, TestCase
 try:
     from cStringIO import StringIO
 except:
     from StringIO import StringIO
+import mock_decorators
 
 debug = False
 
+sys.path.append(os.path.abspath(__file__))
+
 def debug_print(*args, **kwargs):
     if not debug:
         return
@@ -29,17 +33,18 @@ class BSTestRunner(object):
     TIMEOUT = 2
     CRASH = 3
 
-    def __init__(self, spawn_obj, name):
+    def __init__(self, spawn_obj, name, mocks):
         self.sp = spawn_obj
         self.tests = []
         self.reset_timeout = 2
         self.name = name
+        self.mocks = mocks
 
     def get_test_list(self):
         self.sp.sendline('-1')
         timeout = 10
         while timeout > 0:
-            res = self.sp.expect(['>>>>>bs_test_menu_begin', EOF])
+            res = self.sp.expect(['>>>>>bs_test_menu_begin', EOF, TIMEOUT])
             if res == 0:
                 break
             timeout-=1
@@ -49,15 +54,15 @@ def get_test_list(self):
         while True:
             res = self.sp.expect(['>>>>>bs_test_item id\=(\d+) name\="([^\"]*?)" desc="([^"]*?)"',
                                   '>>>>>bs_test_menu_end',
-                                  EOF])
+                                  EOF, TIMEOUT])
             if res == 0:
                 m = self.sp.match
                 t = {'id': m.group(1), 'name': m.group(2), 'desc': m.group(3)}
                 self.tests.append(t)
                 debug_print('added test', t)
             elif res == 1:
                 break
-            elif res == 2:
+            elif res >= 2:
                 time.sleep(0.1)
 
         debug_print('got {} tests'.format(len(self.tests)))
@@ -75,8 +80,14 @@ def run_tests(self):
             else:
                 test_output = StringIO()
                 self.sp.logfile = test_output
+                if name in self.mocks:
+                    print('setting up mocks')
+                    self.mocks[name]['setup']()
                 t_start = time.time()
                 result = self.run_test(index)
+                if name in self.mocks:
+                    print('tearing down mocks')
+                    self.mocks[name]['teardown']()
                 t_stop = time.time()
                 self.sp.logfile = None
                 test_case.elapsed_sec = t_stop - t_start
@@ -96,7 +107,7 @@ def run_test(self, index):
         self.sp.sendline('{}'.format(index))
         timeout = 10
         while timeout > 0:
-            res = self.sp.expect(['>>>>>bs_test_start', EOF])
+            res = self.sp.expect(['>>>>>bs_test_start', EOF, TIMEOUT])
             if res == 0:
                 break
             time.sleep(0.1)
@@ -147,8 +158,8 @@ def spawn_port(port_name, baudrate=115200):
 def spawn_exec(name):
     return pexpect.spawn(name, timeout=0)
 
-def run_tests(spawn, name):
-    tw = BSTestRunner(spawn, name)
+def run_tests(spawn, name, mocks):
+    tw = BSTestRunner(spawn, name, mocks)
     tw.get_test_list()
     return tw.run_tests()
 
@@ -159,6 +170,7 @@ def parse_args():
     parser.add_argument('-e', '--executable', help='Talk to the test executable')
     parser.add_argument('-n', '--name', help='Test run name')
     parser.add_argument('-o', '--output', help='Output JUnit format test report')
+    parser.add_argument('-m', '--mock', help='Set python script to use for mocking purposes')
     return parser.parse_args()
 
 def main():
@@ -178,8 +190,12 @@ def main():
     if spawn_func is None:
         debug_print("Please specify port or executable", file=sys.stderr)
         return 1
+    mocks = {}
+    if args.mock is not None:
+        mocks_mod = imp.load_source('mocks', args.mock)
+        mocks = mock_decorators.env
     with spawn_func(spawn_arg) as sp:
-        ts = run_tests(sp, name)
+        ts = run_tests(sp, name, mocks)
         if args.output:
             with open(args.output, "w") as f:
                 TestSuite.to_file(f, [ts])

tests/device/test_FS/test_FS.ino

@@ -90,7 +90,7 @@ TEST_CASE("files can be renamed", "[fs]")
     }
 }
 
-TEST_CASE("FS::info works")
+TEST_CASE("FS::info works","[fs]")
 {
     REQUIRE(SPIFFS.begin());
     FSInfo info;

tests/device/test_http_client/server.crt

@@ -0,0 +1,18 @@
+-----BEGIN CERTIFICATE-----
+MIICzzCCAbcCCQCUajf39FoF8jANBgkqhkiG9w0BAQsFADAWMRQwEgYDVQQDDAtj
+YS50bHMudGVzdDAeFw0xNjA0MjIxMTU1MTlaFw0xNzA0MjIxMTU1MTlaMD0xITAf
+BgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEYMBYGA1UEAwwPc2VydmVy
+LnRscy50ZXN0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAwhyGSO13
+/ozG0rL6FxDaaxYgMyPVK3hIhTHfKOK/eRZFRDFAhYudz9NbkvUMIF1SToED9FBh
+MbJrkPzCU3cLC1zgnTLseoOS3FKC8xVfJVwsWD8YjMvyQUV/Uo9TAHQSA4SfOB4W
+JjRBMX2GCoWLK5wVzxX+XGd5DnqME4n/CG+Il1t8mB+ACeA1FKwVPTi1wGvDzuCo
+swSEX3J08JB9rP5dix7t+Fezgr6PkvUcnAcu4utAw6f6c0LoHk0SnyXJg1jTdKrl
+J8dIyWQR0cvaxbF+04hvpDepx/62CP+aRs1zcC37eEQ3BVf4phJXbwuXbT7a1CUf
+XQ0cWnJg8mdfcQIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQBV7ZiwVM8alxPN9vd/
+Ze8EiidSuztkbtZX3mhHfkcJ2Q5c+tZ0Oxy4/g6oISe1+NOQJDMrsARrdWVPmmsb
+ihSHNdS6t0DTcWR5h+i/rnGh+cGiWOJKfhNWvpCVuU6YRZWhYOztL8p6iHoG3ZBi
++zo9GD71FNSRQxMois9sR9q/IgOmkm8CjQgYsv9bQ+Le4mRqfaOCOSoQvsRyLoam
+lNl85gzvVygHYPP9ypiia8btyOHwDSHiV8UhKaERSGKFqznPmTTwTQNXEtQylCXG
+C+13mMGVr49yP4cuYaM8mfL8Rg7Im8Mfa0GXq5PBwEFFYpR9xnbBouQv5erYTQdl
+Oxqk
+-----END CERTIFICATE-----

tests/device/test_http_client/server.csr

@@ -0,0 +1,16 @@
+-----BEGIN CERTIFICATE REQUEST-----
+MIICgjCCAWoCAQAwPTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRk
+MRgwFgYDVQQDDA9zZXJ2ZXIudGxzLnRlc3QwggEiMA0GCSqGSIb3DQEBAQUAA4IB
+DwAwggEKAoIBAQDCHIZI7Xf+jMbSsvoXENprFiAzI9UreEiFMd8o4r95FkVEMUCF
+i53P01uS9QwgXVJOgQP0UGExsmuQ/MJTdwsLXOCdMux6g5LcUoLzFV8lXCxYPxiM
+y/JBRX9Sj1MAdBIDhJ84HhYmNEExfYYKhYsrnBXPFf5cZ3kOeowTif8Ib4iXW3yY
+H4AJ4DUUrBU9OLXAa8PO4KizBIRfcnTwkH2s/l2LHu34V7OCvo+S9RycBy7i60DD
+p/pzQugeTRKfJcmDWNN0quUnx0jJZBHRy9rFsX7TiG+kN6nH/rYI/5pGzXNwLft4
+RDcFV/imEldvC5dtPtrUJR9dDRxacmDyZ19xAgMBAAGgADANBgkqhkiG9w0BAQsF
+AAOCAQEAE2xZjAkmVr7/p7LkV3UV3Y0wzeVP7kDBN7VnhNmEMw8xTKwXKsc6z5pY
+fa4/lmMMCIJ04dUPIgPC2qiYXJ1AXevLm0A7Blpay6HJilw51NMjhF6SLUkShl6e
+k3zyj9LnA5TxijsTrFy/km7qIoo6l7sR7+DwTOIlO/Sj/SDNJn+GAVL153zjxYCy
+eBYdEz07kxkeONLUjAW2bV+TkliqTdM6meRpf9GQGQksQrkHrudl7JyyKZTf2m6Z
+U33AnUp2SRIXESJmst824LKwkjLMYiXgxRK8ZSSP7iiNBj71DK1vQOZQwEZ2cCjW
+/pQxKJnHnRvZ3rityuShtHTUF+C39w==
+-----END CERTIFICATE REQUEST-----

tests/device/test_http_client/server.key

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEAwhyGSO13/ozG0rL6FxDaaxYgMyPVK3hIhTHfKOK/eRZFRDFA
+hYudz9NbkvUMIF1SToED9FBhMbJrkPzCU3cLC1zgnTLseoOS3FKC8xVfJVwsWD8Y
+jMvyQUV/Uo9TAHQSA4SfOB4WJjRBMX2GCoWLK5wVzxX+XGd5DnqME4n/CG+Il1t8
+mB+ACeA1FKwVPTi1wGvDzuCoswSEX3J08JB9rP5dix7t+Fezgr6PkvUcnAcu4utA
+w6f6c0LoHk0SnyXJg1jTdKrlJ8dIyWQR0cvaxbF+04hvpDepx/62CP+aRs1zcC37
+eEQ3BVf4phJXbwuXbT7a1CUfXQ0cWnJg8mdfcQIDAQABAoIBAQCKfl2VFNEjbf/B
+fu8Om/iqpOuucSd7z2M2nSGSg02HsEsVX2qgnb+n8y4ICQxw3wSFfGl83Aiss5cp
+qB1h/bKEleywXA/7TXrv9XL2ys3K0xvUjpgOhMjApzwzhIz86tObbPlIybaV5XyK
+Ofvz79oe8EtjKFcGSNFHckoU+sValzH/gyaeIFuPFlJdsW9MOgN5lcyLDtddg2lr
+Z9k+OEEob9BC1dDmc5hxtnClfyiXW1ugjZovOBv79Mq2N8G/V8JZaTHxzEFdOGy3
+lUQ1B/s2xPDejQYGddc3N5+wF4ZuGHAYOmcv8IRaMvLdBMPmVnKdgDJeY4HheJY0
+z0t4BEYtAoGBAPL1exWZputT8GitBxTJcJB1crMzxvlSplS5mGxM36GgZxf2ql72
+8Ufa1hCAQLP7keKPWWl26Bt6Bm4uv/jsB/F8ezMloPvwwQ3nQrN8QY4ihtLLFcvK
+OMaErweMEVvQ6GdQHPGow5RqAkYj/X6jJjoAI88uV1yOx+TucmDneEnbAoGBAMyH
+04ZR3VNvzHSjDx2wo48FP7Cs/uCynOEd/bRb1I8n+fazIXIlWdWX2drFr2YRM+Uy
+UzAadgqFOu3Bvuk64UKwAlxX9jCIm1Ed5O1MUGPj+6OnW1oq/eYIbJ9RlFDyIEBv
+Ic+jV/zxojgpP38jpTzn2oC5jWWS+CjHpZNThtujAoGBAOquRB4xNRLLGcWCnPxv
+N9PSHuEKeoAKXbApNhomhz1P+0Uidp8UWSvXLj7yI422ysvVO8Crorgnvl5fuf9v
+vpx8aWSWTFIP0+riH4PP2mK45xJmKL+Yrg7Ty1225m2R9WsV719ebMzHOTsXOJ1C
+aoJL8EsHsEsvf9aanNENxRtRAoGAVIP34zQm2diDytqqX2FjZLENjWse8yi7bMag
+1ItxvSoOv2Nr+af3hCx4aE9x2CJZqGbwOxtkFZrSK/b4dZXQCWeDwjbS02FPlOhe
+dbQoL+7AR/La69qCCjEG+ZqTSBOVQirp9MwRisMqfjyFMXtAR3ejMbf69rMReoBt
+KgDE5DMCgYAkcBd7XyPjR6t09BXAcz8n5J8QCsoY4taNVeCmCCkl8rUt9y8tjQah
+5NtBk6i834p5/GbGhI4en/sisVUWzpOgc9G80lUAFQSR3weR6lOL1T+/cy691GqO
+wKAWbpOH01fJYmnbUjC7TC2idx033ZA9vJq/s9sNfG2CS2w3egcxag==
+-----END RSA PRIVATE KEY-----