add simple include file processing

This will not work on the ESP32 for large files, due to limited memory.
But this is only the first step. Next we'll add a database for storing
defines from include files.

Author: wnienhaus

esp32_ulp/preprocess.py

@@ -32,29 +32,31 @@ def WRITE_RTC_FIELD(rtc_reg, low_bit, value):
 
 class Preprocessor:
     def __init__(self):
+        self._defines_db = None
         self._defines = {}
 
+    def parse_define_line(self, line):
+        line = line.strip()
+        if not line.startswith("#define"):
+            # skip lines not containing #define
+            return {}
+        line = line[8:].strip()  # remove #define
+        parts = line.split(None, 1)
+        if len(parts) != 2:
+            # skip defines without value
+            return {}
+        identifier, value = parts
+        tmp = identifier.split('(', 1)
+        if len(tmp) == 2:
+            # skip parameterised defines (macros)
+            return {}
+        value = "".join(nocomment.remove_comments(value)).strip()
+        return {identifier: value}
+
     def parse_defines(self, content):
-        result = {}
         for line in content.splitlines():
-            line = line.strip()
-            if not line.startswith("#define"):
-                # skip lines not containing #define
-                continue
-            line = line[8:].strip()  # remove #define
-            parts = line.split(None, 1)
-            if len(parts) != 2:
-                # skip defines without value
-                continue
-            identifier, value = parts
-            tmp = identifier.split('(', 1)
-            if len(tmp) == 2:
-                # skip parameterised defines (macros)
-                continue
-            value = "".join(nocomment.remove_comments(value)).strip()
-            result[identifier] = value
-        self._defines = result
-        return result
+            self._defines.update(self.parse_define_line(line))
+        return self._defines
 
     def expand_defines(self, line):
         found = True
@@ -70,6 +72,16 @@ def expand_defines(self, line):
 
         return line
 
+    def process_include_file(self, filename):
+        defines = self._defines
+
+        with open(filename, 'r') as f:
+            for line in f:
+                result = self.parse_defines(line)
+                defines.update(result)
+
+        return defines
+
     def expand_rtc_macros(self, line):
         clean_line = line.strip()
         if not clean_line:

tests/fixtures/incl.h

@@ -0,0 +1,5 @@
+#define CONST1 42
+#define MACRO(x,y) x+y
+#define MULTI_LINE abc \
+                   xyz
+#define CONST2 99

tests/fixtures/incl2.h

@@ -0,0 +1,2 @@
+#define CONST2 123
+#define CONST3 777

tests/preprocess.py

@@ -27,19 +27,19 @@ def replace_defines_should_return_remove_comments():
 def test_parse_defines():
     p = Preprocessor()
 
-    assert p.parse_defines("") == {}
-    assert p.parse_defines("// comment") == {}
-    assert p.parse_defines("  // comment") == {}
-    assert p.parse_defines("  /* comment */") == {}
-    assert p.parse_defines("  /* comment */ #define A 42") == {}  # #define must be the first thing on a line
-    assert p.parse_defines("#define a 1") == {"a": "1"}
-    assert p.parse_defines(" #define a 1") == {"a": "1"}
-    assert p.parse_defines("#define a 1 2") == {"a": "1 2"}
-    assert p.parse_defines("#define f(a,b) 1") == {}  # macros not supported
-    assert p.parse_defines("#define f(a, b) 1") == {}  # macros not supported
-    assert p.parse_defines("#define f (a,b) 1") == {"f": "(a,b) 1"}  # f is not a macro
-    assert p.parse_defines("#define f (a, b) 1") == {"f": "(a, b) 1"}  # f is not a macro
-    assert p.parse_defines("#define RTC_ADDR       0x12345    // start of range") == {"RTC_ADDR": "0x12345"}
+    assert p.parse_define_line("") == {}
+    assert p.parse_define_line("// comment") == {}
+    assert p.parse_define_line("  // comment") == {}
+    assert p.parse_define_line("  /* comment */") == {}
+    assert p.parse_define_line("  /* comment */ #define A 42") == {}  # #define must be the first thing on a line
+    assert p.parse_define_line("#define a 1") == {"a": "1"}
+    assert p.parse_define_line(" #define a 1") == {"a": "1"}
+    assert p.parse_define_line("#define a 1 2") == {"a": "1 2"}
+    assert p.parse_define_line("#define f(a,b) 1") == {}  # macros not supported
+    assert p.parse_define_line("#define f(a, b) 1") == {}  # macros not supported
+    assert p.parse_define_line("#define f (a,b) 1") == {"f": "(a,b) 1"}  # f is not a macro
+    assert p.parse_define_line("#define f (a, b) 1") == {"f": "(a, b) 1"}  # f is not a macro
+    assert p.parse_define_line("#define RTC_ADDR       0x12345    // start of range") == {"RTC_ADDR": "0x12345"}
 
 
 @test
@@ -181,6 +181,29 @@ def test_expand_rtc_macros():
     assert p.expand_rtc_macros("READ_RTC_FIELD(1, 2)") == "\treg_rd 1, 2 + 1 - 1, 2"
 
 
+@test
+def test_process_include_file():
+    p = Preprocessor()
+
+    defines = p.process_include_file('fixtures/incl.h')
+    assert defines['CONST1'] == '42'
+    assert defines['CONST2'] == '99'
+    assert defines.get('MULTI_LINE', None) == 'abc \\'  # correct. line continuations not supported
+    assert 'MACRO' not in defines
+
+
+@test
+def test_process_include_file_with_multiple_files():
+    p = Preprocessor()
+
+    defines = p.process_include_file('fixtures/incl.h')
+    defines = p.process_include_file('fixtures/incl2.h')
+
+    assert defines['CONST1'] == '42', "constant from incl.h"
+    assert defines['CONST2'] == '123', "constant overridden by incl2.h"
+    assert defines['CONST3'] == '777', "constant from incl2.h"
+
+
 if __name__ == '__main__':
     # run all methods marked with @test
     for t in tests: