comment removal by state machine, fixes #17

Now supported:
/* C-style */
// C-style

Author: ThomasWaldmann

esp32_ulp/__main__.py

@@ -4,19 +4,19 @@
 from .link import make_binary
 
 
-def src_to_binary(lines):
+def src_to_binary(src):
     assembler = Assembler()
-    assembler.assemble(lines)
+    assembler.assemble(src)
     assembler.dump()
     text, data, bss_len = assembler.fetch()
     return make_binary(text, data, bss_len)
 
 
 def main(fn):
     with open(fn) as f:
-        lines = f.readlines()
+        src = f.read()
 
-    binary = src_to_binary(lines)
+    binary = src_to_binary(src)
 
     if fn.endswith('.s') or fn.endswith('.S'):
         fn = fn[:-2]

esp32_ulp/assemble.py

@@ -3,9 +3,11 @@
 """
 
 from . import opcodes
+from .nocomment import remove_comments
 
 TEXT, DATA, BSS = 'text', 'data', 'bss'
 
+
 class Assembler:
 
     def __init__(self):
@@ -16,15 +18,13 @@ def __init__(self):
 
     def parse_line(self, line):
         """
-        parse one line of assembler into label, opcode, args
+        parse one line of assembler into label, opcode, args.
+        comments already have been removed by pre-processing.
 
         a line looks like (label, opcode, args, comment are all optional):
 
-        label:    opcode arg1, arg2, ...    # rest-of-line comment
+        label:    opcode arg1, arg2, ...
         """
-        line = line.split('#', 1)[0]
-        line = line.split('//', 1)[0]
-        line = line.rstrip()
         if not line:
             return
         has_label = line[0] not in '\t '
@@ -49,7 +49,8 @@ def parse_line(self, line):
         return label, opcode, args
 
 
-    def parse(self, lines):
+    def parse(self, text):
+        lines = remove_comments(text)
         parsed = [self.parse_line(line) for line in lines]
         return [p for p in parsed if p is not None]
 

esp32_ulp/nocomment.py

@@ -0,0 +1,119 @@
+def remove_comments(s):
+    """
+    Remove comments of these styles:
+
+        CHASH:       # comment python style, up to: EOL
+        CSLASHSLASH: // comment C style, up to: EOL
+        CSLASHSTAR:  /* comment C style (single/multi line), up to: */
+
+    Strings can be like 'strings' or "strings".
+    Any comment-starting chars within strings are not considered.
+    Escaping of (string-end) chars via backslash in strings is considered.
+
+    Also, leading and trailing whitespace is removed (after comment removal).
+    Indented lines are re-indented afterwards with a single tab char.
+
+    Line numbers stay as in input file because empty lines are kept.
+
+    s: string with comments (can include newlines)
+    returns: list of text lines
+    """
+    # note: micropython's ure module was not capable enough to process this:
+    # missing methods, re modes, recursion limit exceeded, ...
+    # simpler hacks also didn't seem powerful enough to address all the
+    # corner cases of CSLASHSTAR vs. *STR, so this state machine came to life:
+    SRC, CHASH, CSLASHSLASH, CSLASHSTAR, DSTR, SSTR = range(6)  # states
+
+    line = []  # collect chars of one line
+    lines = []  # collect result lines
+
+    def finish_line():
+        # assemble a line from characters, try to get rid of trailing and
+        # most of leading whitespace (keep/put one tab for indented lines).
+        nonlocal line
+        line = ''.join(line)
+        is_indented = line.startswith(' ') or line.startswith('\t')
+        line = line.strip()
+        if line and is_indented:
+            line = '\t' + line
+        lines.append(line)
+        line = []
+
+    state = SRC
+    i = 0
+    length = len(s)
+    while i < length:
+        c = s[i]
+        cn = s[i + 1] if i + 1 < length else '\0'
+        if state == SRC:
+            if c == '#':  # starting to-EOL comment
+                state = CHASH
+                i += 1
+            elif c == '/':
+                if cn == '/':  # starting to-EOL comment
+                    state = CSLASHSLASH
+                    i += 2
+                elif cn == '*':  # starting a /* comment
+                    state = CSLASHSTAR
+                    i += 2
+                else:
+                    i += 1
+                    line.append(c)
+            elif c == '"':
+                state = DSTR
+                i += 1
+                line.append(c)
+            elif c == "'":
+                state = SSTR
+                i += 1
+                line.append(c)
+            elif c == '\n':
+                i += 1
+                finish_line()
+            else:
+                i += 1
+                line.append(c)
+        elif state == CHASH or state == CSLASHSLASH:
+            if c != '\n':  # comment runs until EOL
+                i += 1
+            else:
+                state = SRC
+                i += 1
+                finish_line()
+        elif state == CSLASHSTAR:
+            if c == '*' and cn == '/':  # ending a comment */
+                state = SRC
+                i += 2
+            elif c == '\n':
+                i += 1
+                finish_line()
+            else:
+                i += 1
+        elif state == DSTR and c == '"' or state == SSTR and c == "'":  # string end
+            state = SRC
+            i += 1
+            line.append(c)
+        elif state == DSTR or state == SSTR:
+            i += 1
+            line.append(c)
+            if c == '\\':  # escaping backslash
+                i += 1  # do not look at char after the backslash
+                line.append(cn)
+        else:
+            raise Exception("state: %d c: %s cn: %s" % (state, c, cn))
+    if line:
+        # no final \n triggered processing these chars yet, do it now
+        finish_line()
+    return lines
+
+
+if __name__ == '__main__':
+    import sys
+    filename = sys.argv[1]
+    with open(filename, "r") as f:
+        text = f.read()
+    lines = remove_comments(text)
+    with open(filename + ".nocomments", "w") as f:
+        for line in lines:
+            f.write(line + '\n')
+

tests/assemble.py

@@ -1,15 +1,12 @@
 from esp32_ulp.assemble import Assembler, TEXT, DATA, BSS
 
 src = """\
-# line 1
 
-start:  wait 42    # line 3
-
-        # line 5
+start:  wait 42
         ld r0, r1, 0
-        st  r0,  r1,0    # line 7
+        st  r0,  r1,0
         halt
-end:    // C style comment
+end:
 """
 
 
@@ -18,27 +15,22 @@ def test_parse_line():
     lines = src.splitlines()
     # note: line number = index + 1
     assert a.parse_line(lines[0]) == None
-    assert a.parse_line(lines[1]) == None
-    assert a.parse_line(lines[2]) == ('start', 'wait', ('42', ))
-    assert a.parse_line(lines[3]) == None
-    assert a.parse_line(lines[4]) == None
-    assert a.parse_line(lines[5]) == (None, 'ld', ('r0', 'r1', '0', ))
-    assert a.parse_line(lines[6]) == (None, 'st', ('r0', 'r1', '0', ))
-    assert a.parse_line(lines[7]) == (None, 'halt', ())
-    assert a.parse_line(lines[8]) == ('end', None, ())
+    assert a.parse_line(lines[1]) == ('start', 'wait', ('42', ))
+    assert a.parse_line(lines[2]) == (None, 'ld', ('r0', 'r1', '0', ))
+    assert a.parse_line(lines[3]) == (None, 'st', ('r0', 'r1', '0', ))
+    assert a.parse_line(lines[4]) == (None, 'halt', ())
+    assert a.parse_line(lines[5]) == ('end', None, ())
 
 
 def test_parse():
     a = Assembler()
-    lines = src.splitlines()
-    result = a.parse(lines)
+    result = a.parse(src)
     assert None not in result
 
 
 def test_assemble():
     a = Assembler()
-    lines = src.splitlines()
-    a.assemble(lines)
+    a.assemble(src)
     assert {'start', 'end'} <= set(a.symbols)
     assert a.symbols['start'] == (TEXT, 0)
     assert a.symbols['end'] == (TEXT, 4)

tests/nocomment.py

@@ -0,0 +1,60 @@
+from esp32_ulp.nocomment import remove_comments
+
+ORIG = """\
+/*
+ * HEADER
+ */
+
+# single, full line comment
+
+label:  // another rest-of-line comment
+
+        nop     /* partial line */
+
+        .string "try confusing /* with a comment start"
+        .string "should be there 1"  /* another comment */
+        .string 'try confusing */ with a comment end'
+
+        .string "more confusing \\" /* should be there 2"
+        /* comment */
+        .string 'more confusing \\' */'
+
+/***** FOOTER *****/
+"""
+
+EXPECTED = """\
+
+
+
+
+
+
+label:
+
+	nop
+
+	.string "try confusing /* with a comment start"
+	.string "should be there 1"
+	.string 'try confusing */ with a comment end'
+
+	.string "more confusing \\" /* should be there 2"
+
+	.string 'more confusing \\' */'
+
+
+"""
+
+
+def test_remove_comments():
+    lines_orig = ORIG.splitlines()
+    len_orig = len(lines_orig)
+    lines_expected = EXPECTED.splitlines()
+    len_expected = len(lines_expected)
+    lines_got = remove_comments(ORIG)
+    len_got = len(lines_got)
+    assert len_orig == len_expected == len_got, \
+           "line count differs %d %d %d" % (len_orig, len_expected, len_got)
+    assert lines_expected == lines_got, "texts differ"
+
+
+test_remove_comments()