add support for evaluating expressions

This change allows immediate values to be calculated from an expression,
such as 1+1, even including symbols such as "100 << const" (where const
was defined with the .set directive). Expressions are also supported in
the .set directives.

Expressions are evaluated using the built-in eval(). To prevent misuse or
malicious code execution, expressions are validated.

At the point when eval is called, all symbols should have already been resolved
to their values. That means we only need to allow for numeric characters along
with arithmetic and bitwise operators, round brackets and whitespace. The
character 'x' and the characters 'abcdef' are also accepted to allow for hex
numbers such as 0x123abc. These are only allowed however in sequences starting
with 0x. If any other character is encountered the expression is deemed invalid
and an exception is raised.

Author: wnienhaus

esp32_ulp/assemble.py

@@ -240,7 +240,7 @@ def d_align(self, align=4, fill=None):
             self.fill(self.section, amount, fill)
 
     def d_set(self, symbol, expr):
-        value = int(expr)  # TODO: support more than just integers
+        value = int(opcodes.eval_arg(expr))  # TODO: support more than just integers
         self.symbols.set_sym(symbol, ABS, None, value)
 
     def d_global(self, symbol):

esp32_ulp/opcodes.py

@@ -6,6 +6,7 @@
 from uctypes import struct, addressof, LITTLE_ENDIAN, UINT32, BFUINT32, BF_POS, BF_LEN
 
 from .soc import *
+from .util import split_tokens, validate_expression
 
 # XXX dirty hack: use a global for the symbol table
 symbols = None
@@ -267,6 +268,20 @@ def make_ins(layout):
 ARG = namedtuple('ARG', ('type', 'value', 'raw'))
 
 
+def eval_arg(arg):
+    parts = []
+    for token in split_tokens(arg):
+        if symbols.has_sym(token):
+            _, _, sym_value = symbols.get_sym(token)
+            parts.append(str(sym_value))
+        else:
+            parts.append(token)
+    parts = "".join(parts)
+    if not validate_expression(parts):
+        raise ValueError('Unsupported expression: %s' % parts)
+    return eval(parts)
+
+
 def arg_qualify(arg):
     """
     look at arg and qualify its type:
@@ -289,8 +304,12 @@ def arg_qualify(arg):
         return ARG(IMM, int(arg), arg)
     except ValueError:
         pass
-    entry = symbols.get_sym(arg)
-    return ARG(SYM, entry, arg)
+    try:
+        entry = symbols.get_sym(arg)
+        return ARG(SYM, entry, arg)
+    except KeyError:
+        pass
+    return ARG(IMM, int(eval_arg(arg)), arg)
 
 
 def get_reg(arg):

esp32_ulp/util.py

@@ -2,10 +2,68 @@
 
 import gc
 
+NORMAL, WHITESPACE = 0, 1
+
 
 def garbage_collect(msg, verbose=DEBUG):
     free_before = gc.mem_free()
     gc.collect()
     free_after = gc.mem_free()
     if verbose:
         print("%s: %d --gc--> %d bytes free" % (msg, free_before, free_after))
+
+
+def split_tokens(line):
+    buf = ""
+    tokens = []
+    state = NORMAL
+    for c in line:
+        if ('a' <= c <= 'z') or ('A' <= c <= 'Z') or ('0' <= c <= '9') or c == '_':
+            if state != NORMAL:
+                if len(buf) > 0:
+                    tokens.append(buf)
+                    buf = ""
+                state = NORMAL
+            buf += c
+        elif c == ' ' or c == '\t':
+            if state != WHITESPACE:
+                if len(buf) > 0:
+                    tokens.append(buf)
+                    buf = ""
+                state = WHITESPACE
+            buf += c
+        else:
+            if len(buf) > 0:
+                tokens.append(buf)
+                buf = ""
+            tokens.append(c)
+
+    if len(buf) > 0:
+        tokens.append(buf)
+
+    return tokens
+
+
+def validate_expression(param):
+    for token in split_tokens(param):
+        state = 0
+        for c in token:
+            if c not in ' \t+-*/%()<>&|~x0123456789abcdef':
+                return False
+
+            # the following allows hex digits a-f after 0x but not otherwise
+            if state == 0:
+                if c in 'abcdef':
+                    return False
+                if c == '0':
+                    state = 1
+                continue
+
+            if state == 1:
+                state = 2 if c == 'x' else 0
+                continue
+
+            if state == 2:
+                if c not in '0123456789abcdef':
+                    state = 0
+    return True

tests/00_unit_tests.sh

@@ -4,7 +4,7 @@
 
 set -e
 
-for file in opcodes assemble link ; do
+for file in opcodes assemble link util; do
     echo testing $file...
     micropython $file.py
 done

tests/assemble.py

@@ -135,6 +135,28 @@ def test_assemble_uppercase_opcode():
     assert not raised
 
 
+def test_assemble_evalulate_expressions():
+    src_w_expr = """\
+    .set shft, 2
+    .set loops, (1 << shft)
+
+entry:
+    move r0, 1+1
+    move r1, loops
+    move r2, (shft + 10) * 2
+    move r3, entry << 2
+"""
+    a = Assembler()
+    a.assemble(src_w_expr)
+
+    assert a.symbols.has_sym('shft')
+    assert a.symbols.has_sym('loops')
+    assert a.symbols.has_sym('entry')
+    assert a.symbols.get_sym('shft') == (ABS, None, 2)
+    assert a.symbols.get_sym('loops') == (ABS, None, 4)
+    assert a.symbols.get_sym('entry') == (REL, TEXT, 0)
+
+
 def test_symbols():
     st = SymbolTable({}, {}, {})
     for entry in [
@@ -195,4 +217,5 @@ def test_symbols():
 test_assemble_bss_with_value()
 test_assemble_global()
 test_assemble_uppercase_opcode()
+test_assemble_evalulate_expressions()
 test_symbols()

tests/opcodes.py

@@ -1,6 +1,8 @@
 from uctypes import UINT32, BFUINT32, BF_POS, BF_LEN
 from esp32_ulp.opcodes import make_ins, make_ins_struct_def
-from esp32_ulp.opcodes import get_reg, get_imm, get_cond, arg_qualify, ARG, REG, IMM, COND
+from esp32_ulp.opcodes import get_reg, get_imm, get_cond, arg_qualify, eval_arg, ARG, REG, IMM, SYM, COND
+from esp32_ulp.assemble import SymbolTable, ABS, REL, TEXT
+import esp32_ulp.opcodes as opcodes
 
 OPCODE_DELAY = 4
 LAYOUT_DELAY = """
@@ -43,6 +45,19 @@ def test_arg_qualify():
     assert arg_qualify('Eq') == ARG(COND, 'eq', 'Eq')
     assert arg_qualify('EQ') == ARG(COND, 'eq', 'EQ')
 
+    # for the next tests, ensure the opcodes module has a SymbolTable
+    opcodes.symbols = SymbolTable({}, {}, {})
+    opcodes.symbols.set_sym('const', ABS, None, 42)  # constant as defined by .set
+    opcodes.symbols.set_sym('entry', REL, TEXT, 4)  # label pointing to code
+
+    assert arg_qualify('1+1') == ARG(IMM, 2, '1+1')
+    assert arg_qualify('const >> 1') == ARG(IMM, 21, 'const >> 1')
+    assert arg_qualify('entry') == ARG(SYM, (REL, TEXT, 4), 'entry')  # symbols should not (yet) be evaluated
+    assert arg_qualify('entry + const') == ARG(IMM, 46, 'entry + const')
+
+    # clean up
+    opcodes.symbols = None
+
 
 def test_get_reg():
     assert get_reg('r0') == 0
@@ -57,9 +72,46 @@ def test_get_cond():
     assert get_cond('Eq') == 'eq'
 
 
+def test_eval_arg():
+    opcodes.symbols = SymbolTable({}, {}, {})
+    opcodes.symbols.set_sym('const', ABS, None, 42)  # constant
+    opcodes.symbols.set_sym('raise', ABS, None, 99)  # constant using a python keyword as name (is allowed)
+
+    assert eval_arg('1+1') == 2
+    assert eval_arg('1+const') == 43
+    assert eval_arg('raise*2/3') == 66
+    assert eval_arg('raise-const') == 57
+    assert eval_arg('(raise-const)*2') == 114
+    assert eval_arg('const    % 5') == 2
+    assert eval_arg('const + 0x19af') == 0x19af + 42
+    assert eval_arg('const & ~2') == 40
+    assert eval_arg('const << 3') == 336
+    assert eval_arg('const >> 1') == 21
+    assert eval_arg('(const|4)&0xf') == 0xe
+
+    assert_raises(ValueError, eval_arg, 'evil()')
+    assert_raises(ValueError, eval_arg, 'def cafe()')
+    assert_raises(ValueError, eval_arg, '1 ^ 2')
+    assert_raises(ValueError, eval_arg, '!100')
+
+    # clean up
+    opcodes.symbols = None
+
+
+def assert_raises(exception, func, *args):
+    try:
+        func(*args)
+    except exception:
+        raised = True
+    else:
+        raised = False
+    assert raised
+
+
 test_make_ins_struct_def()
 test_make_ins()
 test_arg_qualify()
 test_get_reg()
 test_get_imm()
 test_get_cond()
+test_eval_arg()

tests/util.py

@@ -0,0 +1,62 @@
+from esp32_ulp.util import split_tokens, validate_expression
+
+tests = []
+
+
+def test(param):
+    """
+    the @test decorator
+    """
+    tests.append(param)
+
+
+@test
+def test_split_tokens():
+    assert split_tokens("") == []
+    assert split_tokens("t") == ['t']
+    assert split_tokens("test") == ['test']
+    assert split_tokens("t t") == ['t', ' ', 't']
+    assert split_tokens("t,t") == ['t', ',', 't']
+    assert split_tokens("test(arg)") == ['test', '(', 'arg', ')']
+    assert split_tokens("test(arg,arg2)") == ['test', '(', 'arg', ',', 'arg2', ')']
+    assert split_tokens("test(arg,arg2)") == ['test', '(', 'arg', ',', 'arg2', ')']
+    assert split_tokens("  test(  arg,  arg2)") == ['  ', 'test', '(', '  ', 'arg', ',', '  ', 'arg2', ')']
+    assert split_tokens("  test(  arg )  ") == ['  ', 'test', '(', '  ', 'arg', ' ', ')', '  ']
+    assert split_tokens("\t  test  \t  ") == ['\t  ', 'test', "  \t  "]
+    assert split_tokens("test\nrow2") == ['test', "\n", "row2"]
+
+    # split_token does not support comments. should generally only be used after comments are already stripped
+    assert split_tokens("test(arg /*comment*/)") == ['test', '(', 'arg', ' ', '/', '*', 'comment', '*', '/', ')']
+    assert split_tokens("#test") == ['#', 'test']
+
+
+@test
+def test_validate_expression():
+    assert validate_expression('') is True
+    assert validate_expression('1') is True
+    assert validate_expression('1+1') is True
+    assert validate_expression('(1+1)') is True
+    assert validate_expression('(1+1)*2') is True
+    assert validate_expression('(1 + 1)') is True
+    assert validate_expression('10 % 2') is True
+    assert validate_expression('0x100 << 2') is True
+    assert validate_expression('0x100 & ~2') is True
+    assert validate_expression('0xabcdef') is True
+    assert validate_expression('0x123def') is True
+    assert validate_expression('2*3+4/5&6|7') is True
+    assert validate_expression('(((((1+1) * 2') is True  # valid characters, even if expression is not valid
+
+    assert validate_expression(':') is False
+    assert validate_expression('_') is False
+    assert validate_expression('=') is False
+    assert validate_expression('.') is False
+    assert validate_expression('!') is False
+    assert validate_expression('123 ^ 4') is False  # operator not supported for now
+    assert validate_expression('evil()') is False
+    assert validate_expression('def cafe()') is False  # valid hex digits, but potentially dangerous code
+
+
+if __name__ == '__main__':
+    # run all methods marked with @test
+    for t in tests:
+        t()