Merge branch 'bench-compliance' into develop

* bench-compliance:
  Use benchmarks.json file for benchmarking
  Change perf test to be time based

Author: jamesls

perf/perftest.py

@@ -11,20 +11,21 @@
 import os
 import json
 import sys
+import timeit
 
-try:
-    _clock = time.process_time
-except AttributeError:
-    # Python 2.x does not have time.process_time
-    _clock = time.clock
+_clock = timeit.default_timer
 
 
 from jmespath.parser import Parser
 from jmespath.lexer import Lexer
 
 
-DIRECTORY = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'cases')
-DEFAULT_NUM_LOOP = 100
+BENCHMARK_FILE = os.path.join(
+    os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
+    'tests',
+    'compliance',
+    'benchmarks.json')
+APPROX_RUN_TIME = 0.5
 
 
 def run_tests(tests):
@@ -33,64 +34,82 @@ def run_tests(tests):
         given = test['given']
         expression = test['expression']
         result = test['result']
+        should_search = test['bench_type'] == 'full'
         lex_time = _lex_time(expression)
         parse_time = _parse_time(expression)
-        search_time = _search_time(expression, given)
-        combined_time = _combined_time(expression, given, result)
+        if should_search:
+            search_time = _search_time(expression, given)
+            combined_time = _combined_time(expression, given, result)
+        else:
+            search_time = 0
+            combined_time = 0
         sys.stdout.write(
-            "lex_time: %.5fms, parse_time: %.5fms, search_time: %.5fms "
-            "combined_time: %.5fms " % (1000 * lex_time,
-                                        1000 * parse_time,
-                                        1000 * search_time,
-                                        1000 * combined_time))
+            "lex_time: %10.5fus, parse_time: %10.5fus, search_time: %10.5fus "
+            "combined_time: %10.5fus " % (1000000 * lex_time,
+                                          1000000 * parse_time,
+                                          1000000 * search_time,
+                                          1000000 * combined_time))
         sys.stdout.write("name: %s\n" % test['name'])
 
 
 def _lex_time(expression, clock=_clock):
-    best = float('inf')
     lex = Lexer()
-    for i in range(DEFAULT_NUM_LOOP):
+    duration = 0
+    i = 0
+    while True:
+        i += 1
         start = clock()
         list(lex.tokenize(expression))
         end = clock()
         total = end - start
-        if total < best:
-            best = total
-    return best
+        duration += total
+        if duration >= APPROX_RUN_TIME:
+            break
+    return duration / i
 
 
 def _search_time(expression, given, clock=_clock):
     p = Parser()
     parsed = p.parse(expression)
-    best = float('inf')
-    for i in range(DEFAULT_NUM_LOOP):
+    duration =  0
+    i = 0
+    while True:
+        i += 1
         start = clock()
         parsed.search(given)
         end = clock()
         total = end - start
-        if total < best:
-            best = total
-    return best
+        duration += total
+        if duration >= APPROX_RUN_TIME:
+            break
+    return duration / i
 
 
 def _parse_time(expression, clock=_clock):
     best = float('inf')
     p = Parser()
-    for i in range(DEFAULT_NUM_LOOP):
+    duration = 0
+    i = 0
+    while True:
+        i += 1
         p.purge()
         start = clock()
         p.parse(expression)
         end = clock()
         total = end - start
-        if total < best:
-            best = total
-    return best
+        duration += total
+        if duration >= APPROX_RUN_TIME:
+            break
+    return duration / i
 
 
 def _combined_time(expression, given, result, clock=_clock):
     best = float('inf')
     p = Parser()
-    for i in range(DEFAULT_NUM_LOOP):
+    duration = 0
+    i = 0
+    while True:
+        i += 1
         p.purge()
         start = clock()
         r = p.parse(expression).search(given)
@@ -99,9 +118,10 @@ def _combined_time(expression, given, result, clock=_clock):
         if r != result:
             raise RuntimeError("Unexpected result, received: %s, "
                                "expected: %s" % (r, result))
-        if total < best:
-            best = total
-    return best
+        duration += total
+        if duration >= APPROX_RUN_TIME:
+            break
+    return duration / i
 
 
 def load_tests(filename):
@@ -118,28 +138,23 @@ def load_tests(filename):
 
 def _add_cases(data, loaded, filename):
     for case in data['cases']:
-        current = {'description': data.get('description', filename),
-                   'given': data['given'],
-                   'name': case.get('name', case['expression']),
-                   'expression': case['expression'],
-                   'result': case.get('result')}
+        current = {
+            'given': data['given'],
+            'name': case.get('comment', case['expression']),
+            'expression': case['expression'],
+            'result': case.get('result'),
+            'bench_type': case['bench'],
+        }
         loaded.append(current)
     return loaded
 
 
 def main():
     parser = argparse.ArgumentParser()
-    parser.add_argument('-d', '--directory', default=DIRECTORY)
-    parser.add_argument('-f', '--filename')
+    parser.add_argument('-f', '--filename', default=BENCHMARK_FILE)
     args = parser.parse_args()
     collected_tests = []
-    if args.filename:
-        collected_tests.extend(load_tests(args.filename))
-    else:
-        for filename in os.listdir(args.directory):
-            if filename.endswith('.json'):
-                full_path = os.path.join(args.directory, filename)
-                collected_tests.extend(load_tests(full_path))
+    collected_tests.extend(load_tests(args.filename))
     run_tests(collected_tests)
 
 

tests/compliance/benchmarks.json

@@ -32,27 +32,41 @@
             }
           }
         }
+      },
+      "b": true,
+      "c": {
+        "d": true
       }
     },
     "cases": [
       {
         "comment": "simple field",
-        "expression": "a",
+        "expression": "b",
+        "result": true,
         "bench": "full"
       },
       {
         "comment": "simple subexpression",
-        "expression": "a.b",
+        "expression": "c.d",
+        "result": true,
         "bench": "full"
       },
       {
-        "comment": "deep field selection",
+        "comment": "deep field selection no match",
         "expression": "a.b.c.d.e.f.g.h.i.j.k.l.m.n.o.p.q.r.s",
+        "result": null,
+        "bench": "full"
+      },
+      {
+        "comment": "deep field selection",
+        "expression": "a.b.c.d.e.f.g.h.i.j.k.l.m.n.o.p",
+        "result": true,
         "bench": "full"
       },
       {
         "comment": "simple or",
-        "expression": "not_there || a",
+        "expression": "not_there || b",
+        "result": true,
         "bench": "full"
       }
     ]
@@ -67,21 +81,31 @@
       {
         "comment": "deep ands",
         "expression": "a && b && c && d && e && f && g && h && i && j && k && l && m && n && o && p && q && r && s && t && u && v && w && x && y && z",
+        "result": 25,
         "bench": "full"
       },
       {
         "comment": "deep ors",
         "expression": "z || y || x || w || v || u || t || s || r || q || p || o || n || m || l || k || j || i || h || g || f || e || d || c || b || a",
+        "result": 25,
         "bench": "full"
       },
       {
         "comment": "lots of summing",
-        "expression": "sum(z, y, x, w, v, u, t, s, r, q, p, o, n, m, l, k, j, i, h, g, f, e, d, c, b, a)",
+        "expression": "sum([z, y, x, w, v, u, t, s, r, q, p, o, n, m, l, k, j, i, h, g, f, e, d, c, b, a])",
+        "result": 325,
+        "bench": "full"
+      },
+      {
+        "comment": "lots of function application",
+        "expression": "sum([z, sum([y, sum([x, sum([w, sum([v, sum([u, sum([t, sum([s, sum([r, sum([q, sum([p, sum([o, sum([n, sum([m, sum([l, sum([k, sum([j, sum([i, sum([h, sum([g, sum([f, sum([e, sum([d, sum([c, sum([b, a])])])])])])])])])])])])])])])])])])])])])])])])])",
+        "result": 325,
         "bench": "full"
       },
       {
         "comment": "lots of multi list",
         "expression": "[z, y, x, w, v, u, t, s, r, q, p, o, n, m, l, k, j, i, h, g, f, e, d, c, b, a]",
+        "result": [25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0],
         "bench": "full"
       }
     ]
@@ -116,7 +140,7 @@
       },
       {
         "comment": "filter projection",
-        "expression": "foo[bar > baz][qux > baz]",
+        "expression": "foo[?bar > baz][?qux > baz]",
         "bench": "parse"
       }
     ]