Add initial benchmarks: djangocms + flaskblogging + mypy + pylint

Author: kmod

.gitignore

@@ -127,3 +127,5 @@ dmypy.json
 
 # Pyre type checker
 .pyre/
+
+results

benchmarks/djangocms.py

@@ -0,0 +1,128 @@
+"""
+Django-cms test
+Sets up a djangocms installation, and hits '/' a number of times.
+'/' is not super interesting, but it still exercises a little bit of
+functionality; looking at cms/templates/cms/welcome.html, it seems
+to do a decent amount of template logic, as well as do some basic
+user auth.
+We could probably improve the flow though, perhaps by logging in
+and browsing around.
+"""
+
+import os
+import requests
+import socket
+import subprocess
+import sys
+import tempfile
+import time
+import json
+
+def setup():
+    """
+    Set up a djangocms installation.
+    Runs the initial bootstrapping without the db migration,
+    so that we can turn off sqlite synchronous and avoid fs time.
+    Rough testing shows that setting synchronous=OFF is basically
+    the same performance as running on /dev/shm
+    """
+
+    subprocess.check_call([exe.replace("python3", "djangocms"), "testsite", "--verbose", "--no-sync"])
+
+    with open("testsite/testsite/settings.py", "a") as f:
+        f.write("""
+from django.db.backends.signals import connection_created
+def set_no_sychronous(sender, connection, **kwargs):
+    if connection.vendor == 'sqlite':
+        cursor = connection.cursor()
+        cursor.execute('PRAGMA synchronous = OFF;')
+
+connection_created.connect(set_no_sychronous)
+""")
+    start = time.time()
+    subprocess.check_call([exe, "manage.py", "migrate"], cwd="testsite")
+    elapsed = time.time() - start
+    print("%.2fs to initialize db" % (elapsed,))
+
+def waitUntilUp(addr, timeout=10.0):
+    start = time.time()
+    while True:
+        try:
+            with socket.create_connection(addr) as sock:
+                return
+        except ConnectionRefusedError:
+            if time.time() > start + timeout:
+                raise Exception("Timeout reached when trying to connect")
+            time.sleep(0.001)
+
+def runbenchmark(n=800, out_file=None):
+    p = subprocess.Popen([exe, "manage.py", "runserver", "--noreload"], cwd="testsite", stdout=open("/dev/null", "w"), stderr=subprocess.STDOUT)
+    try:
+        waitUntilUp(("127.0.0.1", 8000))
+
+        start = time.time()
+        times = []
+        for i in range(n):
+            times.append(time.time())
+            if i % 100 == 0:
+                print(i, time.time() - start)
+            requests.get("http://localhost:8000/").text
+        times.append(time.time())
+        elapsed = time.time() - start
+        print("%.2fs (%.3freq/s)" % (elapsed, n / elapsed))
+
+        exitcode = p.poll()
+        assert exitcode is None, exitcode
+
+        if out_file:
+            json.dump(times, open(out_file, 'w'))
+
+    finally:
+        p.terminate()
+
+if __name__ == "__main__":
+    exe = sys.executable
+    # Hack: make sure this file gets run as "python3" so that perf will collate across different processes
+    if not exe.endswith('3'):
+        os.execv(exe + '3', [exe + '3'] + sys.argv)
+
+    os.environ["PATH"] = os.path.dirname(exe) + ":" + os.environ["PATH"]
+
+    """
+    Usage:
+        python djangocms.py
+        python djangocms.py --setup DIR
+        python djangocms.py --serve DIR
+
+    The first form creates a temporary directory, sets up djangocms in it,
+    serves out of it, and removes the directory.
+    The second form sets up a djangocms installation in the given directory.
+    The third form runs a benchmark out of an already-set-up directory
+    The second and third forms are useful if you want to benchmark the
+    initial migration phase separately from the second serving phase.
+    """
+    if "--setup" in sys.argv:
+        assert len(sys.argv) > 2
+        dir = sys.argv[-1]
+        os.makedirs(dir, exist_ok=True)
+        os.chdir(dir)
+        setup()
+    elif "--serve" in sys.argv:
+        assert len(sys.argv) > 2
+        os.chdir(sys.argv[-1])
+        runbenchmark()
+    else:
+        n = 800
+        if len(sys.argv) > 1:
+            n = int(sys.argv[1])
+        out_file = None
+        if len(sys.argv) > 2:
+            out_file = os.path.abspath(sys.argv[2])
+
+        # It might be interesting to put the temporary directory in /dev/shm,
+        # which makes the initial db migration about 20% faster.
+        with tempfile.TemporaryDirectory(prefix="djangocms_test_") as d:
+            os.chdir(d)
+
+            setup()
+            runbenchmark(n, out_file)

benchmarks/djangocms_requirements.txt

@@ -0,0 +1,39 @@
+asgiref==3.3.0
+certifi==2020.6.20
+chardet==3.0.4
+dj-database-url==0.5.0
+Django==3.0.10
+django-classy-tags==2.0.0
+django-cms==3.7.4
+django-filer==2.0.2
+django-formtools==2.2
+django-js-asset==1.2.2
+django-mptt==0.11.0
+django-polymorphic==3.0.0
+django-sekizai==2.0.0
+django-treebeard==4.3.1
+djangocms-admin-style==1.5.0
+djangocms-attributes-field==2.0.0
+djangocms-bootstrap4==1.6.0
+djangocms-file==2.4.0
+djangocms-googlemap==1.4.0
+djangocms-icon==1.5.0
+djangocms-installer==1.2.3
+djangocms-link==2.6.1
+djangocms-picture==2.4.0
+djangocms-snippet==2.3.0
+djangocms-style==2.3.0
+djangocms-text-ckeditor==3.10.0
+djangocms-video==2.3.0
+easy-thumbnails==2.7
+html5lib==1.1
+idna==2.10
+Pillow==8.0.0
+pytz==2020.1
+requests==2.24.0
+six==1.15.0
+sqlparse==0.4.1
+tzlocal==2.1
+Unidecode==1.1.1
+urllib3==1.25.11
+webencodings==0.5.1

benchmarks/flaskblogging.py

@@ -0,0 +1,40 @@
+import json
+import os
+import requests
+import subprocess
+import sys
+import threading
+import time
+
+from djangocms import waitUntilUp
+
+if __name__ == "__main__":
+    exe = sys.executable
+
+    times = []
+
+    p = subprocess.Popen([exe, "../data/flaskblogging_serve.py"], stdout=open("/dev/null", "w"), stderr=subprocess.STDOUT, cwd=os.path.dirname(__file__))
+    try:
+        waitUntilUp(("127.0.0.1", 8000))
+
+        n = 1800
+        if len(sys.argv) > 1:
+            n = int(sys.argv[1])
+
+        start = time.time()
+        for i in range(n):
+            times.append(time.time())
+            if i % 100 == 0:
+                print(i, time.time() - start)
+            requests.get("http://localhost:8000/blog/").text
+        times.append(time.time())
+        elapsed = time.time() - start
+        print("%.2fs (%.3freq/s)" % (elapsed, n / elapsed))
+
+        assert p.poll() is None, p.poll()
+
+    finally:
+        p.terminate()
+
+    if len(sys.argv) > 2:
+        json.dump(times, open(sys.argv[2], 'w'))

benchmarks/flaskblogging_requirements.txt

@@ -0,0 +1,24 @@
+blinker==1.4
+certifi==2020.6.20
+chardet==3.0.4
+click==7.1.2
+Flask==1.1.2
+Flask-Blogging==1.2.2
+Flask-Caching==1.9.0
+Flask-FileUpload==0.5.0
+Flask-Login==0.5.0
+Flask-Principal==0.4.0
+Flask-WTF==0.14.3
+idna==2.10
+itsdangerous==1.1.0
+Jinja2==2.11.2
+Markdown==3.3.2
+MarkupSafe==1.1.1
+python-slugify==4.0.1
+requests==2.24.0
+shortuuid==1.0.1
+SQLAlchemy==1.3.20
+text-unidecode==1.3
+urllib3==1.25.11
+Werkzeug==0.16.0
+WTForms==2.3.3

benchmarks/mypy_bench.py

@@ -0,0 +1,33 @@
+import json
+import os
+import sys
+import time
+
+"""
+I tested it, and it looks like we get the same performance conclusions
+when we run on the same file multiple times as if we run on a set of files once.
+
+So for convenience run on a single file multiple times.
+"""
+
+if __name__ == "__main__":
+    from mypy.main import main
+
+    n = 20
+    if len(sys.argv) > 1:
+        n = int(sys.argv[1])
+    target = os.path.join(os.path.dirname(__file__), "../data/mypy_target.py")
+
+    times = []
+    devnull = open("/dev/null", "w")
+    for i in range(n):
+        times.append(time.time())
+        print(i)
+        try:
+            main(None, devnull, devnull, [target])
+        except SystemExit:
+            pass
+    times.append(time.time())
+
+    if len(sys.argv) > 2:
+        json.dump(times, open(sys.argv[2], 'w'))

benchmarks/mypy_bench_requirements.txt

@@ -0,0 +1,4 @@
+mypy==0.790
+mypy-extensions==0.4.3
+typed-ast==1.4.1
+typing-extensions==3.7.4.3

benchmarks/pylint_bench.py

@@ -0,0 +1,37 @@
+import json
+import os
+import subprocess
+import sys
+import time
+
+from pylint import epylint as lint
+from pylint.lint import Run
+
+"""
+pylint benchmark
+
+pylint seems to speed up considerably as it progresses, and this
+benchmark includes that
+"""
+
+if __name__ == "__main__":
+    def noop(*args, **kw):
+        pass
+    class NullReporter:
+        path_strip_prefix = "/"
+        def __getattr__(self, attr):
+            return noop
+
+    n = 10
+    if len(sys.argv) > 1:
+        n = int(sys.argv[1])
+
+    times = []
+    for i in range(n):
+        times.append(time.time())
+        print(i)
+        Run([os.path.join(os.path.dirname(__file__), "../data/pylint_target/dist.py")], exit=False, reporter=NullReporter())
+    times.append(time.time())
+
+    if len(sys.argv) > 2:
+        json.dump(times, open(sys.argv[2], 'w'))

benchmarks/pylint_bench_requirements.txt

@@ -0,0 +1,8 @@
+astroid==2.4.2
+isort==5.6.4
+lazy-object-proxy==1.4.3
+mccabe==0.6.1
+pylint==2.6.0
+six==1.15.0
+toml==0.10.1
+wrapt==1.12.1