Merge pull request #93 from dflook/constants

dflook · web-flow · commit eb46ee627bcb · 2023-09-02T18:19:47.000+01:00
Add simple constant folding
diff --git a/.github/workflows/test_corpus.yaml b/.github/workflows/test_corpus.yaml
@@ -74,6 +74,13 @@ jobs:
 
       - name: Run tests
         run: |
+          if [[ "${{ matrix.python }}" == "python3.3" || "${{ matrix.python }}" == "python3.4" || "${{ matrix.python }}" == "python3.5" ]]; then
+            # These versions randomise the hash seed, but don't preserve dict order
+            # This affects how names are assigned. Disable the hash randomisation for
+            # deterministic results.
+            export PYTHONHASHSEED=0
+          fi
+          
           ${{matrix.python}} workflow/corpus_test/generate_results.py /corpus /corpus-results $(<sha.txt) ${{ inputs.regenerate-results }}
 
   generate_report:
diff --git a/docker/Dockerfile-fuzz b/docker/Dockerfile-fuzz
@@ -0,0 +1,16 @@
+FROM fedora:38 AS fuzz
+
+RUN dnf install -y \
+      python3 \
+      python3-pip \
+  && dnf clean all && rm -rf /var/cache/dnf/*
+
+RUN pip install hypothesis[cli] hypofuzz
+
+COPY fuzz.sh /fuzz.sh
+
+WORKDIR /tmp/work
+ENTRYPOINT ["/fuzz.sh"]
+
+EXPOSE 9999/tcp
+VOLUME /tmp/work
diff --git a/docker/build-images.sh b/docker/build-images.sh
@@ -26,6 +26,9 @@ docker build --tag danielflook/python-minifier-build:python3.10-$DATE -f Dockerf
 docker pull fedora:36
 docker build --tag danielflook/python-minifier-build:python3.11-$DATE -f Dockerfile-fedora36 --target python3.11 .
 
+docker pull fedora:38
+docker build --tag danielflook/python-minifier-build:fuzz-$DATE -f Dockerfile-fuzz --target fuzz .
+
 docker push danielflook/python-minifier-build:python3.3-$DATE
 docker push danielflook/python-minifier-build:python2.7-$DATE
 docker push danielflook/python-minifier-build:python3.4-$DATE
@@ -38,3 +41,4 @@ docker push danielflook/python-minifier-build:python3.10-$DATE
 docker push danielflook/python-minifier-build:python3.11-$DATE
 docker push danielflook/python-minifier-build:pypy-$DATE
 docker push danielflook/python-minifier-build:pypy3-$DATE
+docker push danielflook/python-minifier-build:fuzz-$DATE
diff --git a/docker/fuzz.sh b/docker/fuzz.sh
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+pip install .
+
+exec hypothesis fuzz hypo_test
diff --git a/docs/source/transforms/constant_folding.py b/docs/source/transforms/constant_folding.py
@@ -0,0 +1,2 @@
+SECONDS_IN_A_DAY = 60 * 60 * 24
+SECONDS_IN_A_WEEK = SECONDS_IN_A_DAY * 7
diff --git a/docs/source/transforms/constant_folding.rst b/docs/source/transforms/constant_folding.rst
@@ -0,0 +1,25 @@
+Constant Folding
+================
+
+This transform evaluates constant expressions with literal operands when minifying and replaces the expression with the resulting value, if the value is shorter than the expression.
+
+There are some limitations, notably the division and power operators are not evaluated.
+
+This will be most effective with numeric literals.
+
+This transform is always safe and enabled by default. Disable by passing the ``constant_folding=False`` argument to the :func:`python_minifier.minify` function,
+or passing ``--no-constant-folding`` to the pyminify command.
+
+Example
+-------
+
+Input
+~~~~~
+
+.. literalinclude:: constant_folding.py
+
+Output
+~~~~~~
+
+.. literalinclude:: constant_folding.min.py
+    :language: python
diff --git a/docs/source/transforms/index.rst b/docs/source/transforms/index.rst
@@ -19,6 +19,7 @@ They can be enabled or disabled through the minify function, or passing options
    convert_posargs_to_args
    preserve_shebang
    remove_explicit_return_none
+   constant_folding
 
 .. toctree::
    :caption: Disabled by default
diff --git a/hypo_test/expressions.py b/hypo_test/expressions.py
@@ -240,12 +240,14 @@ def arguments(draw, for_lambda=False) -> ast.arguments:
     allow_annotation = False if for_lambda else True
 
     args = draw(lists(arg(allow_annotation), max_size=2))
+    posonlyargs = draw(lists(arg(allow_annotation), max_size=2))
     kwonlyargs = draw(lists(arg(allow_annotation), max_size=2))
     vararg = draw(none() | arg(allow_annotation))
     kwarg = draw(none() | arg(allow_annotation))
     defaults=[]
     kw_defaults=draw(lists(none() | expression(), max_size=len(kwonlyargs), min_size=len(kwonlyargs)))
     return ast.arguments(
+        posonlyargs=posonlyargs,
         args=args,
         vararg=vararg,
         kwonlyargs=kwonlyargs,
diff --git a/hypo_test/folding.py b/hypo_test/folding.py
@@ -0,0 +1,44 @@
+from hypothesis import assume
+from hypothesis.strategies import integers, lists, binary, sampled_from, recursive, dictionaries, booleans, SearchStrategy, text, composite, one_of, floats, complex_numbers, characters, none
+import ast
+
+from expressions import NameConstant, Num
+
+leaves = NameConstant() | Num()
+
+@composite
+def BinOp(draw, expression) -> ast.BinOp:
+    op = draw(sampled_from([
+        ast.Add(),
+        ast.Sub(),
+        ast.Mult(),
+        ast.Div(),
+        ast.FloorDiv(),
+        ast.Mod(),
+        ast.Pow(),
+        ast.LShift(),
+        ast.RShift(),
+        ast.BitOr(),
+        ast.BitXor(),
+        ast.BitAnd(),
+        ast.MatMult()
+    ]))
+
+    le = draw(lists(expression, min_size=2, max_size=2))
+
+    return ast.BinOp(le[0], op, le[1])
+
+
+def expression() -> SearchStrategy:
+    return recursive(
+        leaves,
+        lambda expression:
+        BinOp(expression),
+        max_leaves=150
+    )
+
+@composite
+def FoldableExpression(draw) -> ast.Expression:
+    """ An eval expression """
+    e = draw(expression())
+    return ast.Expression(e)
diff --git a/hypo_test/module.py b/hypo_test/module.py
@@ -269,4 +269,4 @@ def suite() -> SearchStrategy:
 @composite
 def Module(draw) -> ast.Module:
     b = draw(lists(suite(), min_size=1, max_size=10))
-    return ast.Module(body=b)
+    return ast.Module(body=b, type_ignores=[])
diff --git a/hypo_test/test.py b/hypo_test/test.py
diff --git a/hypo_test/test_it.py b/hypo_test/test_it.py
@@ -0,0 +1,74 @@
+import ast
+
+from datetime import timedelta
+from hypothesis import given, settings, Verbosity, note, HealthCheck
+
+from folding import FoldableExpression
+from patterns import Pattern
+from python_minifier import ModulePrinter
+from python_minifier.ast_compare import compare_ast
+from python_minifier.ast_printer import print_ast
+from python_minifier.expression_printer import ExpressionPrinter
+from expressions import Expression
+from module import Module
+from python_minifier.rename.mapper import add_parent
+from python_minifier.transforms.constant_folding import FoldConstants
+
+
+@given(node=Expression())
+@settings(report_multiple_bugs=False, deadline=timedelta(seconds=1), max_examples=100, suppress_health_check=[HealthCheck.too_slow]) #verbosity=Verbosity.verbose
+def test_expression(node):
+    assert isinstance(node, ast.AST)
+
+    note(ast.dump(node))
+    printer = ExpressionPrinter()
+    code = printer(node)
+    note(code)
+    compare_ast(node, ast.parse(code, 'test_expression', 'eval'))
+
+
+@given(node=Module())
+@settings(report_multiple_bugs=False, deadline=timedelta(seconds=1), max_examples=100, suppress_health_check=[HealthCheck.too_slow]) #verbosity=Verbosity.verbose
+def test_module(node):
+    assert isinstance(node, ast.Module)
+
+    note(ast.dump(node))
+    printer = ModulePrinter()
+    code = printer(node)
+    note(code)
+    compare_ast(node, ast.parse(code, 'test_module'))
+
+
+@given(node=Pattern())
+@settings(report_multiple_bugs=False, deadline=timedelta(seconds=2), max_examples=100, verbosity=Verbosity.verbose)
+def test_pattern(node):
+
+    module = ast.Module(
+        body=[ast.Match(subject=ast.Constant(value=None),
+                        cases=[
+                            ast.match_case(
+                                pattern=node,
+                                guard=None,
+                                body=[ast.Pass()]
+                            )
+                        ])],
+        type_ignores=[]
+    )
+
+    printer = ModulePrinter()
+    code = printer(module)
+    note(code)
+    compare_ast(module, ast.parse(code, 'test_pattern'))
+
+@given(node=FoldableExpression())
+@settings(report_multiple_bugs=False, deadline=timedelta(seconds=1), max_examples=100000, suppress_health_check=[HealthCheck.too_slow]) #verbosity=Verbosity.verbose
+def test_folding(node):
+    assert isinstance(node, ast.AST)
+    note(print_ast(node))
+
+    add_parent(node)
+
+    constant_folder = FoldConstants()
+
+    # The constant folder asserts the value is correct
+    constant_folder(node)
diff --git a/requirements-dev.txt b/requirements-dev.txt
@@ -6,4 +6,5 @@ sphinxcontrib-programoutput
 sphinx_rtd_theme
 pyyaml
 sh
-hypothesis
+hypothesis
+hypofuzz
diff --git a/src/python_minifier/__init__.py b/src/python_minifier/__init__.py
@@ -8,6 +8,7 @@
 import re
 
 from python_minifier.ast_compare import CompareError, compare_ast
+from python_minifier.ast_printer import print_ast
 from python_minifier.module_printer import ModulePrinter
 from python_minifier.rename import (
     rename_literals,
@@ -20,6 +21,7 @@
 )
 
 from python_minifier.transforms.combine_imports import CombineImports
+from python_minifier.transforms.constant_folding import FoldConstants
 from python_minifier.transforms.remove_annotations import RemoveAnnotations
 from python_minifier.transforms.remove_annotations_options import RemoveAnnotationsOptions
 from python_minifier.transforms.remove_asserts import RemoveAsserts
@@ -69,7 +71,8 @@ def minify(
     remove_asserts=False,
     remove_debug=False,
     remove_explicit_return_none=True,
-    remove_builtin_exception_brackets=True
+    remove_builtin_exception_brackets=True,
+    constant_folding=True
 ):
     """
     Minify a python module
@@ -102,6 +105,7 @@ def minify(
     :param bool remove_debug: If conditional statements that test '__debug__ is True' should be removed
     :param bool remove_explicit_return_none: If explicit return None statements should be replaced with a bare return
     :param bool remove_builtin_exception_brackets: If brackets should be removed when raising exceptions with no arguments
+    :param bool constant_folding: If literal expressions should be evaluated
 
     :rtype: str
 
@@ -150,6 +154,9 @@ def minify(
     if remove_explicit_return_none:
         module = RemoveExplicitReturnNone()(module)
 
+    if constant_folding:
+        module = FoldConstants()(module)
+
     bind_names(module)
     resolve_names(module)
 
diff --git a/src/python_minifier/__init__.pyi b/src/python_minifier/__init__.pyi
@@ -24,7 +24,8 @@ def minify(
     remove_asserts: bool = ...,
     remove_debug: bool = ...,
     remove_explicit_return_none: bool = ...,
-    remove_builtin_exception_brackets: bool = ...
+    remove_builtin_exception_brackets: bool = ...,
+    constant_folding: bool = ...
 ) -> Text: ...
 
 def unparse(module: ast.Module) -> Text: ...
diff --git a/src/python_minifier/__main__.py b/src/python_minifier/__main__.py
@@ -190,6 +190,12 @@ def parse_args():
         help='Disable removing brackets when raising builtin exceptions with no arguments',
         dest='remove_exception_brackets',
     )
+    minification_options.add_argument(
+        '--no-constant-folding',
+        action='store_false',
+        help='Disable evaluating literal expressions',
+        dest='constant_folding',
+    )
 
     annotation_options = parser.add_argument_group('remove annotations options', 'Options that affect how annotations are removed')
     annotation_options.add_argument(
@@ -308,7 +314,8 @@ def do_minify(source, filename, minification_args):
         remove_asserts=minification_args.remove_asserts,
         remove_debug=minification_args.remove_debug,
         remove_explicit_return_none=minification_args.remove_explicit_return_none,
-        remove_builtin_exception_brackets=minification_args.remove_exception_brackets
+        remove_builtin_exception_brackets=minification_args.remove_exception_brackets,
+        constant_folding=minification_args.constant_folding
     )
 
 
diff --git a/src/python_minifier/transforms/constant_folding.py b/src/python_minifier/transforms/constant_folding.py
diff --git a/test/test_folding.py b/test/test_folding.py
diff --git a/tox.ini b/tox.ini
diff --git a/tox/pyyaml-5.4.1-constraints.txt b/tox/pyyaml-5.4.1-constraints.txt

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+SECONDS_IN_A_DAY = 60 * 60 * 24`
	`2`	`+SECONDS_IN_A_WEEK = SECONDS_IN_A_DAY * 7`