diff --git a/.github/workflows/test_corpus.yaml b/.github/workflows/test_corpus.yaml index 99da2e12..baf8472b 100644 --- a/.github/workflows/test_corpus.yaml +++ b/.github/workflows/test_corpus.yaml @@ -46,6 +46,7 @@ jobs: matrix: python: ["2.7", "3.3", "3.4", "3.5", "3.6", "3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] ref: ["${{ inputs.ref }}", "${{ inputs.base-ref }}"] + target_current_python: ["true"] container: image: danielflook/python-minifier-build:python${{ matrix.python }}-2024-01-12 volumes: @@ -96,7 +97,7 @@ jobs: export PYTHONHASHSEED=0 fi - python${{matrix.python}} workflow/corpus_test/generate_results.py /corpus /corpus-results $(= minimum + + if minimum > self.minimum: + self.minimum = minimum + + if maximum < self.maximum: + self.maximum = maximum + + if self.minimum > self.maximum: + self.maximum = self.minimum + + self._constrained = True + + def __repr__(self): + return 'TargetPythonOptions(minimum=%r, target_maximum_python=%r)' % (self.minimum, self.maximum) + def minify( source, @@ -71,7 +113,8 @@ def minify( remove_debug=False, remove_explicit_return_none=True, remove_builtin_exception_brackets=True, - constant_folding=True + constant_folding=True, + target_python=None ): """ Minify a python module @@ -105,6 +148,8 @@ def minify( :param bool remove_explicit_return_none: If explicit return None statements should be replaced with a bare return :param bool remove_builtin_exception_brackets: If brackets should be removed when raising exceptions with no arguments :param bool constant_folding: If literal expressions should be evaluated + :param target_python: Options for the target python version + :type target_python: :class:`TargetPythonOptions` :rtype: str @@ -115,6 +160,11 @@ def minify( # This will raise if the source file can't be parsed module = ast.parse(source, filename) + if target_python is None: + target_python = TargetPythonOptions((2, 7), (sys.version_info.major, sys.version_info.minor)) + if target_python._constrained is False: + target_python.apply_constraint(*compat.find_syntax_versions(module)) + add_namespace(module) if remove_literal_statements: @@ -141,7 +191,7 @@ def minify( if remove_pass: module = RemovePass()(module) - if remove_object_base: + if target_python.minimum >= (3, 0) and remove_object_base: module = RemoveObject()(module) if remove_asserts: @@ -189,7 +239,7 @@ def minify( if convert_posargs_to_args: module = remove_posargs(module) - minified = unparse(module) + minified = unparse(module, target_python) if preserve_shebang is True: shebang_line = _find_shebang(source) @@ -214,7 +264,7 @@ def _find_shebang(source): return None -def unparse(module): +def unparse(module, target_python=None): """ Turn a module AST into python code @@ -223,13 +273,22 @@ def unparse(module): :param module: The module to turn into python code :type: module: :class:`ast.Module` + :param target_python: Options for the target python version + :type target_python: :class:`TargetPythonOptions` :rtype: str """ assert isinstance(module, ast.Module) - printer = ModulePrinter() + if target_python is None: + target_python = TargetPythonOptions((2, 7), (sys.version_info.major, sys.version_info.minor)) + if target_python._constrained is False: + target_python.apply_constraint(*compat.find_syntax_versions(module)) + + sys.stderr.write('Target Python: %r\n' % target_python) + + printer = ModulePrinter(target_python) printer(module) try: @@ -245,7 +304,7 @@ def unparse(module): return printer.code -def awslambda(source, filename=None, entrypoint=None): +def awslambda(source, filename=None, entrypoint=None, target_python=None): """ Minify a python module for use as an AWS Lambda function @@ -256,6 +315,8 @@ def awslambda(source, filename=None, entrypoint=None): :param str filename: The original source filename if known :param entrypoint: The lambda entrypoint function :type entrypoint: str or NoneType + :param target_python: Options for the target python version + :type target_python: :class:`TargetPythonOptions` :rtype: str """ @@ -265,5 +326,10 @@ def awslambda(source, filename=None, entrypoint=None): rename_globals = False return minify( - source, filename, remove_literal_statements=True, rename_globals=rename_globals, preserve_globals=[entrypoint], + source, + filename, + remove_literal_statements=True, + rename_globals=rename_globals, + preserve_globals=[entrypoint], + target_python=target_python ) diff --git a/src/python_minifier/__init__.pyi b/src/python_minifier/__init__.pyi index a414796d..abfb1edc 100644 --- a/src/python_minifier/__init__.pyi +++ b/src/python_minifier/__init__.pyi @@ -1,11 +1,16 @@ import ast -from typing import List, Text, AnyStr, Optional, Any, Union +from typing import List, Text, AnyStr, Optional, Any, Union, Tuple from .transforms.remove_annotations_options import RemoveAnnotationsOptions as RemoveAnnotationsOptions class UnstableMinification(RuntimeError): def __init__(self, exception: Any, source: Any, minified: Any): ... +class TargetPythonOptions(object): + def __init__(self, minimum: Optional[Tuple[int, int]], maximum: Optional[Tuple[int, int]]): ... + + def apply_constraint(self, minimum: Tuple[int, int], maximum: Tuple[int, int]) -> None: ... + def minify( source: AnyStr, filename: Optional[str] = ..., @@ -25,13 +30,18 @@ def minify( remove_debug: bool = ..., remove_explicit_return_none: bool = ..., remove_builtin_exception_brackets: bool = ..., - constant_folding: bool = ... + constant_folding: bool = ..., + target_python: Optional[TargetPythonOptions] = ... ) -> Text: ... -def unparse(module: ast.Module) -> Text: ... +def unparse( + module: ast.Module, + target_python: Optional[TargetPythonOptions] = ... +) -> Text: ... def awslambda( source: AnyStr, filename: Optional[Text] = ..., - entrypoint: Optional[Text] = ... + entrypoint: Optional[Text] = ..., + target_python: Optional[TargetPythonOptions] = ... ) -> Text: ... diff --git a/src/python_minifier/__main__.py b/src/python_minifier/__main__.py index 056fe5ca..d65c5469 100644 --- a/src/python_minifier/__main__.py +++ b/src/python_minifier/__main__.py @@ -4,7 +4,7 @@ import os import sys -from python_minifier import minify +from python_minifier import minify, TargetPythonOptions from python_minifier.transforms.remove_annotations_options import RemoveAnnotationsOptions if sys.version_info >= (3, 8): @@ -75,6 +75,10 @@ def main(): else: sys.stdout.write(minified) +def split_version(version): + if version is None: + return None + return tuple(map(int, version.split('.'))) def parse_args(): parser = argparse.ArgumentParser(prog='pyminify', description='Minify Python source code', formatter_class=argparse.RawDescriptionHelpFormatter, epilog=main.__doc__) @@ -236,6 +240,36 @@ def parse_args(): dest='remove_class_attribute_annotations', ) + available_python_versions = ['2.7', '3.3', '3.4', '3.5', '3.6', '3.7', '3.8', '3.9', '3.10', '3.11', '3.12'] + target_python_options = parser.add_argument_group('target python options', 'Options that affect which python versions the minified code should target') + target_python_options.add_argument( + '--target-python', + type=str, + choices=available_python_versions, + action='store', + help='The version of Python that the minified code should target.', + dest='target_python', + metavar='VERSION' + ) + target_python_options.add_argument( + '--target-minimum-python', + type=str, + choices=available_python_versions, + action='store', + help='The minimum version of Python that the minified code should target. The default is the minimum version required by the source code.', + dest='target_minimum_python', + metavar='VERSION' + ) + target_python_options.add_argument( + '--target-maximum-python', + type=str, + choices=available_python_versions, + action='store', + help='The maximum version of Python that the minified code should target. The default is the version pyminify is currently using. (Currently %s)' % '.'.join(str(v) for v in sys.version_info[:2]), + dest='target_maximum_python', + metavar='VERSION' + ) + parser.add_argument('--version', '-v', action='version', version=version) args = parser.parse_args() @@ -258,6 +292,22 @@ def parse_args(): sys.stderr.write('error: --remove-class-attribute-annotations would do nothing when used with --no-remove-annotations\n') sys.exit(1) + if args.target_python and (args.target_minimum_python or args.target_maximum_python): + sys.stderr.write('error: --target-python cannot be used with --target-minimum-python or --target-maximum-python\n') + sys.exit(1) + + if args.target_python and split_version(args.target_python) > sys.version_info[:2]: + sys.stderr.write('error: --target-python cannot be greater than the version of Python running pyminify\n') + sys.exit(1) + + if args.target_maximum_python and split_version(args.target_maximum_python) > sys.version_info[:2]: + sys.stderr.write('error: --target-maximum-python cannot be greater than the version of Python running pyminify\n') + sys.exit(1) + + if args.target_maximum_python and args.target_minimum_python and split_version(args.target_maximum_python) < split_version(args.target_minimum_python): + sys.stderr.write('error: --target-maximum-python cannot be less than --target-minimum-python\n') + sys.exit(1) + return args @@ -305,6 +355,17 @@ def do_minify(source, filename, minification_args): remove_class_attribute_annotations=minification_args.remove_class_attribute_annotations, ) + if minification_args.target_python: + target_python = TargetPythonOptions( + minimum=split_version(minification_args.target_python), + maximum=split_version(minification_args.target_python) + ) + else: + target_python = TargetPythonOptions( + minimum=split_version(minification_args.target_minimum_python) if minification_args.target_minimum_python else (2, 7), + maximum=split_version(minification_args.target_maximum_python) if minification_args.target_maximum_python else sys.version_info[:2] + ) + return minify( source, filename=filename, @@ -324,7 +385,8 @@ def do_minify(source, filename, minification_args): remove_debug=minification_args.remove_debug, remove_explicit_return_none=minification_args.remove_explicit_return_none, remove_builtin_exception_brackets=minification_args.remove_exception_brackets, - constant_folding=minification_args.constant_folding + constant_folding=minification_args.constant_folding, + target_python=target_python ) diff --git a/src/python_minifier/compat.py b/src/python_minifier/compat.py new file mode 100644 index 00000000..8656b086 --- /dev/null +++ b/src/python_minifier/compat.py @@ -0,0 +1,184 @@ +import ast +import sys + +from python_minifier.util import NodeVisitor + + +class PythonSourceCompatibility(NodeVisitor): + """ + Determine the minimum python version that can parse this source + + It may or may not be parsable by more recent versions of python. + + The AST will have been parsed by the current version of python. + + This only cares about syntax features. + """ + + class Version(Exception): + def __init__(self, version): + self.version = version + + def __init__(self): + self._min_version = 2, 7 + self._max_version = sys.version_info[1], sys.version_info[2] + + self.f_string_nesting = 0 + + def set_minimum(self, major, minor): + if (major, minor) > self._min_version: + self._min_version = major, minor + + if self._max_version < (major, minor): + self._max_version = major, minor + + def set_version(self, major, minor): + raise self.Version((major, minor)) + + def __call__(self, module): + assert isinstance(module, ast.Module) + + try: + self.visit(module) + + return self._min_version, self._max_version + except self.Version as v: + return v.version, v.version + + # region Literals + def visit_JoinedStr(self, node): + self.set_minimum(3, 6) + self.f_string_nesting += 1 + if self.f_string_nesting > 4: + raise self.Version((3, 12)) + self.generic_visit(node) + self.f_string_nesting -= 1 + + def visit_FormattedValue(self, node): + # Do not visit the format_spec + for field, value in ast.iter_fields(node): + if field == 'format_spec': + continue + + if isinstance(value, list): + for item in value: + if isinstance(item, ast.AST): + self.visit(item) + elif isinstance(value, ast.AST): + self.visit(value) + + def visit_Str(self, node): + if self.f_string_nesting + 1 > 4: + raise self.Version((3, 12)) + + def visit_Bytes(self, node): + if self.f_string_nesting + 1 > 4: + raise self.Version((3, 12)) + + # endregion + + # region Expressions + def visit_NamedExpr(self, node): + self.set_minimum(3, 8) + self.generic_visit(node) + + def visit_MatMult(self, node): + self.set_minimum(3, 5) + self.generic_visit(node) + + # endregion + + # region Assignments + def visit_AnnAssign(self, node): + self.set_minimum(3, 6) + self.generic_visit(node) + + # endregion + + # region Function and Class definitions + def visit_arguments(self, node): + if getattr(node, 'posonlyargs', []): + self.set_minimum(3, 8) + + if getattr(node, 'kwonlyargs', []): + self.set_minimum(3, 0) + + if getattr(node, 'varargannotation', None): + self.set_minimum(3, 0) + + if getattr(node, 'kwargannotation', None): + self.set_minimum(3, 0) + + self.generic_visit(node) + + def visit_arg(self, node): + if getattr(node, 'annotation'): + self.set_minimum(3, 0) + + def visit_YieldFrom(self, node): + self.set_minimum(3, 3) + self.generic_visit(node) + + def visit_nonlocal(self, node): + self.set_minimum(3, 0) + self.generic_visit(node) + + # endregion + + # region Async and await + def visit_AsyncFunctionDef(self, node): + self.set_minimum(3, 5) + self.generic_visit(node) + + def visit_Await(self, node): + self.set_minimum(3, 5) + self.generic_visit(node) + + def visit_AsyncFor(self, node): + self.set_minimum(3, 5) + self.generic_visit(node) + + def visit_AsyncWith(self, node): + self.set_minimum(3, 5) + self.generic_visit(node) + + # endregion + + # region Pattern Matching + def visit_Match(self, node): + self.set_minimum(3, 10) + self.generic_visit(node) + + # endregion + + def visit_Repr(self, node): + self.set_version(2, 7) + self.generic_visit(node) + + def visit_comprehension(self, node): + if getattr(node, 'is_async', False): + self.set_minimum(3, 6) + self.generic_visit(node) + + def visit_TryStar(self, node): + self.set_minimum(3, 11) + self.generic_visit(node) + + # region Type Parameters + def visit_TypeVar(self, node): + self.set_version(3, 12) + self.generic_visit(node) + + def visit_TypeVarTuple(self, node): + self.set_version(3, 12) + self.generic_visit(node) + + def visit_ParamSpec(self, node): + self.set_version(3, 12) + self.generic_visit(node) + + # endregion + + +def find_syntax_versions(module): + return PythonSourceCompatibility()(module) diff --git a/src/python_minifier/expression_printer.py b/src/python_minifier/expression_printer.py index 300f399c..e7bc19f6 100644 --- a/src/python_minifier/expression_printer.py +++ b/src/python_minifier/expression_printer.py @@ -11,7 +11,7 @@ class ExpressionPrinter(object): Builds the smallest possible exact representation of an ast """ - def __init__(self): + def __init__(self, target_python=None): self.precedences = { 'Lambda': 2, # Lambda @@ -35,6 +35,7 @@ def __init__(self): } self.printer = TokenPrinter() + self._target_python = target_python def __call__(self, module): """ @@ -735,7 +736,7 @@ def visit_JoinedStr(self, node): import python_minifier.f_string - if sys.version_info < (3, 12): + if self._target_python.minimum < (3, 12): pep701 = False else: pep701 = True diff --git a/src/python_minifier/module_printer.py b/src/python_minifier/module_printer.py index 3d57178b..0cb8b707 100644 --- a/src/python_minifier/module_printer.py +++ b/src/python_minifier/module_printer.py @@ -1,5 +1,4 @@ import python_minifier.ast_compat as ast -import sys from .expression_printer import ExpressionPrinter from .token_printer import Delimiter @@ -11,8 +10,8 @@ class ModulePrinter(ExpressionPrinter): Builds the smallest possible exact representation of an ast """ - def __init__(self, indent_char='\t'): - super(ModulePrinter, self).__init__() + def __init__(self, target_python, indent_char='\t'): + super(ModulePrinter, self).__init__(target_python) self.indent_char = indent_char def __call__(self, module): @@ -143,7 +142,7 @@ def visit_Return(self, node): self.printer.keyword('return') if isinstance(node.value, ast.Tuple): - if sys.version_info < (3, 8) and [n for n in node.value.elts if is_ast_node(n, 'Starred')]: + if self._target_python.minimum < (3, 8) and [n for n in node.value.elts if is_ast_node(n, 'Starred')]: self.printer.delimiter('(') self._testlist(node.value) self.printer.delimiter(')') diff --git a/src/python_minifier/token_printer.py b/src/python_minifier/token_printer.py index e8813fd2..709865a4 100644 --- a/src/python_minifier/token_printer.py +++ b/src/python_minifier/token_printer.py @@ -82,7 +82,7 @@ class TokenPrinter(object): def __init__(self, prefer_single_line=False, allow_invalid_num_warnings=False): """ :param prefer_single_line: If True, chooses to put as much code as possible on a single line. - :param allow_invalid_num_warnings: If True, allows invalid number literals to be printe that may cause warnings. + :param allow_invalid_num_warnings: If True, allows invalid number literals to be printed that may cause warnings. """ self._prefer_single_line = prefer_single_line diff --git a/src/python_minifier/transforms/remove_object_base.py b/src/python_minifier/transforms/remove_object_base.py index 9a34b9c9..d974809d 100644 --- a/src/python_minifier/transforms/remove_object_base.py +++ b/src/python_minifier/transforms/remove_object_base.py @@ -6,9 +6,6 @@ class RemoveObject(SuiteTransformer): def __call__(self, node): - if sys.version_info < (3, 0): - return node - return self.visit(node) def visit_ClassDef(self, node): diff --git a/src/python_minifier/transforms/suite_transformer.py b/src/python_minifier/transforms/suite_transformer.py index 0b0982a7..1f16e580 100644 --- a/src/python_minifier/transforms/suite_transformer.py +++ b/src/python_minifier/transforms/suite_transformer.py @@ -1,47 +1,12 @@ import python_minifier.ast_compat as ast from python_minifier.rename.mapper import add_parent -from python_minifier.util import is_ast_node - - -class NodeVisitor(object): - def visit(self, node): - """Visit a node.""" - method = 'visit_' + node.__class__.__name__ - visitor = getattr(self, method, self.generic_visit) - return visitor(node) - - def generic_visit(self, node): - """Called if no explicit visitor function exists for a node.""" - for field, value in ast.iter_fields(node): - if isinstance(value, list): - for item in value: - if isinstance(item, ast.AST): - self.visit(item) - elif isinstance(value, ast.AST): - self.visit(value) - - def visit_Constant(self, node): - if node.value in [None, True, False]: - method = 'visit_NameConstant' - elif isinstance(node.value, (int, float, complex)): - method = 'visit_Num' - elif isinstance(node.value, str): - method = 'visit_Str' - elif isinstance(node.value, bytes): - method = 'visit_Bytes' - elif node.value == Ellipsis: - method = 'visit_Ellipsis' - else: - raise RuntimeError('Unknown Constant value %r' % type(node.value)) - - visitor = getattr(self, method, self.generic_visit) - return visitor(node) +from python_minifier.util import is_ast_node, NodeVisitor class SuiteTransformer(NodeVisitor): """ - Transform suites of instructions + Transform suites of statements """ def __call__(self, node): diff --git a/src/python_minifier/util.py b/src/python_minifier/util.py index 9e80a73b..bf1b489c 100644 --- a/src/python_minifier/util.py +++ b/src/python_minifier/util.py @@ -46,3 +46,38 @@ def is_ast_node(node, types): raise RuntimeError('Unknown Constant value %r' % type(node.value)) return False + + +class NodeVisitor(object): + def visit(self, node): + """Visit a node.""" + method = 'visit_' + node.__class__.__name__ + visitor = getattr(self, method, self.generic_visit) + return visitor(node) + + def generic_visit(self, node): + """Called if no explicit visitor function exists for a node.""" + for field, value in ast.iter_fields(node): + if isinstance(value, list): + for item in value: + if isinstance(item, ast.AST): + self.visit(item) + elif isinstance(value, ast.AST): + self.visit(value) + + def visit_Constant(self, node): + if node.value in [None, True, False]: + method = 'visit_NameConstant' + elif isinstance(node.value, (int, float, complex)): + method = 'visit_Num' + elif isinstance(node.value, str): + method = 'visit_Str' + elif isinstance(node.value, bytes): + method = 'visit_Bytes' + elif node.value == Ellipsis: + method = 'visit_Ellipsis' + else: + raise RuntimeError('Unknown Constant value %r' % type(node.value)) + + visitor = getattr(self, method, self.generic_visit) + return visitor(node) diff --git a/test/test_compat.py b/test/test_compat.py new file mode 100644 index 00000000..ee1a019c --- /dev/null +++ b/test/test_compat.py @@ -0,0 +1,260 @@ +import ast +import sys + +import pytest + +from python_minifier.compat import find_syntax_versions + +def test_no_special_syntax(): + source = ''' +a = 'Hello' +''' + + min_version, max_version = find_syntax_versions(ast.parse(source)) + assert min_version, max_version == ((2, 7), sys.version_info[:2]) + +def test_named_expr(): + if sys.version_info < (3, 8): + pytest.skip('Python < 3.8 does not have named expressions') + + source = ''' +if a := 1: + pass +''' + + min_version, max_version = find_syntax_versions(ast.parse(source)) + assert min_version, max_version == ((3, 8), sys.version_info[:2]) + +def test_matmult(): + if sys.version_info < (3, 5): + pytest.skip('Python < 3.5 does not have matmult') + + source = ''' +a @ b +''' + + min_version, max_version = find_syntax_versions(ast.parse(source)) + assert min_version, max_version == ((3, 5), sys.version_info[:2]) + +def test_annassign(): + if sys.version_info < (3, 6): + pytest.skip('Python < 3.6 does not have annotated assignments') + + source = ''' +a: int = 1 +''' + + min_version, max_version = find_syntax_versions(ast.parse(source)) + assert min_version, max_version == ((3, 6), sys.version_info[:2]) + +def test_kwonlyargs(): + if sys.version_info < (3, 0): + pytest.skip('Python 2 does not have kwonlyargs') + + source = ''' +def f(a, b, *, c): + pass +''' + + min_version, max_version = find_syntax_versions(ast.parse(source)) + assert min_version, max_version == ((3, 0), sys.version_info[:2]) + +def test_posonlyargs(): + if sys.version_info < (3, 8): + pytest.skip('Python < 3.8 does not have posonlyargs') + + source = ''' +def f(a, b, /, c): + pass +''' + + min_version, max_version = find_syntax_versions(ast.parse(source)) + assert min_version, max_version == ((3, 8), sys.version_info[:2]) + +def test_vararg_annotation(): + if sys.version_info < (3, 0): + pytest.skip('Python < 3.0 does not have annotations') + + source = ''' +def f(*args: int): + pass +''' + + min_version, max_version = find_syntax_versions(ast.parse(source)) + assert min_version, max_version == ((3, 0), sys.version_info[:2]) + +def test_kwarg_annotation(): + if sys.version_info < (3, 0): + pytest.skip('Python < 3.0 does not have annotations') + + source = ''' +def f(**kwargs: int): + pass +''' + + min_version, max_version = find_syntax_versions(ast.parse(source)) + assert min_version, max_version == ((3, 0), sys.version_info[:2]) + +def test_arg_annotation(): + if sys.version_info < (3, 0): + pytest.skip('Python < 3.0 does not have annotations') + + source = ''' +def f(a: int): + pass +''' + + min_version, max_version = find_syntax_versions(ast.parse(source)) + assert min_version, max_version == ((3, 0), sys.version_info[:2]) + +def test_nonlocal(): + if sys.version_info < (3, 0): + pytest.skip('Python < 3.0 does not have nonlocal') + + source = ''' +def f(): + nonlocal a +''' + + min_version, max_version = find_syntax_versions(ast.parse(source)) + assert min_version, max_version == ((3, 0), sys.version_info[:2]) + +def test_async_function(): + if sys.version_info < (3, 5): + pytest.skip('Python < 3.5 does not have async functions') + + source = ''' +async def f(): + pass +''' + + min_version, max_version = find_syntax_versions(ast.parse(source)) + assert min_version, max_version == ((3, 5), sys.version_info[:2]) + +def test_async_with(): + if sys.version_info < (3, 7): + pytest.skip('Python < 3.7 does not support top level async with') + + source = ''' +async with a: + pass +''' + + min_version, max_version = find_syntax_versions(ast.parse(source)) + assert min_version, max_version == ((3, 5), sys.version_info[:2]) + +def test_async_for(): + if sys.version_info < (3, 7): + pytest.skip('Python < 3.7 does not support top level async for') + + source = ''' +async for a in b: + pass +''' + + min_version, max_version = find_syntax_versions(ast.parse(source)) + assert min_version, max_version == ((3, 5), sys.version_info[:2]) + +def test_async_comprehension(): + if sys.version_info < (3, 7): + pytest.skip('Python < 3.7 does not have async comprehensions') + + source = ''' +result = [i async for i in aiter() if i % 2] +''' + + min_version, max_version = find_syntax_versions(ast.parse(source)) + assert min_version, max_version == ((3, 6), sys.version_info[:2]) + +def test_await(): + if sys.version_info < (3, 7): + pytest.skip('Python < 3.7 does not have top level await') + + source = ''' +await a +''' + + min_version, max_version = find_syntax_versions(ast.parse(source)) + assert min_version, max_version == ((3, 5), sys.version_info[:2]) + +def test_match(): + if sys.version_info < (3, 10): + pytest.skip('Python < 3.10 does not have match') + + source = ''' +match a: + case b: + pass +''' + + min_version, max_version = find_syntax_versions(ast.parse(source)) + assert min_version, max_version == ((3, 10), sys.version_info[:2]) + +def test_repr(): + if sys.version_info > (2, 7): + pytest.skip('Python 3 does not have backtick syntax') + + source = ''' +`1+2` +''' + + min_version, max_version = find_syntax_versions(ast.parse(source)) + assert min_version, max_version == ((2, 7), (2,7)) + +def test_try_star(): + if sys.version_info < (3, 11): + pytest.skip('Python < 3.11 does not have try star') + + source = ''' +try: + pass +except* Error as e: + pass +''' + + min_version, max_version = find_syntax_versions(ast.parse(source)) + assert min_version, max_version == ((3, 11), sys.version_info[:2]) + +def test_function_type_var(): + if sys.version_info < (3, 12): + pytest.skip('Python < 3.12 does not have type vars') + + source = ''' +def a[T](): pass +''' + + min_version, max_version = find_syntax_versions(ast.parse(source)) + assert min_version, max_version == ((3, 12), sys.version_info[:2]) + +def test_class_type_var(): + if sys.version_info < (3, 12): + pytest.skip('Python < 3.12 does not have type vars') + + source = ''' +class a[T]: pass +''' + + min_version, max_version = find_syntax_versions(ast.parse(source)) + assert min_version, max_version == ((3, 12), sys.version_info[:2]) + +def test_typevar_tuple(): + if sys.version_info < (3, 12): + pytest.skip('Python < 3.12 does not have type vars') + + source = ''' +class a[*T]: pass +''' + + min_version, max_version = find_syntax_versions(ast.parse(source)) + assert min_version, max_version == ((3, 12), sys.version_info[:2]) + +def test_paramspec(): + if sys.version_info < (3, 12): + pytest.skip('Python < 3.12 does not have type vars') + + source = ''' +class a[**T]: pass +''' + + min_version, max_version = find_syntax_versions(ast.parse(source)) + assert min_version, max_version == ((3, 12), sys.version_info[:2]) \ No newline at end of file diff --git a/test/test_fstring.py b/test/test_fstring.py index fb9821cf..64cc70c8 100644 --- a/test/test_fstring.py +++ b/test/test_fstring.py @@ -3,7 +3,7 @@ import pytest -from python_minifier import unparse +from python_minifier import unparse, TargetPythonOptions from python_minifier.ast_compare import compare_ast @@ -27,11 +27,16 @@ def test_pep0701(): if sys.version_info < (3, 12): pytest.skip('f-string syntax is bonkers before python 3.12') + python_312 = TargetPythonOptions((sys.version_info.major, sys.version_info.minor), (sys.version_info.major, sys.version_info.minor)) + statement = 'f"{f"{f"{f"{"hello"}"}"}"}"' assert unparse(ast.parse(statement)) == statement statement = 'f"This is the playlist: {", ".join([])}"' - assert unparse(ast.parse(statement)) == statement + # since this snippet doesn't require nested quotes, we may not use them in the output if it isn't smaller + assert unparse(ast.parse(statement)) == '''f"This is the playlist: {', '.join([])}"''' + assert unparse(ast.parse(statement), target_python=python_312) == statement + assert len(unparse(ast.parse(statement))) == len(statement) statement = 'f"{f"{f"{f"{f"{f"{1+1}"}"}"}"}"}"' assert unparse(ast.parse(statement)) == statement @@ -43,38 +48,39 @@ def test_pep0701(): 'Ascensionism' # Take to the broken skies at last ])}" """ - assert unparse(ast.parse(statement)) == 'f"This is the playlist: {", ".join(["Take me back to Eden","Alkaline","Ascensionism"])}"' + assert unparse(ast.parse(statement), target_python=python_312) == 'f"This is the playlist: {", ".join(["Take me back to Eden","Alkaline","Ascensionism"])}"' #statement = '''print(f"This is the playlist: {"\N{BLACK HEART SUIT}".join(songs)}")''' #assert unparse(ast.parse(statement)) == statement statement = '''f"Magic wand: {bag["wand"]}"''' - assert unparse(ast.parse(statement)) == statement + assert unparse(ast.parse(statement), target_python=python_312) == statement statement = """ f'''A complex trick: { bag['bag'] # recursive bags! }''' """ - assert unparse(ast.parse(statement)) == 'f"A complex trick: {bag["bag"]}"' + assert unparse(ast.parse(statement), target_python=python_312) == 'f"A complex trick: {bag["bag"]}"' statement = '''f"These are the things: {", ".join(things)}"''' - assert unparse(ast.parse(statement)) == statement + assert unparse(ast.parse(statement), target_python=python_312) == statement statement = '''f"{source.removesuffix(".py")}.c: $(srcdir)/{source}"''' - assert unparse(ast.parse(statement)) == statement + assert unparse(ast.parse(statement), target_python=python_312) == statement statement = '''f"{f"{f"infinite"}"}"+' '+f"{f"nesting!!!"}"''' - assert unparse(ast.parse(statement)) == statement + # Weirdly, this example doesn't require infinite nesting + assert unparse(ast.parse(statement), target_python=python_312) == statement statement = '''f"{"\\n".join(a)}"''' - assert unparse(ast.parse(statement)) == statement + assert unparse(ast.parse(statement), target_python=python_312) == statement statement = '''f"{f"{f"{f"{f"{f"{1+1}"}"}"}"}"}"''' assert unparse(ast.parse(statement)) == statement statement = '''f"{"":*^{1:{1}}}"''' - assert unparse(ast.parse(statement)) == statement + assert unparse(ast.parse(statement), target_python=python_312) == statement #statement = '''f"{"":*^{1:{1:{1}}}}"''' #assert unparse(ast.parse(statement)) == statement diff --git a/test/test_iterable_unpacking.py b/test/test_iterable_unpacking.py index dd95a2b8..b39256c7 100644 --- a/test/test_iterable_unpacking.py +++ b/test/test_iterable_unpacking.py @@ -1,22 +1,41 @@ import ast import sys import pytest -from python_minifier import unparse +from python_minifier import unparse, TargetPythonOptions from python_minifier.ast_compare import compare_ast -def test_return(): +def test_required_parens(): if sys.version_info < (3, 0): pytest.skip('Iterable unpacking in return not allowed in python < 3.0') - elif sys.version_info < (3, 8): - # Parenthesis are required - source = 'def a():return(True,*[False])' + if sys.version_info > (3, 7): + pytest.skip('Parens not required in python > 3.7') - else: - # Parenthesis not required - source = 'def a():return True,*[False]' + # Parenthesis are required + source = 'def a():return(True,*[False])' expected_ast = ast.parse(source) minified = unparse(expected_ast) compare_ast(expected_ast, ast.parse(minified)) assert source == minified + +def test_target_required_parens(): + if sys.version_info < (3, 8): + pytest.skip('Parens always required in python < 3.8') + + source = 'def a():return True,*[False]' + with_parens = 'def a():return(True,*[False])' + + expected_ast = ast.parse(source) + + # Without constraining the python version, compatibility is assumed to be down to 3.0, so use parens + + minified = unparse(expected_ast) + compare_ast(expected_ast, ast.parse(minified)) + assert with_parens == minified + + # When constraining to min python >= 3.8, parens are not required + python_38_minimum = TargetPythonOptions((3, 8), (sys.version_info.major, sys.version_info.minor)) + minified = unparse(expected_ast, target_python=python_38_minimum) + compare_ast(expected_ast, ast.parse(minified)) + assert source == minified diff --git a/test/test_remove_object.py b/test/test_remove_object.py index d4b45418..96f2c575 100644 --- a/test/test_remove_object.py +++ b/test/test_remove_object.py @@ -2,6 +2,8 @@ import pytest import sys +from python_minifier import minify, TargetPythonOptions +from python_minifier.compat import find_syntax_versions from python_minifier.transforms.remove_object_base import RemoveObject from python_minifier.ast_compare import compare_ast @@ -50,6 +52,37 @@ class Test(other_base): actual_ast = RemoveObject()(ast.parse(source)) compare_ast(expected_ast, actual_ast) +def test_no_remove_python2_target(): + if sys.version_info < (3, 0): + pytest.skip('This test is python3 only') + + source = ''' +class Test(object): + pass +''' + removed = ''' +class Test: pass +''' + + python_30_minimum = TargetPythonOptions((3, 0), (sys.version_info.major, sys.version_info.minor)) + python_27_minimum = TargetPythonOptions((2, 7), (sys.version_info.major, sys.version_info.minor)) + + minified = minify( + source, + remove_object_base=True, + remove_pass=False, + target_python=python_30_minimum + ) + compare_ast(ast.parse(minified), ast.parse(removed)) + + minified = minify( + source, + remove_object_base=True, + remove_pass=False, + target_python=python_27_minimum + ) + compare_ast(ast.parse(minified), ast.parse(source)) + def test_no_remove_object_py2(): if sys.version_info >= (3, 0): @@ -64,7 +97,5 @@ class Test(object): pass ''' - expected_ast = ast.parse(expected) - actual_ast = RemoveObject()(ast.parse(source)) - compare_ast(expected_ast, actual_ast) - + min_version, max_version = find_syntax_versions(ast.parse(source)) + assert min_version, max_version == ((2, 7), sys.version_info[:2]) diff --git a/xtest/manifests/python3.10_test_manifest.yaml b/xtest/manifests/python3.10_test_manifest.yaml index 5603f85c..421bbf43 100644 --- a/xtest/manifests/python3.10_test_manifest.yaml +++ b/xtest/manifests/python3.10_test_manifest.yaml @@ -3825,15 +3825,7 @@ remove_literal_statements: true rename_globals: true /usr/local/lib/python3.10/test/test_urllib.py: [] -/usr/local/lib/python3.10/test/test_urllib2.py: -- options: {} -- options: - rename_globals: true -- options: - remove_literal_statements: true -- options: - remove_literal_statements: true - rename_globals: true +/usr/local/lib/python3.10/test/test_urllib2.py: [] /usr/local/lib/python3.10/test/test_urllib2_localnet.py: - options: {} - options: diff --git a/xtest/manifests/python3.8_test_manifest.yaml b/xtest/manifests/python3.8_test_manifest.yaml index cef77a00..62870878 100644 --- a/xtest/manifests/python3.8_test_manifest.yaml +++ b/xtest/manifests/python3.8_test_manifest.yaml @@ -3775,15 +3775,7 @@ remove_literal_statements: true rename_globals: true /usr/lib64/python3.8/test/test_urllib.py: [] -/usr/lib64/python3.8/test/test_urllib2.py: -- options: {} -- options: - rename_globals: true -- options: - remove_literal_statements: true -- options: - remove_literal_statements: true - rename_globals: true +/usr/lib64/python3.8/test/test_urllib2.py: [] /usr/lib64/python3.8/test/test_urllib2_localnet.py: - options: {} - options: diff --git a/xtest/manifests/python3.9_test_manifest.yaml b/xtest/manifests/python3.9_test_manifest.yaml index e3f60342..0281205d 100644 --- a/xtest/manifests/python3.9_test_manifest.yaml +++ b/xtest/manifests/python3.9_test_manifest.yaml @@ -3720,15 +3720,7 @@ remove_literal_statements: true rename_globals: true /usr/local/lib/python3.9/test/test_urllib.py: [] -/usr/local/lib/python3.9/test/test_urllib2.py: -- options: {} -- options: - rename_globals: true -- options: - remove_literal_statements: true -- options: - remove_literal_statements: true - rename_globals: true +/usr/local/lib/python3.9/test/test_urllib2.py: [] /usr/local/lib/python3.9/test/test_urllib2_localnet.py: - options: {} - options: