From 6fc008c9b2e11a1e14cc4150d2f4c76881372de8 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sat, 7 Sep 2024 23:40:47 +0100 Subject: [PATCH 1/9] Don't compare to empty base result set --- corpus_test/generate_report.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/corpus_test/generate_report.py b/corpus_test/generate_report.py index b0ee2d04..f8373d32 100644 --- a/corpus_test/generate_report.py +++ b/corpus_test/generate_report.py @@ -325,7 +325,17 @@ def report(results_dir: str, minifier_ref: str, minifier_sha: str, base_ref: str try: base_summary = result_summary(results_dir, python_version, base_sha) except FileNotFoundError: - base_summary = ResultSet(python_version, base_ref) + yield ( + f'| {python_version} ' + + f'| {summary.valid_count} ' + + f'| {summary.mean_time:.3f} ' + + f'| {summary.mean_percent_of_original:.3f}% ' + + f'| {len(list(summary.larger_than_original()))} ' + + f'| {len(list(summary.recursion_error()))} ' + + f'| {len(list(summary.unstable_minification()))} ' + + f'| {len(list(summary.exception()))} ' + ) + continue mean_time_change = summary.mean_time - base_summary.mean_time From e35f88068c9854eb1500526cf497bfd173c6b8b4 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Thu, 12 Sep 2024 08:11:53 +0100 Subject: [PATCH 2/9] Add target python version options --- docs/source/api_usage.rst | 2 +- docs/source/index.rst | 1 + docs/source/python_target_version.rst | 18 ++ docs/source/transforms/remove_object_base.rst | 2 +- src/python_minifier/__init__.py | 80 +++++++- src/python_minifier/__init__.pyi | 16 +- src/python_minifier/__main__.py | 66 ++++++- src/python_minifier/compat.py | 179 ++++++++++++++++++ src/python_minifier/expression_printer.py | 5 +- src/python_minifier/module_printer.py | 7 +- src/python_minifier/token_printer.py | 2 +- .../transforms/remove_object_base.py | 3 - .../transforms/suite_transformer.py | 39 +--- src/python_minifier/util.py | 35 ++++ 14 files changed, 393 insertions(+), 62 deletions(-) create mode 100644 docs/source/python_target_version.rst create mode 100644 src/python_minifier/compat.py diff --git a/docs/source/api_usage.rst b/docs/source/api_usage.rst index 4a889061..c63ada7f 100644 --- a/docs/source/api_usage.rst +++ b/docs/source/api_usage.rst @@ -2,7 +2,7 @@ Package Reference ================= .. automodule:: python_minifier - +.. autoclass:: TargetPythonOptions .. autofunction:: minify .. autoclass:: RemoveAnnotationsOptions .. autofunction:: awslambda diff --git a/docs/source/index.rst b/docs/source/index.rst index 639a5d5d..27396df6 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -8,6 +8,7 @@ This package transforms python source code into a 'minified' representation of t :caption: Contents: installation + python_target_version command_usage api_usage transforms/index diff --git a/docs/source/python_target_version.rst b/docs/source/python_target_version.rst new file mode 100644 index 00000000..55ad6cfa --- /dev/null +++ b/docs/source/python_target_version.rst @@ -0,0 +1,18 @@ +Python Target Version +===================== + +This package uses the version of Python that it is installed with to parse your source code. +This means that you should install python-minifier using a version of Python that is appropriate for the source code you want to minify. + +The output aims to match the Python compatibility of the original source code. + +There are options to configure the target versions of Python that the minified code should be compatible with, which will affect the output of the minification process. +You can specify the minimum and maximum target versions of Python that the minified code should be compatible with. + +If the input source module uses syntax that is not compatible with the specified target versions, the target version range is automatically adjusted to include the syntax used in the input source module. + +.. note:: + The target version options will not increase the Python compatibility of the minified code beyond the compatibility of the original source code. + + They can only be used to reduce the compatibility of the minified code to a subset of the compatibility of the original source code. + diff --git a/docs/source/transforms/remove_object_base.rst b/docs/source/transforms/remove_object_base.rst index 05a2c53b..f0204ce1 100644 --- a/docs/source/transforms/remove_object_base.rst +++ b/docs/source/transforms/remove_object_base.rst @@ -2,7 +2,7 @@ Remove Object Base ================== In Python 3 all classes implicitly inherit from ``object``. This transform removes ``object`` from the base class list -of all classes. This transform does nothing on Python 2. +of all classes. This transform is only applied if the target Python version is 3.0 or higher. This transform is always safe to use and enabled by default. diff --git a/src/python_minifier/__init__.py b/src/python_minifier/__init__.py index a7ab2b67..fceac546 100644 --- a/src/python_minifier/__init__.py +++ b/src/python_minifier/__init__.py @@ -3,10 +3,12 @@ a 'minified' representation of the same source code. """ +import sys import python_minifier.ast_compat as ast import re +from python_minifier import compat from python_minifier.ast_compare import CompareError, compare_ast from python_minifier.module_printer import ModulePrinter from python_minifier.rename import ( @@ -51,6 +53,46 @@ def __init__(self, exception, source, minified): def __str__(self): return 'Minification was unstable! Please create an issue at https://github.com/dflook/python-minifier/issues' +class TargetPythonOptions(object): + """ + Options that can be passed to the minify function to specify the target python version + + :param minimum: The minimum python version that the minified code should be compatible with + :type minimum: tuple[int, int] or None + :param maximum: The maximum python version that the minified code should be compatible with + :type maximum: tuple[int, int] or None + """ + + def __init__(self, minimum, maximum): + self.minimum = minimum + self.maximum = maximum + self._constrained = False + + def apply_constraint(self, minimum, maximum): + """ + Apply a constraint to the target python version + + :param minimum: The minimum python version that the minified code should be compatible with + :type minimum: tuple[int, int] + :param maximum: The maximum python version that the minified code should be compatible with + :type maximum: tuple[int, int] + """ + assert maximum >= minimum + + if minimum > self.minimum: + self.minimum = minimum + + if maximum < self.maximum: + self.maximum = maximum + + if self.minimum > self.maximum: + self.maximum = self.minimum + + self._constrained = True + + def __repr__(self): + return 'TargetPythonOptions(minimum=%r, target_maximum_python=%r)' % (self.minimum, self.maximum) + def minify( source, @@ -71,7 +113,8 @@ def minify( remove_debug=False, remove_explicit_return_none=True, remove_builtin_exception_brackets=True, - constant_folding=True + constant_folding=True, + target_python=None ): """ Minify a python module @@ -105,6 +148,8 @@ def minify( :param bool remove_explicit_return_none: If explicit return None statements should be replaced with a bare return :param bool remove_builtin_exception_brackets: If brackets should be removed when raising exceptions with no arguments :param bool constant_folding: If literal expressions should be evaluated + :param target_python: Options for the target python version + :type target_python: :class:`TargetPythonOptions` :rtype: str @@ -115,6 +160,11 @@ def minify( # This will raise if the source file can't be parsed module = ast.parse(source, filename) + if target_python is None: + target_python = TargetPythonOptions((2, 7), (sys.version_info.major, sys.version_info.minor)) + if target_python._constrained is False: + target_python.apply_constraint(*compat.find_syntax_versions(module)) + add_namespace(module) if remove_literal_statements: @@ -141,7 +191,7 @@ def minify( if remove_pass: module = RemovePass()(module) - if remove_object_base: + if target_python.minimum > (3, 0) and remove_object_base: module = RemoveObject()(module) if remove_asserts: @@ -189,7 +239,7 @@ def minify( if convert_posargs_to_args: module = remove_posargs(module) - minified = unparse(module) + minified = unparse(module, target_python) if preserve_shebang is True: shebang_line = _find_shebang(source) @@ -214,7 +264,7 @@ def _find_shebang(source): return None -def unparse(module): +def unparse(module, target_python=None): """ Turn a module AST into python code @@ -223,13 +273,22 @@ def unparse(module): :param module: The module to turn into python code :type: module: :class:`ast.Module` + :param target_python: Options for the target python version + :type target_python: :class:`TargetPythonOptions` :rtype: str """ assert isinstance(module, ast.Module) - printer = ModulePrinter() + if target_python is None: + target_python = TargetPythonOptions((2, 7), (sys.version_info.major, sys.version_info.minor)) + if target_python._constrained is False: + target_python.apply_constraint(*compat.find_syntax_versions(module)) + + sys.stderr.write('Target Python: %r\n' % target_python) + + printer = ModulePrinter(target_python) printer(module) try: @@ -245,7 +304,7 @@ def unparse(module): return printer.code -def awslambda(source, filename=None, entrypoint=None): +def awslambda(source, filename=None, entrypoint=None, target_python=None): """ Minify a python module for use as an AWS Lambda function @@ -256,6 +315,8 @@ def awslambda(source, filename=None, entrypoint=None): :param str filename: The original source filename if known :param entrypoint: The lambda entrypoint function :type entrypoint: str or NoneType + :param target_python: Options for the target python version + :type target_python: :class:`TargetPythonOptions` :rtype: str """ @@ -265,5 +326,10 @@ def awslambda(source, filename=None, entrypoint=None): rename_globals = False return minify( - source, filename, remove_literal_statements=True, rename_globals=rename_globals, preserve_globals=[entrypoint], + source, + filename, + remove_literal_statements=True, + rename_globals=rename_globals, + preserve_globals=[entrypoint], + target_python=target_python ) diff --git a/src/python_minifier/__init__.pyi b/src/python_minifier/__init__.pyi index a414796d..610c56c3 100644 --- a/src/python_minifier/__init__.pyi +++ b/src/python_minifier/__init__.pyi @@ -1,11 +1,14 @@ import ast -from typing import List, Text, AnyStr, Optional, Any, Union +from typing import List, Text, AnyStr, Optional, Any, Union, Tuple from .transforms.remove_annotations_options import RemoveAnnotationsOptions as RemoveAnnotationsOptions class UnstableMinification(RuntimeError): def __init__(self, exception: Any, source: Any, minified: Any): ... +class TargetPythonOptions(object): + def __init__(self, minimum: Optional[Tuple[int, int]], maximum: Optional[Tuple[int, int]]): ... + def minify( source: AnyStr, filename: Optional[str] = ..., @@ -25,13 +28,18 @@ def minify( remove_debug: bool = ..., remove_explicit_return_none: bool = ..., remove_builtin_exception_brackets: bool = ..., - constant_folding: bool = ... + constant_folding: bool = ..., + target_python: Optional[TargetPythonOptions] = ... ) -> Text: ... -def unparse(module: ast.Module) -> Text: ... +def unparse( + module: ast.Module, + target_python: Optional[TargetPythonOptions] = ... +) -> Text: ... def awslambda( source: AnyStr, filename: Optional[Text] = ..., - entrypoint: Optional[Text] = ... + entrypoint: Optional[Text] = ..., + target_python: Optional[TargetPythonOptions] = ... ) -> Text: ... diff --git a/src/python_minifier/__main__.py b/src/python_minifier/__main__.py index 056fe5ca..d65c5469 100644 --- a/src/python_minifier/__main__.py +++ b/src/python_minifier/__main__.py @@ -4,7 +4,7 @@ import os import sys -from python_minifier import minify +from python_minifier import minify, TargetPythonOptions from python_minifier.transforms.remove_annotations_options import RemoveAnnotationsOptions if sys.version_info >= (3, 8): @@ -75,6 +75,10 @@ def main(): else: sys.stdout.write(minified) +def split_version(version): + if version is None: + return None + return tuple(map(int, version.split('.'))) def parse_args(): parser = argparse.ArgumentParser(prog='pyminify', description='Minify Python source code', formatter_class=argparse.RawDescriptionHelpFormatter, epilog=main.__doc__) @@ -236,6 +240,36 @@ def parse_args(): dest='remove_class_attribute_annotations', ) + available_python_versions = ['2.7', '3.3', '3.4', '3.5', '3.6', '3.7', '3.8', '3.9', '3.10', '3.11', '3.12'] + target_python_options = parser.add_argument_group('target python options', 'Options that affect which python versions the minified code should target') + target_python_options.add_argument( + '--target-python', + type=str, + choices=available_python_versions, + action='store', + help='The version of Python that the minified code should target.', + dest='target_python', + metavar='VERSION' + ) + target_python_options.add_argument( + '--target-minimum-python', + type=str, + choices=available_python_versions, + action='store', + help='The minimum version of Python that the minified code should target. The default is the minimum version required by the source code.', + dest='target_minimum_python', + metavar='VERSION' + ) + target_python_options.add_argument( + '--target-maximum-python', + type=str, + choices=available_python_versions, + action='store', + help='The maximum version of Python that the minified code should target. The default is the version pyminify is currently using. (Currently %s)' % '.'.join(str(v) for v in sys.version_info[:2]), + dest='target_maximum_python', + metavar='VERSION' + ) + parser.add_argument('--version', '-v', action='version', version=version) args = parser.parse_args() @@ -258,6 +292,22 @@ def parse_args(): sys.stderr.write('error: --remove-class-attribute-annotations would do nothing when used with --no-remove-annotations\n') sys.exit(1) + if args.target_python and (args.target_minimum_python or args.target_maximum_python): + sys.stderr.write('error: --target-python cannot be used with --target-minimum-python or --target-maximum-python\n') + sys.exit(1) + + if args.target_python and split_version(args.target_python) > sys.version_info[:2]: + sys.stderr.write('error: --target-python cannot be greater than the version of Python running pyminify\n') + sys.exit(1) + + if args.target_maximum_python and split_version(args.target_maximum_python) > sys.version_info[:2]: + sys.stderr.write('error: --target-maximum-python cannot be greater than the version of Python running pyminify\n') + sys.exit(1) + + if args.target_maximum_python and args.target_minimum_python and split_version(args.target_maximum_python) < split_version(args.target_minimum_python): + sys.stderr.write('error: --target-maximum-python cannot be less than --target-minimum-python\n') + sys.exit(1) + return args @@ -305,6 +355,17 @@ def do_minify(source, filename, minification_args): remove_class_attribute_annotations=minification_args.remove_class_attribute_annotations, ) + if minification_args.target_python: + target_python = TargetPythonOptions( + minimum=split_version(minification_args.target_python), + maximum=split_version(minification_args.target_python) + ) + else: + target_python = TargetPythonOptions( + minimum=split_version(minification_args.target_minimum_python) if minification_args.target_minimum_python else (2, 7), + maximum=split_version(minification_args.target_maximum_python) if minification_args.target_maximum_python else sys.version_info[:2] + ) + return minify( source, filename=filename, @@ -324,7 +385,8 @@ def do_minify(source, filename, minification_args): remove_debug=minification_args.remove_debug, remove_explicit_return_none=minification_args.remove_explicit_return_none, remove_builtin_exception_brackets=minification_args.remove_exception_brackets, - constant_folding=minification_args.constant_folding + constant_folding=minification_args.constant_folding, + target_python=target_python ) diff --git a/src/python_minifier/compat.py b/src/python_minifier/compat.py new file mode 100644 index 00000000..013d0130 --- /dev/null +++ b/src/python_minifier/compat.py @@ -0,0 +1,179 @@ +import ast +import sys + +from python_minifier.util import NodeVisitor + + +class PythonSourceCompatibility(NodeVisitor): + """ + Determine the minimum python version that can parse this source + + It may or may not be parsable by more recent versions of python. + + The AST will have been parsed by the current version of python. + + This only cares about syntax features. + """ + + class Version(Exception): + def __init__(self, version): + self.version = version + + def __init__(self): + self._min_version = 2, 7 + self._max_version = sys.version_info[1], sys.version_info[2] + + self.f_string_nesting = 0 + self.pep701_required = False + + def set_minimum(self, major, minor): + if (major, minor) > self._min_version: + self._min_version = major, minor + + if self._max_version < (major, minor): + self._max_version = major, minor + + def set_version(self, major, minor): + raise self.Version((major, minor)) + + def __call__(self, module): + assert isinstance(module, ast.Module) + + try: + self.visit(module) + + if self.pep701_required: + self.set_minimum(3, 12) + + return self._min_version, self._max_version + except self.Version as v: + return v.version, v.version + + # region Literals + def visit_JoinedStr(self, node): + self.set_minimum(3, 6) + self.f_string_nesting += 1 + if self.f_string_nesting > 4: + self.pep701_required = True + self.generic_visit(node) + self.f_string_nesting -= 1 + + def visit_Str(self, node): + self.f_string_nesting += 1 + if self.f_string_nesting > 4: + self.pep701_required = True + self.f_string_nesting -= 1 + + def visit_Bytes(self, node): + self.f_string_nesting += 1 + if self.f_string_nesting > 4: + self.pep701_required = True + self.f_string_nesting -= 1 + + # endregion + + # region Expressions + def visit_NamedExpr(self, node): + self.set_minimum(3, 8) + self.generic_visit(node) + + def visit_MatMult(self, node): + self.set_minimum(3, 5) + self.generic_visit(node) + + # endregion + + # region Assignments + def visit_AnnAssign(self, node): + self.set_minimum(3, 6) + self.generic_visit(node) + + # endregion + + # region Function and Class definitions + def visit_arguments(self, node): + if getattr(node, 'posonlyargs', []): + self.set_minimum(3, 8) + + if getattr(node, 'kwonlyargs', []): + self.set_minimum(3, 0) + + if getattr(node, 'varargannotation', None): + self.set_minimum(3, 0) + + if getattr(node, 'kwargannotation', None): + self.set_minimum(3, 0) + + self.generic_visit(node) + + def visit_arg(self, node): + if getattr(node, 'annotation'): + self.set_minimum(3, 0) + + def visit_YieldFrom(self, node): + self.set_minimum(3, 3) + self.generic_visit(node) + + def visit_nonlocal(self, node): + self.set_minimum(3, 0) + self.generic_visit(node) + + # endregion + + # region Async and await + def visit_AsyncFunctionDef(self, node): + self.set_minimum(3, 5) + self.generic_visit(node) + + def visit_Await(self, node): + self.set_minimum(3, 5) + self.generic_visit(node) + + def visit_AsyncFor(self, node): + self.set_minimum(3, 5) + self.generic_visit(node) + + def visit_AsyncWith(self, node): + self.set_minimum(3, 5) + self.generic_visit(node) + + # endregion + + # region Pattern Matching + def visit_Match(self, node): + self.set_minimum(3, 10) + self.generic_visit(node) + + # endregion + + def visit_Repr(self, node): + self.set_version(2, 7) + self.generic_visit(node) + + def visit_comprehension(self, node): + if getattr(node, 'is_async', False): + self.set_minimum(3, 6) + self.generic_visit(node) + + def visit_TryStar(self, node): + self.set_minimum(3, 11) + self.generic_visit(node) + + # region Type Parameters + def visit_TypeVar(self, node): + self.set_version(3, 12) + self.generic_visit(node) + + def visit_TypeVarTuple(self, node): + self.set_version(3, 12) + self.generic_visit(node) + + def visit_ParamSpec(self, node): + self.set_version(3, 12) + self.generic_visit(node) + + # endregion + + +def find_syntax_versions(module): + return PythonSourceCompatibility()(module) diff --git a/src/python_minifier/expression_printer.py b/src/python_minifier/expression_printer.py index 300f399c..e7bc19f6 100644 --- a/src/python_minifier/expression_printer.py +++ b/src/python_minifier/expression_printer.py @@ -11,7 +11,7 @@ class ExpressionPrinter(object): Builds the smallest possible exact representation of an ast """ - def __init__(self): + def __init__(self, target_python=None): self.precedences = { 'Lambda': 2, # Lambda @@ -35,6 +35,7 @@ def __init__(self): } self.printer = TokenPrinter() + self._target_python = target_python def __call__(self, module): """ @@ -735,7 +736,7 @@ def visit_JoinedStr(self, node): import python_minifier.f_string - if sys.version_info < (3, 12): + if self._target_python.minimum < (3, 12): pep701 = False else: pep701 = True diff --git a/src/python_minifier/module_printer.py b/src/python_minifier/module_printer.py index 3d57178b..0cb8b707 100644 --- a/src/python_minifier/module_printer.py +++ b/src/python_minifier/module_printer.py @@ -1,5 +1,4 @@ import python_minifier.ast_compat as ast -import sys from .expression_printer import ExpressionPrinter from .token_printer import Delimiter @@ -11,8 +10,8 @@ class ModulePrinter(ExpressionPrinter): Builds the smallest possible exact representation of an ast """ - def __init__(self, indent_char='\t'): - super(ModulePrinter, self).__init__() + def __init__(self, target_python, indent_char='\t'): + super(ModulePrinter, self).__init__(target_python) self.indent_char = indent_char def __call__(self, module): @@ -143,7 +142,7 @@ def visit_Return(self, node): self.printer.keyword('return') if isinstance(node.value, ast.Tuple): - if sys.version_info < (3, 8) and [n for n in node.value.elts if is_ast_node(n, 'Starred')]: + if self._target_python.minimum < (3, 8) and [n for n in node.value.elts if is_ast_node(n, 'Starred')]: self.printer.delimiter('(') self._testlist(node.value) self.printer.delimiter(')') diff --git a/src/python_minifier/token_printer.py b/src/python_minifier/token_printer.py index e8813fd2..709865a4 100644 --- a/src/python_minifier/token_printer.py +++ b/src/python_minifier/token_printer.py @@ -82,7 +82,7 @@ class TokenPrinter(object): def __init__(self, prefer_single_line=False, allow_invalid_num_warnings=False): """ :param prefer_single_line: If True, chooses to put as much code as possible on a single line. - :param allow_invalid_num_warnings: If True, allows invalid number literals to be printe that may cause warnings. + :param allow_invalid_num_warnings: If True, allows invalid number literals to be printed that may cause warnings. """ self._prefer_single_line = prefer_single_line diff --git a/src/python_minifier/transforms/remove_object_base.py b/src/python_minifier/transforms/remove_object_base.py index 9a34b9c9..d974809d 100644 --- a/src/python_minifier/transforms/remove_object_base.py +++ b/src/python_minifier/transforms/remove_object_base.py @@ -6,9 +6,6 @@ class RemoveObject(SuiteTransformer): def __call__(self, node): - if sys.version_info < (3, 0): - return node - return self.visit(node) def visit_ClassDef(self, node): diff --git a/src/python_minifier/transforms/suite_transformer.py b/src/python_minifier/transforms/suite_transformer.py index 0b0982a7..1f16e580 100644 --- a/src/python_minifier/transforms/suite_transformer.py +++ b/src/python_minifier/transforms/suite_transformer.py @@ -1,47 +1,12 @@ import python_minifier.ast_compat as ast from python_minifier.rename.mapper import add_parent -from python_minifier.util import is_ast_node - - -class NodeVisitor(object): - def visit(self, node): - """Visit a node.""" - method = 'visit_' + node.__class__.__name__ - visitor = getattr(self, method, self.generic_visit) - return visitor(node) - - def generic_visit(self, node): - """Called if no explicit visitor function exists for a node.""" - for field, value in ast.iter_fields(node): - if isinstance(value, list): - for item in value: - if isinstance(item, ast.AST): - self.visit(item) - elif isinstance(value, ast.AST): - self.visit(value) - - def visit_Constant(self, node): - if node.value in [None, True, False]: - method = 'visit_NameConstant' - elif isinstance(node.value, (int, float, complex)): - method = 'visit_Num' - elif isinstance(node.value, str): - method = 'visit_Str' - elif isinstance(node.value, bytes): - method = 'visit_Bytes' - elif node.value == Ellipsis: - method = 'visit_Ellipsis' - else: - raise RuntimeError('Unknown Constant value %r' % type(node.value)) - - visitor = getattr(self, method, self.generic_visit) - return visitor(node) +from python_minifier.util import is_ast_node, NodeVisitor class SuiteTransformer(NodeVisitor): """ - Transform suites of instructions + Transform suites of statements """ def __call__(self, node): diff --git a/src/python_minifier/util.py b/src/python_minifier/util.py index 9e80a73b..bf1b489c 100644 --- a/src/python_minifier/util.py +++ b/src/python_minifier/util.py @@ -46,3 +46,38 @@ def is_ast_node(node, types): raise RuntimeError('Unknown Constant value %r' % type(node.value)) return False + + +class NodeVisitor(object): + def visit(self, node): + """Visit a node.""" + method = 'visit_' + node.__class__.__name__ + visitor = getattr(self, method, self.generic_visit) + return visitor(node) + + def generic_visit(self, node): + """Called if no explicit visitor function exists for a node.""" + for field, value in ast.iter_fields(node): + if isinstance(value, list): + for item in value: + if isinstance(item, ast.AST): + self.visit(item) + elif isinstance(value, ast.AST): + self.visit(value) + + def visit_Constant(self, node): + if node.value in [None, True, False]: + method = 'visit_NameConstant' + elif isinstance(node.value, (int, float, complex)): + method = 'visit_Num' + elif isinstance(node.value, str): + method = 'visit_Str' + elif isinstance(node.value, bytes): + method = 'visit_Bytes' + elif node.value == Ellipsis: + method = 'visit_Ellipsis' + else: + raise RuntimeError('Unknown Constant value %r' % type(node.value)) + + visitor = getattr(self, method, self.generic_visit) + return visitor(node) From 58d04fd2b20c6e5cec76b46a13f93ef6dff20d90 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Thu, 12 Sep 2024 23:30:15 +0100 Subject: [PATCH 3/9] Format strings don't require quotes --- src/python_minifier/compat.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/python_minifier/compat.py b/src/python_minifier/compat.py index 013d0130..639ff647 100644 --- a/src/python_minifier/compat.py +++ b/src/python_minifier/compat.py @@ -58,17 +58,17 @@ def visit_JoinedStr(self, node): self.generic_visit(node) self.f_string_nesting -= 1 + def visit_FormattedValue(self, node): + # Do not visit the format_spec + self.generic_visit(node.value) + def visit_Str(self, node): - self.f_string_nesting += 1 - if self.f_string_nesting > 4: + if self.f_string_nesting + 1 > 4: self.pep701_required = True - self.f_string_nesting -= 1 def visit_Bytes(self, node): - self.f_string_nesting += 1 - if self.f_string_nesting > 4: + if self.f_string_nesting + 1 > 4: self.pep701_required = True - self.f_string_nesting -= 1 # endregion From b944028d8d26cab547cd4d49951b9e0ff358ae8f Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Fri, 13 Sep 2024 21:56:15 +0100 Subject: [PATCH 4/9] Add and Update tests for specifying target python versions --- src/python_minifier/__init__.py | 2 +- src/python_minifier/__init__.pyi | 2 + src/python_minifier/compat.py | 21 ++- test/test_compat.py | 260 +++++++++++++++++++++++++++++++ test/test_fstring.py | 26 ++-- test/test_iterable_unpacking.py | 35 ++++- test/test_remove_object.py | 39 ++++- 7 files changed, 354 insertions(+), 31 deletions(-) create mode 100644 test/test_compat.py diff --git a/src/python_minifier/__init__.py b/src/python_minifier/__init__.py index fceac546..615fabd8 100644 --- a/src/python_minifier/__init__.py +++ b/src/python_minifier/__init__.py @@ -191,7 +191,7 @@ def minify( if remove_pass: module = RemovePass()(module) - if target_python.minimum > (3, 0) and remove_object_base: + if target_python.minimum >= (3, 0) and remove_object_base: module = RemoveObject()(module) if remove_asserts: diff --git a/src/python_minifier/__init__.pyi b/src/python_minifier/__init__.pyi index 610c56c3..abfb1edc 100644 --- a/src/python_minifier/__init__.pyi +++ b/src/python_minifier/__init__.pyi @@ -9,6 +9,8 @@ class UnstableMinification(RuntimeError): class TargetPythonOptions(object): def __init__(self, minimum: Optional[Tuple[int, int]], maximum: Optional[Tuple[int, int]]): ... + def apply_constraint(self, minimum: Tuple[int, int], maximum: Tuple[int, int]) -> None: ... + def minify( source: AnyStr, filename: Optional[str] = ..., diff --git a/src/python_minifier/compat.py b/src/python_minifier/compat.py index 639ff647..8656b086 100644 --- a/src/python_minifier/compat.py +++ b/src/python_minifier/compat.py @@ -24,7 +24,6 @@ def __init__(self): self._max_version = sys.version_info[1], sys.version_info[2] self.f_string_nesting = 0 - self.pep701_required = False def set_minimum(self, major, minor): if (major, minor) > self._min_version: @@ -42,9 +41,6 @@ def __call__(self, module): try: self.visit(module) - if self.pep701_required: - self.set_minimum(3, 12) - return self._min_version, self._max_version except self.Version as v: return v.version, v.version @@ -54,21 +50,30 @@ def visit_JoinedStr(self, node): self.set_minimum(3, 6) self.f_string_nesting += 1 if self.f_string_nesting > 4: - self.pep701_required = True + raise self.Version((3, 12)) self.generic_visit(node) self.f_string_nesting -= 1 def visit_FormattedValue(self, node): # Do not visit the format_spec - self.generic_visit(node.value) + for field, value in ast.iter_fields(node): + if field == 'format_spec': + continue + + if isinstance(value, list): + for item in value: + if isinstance(item, ast.AST): + self.visit(item) + elif isinstance(value, ast.AST): + self.visit(value) def visit_Str(self, node): if self.f_string_nesting + 1 > 4: - self.pep701_required = True + raise self.Version((3, 12)) def visit_Bytes(self, node): if self.f_string_nesting + 1 > 4: - self.pep701_required = True + raise self.Version((3, 12)) # endregion diff --git a/test/test_compat.py b/test/test_compat.py new file mode 100644 index 00000000..9fa4cb99 --- /dev/null +++ b/test/test_compat.py @@ -0,0 +1,260 @@ +import ast +import sys + +import pytest + +from python_minifier.compat import find_syntax_versions + +def test_no_special_syntax(): + source = ''' +a = 'Hello' +''' + + min_version, max_version = find_syntax_versions(ast.parse(source)) + assert min_version, max_version == ((2, 7), sys.version_info[:2]) + +def test_named_expr(): + if sys.version_info < (3, 8): + pytest.skip('Python < 3.8 does not have named expressions') + + source = ''' +if a := 1: + pass +''' + + min_version, max_version = find_syntax_versions(ast.parse(source)) + assert min_version, max_version == ((3, 8), sys.version_info[:2]) + +def test_matmult(): + if sys.version_info < (3, 5): + pytest.skip('Python < 3.5 does not have matmult') + + source = ''' +a @ b +''' + + min_version, max_version = find_syntax_versions(ast.parse(source)) + assert min_version, max_version == ((3, 5), sys.version_info[:2]) + +def test_annassign(): + if sys.version_info < (3, 6): + pytest.skip('Python < 3.6 does not have annotated assignments') + + source = ''' +a: int = 1 +''' + + min_version, max_version = find_syntax_versions(ast.parse(source)) + assert min_version, max_version == ((3, 6), sys.version_info[:2]) + +def test_kwonlyargs(): + if sys.version_info < (3, 0): + pytest.skip('Python 2 does not have kwonlyargs') + + source = ''' +def f(a, b, *, c): + pass +''' + + min_version, max_version = find_syntax_versions(ast.parse(source)) + assert min_version, max_version == ((3, 0), sys.version_info[:2]) + +def test_posonlyargs(): + if sys.version_info < (3, 8): + pytest.skip('Python < 3.8 does not have posonlyargs') + + source = ''' +def f(a, b, /, c): + pass +''' + + min_version, max_version = find_syntax_versions(ast.parse(source)) + assert min_version, max_version == ((3, 8), sys.version_info[:2]) + +def test_vararg_annotation(): + if sys.version_info < (3, 0): + pytest.skip('Python < 3.0 does not have annotations') + + source = ''' +def f(*args: int): + pass +''' + + min_version, max_version = find_syntax_versions(ast.parse(source)) + assert min_version, max_version == ((3, 0), sys.version_info[:2]) + +def test_kwarg_annotation(): + if sys.version_info < (3, 0): + pytest.skip('Python < 3.0 does not have annotations') + + source = ''' +def f(**kwargs: int): + pass +''' + + min_version, max_version = find_syntax_versions(ast.parse(source)) + assert min_version, max_version == ((3, 0), sys.version_info[:2]) + +def test_arg_annotation(): + if sys.version_info < (3, 0): + pytest.skip('Python < 3.0 does not have annotations') + + source = ''' +def f(a: int): + pass +''' + + min_version, max_version = find_syntax_versions(ast.parse(source)) + assert min_version, max_version == ((3, 0), sys.version_info[:2]) + +def test_nonlocal(): + if sys.version_info < (3, 0): + pytest.skip('Python < 3.0 does not have nonlocal') + + source = ''' +def f(): + nonlocal a +''' + + min_version, max_version = find_syntax_versions(ast.parse(source)) + assert min_version, max_version == ((3, 0), sys.version_info[:2]) + +def test_async_function(): + if sys.version_info < (3, 5): + pytest.skip('Python < 3.5 does not have async functions') + + source = ''' +async def f(): + pass +''' + + min_version, max_version = find_syntax_versions(ast.parse(source)) + assert min_version, max_version == ((3, 5), sys.version_info[:2]) + +def test_async_with(): + if sys.version_info < (3, 5): + pytest.skip('Python < 3.5 does not have async with') + + source = ''' +async with a: + pass +''' + + min_version, max_version = find_syntax_versions(ast.parse(source)) + assert min_version, max_version == ((3, 5), sys.version_info[:2]) + +def test_async_for(): + if sys.version_info < (3, 5): + pytest.skip('Python < 3.5 does not have async for') + + source = ''' +async for a in b: + pass +''' + + min_version, max_version = find_syntax_versions(ast.parse(source)) + assert min_version, max_version == ((3, 5), sys.version_info[:2]) + +def test_async_comprehension(): + if sys.version_info < (3, 6): + pytest.skip('Python < 3.6 does not have async comprehensions') + + source = ''' +[a async for a in b] +''' + + min_version, max_version = find_syntax_versions(ast.parse(source)) + assert min_version, max_version == ((3, 6), sys.version_info[:2]) + +def test_await(): + if sys.version_info < (3, 5): + pytest.skip('Python < 3.5 does not have await') + + source = ''' +await a +''' + + min_version, max_version = find_syntax_versions(ast.parse(source)) + assert min_version, max_version == ((3, 5), sys.version_info[:2]) + +def test_match(): + if sys.version_info < (3, 10): + pytest.skip('Python < 3.10 does not have match') + + source = ''' +match a: + case b: + pass +''' + + min_version, max_version = find_syntax_versions(ast.parse(source)) + assert min_version, max_version == ((3, 10), sys.version_info[:2]) + +def test_repr(): + if sys.version_info > (2, 7): + pytest.skip('Python 3 does not have backtick syntax') + + source = ''' +`1+2` +''' + + min_version, max_version = find_syntax_versions(ast.parse(source)) + assert min_version, max_version == ((2, 7), (2,7)) + +def test_try_star(): + if sys.version_info < (3, 11): + pytest.skip('Python < 3.11 does not have try star') + + source = ''' +try: + pass +except* Error as e: + pass +''' + + min_version, max_version = find_syntax_versions(ast.parse(source)) + assert min_version, max_version == ((3, 11), sys.version_info[:2]) + +def test_function_type_var(): + if sys.version_info < (3, 12): + pytest.skip('Python < 3.12 does not have type vars') + + source = ''' +def a[T](): pass +''' + + min_version, max_version = find_syntax_versions(ast.parse(source)) + assert min_version, max_version == ((3, 12), sys.version_info[:2]) + +def test_class_type_var(): + if sys.version_info < (3, 12): + pytest.skip('Python < 3.12 does not have type vars') + + source = ''' +class a[T]: pass +''' + + min_version, max_version = find_syntax_versions(ast.parse(source)) + assert min_version, max_version == ((3, 12), sys.version_info[:2]) + +def test_typevar_tuple(): + if sys.version_info < (3, 12): + pytest.skip('Python < 3.12 does not have type vars') + + source = ''' +class a[*T]: pass +''' + + min_version, max_version = find_syntax_versions(ast.parse(source)) + assert min_version, max_version == ((3, 12), sys.version_info[:2]) + +def test_paramspec(): + if sys.version_info < (3, 12): + pytest.skip('Python < 3.12 does not have type vars') + + source = ''' +class a[**T]: pass +''' + + min_version, max_version = find_syntax_versions(ast.parse(source)) + assert min_version, max_version == ((3, 12), sys.version_info[:2]) \ No newline at end of file diff --git a/test/test_fstring.py b/test/test_fstring.py index fb9821cf..64cc70c8 100644 --- a/test/test_fstring.py +++ b/test/test_fstring.py @@ -3,7 +3,7 @@ import pytest -from python_minifier import unparse +from python_minifier import unparse, TargetPythonOptions from python_minifier.ast_compare import compare_ast @@ -27,11 +27,16 @@ def test_pep0701(): if sys.version_info < (3, 12): pytest.skip('f-string syntax is bonkers before python 3.12') + python_312 = TargetPythonOptions((sys.version_info.major, sys.version_info.minor), (sys.version_info.major, sys.version_info.minor)) + statement = 'f"{f"{f"{f"{"hello"}"}"}"}"' assert unparse(ast.parse(statement)) == statement statement = 'f"This is the playlist: {", ".join([])}"' - assert unparse(ast.parse(statement)) == statement + # since this snippet doesn't require nested quotes, we may not use them in the output if it isn't smaller + assert unparse(ast.parse(statement)) == '''f"This is the playlist: {', '.join([])}"''' + assert unparse(ast.parse(statement), target_python=python_312) == statement + assert len(unparse(ast.parse(statement))) == len(statement) statement = 'f"{f"{f"{f"{f"{f"{1+1}"}"}"}"}"}"' assert unparse(ast.parse(statement)) == statement @@ -43,38 +48,39 @@ def test_pep0701(): 'Ascensionism' # Take to the broken skies at last ])}" """ - assert unparse(ast.parse(statement)) == 'f"This is the playlist: {", ".join(["Take me back to Eden","Alkaline","Ascensionism"])}"' + assert unparse(ast.parse(statement), target_python=python_312) == 'f"This is the playlist: {", ".join(["Take me back to Eden","Alkaline","Ascensionism"])}"' #statement = '''print(f"This is the playlist: {"\N{BLACK HEART SUIT}".join(songs)}")''' #assert unparse(ast.parse(statement)) == statement statement = '''f"Magic wand: {bag["wand"]}"''' - assert unparse(ast.parse(statement)) == statement + assert unparse(ast.parse(statement), target_python=python_312) == statement statement = """ f'''A complex trick: { bag['bag'] # recursive bags! }''' """ - assert unparse(ast.parse(statement)) == 'f"A complex trick: {bag["bag"]}"' + assert unparse(ast.parse(statement), target_python=python_312) == 'f"A complex trick: {bag["bag"]}"' statement = '''f"These are the things: {", ".join(things)}"''' - assert unparse(ast.parse(statement)) == statement + assert unparse(ast.parse(statement), target_python=python_312) == statement statement = '''f"{source.removesuffix(".py")}.c: $(srcdir)/{source}"''' - assert unparse(ast.parse(statement)) == statement + assert unparse(ast.parse(statement), target_python=python_312) == statement statement = '''f"{f"{f"infinite"}"}"+' '+f"{f"nesting!!!"}"''' - assert unparse(ast.parse(statement)) == statement + # Weirdly, this example doesn't require infinite nesting + assert unparse(ast.parse(statement), target_python=python_312) == statement statement = '''f"{"\\n".join(a)}"''' - assert unparse(ast.parse(statement)) == statement + assert unparse(ast.parse(statement), target_python=python_312) == statement statement = '''f"{f"{f"{f"{f"{f"{1+1}"}"}"}"}"}"''' assert unparse(ast.parse(statement)) == statement statement = '''f"{"":*^{1:{1}}}"''' - assert unparse(ast.parse(statement)) == statement + assert unparse(ast.parse(statement), target_python=python_312) == statement #statement = '''f"{"":*^{1:{1:{1}}}}"''' #assert unparse(ast.parse(statement)) == statement diff --git a/test/test_iterable_unpacking.py b/test/test_iterable_unpacking.py index dd95a2b8..b39256c7 100644 --- a/test/test_iterable_unpacking.py +++ b/test/test_iterable_unpacking.py @@ -1,22 +1,41 @@ import ast import sys import pytest -from python_minifier import unparse +from python_minifier import unparse, TargetPythonOptions from python_minifier.ast_compare import compare_ast -def test_return(): +def test_required_parens(): if sys.version_info < (3, 0): pytest.skip('Iterable unpacking in return not allowed in python < 3.0') - elif sys.version_info < (3, 8): - # Parenthesis are required - source = 'def a():return(True,*[False])' + if sys.version_info > (3, 7): + pytest.skip('Parens not required in python > 3.7') - else: - # Parenthesis not required - source = 'def a():return True,*[False]' + # Parenthesis are required + source = 'def a():return(True,*[False])' expected_ast = ast.parse(source) minified = unparse(expected_ast) compare_ast(expected_ast, ast.parse(minified)) assert source == minified + +def test_target_required_parens(): + if sys.version_info < (3, 8): + pytest.skip('Parens always required in python < 3.8') + + source = 'def a():return True,*[False]' + with_parens = 'def a():return(True,*[False])' + + expected_ast = ast.parse(source) + + # Without constraining the python version, compatibility is assumed to be down to 3.0, so use parens + + minified = unparse(expected_ast) + compare_ast(expected_ast, ast.parse(minified)) + assert with_parens == minified + + # When constraining to min python >= 3.8, parens are not required + python_38_minimum = TargetPythonOptions((3, 8), (sys.version_info.major, sys.version_info.minor)) + minified = unparse(expected_ast, target_python=python_38_minimum) + compare_ast(expected_ast, ast.parse(minified)) + assert source == minified diff --git a/test/test_remove_object.py b/test/test_remove_object.py index d4b45418..96f2c575 100644 --- a/test/test_remove_object.py +++ b/test/test_remove_object.py @@ -2,6 +2,8 @@ import pytest import sys +from python_minifier import minify, TargetPythonOptions +from python_minifier.compat import find_syntax_versions from python_minifier.transforms.remove_object_base import RemoveObject from python_minifier.ast_compare import compare_ast @@ -50,6 +52,37 @@ class Test(other_base): actual_ast = RemoveObject()(ast.parse(source)) compare_ast(expected_ast, actual_ast) +def test_no_remove_python2_target(): + if sys.version_info < (3, 0): + pytest.skip('This test is python3 only') + + source = ''' +class Test(object): + pass +''' + removed = ''' +class Test: pass +''' + + python_30_minimum = TargetPythonOptions((3, 0), (sys.version_info.major, sys.version_info.minor)) + python_27_minimum = TargetPythonOptions((2, 7), (sys.version_info.major, sys.version_info.minor)) + + minified = minify( + source, + remove_object_base=True, + remove_pass=False, + target_python=python_30_minimum + ) + compare_ast(ast.parse(minified), ast.parse(removed)) + + minified = minify( + source, + remove_object_base=True, + remove_pass=False, + target_python=python_27_minimum + ) + compare_ast(ast.parse(minified), ast.parse(source)) + def test_no_remove_object_py2(): if sys.version_info >= (3, 0): @@ -64,7 +97,5 @@ class Test(object): pass ''' - expected_ast = ast.parse(expected) - actual_ast = RemoveObject()(ast.parse(source)) - compare_ast(expected_ast, actual_ast) - + min_version, max_version = find_syntax_versions(ast.parse(source)) + assert min_version, max_version == ((2, 7), sys.version_info[:2]) From bf205c3af425efcbe259af421a8de12943b2398c Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Fri, 13 Sep 2024 22:04:01 +0100 Subject: [PATCH 5/9] Add and Update tests for specifying target python versions --- test/test_compat.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/test_compat.py b/test/test_compat.py index 9fa4cb99..b29fbfc0 100644 --- a/test/test_compat.py +++ b/test/test_compat.py @@ -132,8 +132,8 @@ async def f(): assert min_version, max_version == ((3, 5), sys.version_info[:2]) def test_async_with(): - if sys.version_info < (3, 5): - pytest.skip('Python < 3.5 does not have async with') + if sys.version_info < (3, 7): + pytest.skip('Python < 3.7 does not support top level async with') source = ''' async with a: @@ -144,8 +144,8 @@ def test_async_with(): assert min_version, max_version == ((3, 5), sys.version_info[:2]) def test_async_for(): - if sys.version_info < (3, 5): - pytest.skip('Python < 3.5 does not have async for') + if sys.version_info < (3, 7): + pytest.skip('Python < 3.7 does not support top level async for') source = ''' async for a in b: @@ -167,8 +167,8 @@ def test_async_comprehension(): assert min_version, max_version == ((3, 6), sys.version_info[:2]) def test_await(): - if sys.version_info < (3, 5): - pytest.skip('Python < 3.5 does not have await') + if sys.version_info < (3, 7): + pytest.skip('Python < 3.7 does not have top level await') source = ''' await a From 9a17dde724b1aa0fef7b4dfd82e8462733cbfd2f Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Fri, 13 Sep 2024 22:09:20 +0100 Subject: [PATCH 6/9] Add and Update tests for specifying target python versions --- test/test_compat.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/test_compat.py b/test/test_compat.py index b29fbfc0..ee1a019c 100644 --- a/test/test_compat.py +++ b/test/test_compat.py @@ -156,11 +156,11 @@ def test_async_for(): assert min_version, max_version == ((3, 5), sys.version_info[:2]) def test_async_comprehension(): - if sys.version_info < (3, 6): - pytest.skip('Python < 3.6 does not have async comprehensions') + if sys.version_info < (3, 7): + pytest.skip('Python < 3.7 does not have async comprehensions') source = ''' -[a async for a in b] +result = [i async for i in aiter() if i % 2] ''' min_version, max_version = find_syntax_versions(ast.parse(source)) From f37cbe78c8662b46c34ac6bb9ad5b16b0cecf957 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sat, 14 Sep 2024 09:18:36 +0100 Subject: [PATCH 7/9] Remove tests that fail because of a change in external systems --- xtest/manifests/python3.10_test_manifest.yaml | 10 +--------- xtest/manifests/python3.8_test_manifest.yaml | 10 +--------- xtest/manifests/python3.9_test_manifest.yaml | 10 +--------- 3 files changed, 3 insertions(+), 27 deletions(-) diff --git a/xtest/manifests/python3.10_test_manifest.yaml b/xtest/manifests/python3.10_test_manifest.yaml index 5603f85c..421bbf43 100644 --- a/xtest/manifests/python3.10_test_manifest.yaml +++ b/xtest/manifests/python3.10_test_manifest.yaml @@ -3825,15 +3825,7 @@ remove_literal_statements: true rename_globals: true /usr/local/lib/python3.10/test/test_urllib.py: [] -/usr/local/lib/python3.10/test/test_urllib2.py: -- options: {} -- options: - rename_globals: true -- options: - remove_literal_statements: true -- options: - remove_literal_statements: true - rename_globals: true +/usr/local/lib/python3.10/test/test_urllib2.py: [] /usr/local/lib/python3.10/test/test_urllib2_localnet.py: - options: {} - options: diff --git a/xtest/manifests/python3.8_test_manifest.yaml b/xtest/manifests/python3.8_test_manifest.yaml index cef77a00..62870878 100644 --- a/xtest/manifests/python3.8_test_manifest.yaml +++ b/xtest/manifests/python3.8_test_manifest.yaml @@ -3775,15 +3775,7 @@ remove_literal_statements: true rename_globals: true /usr/lib64/python3.8/test/test_urllib.py: [] -/usr/lib64/python3.8/test/test_urllib2.py: -- options: {} -- options: - rename_globals: true -- options: - remove_literal_statements: true -- options: - remove_literal_statements: true - rename_globals: true +/usr/lib64/python3.8/test/test_urllib2.py: [] /usr/lib64/python3.8/test/test_urllib2_localnet.py: - options: {} - options: diff --git a/xtest/manifests/python3.9_test_manifest.yaml b/xtest/manifests/python3.9_test_manifest.yaml index e3f60342..0281205d 100644 --- a/xtest/manifests/python3.9_test_manifest.yaml +++ b/xtest/manifests/python3.9_test_manifest.yaml @@ -3720,15 +3720,7 @@ remove_literal_statements: true rename_globals: true /usr/local/lib/python3.9/test/test_urllib.py: [] -/usr/local/lib/python3.9/test/test_urllib2.py: -- options: {} -- options: - rename_globals: true -- options: - remove_literal_statements: true -- options: - remove_literal_statements: true - rename_globals: true +/usr/local/lib/python3.9/test/test_urllib2.py: [] /usr/local/lib/python3.9/test/test_urllib2_localnet.py: - options: {} - options: From 78cd054b63c1e7e16fdd12639e06821c97a44432 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sat, 14 Sep 2024 17:23:54 +0100 Subject: [PATCH 8/9] Target current python version in corpus tests This will need to be changed in the future to also test compatible target versions. But previous pytohn-minifier versions only targeted current python, so that is the only useful comparison right now. --- .github/workflows/test_corpus.yaml | 3 ++- corpus_test/generate_results.py | 28 ++++++++++++++++++++-------- 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test_corpus.yaml b/.github/workflows/test_corpus.yaml index 99da2e12..baf8472b 100644 --- a/.github/workflows/test_corpus.yaml +++ b/.github/workflows/test_corpus.yaml @@ -46,6 +46,7 @@ jobs: matrix: python: ["2.7", "3.3", "3.4", "3.5", "3.6", "3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] ref: ["${{ inputs.ref }}", "${{ inputs.base-ref }}"] + target_current_python: ["true"] container: image: danielflook/python-minifier-build:python${{ matrix.python }}-2024-01-12 volumes: @@ -96,7 +97,7 @@ jobs: export PYTHONHASHSEED=0 fi - python${{matrix.python}} workflow/corpus_test/generate_results.py /corpus /corpus-results $( Date: Sun, 15 Sep 2024 10:24:19 +0100 Subject: [PATCH 9/9] Target current python version in corpus tests This will need to be changed in the future to also test compatible target versions. But previous pytohn-minifier versions only targeted current python, so that is the only useful comparison right now. --- corpus_test/generate_results.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/corpus_test/generate_results.py b/corpus_test/generate_results.py index e4098b62..18d202be 100644 --- a/corpus_test/generate_results.py +++ b/corpus_test/generate_results.py @@ -87,18 +87,23 @@ def corpus_test(corpus_path, results_path, sha, regenerate_results, target_curre """ python_version = '.'.join([str(s) for s in sys.version_info[:2]]) + target_python = None + if target_current_python: log_path = 'results_' + python_version + '_' + sha + '.log' results_file_path = os.path.join(results_path, 'results_' + python_version + '_' + sha + '.csv') - target_python = python_minifier.TargetPythonOptions( - minimum=sys.version_info[:2], - maximum=sys.version_info[:2] - ) - print(target_python) + + if hasattr(python_minifier, 'TargetPythonOptions'): + target_python = python_minifier.TargetPythonOptions( + minimum=sys.version_info[:2], + maximum=sys.version_info[:2] + ) + print(target_python) + else: + print('Old version of python-minifier which always targets current version') else: log_path = 'results_' + python_version + '_compatible_target_' + sha + '.log' results_file_path = os.path.join(results_path, 'results_' + python_version + '_compatible_target_' + sha + '.csv') - target_python = None print('Targeting compatible Python versions') print('Logging in GitHub Actions is absolute garbage. Logs are going to ' + log_path)