Merge pull request #540 from onekey-sec/ruff

Use ruff instead of flake8 and isort

Author: vlaci

.github/workflows/build-nix.yml

@@ -29,6 +29,7 @@ jobs:
       - name: Setup emulation
         if: ${{ matrix.arch == 'aarch64-linux' }}
         run: |
+          sudo apt update
           sudo apt install -q -y qemu-system-aarch64 qemu-efi binfmt-support qemu-user-static
           mkdir -p ~/.config/nix
           echo "system-features = aarch64-linux arm-linux" | sudo tee -a /etc/nix/nix.conf

.pre-commit-config.yaml

@@ -13,24 +13,19 @@ repos:
       - id: check-yaml
       - id: check-added-large-files
 
+  - repo: https://github.com/charliermarsh/ruff-pre-commit
+    rev: "v0.0.259"
+    hooks:
+      - id: ruff
+        name: Check python (ruff)
+        args: [--show-source, --fix, --exit-non-zero-on-fix]
+
   - repo: https://github.com/psf/black
     rev: 23.1.0
     hooks:
       - id: black
         name: Check black
 
-  - repo: https://github.com/PyCQA/flake8
-    rev: 6.0.0
-    hooks:
-      - id: flake8
-        name: Check flake8
-
-  - repo: https://github.com/PyCQA/isort
-    rev: 5.12.0
-    hooks:
-      - id: isort
-        name: Check isort
-
   - repo: https://github.com/jendrikseipp/vulture
     rev: v2.7
     hooks:
@@ -46,7 +41,7 @@ repos:
   - repo: https://github.com/nix-community/nixpkgs-fmt
     rev: v1.3.0
     hooks:
-    -   id: nixpkgs-fmt
+      - id: nixpkgs-fmt
 
   - repo: https://github.com/python-jsonschema/check-jsonschema
     rev: 0.21.0

docs/development.md

@@ -432,6 +432,10 @@ We adhere to PEP8 and enforce proper formatting of source files using
 [black](https://github.com/psf/black) so you should not worry about formatting
 source code at all, `pre-commit` will take care of it.
 
+For linting we use [ruff](https://ruff.rs). Lint errors can be shown in your
+editor of choice by one of the [editor
+plugins](https://beta.ruff.rs/docs/editor-integrations/).
+
 ### File Format Correctness
 
 We want to strike the right balance between false positive reduction and a

poetry.lock

@@ -39,6 +39,7 @@ pytest = "^6.2.4"
 pyright = "^0.0.12"
 pre-commit = "^2.15.0"
 pytest-cov = "^3.0.0"
+ruff = "^0.0.259"
 setuptools-rust = "^1.1.2"
 
 [tool.poetry.group.docs]
@@ -51,12 +52,89 @@ pillow = "^9.4.0"
 cairosvg = "^2.7.0"
 mkdocstrings-python = "^0.8.3"
 
-[tool.isort]
-profile = "black"
-
 [tool.poetry.scripts]
 unblob = "unblob.cli:main"
 
+[tool.ruff]
+target-version = "py38"
+
+select = [
+  "A",      # flake8-builtins
+  "ARG",    # flake8-unused-arguments
+  "B",      # flake8-bugbear
+  "C90",    # mccabe
+  "C4",     # flake8-comprehensions
+  "COM818", # flake8-commas; trailing-comma-on-bare-tuple
+  "D",      # pydocstyle
+  "E",      # pycodestyle (errors)
+  "F",      # pyflakes
+  "FBT",    # flake8-boolean-trap
+  "G",      # flake8-logging-format
+  "I",      # isort
+  "ISC",    # flake8-implicit-str-concats
+  "N",      # pep8-naming
+  "PIE",    # flake8-pie
+  "PL",     # pylint
+  "PT",     # flake8-pytest-style
+  "PTH",    # flake8-use-pathlib
+  "RET",    # flake8-return
+  "RUF",    # ruff's own lints
+  "RSE",    # flake8-raise
+  "S",      # flake8-bandit
+  "SIM",    # flake8-simplify
+  "SLF",    # flake8-self
+  "T10",    # flake8-debugger
+  "T20",    # flake8-print
+  "TCH",    # flake8-type-checking
+  "TRY",    # tryceratops
+  "UP",     # pyupgrade
+  "W",      # pycodestyle (warnings)
+]
+
+ignore = [
+  "B027",    # empty-method-without-abstract-decorator: It is okay to have empty methods in abstract classes
+  "D1",      # undocumented-*:                          We are not documenting every public symbol
+  "D203",    # one-blank-line-before-class:             D211 (no-blank-line-before-class) is used instead
+  "D213",    # multi-line-summary-second-line:          D212 (multi-line-summary-first-line) is used instead
+  "E501",    # line-too-long:                           Let black handle line length violations
+  "N818",    # error-suffix-on-exception-name:          Exception names can be meaningful without smurfs
+  "PLC0414", # useless-import-alias:                    Enable explicitly re-exporting public symbols
+  "PLR09",   # too-many-{arguments,branches,...}:       We do not want to impose hard limits
+  "S101",    # assert:                                  Enable usage of asserts
+  "TRY003",  # raise-vanilla-args:                      We are adding contextual information in exception messages
+  "TRY400",  # error-instead-of-exception:              It is okay to log without backtrace in except blocks
+]
+
+# Do not remove unused imports automatically in __init__.py files
+ignore-init-module-imports = true
+
+[tool.ruff.per-file-ignores]
+"build.py" = [
+  "T201"  # print
+]
+"tests/*" = [
+  "FBT001",  # boolean-positional-arg-in-function-definition: allowed in `pytest.param()`
+  "FBT003",  # boolean-positional-value-in-function-call:     allowed test parameters
+]
+
+[tool.ruff.flake8-comprehensions]
+allow-dict-calls-with-keyword-arguments = true  # code like `dict(a=1, b=2)` is allowed
+
+[tool.ruff.mccabe]
+max-complexity = 8
+
+[tool.ruff.pylint]
+allow-magic-value-types = [
+  "bytes",
+  "int",   # bunch of ad-hoc bitmask values
+  "str",
+]
+
+[tool.ruff.flake8-pytest-style]
+fixture-parentheses = false
+mark-parentheses = false
+parametrize-names-type = "csv"
+
 [tool.pytest.ini_options]
 addopts = "--cov=unblob --cov=tests --cov-branch --cov-fail-under=90"
 norecursedirs = """

pyproject.toml

@@ -11,7 +11,7 @@
 
 def test_command_templating():
     command = Command("{outdir}", "{inpath},{outdir}")
-    cmdline = command._make_extract_command(
+    cmdline = command._make_extract_command(  # noqa: SLF001
         Path("inputdir") / "input.file", Path("output")
     )
 
@@ -20,16 +20,16 @@ def test_command_templating():
 
 @pytest.mark.parametrize(
     "template",
-    (
+    [
         "{no_such_placeholder}",
         "{malformed",
-    ),
+    ],
 )
 def test_command_templating_with_invalid_substitution(template):
     command = Command(template)
 
     with pytest.raises(InvalidCommandTemplate, match=template):
-        command._make_extract_command(Path("input"), Path("output"))
+        command._make_extract_command(Path("input"), Path("output"))  # noqa: SLF001
 
 
 def test_command_execution(tmpdir: Path):
@@ -45,30 +45,28 @@ def test_command_execution_failure(tmpdir: Path):
     outdir = PosixPath(tmpdir)
     command = Command("sh", "-c", ">&1 echo -n stdout; >&2 echo -n stderr; false")
 
-    try:
+    with pytest.raises(ExtractError) as excinfo:
         command.extract(Path("input"), outdir)
-        pytest.fail("ExtractError not raised")
-    except ExtractError as e:
-        assert list(e.reports) == [
-            ExtractCommandFailedReport(
-                command=mock.ANY,
-                stdout=b"stdout",
-                stderr=b"stderr",
-                exit_code=1,
-            )
-        ]
+
+    assert list(excinfo.value.reports) == [
+        ExtractCommandFailedReport(
+            command=mock.ANY,
+            stdout=b"stdout",
+            stderr=b"stderr",
+            exit_code=1,
+        )
+    ]
 
 
 def test_command_not_found(tmpdir: Path):
     outdir = PosixPath(tmpdir)
     command = Command("this-command-should-not-exist-in-any-system")
 
-    try:
+    with pytest.raises(ExtractError) as excinfo:
         command.extract(Path("input"), outdir)
-        pytest.fail("ExtractError not raised")
-    except ExtractError as e:
-        assert list(e.reports) == [
-            ExtractorDependencyNotFoundReport(
-                dependencies=["this-command-should-not-exist-in-any-system"],
-            )
-        ]
+
+    assert list(excinfo.value.reports) == [
+        ExtractorDependencyNotFoundReport(
+            dependencies=["this-command-should-not-exist-in-any-system"],
+        )
+    ]

setup.cfg

@@ -50,11 +50,11 @@ def test_valid_calculation():
 
 @pytest.mark.parametrize(
     "header_size",
-    (
+    [
         pytest.param(0, id="size-small"),
         pytest.param(2700, id="size-large"),
         pytest.param(31, id="size-smaller-then-first_hdr_size"),
-    ),
+    ],
 )
 def test_invalid_block_size(header_size):
     contents = bytearray(copy.copy(ARJ_CONTENTS))
@@ -65,7 +65,7 @@ def test_invalid_block_size(header_size):
 
     handler = ARJHandler()
     with pytest.raises(InvalidARJSize):
-        handler._read_arj_main_header(f, 0)
+        handler._read_arj_main_header(f, 0)  # noqa: SLF001
 
 
 def test_invalid_checksum():
@@ -76,4 +76,4 @@ def test_invalid_checksum():
 
     handler = ARJHandler()
     with pytest.raises(ARJChecksumError):
-        handler._read_arj_main_header(f, 0)
+        handler._read_arj_main_header(f, 0)  # noqa: SLF001

tests/extractors/test_command.py

@@ -136,7 +136,7 @@
 
 @pytest.mark.parametrize(
     "contents, expected_length, message",
-    (
+    [
         pytest.param(
             GNU_TAR_CONTENTS + PADDING_TO_DEFAULT_BLOCKING_FACTOR,
             len(GNU_TAR_CONTENTS + PADDING_TO_DEFAULT_BLOCKING_FACTOR),
@@ -173,7 +173,7 @@
             "File end shouldn't include partial zero filled blocks",
             id="posix-padded-after-end",
         ),
-    ),
+    ],
 )
 def test_offset(contents: bytes, expected_length: int, message: str):
     f = File.from_bytes(contents)
@@ -184,7 +184,7 @@ def test_offset(contents: bytes, expected_length: int, message: str):
 
 @pytest.mark.parametrize(
     "contents",
-    (
+    [
         pytest.param(
             GNU_TAR_CONTENTS,
             id="gnu-tar",
@@ -193,11 +193,11 @@ def test_offset(contents: bytes, expected_length: int, message: str):
             POSIX_TAR_CONTENTS,
             id="posix-tar",
         ),
-    ),
+    ],
 )
 @pytest.mark.parametrize(
     "start_complete, message",
-    (
+    [
         pytest.param(
             False,
             "File is truncated and no content can be recovered",
@@ -208,7 +208,7 @@ def test_offset(contents: bytes, expected_length: int, message: str):
             "File is truncated but valid parts should be recovered",
             id="truncated",
         ),
-    ),
+    ],
 )
 def test_truncated_files(contents: bytes, start_complete: bool, message: str):
     truncated = contents[:0x180]
@@ -298,10 +298,10 @@ def test_different_blocking_factor():
 
 @pytest.mark.parametrize(
     "prefix",
-    (
+    [
         pytest.param(b"", id="zero-prefix"),
         pytest.param(b"some prefix ", id="nonzero-prefix"),
-    ),
+    ],
 )
 def test_calculate_chunk(prefix):
     tar_file = File.from_bytes(prefix + GNU_TAR_CONTENTS)