Merge pull request #1134 from onekey-sec/arpy-based-ar-extractor

Arpy based ar extractor

Author: qkaiser

python/unblob/file_utils.py

@@ -12,7 +12,7 @@
 import unicodedata
 from collections.abc import Iterable, Iterator
 from pathlib import Path
-from typing import Literal, Optional, Union
+from typing import Literal, Optional, Protocol, Union, overload
 
 from dissect.cstruct import cstruct
 from structlog import get_logger
@@ -269,8 +269,18 @@ def iterate_patterns(
         file.seek(initial_position)
 
 
+class RandomReader(Protocol):
+    # File implements this interface
+
+    @overload
+    def read(self) -> bytes: ...
+    @overload
+    def read(self, n: int, /) -> bytes: ...
+    def seek(self, pos: int, /, whence: int = io.SEEK_SET) -> int: ...
+
+
 def iterate_file(
-    file: File,
+    file: RandomReader,
     start_offset: int,
     size: int,
     # default buffer size in shutil for unix based systems
@@ -297,7 +307,7 @@ def iterate_file(
         yield data
 
 
-def carve(carve_path: Path, file: File, start_offset: int, size: int):
+def carve(carve_path: Path, file: RandomReader, start_offset: int, size: int):
     """Extract part of a file."""
     carve_path.parent.mkdir(parents=True, exist_ok=True)
 

python/unblob/handlers/archive/ar.py

@@ -1,12 +1,13 @@
 import os
+from pathlib import Path
 from typing import Optional
 
 import arpy
 from structlog import get_logger
 
-from ...extractors import Command
-from ...file_utils import OffsetFile
-from ...models import File, Handler, HexString, ValidChunk
+from ...file_utils import FileSystem, OffsetFile, iterate_file
+from ...models import Extractor, ExtractResult, File, Handler, HexString, ValidChunk
+from ...report import ExtractionProblem
 
 logger = get_logger()
 
@@ -15,6 +16,40 @@
 SIGNATURE_LENGTH = 0x8
 
 
+class ArExtractor(Extractor):
+    def extract(self, inpath: Path, outdir: Path) -> Optional[ExtractResult]:
+        fs = FileSystem(outdir)
+
+        with arpy.Archive(inpath.as_posix()) as archive:
+            archive.read_all_headers()
+
+            for name in sorted(archive.archived_files):
+                archived_file = archive.archived_files[name]
+
+                try:
+                    path = Path(name.decode())
+                except UnicodeDecodeError:
+                    path = Path(name.decode(errors="replace"))
+                    fs.record_problem(
+                        ExtractionProblem(
+                            path=repr(name),
+                            problem="Path is not a valid UTF/8 string",
+                            resolution=f"Converted to {path}",
+                        )
+                    )
+
+                fs.write_chunks(
+                    path,
+                    chunks=iterate_file(
+                        archived_file,
+                        0,
+                        archived_file.header.size,
+                    ),
+                )
+
+        return ExtractResult(reports=fs.problems)
+
+
 class ARHandler(Handler):
     NAME = "ar"
 
@@ -27,7 +62,7 @@ class ARHandler(Handler):
         )
     ]
 
-    EXTRACTOR = Command("unar", "-no-directory", "-o", "{outdir}", "{inpath}")
+    EXTRACTOR = ArExtractor()
 
     def calculate_chunk(self, file: File, start_offset: int) -> Optional[ValidChunk]:
         offset_file = OffsetFile(file, start_offset)

tests/integration/archive/ar/__input__/a1x01.ar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7981c6fabc0cb521c443d3b2e8e54713cae8bbccf8208b1950a078367567c3bb
+size 194

tests/integration/archive/ar/__input__/bsd_mixed.ar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1822aa7c0a030fa6ae9ea664b4ba2d804bb0bf5014cc7366d199374a1d64cc7c
+size 358

tests/integration/archive/ar/__input__/bsd_multi_names.ar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9b57b0fc4cff4d4c8c37dbbc1f2f51ed49859f90a931da3465bb27b7ea75a412
+size 552

tests/integration/archive/ar/__input__/bsd_single_name.ar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1d74d5cbf166638434c696563f88b13db1438d34f8e58675ae6bcf337edbcf9c
+size 298

tests/integration/archive/ar/__input__/contents.ar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:34c790979dc722273410bc18d2d83712f763e111aa252f4f248d7785a5b2014c
+size 160

tests/integration/archive/ar/__input__/empty.ar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f0a17a43c74d2fe5474fa2fd29c8f14799e777d7d75a2cc4d11c20a6e7b161c5
+size 8

tests/integration/archive/ar/__input__/gnu_mixed.ar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a43964b9c61030b12b2320f7782e25f2bab411e0529b85b7e92da969ca1d92a9
+size 274

tests/integration/archive/ar/__input__/gnu_multi_names.ar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3aee5062fe584e07862f99d75a4a28613cff6b261d49eab336d59273268fa6d8
+size 372