Merge pull request #399 from onekey-sec/magic-wrapper-change

Change from file-magic to python-magic, use MAGIC_CONTINUE

Author: martonilles

default.nix

@@ -63,9 +63,9 @@ let
         ];
       });
 
-      file-magic = (super.file-magic.override { preferWheel = false; }).overridePythonAttrs (_: {
+      python-magic = (super.python-magic.override { preferWheel = false; }).overridePythonAttrs (_: {
         patchPhase = ''
-          substituteInPlace magic.py --replace "find_library('magic')" "'${file}/lib/libmagic.so'"
+          substituteInPlace magic/loader.py --replace "find_library('magic')" "'${file}/lib/libmagic.so'"
         '';
       });
 

poetry.lock

@@ -25,7 +25,7 @@ jefferson = { git = "https://github.com/onekey-sec/jefferson.git", rev = "ddbc59
 yaffshiv = { git = "https://github.com/onekey-sec/yaffshiv.git", rev = "24e6e453a36a02144ae2d159eb3229f9c6312828" }
 plotext = "^4.1.5"
 pluggy = "^1.0.0"
-file-magic = "^0.4.0"
+python-magic = "^0.4.27"
 hyperscan = "^0.3.0"
 lark = "^1.1.2"
 lz4 = "^4.0.0"

pyproject.toml

@@ -140,15 +140,26 @@ def from_path(cls, path: Path):
         )
 
 
+# libmagic helpers
+# file magic uses a rule-set to guess the file type, however as rules are added they could
+# shadow each other. File magic uses rule priorities to determine which is the best matching
+# rule, however this could shadow other valid matches as well, which could eventually break
+# any further processing that depends on magic.
+# By enabling keep_going (which eventually enables MAGIC_CONTINUE) all matching patterns
+# will be included in the magic string at the cost of being a bit slower, but increasing
+# accuracy by no shadowing rules.
+get_magic = magic.Magic(keep_going=True).from_file
+get_mime_type = magic.Magic(mime=True).from_file
+
+
 @attr.define(kw_only=True)
 class FileMagicReport(Report):
     magic: str
     mime_type: str
 
     @classmethod
     def from_path(cls, path: Path):
-        detected = magic.detect_from_filename(path)
-        return cls(magic=detected.name, mime_type=detected.mime_type)
+        return cls(magic=get_magic(path), mime_type=get_mime_type(path))
 
 
 @attr.define(kw_only=True)

unblob/report.py

@@ -5,7 +5,7 @@
 from unblob.file_utils import File, iterbits, round_down
 from unblob.models import _JSONEncoder
 from unblob.parser import _HexStringToRegex
-from unblob.report import ChunkReport
+from unblob.report import ChunkReport, FileMagicReport
 
 _HexStringToRegex.literal
 _HexStringToRegex.wildcard
@@ -16,6 +16,8 @@
 _JSONEncoder.default
 
 ChunkReport.handler_name
+FileMagicReport.magic
+FileMagicReport.mime_type
 
 sys.breakpointhook
 cli.cli.context_class