Change from file-magic to python-magic, use MAGIC_CONTINUE

file magic uses a rule-set to guess the file type, however as rules are added they could
shadow each other. File magic uses rule priorities to determine which is the best matching
rule, however this could shadow other valid matches as well, which could eventually break
any further processing that depends on magic.
By enabling keep_going (which eventually enables MAGIC_CONTINUE) all matching patterns
will be included in the magic string at the cost of being a bit slower, but increasing
accuracy by no shadowing rules.

MAGIC_CONTINUE is more transparently supported with python-magic.
Also file-magic's last release was in 2018,
while there are recent changes on github.

Author: e3krisztian

default.nix

@@ -63,9 +63,9 @@ let
         ];
       });
 
-      file-magic = (super.file-magic.override { preferWheel = false; }).overridePythonAttrs (_: {
+      python-magic = (super.python-magic.override { preferWheel = false; }).overridePythonAttrs (_: {
         patchPhase = ''
-          substituteInPlace magic.py --replace "find_library('magic')" "'${file}/lib/libmagic.so'"
+          substituteInPlace magic/loader.py --replace "find_library('magic')" "'${file}/lib/libmagic.so'"
         '';
       });
 

poetry.lock

@@ -25,7 +25,7 @@ jefferson = { git = "https://github.com/onekey-sec/jefferson.git", rev = "ddbc59
 yaffshiv = { git = "https://github.com/onekey-sec/yaffshiv.git", rev = "24e6e453a36a02144ae2d159eb3229f9c6312828" }
 plotext = "^4.1.5"
 pluggy = "^1.0.0"
-file-magic = "^0.4.0"
+python-magic = "^0.4.27"
 hyperscan = "^0.3.0"
 lark = "^1.1.2"
 lz4 = "^4.0.0"

pyproject.toml

@@ -140,15 +140,26 @@ def from_path(cls, path: Path):
         )
 
 
+# libmagic helpers
+# file magic uses a rule-set to guess the file type, however as rules are added they could
+# shadow each other. File magic uses rule priorities to determine which is the best matching
+# rule, however this could shadow other valid matches as well, which could eventually break
+# any further processing that depends on magic.
+# By enabling keep_going (which eventually enables MAGIC_CONTINUE) all matching patterns
+# will be included in the magic string at the cost of being a bit slower, but increasing
+# accuracy by no shadowing rules.
+get_magic = magic.Magic(keep_going=True).from_file
+get_mime_type = magic.Magic(mime=True).from_file
+
+
 @attr.define(kw_only=True)
 class FileMagicReport(Report):
     magic: str
     mime_type: str
 
     @classmethod
     def from_path(cls, path: Path):
-        detected = magic.detect_from_filename(path)
-        return cls(magic=detected.name, mime_type=detected.mime_type)
+        return cls(magic=get_magic(path), mime_type=get_mime_type(path))
 
 
 @attr.define(kw_only=True)

unblob/report.py

@@ -5,7 +5,7 @@
 from unblob.file_utils import File, iterbits, round_down
 from unblob.models import _JSONEncoder
 from unblob.parser import _HexStringToRegex
-from unblob.report import ChunkReport
+from unblob.report import ChunkReport, FileMagicReport
 
 _HexStringToRegex.literal
 _HexStringToRegex.wildcard
@@ -16,6 +16,8 @@
 _JSONEncoder.default
 
 ChunkReport.handler_name
+FileMagicReport.magic
+FileMagicReport.mime_type
 
 sys.breakpointhook
 cli.cli.context_class