@@ -11,6 +11,7 @@ import {
1111 GGUF_QUANT_ORDER ,
1212 findNearestQuantType ,
1313 serializeGgufMetadata ,
14+ buildGgufHeader ,
1415} from "./gguf" ;
1516import fs from "node:fs" ;
1617import { tmpdir } from "node:os" ;
@@ -832,7 +833,6 @@ describe("gguf", () => {
832833 typedMetadata : originalMetadata ,
833834 tensorDataOffset,
834835 littleEndian,
835- tensorInfos,
836836 } = await gguf ( testUrl , {
837837 typedMetadata : true ,
838838 } ) ;
@@ -895,4 +895,288 @@ describe("gguf", () => {
895895 }
896896 } , 30000 ) ;
897897 } ) ;
898+
899+ describe ( "buildGgufHeader" , ( ) => {
900+ it ( "should rebuild GGUF header with updated metadata" , async ( ) => {
901+ // Parse a smaller GGUF file to get original metadata and structure
902+ const {
903+ typedMetadata : originalMetadata ,
904+ tensorInfoByteRange,
905+ littleEndian,
906+ } = await gguf ( URL_V1 , {
907+ typedMetadata : true ,
908+ } ) ;
909+
910+ // Get only the header portion of the original file to avoid memory issues
911+ const headerSize = tensorInfoByteRange [ 1 ] + 1000 ; // Add some padding
912+ const originalResponse = await fetch ( URL_V1 , {
913+ headers : { Range : `bytes=0-${ headerSize - 1 } ` } ,
914+ } ) ;
915+ const originalBlob = new Blob ( [ await originalResponse . arrayBuffer ( ) ] ) ;
916+
917+ // Create updated metadata with a modified name
918+ const updatedMetadata = {
919+ ...originalMetadata ,
920+ "general.name" : {
921+ value : "Modified Test Model" ,
922+ type : GGUFValueType . STRING ,
923+ } ,
924+ } as GGUFTypedMetadata ;
925+
926+ // Build the new header
927+ const newHeaderBlob = await buildGgufHeader ( originalBlob , updatedMetadata , {
928+ littleEndian,
929+ tensorInfoByteRange,
930+ alignment : Number ( originalMetadata [ "general.alignment" ] ?. value ?? 32 ) ,
931+ } ) ;
932+
933+ expect ( newHeaderBlob ) . toBeInstanceOf ( Blob ) ;
934+ expect ( newHeaderBlob . size ) . toBeGreaterThan ( 0 ) ;
935+
936+ // Test that the new header can be parsed by creating a minimal test file
937+ const tempFilePath = join ( tmpdir ( ) , `test-build-header-${ Date . now ( ) } .gguf` ) ;
938+
939+ // Just write the header to test parsing (without tensor data to avoid size issues)
940+ fs . writeFileSync ( tempFilePath , Buffer . from ( await newHeaderBlob . arrayBuffer ( ) ) ) ;
941+
942+ try {
943+ const { typedMetadata : parsedMetadata } = await gguf ( tempFilePath , {
944+ typedMetadata : true ,
945+ allowLocalFile : true ,
946+ } ) ;
947+
948+ // Verify the updated metadata is preserved
949+ expect ( parsedMetadata [ "general.name" ] ) . toEqual ( {
950+ value : "Modified Test Model" ,
951+ type : GGUFValueType . STRING ,
952+ } ) ;
953+
954+ // Verify other metadata fields are preserved
955+ expect ( parsedMetadata . version ) . toEqual ( originalMetadata . version ) ;
956+ expect ( parsedMetadata . tensor_count ) . toEqual ( originalMetadata . tensor_count ) ;
957+ expect ( parsedMetadata [ "general.architecture" ] ) . toEqual ( originalMetadata [ "general.architecture" ] ) ;
958+ } finally {
959+ try {
960+ fs . unlinkSync ( tempFilePath ) ;
961+ } catch ( error ) {
962+ // Ignore cleanup errors
963+ }
964+ }
965+ } , 30_000 ) ;
966+
967+ it ( "should handle metadata with array modifications" , async ( ) => {
968+ // Parse a smaller GGUF file
969+ const {
970+ typedMetadata : originalMetadata ,
971+ tensorInfoByteRange,
972+ littleEndian,
973+ } = await gguf ( URL_V1 , {
974+ typedMetadata : true ,
975+ } ) ;
976+
977+ // Get only the header portion
978+ const headerSize = tensorInfoByteRange [ 1 ] + 1000 ;
979+ const originalResponse = await fetch ( URL_V1 , {
980+ headers : { Range : `bytes=0-${ headerSize - 1 } ` } ,
981+ } ) ;
982+ const originalBlob = new Blob ( [ await originalResponse . arrayBuffer ( ) ] ) ;
983+
984+ // Create updated metadata with a simple array
985+ const updatedMetadata = {
986+ ...originalMetadata ,
987+ "test.array" : {
988+ value : [ "item1" , "item2" , "item3" ] ,
989+ type : GGUFValueType . ARRAY ,
990+ subType : GGUFValueType . STRING ,
991+ } ,
992+ kv_count : {
993+ value : originalMetadata . kv_count . value + 1n ,
994+ type : originalMetadata . kv_count . type ,
995+ } ,
996+ } as GGUFTypedMetadata ;
997+
998+ // Build the new header
999+ const newHeaderBlob = await buildGgufHeader ( originalBlob , updatedMetadata , {
1000+ littleEndian,
1001+ tensorInfoByteRange,
1002+ alignment : Number ( originalMetadata [ "general.alignment" ] ?. value ?? 32 ) ,
1003+ } ) ;
1004+
1005+ expect ( newHeaderBlob ) . toBeInstanceOf ( Blob ) ;
1006+ expect ( newHeaderBlob . size ) . toBeGreaterThan ( 0 ) ;
1007+
1008+ // Test that the new header can be parsed
1009+ const tempFilePath = join ( tmpdir ( ) , `test-build-header-array-${ Date . now ( ) } .gguf` ) ;
1010+ fs . writeFileSync ( tempFilePath , Buffer . from ( await newHeaderBlob . arrayBuffer ( ) ) ) ;
1011+
1012+ try {
1013+ const { typedMetadata : parsedMetadata } = await gguf ( tempFilePath , {
1014+ typedMetadata : true ,
1015+ allowLocalFile : true ,
1016+ } ) ;
1017+
1018+ // Verify the array was added correctly
1019+ expect ( parsedMetadata [ "test.array" ] ) . toEqual ( {
1020+ value : [ "item1" , "item2" , "item3" ] ,
1021+ type : GGUFValueType . ARRAY ,
1022+ subType : GGUFValueType . STRING ,
1023+ } ) ;
1024+
1025+ // Verify structure integrity
1026+ expect ( parsedMetadata . version ) . toEqual ( originalMetadata . version ) ;
1027+ expect ( parsedMetadata . tensor_count ) . toEqual ( originalMetadata . tensor_count ) ;
1028+ expect ( parsedMetadata . kv_count . value ) . toBe ( originalMetadata . kv_count . value + 1n ) ;
1029+ } finally {
1030+ try {
1031+ fs . unlinkSync ( tempFilePath ) ;
1032+ } catch ( error ) {
1033+ // Ignore cleanup errors
1034+ }
1035+ }
1036+ } , 30_000 ) ;
1037+
1038+ it ( "should preserve tensor info correctly" , async ( ) => {
1039+ // Parse a smaller GGUF file
1040+ const {
1041+ typedMetadata : originalMetadata ,
1042+ tensorInfoByteRange,
1043+ tensorInfos : originalTensorInfos ,
1044+ littleEndian,
1045+ } = await gguf ( URL_V1 , {
1046+ typedMetadata : true ,
1047+ } ) ;
1048+
1049+ // Get only the header portion
1050+ const headerSize = tensorInfoByteRange [ 1 ] + 1000 ;
1051+ const originalResponse = await fetch ( URL_V1 , {
1052+ headers : { Range : `bytes=0-${ headerSize - 1 } ` } ,
1053+ } ) ;
1054+ const originalBlob = new Blob ( [ await originalResponse . arrayBuffer ( ) ] ) ;
1055+
1056+ // Create updated metadata with minor changes
1057+ const updatedMetadata = {
1058+ ...originalMetadata ,
1059+ "test.custom" : {
1060+ value : "custom_value" ,
1061+ type : GGUFValueType . STRING ,
1062+ } ,
1063+ kv_count : {
1064+ value : originalMetadata . kv_count . value + 1n ,
1065+ type : originalMetadata . kv_count . type ,
1066+ } ,
1067+ } as GGUFTypedMetadata ;
1068+
1069+ // Build the new header
1070+ const newHeaderBlob = await buildGgufHeader ( originalBlob , updatedMetadata , {
1071+ littleEndian,
1072+ tensorInfoByteRange,
1073+ alignment : Number ( originalMetadata [ "general.alignment" ] ?. value ?? 32 ) ,
1074+ } ) ;
1075+
1076+ // Test that the new header can be parsed
1077+ const tempFilePath = join ( tmpdir ( ) , `test-build-header-tensors-${ Date . now ( ) } .gguf` ) ;
1078+ fs . writeFileSync ( tempFilePath , Buffer . from ( await newHeaderBlob . arrayBuffer ( ) ) ) ;
1079+
1080+ try {
1081+ const { typedMetadata : parsedMetadata , tensorInfos : parsedTensorInfos } = await gguf ( tempFilePath , {
1082+ typedMetadata : true ,
1083+ allowLocalFile : true ,
1084+ } ) ;
1085+
1086+ // Verify tensor info is preserved exactly
1087+ expect ( parsedTensorInfos . length ) . toBe ( originalTensorInfos . length ) ;
1088+ expect ( parsedTensorInfos [ 0 ] ) . toEqual ( originalTensorInfos [ 0 ] ) ;
1089+ expect ( parsedTensorInfos [ parsedTensorInfos . length - 1 ] ) . toEqual (
1090+ originalTensorInfos [ originalTensorInfos . length - 1 ]
1091+ ) ;
1092+
1093+ // Verify our custom metadata was added
1094+ expect ( parsedMetadata [ "test.custom" ] ) . toEqual ( {
1095+ value : "custom_value" ,
1096+ type : GGUFValueType . STRING ,
1097+ } ) ;
1098+
1099+ // Verify kv_count was updated
1100+ expect ( parsedMetadata . kv_count . value ) . toBe ( originalMetadata . kv_count . value + 1n ) ;
1101+ } finally {
1102+ try {
1103+ fs . unlinkSync ( tempFilePath ) ;
1104+ } catch ( error ) {
1105+ // Ignore cleanup errors
1106+ }
1107+ }
1108+ } , 30_000 ) ;
1109+
1110+ it ( "should handle different alignment values" , async ( ) => {
1111+ // Parse a smaller GGUF file
1112+ const {
1113+ typedMetadata : originalMetadata ,
1114+ tensorInfoByteRange,
1115+ littleEndian,
1116+ } = await gguf ( URL_V1 , {
1117+ typedMetadata : true ,
1118+ } ) ;
1119+
1120+ // Get only the header portion
1121+ const headerSize = tensorInfoByteRange [ 1 ] + 1000 ;
1122+ const originalResponse = await fetch ( URL_V1 , {
1123+ headers : { Range : `bytes=0-${ headerSize - 1 } ` } ,
1124+ } ) ;
1125+ const originalBlob = new Blob ( [ await originalResponse . arrayBuffer ( ) ] ) ;
1126+
1127+ // Create updated metadata
1128+ const updatedMetadata = {
1129+ ...originalMetadata ,
1130+ "general.name" : {
1131+ value : "Alignment Test Model" ,
1132+ type : GGUFValueType . STRING ,
1133+ } ,
1134+ } as GGUFTypedMetadata ;
1135+
1136+ // Test different alignment values
1137+ const alignments = [ 16 , 32 , 64 ] ;
1138+
1139+ for ( const alignment of alignments ) {
1140+ const newHeaderBlob = await buildGgufHeader ( originalBlob , updatedMetadata , {
1141+ littleEndian,
1142+ tensorInfoByteRange,
1143+ alignment,
1144+ } ) ;
1145+
1146+ expect ( newHeaderBlob ) . toBeInstanceOf ( Blob ) ;
1147+ expect ( newHeaderBlob . size ) . toBeGreaterThan ( 0 ) ;
1148+
1149+ // Verify the header size is aligned correctly
1150+ expect ( newHeaderBlob . size % alignment ) . toBe ( 0 ) ;
1151+ }
1152+ } , 15_000 ) ;
1153+
1154+ it ( "should validate tensorInfoByteRange parameters" , async ( ) => {
1155+ // Parse a smaller GGUF file
1156+ const { typedMetadata : originalMetadata , littleEndian } = await gguf ( URL_V1 , {
1157+ typedMetadata : true ,
1158+ } ) ;
1159+
1160+ // Create a small test blob
1161+ const testBlob = new Blob ( [ new Uint8Array ( 1000 ) ] ) ;
1162+
1163+ // Test with valid range first to ensure function works
1164+ const validResult = await buildGgufHeader ( testBlob , originalMetadata , {
1165+ littleEndian,
1166+ tensorInfoByteRange : [ 100 , 200 ] , // Valid: start < end
1167+ alignment : 32 ,
1168+ } ) ;
1169+
1170+ expect ( validResult ) . toBeInstanceOf ( Blob ) ;
1171+
1172+ // Test with edge case: start == end (should work as empty range)
1173+ const emptyRangeResult = await buildGgufHeader ( testBlob , originalMetadata , {
1174+ littleEndian,
1175+ tensorInfoByteRange : [ 100 , 100 ] , // Edge case: empty range
1176+ alignment : 32 ,
1177+ } ) ;
1178+
1179+ expect ( emptyRangeResult ) . toBeInstanceOf ( Blob ) ;
1180+ } , 15_000 ) ;
1181+ } ) ;
8981182} ) ;
0 commit comments