Tokenizers don't need config.json (#261)

Author: pcuenca

Sources/Hub/Hub.swift

@@ -70,7 +70,7 @@ public extension Hub {
 
 public final class LanguageModelConfigurationFromHub: Sendable {
     struct Configurations {
-        var modelConfig: Config
+        var modelConfig: Config?
         var tokenizerConfig: Config?
         var tokenizerData: Config
     }
@@ -96,7 +96,7 @@ public final class LanguageModelConfigurationFromHub: Sendable {
         }
     }
 
-    public var modelConfig: Config {
+    public var modelConfig: Config? {
         get async throws {
             try await configPromise.value.modelConfig
         }
@@ -135,7 +135,7 @@ public final class LanguageModelConfigurationFromHub: Sendable {
 
     public var modelType: String? {
         get async throws {
-            try await modelConfig.modelType.string()
+            try await modelConfig?.modelType.string()
         }
     }
 
@@ -174,11 +174,11 @@ public final class LanguageModelConfigurationFromHub: Sendable {
         do {
             // Load required configurations
             let modelConfigURL = modelFolder.appending(path: "config.json")
-            guard FileManager.default.fileExists(atPath: modelConfigURL.path) else {
-                throw Hub.HubClientError.configurationMissing("config.json")
-            }
 
-            let modelConfig = try hubApi.configuration(fileURL: modelConfigURL)
+            var modelConfig: Config? = nil
+            if FileManager.default.fileExists(atPath: modelConfigURL.path) {
+                modelConfig = try hubApi.configuration(fileURL: modelConfigURL)
+            }
 
             let tokenizerDataURL = modelFolder.appending(path: "tokenizer.json")
             guard FileManager.default.fileExists(atPath: tokenizerDataURL.path) else {

Sources/Models/LanguageModel.swift

@@ -141,7 +141,7 @@ public extension LanguageModel {
 
 /// async properties downloaded from the configuration
 public extension LanguageModel {
-    var modelConfig: Config {
+    var modelConfig: Config? {
         get async throws {
             try await configuration!.modelConfig
         }
@@ -161,13 +161,13 @@ public extension LanguageModel {
 
     var modelType: String? {
         get async throws {
-            try await modelConfig.modelType.string()
+            try await modelConfig?.modelType.string()
         }
     }
 
     var textGenerationParameters: Config? {
         get async throws {
-            try await modelConfig.taskSpecificParams.textGeneration
+            try await modelConfig?.taskSpecificParams.textGeneration
         }
     }
 
@@ -180,14 +180,14 @@ public extension LanguageModel {
     var bosTokenId: Int? {
         get async throws {
             let modelConfig = try await modelConfig
-            return modelConfig.bosTokenId.integer()
+            return modelConfig?.bosTokenId.integer()
         }
     }
 
     var eosTokenId: Int? {
         get async throws {
             let modelConfig = try await modelConfig
-            return modelConfig.eosTokenId.integer()
+            return modelConfig?.eosTokenId.integer()
         }
     }
 

Tests/HubTests/HubTests.swift

@@ -29,7 +29,10 @@ class HubTests: XCTestCase {
     func testConfigDownload() async {
         do {
             let configLoader = LanguageModelConfigurationFromHub(modelName: "t5-base", hubApi: hubApi)
-            let config = try await configLoader.modelConfig
+            guard let config = try await configLoader.modelConfig else {
+                XCTFail("Test repo is expected to have a config.json file")
+                return
+            }
 
             // Test leaf value (Int)
             guard let eos = config["eos_token_id"].integer() else {
@@ -71,7 +74,10 @@ class HubTests: XCTestCase {
     func testConfigCamelCase() async {
         do {
             let configLoader = LanguageModelConfigurationFromHub(modelName: "t5-base", hubApi: hubApi)
-            let config = try await configLoader.modelConfig
+            guard let config = try await configLoader.modelConfig else {
+                XCTFail("Test repo is expected to have a config.json file")
+                return
+            }
 
             // Test leaf value (Int)
             guard let eos = config["eosTokenId"].integer() else {