Add trigram tokenizer

Author: jberkel

Sources/SQLite/Extensions/FTS4.swift

@@ -95,10 +95,10 @@ extension VirtualTable {
 public struct Tokenizer {
 
     public static let Simple = Tokenizer("simple")
-
     public static let Porter = Tokenizer("porter")
 
-    public static func Unicode61(removeDiacritics: Bool? = nil, tokenchars: Set<Character> = [],
+    public static func Unicode61(removeDiacritics: Bool? = nil,
+                                 tokenchars: Set<Character> = [],
                                  separators: Set<Character> = []) -> Tokenizer {
         var arguments = [String]()
 
@@ -119,6 +119,11 @@ public struct Tokenizer {
         return Tokenizer("unicode61", arguments)
     }
 
+    // https://sqlite.org/fts5.html#the_experimental_trigram_tokenizer
+    public static func Trigram(caseSensitive: Bool = false) -> Tokenizer {
+        return Tokenizer("trigram", ["case_sensitive", caseSensitive ? "1" : "0"])
+    }
+
     public static func Custom(_ name: String) -> Tokenizer {
         Tokenizer(Tokenizer.moduleName.quote(), [name.quote()])
     }

Tests/SQLiteTests/FTS5Tests.swift

@@ -81,6 +81,16 @@ class FTS5Tests: XCTestCase {
             sql(config.tokenizer(.Unicode61(removeDiacritics: true, tokenchars: ["."], separators: ["X"]))))
     }
 
+    func test_tokenizer_trigram() {
+        XCTAssertEqual(
+                "CREATE VIRTUAL TABLE \"virtual_table\" USING fts5(tokenize=trigram case_sensitive 0)",
+                sql(config.tokenizer(.Trigram())))
+
+        XCTAssertEqual(
+                "CREATE VIRTUAL TABLE \"virtual_table\" USING fts5(tokenize=trigram case_sensitive 1)",
+                sql(config.tokenizer(.Trigram(caseSensitive: true))))
+    }
+
     func test_column_size() {
         XCTAssertEqual(
             "CREATE VIRTUAL TABLE \"virtual_table\" USING fts5(columnsize=1)",