Merge pull request #160 from weaviate/add-kagome-kr-tokenizer

Add support for the kagome_kr tokenizer

Author: tsmith023

src/collections/configure/index.ts

@@ -43,6 +43,7 @@ const tokenization = {
   FIELD: 'field' as const,
   TRIGRAM: 'trigram' as const,
   GSE: 'gse' as const,
+  KAGOME_KR: 'kagome_kr' as const,
 };
 
 const vectorDistances = {

src/openapi/schema.ts

@@ -518,7 +518,7 @@ export interface definitions {
      * @description Determines tokenization of the property as separate words or whole field. Optional. Applies to text and text[] data types. Allowed values are `word` (default; splits on any non-alphanumerical, lowercases), `lowercase` (splits on white spaces, lowercases), `whitespace` (splits on white spaces), `field` (trims). Not supported for remaining data types
      * @enum {string}
      */
-    tokenization?: 'word' | 'lowercase' | 'whitespace' | 'field' | 'trigram' | 'gse';
+    tokenization?: 'word' | 'lowercase' | 'whitespace' | 'field' | 'trigram' | 'gse' | 'kagome_kr';
     /** @description The properties of the nested object(s). Applies to object and object[] data types. */
     nestedProperties?: definitions['NestedProperty'][];
   };