tokenizer added

Author: rawatSingh2022

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@codebolt/codeboltjs",
-  "version": "1.1.22",
+  "version": "1.1.23",
   "description": "",
   "keywords": [],
   "author": "",

src/index.ts

@@ -19,6 +19,7 @@ import cbstate from './modules/state';
 import task from './modules/task';
 import vectorDB from './modules/vectordb';
 import debug from './modules/debug'
+import tokenizer from './modules/tokenizer'
 import WebSocket from 'ws';
 
 
@@ -95,6 +96,7 @@ class Codebolt {
     taskplaner=task;
     vectordb=vectorDB;
     debug=debug;
+    tokenizer=tokenizer;
 }
 
 export default new Codebolt();

src/modules/tokenizer.ts

@@ -0,0 +1,32 @@
+import cbws from './websocket';
+
+/**
+ * Tokenizer module for handling token-related operations.
+ */
+const tokenizer = {
+  
+    /**
+     * Adds a token asynchronously.
+     * @param {string} key - The key of the token to add.
+     * @returns {Promise<any>} A promise that resolves with the response.
+     */
+    addToken: async (key: string): Promise<any> => {
+        return new Promise((resolve, reject) => {
+            cbws.getWebsocket.send(JSON.stringify({
+                "type":"tokenizerEvent",
+                "action": "addToken",
+                "message": {
+                    item: key
+                },
+            }));
+            cbws.getWebsocket.on('message', (data: string) => {
+                const response = JSON.parse(data);
+                if (response.type === "addTokenResponse") {
+                    resolve(response);
+                }
+            });
+        });
+    }
+}
+
+export default tokenizer