chore: upgrade core dependencies

Author: manekinekko

app/api/chat/config/llamacloud/route.ts

@@ -0,0 +1,24 @@
+import { LLamaCloudFileService } from "llamaindex";
+import { NextResponse } from "next/server";
+
+/**
+ * This API is to get config from the backend envs and expose them to the frontend
+ */
+export async function GET() {
+  if (!process.env.LLAMA_CLOUD_API_KEY) {
+    return NextResponse.json(
+      {
+        error: "env variable LLAMA_CLOUD_API_KEY is required to use LlamaCloud",
+      },
+      { status: 500 },
+    );
+  }
+  const config = {
+    projects: await LLamaCloudFileService.getAllProjectsWithPipelines(),
+    pipeline: {
+      pipeline: process.env.LLAMA_CLOUD_INDEX_NAME,
+      project: process.env.LLAMA_CLOUD_PROJECT_NAME,
+    },
+  };
+  return NextResponse.json(config, { status: 200 });
+}

app/api/chat/config/route.ts

@@ -0,0 +1,11 @@
+import { NextResponse } from "next/server";
+
+/**
+ * This API is to get config from the backend envs and expose them to the frontend
+ */
+export async function GET() {
+  const config = {
+    starterQuestions: process.env.CONVERSATION_STARTERS?.trim().split("\n"),
+  };
+  return NextResponse.json(config, { status: 200 });
+}

app/api/chat/engine/chat.ts

@@ -1,21 +1,36 @@
-import { ContextChatEngine, Settings } from "llamaindex";
+import { BaseChatEngine, BaseToolWithCall, LLMAgent } from "llamaindex";
+import fs from "node:fs/promises";
+import path from "node:path";
 import { getDataSource } from "./index";
+import { createTools } from "./tools";
+import { createQueryEngineTool } from "./tools/query-engine";
 
-export async function createChatEngine() {
-  const index = await getDataSource();
-  if (!index) {
-    throw new Error(
-      `StorageContext is empty - call 'npm run generate' to generate the storage first`,
-    );
+export async function createChatEngine(documentIds?: string[], params?: any) {
+  const tools: BaseToolWithCall[] = [];
+
+  // Add a query engine tool if we have a data source
+  // Delete this code if you don't have a data source
+  const index = await getDataSource(params);
+  if (index) {
+    tools.push(createQueryEngineTool(index, { documentIds }));
   }
-  const retriever = index.asRetriever();
-  retriever.similarityTopK = process.env.TOP_K
-    ? parseInt(process.env.TOP_K)
-    : 3;
 
-  return new ContextChatEngine({
-    chatModel: Settings.llm,
-    retriever,
+  const configFile = path.join("config", "tools.json");
+  let toolConfig: any;
+  try {
+    // add tools from config file if it exists
+    toolConfig = JSON.parse(await fs.readFile(configFile, "utf8"));
+  } catch (e) {
+    console.info(`Could not read ${configFile} file. Using no tools.`);
+  }
+  if (toolConfig) {
+    tools.push(...(await createTools(toolConfig)));
+  }
+
+  const agent = new LLMAgent({
+    tools,
     systemPrompt: process.env.SYSTEM_PROMPT,
-  });
+  }) as unknown as BaseChatEngine;
+
+  return agent;
 }

app/api/chat/engine/generate.ts

@@ -5,7 +5,6 @@ import * as dotenv from "dotenv";
 
 import { getDocuments } from "./loader";
 import { initSettings } from "./settings";
-import { STORAGE_CACHE_DIR } from "./shared";
 
 // Load environment variables from local .env file
 dotenv.config();
@@ -20,11 +19,16 @@ async function getRuntime(func: any) {
 async function generateDatasource() {
   console.log(`Generating storage context...`);
   // Split documents, create embeddings and store them in the storage context
+  const persistDir = process.env.STORAGE_CACHE_DIR;
+  if (!persistDir) {
+    throw new Error("STORAGE_CACHE_DIR environment variable is required!");
+  }
   const ms = await getRuntime(async () => {
     const storageContext = await storageContextFromDefaults({
-      persistDir: STORAGE_CACHE_DIR,
+      persistDir,
     });
     const documents = await getDocuments();
+
     await VectorStoreIndex.fromDocuments(documents, {
       storageContext,
     });

app/api/chat/engine/index.ts

@@ -1,10 +1,13 @@
 import { SimpleDocumentStore, VectorStoreIndex } from "llamaindex";
 import { storageContextFromDefaults } from "llamaindex/storage/StorageContext";
-import { STORAGE_CACHE_DIR } from "./shared";
 
-export async function getDataSource() {
+export async function getDataSource(params?: any) {
+  const persistDir = process.env.STORAGE_CACHE_DIR;
+  if (!persistDir) {
+    throw new Error("STORAGE_CACHE_DIR environment variable is required!");
+  }
   const storageContext = await storageContextFromDefaults({
-    persistDir: `${STORAGE_CACHE_DIR}`,
+    persistDir,
   });
 
   const numberOfDocs = Object.keys(

app/api/chat/engine/loader.ts

@@ -1,9 +1,24 @@
-import { SimpleDirectoryReader } from "llamaindex";
+import {
+  FILE_EXT_TO_READER,
+  SimpleDirectoryReader,
+} from "@llamaindex/readers/directory";
 
 export const DATA_DIR = "./data";
 
+export function getExtractors() {
+  return FILE_EXT_TO_READER;
+}
+
 export async function getDocuments() {
-  return await new SimpleDirectoryReader().loadData({
+  const documents = await new SimpleDirectoryReader().loadData({
     directoryPath: DATA_DIR,
   });
+  // Set private=false to mark the document as public (required for filtering)
+  for (const document of documents) {
+    document.metadata = {
+      ...document.metadata,
+      private: "false",
+    };
+  }
+  return documents;
 }

app/api/chat/engine/provider.ts

@@ -0,0 +1,37 @@
+import { OpenAI, OpenAIEmbedding } from "@llamaindex/openai";
+import { Settings } from "llamaindex";
+import {
+  DefaultAzureCredential,
+  getBearerTokenProvider,
+} from "@azure/identity";
+
+const AZURE_COGNITIVE_SERVICES_SCOPE =
+  "https://cognitiveservices.azure.com/.default";
+
+export function setupProvider() {
+  const credential = new DefaultAzureCredential();
+    const azureADTokenProvider = getBearerTokenProvider(
+      credential,
+      AZURE_COGNITIVE_SERVICES_SCOPE,
+    );
+  
+    const azure = {
+      azureADTokenProvider,
+      deployment: process.env.AZURE_DEPLOYMENT_NAME ?? "gpt-35-turbo",
+    };
+  
+    // configure LLM model
+    Settings.llm = new OpenAI({
+      azure,
+    }) as any;
+  
+    // configure embedding model
+    azure.deployment = process.env.EMBEDDING_MODEL as string;
+    Settings.embedModel = new OpenAIEmbedding({
+      azure,
+      model: process.env.EMBEDDING_MODEL,
+      dimensions: process.env.EMBEDDING_DIM
+        ? parseInt(process.env.EMBEDDING_DIM)
+        : undefined,
+    });
+}

app/api/chat/engine/queryFilter.ts

@@ -0,0 +1,25 @@
+import { MetadataFilter, MetadataFilters } from "llamaindex";
+
+export function generateFilters(documentIds: string[]): MetadataFilters {
+  // filter all documents have the private metadata key set to true
+  const publicDocumentsFilter: MetadataFilter = {
+    key: "private",
+    value: "true",
+    operator: "!=",
+  };
+
+  // if no documentIds are provided, only retrieve information from public documents
+  if (!documentIds.length) return { filters: [publicDocumentsFilter] };
+
+  const privateDocumentsFilter: MetadataFilter = {
+    key: "doc_id",
+    value: documentIds,
+    operator: "in",
+  };
+
+  // if documentIds are provided, retrieve information from public and private documents
+  return {
+    filters: [publicDocumentsFilter, privateDocumentsFilter],
+    condition: "or",
+  };
+}

app/api/chat/engine/settings.ts

@@ -1,106 +1,18 @@
-import { OpenAI, OpenAIEmbedding, Settings } from "llamaindex";
-
-import {
-  DefaultAzureCredential,
-  getBearerTokenProvider,
-} from "@azure/identity";
-import { OllamaEmbedding } from "llamaindex/embeddings/OllamaEmbedding";
-import { Ollama } from "llamaindex/llm/ollama";
+import { Settings } from "llamaindex";
+import { setupProvider } from "./provider";
 
 const CHUNK_SIZE = 512;
 const CHUNK_OVERLAP = 20;
-const AZURE_COGNITIVE_SERVICES_SCOPE =
-  "https://cognitiveservices.azure.com/.default";
 
 export const initSettings = async () => {
   console.log(`Using '${process.env.MODEL_PROVIDER}' model provider`);
 
-  // if provider is OpenAI, MODEL must be set
-  if (process.env.MODEL_PROVIDER === 'openai' && process.env.OPENAI_API_TYPE !== 'AzureOpenAI' && !process.env.MODEL) {
-    throw new Error("'MODEL' env variable must be set.");
-  }
-
-  // if provider is Azure OpenAI, AZURE_DEPLOYMENT_NAME must be set
-  if (process.env.MODEL_PROVIDER === 'openai' && process.env.OPENAI_API_TYPE === 'AzureOpenAI' && !process.env.AZURE_DEPLOYMENT_NAME) {
-    throw new Error("'AZURE_DEPLOYMENT_NAME' env variables must be set.");
+  if (!process.env.MODEL || !process.env.EMBEDDING_MODEL) {
+    throw new Error("'MODEL' and 'EMBEDDING_MODEL' env variables must be set.");
   }
 
-  if (!process.env.EMBEDDING_MODEL) {
-    throw new Error("'EMBEDDING_MODEL' env variable must be set.");
-  }
-
-  switch (process.env.MODEL_PROVIDER) {
-    case "ollama":
-      initOllama();
-      break;
-    case "openai":
-      if (process.env.OPENAI_API_TYPE === "AzureOpenAI") {
-        await initAzureOpenAI();
-      } else {
-        initOpenAI();
-      }
-      break;
-    default:
-      throw new Error(
-        `Model provider '${process.env.MODEL_PROVIDER}' not supported.`,
-      );
-  }
   Settings.chunkSize = CHUNK_SIZE;
   Settings.chunkOverlap = CHUNK_OVERLAP;
-};
-
-function initOpenAI() {
-  Settings.llm = new OpenAI({
-    model: process.env.MODEL ?? "gpt-3.5-turbo",
-    maxTokens: 512,
-  });
-  Settings.embedModel = new OpenAIEmbedding({
-    model: process.env.EMBEDDING_MODEL,
-    dimensions: process.env.EMBEDDING_DIM
-      ? parseInt(process.env.EMBEDDING_DIM)
-      : undefined,
-  });
-}
-
-async function initAzureOpenAI() {
-  const credential = new DefaultAzureCredential();
-  const azureADTokenProvider = getBearerTokenProvider(
-    credential,
-    AZURE_COGNITIVE_SERVICES_SCOPE,
-  );
-
-  const azure = {
-    azureADTokenProvider,
-    deployment: process.env.AZURE_OPENAI_DEPLOYMENT ?? "gpt-35-turbo",
-  };
-
-  // configure LLM model
-  Settings.llm = new OpenAI({
-    azure,
-  }) as any;
-
-  // configure embedding model
-  azure.deployment = process.env.EMBEDDING_MODEL as string;
-  Settings.embedModel = new OpenAIEmbedding({
-    azure,
-    model: process.env.EMBEDDING_MODEL,
-    dimensions: process.env.EMBEDDING_DIM
-      ? parseInt(process.env.EMBEDDING_DIM)
-      : undefined,
-  });
-}
-
-function initOllama() {
-  const config = {
-    host: process.env.OLLAMA_BASE_URL ?? "http://127.0.0.1:11434",
-  };
-  Settings.llm = new Ollama({
-    model: process.env.MODEL ?? "",
-    config,
-  });
-  Settings.embedModel = new OllamaEmbedding({
-    model: process.env.EMBEDDING_MODEL ?? "",
-    config,
-  });
-}
 
+  setupProvider();
+};