Merge pull request #17 from lpm0073/next

HybridSearchRetriever refactoring

Author: lpm0073

Makefile

@@ -3,7 +3,19 @@ SHELL := /bin/bash
 ifneq ("$(wildcard .env)","")
     include .env
 else
-    $(shell echo -e "OPENAI_API_ORGANIZATION=PLEASE-ADD-ME\nOPENAI_API_KEY=PLEASE-ADD-ME\nPINECONE_API_KEY=PLEASE-ADD-ME\nPINECONE_ENVIRONMENT=gcp-starter\nPINECONE_INDEX_NAME=hsr\nOPENAI_CHAT_MODEL_NAME=gpt-3.5-turbo\nOPENAI_PROMPT_MODEL_NAME=text-davinci-003\nOPENAI_CHAT_TEMPERATURE=0.0\nOPENAI_CHAT_MAX_RETRIES=3\nDEBUG_MODE=True\n" >> .env)
+$(shell echo -e "OPENAI_API_ORGANIZATION=PLEASE-ADD-ME\n\
+OPENAI_API_KEY=PLEASE-ADD-ME\n\
+PINECONE_API_KEY=PLEASE-ADD-ME\n\
+PINECONE_ENVIRONMENT=gcp-starter\n\
+PINECONE_INDEX_NAME=hsr\n\
+PINECONE_VECTORSTORE_TEXT_KEY=lc_id\n\
+PINECONE_METRIC=dotproduct\n\
+PINECONE_DIMENSIONS=1536\n\
+OPENAI_CHAT_MODEL_NAME=gpt-3.5-turbo\n\
+OPENAI_PROMPT_MODEL_NAME=text-davinci-003\n\
+OPENAI_CHAT_TEMPERATURE=0.0\n\
+OPENAI_CHAT_MAX_RETRIES=3\n\
+DEBUG_MODE=True\n" >> .env)
 endif
 
 .PHONY: analyze init activate test lint clean

models/const.py

@@ -16,6 +16,9 @@
     PINECONE_API_KEY = os.environ["PINECONE_API_KEY"]
     PINECONE_ENVIRONMENT = os.environ["PINECONE_ENVIRONMENT"]
     PINECONE_INDEX_NAME = os.environ.get("PINECONE_INDEX_NAME", "hsr")
+    PINECONE_VECTORSTORE_TEXT_KEY = os.environ.get("PINECONE_VECTORSTORE_TEXT_KEY", "lc_id")
+    PINECONE_METRIC = os.environ.get("PINECONE_METRIC", "dotproduct")
+    PINECONE_DIMENSIONS = int(os.environ.get("PINECONE_DIMENSIONS", 1536))
     OPENAI_CHAT_MODEL_NAME = os.environ.get("OPENAI_CHAT_MODEL_NAME", "gpt-3.5-turbo")
     OPENAI_PROMPT_MODEL_NAME = os.environ.get("OPENAI_PROMPT_MODEL_NAME", "text-davinci-003")
     OPENAI_CHAT_TEMPERATURE = float(os.environ.get("OPENAI_CHAT_TEMPERATURE", 0.0))
@@ -26,7 +29,16 @@
     raise FileNotFoundError("No .env file found in root directory of repository")
 
 
-class Config:
+class ReadOnly(type):
+    """Metaclass to make all class attributes read-only."""
+
+    def __setattr__(cls, name, value):
+        if name in cls.__dict__:
+            raise TypeError(f"Cannot change a read-only attribute {name}")
+        super().__setattr__(name, value)
+
+
+class Config(metaclass=ReadOnly):
     """Configuration parameters."""
 
     DEBUG_MODE: bool = DEBUG_MODE
@@ -35,13 +47,16 @@ class Config:
     OPENAI_CHAT_TEMPERATURE: float = OPENAI_CHAT_TEMPERATURE
     OPENAI_CHAT_MAX_RETRIES: int = OPENAI_CHAT_MAX_RETRIES
     OPENAI_CHAT_CACHE: bool = OPENAI_CHAT_CACHE
+    PINECONE_ENVIRONMENT = PINECONE_ENVIRONMENT
+    PINECONE_INDEX_NAME = PINECONE_INDEX_NAME
+    PINECONE_VECTORSTORE_TEXT_KEY: str = PINECONE_VECTORSTORE_TEXT_KEY
+    PINECONE_METRIC: str = PINECONE_METRIC
+    PINECONE_DIMENSIONS: int = PINECONE_DIMENSIONS
 
 
-class Credentials:
+class Credentials(metaclass=ReadOnly):
     """Credentials."""
 
     OPENAI_API_KEY = OPENAI_API_KEY
     OPENAI_API_ORGANIZATION = OPENAI_API_ORGANIZATION
     PINECONE_API_KEY = PINECONE_API_KEY
-    PINECONE_ENVIRONMENT = PINECONE_ENVIRONMENT
-    PINECONE_INDEX_NAME = PINECONE_INDEX_NAME

models/hybrid_search_retreiver.py

@@ -53,9 +53,6 @@
 ###############################################################################
 # initializations
 ###############################################################################
-DEFAULT_MODEL_NAME = Config.OPENAI_PROMPT_MODEL_NAME
-pinecone.init(api_key=Credentials.PINECONE_API_KEY, environment=Credentials.PINECONE_ENVIRONMENT)
-set_llm_cache(InMemoryCache())
 logging.basicConfig(level=logging.DEBUG if Config.DEBUG_MODE else logging.INFO)
 
 
@@ -78,25 +75,74 @@ def create_documents(self, texts):
 class HybridSearchRetriever:
     """Hybrid Search Retriever (OpenAI + Pinecone)"""
 
+    _chat: ChatOpenAI = None
+    _openai_embeddings: OpenAIEmbeddings = None
+    _pinecone_index: pinecone.Index = None
+    _vector_store: Pinecone = None
+    _text_splitter: TextSplitter = None
+    _b25_encoder: BM25Encoder = None
+
+    def __init__(self):
+        """Constructor"""
+        pinecone.init(api_key=Credentials.PINECONE_API_KEY, environment=Config.PINECONE_ENVIRONMENT)
+        set_llm_cache(InMemoryCache())
+
     # prompting wrapper
-    chat = ChatOpenAI(
-        api_key=Credentials.OPENAI_API_KEY,
-        organization=Credentials.OPENAI_API_ORGANIZATION,
-        cache=Config.OPENAI_CHAT_CACHE,
-        max_retries=Config.OPENAI_CHAT_MAX_RETRIES,
-        model=Config.OPENAI_CHAT_MODEL_NAME,
-        temperature=Config.OPENAI_CHAT_TEMPERATURE,
-    )
+    @property
+    def chat(self) -> ChatOpenAI:
+        """ChatOpenAI lazy read-only property."""
+        if self._chat is None:
+            self._chat = ChatOpenAI(
+                api_key=Credentials.OPENAI_API_KEY,
+                organization=Credentials.OPENAI_API_ORGANIZATION,
+                cache=Config.OPENAI_CHAT_CACHE,
+                max_retries=Config.OPENAI_CHAT_MAX_RETRIES,
+                model=Config.OPENAI_CHAT_MODEL_NAME,
+                temperature=Config.OPENAI_CHAT_TEMPERATURE,
+            )
+        return self._chat
 
     # embeddings
-    openai_embeddings = OpenAIEmbeddings(
-        api_key=Credentials.OPENAI_API_KEY, organization=Credentials.OPENAI_API_ORGANIZATION
-    )
-    pinecone_index = pinecone.Index(index_name=Credentials.PINECONE_INDEX_NAME)
-    vector_store = Pinecone(index=pinecone_index, embedding=openai_embeddings, text_key="lc_id")
-
-    text_splitter = TextSplitter()
-    bm25_encoder = BM25Encoder().default()
+    @property
+    def openai_embeddings(self) -> OpenAIEmbeddings:
+        """OpenAIEmbeddings lazy read-only property."""
+        if self._openai_embeddings is None:
+            self._openai_embeddings = OpenAIEmbeddings(
+                api_key=Credentials.OPENAI_API_KEY, organization=Credentials.OPENAI_API_ORGANIZATION
+            )
+        return self._openai_embeddings
+
+    @property
+    def pinecone_index(self) -> pinecone.Index:
+        """pinecone.Index lazy read-only property."""
+        if self._pinecone_index is None:
+            self._pinecone_index = pinecone.Index(index_name=Config.PINECONE_INDEX_NAME)
+        return self._pinecone_index
+
+    @property
+    def vector_store(self) -> Pinecone:
+        """Pinecone lazy read-only property."""
+        if self._vector_store is None:
+            self._vector_store = Pinecone(
+                index=self.pinecone_index,
+                embedding=self.openai_embeddings,
+                text_key=Config.PINECONE_VECTORSTORE_TEXT_KEY,
+            )
+        return self._vector_store
+
+    @property
+    def text_splitter(self) -> TextSplitter:
+        """TextSplitter lazy read-only property."""
+        if self._text_splitter is None:
+            self._text_splitter = TextSplitter()
+        return self._text_splitter
+
+    @property
+    def bm25_encoder(self) -> BM25Encoder:
+        """BM25Encoder lazy read-only property."""
+        if self._b25_encoder is None:
+            self._b25_encoder = BM25Encoder().default()
+        return self._b25_encoder
 
     def cached_chat_request(
         self, system_message: Union[str, SystemMessage], human_message: Union[str, HumanMessage]
@@ -114,7 +160,9 @@ def cached_chat_request(
         retval = self.chat(messages)
         return retval
 
-    def prompt_with_template(self, prompt: PromptTemplate, concept: str, model: str = DEFAULT_MODEL_NAME) -> str:
+    def prompt_with_template(
+        self, prompt: PromptTemplate, concept: str, model: str = Config.OPENAI_PROMPT_MODEL_NAME
+    ) -> str:
         """Prompt with template."""
         llm = OpenAI(model=model)
         retval = llm(prompt.format(concept=concept))
@@ -135,17 +183,20 @@ def load(self, filepath: str):
         """
         try:
             logging.debug("Deleting index...")
-            pinecone.delete_index(Credentials.PINECONE_INDEX_NAME)
+            pinecone.delete_index(Config.PINECONE_INDEX_NAME)
         except pinecone.exceptions.PineconeException:
             logging.debug("Index does not exist. Continuing...")
 
         metadata_config = {
-            "indexed": ["lc_id", "lc_type"],
+            "indexed": [Config.PINECONE_VECTORSTORE_TEXT_KEY, "lc_type"],
             "context": ["lc_text"],
         }
         logging.debug("Creating index. This may take a few minutes...")
         pinecone.create_index(
-            Credentials.PINECONE_INDEX_NAME, dimension=1536, metric="dotproduct", metadata_config=metadata_config
+            Config.PINECONE_INDEX_NAME,
+            dimension=Config.PINECONE_DIMENSIONS,
+            metric=Config.PINECONE_METRIC,
+            metadata_config=metadata_config,
         )
 
         pdf_files = glob.glob(os.path.join(filepath, "*.pdf"))
@@ -187,11 +238,13 @@ def rag(self, human_message: Union[str, HumanMessage]):
             logging.debug("Converting human_message to HumanMessage")
             human_message = HumanMessage(content=human_message)
 
+        # ---------------------------------------------------------------------
+        # 1.) Retrieve relevant documents from Pinecone vector database
+        # ---------------------------------------------------------------------
         retriever = PineconeHybridSearchRetriever(
             embeddings=self.openai_embeddings, sparse_encoder=self.bm25_encoder, index=self.pinecone_index
         )
         documents = retriever.get_relevant_documents(query=human_message.content)
-        logging.debug("Retrieved %i related documents from Pinecone", len(documents))
 
         # Extract the text from the documents
         document_texts = [doc.page_content for doc in documents]
@@ -202,13 +255,19 @@ def rag(self, human_message: Union[str, HumanMessage]):
             into your responses:\n\n
         """
         )
-        system_message = f"{leader} {'. '.join(document_texts)}"
+        system_message_content = f"{leader} {'. '.join(document_texts)}"
+        system_message = SystemMessage(content=system_message_content)
+        # ---------------------------------------------------------------------
+        # finished with hybrid search setup
+        # ---------------------------------------------------------------------
 
-        logging.debug("System messages contains %i words", len(system_message.split()))
-        logging.debug("Prompt: %s", system_message)
-        system_message = SystemMessage(content=system_message)
+        # 2.) get a response from the chat model
         response = self.cached_chat_request(system_message=system_message, human_message=human_message)
 
+        logging.debug("------------------------------------------------------")
+        logging.debug("Retrieved %i related documents from Pinecone", len(documents))
+        logging.debug("System messages contains %i words", len(system_message.content.split()))
+        logging.debug("Prompt: %s", system_message.content)
         logging.debug("Response:")
         logging.debug("------------------------------------------------------")
         return response.content

models/tests/test_pinecone.py

@@ -9,7 +9,7 @@
 from langchain.embeddings import OpenAIEmbeddings
 from langchain.vectorstores.pinecone import Pinecone
 
-from ..const import Credentials
+from ..const import Config, Credentials
 
 
 class TestPinecone:
@@ -19,22 +19,20 @@ def test_01_test_pinecone_connectivity(self):
         """Ensure that we have connectivity to Pinecone."""
         # pylint: disable=broad-except
         try:
-            pinecone.init(api_key=Credentials.PINECONE_API_KEY, environment=Credentials.PINECONE_ENVIRONMENT)
+            pinecone.init(api_key=Credentials.PINECONE_API_KEY, environment=Config.PINECONE_ENVIRONMENT)
         except Exception as e:
             assert False, f"pinecone.init() failed with exception: {e}"
 
     def test_02_test_pinecone_index(self):
         """Ensure that the Pinecone index exists and that we can connect to it."""
-        pinecone.init(api_key=Credentials.PINECONE_API_KEY, environment=Credentials.PINECONE_ENVIRONMENT)
+        pinecone.init(api_key=Credentials.PINECONE_API_KEY, environment=Config.PINECONE_ENVIRONMENT)
         openai_embedding = OpenAIEmbeddings()
 
         # pylint: disable=broad-except
         try:
             Pinecone.from_existing_index(
-                Credentials.PINECONE_INDEX_NAME,
+                Config.PINECONE_INDEX_NAME,
                 embedding=openai_embedding,
             )
         except Exception as e:
-            assert (
-                False
-            ), f"Pinecone initialization of index {Credentials.PINECONE_INDEX_NAME,} failed with exception: {e}"
+            assert False, f"Pinecone initialization of index {Config.PINECONE_INDEX_NAME,} failed with exception: {e}"

models/yt.py

@@ -37,6 +37,8 @@
 # 8.) LangChain agents
 from langchain_experimental.agents.agent_toolkits.python.base import create_python_agent
 
+from models.const import Config, Credentials
+
 
 # Load environment variables from .env file in all folders
 # pylint: disable=duplicate-code
@@ -45,8 +47,6 @@
     load_dotenv(dotenv_path=dotenv_path, verbose=True)
     OPENAI_API_KEY = os.environ["OPENAI_API_KEY"]
     OPENAI_API_ORGANIZATION = os.environ["OPENAI_API_ORGANIZATION"]
-    PINECONE_API_KEY = os.environ["PINECONE_API_KEY"]
-    PINECONE_ENVIRONMENT = os.environ["PINECONE_ENVIRONMENT"]
 else:
     raise FileNotFoundError("No .env file found in root directory of repository")
 
@@ -66,7 +66,7 @@ class LangChainDev:
         tool=PythonREPL(),
         verbose=True,
     )
-    pinecone.init(api_key=PINECONE_API_KEY, environment=PINECONE_ENVIRONMENT)  # minute 10:43
+    pinecone.init(api_key=Credentials.PINECONE_API_KEY, environment=Config.PINECONE_ENVIRONMENT)  # minute 10:43
 
     # LLM wrappers. minute 5:46
     def test_01_basic(self):