chore: code, test, debug PineconeIndex

Author: lpm0073

Makefile

@@ -7,7 +7,7 @@ $(shell echo -e "OPENAI_API_ORGANIZATION=PLEASE-ADD-ME\n\
 OPENAI_API_KEY=PLEASE-ADD-ME\n\
 PINECONE_API_KEY=PLEASE-ADD-ME\n\
 PINECONE_ENVIRONMENT=gcp-starter\n\
-PINECONE_INDEX_NAME=hsr\n\
+PINECONE_INDEX_NAME=rag\n\
 PINECONE_VECTORSTORE_TEXT_KEY=lc_id\n\
 PINECONE_METRIC=dotproduct\n\
 PINECONE_DIMENSIONS=1536\n\

README.md

@@ -42,10 +42,9 @@ python3 -m models.examples.training_services "Microsoft certified Azure AI engin
 # example 4 - prompted assistant
 python3 -m models.examples.training_services_oracle "Oracle database administrator"
 
-# example 5 - Load PDF documents
+# example 5 - Retrieval Augmented Generation
+python3 -m models.examples.pinecone_init
 python3 -m models.examples.load "./data/"
-
-# example 6 - Retrieval Augmented Generation
 python3 -m models.examples.rag "What analytics and accounting courses does Wharton offer?"
 ```
 
@@ -67,7 +66,7 @@ DEBUG_MODE=False
 You'll need to manually create an index with the following characteristics
 
 - Environment: gcp-starter
-- Index name: netec-rag
+- Index name: rag
 - Metric: dotproduct
 - Dimensions: 1536
 - Pod Type: starter

models/examples/pinecone_init.py

@@ -0,0 +1,12 @@
+# -*- coding: utf-8 -*-
+"""Sales Support Model (hsr) Retrieval Augmented Generation (RAG)"""
+
+from models.pinecone import PineconeIndex
+
+
+pinecone = PineconeIndex()
+
+if __name__ == "__main__":
+    pinecone.initialize()
+    print("Pinecone index initialized. name: ", pinecone.index_name)
+    print(pinecone.index.describe_index_stats())

models/examples/rag.py

@@ -14,6 +14,6 @@
     parser.add_argument("prompt", type=str, help="A question about the PDF contents")
     args = parser.parse_args()
 
-    human_message = HumanMessage(text=args.prompt)
+    human_message = HumanMessage(content=args.prompt)
     result = hsr.rag(human_message=human_message)
     print(result)

models/hybrid_search_retreiver.py

@@ -38,7 +38,7 @@
 
 # this project
 from models.const import Config, Credentials
-from models.pinecone import PineConeIndex
+from models.pinecone import PineconeIndex
 
 
 ###############################################################################
@@ -52,18 +52,18 @@ class HybridSearchRetriever:
 
     _chat: ChatOpenAI = None
     _b25_encoder: BM25Encoder = None
-    _pinecone: PineConeIndex = None
+    _pinecone: PineconeIndex = None
     _retriever: PineconeHybridSearchRetriever = None
 
     def __init__(self):
         """Constructor"""
         set_llm_cache(InMemoryCache())
 
     @property
-    def pinecone(self) -> PineConeIndex:
-        """PineConeIndex lazy read-only property."""
+    def pinecone(self) -> PineconeIndex:
+        """PineconeIndex lazy read-only property."""
         if self._pinecone is None:
-            self._pinecone = PineConeIndex()
+            self._pinecone = PineconeIndex()
         return self._pinecone
 
     # prompting wrapper
@@ -148,7 +148,8 @@ def rag(self, human_message: Union[str, HumanMessage]):
         # ---------------------------------------------------------------------
         # 1.) Retrieve relevant documents from Pinecone vector database
         # ---------------------------------------------------------------------
-        documents = self.retriever.get_relevant_documents(query=human_message.content)
+        # documents = self.retriever.get_relevant_documents(query=human_message.content)
+        documents = self.pinecone.vector_store.similarity_search(query=human_message.content)
 
         # Extract the text from the documents
         document_texts = [doc.page_content for doc in documents]
@@ -165,14 +166,15 @@ def rag(self, human_message: Union[str, HumanMessage]):
         # finished with hybrid search setup
         # ---------------------------------------------------------------------
 
-        # 2.) get a response from the chat model
-        response = self.cached_chat_request(system_message=system_message, human_message=human_message)
-
         logging.debug("------------------------------------------------------")
         logging.debug("rag() Retrieval Augmented Generation prompt")
         logging.debug("Diagnostic information:")
         logging.debug("  Retrieved %i related documents from Pinecone", len(documents))
         logging.debug("  System messages contains %i words", len(system_message.content.split()))
         logging.debug("  Prompt: %s", system_message.content)
         logging.debug("------------------------------------------------------")
+
+        # 2.) get a response from the chat model
+        response = self.cached_chat_request(system_message=system_message, human_message=human_message)
+
         return response.content

models/pinecone.py

@@ -5,6 +5,7 @@
 import glob
 
 # general purpose imports
+import json
 import logging
 import os
 
@@ -36,7 +37,7 @@ def create_documents(self, texts):
         return documents
 
 
-class PineConeIndex:
+class PineconeIndex:
     """Pinecone helper class."""
 
     _index: pinecone.Index = None
@@ -46,13 +47,42 @@ class PineConeIndex:
     _vector_store: LCPinecone = None
 
     def __init__(self, index_name: str = None):
-        self._index_name = index_name or Config.PINECONE_INDEX_NAME
         self.init()
+        self.index_name = index_name or Config.PINECONE_INDEX_NAME
+
+    @property
+    def index_name(self) -> str:
+        """index name."""
+        return self._index_name
+
+    @index_name.setter
+    def index_name(self, value: str) -> None:
+        """Set index name."""
+        if self._index_name != value:
+            self.init()
+            self._index_name = value
+            self.init_index()
+
+    @property
+    def index(self) -> pinecone.Index:
+        """pinecone.Index lazy read-only property."""
+        if self._index is None:
+            self.init_index()
+            self._index = pinecone.Index(index_name=self.index_name)
+        return self._index
+
+    @property
+    def initialized(self) -> bool:
+        """initialized read-only property."""
+        indexes = pinecone.manage.list_indexes()
+        return self.index_name in indexes
 
     @property
     def vector_store(self) -> LCPinecone:
         """Pinecone lazy read-only property."""
         if self._vector_store is None:
+            if not self.initialized:
+                self.init_index()
             self._vector_store = LCPinecone(
                 index=self.index,
                 embedding=self.openai_embeddings,
@@ -76,41 +106,29 @@ def text_splitter(self) -> TextSplitter:
             self._text_splitter = TextSplitter()
         return self._text_splitter
 
-    @property
-    def index_name(self) -> str:
-        """index name."""
-        return self._index_name
-
-    @index_name.setter
-    def index_name(self, value: str) -> None:
-        """Set index name."""
-        if self._index_name != value:
-            self._index_name = value
-            self.initialize()
-
-    @property
-    def index(self) -> pinecone.Index:
-        """pinecone.Index lazy read-only property."""
-        if self._index is None:
-            try:
-                self._index = pinecone.Index(index_name=self.index_name)
-            except pinecone.exceptions.PineconeException:
-                # index does not exist, so create it.
-                self.create()
-                self._index = pinecone.Index(index_name=self.index_name)
-        return self._index
+    def init_index(self):
+        """Verify that an index named self.index_name exists in Pinecone. If not, create it."""
+        indexes = pinecone.manage.list_indexes()
+        if self.index_name not in indexes:
+            logging.info("Index does not exist.")
+            self.create()
 
     def init(self):
         """Initialize Pinecone."""
         pinecone.init(api_key=Credentials.PINECONE_API_KEY, environment=Config.PINECONE_ENVIRONMENT)
+        self._index = None
+        self._index_name = None
+        self._text_splitter = None
+        self._openai_embeddings = None
+        self._vector_store = None
 
     def delete(self):
         """Delete index."""
-        try:
-            logging.info("Deleting index...")
-            pinecone.delete_index(self.index_name)
-        except pinecone.exceptions.PineconeException:
-            logging.info("Index does not exist. Continuing...")
+        if not self.initialized:
+            logging.info("Index does not exist. Nothing to delete.")
+            return
+        logging.info("Deleting index...")
+        pinecone.delete_index(self.index_name)
 
     def create(self):
         """Create index."""
@@ -121,11 +139,12 @@ def create(self):
         logging.info("Creating index. This may take a few minutes...")
 
         pinecone.create_index(
-            self.index_name,
+            name=self.index_name,
             dimension=Config.PINECONE_DIMENSIONS,
             metric=Config.PINECONE_METRIC,
             metadata_config=metadata_config,
         )
+        logging.info("Index created.")
 
     def initialize(self):
         """Initialize index."""
@@ -152,16 +171,17 @@ def pdf_loader(self, filepath: str):
         for pdf_file in pdf_files:
             i += 1
             j = len(pdf_files)
-            logging.info("Loading PDF %s of %s: %s", i, j, pdf_file)
+            print("Loading PDF %s of %s: %s", i, j, pdf_file)
             loader = PyPDFLoader(file_path=pdf_file)
             docs = loader.load()
             k = 0
             for doc in docs:
                 k += 1
-                logging.info(k * "-", end="\r")
+                print(k * "-", end="\r")
                 documents = self.text_splitter.create_documents([doc.page_content])
                 document_texts = [doc.page_content for doc in documents]
                 embeddings = self.openai_embeddings.embed_documents(document_texts)
                 self.vector_store.add_documents(documents=documents, embeddings=embeddings)
 
-        logging.info("Finished loading PDFs")
+        index_stats_string = json.dumps(self.index.describe_index_stats(), indent=4)
+        print("Finished loading PDFs. \n" + index_stats_string)

models/tests/test_hsr.py

@@ -7,7 +7,7 @@
 from langchain.chat_models import ChatOpenAI
 
 from models.hybrid_search_retreiver import HybridSearchRetriever
-from models.pinecone import PineConeIndex
+from models.pinecone import PineconeIndex
 
 
 class TestSalesSupportModel:
@@ -27,4 +27,4 @@ def test_02_class_aatribute_types(self):
 
         hsr = HybridSearchRetriever()
         assert isinstance(hsr.chat, ChatOpenAI)
-        assert isinstance(hsr.pinecone, PineConeIndex)
+        assert isinstance(hsr.pinecone, PineconeIndex)

models/tests/test_pinecone.py

@@ -10,7 +10,7 @@
 import pytest  # pylint: disable=unused-import
 
 from models.const import Config
-from models.pinecone import PineConeIndex
+from models.pinecone import PineconeIndex
 
 
 class TestPinecone:
@@ -20,13 +20,13 @@ def test_01_can_instantiate(self):
         """Ensure that we instantiate the object."""
         # pylint: disable=broad-except
         try:
-            PineConeIndex()
+            PineconeIndex()
         except Exception as e:
             assert False, f"Pinecone() failed with exception: {e}"
 
     def test_02_init(self):
         """Ensure that we can initialize Pinecone."""
-        pinecone = PineConeIndex()
+        pinecone = PineconeIndex()
         # pylint: disable=broad-except
         try:
             pinecone.init()
@@ -35,12 +35,12 @@ def test_02_init(self):
 
     def test_03_index(self):
         """Test that the index name is correct."""
-        pinecone = PineConeIndex()
+        pinecone = PineconeIndex()
         assert pinecone.index_name == Config.PINECONE_INDEX_NAME
 
     def test_04_initialize(self):
         """Test that the index initializes."""
-        pinecone = PineConeIndex()
+        pinecone = PineconeIndex()
         # pylint: disable=broad-except
         try:
             pinecone.initialize()
@@ -50,7 +50,9 @@ def test_04_initialize(self):
 
     def test_05_delete(self):
         """Test that the index can be deleted."""
-        pinecone = PineConeIndex()
+        pinecone = PineconeIndex()
+        indexes = oem_pinecone.manage.list_indexes()
+        assert pinecone.index_name in indexes
         # pylint: disable=broad-except
         try:
             pinecone.delete()
@@ -59,20 +61,25 @@ def test_05_delete(self):
 
     def test_06_create(self):
         """Test that the index can be created."""
-        pinecone = PineConeIndex()
+        pinecone = PineconeIndex()
+        indexes = oem_pinecone.manage.list_indexes()
+        if pinecone.index_name in indexes:
+            pinecone.delete()
+
         # pylint: disable=broad-except
         try:
             pinecone.create()
         except Exception as e:
             assert False, f"Pinecone.create() failed with exception: {e}"
+        assert isinstance(pinecone.index, oem_pinecone.Index)
         pinecone.delete()
 
     def test_07_load_pdf(self):
         """Test that we can load a PDF document to the index."""
         if not os.path.exists("./data/test_07_load.pdf"):
             pytest.skip("File './data/test_07_load.pdf' does not exist")
 
-        pinecone = PineConeIndex()
+        pinecone = PineconeIndex()
         # pylint: disable=broad-except
         try:
             pinecone.pdf_loader(filepath="./data/test_07_load.pdf")

models/yt.py

@@ -172,7 +172,7 @@ def test_06_embeddings_b(self):
 
         # 7.) pinecone client. minute 11:00
         self.pinecone_search = Pinecone.from_documents(
-            self.texts_splitter_results,
+            documents=self.texts_splitter_results,
             embedding=self.openai_embedding,
             index_name=self.PINECONE_INDEX_NAME,
         )