refactor: parameterize PINECONE_METRIC and PINECONE_DIMENSIONS. strongly type all properties

Author: lpm0073

Makefile

@@ -3,7 +3,19 @@ SHELL := /bin/bash
 ifneq ("$(wildcard .env)","")
     include .env
 else
-    $(shell echo -e "OPENAI_API_ORGANIZATION=PLEASE-ADD-ME\nOPENAI_API_KEY=PLEASE-ADD-ME\nPINECONE_API_KEY=PLEASE-ADD-ME\nPINECONE_ENVIRONMENT=gcp-starter\nPINECONE_INDEX_NAME=hsr\nPINECONE_VECTORSTORE_TEXT_KEY=lc_id\nOPENAI_CHAT_MODEL_NAME=gpt-3.5-turbo\nOPENAI_PROMPT_MODEL_NAME=text-davinci-003\nOPENAI_CHAT_TEMPERATURE=0.0\nOPENAI_CHAT_MAX_RETRIES=3\nDEBUG_MODE=True\n" >> .env)
+$(shell echo -e "OPENAI_API_ORGANIZATION=PLEASE-ADD-ME\n\
+OPENAI_API_KEY=PLEASE-ADD-ME\n\
+PINECONE_API_KEY=PLEASE-ADD-ME\n\
+PINECONE_ENVIRONMENT=gcp-starter\n\
+PINECONE_INDEX_NAME=hsr\n\
+PINECONE_VECTORSTORE_TEXT_KEY=lc_id\n\
+PINECONE_METRIC=dotproduct\n\
+PINECONE_DIMENSIONS=1536\n\
+OPENAI_CHAT_MODEL_NAME=gpt-3.5-turbo\n\
+OPENAI_PROMPT_MODEL_NAME=text-davinci-003\n\
+OPENAI_CHAT_TEMPERATURE=0.0\n\
+OPENAI_CHAT_MAX_RETRIES=3\n\
+DEBUG_MODE=True\n" >> .env)
 endif
 
 .PHONY: analyze init activate test lint clean

models/const.py

@@ -17,6 +17,8 @@
     PINECONE_ENVIRONMENT = os.environ["PINECONE_ENVIRONMENT"]
     PINECONE_INDEX_NAME = os.environ.get("PINECONE_INDEX_NAME", "hsr")
     PINECONE_VECTORSTORE_TEXT_KEY = os.environ.get("PINECONE_VECTORSTORE_TEXT_KEY", "lc_id")
+    PINECONE_METRIC = os.environ.get("PINECONE_METRIC", "dotproduct")
+    PINECONE_DIMENSIONS = int(os.environ.get("PINECONE_DIMENSIONS", 1536))
     OPENAI_CHAT_MODEL_NAME = os.environ.get("OPENAI_CHAT_MODEL_NAME", "gpt-3.5-turbo")
     OPENAI_PROMPT_MODEL_NAME = os.environ.get("OPENAI_PROMPT_MODEL_NAME", "text-davinci-003")
     OPENAI_CHAT_TEMPERATURE = float(os.environ.get("OPENAI_CHAT_TEMPERATURE", 0.0))
@@ -48,6 +50,8 @@ class Config(metaclass=ReadOnly):
     PINECONE_ENVIRONMENT = PINECONE_ENVIRONMENT
     PINECONE_INDEX_NAME = PINECONE_INDEX_NAME
     PINECONE_VECTORSTORE_TEXT_KEY: str = PINECONE_VECTORSTORE_TEXT_KEY
+    PINECONE_METRIC: str = PINECONE_METRIC
+    PINECONE_DIMENSIONS: int = PINECONE_DIMENSIONS
 
 
 class Credentials(metaclass=ReadOnly):

models/hybrid_search_retreiver.py

@@ -89,7 +89,7 @@ def __init__(self):
 
     # prompting wrapper
     @property
-    def chat(self):
+    def chat(self) -> ChatOpenAI:
         """ChatOpenAI lazy read-only property."""
         if self._chat is None:
             self._chat = ChatOpenAI(
@@ -104,7 +104,7 @@ def chat(self):
 
     # embeddings
     @property
-    def openai_embeddings(self):
+    def openai_embeddings(self) -> OpenAIEmbeddings:
         """OpenAIEmbeddings lazy read-only property."""
         if self._openai_embeddings is None:
             self._openai_embeddings = OpenAIEmbeddings(
@@ -113,14 +113,14 @@ def openai_embeddings(self):
         return self._openai_embeddings
 
     @property
-    def pinecone_index(self):
+    def pinecone_index(self) -> pinecone.Index:
         """pinecone.Index lazy read-only property."""
         if self._pinecone_index is None:
             self._pinecone_index = pinecone.Index(index_name=Config.PINECONE_INDEX_NAME)
         return self._pinecone_index
 
     @property
-    def vector_store(self):
+    def vector_store(self) -> Pinecone:
         """Pinecone lazy read-only property."""
         if self._vector_store is None:
             self._vector_store = Pinecone(
@@ -131,14 +131,14 @@ def vector_store(self):
         return self._vector_store
 
     @property
-    def text_splitter(self):
+    def text_splitter(self) -> TextSplitter:
         """TextSplitter lazy read-only property."""
         if self._text_splitter is None:
             self._text_splitter = TextSplitter()
         return self._text_splitter
 
     @property
-    def bm25_encoder(self):
+    def bm25_encoder(self) -> BM25Encoder:
         """BM25Encoder lazy read-only property."""
         if self._b25_encoder is None:
             self._b25_encoder = BM25Encoder().default()
@@ -193,7 +193,10 @@ def load(self, filepath: str):
         }
         logging.debug("Creating index. This may take a few minutes...")
         pinecone.create_index(
-            Config.PINECONE_INDEX_NAME, dimension=1536, metric="dotproduct", metadata_config=metadata_config
+            Config.PINECONE_INDEX_NAME,
+            dimension=Config.PINECONE_DIMENSIONS,
+            metric=Config.PINECONE_METRIC,
+            metadata_config=metadata_config,
         )
 
         pdf_files = glob.glob(os.path.join(filepath, "*.pdf"))