Add MistralAI text vectorizer (#169)

As requested in [issue
160](#160)

Author: justin-cechmanek

.github/workflows/run_tests.yml

@@ -61,6 +61,7 @@ jobs:
         GCP_LOCATION: ${{ secrets.GCP_LOCATION }}
         GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
         COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}
+        MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }}
         AZURE_OPENAI_API_KEY: ${{secrets.AZURE_OPENAI_API_KEY}}
         AZURE_OPENAI_ENDPOINT: ${{secrets.AZURE_OPENAI_ENDPOINT}}
         AZURE_OPENAI_DEPLOYMENT_NAME: ${{secrets.AZURE_OPENAI_DEPLOYMENT_NAME}}
@@ -80,6 +81,7 @@ jobs:
         GCP_LOCATION: ${{ secrets.GCP_LOCATION }}
         GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
         COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}
+        MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }}
         AZURE_OPENAI_API_KEY: ${{secrets.AZURE_OPENAI_API_KEY}}
         AZURE_OPENAI_ENDPOINT: ${{secrets.AZURE_OPENAI_ENDPOINT}}
         AZURE_OPENAI_DEPLOYMENT_NAME: ${{secrets.AZURE_OPENAI_DEPLOYMENT_NAME}}
@@ -92,4 +94,4 @@ jobs:
       with:
         token: ${{ secrets.CODECOV_TOKEN }}
         files: ./coverage.xml
-        fail_ci_if_error: false
+        fail_ci_if_error: false

README.md

@@ -213,6 +213,7 @@ Commands:
 ### ⚡  Community Integrations
 Integrate with popular embedding models and providers to greatly simplify the process of vectorizing unstructured data for your index and queries:
 - [Cohere](https://www.redisvl.com/api/vectorizer/html#coheretextvectorizer)
+- [Mistral](https://www.redisvl.com/api/vectorizer/html#mistralaitextvectorizer)
 - [OpenAI](https://www.redisvl.com/api/vectorizer.html#openaitextvectorizer)
 - [HuggingFace](https://www.redisvl.com/api/vectorizer.html#hftextvectorizer)
 - [GCP VertexAI](https://www.redisvl.com/api/vectorizer.html#vertexaitextvectorizer)

conftest.py

@@ -59,6 +59,10 @@ def azure_endpoint():
 def cohere_key():
     return os.getenv("COHERE_API_KEY")
 
+@pytest.fixture
+def mistral_key():
+    return os.getenv("MISTRAL_API_KEY")
+
 @pytest.fixture
 def gcp_location():
     return os.getenv("GCP_LOCATION")
@@ -133,4 +137,4 @@ def sample_data():
 def clear_db(redis):
     redis.flushall()
     yield
-    redis.flushall()
+    redis.flushall()

docs/user_guide/vectorizers_04.ipynb

@@ -12,6 +12,7 @@
     "2. HuggingFace\n",
     "3. Vertex AI\n",
     "4. Cohere\n",
+    "5. Mistral AI\n",
     "\n",
     "Before running this notebook, be sure to\n",
     "1. Have installed ``redisvl`` and have that environment active for this notebook.\n",
@@ -500,6 +501,44 @@
     "Learn more about using RedisVL and Cohere together through [this dedicated user guide](https://docs.cohere.com/docs/redis-and-cohere)."
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Mistral AI\n",
+    "\n",
+    "[Mistral](https://console.mistral.ai/) offers LLM and embedding APIs you to implement into your product. The `MistralAITextVectorizer` makes it simple to use RedisVL with their embeddings model. You will need to install `mistralai`.\n",
+    "\n",
+    "```bash\n",
+    "pip install mistralai\n",
+    "```"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "vector dimensions: 1024\n",
+      "[-0.02801513671875, 0.02532958984375, 0.04278564453125, 0.0185699462890625, 0.041015625, 0.006053924560546875, 0.03607177734375, -0.0030155181884765625, 0.0033893585205078125, -0.01390838623046875]\n"
+     ]
+    }
+   ],
+   "source": [
+    "from redisvl.utils.vectorize import MistralAITextVectorizer\n",
+    "\n",
+    "mistral = MistralAITextVectorizer()\n",
+    "\n",
+    "# embed a sentence using their asyncronous method\n",
+    "test = await mistral.aembed(\"This is a test sentence.\")\n",
+    "print(\"vector dimensions:\", len(test))\n",
+    "print(test[:10])"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -658,7 +697,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.5"
+   "version": "3.12.2"
   },
   "orig_nbformat": 4,
   "vscode": {

poetry.lock

@@ -30,12 +30,14 @@ openai = { version = ">=1.13.0", optional = true }
 sentence-transformers = { version = ">=2.2.2", optional = true }
 google-cloud-aiplatform = { version = ">=1.26", optional = true }
 cohere = { version = ">=4.44", optional = true }
+mistralai = { version = ">=0.2.0", optional = true }
 
 [tool.poetry.extras]
 openai = ["openai"]
 sentence-transformers = ["sentence-transformers"]
 google_cloud_aiplatform = ["google_cloud_aiplatform"]
 cohere = ["cohere"]
+mistralai = ["mistralai"]
 
 [tool.poetry.group.dev.dependencies]
 black = ">=20.8b1"
@@ -116,4 +118,4 @@ directory = "htmlcov"
 
 [tool.mypy]
 warn_unused_configs = true
-ignore_missing_imports = true
+ignore_missing_imports = true

pyproject.toml

@@ -2,6 +2,7 @@
 from redisvl.utils.vectorize.text.azureopenai import AzureOpenAITextVectorizer
 from redisvl.utils.vectorize.text.cohere import CohereTextVectorizer
 from redisvl.utils.vectorize.text.huggingface import HFTextVectorizer
+from redisvl.utils.vectorize.text.mistral import MistralAITextVectorizer
 from redisvl.utils.vectorize.text.openai import OpenAITextVectorizer
 from redisvl.utils.vectorize.text.vertexai import VertexAITextVectorizer
 
@@ -12,4 +13,5 @@
     "OpenAITextVectorizer",
     "VertexAITextVectorizer",
     "AzureOpenAITextVectorizer",
+    "MistralAITextVectorizer",
 ]