From cecd9152f73d04278d44064c2d77a0f179ab84a1 Mon Sep 17 00:00:00 2001 From: "redisdocsapp[bot]" <177626021+redisdocsapp[bot]@users.noreply.github.com> Date: Sat, 8 Nov 2025 00:22:44 +0000 Subject: [PATCH] Update for redisvl 0.11.0 --- content/develop/ai/redisvl/api/_index.md | 5 +- content/develop/ai/redisvl/api/query.md | 81 +- content/develop/ai/redisvl/api/schema.md | 1135 ++++++++++++++++- content/develop/ai/redisvl/api/vector.md | 7 + .../develop/ai/redisvl/user_guide/_index.md | 45 +- .../ai/redisvl/user_guide/advanced_queries.md | 729 +++++++++++ .../ai/redisvl/user_guide/hash_vs_json.md | 3 +- .../ai/redisvl/user_guide/svs_vamana.md | 554 ++++++++ 8 files changed, 2478 insertions(+), 81 deletions(-) create mode 100644 content/develop/ai/redisvl/user_guide/advanced_queries.md create mode 100644 content/develop/ai/redisvl/user_guide/svs_vamana.md diff --git a/content/develop/ai/redisvl/api/_index.md b/content/develop/ai/redisvl/api/_index.md index c4982e0635..7b8e8250ac 100644 --- a/content/develop/ai/redisvl/api/_index.md +++ b/content/develop/ai/redisvl/api/_index.md @@ -15,7 +15,10 @@ Reference documentation for the RedisVL API. * [Schema](schema/) * [IndexSchema](schema/#indexschema) * [Defining Fields](schema/#defining-fields) - * [Supported Field Types and Attributes](schema/#supported-field-types-and-attributes) + * [Basic Field Types](schema/#basic-field-types) + * [Vector Field Types](schema/#vector-field-types) + * [SVS-VAMANA Configuration Utilities](schema/#svs-vamana-configuration-utilities) + * [Vector Algorithm Comparison](schema/#vector-algorithm-comparison) * [Search Index Classes](searchindex/) * [SearchIndex](searchindex/#searchindex) * [AsyncSearchIndex](searchindex/#asyncsearchindex) diff --git a/content/develop/ai/redisvl/api/query.md b/content/develop/ai/redisvl/api/query.md index adc3b8fa04..412435209f 100644 --- a/content/develop/ai/redisvl/api/query.md +++ b/content/develop/ai/redisvl/api/query.md @@ -730,39 +730,18 @@ Return self as the query object. ## HybridQuery -### `class HybridQuery(text, text_field_name, vector, vector_field_name, text_scorer='BM25STD', filter_expression=None, alpha=0.7, dtype='float32', num_results=10, return_fields=None, stopwords='english', dialect=2)` +### `class HybridQuery(*args, **kwargs)` -Bases: `AggregationQuery` - -HybridQuery combines text and vector search in Redis. -It allows you to perform a hybrid search using both text and vector similarity. -It scores documents based on a weighted combination of text and vector similarity. - -```python -from redisvl.query import HybridQuery -from redisvl.index import SearchIndex +Bases: `AggregateHybridQuery` -index = SearchIndex.from_yaml("path/to/index.yaml") +Backward compatibility wrapper for AggregateHybridQuery. -query = HybridQuery( - text="example text", - text_field_name="text_field", - vector=[0.1, 0.2, 0.3], - vector_field_name="vector_field", - text_scorer="BM25STD", - filter_expression=None, - alpha=0.7, - dtype="float32", - num_results=10, - return_fields=["field1", "field2"], - stopwords="english", - dialect=2, -) - -results = index.query(query) -``` +#### `Deprecated` +Deprecated since version HybridQuery: is a backward compatibility wrapper around AggregateHybridQuery +and will eventually be replaced with a new hybrid query implementation. +To maintain current functionality please use AggregateHybridQuery directly.", -Instantiates a HybridQuery object. +Instantiates a AggregateHybridQuery object. * **Parameters:** * **text** (*str*) – The text to search for. @@ -785,6 +764,9 @@ Instantiates a HybridQuery object. set, or tuple of strings is provided then those will be used as stopwords. Defaults to "english". if set to "None" then no stopwords will be removed. * **dialect** (*int* *,* *optional*) – The Redis dialect version. Defaults to 2. + * **text_weights** (*Optional* *[* *Dict* *[* *str* *,* *float* *]* *]*) – The importance weighting of individual words + within the query text. Defaults to None, as no modifications will be made to the + text_scorer score. * **Raises:** * **ValueError** – If the text string is empty, or if the text string becomes empty after stopwords are removed. @@ -922,6 +904,14 @@ Default is TFIDF. * **Return type:** *AggregateRequest* +#### `set_text_weights(weights)` + +Set or update the text weights for the query. + +* **Parameters:** + * **text_weights** – Dictionary of word:weight mappings + * **weights** (*Dict* *[* *str* *,* *float* *]*) + #### `sort_by(*fields, **kwargs)` Indicate how the results should be sorted. This can also be used for @@ -975,9 +965,18 @@ Return the stopwords used in the query. :returns: The stopwords used in the query. :rtype: Set[str] +#### `property text_weights: Dict[str, float]` + +Get the text weights. + +* **Returns:** + weight mappings. +* **Return type:** + Dictionary of word + ## TextQuery -### `class TextQuery(text, text_field_name, text_scorer='BM25STD', filter_expression=None, return_fields=None, num_results=10, return_score=True, dialect=2, sort_by=None, in_order=False, params=None, stopwords='english')` +### `class TextQuery(text, text_field_name, text_scorer='BM25STD', filter_expression=None, return_fields=None, num_results=10, return_score=True, dialect=2, sort_by=None, in_order=False, params=None, stopwords='english', text_weights=None)` Bases: `BaseQuery` @@ -1038,6 +1037,9 @@ A query for running a full text search, along with an optional filter expression a default set of stopwords for that language will be used. Users may specify their own stop words by providing a List or Set of words. if set to None, then no words will be removed. Defaults to ‘english’. + * **text_weights** (*Optional* *[* *Dict* *[* *str* *,* *float* *]* *]*) – The importance weighting of individual words + within the query text. Defaults to None, as no modifications will be made to the + text_scorer score. * **Raises:** * **ValueError** – if stopwords language string cannot be loaded. * **TypeError** – If stopwords is not a valid iterable set of strings. @@ -1184,6 +1186,14 @@ Set the filter expression for the query. * **Raises:** **TypeError** – If filter_expression is not a valid FilterExpression or string. +#### `set_text_weights(weights)` + +Set or update the text weights for the query. + +* **Parameters:** + * **text_weights** – Dictionary of word:weight mappings + * **weights** (*Dict* *[* *str* *,* *float* *]*) + #### `slop(slop)` Allow a maximum of N intervening non matched terms between @@ -1289,6 +1299,15 @@ Get the text field name(s) - for backward compatibility. Either a single field name string (if only one field with weight 1.0) or a dictionary of field:weight mappings. +#### `property text_weights: Dict[str, float]` + +Get the text weights. + +* **Returns:** + weight mappings. +* **Return type:** + Dictionary of word + ## FilterQuery ### `class FilterQuery(filter_expression=None, return_fields=None, num_results=10, dialect=2, sort_by=None, in_order=False, params=None)` @@ -1797,7 +1816,7 @@ Return self as the query object. Bases: `AggregationQuery` -MultiVectorQuery allows for search over multiple vector fields in a document simulateously. +MultiVectorQuery allows for search over multiple vector fields in a document simultaneously. The final score will be a weighted combination of the individual vector similarity scores following the formula: diff --git a/content/develop/ai/redisvl/api/schema.md b/content/develop/ai/redisvl/api/schema.md index b8216eeb55..a7b811dfae 100644 --- a/content/develop/ai/redisvl/api/schema.md +++ b/content/develop/ai/redisvl/api/schema.md @@ -306,52 +306,1117 @@ Fields in the schema can be defined in YAML format or as a Python dictionary, sp } ``` -## Supported Field Types and Attributes +## Basic Field Types -Each field type supports specific attributes that customize its behavior. Below are the field types and their available attributes: +RedisVL supports several basic field types for indexing different kinds of data. Each field type has specific attributes that customize its indexing and search behavior. -**Text Field Attributes**: +### `Text Fields` -- weight: Importance of the field in result calculation. -- no_stem: Disables stemming during indexing. -- withsuffixtrie: Optimizes queries by maintaining a suffix trie. -- phonetic_matcher: Enables phonetic matching. -- sortable: Allows sorting on this field. -- no_index: When True, field is not indexed but can be returned in results (requires sortable=True). -- unf: Un-normalized form. When True, preserves original case for sorting (requires sortable=True). +Text fields support full-text search with stemming, phonetic matching, and other text analysis features. -**Tag Field Attributes**: +### `class TextField(*, name, type=FieldTypes.TEXT, path=None, attrs=)` -- separator: Character for splitting text into individual tags. -- case_sensitive: Case sensitivity in tag matching. -- withsuffixtrie: Suffix trie optimization for queries. -- sortable: Enables sorting based on the tag field. -- no_index: When True, field is not indexed but can be returned in results (requires sortable=True). +Bases: `BaseField` -**Numeric Field Attributes**: +Text field supporting a full text search index -- sortable: Enables sorting on the numeric field. -- no_index: When True, field is not indexed but can be returned in results (requires sortable=True). -- unf: Un-normalized form. When True, maintains original numeric representation for sorting (requires sortable=True). +Create a new model by parsing and validating input data from keyword arguments. + +Raises [ValidationError][pydantic_core.ValidationError] if the input data cannot be +validated to form a valid model. + +self is explicitly positional-only to allow self as a field name. + +* **Parameters:** + * **name** (*str*) + * **type** (*Literal* *[* *FieldTypes.TEXT* *]*) + * **path** (*str* *|* *None*) + * **attrs** ([TextFieldAttributes](#textfieldattributes)) + +#### `as_redis_field()` + +Convert schema field to Redis Field object + +* **Return type:** + *Field* + +#### `attrs: `[TextFieldAttributes](#textfieldattributes) + +Specified field attributes + +#### `model_config: ClassVar[ConfigDict] = {}` + +Configuration for the model, should be a dictionary conforming to [ConfigDict][pydantic.config.ConfigDict]. + +#### `type: Literal[FieldTypes.TEXT]` + +Field type + +### `class TextFieldAttributes(*, sortable=False, index_missing=False, no_index=False, weight=1, no_stem=False, withsuffixtrie=False, phonetic_matcher=None, index_empty=False, unf=False)` + +Full text field attributes + +Create a new model by parsing and validating input data from keyword arguments. + +Raises [ValidationError][pydantic_core.ValidationError] if the input data cannot be +validated to form a valid model. + +self is explicitly positional-only to allow self as a field name. + +* **Parameters:** + * **sortable** (*bool*) + * **index_missing** (*bool*) + * **no_index** (*bool*) + * **weight** (*float*) + * **no_stem** (*bool*) + * **withsuffixtrie** (*bool*) + * **phonetic_matcher** (*str* *|* *None*) + * **index_empty** (*bool*) + * **unf** (*bool*) + +#### `index_empty: bool` + +Allow indexing and searching for empty strings + +#### `model_config: ClassVar[ConfigDict] = {}` + +Configuration for the model, should be a dictionary conforming to [ConfigDict][pydantic.config.ConfigDict]. + +#### `no_stem: bool` + +Disable stemming on the text field during indexing + +#### `phonetic_matcher: str | None` + +Used to perform phonetic matching during search + +#### `unf: bool` + +Un-normalized form - disable normalization on sortable fields (only applies when sortable=True) + +#### `weight: float` + +Declares the importance of this field when calculating results + +#### `withsuffixtrie: bool` + +Keep a suffix trie with all terms which match the suffix to optimize certain queries + +### `Tag Fields` + +Tag fields are optimized for exact-match filtering and faceted search on categorical data. + +### `class TagField(*, name, type=FieldTypes.TAG, path=None, attrs=)` + +Bases: `BaseField` + +Tag field for simple boolean-style filtering + +Create a new model by parsing and validating input data from keyword arguments. + +Raises [ValidationError][pydantic_core.ValidationError] if the input data cannot be +validated to form a valid model. + +self is explicitly positional-only to allow self as a field name. + +* **Parameters:** + * **name** (*str*) + * **type** (*Literal* *[* *FieldTypes.TAG* *]*) + * **path** (*str* *|* *None*) + * **attrs** ([TagFieldAttributes](#tagfieldattributes)) + +#### `as_redis_field()` + +Convert schema field to Redis Field object + +* **Return type:** + *Field* + +#### `attrs: `[TagFieldAttributes](#tagfieldattributes) + +Specified field attributes + +#### `model_config: ClassVar[ConfigDict] = {}` + +Configuration for the model, should be a dictionary conforming to [ConfigDict][pydantic.config.ConfigDict]. + +#### `type: Literal[FieldTypes.TAG]` + +Field type + +### `class TagFieldAttributes(*, sortable=False, index_missing=False, no_index=False, separator=',', case_sensitive=False, withsuffixtrie=False, index_empty=False)` + +Tag field attributes + +Create a new model by parsing and validating input data from keyword arguments. + +Raises [ValidationError][pydantic_core.ValidationError] if the input data cannot be +validated to form a valid model. + +self is explicitly positional-only to allow self as a field name. + +* **Parameters:** + * **sortable** (*bool*) + * **index_missing** (*bool*) + * **no_index** (*bool*) + * **separator** (*str*) + * **case_sensitive** (*bool*) + * **withsuffixtrie** (*bool*) + * **index_empty** (*bool*) + +#### `case_sensitive: bool` + +Treat text as case sensitive or not. By default, tag characters are converted to lowercase + +#### `index_empty: bool` + +Allow indexing and searching for empty strings + +#### `model_config: ClassVar[ConfigDict] = {}` + +Configuration for the model, should be a dictionary conforming to [ConfigDict][pydantic.config.ConfigDict]. + +#### `separator: str` + +Indicates how the text in the original attribute is split into individual tags + +#### `withsuffixtrie: bool` + +Keep a suffix trie with all terms which match the suffix to optimize certain queries + +### `Numeric Fields` + +Numeric fields support range queries and sorting on numeric data. + +### `class NumericField(*, name, type=FieldTypes.NUMERIC, path=None, attrs=)` + +Bases: `BaseField` + +Numeric field for numeric range filtering + +Create a new model by parsing and validating input data from keyword arguments. + +Raises [ValidationError][pydantic_core.ValidationError] if the input data cannot be +validated to form a valid model. + +self is explicitly positional-only to allow self as a field name. + +* **Parameters:** + * **name** (*str*) + * **type** (*Literal* *[* *FieldTypes.NUMERIC* *]*) + * **path** (*str* *|* *None*) + * **attrs** ([NumericFieldAttributes](#numericfieldattributes)) + +#### `as_redis_field()` + +Convert schema field to Redis Field object + +* **Return type:** + *Field* + +#### `attrs: `[NumericFieldAttributes](#numericfieldattributes) + +Specified field attributes + +#### `model_config: ClassVar[ConfigDict] = {}` + +Configuration for the model, should be a dictionary conforming to [ConfigDict][pydantic.config.ConfigDict]. + +#### `type: Literal[FieldTypes.NUMERIC]` + +Field type + +### `class NumericFieldAttributes(*, sortable=False, index_missing=False, no_index=False, unf=False)` + +Numeric field attributes + +Create a new model by parsing and validating input data from keyword arguments. + +Raises [ValidationError][pydantic_core.ValidationError] if the input data cannot be +validated to form a valid model. + +self is explicitly positional-only to allow self as a field name. + +* **Parameters:** + * **sortable** (*bool*) + * **index_missing** (*bool*) + * **no_index** (*bool*) + * **unf** (*bool*) + +#### `model_config: ClassVar[ConfigDict] = {}` + +Configuration for the model, should be a dictionary conforming to [ConfigDict][pydantic.config.ConfigDict]. + +#### `unf: bool` + +Un-normalized form - disable normalization on sortable fields (only applies when sortable=True) + +### `Geo Fields` + +Geo fields enable location-based search with geographic coordinates. + +### `class GeoField(*, name, type=FieldTypes.GEO, path=None, attrs=)` + +Bases: `BaseField` + +Geo field with a geo-spatial index for location based search + +Create a new model by parsing and validating input data from keyword arguments. + +Raises [ValidationError][pydantic_core.ValidationError] if the input data cannot be +validated to form a valid model. + +self is explicitly positional-only to allow self as a field name. + +* **Parameters:** + * **name** (*str*) + * **type** (*Literal* *[* *FieldTypes.GEO* *]*) + * **path** (*str* *|* *None*) + * **attrs** ([GeoFieldAttributes](#geofieldattributes)) + +#### `as_redis_field()` + +Convert schema field to Redis Field object + +* **Return type:** + *Field* + +#### `attrs: `[GeoFieldAttributes](#geofieldattributes) + +Specified field attributes + +#### `model_config: ClassVar[ConfigDict] = {}` + +Configuration for the model, should be a dictionary conforming to [ConfigDict][pydantic.config.ConfigDict]. + +#### `type: Literal[FieldTypes.GEO]` + +Field type + +### `class GeoFieldAttributes(*, sortable=False, index_missing=False, no_index=False)` + +Numeric field attributes + +Create a new model by parsing and validating input data from keyword arguments. + +Raises [ValidationError][pydantic_core.ValidationError] if the input data cannot be +validated to form a valid model. + +self is explicitly positional-only to allow self as a field name. + +* **Parameters:** + * **sortable** (*bool*) + * **index_missing** (*bool*) + * **no_index** (*bool*) + +#### `model_config: ClassVar[ConfigDict] = {}` + +Configuration for the model, should be a dictionary conforming to [ConfigDict][pydantic.config.ConfigDict]. + +## Vector Field Types + +Vector fields enable semantic similarity search using various algorithms. All vector fields share common attributes but have algorithm-specific configurations. + +### `Common Vector Attributes` + +All vector field types share these base attributes: + +### `class BaseVectorFieldAttributes(*, dims, algorithm, datatype=VectorDataType.FLOAT32, distance_metric=VectorDistanceMetric.COSINE, initial_cap=None, index_missing=False)` + +Base vector field attributes shared by FLAT, HNSW, and SVS-VAMANA fields + +Create a new model by parsing and validating input data from keyword arguments. + +Raises [ValidationError][pydantic_core.ValidationError] if the input data cannot be +validated to form a valid model. + +self is explicitly positional-only to allow self as a field name. + +* **Parameters:** + * **dims** (*int*) + * **algorithm** (*VectorIndexAlgorithm*) + * **datatype** (*VectorDataType*) + * **distance_metric** (*VectorDistanceMetric*) + * **initial_cap** (*int* *|* *None*) + * **index_missing** (*bool*) + +#### `classmethod uppercase_strings(v)` + +Validate that provided values are cast to uppercase + +#### `algorithm: VectorIndexAlgorithm` + +FLAT, HNSW, or SVS-VAMANA + +* **Type:** + The indexing algorithm for the field + +#### `datatype: VectorDataType` + +The float datatype for the vector embeddings + +#### `dims: int` + +Dimensionality of the vector embeddings field + +#### `distance_metric: VectorDistanceMetric` + +The distance metric used to measure query relevance + +#### `property field_data: Dict[str, Any]` + +Select attributes required by the Redis API + +#### `index_missing: bool` + +Allow indexing and searching for missing values (documents without the field) + +#### `initial_cap: int | None` + +Initial vector capacity in the index affecting memory allocation size of the index + +#### `model_config: ClassVar[ConfigDict] = {}` + +Configuration for the model, should be a dictionary conforming to [ConfigDict][pydantic.config.ConfigDict]. + +**Key Attributes:** + +- dims: Dimensionality of the vector (e.g., 768, 1536). +- algorithm: Indexing algorithm for vector search: + - flat: Brute-force exact search. 100% recall, slower for large datasets. Best for <10K vectors. + - hnsw: Graph-based approximate search. Fast with high recall (95-99%). Best for general use. + - svs-vamana: SVS-VAMANA (Scalable Vector Search with VAMANA graph algorithm) provides fast approximate nearest neighbor search with optional compression support. This algorithm is optimized for Intel hardware and offers reduced memory usage through vector compression. + + #### NOTE + For detailed algorithm comparison and selection guidance, see [Vector Algorithm Comparison](#vector-algorithm-comparison). +- datatype: Float precision (bfloat16, float16, float32, float64). Note: SVS-VAMANA only supports float16 and float32. +- distance_metric: Similarity metric (COSINE, L2, IP). +- initial_cap: Initial capacity hint for memory allocation (optional). +- index_missing: When True, allows searching for documents missing this field (optional). + +### `HNSW Vector Fields` + +HNSW (Hierarchical Navigable Small World) - Graph-based approximate search with excellent recall. **Best for general-purpose vector search (10K-1M+ vectors).** + +### `When to use HNSW & Performance Details` + +**Use HNSW when:** + +- Medium to large datasets (100K-1M+ vectors) requiring high recall rates +- Search accuracy is more important than memory usage +- Need general-purpose vector search with balanced performance +- Cross-platform deployments where hardware-specific optimizations aren’t available + +**Performance characteristics:** + +- **Search speed**: Very fast approximate search with tunable accuracy +- **Memory usage**: Higher than compressed SVS-VAMANA but reasonable for most applications +- **Recall quality**: Excellent recall rates (95-99%), often better than other approximate methods +- **Build time**: Moderate construction time, faster than SVS-VAMANA for smaller datasets + +### `class HNSWVectorField(*, name, type='vector', path=None, attrs)` + +Bases: `BaseField` + +Vector field with HNSW (Hierarchical Navigable Small World) indexing for approximate nearest neighbor search. + +Create a new model by parsing and validating input data from keyword arguments. + +Raises [ValidationError][pydantic_core.ValidationError] if the input data cannot be +validated to form a valid model. + +self is explicitly positional-only to allow self as a field name. + +* **Parameters:** + * **name** (*str*) + * **type** (*Literal* *[* *'vector'* *]*) + * **path** (*str* *|* *None*) + * **attrs** ([HNSWVectorFieldAttributes](#hnswvectorfieldattributes)) + +#### `as_redis_field()` + +Convert schema field to Redis Field object + +* **Return type:** + *Field* + +#### `attrs: `[HNSWVectorFieldAttributes](#hnswvectorfieldattributes) + +Specified field attributes + +#### `model_config: ClassVar[ConfigDict] = {}` + +Configuration for the model, should be a dictionary conforming to [ConfigDict][pydantic.config.ConfigDict]. + +#### `type: Literal['vector']` + +Field type + +### `class HNSWVectorFieldAttributes(*, dims, algorithm=VectorIndexAlgorithm.HNSW, datatype=VectorDataType.FLOAT32, distance_metric=VectorDistanceMetric.COSINE, initial_cap=None, index_missing=False, m=16, ef_construction=200, ef_runtime=10, epsilon=0.01)` + +HNSW vector field attributes for approximate nearest neighbor search. + +Create a new model by parsing and validating input data from keyword arguments. + +Raises [ValidationError][pydantic_core.ValidationError] if the input data cannot be +validated to form a valid model. + +self is explicitly positional-only to allow self as a field name. + +* **Parameters:** + * **dims** (*int*) + * **algorithm** (*Literal* *[* *VectorIndexAlgorithm.HNSW* *]*) + * **datatype** (*VectorDataType*) + * **distance_metric** (*VectorDistanceMetric*) + * **initial_cap** (*int* *|* *None*) + * **index_missing** (*bool*) + * **m** (*int*) + * **ef_construction** (*int*) + * **ef_runtime** (*int*) + * **epsilon** (*float*) + +#### `algorithm: Literal[VectorIndexAlgorithm.HNSW]` + +The indexing algorithm (fixed as ‘hnsw’) + +#### `ef_construction: int` + +100-800) + +* **Type:** + Max edge candidates during build time (default +* **Type:** + 200, range + +#### `ef_runtime: int` + +1. - primary tuning parameter + +* **Type:** + Max top candidates during search (default + +#### `epsilon: float` + +0.01) + +* **Type:** + Range search boundary factor (default + +#### `m: int` + +8-64) + +* **Type:** + Max outgoing edges per node in each layer (default +* **Type:** + 16, range + +#### `model_config: ClassVar[ConfigDict] = {}` + +Configuration for the model, should be a dictionary conforming to [ConfigDict][pydantic.config.ConfigDict]. + +**HNSW Examples:** + +**Balanced configuration (recommended starting point):** + +```yaml +- name: embedding + type: vector + attrs: + algorithm: hnsw + dims: 768 + distance_metric: cosine + datatype: float32 + # Balanced settings for good recall and performance + m: 16 + ef_construction: 200 + ef_runtime: 10 +``` + +**High-recall configuration:** + +```yaml +- name: embedding + type: vector + attrs: + algorithm: hnsw + dims: 768 + distance_metric: cosine + datatype: float32 + # Tuned for maximum accuracy + m: 32 + ef_construction: 400 + ef_runtime: 50 +``` + +### `SVS-VAMANA Vector Fields` + +SVS-VAMANA (Scalable Vector Search with VAMANA graph algorithm) provides fast approximate nearest neighbor search with optional compression support. This algorithm is optimized for Intel hardware and offers reduced memory usage through vector compression. **Best for large datasets (>100K vectors) on Intel hardware with memory constraints.** + +### `When to use SVS-VAMANA & Detailed Guide` + +**Requirements:** +: - Redis >= 8.2.0 with RediSearch >= 2.8.10 + - datatype must be ‘float16’ or ‘float32’ (float64/bfloat16 not supported) + +**Use SVS-VAMANA when:** +: - Large datasets where memory is expensive + - Cloud deployments with memory-based pricing + - When 90-95% recall is acceptable + - High-dimensional vectors (>1024 dims) with LeanVec compression + +**Performance vs other algorithms:** +: - **vs FLAT**: Much faster search, significantly lower memory usage with compression, but approximate results + - **vs HNSW**: Better memory efficiency with compression, similar or better recall, Intel-optimized + +**Compression selection guide:** + +- **No compression**: Best performance, standard memory usage +- **LVQ4/LVQ8**: Good balance of compression (2x-4x) and performance +- **LeanVec4x8/LeanVec8x8**: Maximum compression (up to 8x) with dimensionality reduction + +**Memory Savings Examples (1M vectors, 768 dims):** +: - No compression (float32): 3.1 GB + - LVQ4x4 compression: 1.6 GB (~48% savings) + - LeanVec4x8 + reduce to 384: 580 MB (~81% savings) + +### `class SVSVectorField(*, name, type=FieldTypes.VECTOR, path=None, attrs)` + +Bases: `BaseField` + +Vector field with SVS-VAMANA indexing and compression for memory-efficient approximate nearest neighbor search. + +Create a new model by parsing and validating input data from keyword arguments. + +Raises [ValidationError][pydantic_core.ValidationError] if the input data cannot be +validated to form a valid model. + +self is explicitly positional-only to allow self as a field name. + +* **Parameters:** + * **name** (*str*) + * **type** (*Literal* *[* *FieldTypes.VECTOR* *]*) + * **path** (*str* *|* *None*) + * **attrs** ([SVSVectorFieldAttributes](#svsvectorfieldattributes)) + +#### `as_redis_field()` + +Convert schema field to Redis Field object + +* **Return type:** + *Field* + +#### `attrs: `[SVSVectorFieldAttributes](#svsvectorfieldattributes) + +Specified field attributes + +#### `model_config: ClassVar[ConfigDict] = {}` + +Configuration for the model, should be a dictionary conforming to [ConfigDict][pydantic.config.ConfigDict]. + +#### `type: Literal[FieldTypes.VECTOR]` + +Field type + +### `class SVSVectorFieldAttributes(*, dims, algorithm=VectorIndexAlgorithm.SVS_VAMANA, datatype=VectorDataType.FLOAT32, distance_metric=VectorDistanceMetric.COSINE, initial_cap=None, index_missing=False, graph_max_degree=40, construction_window_size=250, search_window_size=20, epsilon=0.01, compression=None, reduce=None, training_threshold=None)` + +SVS-VAMANA vector field attributes with compression support. + +Create a new model by parsing and validating input data from keyword arguments. + +Raises [ValidationError][pydantic_core.ValidationError] if the input data cannot be +validated to form a valid model. + +self is explicitly positional-only to allow self as a field name. + +* **Parameters:** + * **dims** (*int*) + * **algorithm** (*Literal* *[* *VectorIndexAlgorithm.SVS_VAMANA* *]*) + * **datatype** (*VectorDataType*) + * **distance_metric** (*VectorDistanceMetric*) + * **initial_cap** (*int* *|* *None*) + * **index_missing** (*bool*) + * **graph_max_degree** (*int*) + * **construction_window_size** (*int*) + * **search_window_size** (*int*) + * **epsilon** (*float*) + * **compression** (*CompressionType* *|* *None*) + * **reduce** (*int* *|* *None*) + * **training_threshold** (*int* *|* *None*) + +#### `validate_svs_params()` + +Validate SVS-VAMANA specific constraints + +#### `algorithm: Literal[VectorIndexAlgorithm.SVS_VAMANA]` + +The indexing algorithm for the vector field + +#### `compression: CompressionType | None` + +LVQ4, LVQ8, LeanVec4x8, LeanVec8x8 + +* **Type:** + Vector compression + +#### `construction_window_size: int` + +1. - affects quality vs build time + +* **Type:** + Build-time candidates (default + +#### `epsilon: float` + +0.01) + +* **Type:** + Range query boundary factor (default + +#### `graph_max_degree: int` + +1. - affects recall vs memory + +* **Type:** + Max edges per node (default + +#### `model_config: ClassVar[ConfigDict] = {}` + +Configuration for the model, should be a dictionary conforming to [ConfigDict][pydantic.config.ConfigDict]. + +#### `reduce: int | None` + +Dimensionality reduction for LeanVec types (must be < dims) + +#### `search_window_size: int` + +1. - primary tuning parameter + +* **Type:** + Search candidates (default + +#### `training_threshold: int | None` + +10,240) + +* **Type:** + Min vectors before compression training (default + +**SVS-VAMANA Examples:** + +**Basic configuration (no compression):** + +```yaml +- name: embedding + type: vector + attrs: + algorithm: svs-vamana + dims: 768 + distance_metric: cosine + datatype: float32 + # Standard settings for balanced performance + graph_max_degree: 40 + construction_window_size: 250 + search_window_size: 20 +``` + +**High-performance configuration with compression:** + +```yaml +- name: embedding + type: vector + attrs: + algorithm: svs-vamana + dims: 768 + distance_metric: cosine + datatype: float32 + # Tuned for better recall + graph_max_degree: 64 + construction_window_size: 500 + search_window_size: 40 + # Maximum compression with dimensionality reduction + compression: LeanVec4x8 + reduce: 384 # 50% dimensionality reduction + training_threshold: 1000 +``` + +**Important Notes:** + +- **Requirements**: SVS-VAMANA requires Redis >= 8.2 with RediSearch >= 2.8.10. +- **Datatype limitations**: SVS-VAMANA only supports float16 and float32 datatypes (not bfloat16 or float64). +- **Compression compatibility**: The reduce parameter is only valid with LeanVec compression types (LeanVec4x8 or LeanVec8x8). +- **Platform considerations**: Intel’s proprietary LVQ and LeanVec optimizations are not available in Redis Open Source. On non-Intel platforms and Redis Open Source, SVS-VAMANA with compression falls back to basic 8-bit scalar quantization. +- **Performance tip**: Start with default parameters and tune search_window_size first for your speed vs accuracy requirements. + +### `FLAT Vector Fields` + +FLAT - Brute-force exact search. **Best for small datasets (<10K vectors) requiring 100% accuracy.** + +### `When to use FLAT & Performance Details` + +**Use FLAT when:** +: - Small datasets (<100K vectors) where exact results are required + - Search accuracy is critical and approximate results are not acceptable + - Baseline comparisons when evaluating approximate algorithms + - Simple use cases where setup simplicity is more important than performance + +**Performance characteristics:** +: - **Search accuracy**: 100% exact results (no approximation) + - **Search speed**: Linear time O(n) - slower as dataset grows + - **Memory usage**: Minimal overhead, stores vectors as-is + - **Build time**: Fastest index construction (no preprocessing) + +**Trade-offs vs other algorithms:** +: - **vs HNSW**: Much slower search but exact results, faster index building + - **vs SVS-VAMANA**: Slower search and higher memory usage, but exact results + +### `class FlatVectorField(*, name, type=FieldTypes.VECTOR, path=None, attrs)` + +Bases: `BaseField` + +Vector field with FLAT (exact search) indexing for exact nearest neighbor search. + +Create a new model by parsing and validating input data from keyword arguments. + +Raises [ValidationError][pydantic_core.ValidationError] if the input data cannot be +validated to form a valid model. + +self is explicitly positional-only to allow self as a field name. + +* **Parameters:** + * **name** (*str*) + * **type** (*Literal* *[* *FieldTypes.VECTOR* *]*) + * **path** (*str* *|* *None*) + * **attrs** ([FlatVectorFieldAttributes](#flatvectorfieldattributes)) + +#### `as_redis_field()` + +Convert schema field to Redis Field object + +* **Return type:** + *Field* + +#### `attrs: `[FlatVectorFieldAttributes](#flatvectorfieldattributes) + +Specified field attributes + +#### `model_config: ClassVar[ConfigDict] = {}` + +Configuration for the model, should be a dictionary conforming to [ConfigDict][pydantic.config.ConfigDict]. + +#### `type: Literal[FieldTypes.VECTOR]` + +Field type + +### `class FlatVectorFieldAttributes(*, dims, algorithm=VectorIndexAlgorithm.FLAT, datatype=VectorDataType.FLOAT32, distance_metric=VectorDistanceMetric.COSINE, initial_cap=None, index_missing=False, block_size=None)` + +FLAT vector field attributes for exact nearest neighbor search. + +Create a new model by parsing and validating input data from keyword arguments. + +Raises [ValidationError][pydantic_core.ValidationError] if the input data cannot be +validated to form a valid model. + +self is explicitly positional-only to allow self as a field name. + +* **Parameters:** + * **dims** (*int*) + * **algorithm** (*Literal* *[* *VectorIndexAlgorithm.FLAT* *]*) + * **datatype** (*VectorDataType*) + * **distance_metric** (*VectorDistanceMetric*) + * **initial_cap** (*int* *|* *None*) + * **index_missing** (*bool*) + * **block_size** (*int* *|* *None*) + +#### `algorithm: Literal[VectorIndexAlgorithm.FLAT]` + +The indexing algorithm (fixed as ‘flat’) + +#### `block_size: int | None` + +Block size for processing (optional) - improves batch operation throughput + +#### `model_config: ClassVar[ConfigDict] = {}` + +Configuration for the model, should be a dictionary conforming to [ConfigDict][pydantic.config.ConfigDict]. + +**FLAT Example:** + +```yaml +- name: embedding + type: vector + attrs: + algorithm: flat + dims: 768 + distance_metric: cosine + datatype: float32 + # Optional: tune for batch processing + block_size: 1024 +``` + +**Note**: FLAT is recommended for small datasets or when exact results are mandatory. For larger datasets, consider HNSW or SVS-VAMANA for better performance. + +## SVS-VAMANA Configuration Utilities + +For SVS-VAMANA indices, RedisVL provides utilities to help configure compression settings and estimate memory savings. + +### `CompressionAdvisor` + +### `class CompressionAdvisor` + +Bases: `object` + +Helper to recommend compression settings based on vector characteristics. + +This class provides utilities to: +- Recommend optimal SVS-VAMANA configurations based on vector dimensions and priorities +- Estimate memory savings from compression and dimensionality reduction + +### `Examples` + +```pycon +>> # Get recommendations for high-dimensional vectors +>> config = CompressionAdvisor.recommend(dims=1536, priority="balanced") +>> config.compression +'LeanVec4x8' +>> config.reduce +768 +``` + +```pycon +>> # Estimate memory savings +>> savings = CompressionAdvisor.estimate_memory_savings( +... compression="LeanVec4x8", +... dims=1536, +... reduce=768 +... ) +>> savings +81.2 +``` + +#### `static estimate_memory_savings(compression, dims, reduce=None)` + +Estimate memory savings percentage from compression. + +Calculates the percentage of memory saved compared to uncompressed float32 vectors. + +* **Parameters:** + * **compression** (*str*) – Compression type (e.g., "LVQ4", "LeanVec4x8") + * **dims** (*int*) – Original vector dimensionality + * **reduce** (*int* *|* *None*) – Reduced dimensionality (for LeanVec compression) +* **Returns:** + Memory savings percentage (0-100) +* **Return type:** + float + +### `Examples` + +```pycon +>> # LeanVec with dimensionality reduction +>> CompressionAdvisor.estimate_memory_savings( +... compression="LeanVec4x8", +... dims=1536, +... reduce=768 +... ) +81.2 +``` + +```pycon +>> # LVQ without dimensionality reduction +>> CompressionAdvisor.estimate_memory_savings( +... compression="LVQ4", +... dims=384 +... ) +87.5 +``` + +#### `static recommend(dims, priority='balanced', datatype=None)` + +Recommend compression settings based on dimensions and priorities. + +* **Parameters:** + * **dims** (*int*) – Vector dimensionality (must be > 0) + * **priority** (*Literal* *[* *'speed'* *,* *'memory'* *,* *'balanced'* *]*) – Optimization priority: + - "memory": Maximize memory savings + - "speed": Optimize for query speed + - "balanced": Balance between memory and speed + * **datatype** (*str* *|* *None*) – Override datatype (default: float16 for high-dim, float32 for low-dim) +* **Returns:** + Complete SVS-VAMANA configuration including: + : - algorithm: "svs-vamana" + - datatype: Recommended datatype + - compression: Compression type + - reduce: Dimensionality reduction (for LeanVec only) + - graph_max_degree: Graph connectivity + - construction_window_size: Build-time candidates + - search_window_size: Query-time candidates +* **Return type:** + dict +* **Raises:** + **ValueError** – If dims <= 0 + +### `Examples` + +```pycon +>> # High-dimensional embeddings (e.g., OpenAI ada-002) +>> config = CompressionAdvisor.recommend(dims=1536, priority="memory") +>> config.compression +'LeanVec4x8' +>> config.reduce +768 +``` + +```pycon +>> # Lower-dimensional embeddings +>> config = CompressionAdvisor.recommend(dims=384, priority="speed") +>> config.compression +'LVQ4x8' +``` + +### `SVSConfig` + +### `class SVSConfig(*, algorithm='svs-vamana', datatype=None, compression=None, reduce=None, graph_max_degree=None, construction_window_size=None, search_window_size=None)` + +Bases: `BaseModel` + +SVS-VAMANA configuration model. + +* **Parameters:** + * **algorithm** (*Literal* *[* *'svs-vamana'* *]*) + * **datatype** (*str* *|* *None*) + * **compression** (*str* *|* *None*) + * **reduce** (*int* *|* *None*) + * **graph_max_degree** (*int* *|* *None*) + * **construction_window_size** (*int* *|* *None*) + * **search_window_size** (*int* *|* *None*) + +#### `algorithm` + +Always "svs-vamana" + +* **Type:** + Literal[‘svs-vamana’] + +#### `datatype` + +Vector datatype (float16, float32) + +* **Type:** + str | None + +#### `compression` + +Compression type (LVQ4, LeanVec4x8, etc.) + +* **Type:** + str | None + +#### `reduce` + +Reduced dimensionality (only for LeanVec) + +* **Type:** + int | None + +#### `graph_max_degree` + +Max edges per node + +* **Type:** + int | None + +#### `construction_window_size` + +Build-time candidates + +* **Type:** + int | None + +#### `search_window_size` + +Query-time candidates + +* **Type:** + int | None + +Create a new model by parsing and validating input data from keyword arguments. + +Raises [ValidationError][pydantic_core.ValidationError] if the input data cannot be +validated to form a valid model. + +self is explicitly positional-only to allow self as a field name. + +#### `model_config: ClassVar[ConfigDict] = {}` + +Configuration for the model, should be a dictionary conforming to [ConfigDict][pydantic.config.ConfigDict]. + + + +## Vector Algorithm Comparison + +This section provides detailed guidance for choosing between vector search algorithms. + +### `Algorithm Selection Guide` + +#### `Vector Algorithm Comparison` + +| Algorithm | Best For | Performance | Memory Usage | Trade-offs | +|----------------|----------------------------------------|--------------------------------|---------------------------|---------------------------------------| +| **FLAT** | Small datasets (<100K vectors) | 100% recall, O(n) search | Minimal overhead | Exact but slow for large data | +| **HNSW** | General purpose (100K-1M+ vectors) | 95-99% recall, O(log n) search | Moderate (graph overhead) | Fast approximate search | +| **SVS-VAMANA** | Large datasets with memory constraints | 90-95% recall, O(log n) search | Low (with compression) | Intel-optimized, requires newer Redis | + +### `When to Use Each Algorithm` + +**Choose FLAT when:** +: - Dataset size < 100,000 vectors + - Exact results are mandatory + - Simple setup is preferred + - Query latency is not critical + +**Choose HNSW when:** +: - Dataset size 100K - 1M+ vectors + - Need balanced speed and accuracy + - Cross-platform compatibility required + - Most common choice for production + +**Choose SVS-VAMANA when:** +: - Dataset size > 100K vectors + - Memory usage is a primary concern + - Running on Intel hardware + - Can accept 90-95% recall for memory savings + +### `Performance Characteristics` + +**Search Speed:** +: - FLAT: Linear time O(n) - gets slower as data grows + - HNSW: Logarithmic time O(log n) - scales well + - SVS-VAMANA: Logarithmic time O(log n) - scales well -**Geo Field Attributes**: +**Memory Usage (1M vectors, 768 dims, float32):** +: - FLAT: ~3.1 GB (baseline) + - HNSW: ~3.7 GB (20% overhead for graph) + - SVS-VAMANA: 1.6-3.1 GB (depends on compression) -- sortable: Enables sorting based on the geo field. -- no_index: When True, field is not indexed but can be returned in results (requires sortable=True). +**Recall Quality:** +: - FLAT: 100% (exact search) + - HNSW: 95-99% (tunable via ef_runtime) + - SVS-VAMANA: 90-95% (depends on compression) -**Common Vector Field Attributes**: +### `Migration Considerations` -- dims: Dimensionality of the vector. -- algorithm: Indexing algorithm (flat or hnsw). -- datatype: Float datatype of the vector (bfloat16, float16, float32, float64). -- distance_metric: Metric for measuring query relevance (COSINE, L2, IP). +**From FLAT to HNSW:** +: - Straightforward migration + - Expect slight recall reduction but major speed improvement + - Tune ef_runtime to balance speed vs accuracy -**HNSW Vector Field Specific Attributes**: +**From HNSW to SVS-VAMANA:** +: - Requires Redis >= 8.2 with RediSearch >= 2.8.10 + - Change datatype to float16 or float32 if using others + - Consider compression options for memory savings -- m: Max outgoing edges per node in each layer. -- ef_construction: Max edge candidates during build time. -- ef_runtime: Max top candidates during search. -- epsilon: Range search boundary factor. +**From SVS-VAMANA to others:** +: - May need to change datatype back if using float64/bfloat16 + - HNSW provides similar performance with broader compatibility -Note: -: See fully documented Redis-supported fields and options here: [https://redis.io/commands/ft.create/](https://redis.io/commands/ft.create/) +For complete Redis field documentation, see: [https://redis.io/commands/ft.create/](https://redis.io/commands/ft.create/) diff --git a/content/develop/ai/redisvl/api/vector.md b/content/develop/ai/redisvl/api/vector.md index 94128399a8..1fbc11a312 100644 --- a/content/develop/ai/redisvl/api/vector.md +++ b/content/develop/ai/redisvl/api/vector.md @@ -27,6 +27,13 @@ self is explicitly positional-only to allow self as a field name. * **dtype** (*str*) * **weight** (*float*) +#### `validate_vector()` + +If the vector passed in is an array of float convert it to a byte string. + +* **Return type:** + *Self* + #### `model_config: ClassVar[ConfigDict] = {}` Configuration for the model, should be a dictionary conforming to [ConfigDict][pydantic.config.ConfigDict]. diff --git a/content/develop/ai/redisvl/user_guide/_index.md b/content/develop/ai/redisvl/user_guide/_index.md index 2e8d7f0843..8ac9b40668 100644 --- a/content/develop/ai/redisvl/user_guide/_index.md +++ b/content/develop/ai/redisvl/user_guide/_index.md @@ -37,17 +37,6 @@ User guides provide helpful resources for using RedisVL and its different compon * [Utilize TTL](llmcache/#utilize-ttl) * [Simple Performance Testing](llmcache/#simple-performance-testing) * [Cache Access Controls, Tags & Filters](llmcache/#cache-access-controls-tags-filters) -* [Caching Embeddings](embeddings_cache/) - * [Setup](embeddings_cache/#setup) - * [Initializing the EmbeddingsCache](embeddings_cache/#initializing-the-embeddingscache) - * [Basic Usage](embeddings_cache/#basic-usage) - * [Advanced Usage](embeddings_cache/#advanced-usage) - * [Async Support](embeddings_cache/#async-support) - * [Real-World Example](embeddings_cache/#real-world-example) - * [Performance Benchmark](embeddings_cache/#performance-benchmark) - * [Common Use Cases for Embedding Caching](embeddings_cache/#common-use-cases-for-embedding-caching) - * [Cleanup](embeddings_cache/#cleanup) - * [Summary](embeddings_cache/#summary) * [Vectorizers](vectorizers/) * [Creating Text Embeddings](vectorizers/#creating-text-embeddings) * [Search with Provider Embeddings](vectorizers/#search-with-provider-embeddings) @@ -58,7 +47,6 @@ User guides provide helpful resources for using RedisVL and its different compon * [Working with nested data in JSON](hash_vs_json/#working-with-nested-data-in-json) * [Full JSON Path support](hash_vs_json/#full-json-path-support) * [As an example:](hash_vs_json/#as-an-example) -* [Cleanup](hash_vs_json/#id1) * [Rerankers](rerankers/) * [Simple Reranking](rerankers/#simple-reranking) * [LLM Message History](message_history/) @@ -75,3 +63,36 @@ User guides provide helpful resources for using RedisVL and its different compon * [Get route references](semantic_router/#get-route-references) * [Delete route references](semantic_router/#delete-route-references) * [Clean up the router](semantic_router/#clean-up-the-router) +* [SVS-VAMANA Vector Search](svs_vamana/) + * [Prerequisites](svs_vamana/#prerequisites) + * [Quick Start with CompressionAdvisor](svs_vamana/#quick-start-with-compressionadvisor) + * [Creating an SVS-VAMANA Index](svs_vamana/#creating-an-svs-vamana-index) + * [Loading Sample Data](svs_vamana/#loading-sample-data) + * [Performing Vector Searches](svs_vamana/#performing-vector-searches) + * [Understanding Compression Types](svs_vamana/#understanding-compression-types) + * [Compression Types Explained](svs_vamana/#compression-types-explained) + * [Hybrid Queries with SVS-VAMANA](svs_vamana/#hybrid-queries-with-svs-vamana) + * [Performance Monitoring](svs_vamana/#performance-monitoring) + * [Manual Configuration (Advanced)](svs_vamana/#manual-configuration-advanced) + * [Best Practices and Tips](svs_vamana/#best-practices-and-tips) + * [Cleanup](svs_vamana/#cleanup) +* [Caching Embeddings](embeddings_cache/) + * [Setup](embeddings_cache/#setup) + * [Initializing the EmbeddingsCache](embeddings_cache/#initializing-the-embeddingscache) + * [Basic Usage](embeddings_cache/#basic-usage) + * [Advanced Usage](embeddings_cache/#advanced-usage) + * [Async Support](embeddings_cache/#async-support) + * [Real-World Example](embeddings_cache/#real-world-example) + * [Performance Benchmark](embeddings_cache/#performance-benchmark) + * [Common Use Cases for Embedding Caching](embeddings_cache/#common-use-cases-for-embedding-caching) + * [Cleanup](embeddings_cache/#cleanup) + * [Summary](embeddings_cache/#summary) +* [Advanced Query Types](advanced_queries/) + * [Setup and Data Preparation](advanced_queries/#setup-and-data-preparation) + * [Define the Schema](advanced_queries/#define-the-schema) + * [Create Index and Load Data](advanced_queries/#create-index-and-load-data) + * [1. TextQuery: Full Text Search](advanced_queries/#textquery-full-text-search) + * [2. AggregateHybridQuery: Combining Text and Vector Search](advanced_queries/#aggregatehybridquery-combining-text-and-vector-search) + * [3. MultiVectorQuery: Multi-Vector Search](advanced_queries/#multivectorquery-multi-vector-search) + * [Comparing Query Types](advanced_queries/#comparing-query-types) + * [Best Practices](advanced_queries/#best-practices) diff --git a/content/develop/ai/redisvl/user_guide/advanced_queries.md b/content/develop/ai/redisvl/user_guide/advanced_queries.md new file mode 100644 index 0000000000..e7a01bd039 --- /dev/null +++ b/content/develop/ai/redisvl/user_guide/advanced_queries.md @@ -0,0 +1,729 @@ +--- +linkTitle: Advanced query types +title: Advanced Query Types +aliases: +- /integrate/redisvl/user_guide/11_advanced_queries +weight: 11 +--- + + +In this notebook, we will explore advanced query types available in RedisVL: + +1. **`TextQuery`**: Full text search with advanced scoring +2. **`AggregateHybridQuery`**: Combines text and vector search for hybrid retrieval +3. **`MultiVectorQuery`**: Search over multiple vector fields simultaneously + +These query types are powerful tools for building sophisticated search applications that go beyond simple vector similarity search. + +Prerequisites: +- Ensure `redisvl` is installed in your Python environment. +- Have a running instance of [Redis Stack](https://redis.io/docs/install/install-stack/) or [Redis Cloud](https://redis.io/cloud). + + +## Setup and Data Preparation + +First, let's create a schema and prepare sample data that includes text fields, numeric fields, and vector fields. + + +```python +import numpy as np +from jupyterutils import result_print + +# Sample data with text descriptions, categories, and vectors +data = [ + { + 'product_id': 'prod_1', + 'brief_description': 'comfortable running shoes for athletes', + 'full_description': 'Engineered with a dual-layer EVA foam midsole and FlexWeave breathable mesh upper, these running shoes deliver responsive cushioning for long-distance runs. The anatomical footbed adapts to your stride while the carbon rubber outsole provides superior traction on varied terrain.', + 'category': 'footwear', + 'price': 89.99, + 'rating': 4.5, + 'text_embedding': np.array([0.1, 0.2, 0.1], dtype=np.float32).tobytes(), + 'image_embedding': np.array([0.8, 0.1], dtype=np.float32).tobytes(), + }, + { + 'product_id': 'prod_2', + 'brief_description': 'lightweight running jacket with water resistance', + 'full_description': 'Stay protected with this ultralight 2.5-layer DWR-coated shell featuring laser-cut ventilation zones and reflective piping for low-light visibility. Packs into its own chest pocket and weighs just 4.2 oz, making it ideal for unpredictable weather conditions.', + 'category': 'outerwear', + 'price': 129.99, + 'rating': 4.8, + 'text_embedding': np.array([0.2, 0.3, 0.2], dtype=np.float32).tobytes(), + 'image_embedding': np.array([0.7, 0.2], dtype=np.float32).tobytes(), + }, + { + 'product_id': 'prod_3', + 'brief_description': 'professional tennis racket for competitive players', + 'full_description': 'Competition-grade racket featuring a 98 sq in head size, 16x19 string pattern, and aerospace-grade graphite frame that delivers explosive power with pinpoint control. Tournament-approved specs include 315g weight and 68 RA stiffness rating for advanced baseline play.', + 'category': 'equipment', + 'price': 199.99, + 'rating': 4.9, + 'text_embedding': np.array([0.9, 0.1, 0.05], dtype=np.float32).tobytes(), + 'image_embedding': np.array([0.1, 0.9], dtype=np.float32).tobytes(), + }, + { + 'product_id': 'prod_4', + 'brief_description': 'yoga mat with extra cushioning for comfort', + 'full_description': 'Premium 8mm thick TPE yoga mat with dual-texture surface - smooth side for hot yoga flow and textured side for maximum grip during balancing poses. Closed-cell technology prevents moisture absorption while alignment markers guide proper positioning in asanas.', + 'category': 'accessories', + 'price': 39.99, + 'rating': 4.3, + 'text_embedding': np.array([0.15, 0.25, 0.15], dtype=np.float32).tobytes(), + 'image_embedding': np.array([0.5, 0.5], dtype=np.float32).tobytes(), + }, + { + 'product_id': 'prod_5', + 'brief_description': 'basketball shoes with excellent ankle support', + 'full_description': 'High-top basketball sneakers with Zoom Air units in forefoot and heel, reinforced lateral sidewalls for explosive cuts, and herringbone traction pattern optimized for hardwood courts. The internal bootie construction and extended ankle collar provide lockdown support during aggressive drives.', + 'category': 'footwear', + 'price': 139.99, + 'rating': 4.7, + 'text_embedding': np.array([0.12, 0.18, 0.12], dtype=np.float32).tobytes(), + 'image_embedding': np.array([0.75, 0.15], dtype=np.float32).tobytes(), + }, + { + 'product_id': 'prod_6', + 'brief_description': 'swimming goggles with anti-fog coating', + 'full_description': 'Low-profile competition goggles with curved polycarbonate lenses offering 180-degree peripheral vision and UV protection. Hydrophobic anti-fog coating lasts 10x longer than standard treatments, while the split silicone strap and interchangeable nose bridges ensure a watertight, custom fit.', + 'category': 'accessories', + 'price': 24.99, + 'rating': 4.4, + 'text_embedding': np.array([0.3, 0.1, 0.2], dtype=np.float32).tobytes(), + 'image_embedding': np.array([0.2, 0.8], dtype=np.float32).tobytes(), + }, +] +``` + +## Define the Schema + +Our schema includes: +- **Tag fields**: `product_id`, `category` +- **Text fields**: `brief_description` and `full_description` for full-text search +- **Numeric fields**: `price`, `rating` +- **Vector fields**: `text_embedding` (3 dimensions) and `image_embedding` (2 dimensions) for semantic search + + +```python +schema = { + "index": { + "name": "advanced_queries", + "prefix": "products", + "storage_type": "hash", + }, + "fields": [ + {"name": "product_id", "type": "tag"}, + {"name": "category", "type": "tag"}, + {"name": "brief_description", "type": "text"}, + {"name": "full_description", "type": "text"}, + {"name": "price", "type": "numeric"}, + {"name": "rating", "type": "numeric"}, + { + "name": "text_embedding", + "type": "vector", + "attrs": { + "dims": 3, + "distance_metric": "cosine", + "algorithm": "flat", + "datatype": "float32" + } + }, + { + "name": "image_embedding", + "type": "vector", + "attrs": { + "dims": 2, + "distance_metric": "cosine", + "algorithm": "flat", + "datatype": "float32" + } + } + ], +} +``` + +## Create Index and Load Data + + +```python +from redisvl.index import SearchIndex + +# Create the search index +index = SearchIndex.from_dict(schema, redis_url="redis://localhost:6379") + +# Create the index and load data +index.create(overwrite=True) +keys = index.load(data) + +print(f"Loaded {len(keys)} products into the index") +``` + + Loaded 6 products into the index + + +## 1. TextQuery: Full Text Search + +The `TextQuery` class enables full text search with advanced scoring algorithms. It's ideal for keyword-based search with relevance ranking. + +### Basic Text Search + +Let's search for products related to "running shoes": + + +```python +from redisvl.query import TextQuery + +# Create a text query +text_query = TextQuery( + text="running shoes", + text_field_name="brief_description", + return_fields=["product_id", "brief_description", "category", "price"], + num_results=5 +) + +results = index.query(text_query) +result_print(results) +``` + + +
scoreproduct_idbrief_descriptioncategoryprice
5.953989333038773prod_1comfortable running shoes for athletesfootwear89.99
2.085315593627535prod_5basketball shoes with excellent ankle supportfootwear139.99
2.0410082774474088prod_2lightweight running jacket with water resistanceouterwear129.99
+ + +### Text Search with Different Scoring Algorithms + +RedisVL supports multiple text scoring algorithms. Let's compare `BM25STD` and `TFIDF`: + + +```python +# BM25 standard scoring (default) +bm25_query = TextQuery( + text="comfortable shoes", + text_field_name="brief_description", + text_scorer="BM25STD", + return_fields=["product_id", "brief_description", "price"], + num_results=3 +) + +print("Results with BM25 scoring:") +results = index.query(bm25_query) +result_print(results) +``` + + Results with BM25 scoring: + + + +
scoreproduct_idbrief_descriptionprice
6.031534703977659prod_1comfortable running shoes for athletes89.99
2.085315593627535prod_5basketball shoes with excellent ankle support139.99
1.5268074873573214prod_4yoga mat with extra cushioning for comfort39.99
+ + + +```python +# TFIDF scoring +tfidf_query = TextQuery( + text="comfortable shoes", + text_field_name="brief_description", + text_scorer="TFIDF", + return_fields=["product_id", "brief_description", "price"], + num_results=3 +) + +print("Results with TFIDF scoring:") +results = index.query(tfidf_query) +result_print(results) +``` + + Results with TFIDF scoring: + + + +
scoreproduct_idbrief_descriptionprice
2.3333333333333335prod_1comfortable running shoes for athletes89.99
2.0prod_5basketball shoes with excellent ankle support139.99
1.0prod_4yoga mat with extra cushioning for comfort39.99
+ + +### Text Search with Filters + +Combine text search with filters to narrow results: + + +```python +from redisvl.query.filter import Tag, Num + +# Search for "shoes" only in the footwear category +filtered_text_query = TextQuery( + text="shoes", + text_field_name="brief_description", + filter_expression=Tag("category") == "footwear", + return_fields=["product_id", "brief_description", "category", "price"], + num_results=5 +) + +results = index.query(filtered_text_query) +result_print(results) +``` + + +
scoreproduct_idbrief_descriptioncategoryprice
3.9314935770863046prod_1comfortable running shoes for athletesfootwear89.99
3.1279733904413027prod_5basketball shoes with excellent ankle supportfootwear139.99
+ + + +```python +# Search for products under $100 +price_filtered_query = TextQuery( + text="comfortable", + text_field_name="brief_description", + filter_expression=Num("price") < 100, + return_fields=["product_id", "brief_description", "price"], + num_results=5 +) + +results = index.query(price_filtered_query) +result_print(results) +``` + + +
scoreproduct_idbrief_descriptionprice
3.1541404034996914prod_1comfortable running shoes for athletes89.99
1.5268074873573214prod_4yoga mat with extra cushioning for comfort39.99
+ + +### Text Search with Multiple Fields and Weights + +You can search across multiple text fields with different weights to prioritize certain fields. +Here we'll prioritize the `brief_description` field and make text similarity in that field twice as important as text similarity in `full_description`: + + +```python +weighted_query = TextQuery( + text="shoes", + text_field_name={"brief_description": 1.0, "full_description": 0.5}, + return_fields=["product_id", "brief_description"], + num_results=3 +) + +results = index.query(weighted_query) +result_print(results) +``` + + +
scoreproduct_idbrief_description
5.035440025836444prod_1comfortable running shoes for athletes
2.085315593627535prod_5basketball shoes with excellent ankle support
+ + +### Text Search with Custom Stopwords + +Stopwords are common words that are filtered out before processing the query. You can specify which language's default stopwords should be filtered out, like `english`, `french`, or `german`. You can also define your own list of stopwords: + + +```python +# Use English stopwords (default) +query_with_stopwords = TextQuery( + text="the best shoes for running", + text_field_name="brief_description", + stopwords="english", # Common words like "the", "for" will be removed + return_fields=["product_id", "brief_description"], + num_results=3 +) + +results = index.query(query_with_stopwords) +result_print(results) +``` + + +
scoreproduct_idbrief_description
5.953989333038773prod_1comfortable running shoes for athletes
2.085315593627535prod_5basketball shoes with excellent ankle support
2.0410082774474088prod_2lightweight running jacket with water resistance
+ + + +```python +# Use custom stopwords +custom_stopwords_query = TextQuery( + text="professional equipment for athletes", + text_field_name="brief_description", + stopwords=["for", "with"], # Only these words will be filtered + return_fields=["product_id", "brief_description"], + num_results=3 +) + +results = index.query(custom_stopwords_query) +result_print(results) +``` + + +
scoreproduct_idbrief_description
3.1541404034996914prod_1comfortable running shoes for athletes
3.0864038416103prod_3professional tennis racket for competitive players
+ + + +```python +# No stopwords +no_stopwords_query = TextQuery( + text="the best shoes for running", + text_field_name="brief_description", + stopwords=None, # All words will be included + return_fields=["product_id", "brief_description"], + num_results=3 +) + +results = index.query(no_stopwords_query) +result_print(results) +``` + + +
scoreproduct_idbrief_description
5.953989333038773prod_1comfortable running shoes for athletes
2.085315593627535prod_5basketball shoes with excellent ankle support
2.0410082774474088prod_2lightweight running jacket with water resistance
+ + +## 2. AggregateHybridQuery: Combining Text and Vector Search + +The `AggregateHybridQuery` combines text search and vector similarity to provide the best of both worlds: +- **Text search**: Finds exact keyword matches +- **Vector search**: Captures semantic similarity + +Results are scored using a weighted combination: + +``` +hybrid_score = (alpha) * vector_score + (1 - alpha) * text_score +``` + +Where `alpha` controls the balance between vector and text search (default: 0.7). + +### Basic Aggregate Hybrid Query + +Let's search for "running" with both text and semantic search: + + +```python +from redisvl.query import AggregateHybridQuery + +# Create a hybrid query +hybrid_query = AggregateHybridQuery( + text="running shoes", + text_field_name="brief_description", + vector=[0.1, 0.2, 0.1], # Query vector + vector_field_name="text_embedding", + return_fields=["product_id", "brief_description", "category", "price"], + num_results=5 +) + +results = index.query(hybrid_query) +result_print(results) +``` + + +
vector_distanceproduct_idbrief_descriptioncategorypricevector_similaritytext_scorehybrid_score
5.96046447754e-08prod_1comfortable running shoes for athletesfootwear89.990.9999999701985.953989333042.48619677905
0.00985252857208prod_5basketball shoes with excellent ankle supportfootwear139.990.9950737357142.085315593631.32214629309
0.00985252857208prod_2lightweight running jacket with water resistanceouterwear129.990.9950737357142.041008277451.30885409823
0.0038834810257prod_4yoga mat with extra cushioning for comfortaccessories39.990.99805825948700.698640781641
0.236237406731prod_6swimming goggles with anti-fog coatingaccessories24.990.88188129663500.617316907644
+ + +### Adjusting the Alpha Parameter + +The `alpha` parameter controls the weight between vector and text search: +- `alpha=1.0`: Pure vector search +- `alpha=0.0`: Pure text search +- `alpha=0.7` (default): 70% vector, 30% text + + +```python +# More emphasis on vector search (alpha=0.9) +vector_heavy_query = AggregateHybridQuery( + text="comfortable", + text_field_name="brief_description", + vector=[0.15, 0.25, 0.15], + vector_field_name="text_embedding", + alpha=0.9, # 90% vector, 10% text + return_fields=["product_id", "brief_description"], + num_results=3 +) + +print("Results with alpha=0.9 (vector-heavy):") +results = index.query(vector_heavy_query) +result_print(results) +``` + + Results with alpha=0.9 (vector-heavy): + + + +
vector_distanceproduct_idbrief_descriptionvector_similaritytext_scorehybrid_score
-1.19209289551e-07prod_4yoga mat with extra cushioning for comfort1.00000005961.526807487361.05268080238
0.00136888027191prod_5basketball shoes with excellent ankle support0.99931555986400.899384003878
0.00136888027191prod_2lightweight running jacket with water resistance0.99931555986400.899384003878
+ + +### Aggregate Hybrid Query with Filters + +You can also combine hybrid search with filters: + + +```python +# Hybrid search with a price filter +filtered_hybrid_query = AggregateHybridQuery( + text="professional equipment", + text_field_name="brief_description", + vector=[0.9, 0.1, 0.05], + vector_field_name="text_embedding", + filter_expression=Num("price") > 100, + return_fields=["product_id", "brief_description", "category", "price"], + num_results=5 +) + +results = index.query(filtered_hybrid_query) +result_print(results) +``` + + +
vector_distanceproduct_idbrief_descriptioncategorypricevector_similaritytext_scorehybrid_score
-1.19209289551e-07prod_3professional tennis racket for competitive playersequipment199.991.00000005963.086403841611.62592119421
0.411657452583prod_5basketball shoes with excellent ankle supportfootwear139.990.79417127370800.555919891596
0.411657452583prod_2lightweight running jacket with water resistanceouterwear129.990.79417127370800.555919891596
+ + +### Using Different Text Scorers + +AggregateHybridQuery supports the same text scoring algorithms as TextQuery: + + +```python +# Aggregate Hybrid query with TFIDF scorer +hybrid_tfidf = AggregateHybridQuery( + text="shoes support", + text_field_name="brief_description", + vector=[0.12, 0.18, 0.12], + vector_field_name="text_embedding", + text_scorer="TFIDF", + return_fields=["product_id", "brief_description"], + num_results=3 +) + +results = index.query(hybrid_tfidf) +result_print(results) +``` + + +
vector_distanceproduct_idbrief_descriptionvector_similaritytext_scorehybrid_score
0prod_5basketball shoes with excellent ankle support152.2
0prod_2lightweight running jacket with water resistance100.7
0.00136888027191prod_4yoga mat with extra cushioning for comfort0.99931555986400.699520891905
+ + +## 3. MultiVectorQuery: Multi-Vector Search + +The `MultiVectorQuery` allows you to search over multiple vector fields simultaneously. This is useful when you have different types of embeddings (e.g., text and image embeddings) and want to find results that match across multiple modalities. + +The final score is calculated as a weighted combination: + +``` +combined_score = w_1 * score_1 + w_2 * score_2 + w_3 * score_3 + ... +``` + +### Basic Multi-Vector Query + +First, we need to import the `Vector` class to define our query vectors: + + +```python +from redisvl.query import MultiVectorQuery, Vector + +# Define multiple vectors for the query +text_vector = Vector( + vector=[0.1, 0.2, 0.1], + field_name="text_embedding", + dtype="float32", + weight=0.7 # 70% weight for text embedding +) + +image_vector = Vector( + vector=[0.8, 0.1], + field_name="image_embedding", + dtype="float32", + weight=0.3 # 30% weight for image embedding +) + +# Create a multi-vector query +multi_vector_query = MultiVectorQuery( + vectors=[text_vector, image_vector], + return_fields=["product_id", "brief_description", "category"], + num_results=5 +) + +results = index.query(multi_vector_query) +result_print(results) +``` + + +
distance_0distance_1product_idbrief_descriptioncategoryscore_0score_1combined_score
5.96046447754e-085.96046447754e-08prod_1comfortable running shoes for athletesfootwear0.9999999701980.9999999701980.999999970198
0.009852528572080.00266629457474prod_5basketball shoes with excellent ankle supportfootwear0.9950737357140.9986668527130.996151670814
0.009852528572080.0118260979652prod_2lightweight running jacket with water resistanceouterwear0.9950737357140.9940869510170.994777700305
0.00388348102570.210647821426prod_4yoga mat with extra cushioning for comfortaccessories0.9980582594870.8946760892870.967043608427
0.2362374067310.639005899429prod_6swimming goggles with anti-fog coatingaccessories0.8818812966350.6804970502850.82146602273
+ + +### Adjusting Vector Weights + +You can adjust the weights to prioritize different vector fields: + + +```python +# More emphasis on image similarity +text_vec = Vector( + vector=[0.9, 0.1, 0.05], + field_name="text_embedding", + dtype="float32", + weight=0.2 # 20% weight +) + +image_vec = Vector( + vector=[0.1, 0.9], + field_name="image_embedding", + dtype="float32", + weight=0.8 # 80% weight +) + +image_heavy_query = MultiVectorQuery( + vectors=[text_vec, image_vec], + return_fields=["product_id", "brief_description", "category"], + num_results=3 +) + +print("Results with emphasis on image similarity:") +results = index.query(image_heavy_query) +result_print(results) +``` + + Results with emphasis on image similarity: + + + +
distance_0distance_1product_idbrief_descriptioncategoryscore_0score_1combined_score
-1.19209289551e-070prod_3professional tennis racket for competitive playersequipment1.000000059611.00000001192
0.145393729210.00900757312775prod_6swimming goggles with anti-fog coatingaccessories0.9273031353950.9954962134360.981857597828
0.4366961717610.219131231308prod_4yoga mat with extra cushioning for comfortaccessories0.781651914120.8904343843460.868677890301
+ + +### Multi-Vector Query with Filters + +Combine multi-vector search with filters to narrow results: + + +```python +# Multi-vector search with category filter +text_vec = Vector( + vector=[0.1, 0.2, 0.1], + field_name="text_embedding", + dtype="float32", + weight=0.6 +) + +image_vec = Vector( + vector=[0.8, 0.1], + field_name="image_embedding", + dtype="float32", + weight=0.4 +) + +filtered_multi_query = MultiVectorQuery( + vectors=[text_vec, image_vec], + filter_expression=Tag("category") == "footwear", + return_fields=["product_id", "brief_description", "category", "price"], + num_results=5 +) + +results = index.query(filtered_multi_query) +result_print(results) +``` + + +
distance_0distance_1product_idbrief_descriptioncategorypricescore_0score_1combined_score
5.96046447754e-085.96046447754e-08prod_1comfortable running shoes for athletesfootwear89.990.9999999701980.9999999701980.999999970198
0.009852528572080.00266629457474prod_5basketball shoes with excellent ankle supportfootwear139.990.9950737357140.9986668527130.996510982513
+ + +## Comparing Query Types + +Let's compare the three query types side by side: + + +```python +# TextQuery - keyword-based search +text_q = TextQuery( + text="shoes", + text_field_name="brief_description", + return_fields=["product_id", "brief_description"], + num_results=3 +) + +print("TextQuery Results (keyword-based):") +result_print(index.query(text_q)) +print() +``` + + TextQuery Results (keyword-based): + + + +
scoreproduct_idbrief_description
2.8773943004779676prod_1comfortable running shoes for athletes
2.085315593627535prod_5basketball shoes with excellent ankle support
+ + + + + + +```python +# AggregateHybridQuery - combines text and vector search +hybrid_q = AggregateHybridQuery( + text="shoes", + text_field_name="brief_description", + vector=[0.1, 0.2, 0.1], + vector_field_name="text_embedding", + return_fields=["product_id", "brief_description"], + num_results=3 +) + +print("AggregateHybridQuery Results (text + vector):") +result_print(index.query(hybrid_q)) +print() +``` + + AggregateHybridQuery Results (text + vector): + + + +
vector_distanceproduct_idbrief_descriptionvector_similaritytext_scorehybrid_score
5.96046447754e-08prod_1comfortable running shoes for athletes0.9999999701982.877394300481.56321826928
0.0038834810257prod_4yoga mat with extra cushioning for comfort0.99805825948700.698640781641
0.00985252857208prod_2lightweight running jacket with water resistance0.99507373571400.696551615
+ + + + + + +```python +# MultiVectorQuery - searches multiple vector fields +mv_text = Vector( + vector=[0.1, 0.2, 0.1], + field_name="text_embedding", + dtype="float32", + weight=0.5 +) + +mv_image = Vector( + vector=[0.8, 0.1], + field_name="image_embedding", + dtype="float32", + weight=0.5 +) + +multi_q = MultiVectorQuery( + vectors=[mv_text, mv_image], + return_fields=["product_id", "brief_description"], + num_results=3 +) + +print("MultiVectorQuery Results (multiple vectors):") +result_print(index.query(multi_q)) +``` + + MultiVectorQuery Results (multiple vectors): + + + +
distance_0distance_1product_idbrief_descriptionscore_0score_1combined_score
5.96046447754e-085.96046447754e-08prod_1comfortable running shoes for athletes0.9999999701980.9999999701980.999999970198
0.009852528572080.00266629457474prod_5basketball shoes with excellent ankle support0.9950737357140.9986668527130.996870294213
0.009852528572080.0118260979652prod_2lightweight running jacket with water resistance0.9950737357140.9940869510170.994580343366
+ + +## Best Practices + +### When to Use Each Query Type: + +1. **`TextQuery`**: + - When you need precise keyword matching + - For traditional search engine functionality + - When text relevance scoring is important + - Example: Product search, document retrieval + +2. **`AggregateHybridQuery`**: + - When you want to combine keyword and semantic search + - For improved search quality over pure text or vector search + - When you have both text and vector representations of your data + - Example: E-commerce search, content recommendation + +3. **`MultiVectorQuery`**: + - When you have multiple types of embeddings (text, image, audio, etc.) + - For multi-modal search applications + - When you want to balance multiple semantic signals + - Example: Image-text search, cross-modal retrieval + + +```python +# Cleanup +index.delete() +``` diff --git a/content/develop/ai/redisvl/user_guide/hash_vs_json.md b/content/develop/ai/redisvl/user_guide/hash_vs_json.md index a6f32910a2..6ba665a209 100644 --- a/content/develop/ai/redisvl/user_guide/hash_vs_json.md +++ b/content/develop/ai/redisvl/user_guide/hash_vs_json.md @@ -494,9 +494,8 @@ results -# Cleanup - ```python +# Cleanup bike_index.delete() ``` diff --git a/content/develop/ai/redisvl/user_guide/svs_vamana.md b/content/develop/ai/redisvl/user_guide/svs_vamana.md new file mode 100644 index 0000000000..34c4266f9e --- /dev/null +++ b/content/develop/ai/redisvl/user_guide/svs_vamana.md @@ -0,0 +1,554 @@ +--- +linkTitle: SVS-VAMANA vector search +title: SVS-VAMANA Vector Search +aliases: +- /integrate/redisvl/user_guide/09_svs_vamana +weight: 09 +--- + + +In this notebook, we will explore SVS-VAMANA (Scalable Vector Search with VAMANA graph algorithm), a graph-based vector search algorithm that is optimized to work with compression methods to reduce memory usage. It combines the Vamana graph algorithm with advanced compression techniques (LVQ and LeanVec) and is optimized for Intel hardware. + +**How it works** + +Vamana builds a single-layer proximity graph and prunes edges during construction based on tunable parameters, similar to HNSW but with a simpler structure. The compression methods apply per-vector normalization and scalar quantization, learning parameters directly from the data to enable fast, on-the-fly distance computations with SIMD-optimized layout Vector quantization and compression. + + +**SVS-VAMANA offers:** +- **Fast approximate nearest neighbor search** using graph-based algorithms +- **Vector compression** (LVQ, LeanVec) with up to 87.5% memory savings +- **Dimensionality reduction** (optional, with LeanVec) +- **Automatic performance optimization** through CompressionAdvisor + +**Use SVS-VAMANA when:** +- Large datasets where memory is expensive +- Cloud deployments with memory-based pricing +- When 90-95% recall is acceptable +- High-dimensional vectors (>1024 dims) with LeanVec compression + + + +**Table of Contents** + +1. [Prerequisites](#Prerequisites) +2. [Quick Start with CompressionAdvisor](#Quick-Start-with-CompressionAdvisor) +3. [Creating an SVS-VAMANA Index](#Creating-an-SVS-VAMANA-Index) +4. [Loading Sample Data](#Loading-Sample-Data) +5. [Performing Vector Searches](#Performing-Vector-Searches) +6. [Understanding Compression Types](#Understanding-Compression-Types) +7. [Hybrid Queries with SVS-VAMANA](#Hybrid-Queries-with-SVS-VAMANA) +8. [Performance Monitoring](#Performance-Monitoring) +9. [Manual Configuration (Advanced)](#Manual-Configuration-(Advanced)) +10. [Best Practices and Tips](#Best-Practices-and-Tips) +11. [Cleanup](#Cleanup) + +--- + +## Prerequisites + +Before running this notebook, ensure you have: +1. Installed `redisvl` and have that environment active for this notebook +2. A running Redis Stack instance with: + - Redis >= 8.2.0 + - RediSearch >= 2.8.10 + +For example, you can run Redis Stack locally with Docker: + +```bash +docker run -d -p 6379:6379 -p 8001:8001 redis/redis-stack:latest +``` + +**Note:** SVS-VAMANA only supports FLOAT16 and FLOAT32 datatypes. + + +```python +# Import necessary modules +import numpy as np +from redisvl.index import SearchIndex +from redisvl.query import VectorQuery +from redisvl.utils import CompressionAdvisor +from redisvl.redis.utils import array_to_buffer + +# Set random seed for reproducible results +np.random.seed(42) +``` + + +```python +# Redis connection +REDIS_URL = "redis://localhost:6379" +``` + +## Quick Start with CompressionAdvisor + +The easiest way to get started with SVS-VAMANA is using the `CompressionAdvisor` utility, which automatically recommends optimal configuration based on your vector dimensions and performance priorities. + + +```python +# Get recommended configuration for common embedding dimensions +dims = 1024 # Common embedding dimensions (works reliably with SVS-VAMANA) + +config = CompressionAdvisor.recommend( + dims=dims, + priority="balanced" # Options: "memory", "speed", "balanced" +) + +print("Recommended Configuration:") +for key, value in config.items(): + print(f" {key}: {value}") + +# Estimate memory savings +savings = CompressionAdvisor.estimate_memory_savings( + config["compression"], + dims, + config.get("reduce") +) +print(f"\nEstimated Memory Savings: {savings}%") +``` + + Recommended Configuration: + algorithm: svs-vamana + datatype: float16 + graph_max_degree: 64 + construction_window_size: 300 + compression: LeanVec4x8 + reduce: 512 + search_window_size: 30 + + Estimated Memory Savings: 81.2% + + +## Creating an SVS-VAMANA Index + +Let's create an index using the recommended configuration. We'll use a simple schema with text content and vector embeddings. + + +```python +# Create index schema with recommended SVS-VAMANA configuration +schema = { + "index": { + "name": "svs_demo", + "prefix": "doc", + }, + "fields": [ + {"name": "content", "type": "text"}, + {"name": "category", "type": "tag"}, + { + "name": "embedding", + "type": "vector", + "attrs": { + "dims": dims, + **config, # Use the recommended configuration + "distance_metric": "cosine" + } + } + ] +} + +# Create the index +index = SearchIndex.from_dict(schema, redis_url=REDIS_URL) +index.create(overwrite=True) + +print(f"✅ Created SVS-VAMANA index: {index.name}") +print(f" Algorithm: {config['algorithm']}") +print(f" Compression: {config['compression']}") +print(f" Dimensions: {dims}") +if 'reduce' in config: + print(f" Reduced to: {config['reduce']} dimensions") +``` + + ✅ Created SVS-VAMANA index: svs_demo + Algorithm: svs-vamana + Compression: LeanVec4x8 + Dimensions: 1024 + Reduced to: 512 dimensions + + +## Loading Sample Data + +Let's create some sample documents with embeddings to demonstrate SVS-VAMANA search capabilities. + + +```python +# Generate sample data +sample_documents = [ + {"content": "Machine learning algorithms for data analysis", "category": "technology"}, + {"content": "Natural language processing and text understanding", "category": "technology"}, + {"content": "Computer vision and image recognition systems", "category": "technology"}, + {"content": "Delicious pasta recipes from Italy", "category": "food"}, + {"content": "Traditional French cooking techniques", "category": "food"}, + {"content": "Healthy meal planning and nutrition", "category": "food"}, + {"content": "Travel guide to European destinations", "category": "travel"}, + {"content": "Adventure hiking in mountain regions", "category": "travel"}, + {"content": "Cultural experiences in Asian cities", "category": "travel"}, + {"content": "Financial planning for retirement", "category": "finance"}, +] + +# Generate random embeddings for demonstration +# In practice, you would use a real embedding model +data_to_load = [] + +# Use reduced dimensions if LeanVec compression is applied +vector_dims = config.get("reduce", dims) +print(f"Creating vectors with {vector_dims} dimensions (reduced from {dims} if applicable)") + +for i, doc in enumerate(sample_documents): + # Create a random vector with some category-based clustering + base_vector = np.random.random(vector_dims).astype(np.float32) + + # Add some category-based similarity (optional, for demo purposes) + category_offset = hash(doc["category"]) % 100 / 1000.0 + base_vector[0] += category_offset + + # Convert to the datatype specified in config + if config["datatype"] == "float16": + base_vector = base_vector.astype(np.float16) + + data_to_load.append({ + "content": doc["content"], + "category": doc["category"], + "embedding": array_to_buffer(base_vector, dtype=config["datatype"]) + }) + +# Load data into the index +index.load(data_to_load) +print(f"✅ Loaded {len(data_to_load)} documents into the index") + +# Wait a moment for indexing to complete +import time +time.sleep(2) + +# Verify the data was loaded +info = index.info() +print(f" Index now contains {info.get('num_docs', 0)} documents") +``` + + Creating vectors with 512 dimensions (reduced from 1024 if applicable) + ✅ Loaded 10 documents into the index + Index now contains 0 documents + + +## Performing Vector Searches + +Now let's perform some vector similarity searches using our SVS-VAMANA index. + + +```python +# Create a query vector (in practice, this would be an embedding of your query text) +# Important: Query vector must match the index datatype and dimensions +vector_dims = config.get("reduce", dims) +if config["datatype"] == "float16": + query_vector = np.random.random(vector_dims).astype(np.float16) +else: + query_vector = np.random.random(vector_dims).astype(np.float32) + +# Perform a vector similarity search +query = VectorQuery( + vector=query_vector.tolist(), + vector_field_name="embedding", + return_fields=["content", "category"], + num_results=5 +) + +results = index.query(query) + +print("🔍 Vector Search Results:") +print("=" * 50) +for i, result in enumerate(results, 1): + distance = result.get('vector_distance', 'N/A') + print(f"{i}. [{result['category']}] {result['content']}") + print(f" Distance: {distance:.4f}" if isinstance(distance, (int, float)) else f" Distance: {distance}") + print() +``` + + 🔍 Vector Search Results: + ================================================== + + +## Understanding Compression Types + +SVS-VAMANA supports different compression algorithms that trade off between memory usage and search quality. Let's explore the available options. + + +```python +# Compare different compression priorities +print("Compression Recommendations for Different Priorities:") +print("=" * 60) + +priorities = ["memory", "speed", "balanced"] +for priority in priorities: + config = CompressionAdvisor.recommend(dims=dims, priority=priority) + savings = CompressionAdvisor.estimate_memory_savings( + config["compression"], + dims, + config.get("reduce") + ) + + print(f"\n{priority.upper()} Priority:") + print(f" Compression: {config['compression']}") + print(f" Datatype: {config['datatype']}") + if "reduce" in config: + print(f" Dimensionality reduction: {dims} → {config['reduce']}") + print(f" Search window size: {config['search_window_size']}") + print(f" Memory savings: {savings}%") +``` + + Compression Recommendations for Different Priorities: + ============================================================ + + MEMORY Priority: + Compression: LeanVec4x8 + Datatype: float16 + Dimensionality reduction: 1024 → 512 + Search window size: 20 + Memory savings: 81.2% + + SPEED Priority: + Compression: LeanVec4x8 + Datatype: float16 + Dimensionality reduction: 1024 → 256 + Search window size: 40 + Memory savings: 90.6% + + BALANCED Priority: + Compression: LeanVec4x8 + Datatype: float16 + Dimensionality reduction: 1024 → 512 + Search window size: 30 + Memory savings: 81.2% + + +## Compression Types Explained + +SVS-VAMANA offers several compression algorithms: + +### LVQ (Learned Vector Quantization) +- **LVQ4**: 4 bits per dimension (87.5% memory savings) +- **LVQ4x4**: 8 bits per dimension (75% memory savings) +- **LVQ4x8**: 12 bits per dimension (62.5% memory savings) +- **LVQ8**: 8 bits per dimension (75% memory savings) + +### LeanVec (Compression + Dimensionality Reduction) +- **LeanVec4x8**: 12 bits per dimension + dimensionality reduction +- **LeanVec8x8**: 16 bits per dimension + dimensionality reduction + +The CompressionAdvisor automatically chooses the best compression type based on your vector dimensions and priority. + + +```python +# Demonstrate compression savings for different vector dimensions +test_dimensions = [384, 768, 1024, 1536, 3072] + +print("Memory Savings by Vector Dimension:") +print("=" * 50) +print(f"{'Dims':<6} {'Compression':<12} {'Savings':<8} {'Strategy'}") +print("-" * 50) + +for dims in test_dimensions: + config = CompressionAdvisor.recommend(dims=dims, priority="balanced") + savings = CompressionAdvisor.estimate_memory_savings( + config["compression"], + dims, + config.get("reduce") + ) + + strategy = "LeanVec" if dims >= 1024 else "LVQ" + print(f"{dims:<6} {config['compression']:<12} {savings:>6.1f}% {strategy}") +``` + + Memory Savings by Vector Dimension: + ================================================== + Dims Compression Savings Strategy + -------------------------------------------------- + 384 LVQ4x4 75.0% LVQ + 768 LVQ4x4 75.0% LVQ + 1024 LeanVec4x8 81.2% LeanVec + 1536 LeanVec4x8 81.2% LeanVec + 3072 LeanVec4x8 81.2% LeanVec + + +## Hybrid Queries with SVS-VAMANA + +SVS-VAMANA can be combined with other Redis search capabilities for powerful hybrid queries that filter by metadata while performing vector similarity search. + + +```python +# Perform a hybrid search: vector similarity + category filter +hybrid_query = VectorQuery( + vector=query_vector.tolist(), + vector_field_name="embedding", + return_fields=["content", "category"], + num_results=3 +) + +# Add a filter to only search within "technology" category +hybrid_query.set_filter("@category:{technology}") + +filtered_results = index.query(hybrid_query) + +print("🔍 Hybrid Search Results (Technology category only):") +print("=" * 55) +for i, result in enumerate(filtered_results, 1): + distance = result.get('vector_distance', 'N/A') + print(f"{i}. [{result['category']}] {result['content']}") + print(f" Distance: {distance:.4f}" if isinstance(distance, (int, float)) else f" Distance: {distance}") + print() +``` + + 🔍 Hybrid Search Results (Technology category only): + ======================================================= + + +## Performance Monitoring + +Let's examine the index statistics to understand the performance characteristics of our SVS-VAMANA index. + + +```python +# Get detailed index information +info = index.info() + +print("📊 Index Statistics:") +print("=" * 30) +print(f"Documents: {info.get('num_docs', 0)}") + +# Handle vector_index_sz_mb which might be a string +vector_size = info.get('vector_index_sz_mb', 0) +if isinstance(vector_size, str): + try: + vector_size = float(vector_size) + except ValueError: + vector_size = 0.0 +print(f"Vector index size: {vector_size:.2f} MB") + +# Handle total_indexing_time which might also be a string +indexing_time = info.get('total_indexing_time', 0) +if isinstance(indexing_time, str): + try: + indexing_time = float(indexing_time) + except ValueError: + indexing_time = 0.0 +print(f"Total indexing time: {indexing_time:.2f} seconds") + +# Calculate memory efficiency +if info.get('num_docs', 0) > 0 and vector_size > 0: + mb_per_doc = vector_size / info.get('num_docs', 1) + print(f"Memory per document: {mb_per_doc:.4f} MB") + + # Estimate for larger datasets + for scale in [1000, 10000, 100000]: + estimated_mb = mb_per_doc * scale + print(f"Estimated size for {scale:,} docs: {estimated_mb:.1f} MB") +else: + print("Memory efficiency calculation requires documents and vector index size > 0") +``` + + 📊 Index Statistics: + ============================== + Documents: 0 + Vector index size: 0.00 MB + Total indexing time: 1.58 seconds + Memory efficiency calculation requires documents and vector index size > 0 + + +## Manual Configuration (Advanced) + +For advanced users who want full control over SVS-VAMANA parameters, you can manually configure the algorithm instead of using CompressionAdvisor. + + +```python +# Example of manual SVS-VAMANA configuration +manual_schema = { + "index": { + "name": "svs_manual", + "prefix": "manual", + }, + "fields": [ + {"name": "content", "type": "text"}, + { + "name": "embedding", + "type": "vector", + "attrs": { + "dims": 768, + "algorithm": "svs-vamana", + "datatype": "float32", + "distance_metric": "cosine", + + # Graph construction parameters + "graph_max_degree": 64, # Higher = better recall, more memory + "construction_window_size": 300, # Higher = better quality, slower build + + # Search parameters + "search_window_size": 40, # Higher = better recall, slower search + + # Compression settings + "compression": "LVQ4x4", # Choose compression type + "training_threshold": 10000, # Min vectors before compression training + } + } + ] +} + +print("Manual SVS-VAMANA Configuration:") +print("=" * 40) +vector_attrs = manual_schema["fields"][1]["attrs"] +for key, value in vector_attrs.items(): + if key != "dims": # Skip dims as it's obvious + print(f" {key}: {value}") + +# Calculate memory savings for this configuration +manual_savings = CompressionAdvisor.estimate_memory_savings( + "LVQ4x4", 768, None +) +print(f"\nEstimated memory savings: {manual_savings}%") +``` + + Manual SVS-VAMANA Configuration: + ======================================== + algorithm: svs-vamana + datatype: float32 + distance_metric: cosine + graph_max_degree: 64 + construction_window_size: 300 + search_window_size: 40 + compression: LVQ4x4 + training_threshold: 10000 + + Estimated memory savings: 75.0% + + +## Best Practices and Tips + +### When to Use SVS-VAMANA +- **Large datasets** (>10K vectors) where memory efficiency matters +- **High-dimensional vectors** (>512 dimensions) that benefit from compression +- **Applications** that can tolerate slight recall trade-offs for speed and memory savings + +### Parameter Tuning Guidelines +- **Start with CompressionAdvisor** recommendations +- **Increase search_window_size** if you need higher recall +- **Use LeanVec** for high-dimensional vectors (≥1024 dims) +- **Use LVQ** for lower-dimensional vectors (<1024 dims) + +### Performance Considerations +- **Index build time** increases with higher construction_window_size +- **Search latency** increases with higher search_window_size +- **Memory usage** decreases with more aggressive compression +- **Recall quality** may decrease with more aggressive compression + +## Cleanup + +Clean up the indices created in this demo. + + +```python +# Clean up demo indices +try: + index.delete() + print("Cleaned up svs_demo index") +except: + print("- svs_demo index was already deleted or doesn't exist") +``` + + Cleaned up svs_demo index +