Fixes the hybrid retriever query (#114)

* Fixes the hybrid retriever query

* Updated hybrid query to make it more readable

* Updated CHANGELOG

Author: alexthomas93

CHANGELOG.md

@@ -2,6 +2,11 @@
 
 ## Next
 
+## 0.5.0
+
+### Fixed
+- Corrected the hybrid retriever query to ensure proper normalization of scores in vector search results.
+
 ## 0.4.0
 
 ### Added

src/neo4j_genai/neo4j_queries.py

@@ -53,12 +53,14 @@
 def _get_hybrid_query() -> str:
     return (
         f"CALL {{ {VECTOR_INDEX_QUERY} "
-        f"RETURN node, score "
+        f"WITH collect({{node:node, score:score}}) AS nodes, max(score) AS vector_index_max_score "
+        f"UNWIND nodes AS n "
+        f"RETURN n.node AS node, (n.score / vector_index_max_score) AS score "
         f"UNION "
         f"{FULL_TEXT_SEARCH_QUERY} "
-        f"WITH collect({{node:node, score:score}}) AS nodes, max(score) AS max "
+        f"WITH collect({{node:node, score:score}}) AS nodes, max(score) AS ft_index_max_score "
         f"UNWIND nodes AS n "
-        f"RETURN n.node AS node, (n.score / max) AS score }} "
+        f"RETURN n.node AS node, (n.score / ft_index_max_score) AS score }} "
         f"WITH node, max(score) AS score ORDER BY score DESC LIMIT $top_k"
     )
 

tests/unit/test_neo4j_queries.py

@@ -35,12 +35,14 @@ def test_hybrid_search_basic() -> None:
         "CALL { "
         "CALL db.index.vector.queryNodes($vector_index_name, $top_k, $query_vector) "
         "YIELD node, score "
-        "RETURN node, score UNION "
+        "WITH collect({node:node, score:score}) AS nodes, max(score) AS vector_index_max_score "
+        "UNWIND nodes AS n "
+        "RETURN n.node AS node, (n.score / vector_index_max_score) AS score UNION "
         "CALL db.index.fulltext.queryNodes($fulltext_index_name, $query_text, {limit: $top_k}) "
         "YIELD node, score "
-        "WITH collect({node:node, score:score}) AS nodes, max(score) AS max "
+        "WITH collect({node:node, score:score}) AS nodes, max(score) AS ft_index_max_score "
         "UNWIND nodes AS n "
-        "RETURN n.node AS node, (n.score / max) AS score "
+        "RETURN n.node AS node, (n.score / ft_index_max_score) AS score "
         "} "
         "WITH node, max(score) AS score ORDER BY score DESC LIMIT $top_k "
         "RETURN node { .*, `None`: null } AS node, labels(node) AS nodeLabels, elementId(node) AS id, score"
@@ -125,12 +127,14 @@ def test_hybrid_search_with_retrieval_query() -> None:
         "CALL { "
         "CALL db.index.vector.queryNodes($vector_index_name, $top_k, $query_vector) "
         "YIELD node, score "
-        "RETURN node, score UNION "
+        "WITH collect({node:node, score:score}) AS nodes, max(score) AS vector_index_max_score "
+        "UNWIND nodes AS n "
+        "RETURN n.node AS node, (n.score / vector_index_max_score) AS score UNION "
         "CALL db.index.fulltext.queryNodes($fulltext_index_name, $query_text, {limit: $top_k}) "
         "YIELD node, score "
-        "WITH collect({node:node, score:score}) AS nodes, max(score) AS max "
+        "WITH collect({node:node, score:score}) AS nodes, max(score) AS ft_index_max_score "
         "UNWIND nodes AS n "
-        "RETURN n.node AS node, (n.score / max) AS score "
+        "RETURN n.node AS node, (n.score / ft_index_max_score) AS score "
         "} "
         "WITH node, max(score) AS score ORDER BY score DESC LIMIT $top_k "
         + retrieval_query
@@ -145,12 +149,14 @@ def test_hybrid_search_with_properties() -> None:
         "CALL { "
         "CALL db.index.vector.queryNodes($vector_index_name, $top_k, $query_vector) "
         "YIELD node, score "
-        "RETURN node, score UNION "
+        "WITH collect({node:node, score:score}) AS nodes, max(score) AS vector_index_max_score "
+        "UNWIND nodes AS n "
+        "RETURN n.node AS node, (n.score / vector_index_max_score) AS score UNION "
         "CALL db.index.fulltext.queryNodes($fulltext_index_name, $query_text, {limit: $top_k}) "
         "YIELD node, score "
-        "WITH collect({node:node, score:score}) AS nodes, max(score) AS max "
+        "WITH collect({node:node, score:score}) AS nodes, max(score) AS ft_index_max_score "
         "UNWIND nodes AS n "
-        "RETURN n.node AS node, (n.score / max) AS score "
+        "RETURN n.node AS node, (n.score / ft_index_max_score) AS score "
         "} "
         "WITH node, max(score) AS score ORDER BY score DESC LIMIT $top_k "
         "RETURN node {.name, .age} as node, score"