Merge pull request #104 from semi-technologies/bm25_hybrid

Add Bm25 search and hybrid search

Author: aliszka

ci/docker-compose-okta.yml

@@ -10,7 +10,7 @@ services:
       - --scheme
       - http
       - --write-timeout=600s
-    image: semitechnologies/weaviate:1.15.4-b7811d4
+    image: semitechnologies/weaviate:1.17.0-prealpha-8950d6f
     ports:
       - 8082:8082
     restart: on-failure:0

ci/docker-compose-wcs.yml

@@ -10,7 +10,7 @@ services:
       - --scheme
       - http
       - --write-timeout=600s
-    image: semitechnologies/weaviate:preview-replace-shardingconfig-replicas-with-replication-factor-29e987d
+    image: semitechnologies/weaviate:1.17.0-prealpha-8950d6f
     ports:
       - 8083:8083
     restart: on-failure:0

ci/docker-compose.yml

@@ -2,7 +2,7 @@
 version: '3.4'
 services:
   weaviate:
-    image: semitechnologies/weaviate:preview-replace-shardingconfig-replicas-with-replication-factor-29e987d
+    image: semitechnologies/weaviate:1.17.0-prealpha-8950d6f
     restart: on-failure:0
     ports:
       - "8080:8080"

cluster/journey.test.js

@@ -2,7 +2,7 @@ const weaviate = require("../index");
 const { createTestFoodSchemaAndData, cleanupTestFood, PIZZA_CLASS_NAME, SOUP_CLASS_NAME } = require("../utils/testData");
 
 const EXPECTED_WEAVIATE_VERSION = "1.17.0-prealpha"
-const EXPECTED_WEAVIATE_GIT_HASH = "29e987d"
+const EXPECTED_WEAVIATE_GIT_HASH = "8950d6f"
 
 describe("cluster nodes endpoint", () => {
   const client = weaviate.client({

graphql/bm25.js

@@ -0,0 +1,58 @@
+import { isValidStringArray, isValidStringProperty } from "../validation/string";
+
+export default class GraphQLBm25 {
+
+  constructor(bm25Obj) {
+    this.source = bm25Obj;
+  }
+
+  toString() {
+    this.parse();
+    this.validate();
+
+    let args = [`query:${JSON.stringify(this.query)}`]; // query must always be set
+
+    if (this.properties !== undefined) {
+      args = [...args, `properties:${JSON.stringify(this.properties)}`];
+    }
+
+    return `{${args.join(",")}}`;
+  }
+
+  parse() {
+    for (let key in this.source) {
+      switch (key) {
+        case "query":
+          this.parseQuery(this.source[key]);
+          break;
+        case "properties":
+          this.parseProperties(this.source[key]);
+          break;
+        default:
+          throw new Error(`bm25 filter: unrecognized key '${key}'`);
+      }
+    }
+  }
+
+  parseQuery(query) {
+    if (!isValidStringProperty(query)) {
+      throw new Error("bm25 filter: query must be a string");
+    }
+
+    this.query = query;
+  }
+
+  parseProperties(properties) {
+    if (!isValidStringArray(properties)) {
+      throw new Error("bm25 filter: properties must be an array of strings");
+    }
+
+    this.properties = properties;
+  }
+
+  validate() {
+    if (!this.query) {
+      throw new Error("bm25 filter: query cannot be empty");
+    }
+  }
+}

graphql/getter.js

@@ -1,6 +1,8 @@
 import Where from "./where";
 import NearText from "./nearText";
 import NearVector from "./nearVector";
+import Bm25 from "./bm25";
+import Hybrid from "./hybrid";
 import NearObject from "./nearObject";
 import NearImage from "./nearImage";
 import Ask from "./ask";
@@ -60,6 +62,27 @@ export default class Getter {
     return this;
   };
 
+  withBm25 = (bm25Obj) => {
+    try {
+      this.bm25String = new Bm25(bm25Obj).toString();
+    } catch (e) {
+      this.errors = [...this.errors, e];
+    }
+
+    return this;
+  };
+
+  withHybrid = (hybridObj) => {
+    try {
+      this.hybridString = new Hybrid(hybridObj).toString();
+    } catch (e) {
+      this.errors = [...this.errors, e];
+    }
+
+    return this;
+  };
+
+
   withNearObject = (nearObjectObj) => {
     if (this.includesNearMediaFilter) {
       throw new Error(
@@ -172,8 +195,10 @@ export default class Getter {
       this.nearTextString ||
       this.nearObjectString ||
       this.nearVectorString ||
-      this.askString ||
       this.nearImageString ||
+      this.askString ||
+      this.bm25String ||
+      this.hybridString ||
       this.limit ||
       this.offset ||
       this.groupString ||
@@ -205,6 +230,14 @@ export default class Getter {
         args = [...args, `nearVector:${this.nearVectorString}`];
       }
 
+      if (this.bm25String) {
+        args = [...args, `bm25:${this.bm25String}`];
+      }
+
+      if (this.hybridString) {
+        args = [...args, `hybrid:${this.hybridString}`];
+      }
+
       if (this.groupString) {
         args = [...args, `group:${this.groupString}`];
       }

graphql/getter.test.js

@@ -1377,3 +1377,231 @@ describe("invalid sort filters", () => {
     });
   });
 });
+
+describe("bm25 valid searchers", () => {
+  const mockClient = {
+    query: jest.fn(),
+  };
+
+  test("query and no properties", () => {
+    const expectedQuery =
+      `{Get{Person` + `(bm25:{query:"accountant"})` + `{name}}}`;
+
+    new Getter(mockClient)
+      .withClassName("Person")
+      .withFields("name")
+      .withBm25({ query: "accountant" })
+      .do();
+
+    expect(mockClient.query).toHaveBeenCalledWith(expectedQuery);
+  });
+
+  test("query and properties", () => {
+    const expectedQuery =
+      `{Get{Person` + `(bm25:{query:"accountant",properties:["profession","position"]})` + `{name}}}`;
+
+    new Getter(mockClient)
+      .withClassName("Person")
+      .withFields("name")
+      .withBm25({ query: "accountant", properties: ["profession", "position"] })
+      .do();
+
+    expect(mockClient.query).toHaveBeenCalledWith(expectedQuery);
+  });
+
+  test("query and empty properties", () => {
+    const expectedQuery =
+      `{Get{Person` + `(bm25:{query:"accountant",properties:[]})` + `{name}}}`;
+
+    new Getter(mockClient)
+      .withClassName("Person")
+      .withFields("name")
+      .withBm25({ query: "accountant", properties: [] })
+      .do();
+
+    expect(mockClient.query).toHaveBeenCalledWith(expectedQuery);
+  });
+});
+
+describe("bm25 invalid searchers", () => {
+  const mockClient = {
+    query: jest.fn(),
+  };
+
+  const tests = [
+    {
+      title: "an empty bm25",
+      bm25: {},
+      msg: "bm25 filter: query cannot be empty",
+    },
+    {
+      title: "an empty query",
+      bm25: { query: ""},
+      msg: "bm25 filter: query must be a string",
+    },
+    {
+      title: "query of wrong type",
+      bm25: { query: {} },
+      msg: "bm25 filter: query must be a string",
+    },
+    {
+      title: "an empty property",
+      bm25: { query: "query", properties: [""] },
+      msg: "bm25 filter: properties must be an array of strings",
+    },
+    {
+      title: "property of wrong type",
+      bm25: { query: "query", properties: [123] },
+      msg: "bm25 filter: properties must be an array of strings",
+    },
+    {
+      title: "properties of wrong type",
+      bm25: { query: "query", properties: {} },
+      msg: "bm25 filter: properties must be an array of strings",
+    },
+  ];
+
+  tests.forEach((t) => {
+    test(t.title, () => {
+      new Getter(mockClient)
+        .withClassName("Person")
+        .withFields("name")
+        .withBm25(t.bm25)
+        .do()
+        .then(() => fail("it should have error'd"))
+        .catch((e) => {
+          expect(e.toString()).toContain(t.msg);
+        });
+    });
+  });
+});
+
+
+describe("hybrid valid searchers", () => {
+  const mockClient = {
+    query: jest.fn(),
+  };
+
+  test("query and no alpha, no vector", () => {
+    const expectedQuery =
+      `{Get{Person` + `(hybrid:{query:"accountant"})` + `{name}}}`;
+
+    new Getter(mockClient)
+      .withClassName("Person")
+      .withFields("name")
+      .withHybrid({ query: "accountant" })
+      .do();
+
+    expect(mockClient.query).toHaveBeenCalledWith(expectedQuery);
+  });
+
+  test("query and alpha, no vector", () => {
+    const expectedQuery =
+      `{Get{Person` + `(hybrid:{query:"accountant",alpha:0.75})` + `{name}}}`;
+
+    new Getter(mockClient)
+      .withClassName("Person")
+      .withFields("name")
+      .withHybrid({ query: "accountant", alpha: 0.75 })
+      .do();
+
+    expect(mockClient.query).toHaveBeenCalledWith(expectedQuery);
+  });
+
+  test("query and alpha 0, no vector", () => {
+    const expectedQuery =
+      `{Get{Person` + `(hybrid:{query:"accountant",alpha:0})` + `{name}}}`;
+
+    new Getter(mockClient)
+      .withClassName("Person")
+      .withFields("name")
+      .withHybrid({ query: "accountant", alpha: 0 })
+      .do();
+
+    expect(mockClient.query).toHaveBeenCalledWith(expectedQuery);
+  });
+
+  test("query and vector, no alpha", () => {
+    const expectedQuery =
+      `{Get{Person` + `(hybrid:{query:"accountant",vector:[1,2,3]})` + `{name}}}`;
+
+    new Getter(mockClient)
+      .withClassName("Person")
+      .withFields("name")
+      .withHybrid({ query: "accountant", vector: [1,2,3] })
+      .do();
+
+    expect(mockClient.query).toHaveBeenCalledWith(expectedQuery);
+  });
+
+  test("query and alpha and vector", () => {
+    const expectedQuery =
+      `{Get{Person` + `(hybrid:{query:"accountant",alpha:0.75,vector:[1,2,3]})` + `{name}}}`;
+
+    new Getter(mockClient)
+      .withClassName("Person")
+      .withFields("name")
+      .withHybrid({ query: "accountant", alpha: 0.75, vector: [1,2,3] })
+      .do();
+
+    expect(mockClient.query).toHaveBeenCalledWith(expectedQuery);
+  });
+});
+
+describe("hybrid invalid searchers", () => {
+  const mockClient = {
+    query: jest.fn(),
+  };
+
+  const tests = [
+    {
+      title: "an empty hybrid",
+      hybrid: {},
+      msg: "hybrid filter: query cannot be empty",
+    },
+    {
+      title: "an empty query",
+      hybrid: { query: ""},
+      msg: "hybrid filter: query must be a string",
+    },
+    {
+      title: "query of wrong type",
+      hybrid: { query: {} },
+      msg: "hybrid filter: query must be a string",
+    },
+    {
+      title: "alpha on wrong type",
+      hybrid: { query: "query", alpha: "alpha" },
+      msg: "hybrid filter: alpha must be a number",
+    },
+    {
+      title: "an empty vector",
+      hybrid: { query: "query", vector: [] },
+      msg: "hybrid filter: vector must be an array of numbers",
+    },
+    {
+      title: "vector element of wrong type",
+      hybrid: { query: "query", vector: ["vector"] },
+      msg: "hybrid filter: vector must be an array of numbers",
+    },
+    {
+      title: "vector of wrong type",
+      hybrid: { query: "query", vector: {} },
+      msg: "hybrid filter: vector must be an array of numbers",
+    },
+  ];
+
+  tests.forEach((t) => {
+    test(t.title, () => {
+      new Getter(mockClient)
+        .withClassName("Person")
+        .withFields("name")
+        .withHybrid(t.hybrid)
+        .do()
+        .then(() => fail("it should have error'd"))
+        .catch((e) => {
+          expect(e.toString()).toContain(t.msg);
+        });
+    });
+  });
+});