Add batching of tenants.create and tenants.update with serial method invocations

Author: tsmith023

.github/workflows/main.yaml

@@ -7,9 +7,9 @@ on:
   pull_request:
 
 env:
-  WEAVIATE_124: 1.24.20
-  WEAVIATE_125: 1.25.7
-  WEAVIATE_126: preview--3aca6c5
+  WEAVIATE_124: 1.24.21
+  WEAVIATE_125: 1.25.8
+  WEAVIATE_126: preview--caba86b
 
 jobs:
   checks:

src/collections/serialize/index.ts

@@ -1329,7 +1329,19 @@ export class Serialize {
     });
   };
 
-  public static tenantsCreate(tenant: TenantBC | TenantCreate): {
+  public static tenants<T, M>(tenants: T[], mapper: (tenant: T) => M): M[][] {
+    const mapped = [];
+    const batches = Math.ceil(tenants.length / 100);
+    for (let i = 0; i < batches; i++) {
+      const batch = tenants.slice(i * 100, (i + 1) * 100);
+      mapped.push(batch.map(mapper));
+    }
+    return mapped;
+  }
+
+  public static tenantCreate<T extends TenantBC | TenantCreate>(
+    tenant: T
+  ): {
     name: string;
     activityStatus?: 'HOT' | 'COLD';
   } {
@@ -1361,9 +1373,9 @@ export class Serialize {
     };
   }
 
-  public static tenantUpdate = (
-    tenant: TenantBC | TenantUpdate
-  ): { name: string; activityStatus: 'HOT' | 'COLD' | 'FROZEN' } => {
+  public static tenantUpdate<T extends TenantBC | TenantUpdate>(
+    tenant: T
+  ): { name: string; activityStatus: 'HOT' | 'COLD' | 'FROZEN' } {
     let activityStatus: 'HOT' | 'COLD' | 'FROZEN';
     switch (tenant.activityStatus) {
       case 'ACTIVE':
@@ -1389,5 +1401,5 @@ export class Serialize {
       name: tenant.name,
       activityStatus,
     };
-  };
+  }
 }

src/collections/tenants/index.ts

@@ -44,10 +44,16 @@ const tenants = (
       return result;
     });
   return {
-    create: (tenants: TenantBC | TenantCreate | (TenantBC | TenantCreate)[]) =>
-      new TenantsCreator(connection, collection, parseValueOrValueArray(tenants).map(Serialize.tenantsCreate))
-        .do()
-        .then((res) => res.map(parseTenantREST)),
+    create: async (tenants: TenantBC | TenantCreate | (TenantBC | TenantCreate)[]) => {
+      const out: Tenant[] = [];
+      for await (const res of Serialize.tenants(parseValueOrValueArray(tenants), Serialize.tenantCreate).map(
+        (tenants) =>
+          new TenantsCreator(connection, collection, tenants).do().then((res) => res.map(parseTenantREST))
+      )) {
+        out.push(...res);
+      }
+      return out;
+    },
     get: async function () {
       const check = await dbVersionSupport.supportsTenantsGetGRPCMethod();
       return check.supports ? getGRPC() : getREST();
@@ -63,10 +69,16 @@ const tenants = (
         collection,
         parseValueOrValueArray(tenants).map(parseStringOrTenant)
       ).do(),
-    update: (tenants: TenantBC | TenantUpdate | (TenantBC | TenantUpdate)[]) =>
-      new TenantsUpdater(connection, collection, parseValueOrValueArray(tenants).map(Serialize.tenantUpdate))
-        .do()
-        .then((res) => res.map(parseTenantREST)),
+    update: async (tenants: TenantBC | TenantUpdate | (TenantBC | TenantUpdate)[]) => {
+      const out: Tenant[] = [];
+      for await (const res of Serialize.tenants(parseValueOrValueArray(tenants), Serialize.tenantUpdate).map(
+        (tenants) =>
+          new TenantsUpdater(connection, collection, tenants).do().then((res) => res.map(parseTenantREST))
+      )) {
+        out.push(...res);
+      }
+      return out;
+    },
   };
 };
 

src/collections/tenants/integration.test.ts

@@ -160,4 +160,22 @@ describe('Testing of the collection.tenants methods', () => {
       expect(result).not.toHaveProperty('non-existing');
     });
   });
+
+  it('should be able to create and update 1000 tenants', async () => {
+    const howManyToAdd = 1000;
+    const howManyPreExisting = Object.entries(await collection.tenants.get()).length;
+    const howMany = howManyToAdd + howManyPreExisting;
+    const tenants = Array.from({ length: howManyToAdd }, (_, i) => ({
+      name: `tenant-${i}`,
+    }));
+    await collection.tenants.create(tenants);
+    const getTenants = await collection.tenants.get();
+    expect(Object.entries(getTenants).length).toBe(howMany);
+    expect(Object.values(getTenants).every((tenant) => tenant.activityStatus === 'ACTIVE')).toBe(true);
+    const updated = await collection.tenants.update(
+      Object.values(getTenants).map((tenant) => ({ name: tenant.name, activityStatus: 'INACTIVE' }))
+    );
+    expect(Object.entries(updated).length).toBe(howMany);
+    expect(Object.values(updated).every((tenant) => tenant.activityStatus === 'INACTIVE')).toBe(true);
+  });
 });