feat(compass-collection): Avoid sending sample values for binary fields - CLOUDP-350484 (#7439)

* Skip binary for sample values

* Comment

* WIP

Author: jcobis

packages/compass-collection/src/transform-schema-to-field-info.spec.ts

@@ -148,7 +148,7 @@ describe('processSchema', function () {
     expect(result.arrayLengthMap).to.deep.equal({});
   });
 
-  it('limits sample values to 10', function () {
+  it('limits sample values to 5', function () {
     const manyValues = Array.from({ length: 20 }, (_, i) => `value${i}`);
 
     const schema: Schema = {
@@ -177,9 +177,9 @@ describe('processSchema', function () {
 
     const result = processSchema(schema);
 
-    expect(result.fieldInfo.field.sampleValues).to.have.length(10);
+    expect(result.fieldInfo.field.sampleValues).to.have.length(5);
     expect(result.fieldInfo.field.sampleValues).to.deep.equal(
-      manyValues.slice(0, 10)
+      manyValues.slice(0, 5)
     );
   });
 
@@ -486,7 +486,7 @@ describe('processSchema', function () {
       },
       binary: {
         type: 'Binary',
-        sampleValues: ['dGVzdA=='],
+        // sampleValues property should be absent for Binary fields
         probability: 1.0,
       },
       regex: {
@@ -533,6 +533,132 @@ describe('processSchema', function () {
     expect(result.arrayLengthMap).to.deep.equal({});
   });
 
+  it('excludes sample values for Binary fields to avoid massive payloads', function () {
+    const embedding = new Binary(Buffer.from([1, 2, 3, 4])); // Test Binary field logic
+    const schema: Schema = {
+      fields: [
+        {
+          name: 'plot_embedding',
+          path: ['plot_embedding'],
+          count: 1,
+          type: ['Binary'],
+          probability: 1.0,
+          hasDuplicates: false,
+          types: [
+            {
+              name: 'Binary',
+              bsonType: 'Binary',
+              path: ['plot_embedding'],
+              count: 1,
+              probability: 1.0,
+              values: [embedding],
+            },
+          ],
+        },
+        {
+          name: 'regular_field',
+          path: ['regular_field'],
+          count: 1,
+          type: ['String'],
+          probability: 1.0,
+          hasDuplicates: false,
+          types: [
+            {
+              name: 'String',
+              bsonType: 'String',
+              path: ['regular_field'],
+              count: 1,
+              probability: 1.0,
+              values: ['test'],
+            },
+          ],
+        },
+      ],
+      count: 1,
+    };
+
+    const result = processSchema(schema);
+
+    expect(result.fieldInfo).to.deep.equal({
+      plot_embedding: {
+        type: 'Binary',
+        // sampleValues property should be absent for Binary fields
+        probability: 1.0,
+      },
+      regular_field: {
+        type: 'String',
+        sampleValues: ['test'], // Should still have sample values for non-Binary fields
+        probability: 1.0,
+      },
+    });
+    expect(result.arrayLengthMap).to.deep.equal({});
+  });
+
+  it('truncates very long sample values to prevent massive payloads', function () {
+    const longText = 'A'.repeat(1000);
+    const schema: Schema = {
+      fields: [
+        {
+          name: 'longField',
+          path: ['longField'],
+          count: 1,
+          type: ['String'],
+          probability: 1.0,
+          hasDuplicates: false,
+          types: [
+            {
+              name: 'String',
+              bsonType: 'String',
+              path: ['longField'],
+              count: 1,
+              probability: 1.0,
+              values: [longText, 'short'],
+            },
+          ],
+        },
+      ],
+      count: 1,
+    };
+
+    const result = processSchema(schema);
+
+    expect(result.fieldInfo.longField.sampleValues).to.have.length(2);
+    expect(result.fieldInfo.longField.sampleValues![0]).to.equal(
+      'A'.repeat(300) + '...'
+    );
+    expect(result.fieldInfo.longField.sampleValues![1]).to.equal('short');
+  });
+
+  it('rounds probability to 2 decimal places', function () {
+    const schema: Schema = {
+      fields: [
+        {
+          name: 'field',
+          path: ['field'],
+          count: 1,
+          type: ['String'],
+          probability: 0.23076923076923078, // Very precise decimal
+          hasDuplicates: false,
+          types: [
+            {
+              name: 'String',
+              bsonType: 'String',
+              path: ['field'],
+              count: 1,
+              probability: 1.0,
+              values: ['test'],
+            },
+          ],
+        },
+      ],
+      count: 1,
+    };
+
+    const result = processSchema(schema);
+
+    expect(result.fieldInfo.field.probability).to.equal(0.23); // Rounded to 2 decimal places
+  });
+
   it('transforms nested document field', function () {
     const schema: Schema = {
       fields: [
@@ -908,7 +1034,7 @@ describe('processSchema', function () {
     expect(result.fieldInfo).to.deep.equal({
       'cube[][][]': {
         type: 'Number',
-        sampleValues: [1, 2, 3, 4, 5, 6, 7, 8],
+        sampleValues: [1, 2, 3, 4, 5],
         probability: 1.0,
       },
     });

packages/compass-collection/src/transform-schema-to-field-info.ts

@@ -43,7 +43,14 @@ import {
 /**
  * Maximum number of sample values to include for each field
  */
-const MAX_SAMPLE_VALUES = 10;
+const MAX_SAMPLE_VALUES = 5;
+
+/**
+ * Maximum length for individual sample values (to prevent massive payloads)
+ * 300 chars allows for meaningful text samples while keeping payloads manageable
+ */
+const MAX_STRING_SAMPLE_VALUE_LENGTH = 300;
+
 export const FIELD_NAME_SEPARATOR = '.';
 
 /**
@@ -113,7 +120,11 @@ function convertBSONToPrimitive(value: unknown): SampleValue {
     return value.toString();
   }
   if (value instanceof Binary) {
-    return value.toString('base64');
+    // For Binary data, provide a descriptive placeholder instead of the actual data
+    // to avoid massive base64 strings that can break LLM requests
+    // (Defensive check: should never be called, since sample values for binary are skipped)
+    const sizeInBytes = value.buffer?.length || 0;
+    return `<Binary data: ${sizeInBytes} bytes>`;
   }
   if (value instanceof BSONRegExp) {
     return value.pattern;
@@ -146,6 +157,14 @@ function convertBSONToPrimitive(value: unknown): SampleValue {
     return result as SampleValue;
   }
 
+  // Truncate very long strings to prevent massive payloads
+  if (
+    typeof value === 'string' &&
+    value.length > MAX_STRING_SAMPLE_VALUE_LENGTH
+  ) {
+    return value.substring(0, MAX_STRING_SAMPLE_VALUE_LENGTH) + '...';
+  }
+
   return value as SampleValue;
 }
 
@@ -229,7 +248,7 @@ function processNamedField(
     primaryType,
     currentPath,
     result,
-    field.probability,
+    Math.round(field.probability * 100) / 100, // Round to 2 decimal places
     arrayLengthMap
   );
 }
@@ -281,12 +300,16 @@ function processType(
     // Primitive: Create entry
     const fieldInfo: FieldInfo = {
       type: type.name,
-      sampleValues: type.values
-        .slice(0, MAX_SAMPLE_VALUES)
-        .map(convertBSONToPrimitive),
       probability: fieldProbability,
     };
 
+    // Only add sampleValues if not Binary (to avoid massive payloads from embeddings)
+    if (type.name !== 'Binary') {
+      fieldInfo.sampleValues = type.values
+        .slice(0, MAX_SAMPLE_VALUES)
+        .map(convertBSONToPrimitive);
+    }
+
     result[currentPath] = fieldInfo;
   }
 }