chore(modelarmor): enabled floor settings tests (#4123)

* chore(tests): enabled floor settings tests

* chore(tests): added floor setting enable flag exists on get tests

* chore(tests): enabled floor settings clean-up

* removed empty template related tests

As Katie mentioned earlier in C# repo that service level tests already exists, so anyways these tests won't help. Hence, removing them

* chore: update dependencies and improve floor settings tests

* ci: move custard config to same level as package.json

* ci: may not support ints here, use strings

---------

Co-authored-by: Harsh Nasit <131268456+harshnasitcrest@users.noreply.github.com>
Co-authored-by: Katie McLaughlin <katie@glasnt.com>
Co-authored-by: Harsh Nasit <harsh.nasit@crestdata.ai>

Author: 3 people

model-armor/ci-setup.json

@@ -0,0 +1,6 @@
+{
+    "env": {
+        "MA_FOLDER_ID": "695279264361",
+        "MA_ORG_ID": "951890214235"
+    }
+}

model-armor/package.json

@@ -14,13 +14,13 @@
     "test": "c8 mocha -p -j 2 --recursive test/ --timeout=60000"
   },
   "dependencies": {
-    "@google-cloud/modelarmor": "^0.1.0",
-    "@google-cloud/dlp": "^5.0.0"
+    "@google-cloud/modelarmor": "^0.4.0",
+    "@google-cloud/dlp": "^6.3.0"
   },
   "devDependencies": {
     "c8": "^10.0.0",
-    "chai": "^4.5.0",
-    "mocha": "^10.0.0",
-    "uuid": "^10.0.0"
+    "chai": "^5.2.1",
+    "mocha": "^11.7.1",
+    "uuid": "^11.1.0"
   }
 }

model-armor/snippets/ci-setup.json

@@ -57,7 +57,8 @@ async function main(folderId) {
                 protos.google.cloud.modelarmor.v1.RaiFilterType
                   .SEXUALLY_EXPLICIT,
               confidenceLevel:
-                protos.google.cloud.modelarmor.v1.DetectionConfidenceLevel.HIGH,
+                protos.google.cloud.modelarmor.v1.DetectionConfidenceLevel
+                  .LOW_AND_ABOVE,
             },
           ],
         },

model-armor/snippets/updateFolderFloorSettings.js

@@ -55,7 +55,8 @@ async function main(organizationId) {
                 protos.google.cloud.modelarmor.v1.RaiFilterType
                   .SEXUALLY_EXPLICIT,
               confidenceLevel:
-                protos.google.cloud.modelarmor.v1.DetectionConfidenceLevel.HIGH,
+                protos.google.cloud.modelarmor.v1.DetectionConfidenceLevel
+                  .LOW_AND_ABOVE,
             },
           ],
         },

model-armor/snippets/updateOrganizationFloorSettings.js

@@ -56,7 +56,8 @@ async function main(projectId) {
                 protos.google.cloud.modelarmor.v1.RaiFilterType
                   .SEXUALLY_EXPLICIT,
               confidenceLevel:
-                protos.google.cloud.modelarmor.v1.DetectionConfidenceLevel.HIGH,
+                protos.google.cloud.modelarmor.v1.DetectionConfidenceLevel
+                  .LOW_AND_ABOVE,
             },
           ],
         },

model-armor/snippets/updateProjectFloorSettings.js

@@ -30,7 +30,6 @@ const options = {
 const client = new ModelArmorClient(options);
 const templateIdPrefix = `test-template-${uuidv4().substring(0, 8)}`;
 
-let emptyTemplateId;
 let basicTemplateId;
 let basicSdpTemplateId;
 let advanceSdpTemplateId;
@@ -261,6 +260,7 @@ describe('Model Armor tests', () => {
   before(async () => {
     projectId = await client.getProjectId();
     const {protos} = require('@google-cloud/modelarmor');
+
     // Import necessary enums
     const DetectionConfidenceLevel =
       protos.google.cloud.modelarmor.v1.DetectionConfidenceLevel;
@@ -275,9 +275,7 @@ describe('Model Armor tests', () => {
         .SdpBasicConfigEnforcement;
     const RaiFilterType = protos.google.cloud.modelarmor.v1.RaiFilterType;
 
-    // Create empty template for sanitizeUserPrompt tests
-    emptyTemplateId = `${templateIdPrefix}-empty`;
-    await createTemplate(emptyTemplateId, {});
+    await disableFloorSettings();
 
     // Create basic template with PI/Jailbreak and Malicious URI filters for sanitizeUserPrompt tests
     basicTemplateId = `${templateIdPrefix}-basic`;
@@ -360,7 +358,6 @@ describe('Model Armor tests', () => {
     });
 
     templatesToDelete.push(
-      `projects/${projectId}/locations/${locationId}/templates/${emptyTemplateId}`,
       `projects/${projectId}/locations/${locationId}/templates/${basicTemplateId}`,
       `projects/${projectId}/locations/${locationId}/templates/${basicSdpTemplateId}`,
       `projects/${projectId}/locations/${locationId}/templates/${advanceSdpTemplateId}`,
@@ -371,10 +368,6 @@ describe('Model Armor tests', () => {
 
   after(async () => {
     for (const templateName of templatesToDelete) {
-      // TODO(b/424365799): Uncomment below code once the mentioned issue is resolved
-      // Disable floor settings to restore original state
-      // await disableFloorSettings();
-
       await deleteTemplate(templateName);
     }
 
@@ -728,24 +721,6 @@ describe('Model Armor tests', () => {
     );
   });
 
-  it('should not detect issues in user prompt with empty template', async () => {
-    const sanitizeUserPrompt = require('../snippets/sanitizeUserPrompt');
-    const testUserPrompt =
-      'Can you describe this link? https://testsafebrowsing.appspot.com/s/malware.html,';
-
-    const response = await sanitizeUserPrompt(
-      projectId,
-      locationId,
-      emptyTemplateId,
-      testUserPrompt
-    );
-
-    assert.equal(
-      response.sanitizationResult.filterMatchState,
-      'NO_MATCH_FOUND'
-    );
-  });
-
   // =================== Model Response Sanitization Tests ===================
 
   it('should detect malicious URL in model response', async () => {
@@ -826,24 +801,6 @@ describe('Model Armor tests', () => {
     );
   });
 
-  it('should not detect issues in model response with empty template', async () => {
-    const sanitizeModelResponse = require('../snippets/sanitizeModelResponse');
-    const testModelResponse =
-      'For following email 1l6Y2@example.com found following associated phone number: 954-321-7890 and this ITIN: 988-86-1234';
-
-    const response = await sanitizeModelResponse(
-      projectId,
-      locationId,
-      emptyTemplateId,
-      testModelResponse
-    );
-
-    assert.equal(
-      response.sanitizationResult.filterMatchState,
-      'NO_MATCH_FOUND'
-    );
-  });
-
   it('should detect PII in model response with basic SDP template', async () => {
     const sanitizeModelResponse = require('../snippets/sanitizeModelResponse');
     const testModelResponse =
@@ -874,27 +831,6 @@ describe('Model Armor tests', () => {
 
   // =================== Model Response with User Prompt Tests ===================
 
-  it('should not detect issues in model response with user prompt using empty template', async () => {
-    const sanitizeModelResponseWithUserPrompt = require('../snippets/sanitizeModelResponseWithUserPrompt');
-    const testUserPrompt =
-      'How can I make my email address test@dot.com make available to public for feedback';
-    const testModelResponse =
-      'You can make support email such as contact@email.com for getting feedback from your customer';
-
-    const response = await sanitizeModelResponseWithUserPrompt(
-      projectId,
-      locationId,
-      emptyTemplateId,
-      testModelResponse,
-      testUserPrompt
-    );
-
-    assert.equal(
-      response.sanitizationResult.filterMatchState,
-      'NO_MATCH_FOUND'
-    );
-  });
-
   it('should sanitize model response with user prompt using advanced SDP template', async () => {
     const sanitizeModelResponseWithUserPrompt = require('../snippets/sanitizeModelResponseWithUserPrompt');
     const testUserPrompt =
@@ -937,57 +873,66 @@ describe('Model Armor tests', () => {
       'NO_MATCH_FOUND'
     );
   });
+});
+
+describe('Model Armor floor setting tests', () => {
+  before(async () => {
+    projectId = await client.getProjectId();
+  });
+
+  after(async () => {
+    await disableFloorSettings();
+  });
 
-  // =================== Floor Settings Tests ===================
+  it('should update organization floor settings', async () => {
+    const updateOrganizationFloorSettings = require('../snippets/updateOrganizationFloorSettings');
+    const output = await updateOrganizationFloorSettings.main(organizationId);
+    // Check that the enableFloorSettingEnforcement=true
+    assert.equal(output.enableFloorSettingEnforcement, true);
+  });
 
-  // TODO(b/424365799): Enable below tests once the mentioned issue is resolved
+  it('should update folder floor settings', async () => {
+    const updateFolderFloorSettings = require('../snippets/updateFolderFloorSettings');
+    const output = await updateFolderFloorSettings.main(folderId);
+    // Check that the enableFloorSettingEnforcement=true
+    assert.equal(output.enableFloorSettingEnforcement, true);
+  });
 
-  it.skip('should get organization floor settings', async () => {
+  it('should update project floor settings', async () => {
+    const updateProjectFloorSettings = require('../snippets/updateProjectFloorSettings');
+    const output = await updateProjectFloorSettings.main(projectId);
+    // Check that the enableFloorSettingEnforcement=true
+    assert.equal(output.enableFloorSettingEnforcement, true);
+  });
+
+  it('should get organization floor settings', async () => {
     const getOrganizationFloorSettings = require('../snippets/getOrganizationFloorSettings');
 
     const output = await getOrganizationFloorSettings.main(organizationId);
 
     const expectedName = `organizations/${organizationId}/locations/global/floorSetting`;
     assert.equal(output.name, expectedName);
+    assert.exists(output.enableFloorSettingEnforcement);
   });
 
-  it.skip('should get folder floor settings', async () => {
+  it('should get folder floor settings', async () => {
     const getFolderFloorSettings = require('../snippets/getFolderFloorSettings');
 
     const output = await getFolderFloorSettings.main(folderId);
 
     // Check for expected name format in output
     const expectedName = `folders/${folderId}/locations/global/floorSetting`;
     assert.equal(output.name, expectedName);
+    assert.exists(output.enableFloorSettingEnforcement);
   });
 
-  it.skip('should get project floor settings', async () => {
+  it('should get project floor settings', async () => {
     const getProjectFloorSettings = require('../snippets/getProjectFloorSettings');
 
     const output = await getProjectFloorSettings.main(projectId);
     // Check for expected name format in output
     const expectedName = `projects/${projectId}/locations/global/floorSetting`;
     assert.equal(output.name, expectedName);
-  });
-
-  it.skip('should update organization floor settings', async () => {
-    const updateOrganizationFloorSettings = require('../snippets/updateOrganizationFloorSettings');
-    const output = await updateOrganizationFloorSettings.main(organizationId);
-    // Check that the enableFloorSettingEnforcement=true
-    assert.equal(output.enableFloorSettingEnforcement, true);
-  });
-
-  it.skip('should update folder floor settings', async () => {
-    const updateFolderFloorSettings = require('../snippets/updateFolderFloorSettings');
-    const output = await updateFolderFloorSettings.main(folderId);
-    // Check that the enableFloorSettingEnforcement=true
-    assert.equal(output.enableFloorSettingEnforcement, true);
-  });
-
-  it.skip('should update project floor settings', async () => {
-    const updateProjectFloorSettings = require('../snippets/updateProjectFloorSettings');
-    const output = await updateProjectFloorSettings.main(projectId);
-    // Check that the enableFloorSettingEnforcement=true
-    assert.equal(output.enableFloorSettingEnforcement, true);
+    assert.exists(output.enableFloorSettingEnforcement);
   });
 });