SERVER-42473 Create concurrency workload to validate current cleanupOrphaned command and range deleter

Author: alextaskov

buildscripts/resmokeconfig/suites/concurrency_sharded_causal_consistency.yml

@@ -110,6 +110,7 @@ selector:
 
   exclude_with_any_tags:
   - does_not_support_causal_consistency
+  - assumes_balancer_on
   # This suite uses secondary read preference, which isn't currently compatible with transactions.
   - uses_transactions
   - requires_replication

buildscripts/resmokeconfig/suites/concurrency_sharded_local_read_write_multi_stmt_txn.yml

@@ -154,6 +154,7 @@ selector:
   exclude_with_any_tags:
   - does_not_support_causal_consistency
   - requires_replication
+  - assumes_balancer_on
   # The touch command does not exist on mongos, which is enforced before session information is
   # parsed, causing state functions in these workloads to fail before starting a transaction despite
   # sending a command with startTransaction=true, leading to an infinite loop in the auto

buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn.yml

@@ -154,6 +154,7 @@ selector:
   exclude_with_any_tags:
   - does_not_support_causal_consistency
   - requires_replication
+  - assumes_balancer_on
   # The touch command does not exist on mongos, which is enforced before session information is
   # parsed, causing state functions in these workloads to fail before starting a transaction despite
   # sending a command with startTransaction=true, leading to an infinite loop in the auto

buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_with_stepdowns.yml

@@ -219,6 +219,7 @@ selector:
   exclude_with_any_tags:
   - does_not_support_causal_consistency
   - requires_replication
+  - assumes_balancer_on
   # The touch command does not exist on mongos, which is enforced before session information is
   # parsed, causing state functions in these workloads to fail before starting a transaction despite
   # sending a command with startTransaction=true, leading to an infinite loop in the auto

buildscripts/resmokeconfig/suites/concurrency_sharded_replication.yml

@@ -118,6 +118,7 @@ selector:
 
   exclude_with_any_tags:
   - requires_replication
+  - assumes_balancer_on
 
 executor:
   archive:

buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns.yml

@@ -183,6 +183,7 @@ selector:
   exclude_with_any_tags:
   - requires_replication
   - requires_non_retryable_writes
+  - assumes_balancer_on
   # Curop requires readConcern local.
   - uses_curop_agg_stage
 

jstests/concurrency/fsm_libs/cluster.js

@@ -231,6 +231,22 @@ var Cluster = function(options) {
                 replSets.push(rs);
             }
 
+            if (options.sharded.enableBalancer === true) {
+                st._configServers.forEach((conn) => {
+                    const configDb = conn.getDB('admin');
+
+                    configDb.adminCommand({
+                        configureFailPoint: 'balancerShouldReturnRandomMigrations',
+                        mode: 'alwaysOn'
+                    });
+                    configDb.adminCommand({
+                        configureFailPoint: 'overrideBalanceRoundInterval',
+                        mode: 'alwaysOn',
+                        data: {intervalMs: 100}
+                    });
+                });
+            }
+
         } else if (options.replication.enabled) {
             rst = new ReplSetTest(db.getMongo().host);
 

jstests/concurrency/fsm_workloads/cleanupOrphanedWhileMigrating.js

@@ -0,0 +1,125 @@
+'use strict';
+
+/**
+ * Performs range deletions while chunks are being moved.
+ *
+ * @tags: [requires_sharding, assumes_balancer_on]
+ */
+
+load('jstests/concurrency/fsm_libs/extend_workload.js');
+load('jstests/concurrency/fsm_workloads/sharded_base_partitioned.js');
+
+var $config = extendWorkload($config, function($config, $super) {
+    $config.threadCount = 5;
+    $config.iterations = 50;
+
+    $config.data.shardKey = {skey: 1};
+    $config.data.shardKeyField = 'skey';
+
+    const numChunks = 50;
+    const numDocs = 100;
+
+    // Total count of documents when initialized.
+    $config.data.initialCount = numChunks * numDocs;
+
+    function executeCommandWithRetries(fn, sleepInterval, retries, name) {
+        let result = null;
+        let done = false;
+
+        while (retries > 0 && !done) {
+            result = fn();
+
+            if (result.ok) {
+                print("command succeeded: " + name);
+                done = true;
+            } else {
+                print("command failed: " + name);
+                printjson(result);
+                sleep(sleepInterval);
+            }
+        }
+
+        return result;
+    }
+
+    // Run cleanupOrphaned on a random shard's primary node.
+    $config.states.cleanupOrphans = function(db, collName, connCache) {
+        const ns = db[collName].getFullName();
+
+        // Get index of random shard.
+        const shardNames = Object.keys(connCache.shards);
+        const randomIndex = Math.floor(Math.random() * shardNames.length);
+
+        const shard = connCache.shards[shardNames[randomIndex]];
+        const shardPrimary = ChunkHelper.getPrimary(shard);
+
+        let nextKey = {};
+        let result = null;
+
+        let iteration = 0;
+        while (nextKey != null) {
+            result = executeCommandWithRetries(() => {
+                return shardPrimary.adminCommand(
+                    {cleanupOrphaned: ns, startingFromKey: nextKey, secondaryThrottle: true});
+            }, 100, 1000, "cleanupOrphaned");
+
+            nextKey = result.stoppedAtKey;
+            iteration++;
+        }
+
+        assert(result.ok);
+    };
+
+    // Verify that counts are stable.
+    $config.states.validate = function(db, collName, connCache) {
+        const ns = db[collName].getFullName();
+
+        // Get total count from mongos. Need to specify batch count that is larger than the total
+        // number of records to prevent getmore command from being issued since stepdown suites
+        // ban it.
+        const mongos = ChunkHelper.getRandomMongos(connCache.mongos);
+        const coll = mongos.getCollection(ns);
+        const totalCount = coll.find({}).batchSize($config.data.initialCount + numDocs).itcount();
+
+        // Verify that sum equals original total.
+        assert(this.initialCount === totalCount,
+               "Document count doesn't match initial count: " + this.initialCount +
+                   " != " + totalCount);
+    };
+
+    $config.states.init = function init(db, collName, connCache) {};
+
+    $config.setup = function setup(db, collName, cluster) {
+        const ns = db[collName].getFullName();
+
+        for (let chunkIndex = 0; chunkIndex < numChunks; chunkIndex++) {
+            let bulk = db[collName].initializeUnorderedBulkOp();
+
+            const splitKey = chunkIndex * numDocs;
+            print("splitting at: " + splitKey);
+
+            for (let docIndex = splitKey - numDocs; docIndex < splitKey; docIndex++) {
+                bulk.insert({_id: docIndex, skey: docIndex});
+            }
+
+            assertAlways.commandWorked(bulk.execute());
+
+            if (chunkIndex > 0) {
+                // Need to retry split command to avoid conflicting with moveChunks issued by the
+                // balancer.
+                let result = executeCommandWithRetries(() => {
+                    return db.adminCommand({split: ns, middle: {skey: splitKey}});
+                }, 100, 10, "split");
+                assertAlways.commandWorked(result);
+            }
+        }
+    };
+
+    $config.transitions = {
+        init: {cleanupOrphans: 1},
+        cleanupOrphans: {cleanupOrphans: 0.5, validate: 0.5},
+        validate: {cleanupOrphans: 1}
+    };
+
+    return $config;
+});

jstests/libs/override_methods/network_error_and_txn_override.js

@@ -85,7 +85,6 @@ const kNonRetryableCommands = new Set([
     "appendOplogNote",
     "applyOps",
     "captrunc",
-    "cleanupOrphaned",
     "clone",
     "cloneCollectionAsCapped",
     "collMod",