SERVER-50122: Support running mixed versions with last-continuous in ReplSetTest/ShardingTest

Author: ldennis

jstests/sharding/explainFind_stale_mongos.js

@@ -22,7 +22,8 @@ assert.commandWorked(freshMongos.adminCommand({enableSharding: dbName}));
 assert.commandWorked(freshMongos.adminCommand({shardCollection: ns, key: {"_id": 1}}));
 
 jsTest.log("Ensure the shard knows " + ns + " is sharded");
-if (st.isMixedVersionCluster()) {
+const clusterVersionInfo = st.getClusterVersionInfo();
+if (clusterVersionInfo.isMixedVersion) {
     assert.commandWorked(
         st.shard0.adminCommand({_flushRoutingTableCacheUpdates: ns, syncFromConfig: true}));
 } else {

src/mongo/shell/replsettest.js

@@ -1294,21 +1294,31 @@ var ReplSetTest = function(opts) {
         cmd[cmdKey] = config;
 
         // Initiating a replica set with a single node will use "latest" FCV. This will
-        // cause IncompatibleServerVersion errors if additional "last-lts" binary version
-        // nodes are subsequently added to the set, since such nodes cannot set their FCV to
-        // "latest". Therefore, we make sure the primary is "last-lts" FCV before adding in
-        // nodes of different binary versions to the replica set.
+        // cause IncompatibleServerVersion errors if additional "last-lts"/"last-continuous" binary
+        // version nodes are subsequently added to the set, since such nodes cannot set their FCV to
+        // "latest". Therefore, we make sure the primary is "last-lts"/"last-continuous" FCV before
+        // adding in nodes of different binary versions to the replica set.
         let lastLTSBinVersionWasSpecifiedForSomeNode = false;
+        let lastContinuousBinVersionWasSpecifiedForSomeNode = false;
         let explicitBinVersionWasSpecifiedForSomeNode = false;
         Object.keys(this.nodeOptions).forEach(function(key, index) {
             let val = self.nodeOptions[key];
             if (typeof (val) === "object" && val.hasOwnProperty("binVersion")) {
                 lastLTSBinVersionWasSpecifiedForSomeNode =
                     MongoRunner.areBinVersionsTheSame(val.binVersion, lastLTSFCV);
+                lastContinuousBinVersionWasSpecifiedForSomeNode =
+                    (lastLTSFCV !== lastContinuousFCV) &&
+                    MongoRunner.areBinVersionsTheSame(val.binVersion, lastContinuousFCV);
                 explicitBinVersionWasSpecifiedForSomeNode = true;
             }
         });
 
+        if (lastLTSBinVersionWasSpecifiedForSomeNode &&
+            lastContinuousBinVersionWasSpecifiedForSomeNode) {
+            throw new Error("Can only specify one of 'last-lts' and 'last-continuous' " +
+                            "in binVersion, not both.");
+        }
+
         // If no binVersions have been explicitly set, then we should be using the latest binary
         // version, which allows us to use the failpoint below.
         let explicitBinVersion =
@@ -1347,26 +1357,33 @@ var ReplSetTest = function(opts) {
         print("ReplSetTest initiate command took " + (new Date() - initiateStart) + "ms for " +
               this.nodes.length + " nodes in set '" + this.name + "'");
 
-        // Set the FCV to 'last-lts' if we are running a mixed version replica set. If this is a
-        // config server, the FCV will be set as part of ShardingTest.
+        // Set the FCV to 'last-lts'/'last-continuous' if we are running a mixed version replica
+        // set. If this is a config server, the FCV will be set as part of ShardingTest.
+        // TODO SERVER-50389: Set FCV to 'last-continuous' properly when last-continuous binary
+        // versions are supported with the useRandomBinVersionsWithinReplicaSet option.
         let setLastLTSFCV = (lastLTSBinVersionWasSpecifiedForSomeNode ||
                              jsTest.options().useRandomBinVersionsWithinReplicaSet) &&
             !self.isConfigServer;
-        if (setLastLTSFCV && jsTest.options().replSetFeatureCompatibilityVersion) {
+        let setLastContinuousFCV = !setLastLTSFCV &&
+            lastContinuousBinVersionWasSpecifiedForSomeNode && !self.isConfigServer;
+
+        if ((setLastLTSFCV || setLastContinuousFCV) &&
+            jsTest.options().replSetFeatureCompatibilityVersion) {
+            const fcv = setLastLTSFCV ? lastLTSFCV : lastContinuousFCV;
             throw new Error(
-                "The FCV will be set to 'last-lts' automatically when starting up a replica " +
+                "The FCV will be set to '" + fcv + "' automatically when starting up a replica " +
                 "set with mixed binary versions. Therefore, we expect an empty value for " +
                 "'replSetFeatureCompatibilityVersion'.");
         }
 
-        if (setLastLTSFCV) {
+        if (setLastLTSFCV || setLastContinuousFCV) {
             // Authenticate before running the command.
             asCluster(self.nodes, function setFCV() {
-                let fcv = lastLTSFCV;
+                let fcv = setLastLTSFCV ? lastLTSFCV : lastContinuousFCV;
                 print("Setting feature compatibility version for replica set to '" + fcv + "'");
                 assert.commandWorked(
                     self.getPrimary().adminCommand({setFeatureCompatibilityVersion: fcv}));
-                checkFCV(self.getPrimary().getDB("admin"), lastLTSFCV);
+                checkFCV(self.getPrimary().getDB("admin"), fcv);
                 print("Fetch the config version from primay since 4.4 downgrade runs a reconfig.");
                 config.version = self.getReplSetConfigFromNode().version;
             });
@@ -2946,6 +2963,8 @@ var ReplSetTest = function(opts) {
                 options.binVersion = "latest";
             } else {
                 const rand = Random.rand();
+                // TODO SERVER-50389: Support last-continuous binary version with
+                // useRandomBinVersionsWithinReplicaSet.
                 options.binVersion = rand < 0.5 ? "latest" : "last-lts";
             }
             print("Randomly assigned binary version: " + options.binVersion + " to node: " + n);

src/mongo/shell/shardingtest.js

@@ -980,76 +980,92 @@ var ShardingTest = function(params) {
     };
 
     /**
-     * Returns whether any settings to ShardingTest or jsTestOptions indicate this is a multiversion
-     * cluster.
+     * Returns a document {isMixedVersion: <bool>, oldestBinVersion: <string>}.
+     * The 'isMixedVersion' field is true if any settings to ShardingTest or jsTestOptions indicate
+     * this is a multiversion cluster.
+     * The 'oldestBinVersion' field is set to the oldest binary version used in this cluster, one of
+     * 'latest', 'last-continuous' and 'last-lts'.
+     * Note: Mixed version cluster with binary versions older than 'last-lts' is not supported. If
+     * such binary exists in the cluster, this function assumes this is not a mixed version cluster
+     * and returns 'oldestBinVersion' as 'latest'.
      *
-     * Checks for 'last-lts' bin versions via:
+     * Checks for bin versions via:
      *     jsTestOptions().shardMixedBinVersions, jsTestOptions().mongosBinVersion,
      *     otherParams.configOptions.binVersion, otherParams.shardOptions.binVersion,
      *     otherParams.mongosOptions.binVersion
      */
-    this.isMixedVersionCluster = function() {
-        var lastLTSBinVersion = MongoRunner.getBinVersionFor('last-lts');
-
-        // Must check shardMixedBinVersion because it causes shardOptions.binVersion to be an object
-        // (versionIterator) rather than a version string. Must check mongosBinVersion, as well,
-        // because it does not update mongosOptions.binVersion.
-        if (jsTestOptions().shardMixedBinVersions ||
-            (jsTestOptions().mongosBinVersion &&
-             MongoRunner.areBinVersionsTheSame(lastLTSBinVersion,
-                                               jsTestOptions().mongosBinVersion))) {
-            return true;
-        }
+    this.getClusterVersionInfo = function() {
+        function clusterHasBinVersion(version) {
+            const binVersion = MongoRunner.getBinVersionFor(version);
+            const hasBinVersionInParams = (params) => {
+                return params && params.binVersion &&
+                    MongoRunner.areBinVersionsTheSame(
+                        binVersion, MongoRunner.getBinVersionFor(params.binVersion));
+            };
 
-        // Check for 'last-lts' config servers.
-        if (otherParams.configOptions && otherParams.configOptions.binVersion &&
-            MongoRunner.areBinVersionsTheSame(
-                lastLTSBinVersion,
-                MongoRunner.getBinVersionFor(otherParams.configOptions.binVersion))) {
-            return true;
-        }
-        for (var i = 0; i < numConfigs; ++i) {
-            if (otherParams['c' + i] && otherParams['c' + i].binVersion &&
-                MongoRunner.areBinVersionsTheSame(
-                    lastLTSBinVersion,
-                    MongoRunner.getBinVersionFor(otherParams['c' + i].binVersion))) {
+            // Must check shardMixedBinVersion because it causes shardOptions.binVersion to be an
+            // object (versionIterator) rather than a version string. Must check mongosBinVersion,
+            // as well, because it does not update mongosOptions.binVersion.
+            // TODO SERVER-50389: Differentiate between 'last-lts' and 'last-continuous' when
+            // last-continuous is supported with shardMixedBinVersions.
+            if (jsTestOptions().shardMixedBinVersions ||
+                (jsTestOptions().mongosBinVersion &&
+                 MongoRunner.areBinVersionsTheSame(binVersion, jsTestOptions().mongosBinVersion))) {
                 return true;
             }
-        }
 
-        // Check for 'last-lts' mongod servers.
-        if (otherParams.shardOptions && otherParams.shardOptions.binVersion &&
-            MongoRunner.areBinVersionsTheSame(
-                lastLTSBinVersion,
-                MongoRunner.getBinVersionFor(otherParams.shardOptions.binVersion))) {
-            return true;
-        }
-        for (var i = 0; i < numShards; ++i) {
-            if (otherParams['d' + i] && otherParams['d' + i].binVersion &&
-                MongoRunner.areBinVersionsTheSame(
-                    lastLTSBinVersion,
-                    MongoRunner.getBinVersionFor(otherParams['d' + i].binVersion))) {
+            // Check for config servers.
+            if (hasBinVersionInParams(otherParams.configOptions)) {
                 return true;
             }
-        }
+            for (let i = 0; i < numConfigs; ++i) {
+                if (hasBinVersionInParams(otherParams['c' + i])) {
+                    return true;
+                }
+            }
 
-        // Check for 'last-lts' mongos servers.
-        if (otherParams.mongosOptions && otherParams.mongosOptions.binVersion &&
-            MongoRunner.areBinVersionsTheSame(
-                lastLTSBinVersion,
-                MongoRunner.getBinVersionFor(otherParams.mongosOptions.binVersion))) {
-            return true;
-        }
-        for (var i = 0; i < numMongos; ++i) {
-            if (otherParams['s' + i] && otherParams['s' + i].binVersion &&
-                MongoRunner.areBinVersionsTheSame(
-                    lastLTSBinVersion,
-                    MongoRunner.getBinVersionFor(otherParams['s' + i].binVersion))) {
+            // Check for mongod servers.
+            if (hasBinVersionInParams(otherParams.shardOptions)) {
+                return true;
+            }
+            if (hasBinVersionInParams(otherParams.rs)) {
                 return true;
             }
+            for (let i = 0; i < numShards; ++i) {
+                if (hasBinVersionInParams(otherParams['d' + i])) {
+                    return true;
+                }
+                if (hasBinVersionInParams(otherParams['rs' + i])) {
+                    return true;
+                }
+            }
+
+            // Check for mongos servers.
+            if (hasBinVersionInParams(otherParams.mongosOptions)) {
+                return true;
+            }
+            for (let i = 0; i < numMongos; ++i) {
+                if (hasBinVersionInParams(otherParams['s' + i])) {
+                    return true;
+                }
+            }
+
+            return false;
         }
 
-        return false;
+        let hasLastLTS = clusterHasBinVersion("last-lts");
+        let hasLastContinuous = clusterHasBinVersion("last-continuous");
+        if ((lastLTSFCV !== lastContinuousFCV) && hasLastLTS && hasLastContinuous) {
+            throw new Error("Can only specify one of 'last-lts' and 'last-continuous' " +
+                            "in binVersion, not both.");
+        }
+        if (hasLastLTS) {
+            return {isMixedVersion: true, oldestBinVersion: "last-lts"};
+        } else if (hasLastContinuous) {
+            return {isMixedVersion: true, oldestBinVersion: "last-continuous"};
+        } else {
+            return {isMixedVersion: false, oldestBinVersion: "latest"};
+        }
     };
 
     /**
@@ -1279,6 +1295,8 @@ var ShardingTest = function(params) {
                     // version
                     // shard cluster that randomly assigns shard binVersions, half "latest" and half
                     // "last-lts".
+                    // TODO SERVER-50389: Support last-continuous binary version with
+                    // shardMixedBinVersions.
                     if (!otherParams.shardOptions.binVersion) {
                         Random.setRandomSeed();
                         otherParams.shardOptions.binVersion =
@@ -1354,6 +1372,8 @@ var ShardingTest = function(params) {
                 // If the test doesn't depend on specific shard binVersions, create a mixed version
                 // shard cluster that randomly assigns shard binVersions, half "latest" and half
                 // "last-lts".
+                // TODO SERVER-50389: Support last-continuous binary version with
+                // shardMixedBinVersions.
                 if (!otherParams.shardOptions.binVersion) {
                     Random.setRandomSeed();
                     otherParams.shardOptions.binVersion =
@@ -1646,10 +1666,11 @@ var ShardingTest = function(params) {
     }
 
     const configRS = this.configRS;
-    if (_hasNewFeatureCompatibilityVersion() && this.isMixedVersionCluster()) {
+    const clusterVersionInfo = this.getClusterVersionInfo();
+    if (_hasNewFeatureCompatibilityVersion() && clusterVersionInfo.isMixedVersion) {
+        const fcv = binVersionToFCV(clusterVersionInfo.oldestBinVersion);
         function setFeatureCompatibilityVersion() {
-            assert.commandWorked(
-                csrsPrimary.adminCommand({setFeatureCompatibilityVersion: lastLTSFCV}));
+            assert.commandWorked(csrsPrimary.adminCommand({setFeatureCompatibilityVersion: fcv}));
 
             // Wait for the new featureCompatibilityVersion to propagate to all nodes in the CSRS
             // to ensure that older versions of mongos can successfully connect.
@@ -1798,50 +1819,42 @@ var ShardingTest = function(params) {
     // Ensure that the sessions collection exists so jstests can run things with
     // logical sessions and test them. We do this by forcing an immediate cache refresh
     // on the config server, which auto-shards the collection for the cluster.
-    var lastLTSBinVersion = MongoRunner.getBinVersionFor('last-lts');
-    if ((!otherParams.configOptions) ||
-        (otherParams.configOptions && !otherParams.configOptions.binVersion) ||
-        (otherParams.configOptions && otherParams.configOptions.binVersion &&
-         MongoRunner.areBinVersionsTheSame(
-             lastLTSBinVersion,
-             MongoRunner.getBinVersionFor(otherParams.configOptions.binVersion)))) {
-        this.configRS.getPrimary().getDB("admin").runCommand({refreshLogicalSessionCacheNow: 1});
-
-        const x509AuthRequired = (mongosOptions[0] && mongosOptions[0].clusterAuthMode &&
-                                  mongosOptions[0].clusterAuthMode === "x509");
-
-        // Flushes the routing table cache on connection 'conn'. If 'keyFileLocal' is defined,
-        // authenticates the keyfile user on 'authConn' - a connection or set of connections for
-        // the shard - before executing the flush.
-        const flushRT = function flushRoutingTableAndHandleAuth(conn, authConn, keyFileLocal) {
-            // Invokes the actual execution of cache refresh.
-            const execFlushRT = (conn) => {
-                assert.commandWorked(conn.getDB("admin").runCommand(
-                    {_flushRoutingTableCacheUpdates: "config.system.sessions"}));
-            };
-
-            if (keyFileLocal) {
-                authutil.asCluster(authConn, keyFileLocal, () => execFlushRT(conn));
-            } else {
-                execFlushRT(conn);
-            }
+    this.configRS.getPrimary().getDB("admin").runCommand({refreshLogicalSessionCacheNow: 1});
+
+    const x509AuthRequired = (mongosOptions[0] && mongosOptions[0].clusterAuthMode &&
+                              mongosOptions[0].clusterAuthMode === "x509");
+
+    // Flushes the routing table cache on connection 'conn'. If 'keyFileLocal' is defined,
+    // authenticates the keyfile user on 'authConn' - a connection or set of connections for
+    // the shard - before executing the flush.
+    const flushRT = function flushRoutingTableAndHandleAuth(conn, authConn, keyFileLocal) {
+        // Invokes the actual execution of cache refresh.
+        const execFlushRT = (conn) => {
+            assert.commandWorked(conn.getDB("admin").runCommand(
+                {_flushRoutingTableCacheUpdates: "config.system.sessions"}));
         };
 
-        // TODO SERVER-45108: Enable support for x509 auth for _flushRoutingTableCacheUpdates.
-        if (!otherParams.manualAddShard && !x509AuthRequired) {
-            for (let i = 0; i < numShards; i++) {
-                const keyFileLocal =
-                    (otherParams.shards && otherParams.shards[i] && otherParams.shards[i].keyFile)
-                    ? otherParams.shards[i].keyFile
-                    : this.keyFile;
-
-                if (otherParams.rs || otherParams["rs" + i] || startShardsAsRS) {
-                    const rs = this._rs[i].test;
-                    flushRT(rs.getPrimary(), rs.nodes, keyFileLocal);
-                } else {
-                    // If specified, use the keyFile for the standalone shard.
-                    flushRT(this["shard" + i], this["shard" + i], keyFileLocal);
-                }
+        if (keyFileLocal) {
+            authutil.asCluster(authConn, keyFileLocal, () => execFlushRT(conn));
+        } else {
+            execFlushRT(conn);
+        }
+    };
+
+    // TODO SERVER-45108: Enable support for x509 auth for _flushRoutingTableCacheUpdates.
+    if (!otherParams.manualAddShard && !x509AuthRequired) {
+        for (let i = 0; i < numShards; i++) {
+            const keyFileLocal =
+                (otherParams.shards && otherParams.shards[i] && otherParams.shards[i].keyFile)
+                ? otherParams.shards[i].keyFile
+                : this.keyFile;
+
+            if (otherParams.rs || otherParams["rs" + i] || startShardsAsRS) {
+                const rs = this._rs[i].test;
+                flushRT(rs.getPrimary(), rs.nodes, keyFileLocal);
+            } else {
+                // If specified, use the keyFile for the standalone shard.
+                flushRT(this["shard" + i], this["shard" + i], keyFileLocal);
             }
         }
     }