Files
mongo/jstests/multiVersion/set_feature_compatibility_version.js

581 lines
26 KiB
JavaScript

/**
* Tests setFeatureCompatibilityVersion.
*/
// Checking UUID consistency involves talking to a shard node, which in this test is shutdown
TestData.skipCheckingUUIDsConsistentAcrossCluster = true;
// Some of the test cases in this file clear the FCV document, so the validate command's UUID checks
// will fail.
TestData.skipCollectionAndIndexValidation = true;
// A replset test case checks that replication to a secondary ceases, so we do not expect identical
// data.
TestData.skipCheckDBHashes = true;
(function() {
"use strict";
load("jstests/replsets/rslib.js");
load("jstests/libs/feature_compatibility_version.js");
load("jstests/libs/get_index_helpers.js");
let dbpath = MongoRunner.dataPath + "feature_compatibility_version";
resetDbpath(dbpath);
let res;
const latest = "latest";
const lastStable = "last-stable";
const latestFCV = "4.0";
const lastStableFCV = "3.6";
/**
* If we're using mmapv1, we must recover the journal files from an unclean shutdown before
* attempting to run with --repair.
*/
let recoverMMapJournal = function(isMMAPv1, conn, dbpath) {
if (isMMAPv1) {
let returnCode = runMongoProgram("mongod",
"--port",
conn.port,
"--journalOptions",
/*MMAPV1Options::JournalRecoverOnly*/ 4,
"--dbpath",
dbpath);
assert.eq(returnCode, /*EXIT_NET_ERROR*/ 48);
}
};
/**
* Ensure that a mongod (without using --repair) fails to start up if there are non-local
* collections and either the admin database has been dropped or the FCV document in the admin
* database has been removed.
*
* The mongod has 'version' binary and is started up on 'dbpath'.
*/
let doStartupFailTests = function(version, dbpath) {
// Set up a mongod with a user collection but without an admin database.
setupUserCollAndMissingAdminDB(version, dbpath);
// Now attempt to start up a new mongod without clearing the data files from 'dbpath', which
// contain a user collection but are missing the admin database. The mongod should fail to
// start up if the admin database is missing but there are non-local collections.
conn = MongoRunner.runMongod({dbpath: dbpath, binVersion: version, noCleanData: true});
assert.eq(
null,
conn,
"expected mongod to fail when data files are present but no admin database is found.");
// Set up a mongod with an admin database but without a FCV document in the admin database.
setupMissingFCVDoc(version, dbpath);
// Now attempt to start up a new mongod without clearing the data files from 'dbpath', which
// contain the admin database but are missing the FCV document. The mongod should fail to
// start start up if there is a non-local collection and the FCV document is missing.
conn = MongoRunner.runMongod({dbpath: dbpath, binVersion: version, noCleanData: true});
assert.eq(
null,
conn,
"expected mongod to fail when data files are present but no FCV document is found.");
};
/**
* Starts up a mongod with binary 'version' on 'dbpath', then removes the FCV document from the
* admin database and returns the mongod.
*/
let setupMissingFCVDoc = function(version, dbpath) {
let conn = MongoRunner.runMongod({dbpath: dbpath, binVersion: version});
assert.neq(null,
conn,
"mongod was unable to start up with version=" + version + " and no data files");
adminDB = conn.getDB("admin");
removeFCVDocument(adminDB);
MongoRunner.stopMongod(conn);
return conn;
};
/**
* Starts up a mongod with binary 'version' on 'dbpath' and creates a user collection. Then
* drops the admin database and returns the mongod.
*/
let setupUserCollAndMissingAdminDB = function(version, dbpath) {
conn = MongoRunner.runMongod({dbpath: dbpath, binVersion: version});
assert.neq(null,
conn,
"mongod was unable to start up with version=" + version + " and no data files");
let testDB = conn.getDB("test");
assert.commandWorked(testDB.createCollection("testcoll"));
adminDB = conn.getDB("admin");
assert.commandWorked(adminDB.runCommand({dropDatabase: 1}),
"expected drop of admin database to be successful");
MongoRunner.stopMongod(conn);
return conn;
};
//
// Standalone tests.
//
let conn;
let adminDB;
// A 'latest' binary standalone should default to 'latestFCV'.
conn = MongoRunner.runMongod({dbpath: dbpath, binVersion: latest});
assert.neq(
null, conn, "mongod was unable to start up with version=" + latest + " and no data files");
adminDB = conn.getDB("admin");
checkFCV(adminDB,
lastStableFCV); // TODO: make this 'latestFCV' when SERVER-32597 bumps default
// TODO: remove this when SERVER-32597 bumps the default to 4.0.
assert.commandWorked(adminDB.runCommand({setFeatureCompatibilityVersion: latestFCV}));
// featureCompatibilityVersion cannot be set to invalid value.
assert.commandFailed(adminDB.runCommand({setFeatureCompatibilityVersion: 5}));
assert.commandFailed(adminDB.runCommand({setFeatureCompatibilityVersion: "3.2"}));
assert.commandFailed(adminDB.runCommand({setFeatureCompatibilityVersion: "4.2"}));
assert.commandFailed(adminDB.runCommand({setFeatureCompatibilityVersion: "3.4"}));
// setFeatureCompatibilityVersion rejects unknown fields.
assert.commandFailed(
adminDB.runCommand({setFeatureCompatibilityVersion: lastStable, unknown: 1}));
// setFeatureCompatibilityVersion can only be run on the admin database.
assert.commandFailed(
conn.getDB("test").runCommand({setFeatureCompatibilityVersion: lastStable}));
// featureCompatibilityVersion cannot be set via setParameter.
assert.commandFailed(
adminDB.runCommand({setParameter: 1, featureCompatibilityVersion: lastStable}));
// setFeatureCompatibilityVersion fails to downgrade FCV if the write fails.
assert.commandWorked(adminDB.runCommand({
configureFailPoint: "failCollectionUpdates",
data: {collectionNS: "admin.system.version"},
mode: "alwaysOn"
}));
assert.commandFailed(adminDB.runCommand({setFeatureCompatibilityVersion: lastStableFCV}));
checkFCV(adminDB, latestFCV);
assert.commandWorked(adminDB.runCommand({
configureFailPoint: "failCollectionUpdates",
data: {collectionNS: "admin.system.version"},
mode: "off"
}));
// featureCompatibilityVersion can be downgraded to 'lastStableFCV'.
assert.commandWorked(adminDB.runCommand({setFeatureCompatibilityVersion: lastStableFCV}));
checkFCV(adminDB, lastStableFCV);
// setFeatureCompatibilityVersion fails to upgrade to 'latestFCV' if the write fails.
assert.commandWorked(adminDB.runCommand({
configureFailPoint: "failCollectionUpdates",
data: {collectionNS: "admin.system.version"},
mode: "alwaysOn"
}));
assert.commandFailed(adminDB.runCommand({setFeatureCompatibilityVersion: latestFCV}));
checkFCV(adminDB, lastStableFCV);
assert.commandWorked(adminDB.runCommand({
configureFailPoint: "failCollectionUpdates",
data: {collectionNS: "admin.system.version"},
mode: "off"
}));
// featureCompatibilityVersion can be upgraded to 'latestFCV'.
assert.commandWorked(adminDB.runCommand({setFeatureCompatibilityVersion: latestFCV}));
checkFCV(adminDB, latestFCV);
// TODO: remove these FCV 3.4 / 4.0 tests when FCV 3.4 is removed (SERVER-32597).
// ----------------------------------------------------------------------------------
// Cannot jump down two FCVs from 4.0 to 3.4
assert.commandFailed(adminDB.runCommand({setFeatureCompatibilityVersion: "3.4"}));
checkFCV(adminDB, latestFCV);
assert.commandWorked(adminDB.runCommand({setFeatureCompatibilityVersion: lastStableFCV}));
assert.commandWorked(adminDB.runCommand({setFeatureCompatibilityVersion: "3.4"}));
checkFCV(adminDB, "3.4");
// Cannot jump up two FCVs from 3.4 to 4.0
assert.commandFailed(adminDB.runCommand({setFeatureCompatibilityVersion: latestFCV}));
checkFCV(adminDB, "3.4");
// ------------------ end FCV 3.4/4.0 testing ---------------------------------------
MongoRunner.stopMongod(conn);
// featureCompatibilityVersion is durable.
// TODO: update this to use 'latestFCV' and 'lastStableFCV', rather than 'lastStableFCV' and
// '3.4', respectively, when SERVER-32597 bumps the default to 4.0.
conn = MongoRunner.runMongod({dbpath: dbpath, binVersion: latest});
assert.neq(
null, conn, "mongod was unable to start up with version=" + latest + " and no data files");
adminDB = conn.getDB("admin");
checkFCV(adminDB, lastStableFCV);
assert.commandWorked(adminDB.runCommand({setFeatureCompatibilityVersion: "3.4"}));
checkFCV(adminDB, "3.4");
MongoRunner.stopMongod(conn);
conn = MongoRunner.runMongod({dbpath: dbpath, binVersion: latest, noCleanData: true});
assert.neq(null,
conn,
"mongod was unable to start up with binary version=" + latest +
" and featureCompatibilityVersion=3.4");
adminDB = conn.getDB("admin");
checkFCV(adminDB, "3.4");
MongoRunner.stopMongod(conn);
// If you upgrade from 'lastStable' binary to 'latest' binary and have non-local databases, FCV
// remains 'lastStableFCV'.
conn = MongoRunner.runMongod({dbpath: dbpath, binVersion: lastStable});
assert.neq(null,
conn,
"mongod was unable to start up with version=" + lastStable + " and no data files");
assert.writeOK(conn.getDB("test").coll.insert({a: 5}));
adminDB = conn.getDB("admin");
checkFCV(adminDB, lastStableFCV);
MongoRunner.stopMongod(conn);
conn = MongoRunner.runMongod({dbpath: dbpath, binVersion: latest, noCleanData: true});
assert.neq(null,
conn,
"mongod was unable to start up with binary version=" + latest +
" and featureCompatibilityVersion=" + lastStableFCV);
adminDB = conn.getDB("admin");
checkFCV(adminDB, lastStableFCV);
MongoRunner.stopMongod(conn);
// A 'latest' binary mongod started with --shardsvr and clean data files defaults to
// 'lastStableFCV'.
conn = MongoRunner.runMongod({dbpath: dbpath, binVersion: latest, shardsvr: ""});
assert.neq(
null, conn, "mongod was unable to start up with version=" + latest + " and no data files");
adminDB = conn.getDB("admin");
checkFCV(adminDB, lastStableFCV);
MongoRunner.stopMongod(conn);
// Check that start up without --repair fails if there is non-local DB data and either the admin
// database was dropped or the FCV doc was deleted.
doStartupFailTests(latest, dbpath);
const isMMAPv1 = jsTest.options().storageEngine === "mmapv1";
// --repair can be used to restore a missing admin database, and thus FCV document that resides
// in the admin database, if at least some collections have UUIDs.
conn = setupUserCollAndMissingAdminDB(latest, dbpath);
recoverMMapJournal(isMMAPv1, conn, dbpath);
let returnCode = runMongoProgram("mongod", "--port", conn.port, "--repair", "--dbpath", dbpath);
assert.eq(
returnCode,
0,
"expected mongod --repair to execute successfully when restoring a missing admin database.");
conn = MongoRunner.runMongod({dbpath: dbpath, binVersion: latest, noCleanData: true});
assert.neq(null,
conn,
"mongod was unable to start up with version=" + latest + " and existing data files");
// FCV is 3.4 and targetVersion is 3.6 because the admin database needed to be restored.
// TODO: update this code and comment when SERVER-32597 bumps the FCV defaults.
adminDB = conn.getDB("admin");
assert.eq(adminDB.system.version.findOne({_id: "featureCompatibilityVersion"}).version, "3.4");
assert.eq(adminDB.system.version.findOne({_id: "featureCompatibilityVersion"}).targetVersion,
lastStableFCV);
MongoRunner.stopMongod(conn);
// --repair can be used to restore a missing featureCompatibilityVersion document to an existing
// admin database if at least some collections have UUIDs.
conn = setupMissingFCVDoc(latest, dbpath);
recoverMMapJournal(isMMAPv1, conn, dbpath);
returnCode = runMongoProgram("mongod", "--port", conn.port, "--repair", "--dbpath", dbpath);
assert.eq(
returnCode,
0,
"expected mongod --repair to execute successfully when restoring a missing FCV document.");
conn = MongoRunner.runMongod({dbpath: dbpath, binVersion: latest, noCleanData: true});
assert.neq(null,
conn,
"mongod was unable to start up with version=" + latest + " and existing data files");
// FCV is 'latestFCV' because all collections were left intact with UUIDs.
// TODO: update 'lastStableFCV' to 'latestFCV' when SERVER-32597 bumps the FCV defaults.
adminDB = conn.getDB("admin");
assert.eq(adminDB.system.version.findOne({_id: "featureCompatibilityVersion"}).version,
lastStableFCV);
MongoRunner.stopMongod(conn);
// If the featureCompatibilityVersion document is present but there are no collection UUIDs,
// --repair should not attempt to restore the document and should return success.
conn = MongoRunner.runMongod({dbpath: dbpath, binVersion: lastStable});
assert.neq(null,
conn,
"mongod was unable to start up with version=" + lastStable + " and no data files");
MongoRunner.stopMongod(conn);
recoverMMapJournal(isMMAPv1, conn, dbpath);
returnCode = runMongoProgram("mongod", "--port", conn.port, "--repair", "--dbpath", dbpath);
assert.eq(returnCode, 0);
//
// Replica set tests.
//
let rst;
let rstConns;
let replSetConfig;
let primaryAdminDB;
let secondaryAdminDB;
// 'latest' binary replica set.
rst = new ReplSetTest({nodes: 2, nodeOpts: {binVersion: latest}});
rst.startSet();
rst.initiate();
primaryAdminDB = rst.getPrimary().getDB("admin");
secondaryAdminDB = rst.getSecondary().getDB("admin");
// FCV should default to 'latestFCV' on primary and secondary in a 'latest' binary replica set.
// TODO: update these to 'latestFCV' when SERVER-32597 bumps the FCV default.
checkFCV(primaryAdminDB, lastStableFCV);
rst.awaitReplication();
checkFCV(secondaryAdminDB, lastStableFCV);
// featureCompatibilityVersion propagates to secondary.
// TODO: update these to 'lastStableFCV' when SERVER-32597 bumps the FCV default.
assert.commandWorked(primaryAdminDB.runCommand({setFeatureCompatibilityVersion: "3.4"}));
checkFCV(primaryAdminDB, "3.4");
rst.awaitReplication();
checkFCV(secondaryAdminDB, "3.4");
// setFeatureCompatibilityVersion cannot be run on secondary.
// TODO: update this to 'latestFCV' when SERVER-32597 bumps the FCV default.
assert.commandFailed(
secondaryAdminDB.runCommand({setFeatureCompatibilityVersion: lastStableFCV}));
rst.stopSet();
// A 'latest' binary secondary with a 'lastStable' binary primary will have 'lastStableFCV'
rst = new ReplSetTest({nodes: [{binVersion: lastStable}, {binVersion: latest}]});
rstConns = rst.startSet();
replSetConfig = rst.getReplSetConfig();
replSetConfig.members[1].priority = 0;
replSetConfig.members[1].votes = 0;
rst.initiate(replSetConfig);
rst.waitForState(rstConns[0], ReplSetTest.State.PRIMARY);
secondaryAdminDB = rst.getSecondary().getDB("admin");
checkFCV(secondaryAdminDB, lastStableFCV);
rst.stopSet();
// Test that a 'lastStable' secondary can successfully perform initial sync from a 'latest'
// primary with 'lastStableFCV'.
rst = new ReplSetTest(
{nodes: [{binVersion: latest}, {binVersion: latest, rsConfig: {priority: 0}}]});
rst.startSet();
rst.initiate();
let primary = rst.getPrimary();
primaryAdminDB = primary.getDB("admin");
let secondary = rst.add({binVersion: lastStable});
secondaryAdminDB = secondary.getDB("admin");
// Rig the election so that the first node running latest version remains the primary after the
// 'lastStable' secondary is added to the replica set.
replSetConfig = rst.getReplSetConfig();
replSetConfig.version = 4;
replSetConfig.members[2].priority = 0;
reconfig(rst, replSetConfig);
// Verify that the 'lastStable' secondary successfully performed its initial sync.
assert.writeOK(
primaryAdminDB.getSiblingDB("test").coll.insert({awaitRepl: true}, {writeConcern: {w: 3}}));
// Test that a 'lastStable' secondary can no longer replicate from the primary after the FCV is
// upgraded to 'latestFCV'.
assert.commandWorked(primary.adminCommand({setFeatureCompatibilityVersion: latestFCV}));
checkFCV(secondaryAdminDB, lastStableFCV);
assert.writeOK(primaryAdminDB.getSiblingDB("test").coll.insert({shouldReplicate: false}));
assert.eq(secondaryAdminDB.getSiblingDB("test").coll.find({shouldReplicate: false}).itcount(),
0);
rst.stopSet();
// Test idempotency for setFeatureCompatibilityVersion.
rst = new ReplSetTest({nodes: 2, nodeOpts: {binVersion: latest}});
rst.startSet();
rst.initiate();
// Set FCV to 'lastStableFCV' so that a 'lastStable' binary node can join the set.
primary = rst.getPrimary();
assert.commandWorked(primary.adminCommand({setFeatureCompatibilityVersion: lastStableFCV}));
rst.awaitReplication();
// Add a 'lastStable' binary node to the set.
secondary = rst.add({binVersion: lastStable});
rst.reInitiate();
// Ensure the 'lastStable' binary node succeeded its initial sync.
assert.writeOK(primary.getDB("test").coll.insert({awaitRepl: true}, {writeConcern: {w: 3}}));
// Run {setFCV: lastStableFCV}. This should be idempotent.
assert.commandWorked(primary.adminCommand({setFeatureCompatibilityVersion: lastStableFCV}));
rst.awaitReplication();
// Ensure the secondary is still running.
rst.stopSet();
//
// Sharding tests.
//
let st;
let mongosAdminDB;
let configPrimaryAdminDB;
let shardPrimaryAdminDB;
// A 'latest' binary cluster started with clean data files will set FCV to 'latestFCV'.
st = new ShardingTest({
shards: {rs0: {nodes: [{binVersion: latest}, {binVersion: latest}]}},
other: {useBridge: true}
});
mongosAdminDB = st.s.getDB("admin");
configPrimaryAdminDB = st.configRS.getPrimary().getDB("admin");
shardPrimaryAdminDB = st.rs0.getPrimary().getDB("admin");
// TODO: update this to 'latestFCV' when SERVER-32597 bumps the FCV default.
checkFCV(configPrimaryAdminDB, lastStableFCV);
checkFCV(shardPrimaryAdminDB, lastStableFCV);
// TODO: remove this when SERVER-32597 bumps the FCV default ot 4.0.
assert.commandWorked(mongosAdminDB.runCommand({setFeatureCompatibilityVersion: latestFCV}));
// featureCompatibilityVersion cannot be set to invalid value on mongos.
assert.commandFailed(mongosAdminDB.runCommand({setFeatureCompatibilityVersion: 5}));
assert.commandFailed(mongosAdminDB.runCommand({setFeatureCompatibilityVersion: "3.2"}));
assert.commandFailed(mongosAdminDB.runCommand({setFeatureCompatibilityVersion: "4.2"}));
// setFeatureCompatibilityVersion rejects unknown fields on mongos.
assert.commandFailed(
mongosAdminDB.runCommand({setFeatureCompatibilityVersion: lastStableFCV, unknown: 1}));
// setFeatureCompatibilityVersion can only be run on the admin database on mongos.
assert.commandFailed(
st.s.getDB("test").runCommand({setFeatureCompatibilityVersion: lastStableFCV}));
// featureCompatibilityVersion cannot be set via setParameter on mongos.
assert.commandFailed(
mongosAdminDB.runCommand({setParameter: 1, featureCompatibilityVersion: lastStableFCV}));
// Prevent the shard primary from receiving messages from the config server primary. When we try
// to set FCV to 'lastStableFCV', the command should fail because the shard cannot be contacted.
st.rs0.getPrimary().discardMessagesFrom(st.configRS.getPrimary(), 1.0);
assert.commandFailed(
mongosAdminDB.runCommand({setFeatureCompatibilityVersion: lastStableFCV, maxTimeMS: 1000}));
checkFCV(
configPrimaryAdminDB, lastStableFCV, lastStableFCV /* indicates downgrade in progress */);
st.rs0.getPrimary().discardMessagesFrom(st.configRS.getPrimary(), 0.0);
// FCV can be set to 'lastStableFCV' on mongos.
// This is run through assert.soon() because we've just caused a network interruption
// by discarding messages in the bridge.
assert.soon(function() {
res = mongosAdminDB.runCommand({setFeatureCompatibilityVersion: lastStableFCV});
if (res.ok == 0) {
print("Failed to set feature compatibility version: " + tojson(res));
return false;
}
return true;
});
// featureCompatibilityVersion propagates to config and shard.
checkFCV(configPrimaryAdminDB, lastStableFCV);
checkFCV(shardPrimaryAdminDB, lastStableFCV);
// A 'latest' binary replica set started as a shard server defaults to 'lastStableFCV'.
let latestShard = new ReplSetTest({
name: "latestShard",
nodes: [{binVersion: latest}, {binVersion: latest}],
nodeOptions: {shardsvr: ""},
useHostName: true
});
latestShard.startSet();
latestShard.initiate();
let latestShardPrimaryAdminDB = latestShard.getPrimary().getDB("admin");
checkFCV(latestShardPrimaryAdminDB, lastStableFCV);
assert.commandWorked(mongosAdminDB.runCommand({addShard: latestShard.getURL()}));
checkFCV(latestShardPrimaryAdminDB, lastStableFCV);
// FCV can be set to 'latestFCV' on mongos.
assert.commandWorked(mongosAdminDB.runCommand({setFeatureCompatibilityVersion: latestFCV}));
checkFCV(st.configRS.getPrimary().getDB("admin"), latestFCV);
checkFCV(shardPrimaryAdminDB, latestFCV);
checkFCV(latestShardPrimaryAdminDB, latestFCV);
// Call ShardingTest.stop before shutting down latestShard, so that the UUID check in
// ShardingTest.stop can talk to latestShard.
st.stop();
latestShard.stopSet();
// Create cluster with a 'lastStable' binary mongos so that we can add 'lastStable' binary
// shards.
st = new ShardingTest({shards: 0, other: {mongosOptions: {binVersion: lastStable}}});
mongosAdminDB = st.s.getDB("admin");
configPrimaryAdminDB = st.configRS.getPrimary().getDB("admin");
checkFCV(configPrimaryAdminDB, lastStableFCV);
// Adding a 'lastStable' binary shard to a cluster with 'lastStableFCV' succeeds.
let lastStableShard = new ReplSetTest({
name: "lastStableShard",
nodes: [{binVersion: lastStable}, {binVersion: lastStable}],
nodeOptions: {shardsvr: ""},
useHostName: true
});
lastStableShard.startSet();
lastStableShard.initiate();
assert.commandWorked(mongosAdminDB.runCommand({addShard: lastStableShard.getURL()}));
checkFCV(lastStableShard.getPrimary().getDB("admin"), lastStableFCV);
// call ShardingTest.stop before shutting down lastStableShard, so that the UUID check in
// ShardingTest.stop can talk to lastStableShard.
st.stop();
lastStableShard.stopSet();
// Create a cluster running with 'latestFCV'
st = new ShardingTest({shards: 1, mongos: 1});
mongosAdminDB = st.s.getDB("admin");
configPrimaryAdminDB = st.configRS.getPrimary().getDB("admin");
shardPrimaryAdminDB = st.shard0.getDB("admin");
// TODO: update this to 'latestFCV' when SERVER-32597 updates the FCV defaults.
checkFCV(configPrimaryAdminDB, lastStableFCV);
checkFCV(shardPrimaryAdminDB, lastStableFCV);
// TODO: remove this when SERVER-32597 updates the FCV defaults.
assert.commandWorked(mongosAdminDB.runCommand({setFeatureCompatibilityVersion: latestFCV}));
// Ensure that a 'lastStable' binary mongos can be added to a 'lastStableFCV' cluster.
assert.commandWorked(mongosAdminDB.runCommand({setFeatureCompatibilityVersion: lastStableFCV}));
checkFCV(configPrimaryAdminDB, lastStableFCV);
checkFCV(shardPrimaryAdminDB, lastStableFCV);
let lastStableMongos =
MongoRunner.runMongos({configdb: st.configRS.getURL(), binVersion: lastStable});
assert.neq(null,
lastStableMongos,
"mongos was unable to start up with binary version=" + latest +
" and connect to FCV=" + lastStableFCV + " cluster");
// Ensure that the 'lastStable' binary mongos can perform reads and writes to the shards in the
// cluster.
assert.writeOK(lastStableMongos.getDB("test").foo.insert({x: 1}));
let foundDoc = lastStableMongos.getDB("test").foo.findOne({x: 1});
assert.neq(null, foundDoc);
assert.eq(1, foundDoc.x, tojson(foundDoc));
// The 'lastStable' binary mongos can no longer perform reads and writes after the cluster is
// upgraded to 'latestFCV'.
assert.commandWorked(mongosAdminDB.runCommand({setFeatureCompatibilityVersion: latestFCV}));
assert.writeError(lastStableMongos.getDB("test").foo.insert({x: 1}));
// The 'latest' binary mongos can still perform reads and writes after the FCV is upgraded to
// 'latestFCV'.
assert.writeOK(st.s.getDB("test").foo.insert({x: 1}));
st.stop();
})();