289 lines
10 KiB
JavaScript
289 lines
10 KiB
JavaScript
/**
|
|
* Test that servers with incorrect cluster roles (for the replica set they are in) eventually crash
|
|
* during the upgrade process from 7.0 to 8.0 and that there are no issues with CRUD operations or
|
|
* data inconsistencies.
|
|
*
|
|
* To provide context on this test: SERVER-80249 added code to allow a 7.0 replica set
|
|
* to restart as an 8.0 auto-bootstrapped config shard; however, that code also necessarily allows a
|
|
* replica set to be started with mixed cluster roles. To fix that, SERVER-80249 also added code to
|
|
* shut down a server if its cluster role does not align with the shard identity document it
|
|
* replicated (example: shard server has shard identity document for a config server). This ensures
|
|
* that the replica set eventually all has nodes with the same cluster role.
|
|
*
|
|
* @tags: [
|
|
* multiversion_incompatible,
|
|
* requires_replication,
|
|
* requires_persistence
|
|
* ]
|
|
*/
|
|
|
|
import {ReplSetTest} from "jstests/libs/replsettest.js";
|
|
import {ShardingTest} from "jstests/libs/shardingtest.js";
|
|
|
|
function testCRUD(conn) {
|
|
const db = conn.getDB('apple');
|
|
assert.commandWorked(db.foo.insert({x: 1}));
|
|
assert.commandWorked(db.foo.insert({x: -1}));
|
|
assert.commandWorked(db.foo.update({x: 1}, {$set: {y: 1}}));
|
|
assert.commandWorked(db.foo.update({x: -1}, {$set: {y: 1}}));
|
|
var doc1 = db.foo.findOne({x: 1});
|
|
assert.eq(1, doc1.y);
|
|
var doc2 = db.foo.findOne({x: -1});
|
|
assert.eq(1, doc2.y);
|
|
|
|
assert.commandWorked(db.foo.remove({x: 1}, true));
|
|
assert.commandWorked(db.foo.remove({x: -1}, true));
|
|
assert.eq(null, db.foo.findOne());
|
|
|
|
return true;
|
|
}
|
|
|
|
function ensureShardingCommandsFail(conn) {
|
|
assert.commandFailedWithCode(
|
|
conn.getDB('admin').runCommand({_shardsvrCreateCollection: 'test.test', key: {_id: 1}}),
|
|
ErrorCodes.ShardingStateNotInitialized);
|
|
}
|
|
|
|
/**
|
|
* Upgrading 7.0 replica set to 8.0 one shard cluster (config shard).
|
|
*
|
|
* The test first verifies that CRUD operations behave correctly before the shard identity document
|
|
* is inserted. It does this verification with various primaries (shard server primary, or 7.0
|
|
* replica set node primary) and secondaries (shard server secondary, config server secondary
|
|
* and 7.0 replica set node secondary).
|
|
*
|
|
* The test then verifies that shard servers crash when the config server shard identity
|
|
* document is inserted.
|
|
*
|
|
* Finally it verifies that there were no inconsistences as a result of the mixed cluster roles
|
|
* before the shard identity document was inserted.
|
|
*/
|
|
{
|
|
jsTestLog('Starting a replica set with 4 nodes (only the first node can be elected).');
|
|
let rst = new ReplSetTest({
|
|
name: 'rs',
|
|
nodes: [
|
|
// Only allow elections on node1 to make it easier to control what ClusterRole the
|
|
// primary has.
|
|
{},
|
|
{rsConfig: {priority: 0}},
|
|
{rsConfig: {priority: 0}},
|
|
{rsConfig: {priority: 0}}
|
|
],
|
|
nodeOptions: {binVersion: 'last-lts'}
|
|
});
|
|
rst.startSet();
|
|
rst.initiate();
|
|
rst.getPrimary();
|
|
rst.awaitSecondaryNodes();
|
|
let [node0, node1, node2, node3] = rst.nodes;
|
|
|
|
jsTestLog('Restarting node1 as shard server and node2 as config server.');
|
|
MongoRunner.stopMongod(node1, null, {noCleanData: true});
|
|
MongoRunner.stopMongod(node2, null, {noCleanData: true});
|
|
node1 = MongoRunner.runMongod({
|
|
noCleanData: true,
|
|
shardsvr: '',
|
|
replSet: 'rs',
|
|
dbpath: node1.dbpath,
|
|
port: node1.port,
|
|
setParameter: {
|
|
featureFlagAllMongodsAreSharded: true,
|
|
}
|
|
});
|
|
node2 = MongoRunner.runMongod({
|
|
noCleanData: true,
|
|
configsvr: '',
|
|
replSet: 'rs',
|
|
dbpath: node2.dbpath,
|
|
port: node2.port,
|
|
setParameter: {
|
|
featureFlagAllMongodsAreSharded: true,
|
|
}
|
|
});
|
|
|
|
jsTestLog('Test with 7.0 node as primary before the shard identity document is inserted.');
|
|
assert.eq(rst.getPrimary(), node0);
|
|
testCRUD(node0);
|
|
ensureShardingCommandsFail(node0);
|
|
|
|
jsTestLog('Test with shard server primary before the shard identity document is inserted.');
|
|
MongoRunner.stopMongod(node0, null, {noCleanData: true});
|
|
node0 = MongoRunner.runMongod({
|
|
noCleanData: true,
|
|
shardsvr: '',
|
|
replSet: 'rs',
|
|
dbpath: node0.dbpath,
|
|
port: node0.port,
|
|
setParameter: {
|
|
featureFlagAllMongodsAreSharded: true,
|
|
}
|
|
});
|
|
assert.eq(rst.getPrimary(), node0);
|
|
testCRUD(node0);
|
|
ensureShardingCommandsFail(node0);
|
|
|
|
jsTestLog('Restarting with config server as primary.');
|
|
MongoRunner.stopMongod(node0, null, {noCleanData: true});
|
|
node0 = MongoRunner.runMongod({
|
|
noCleanData: true,
|
|
configsvr: '',
|
|
replSet: 'rs',
|
|
dbpath: node0.dbpath,
|
|
port: node0.port,
|
|
setParameter: {
|
|
featureFlagAllMongodsAreSharded: true,
|
|
}
|
|
});
|
|
|
|
jsTestLog('Upgrading the 7.0 replica set node to a 8.0 config server.');
|
|
MongoRunner.stopMongod(node3, null, {noCleanData: true});
|
|
node3 = MongoRunner.runMongod({
|
|
noCleanData: true,
|
|
configsvr: '',
|
|
replSet: 'rs',
|
|
dbpath: node3.dbpath,
|
|
port: node3.port,
|
|
setParameter: {
|
|
featureFlagAllMongodsAreSharded: true,
|
|
}
|
|
});
|
|
|
|
jsTestLog(
|
|
'Wait for config server primary to be writable primary (which implies shard identity document is inserted).');
|
|
assert.soonNoExcept(() => testCRUD(node0));
|
|
|
|
jsTestLog('Checking that the node1 shard server crashes.');
|
|
assert.soon(() => !checkProgram(node1.pid).alive);
|
|
[node0, node2, node3].map(node => checkProgram(node.pid).alive);
|
|
|
|
jsTestLog('Restarting the crashed node1 server as a config server should work.');
|
|
node1 = MongoRunner.runMongod({
|
|
noCleanData: true,
|
|
configsvr: '',
|
|
replSet: 'rs',
|
|
dbpath: node1.dbpath,
|
|
port: node1.port,
|
|
setParameter: {
|
|
featureFlagAllMongodsAreSharded: true,
|
|
}
|
|
});
|
|
assert.commandWorked(node1.getDB('admin').runCommand({ping: 1}));
|
|
|
|
jsTestLog('Restarting a config server as a shard server should fail.');
|
|
rst.awaitSecondaryNodes(); // Ensure shard identity document replicated to secondaries
|
|
MongoRunner.stopMongod(node2, null, {noCleanData: true});
|
|
assert.throws(() => {
|
|
MongoRunner.runMongod({
|
|
noCleanData: true,
|
|
shardsvr: '',
|
|
replSet: 'rs',
|
|
dbpath: node2.dbpath,
|
|
port: node2.port,
|
|
setParameter: {
|
|
featureFlagAllMongodsAreSharded: true,
|
|
}
|
|
});
|
|
});
|
|
node2 = MongoRunner.runMongod({
|
|
noCleanData: true,
|
|
configsvr: '',
|
|
replSet: 'rs',
|
|
dbpath: node2.dbpath,
|
|
port: node2.port,
|
|
setParameter: {
|
|
featureFlagAllMongodsAreSharded: true,
|
|
}
|
|
});
|
|
|
|
// stopSet() will also check oplogs, preImageCollection, changeCollection and db hashes
|
|
// which ensures that the mixed cluster roles for the replica set at the start did not cause
|
|
// any data inconsistencies.
|
|
//
|
|
// Collection validation is skipped because unclean shutdowns (due to the shard identity
|
|
// document not matching the shard's cluster role) causes fast counts to be wrong for
|
|
// admin.system.version.
|
|
rst.stopSet(null, null, {skipValidation: true});
|
|
}
|
|
|
|
/**
|
|
* Upgrading 7.0 sharded cluster to 8.0 sharded cluster.
|
|
*
|
|
* This test verifies that mixed cluster roles are not possible when upgrading a 7.0 sharded cluster
|
|
* to 8.0. More specifically it verifies a server will crash on startup in 8.0 if its cluster role
|
|
* does not agree with the shard identity for the replica set. For example if a shard server sees
|
|
* the shard identity document for a config server (i.e with _id: 'config') it should crash because
|
|
* it knows the replica set it is a part of is for a config server.
|
|
*/
|
|
{
|
|
const st = new ShardingTest({
|
|
mongos: 1,
|
|
config: 3,
|
|
shards: 1,
|
|
rs: {nodes: 3},
|
|
configShard: false,
|
|
nodeOptions: {binVersion: 'last-lts'}
|
|
});
|
|
|
|
jsTestLog('Restarting a 7.0 config server secondary as a 8.0 shard server should fail.');
|
|
let configSecondary = st.configRS.getSecondary();
|
|
printjson(configSecondary);
|
|
MongoRunner.stopMongod(configSecondary, null, {noCleanData: true});
|
|
assert.throws(() => {
|
|
MongoRunner.runMongod({
|
|
noCleanData: true,
|
|
shardsvr: '',
|
|
replSet: st.configRS.name,
|
|
dbpath: configSecondary.dbpath,
|
|
port: configSecondary.port,
|
|
setParameter: {
|
|
featureFlagAllMongodsAreSharded: true,
|
|
}
|
|
});
|
|
});
|
|
|
|
jsTestLog('Restarting a 7.0 config server secondary as a 8.0 config server should work.');
|
|
configSecondary = MongoRunner.runMongod({
|
|
noCleanData: true,
|
|
configsvr: '',
|
|
replSet: st.configRS.name,
|
|
dbpath: configSecondary.dbpath,
|
|
port: configSecondary.port,
|
|
setParameter: {
|
|
featureFlagAllMongodsAreSharded: true,
|
|
}
|
|
});
|
|
testCRUD(st.s0);
|
|
|
|
jsTestLog('Restarting a 7.0 shard server secondary as a 8.0 config server should fail.');
|
|
let shardSecondary = st.rs0.getSecondary();
|
|
MongoRunner.stopMongod(shardSecondary, null, {noCleanData: true});
|
|
assert.throws(() => {
|
|
shardSecondary = MongoRunner.runMongod({
|
|
noCleanData: true,
|
|
configsvr: '',
|
|
replSet: st.rs0.name,
|
|
dbpath: shardSecondary.dbpath,
|
|
port: shardSecondary.port,
|
|
setParameter: {
|
|
featureFlagAllMongodsAreSharded: true,
|
|
}
|
|
});
|
|
});
|
|
|
|
jsTestLog('Restarting a 7.0 shard server secondary as a 8.0 shard server should work.');
|
|
shardSecondary = MongoRunner.runMongod({
|
|
noCleanData: true,
|
|
shardsvr: '',
|
|
replSet: st.rs0.name,
|
|
dbpath: shardSecondary.dbpath,
|
|
port: shardSecondary.port,
|
|
setParameter: {
|
|
featureFlagAllMongodsAreSharded: true,
|
|
}
|
|
});
|
|
testCRUD(st.s0);
|
|
|
|
st.stop();
|
|
}
|