Files
mongo/jstests/replsets/libs/tenant_migration_test.js

690 lines
26 KiB
JavaScript

/**
* Wrapper around ReplSetTest for testing tenant migration behavior.
*/
"use strict";
load("jstests/aggregation/extras/utils.js");
load("jstests/replsets/libs/tenant_migration_util.js");
/**
* This fixture allows the user to optionally pass in a custom ReplSetTest for the donor and
* recipient replica sets, to be used for the test.
*
* If the caller does not provide their own replica set, a two node replset will be initialized
* instead, with all nodes running the latest version.
*
* @param {string} [name] the name of the replica sets
* @param {boolean} [enableRecipientTesting] whether recipient would actually migrate tenant data
* @param {Object} [donorRst] the ReplSetTest instance to adopt for the donor
* @param {Object} [recipientRst] the ReplSetTest instance to adopt for the recipient
* @param {Object} [sharedOptions] an object that can contain 'nodes' <number>, the number of nodes
* each RST will contain, and 'setParameter' <object>, an object with various server parameters.
* @param {boolean} [allowDonorReadAfterMigration] whether donor would allow reads after a committed
* migration.
* @param {boolean} [initiateRstWithHighElectionTimeout] whether donor and recipient replica sets
* should be initiated with high election timeout.
* @param {boolean} [quickGarbageCollection] whether to set a low garbageCollectionDelayMS.
* @param {string} [insertDataForTenant] create dummy data in <tenantId>_test database.
*/
function TenantMigrationTest({
name = "TenantMigrationTest",
enableRecipientTesting = true,
donorRst,
recipientRst,
sharedOptions = {},
// Default this to true so it is easier for data consistency checks.
allowStaleReadsOnDonor = true,
initiateRstWithHighElectionTimeout = true,
quickGarbageCollection = false,
insertDataForTenant,
}) {
const donorPassedIn = (donorRst !== undefined);
const recipientPassedIn = (recipientRst !== undefined);
const migrationX509Options = TenantMigrationUtil.makeX509OptionsForTest();
const migrationCertificates = TenantMigrationUtil.makeMigrationCertificatesForTest();
const nodes = sharedOptions.nodes || 2;
const setParameterOpts = sharedOptions.setParameter || {};
if (quickGarbageCollection) {
setParameterOpts.tenantMigrationGarbageCollectionDelayMS = 3 * 1000;
setParameterOpts.ttlMonitorSleepSecs = 3;
}
donorRst = donorPassedIn ? donorRst : performSetUp(true /* isDonor */);
recipientRst = recipientPassedIn ? recipientRst : performSetUp(false /* isDonor */);
donorRst.asCluster(donorRst.nodes, () => {
donorRst.getPrimary();
donorRst.awaitReplication();
TenantMigrationUtil.createTenantMigrationRecipientRoleIfNotExist(donorRst);
});
recipientRst.asCluster(recipientRst.nodes, () => {
recipientRst.getPrimary();
recipientRst.awaitReplication();
TenantMigrationUtil.createTenantMigrationDonorRoleIfNotExist(recipientRst);
});
function loadDummyData() {
const numDocs = 20;
const testData = [];
for (let i = 0; i < numDocs; ++i) {
testData.push({_id: i, x: i});
}
return testData;
}
/**
* Inserts documents into the specified collection on the donor primary.
*/
this.insertDonorDB = function(dbName, collName, data = loadDummyData()) {
jsTestLog(`Inserting data into collection ${collName} of DB ${dbName} on the donor`);
const primary = donorRst.getPrimary();
const db = primary.getDB(dbName);
const res = assert.commandWorked(
db.runCommand({insert: collName, documents: data, writeConcern: {w: 'majority'}}));
jsTestLog(`Inserted with w: majority, opTime ${tojson(res.operationTime)}`);
};
// Shard Merge installs TenantRecipientAccessBlockers only for tenants with data, so most tests
// require some data.
if (insertDataForTenant !== undefined) {
this.insertDonorDB(`${insertDataForTenant}_test`, "test");
}
/**
* Creates a ReplSetTest instance. The repl set will have 2 nodes if not otherwise specified.
*/
function performSetUp(isDonor) {
if (TestData.logComponentVerbosity) {
setParameterOpts["logComponentVerbosity"] =
tojsononeline(TestData.logComponentVerbosity);
}
if (!(isDonor || enableRecipientTesting)) {
setParameterOpts["failpoint.returnResponseOkForRecipientSyncDataCmd"] =
tojson({mode: 'alwaysOn'});
}
if (allowStaleReadsOnDonor) {
setParameterOpts["failpoint.tenantMigrationDonorAllowsNonTimestampedReads"] =
tojson({mode: 'alwaysOn'});
}
let nodeOptions = isDonor ? migrationX509Options.donor : migrationX509Options.recipient;
nodeOptions["setParameter"] = setParameterOpts;
const rstName = `${name}_${(isDonor ? "donor" : "recipient")}`;
const rst = new ReplSetTest({name: rstName, nodes, nodeOptions});
rst.startSet();
if (initiateRstWithHighElectionTimeout) {
rst.initiateWithHighElectionTimeout();
} else {
rst.initiate();
}
return rst;
}
/**
* Runs a tenant migration with the given migration options and waits for the migration to
* be committed or aborted.
*
* Returns the result of the initial donorStartMigration if it was unsuccessful. Otherwise,
* returns the command response containing the migration state on the donor after the
* migration has completed.
*/
this.runMigration = function(migrationOpts, opts = {}) {
const {
retryOnRetryableErrors = false,
automaticForgetMigration = true,
enableDonorStartMigrationFsync = false
} = opts;
const startRes = this.startMigration(migrationOpts, opts);
if (!startRes.ok) {
return startRes;
}
const completeRes = this.waitForMigrationToComplete(migrationOpts, retryOnRetryableErrors);
if (automaticForgetMigration &&
(completeRes.state === TenantMigrationTest.State.kCommitted ||
completeRes.state === TenantMigrationTest.State.kAborted)) {
jsTestLog(`Automatically forgetting ${completeRes.state} migration with migrationId: ${
migrationOpts.migrationIdString}`);
this.forgetMigration(migrationOpts.migrationIdString);
}
return completeRes;
};
/**
* Starts a tenant migration by running the 'donorStartMigration' command once.
*
* Returns the result of the 'donorStartMigration' command.
*/
this.startMigration = function(
migrationOpts,
{retryOnRetryableErrors = false, enableDonorStartMigrationFsync = false} = {}) {
return this.runDonorStartMigration(migrationOpts, {
retryOnRetryableErrors,
enableDonorStartMigrationFsync,
});
};
/**
* Waits for a migration to complete by continuously polling the donor primary with
* 'donorStartMigration' until the returned state is committed or aborted. Must be used with
* startMigration, after the migration has been started for the specified tenantId.
*
* Returns the result of the last 'donorStartMigration' command executed.
*/
this.waitForMigrationToComplete = function(
migrationOpts, retryOnRetryableErrors = false, forgetMigration = false) {
// Assert that the migration has already been started.
const tenantId = migrationOpts.tenantId;
assert(this.getDonorPrimary()
.getCollection(TenantMigrationTest.kConfigDonorsNS)
.findOne({tenantId}));
const donorStartReply = this.runDonorStartMigration(
migrationOpts, {waitForMigrationToComplete: true, retryOnRetryableErrors});
if (!forgetMigration) {
return donorStartReply;
}
this.forgetMigration(migrationOpts.migrationIdString, retryOnRetryableErrors);
return donorStartReply;
};
/**
* Executes the 'donorStartMigration' command on the donor primary.
*
* This will return on the first successful command if 'waitForMigrationToComplete' is
* set to false. Otherwise, it will continuously poll the donor primary until the
* migration has been committed or aborted.
*
* If 'retryOnRetryableErrors' is set, this function will retry if the command fails
* with a NotPrimary or network error.
*/
this.runDonorStartMigration = function({
migrationIdString,
tenantId,
recipientConnectionString = recipientRst.getURL(),
readPreference = {mode: "primary"},
donorCertificateForRecipient = migrationCertificates.donorCertificateForRecipient,
recipientCertificateForDonor = migrationCertificates.recipientCertificateForDonor,
},
opts = {}) {
const {
waitForMigrationToComplete = false,
retryOnRetryableErrors = false,
enableDonorStartMigrationFsync = false,
} = opts;
const cmdObj = {
donorStartMigration: 1,
tenantId,
migrationId: UUID(migrationIdString),
recipientConnectionString,
readPreference,
donorCertificateForRecipient,
recipientCertificateForDonor
};
const stateRes = TenantMigrationUtil.runTenantMigrationCommand(cmdObj, this.getDonorRst(), {
enableDonorStartMigrationFsync,
retryOnRetryableErrors,
shouldStopFunc: stateRes =>
(!waitForMigrationToComplete || TenantMigrationUtil.isMigrationCompleted(stateRes))
});
// If the migration has been successfully committed, check the db hashes for the tenantId
// between the donor and recipient.
if (stateRes.state === TenantMigrationTest.State.kCommitted) {
TenantMigrationUtil.checkTenantDBHashes(
this.getDonorRst(), this.getRecipientRst(), tenantId);
}
return stateRes;
};
/**
* Runs the donorForgetMigration command with the given migrationId and returns the response.
*
* If 'retryOnRetryableErrors' is set, this function will retry if the command fails with a
* NotPrimary or network error.
*/
this.forgetMigration = function(migrationIdString, retryOnRetryableErrors = false) {
const cmdObj = {donorForgetMigration: 1, migrationId: UUID(migrationIdString)};
const res = TenantMigrationUtil.runTenantMigrationCommand(
cmdObj, this.getDonorRst(), {retryOnRetryableErrors});
// If the command succeeded, we expect that the migration is marked garbage collectable on
// the donor and the recipient. Check the state docs for expireAt, check that the oplog
// buffer collection has been dropped, and external keys have ttlExpiresAt.
if (res.ok) {
const donorPrimary = this.getDonorPrimary();
const recipientPrimary = this.getRecipientPrimary();
const donorStateDoc =
donorPrimary.getCollection(TenantMigrationTest.kConfigDonorsNS).findOne({
_id: UUID(migrationIdString)
});
const recipientStateDoc =
recipientPrimary.getCollection(TenantMigrationTest.kConfigRecipientsNS).findOne({
_id: UUID(migrationIdString)
});
if (donorStateDoc) {
assert(donorStateDoc.expireAt);
}
if (recipientStateDoc) {
assert(recipientStateDoc.expireAt);
}
const configDBCollections = recipientPrimary.getDB('config').getCollectionNames();
assert(!configDBCollections.includes('repl.migration.oplog_' + migrationIdString),
configDBCollections);
this.getDonorRst().asCluster(donorPrimary, () => {
const donorKeys =
TenantMigrationUtil.getExternalKeys(donorPrimary, UUID(migrationIdString));
if (donorKeys.length) {
donorKeys.forEach(key => {
assert(key.hasOwnProperty("ttlExpiresAt"), tojson(key));
});
}
});
this.getRecipientRst().asCluster(recipientPrimary, () => {
const recipientKeys =
TenantMigrationUtil.getExternalKeys(recipientPrimary, UUID(migrationIdString));
if (recipientKeys.length) {
recipientKeys.forEach(key => {
assert(key.hasOwnProperty("ttlExpiresAt"), tojson(key));
});
}
});
}
return res;
};
/**
* Runs the donorAbortMigration command with the given migration options and returns the
* response.
*/
this.tryAbortMigration = function(migrationOpts, retryOnRetryableErrors = false) {
const cmdObj = {
donorAbortMigration: 1,
migrationId: UUID(migrationOpts.migrationIdString),
};
return TenantMigrationUtil.runTenantMigrationCommand(
cmdObj, this.getDonorRst(), {retryOnRetryableErrors});
};
/**
* Asserts that durable and in-memory state for the migration 'migrationId' and 'tenantId' is
* eventually deleted from the given nodes.
*/
this.waitForMigrationGarbageCollection = function(
migrationId, tenantId, donorNodes, recipientNodes) {
donorNodes = donorNodes || donorRst.nodes;
recipientNodes = recipientNodes || recipientRst.nodes;
donorNodes.forEach(node => {
const configDonorsColl = node.getCollection(TenantMigrationTest.kConfigDonorsNS);
assert.soon(() => 0 === configDonorsColl.count({_id: migrationId}), tojson(node));
let mtab;
assert.soon(() => {
mtab = this.getTenantMigrationAccessBlocker({donorNode: node, tenantId});
return !mtab;
}, tojson(mtab));
});
recipientNodes.forEach(node => {
const configRecipientsColl =
node.getCollection(TenantMigrationTest.kConfigRecipientsNS);
assert.soon(() => 0 === configRecipientsColl.count({_id: migrationId}), tojson(node));
let mtab;
assert.soon(() => {
mtab =
this.getTenantMigrationAccessBlocker({recipientNode: node, tenantId: tenantId});
return !mtab;
}, tojson(mtab));
});
};
/**
* Asserts that the migration 'migrationId' and 'tenantId' eventually goes to the
* expected state on all the given donor nodes.
*/
this.waitForDonorNodesToReachState = function(nodes, migrationId, tenantId, expectedState) {
nodes.forEach(node => {
assert.soon(
() => this.isDonorNodeInExpectedState(node, migrationId, tenantId, expectedState));
});
};
/**
* Asserts that the migration 'migrationId' and 'tenantId' is in the expected state on all the
* given donor nodes.
*/
this.assertDonorNodesInExpectedState = function(nodes, migrationId, tenantId, expectedState) {
nodes.forEach(node => {
assert(this.isDonorNodeInExpectedState(node, migrationId, tenantId, expectedState));
});
};
/**
* Returns true if the durable and in-memory state for the migration 'migrationId' and
* 'tenantId' is in the expected state, and false otherwise.
*/
this.isDonorNodeInExpectedState = function(node, migrationId, tenantId, expectedState) {
const configDonorsColl =
this.getDonorPrimary().getCollection(TenantMigrationTest.kConfigDonorsNS);
const configDoc = configDonorsColl.findOne({_id: migrationId});
if (!configDoc || configDoc.state !== expectedState) {
return false;
}
const expectedAccessState = (expectedState === TenantMigrationTest.State.kCommitted)
? TenantMigrationTest.DonorAccessState.kReject
: TenantMigrationTest.DonorAccessState.kAborted;
const mtab = this.getTenantMigrationAccessBlocker({donorNode: node, tenantId});
return (mtab.donor.state === expectedAccessState);
};
function buildErrorMsg(
migrationId, expectedState, expectedAccessState, configDoc, recipientMtab) {
return tojson({migrationId, expectedState, expectedAccessState, configDoc, recipientMtab});
}
/**
* Asserts that the migration 'migrationId' and 'tenantId' eventually goes to the expected state
* on all the given recipient nodes.
*/
this.waitForRecipientNodesToReachState = function(
nodes, migrationId, tenantId, expectedState, expectedAccessState) {
nodes.forEach(node => {
let result = {};
assert.soon(
() => {
result = this.isRecipientNodeInExpectedState(
node, migrationId, tenantId, expectedState, expectedAccessState);
return result.value;
},
() => {
return "waitForRecipientNodesToReachState failed: " +
buildErrorMsg(migrationId,
expectedState,
expectedAccessState,
result.configDoc,
result.recipientMtab);
});
});
};
/**
* Asserts that the migration 'migrationId' and 'tenantId' is in the expected state on all the
* given recipient nodes.
*/
this.assertRecipientNodesInExpectedState = function(
nodes, migrationId, tenantId, expectedState, expectedAccessState) {
nodes.forEach(node => {
let result = this.isRecipientNodeInExpectedState(
node, migrationId, tenantId, expectedState, expectedAccessState);
assert(result.value, () => {
return "assertRecipientNodesInExpectedState failed: " +
buildErrorMsg(migrationId,
expectedState,
expectedAccessState,
result.configDoc,
result.recipientMtab);
});
});
};
/**
* Returns true if the durable and in-memory state for the migration 'migrationId' and
* 'tenantId' is in the expected state, and false otherwise.
*/
this.isRecipientNodeInExpectedState = function(
node, migrationId, tenantId, expectedState, expectedAccessState) {
const configRecipientsColl =
this.getRecipientPrimary().getCollection("config.tenantMigrationRecipients");
const configDoc = configRecipientsColl.findOne({_id: migrationId});
const mtab = this.getTenantMigrationAccessBlocker({recipientNode: node, tenantId});
let checkStates = () => {
if (!configDoc || configDoc.state !== expectedState) {
return false;
}
return (mtab.recipient.state === expectedAccessState);
};
return {value: checkStates(), configDoc: configDoc, recipientMtab: mtab.recipient};
};
/**
* Verifies that the documents on the recipient primary are correct.
*/
this.verifyRecipientDB = function(
tenantId, dbName, collName, migrationCommitted = true, data = loadDummyData()) {
// We should migrate all data regardless of tenant id for shard merge.
const shouldMigrate = migrationCommitted &&
(TenantMigrationUtil.isShardMergeEnabled(this.getRecipientPrimary().getDB("admin")) ||
TenantMigrationUtil.isNamespaceForTenant(tenantId, dbName));
jsTestLog(`Verifying that data in collection ${collName} of DB ${dbName} was ${
(shouldMigrate ? "" : "not")} migrated to the recipient`);
const db = this.getRecipientPrimary().getDB(dbName);
const coll = db.getCollection(collName);
const findRes = coll.find();
const numDocsFound = findRes.count();
if (!shouldMigrate) {
assert.eq(0,
numDocsFound,
`Find command on recipient collection ${collName} of DB ${
dbName} should return 0 docs, instead has count of ${numDocsFound}`);
return;
}
const numDocsExpected = data.length;
assert.eq(numDocsExpected,
numDocsFound,
`Find command on recipient collection ${collName} of DB ${dbName} should return ${
numDocsExpected} docs, instead has count of ${numDocsFound}`);
const docsReturned = findRes.sort({_id: 1}).toArray();
assert(arrayEq(docsReturned, data),
() => (`${tojson(docsReturned)} is not equal to ${tojson(data)}`));
};
/**
* Crafts a tenant database name.
*/
this.tenantDB = function(tenantId, dbName) {
return `${tenantId}_${dbName}`;
};
/**
* Crafts a database name that does not belong to the tenant.
*/
this.nonTenantDB = function(tenantId, dbName) {
return `non_${tenantId}_${dbName}`;
};
/**
* Returns the TenantMigrationAccessBlocker serverStatus output for the migration or shard merge
* for the given node.
*/
this.getTenantMigrationAccessBlocker = function(obj) {
return TenantMigrationUtil.getTenantMigrationAccessBlocker(obj);
};
/**
* Returns the TenantMigrationStats on the node.
*/
this.getTenantMigrationStats = function(node) {
return assert.commandWorked(node.adminCommand({serverStatus: 1})).tenantMigrations;
};
/**
* Awaits the condition when every stats counter reaches the specified count.
*/
this.awaitTenantMigrationStatsCounts = function(node, {
currentMigrationsDonating = 0,
currentMigrationsReceiving = 0,
totalSuccessfulMigrationsDonated = 0,
totalSuccessfulMigrationsReceived = 0,
totalFailedMigrationsDonated = 0,
totalFailedMigrationsReceived = 0
}) {
const check = function(expectedVal, stats, fieldName) {
if (expectedVal == stats[fieldName]) {
return true; // Condition reached, true means the counter reached the target.
}
assert.gt(expectedVal,
stats[fieldName],
`Stat ${fieldName} value ${stats[fieldName]} exceeded the target`);
return false;
};
let stats;
assert.soon(() => {
stats = this.getTenantMigrationStats(node);
if (check(currentMigrationsDonating, stats, "currentMigrationsDonating") &&
check(currentMigrationsReceiving, stats, "currentMigrationsReceiving") &&
check(
totalSuccessfulMigrationsDonated, stats, "totalSuccessfulMigrationsDonated") &&
check(totalSuccessfulMigrationsReceived,
stats,
"totalSuccessfulMigrationsReceived") &&
check(totalFailedMigrationsDonated, stats, "totalFailedMigrationsDonated") &&
check(totalFailedMigrationsReceived, stats, "totalFailedMigrationsReceived")) {
return true; // Done.
}
return false;
}, `Awaiting for tenant migration stats to reach target, got ${tojson(stats)}`);
};
/**
* Returns the donor ReplSetTest.
*/
this.getDonorRst = function() {
return donorRst;
};
/**
* Returns the recipient ReplSetTest.
*/
this.getRecipientRst = function() {
return recipientRst;
};
/**
* Returns the donor's primary.
*/
this.getDonorPrimary = function() {
return this.getDonorRst().getPrimary();
};
/**
* Returns the recipient's primary.
*/
this.getRecipientPrimary = function() {
return this.getRecipientRst().getPrimary();
};
/**
* Returns the recipient's connection string.
*/
this.getRecipientConnString = function() {
return this.getRecipientRst().getURL();
};
/**
* Shuts down the donor and recipient sets, only if they were not passed in as parameters.
* If they were passed in, the test that initialized them should be responsible for shutting
* them down.
*/
this.stop = function() {
if (!donorPassedIn)
donorRst.stopSet();
if (!recipientPassedIn)
recipientRst.stopSet();
};
}
/**
* Takes in the response to the donarStartMigration command and asserts the command
* works and the state is 'committed'.
*/
TenantMigrationTest.assertCommitted = function(stateRes) {
assert.commandWorked(stateRes);
assert.eq(stateRes.state, TenantMigrationTest.DonorState.kCommitted, tojson(stateRes));
return stateRes;
};
/**
* Takes in the response to the donarStartMigration command and asserts the command
* works and the state is 'aborted', with optional errorCode.
*/
TenantMigrationTest.assertAborted = function(stateRes, errorCode) {
assert.commandWorked(stateRes);
assert.eq(stateRes.state, TenantMigrationTest.DonorState.kAborted, tojson(stateRes));
if (errorCode !== undefined) {
assert.eq(stateRes.abortReason.code, errorCode, tojson(stateRes));
}
return stateRes;
};
TenantMigrationTest.DonorState = {
kCommitted: "committed",
kAborted: "aborted",
kDataSync: "data sync",
kBlocking: "blocking",
kAbortingIndexBuilds: "aborting index builds",
};
TenantMigrationTest.RecipientState = {
kUninitialized: "uninitialized",
kStarted: "started",
kConsistent: "consistent",
kDone: "done",
kLearnedFilenames: "learned filenames",
};
TenantMigrationTest.RecipientStateEnum =
Object.keys(TenantMigrationTest.RecipientState).reduce((acc, key, idx) => {
acc[key] = idx;
return acc;
}, {});
TenantMigrationTest.State = TenantMigrationTest.DonorState;
TenantMigrationTest.DonorAccessState = {
kAllow: "allow",
kBlockWrites: "blockWrites",
kBlockWritesAndReads: "blockWritesAndReads",
kReject: "reject",
kAborted: "aborted",
};
TenantMigrationTest.RecipientAccessState = {
kReject: "reject",
kRejectBefore: "rejectBefore"
};
TenantMigrationTest.kConfigDonorsNS = "config.tenantMigrationDonors";
TenantMigrationTest.kConfigRecipientsNS = "config.tenantMigrationRecipients";