mongo/jstests/libs/election_timing_test.js

/**
 * ElectionTimingTest - set up a ReplSetTest and use default or provided functions to
 *  trigger an election. The time it takes to discover a new primary is recorded.
 */
var ElectionTimingTest = function(opts) {
    // How many times do we start a new ReplSetTest.
    this.testRuns = opts.testRuns || 1;

    // How many times do we step down during a ReplSetTest"s lifetime.
    this.testCycles = opts.testCycles || 1;

    // The config is set to two electable nodes since we use waitForMemberState
    // to wait for the electable secondary to become primary.
    this.nodes = opts.nodes || [
        {},
        {},
        {rsConfig: {arbiterOnly: true}}
    ];

    // The name of the replica set and of the collection.
    this.name = opts.name || "election_timing";

    // Pass additional replicaSet config options.
    this.settings = opts.settings || {};

    // pv1 is the default in master and here.
    this.protocolVersion = opts.hasOwnProperty("protocolVersion") ? opts.protocolVersion : 1;

    // A function that runs after the ReplSetTest is initialized.
    this.testSetup = opts.testSetup || Function.prototype;

    // A function that triggers election, default is to kill the mongod process.
    this.electionTrigger = opts.electionTrigger || this.stopPrimary;

    // A function that waits for new primary to be elected.
    this.waitForNewPrimary = opts.waitForNewPrimary || this.waitForNewPrimary;

    // A function that cleans up after the election trigger.
    this.testReset = opts.testReset || this.stopPrimaryReset;

    // The interval passed to stepdown that primaries may not seek re-election.
    // We also have to wait out this interval before allowing another stepdown.
    this.stepDownGuardTime = opts.stepDownGuardTime || 60;

    // Test results will be stored in these arrays.
    this.testResults = [];
    this.testErrors = [];

    this._runTimingTest();
};

ElectionTimingTest.prototype._runTimingTest = function() {
    for (var run = 0; run < this.testRuns; run++) {
        var collectionName = "test." + this.name;
        var cycleData = {testRun: run, results: []};

        jsTestLog("Starting ReplSetTest for test " + this.name + " run: " + run);
        this.rst = new ReplSetTest({name: this.name, nodes: this.nodes, nodeOptions: {verbose:""}});
        this.rst.startSet();

        // Get the replset config and apply the settings object.
        var conf = this.rst.getReplSetConfig();
        conf.settings = conf.settings || {};
        conf.settings = Object.merge(conf.settings, this.settings);

        // Explicitly setting protocolVersion.
        conf.protocolVersion = this.protocolVersion;
        this.rst.initiate(conf);

        // Run the user supplied testSetup() method. Typical uses would be to set up
        // bridging, or wait for a particular state after initiate().
        try {
            this.testSetup();
        } catch (e) {
            // If testSetup() fails, we are in an unknown state, log and return.
            this.testErrors.push({testRun: run, status: "testSetup() failed", error: e});
            this.rst.stopSet();
            return;
        }

        // Create and populate a collection.
        var primary = this.rst.getPrimary();
        var coll = primary.getCollection(collectionName);
        var secondary = this.rst.getSecondary();

        this.electionTimeoutLimitMillis =
            ElectionTimingTest.calculateElectionTimeoutLimitMillis(primary);
        jsTestLog('Election timeout limit: ' + this.electionTimeoutLimitMillis + ' ms');

        for (var i = 0; i < 100; i++) {
            assert.writeOK(coll.insert({_id: i,
                                        x: i * 3,
                                        arbitraryStr: "this is a string"}));
        }

        // Make sure the secondaries are up then await replication.
        this.rst.awaitSecondaryNodes();
        this.rst.awaitReplication();

        // Run the election tests on this ReplSetTest instance.
        for (var cycle = 0; cycle < this.testCycles; cycle++) {
            jsTestLog("Starting test: " + this.name + " run: " + run + " cycle: " + cycle);
            var oldElectionId = primary.getDB("admin").isMaster().electionId;

            // Time the new election.
            var stepDownTime = Date.now();

            // Run the specified election trigger method. Default is to sigstop the primary.
            try {
                this.electionTrigger();
            } catch (e) {
                // Left empty on purpose.
            }

            // Wait for the electable secondary to become primary.
            try {
                this.waitForNewPrimary(this.rst, secondary);
            } catch (e) {
                // If we didn"t find a primary, save the error, break so this
                // ReplSetTest is stopped. We can"t continue from a flaky state.
                this.testErrors.push({testRun: run,
                                      cycle: cycle,
                                      status: "new primary not elected",
                                      error: e});
                break;
            }

            var electionCompleteTime = Date.now();

            // Verify we had an election and we have a new primary.
            var newPrimary = this.rst.getPrimary();
            var newElectionId = newPrimary.getDB("admin").isMaster().electionId;
            if (bsonWoCompare(oldElectionId, newElectionId) !== 0) {
                this.testErrors.push({testRun: run,
                                      cycle: cycle,
                                      status: "electionId not changed, no election was triggered"});
                break;
            }

            if (primary.host === newPrimary.host) {
                this.testErrors.push({testRun: run,
                                      cycle: cycle,
                                      status: "Previous primary was re-elected"});
                break;
            }

            cycleData.results.push((electionCompleteTime - stepDownTime) / 1000);

            // If we are running another test on this ReplSetTest, call the reset function.
            if (cycle + 1 < this.testCycles) {
                try {
                    this.testReset();
                } catch (e) {
                    this.testErrors.push({testRun: run,
                                          cycle: cycle,
                                          status: "testReset() failed",
                                          error: e});
                    break;
                }
            }
            // Wait for replication. When there are only two nodes in the set,
            // the previous primary should be given a chance to catch up or
            // else there will be rollbacks after the next election cycle.
            this.rst.awaitSecondaryNodes();
            this.rst.awaitReplication();
            primary = newPrimary;
            secondary = this.rst.getSecondary();
        }
        this.testResults.push(cycleData);
        this.rst.stopSet();
    }
};

ElectionTimingTest.prototype.stopPrimary = function() {
    this.originalPrimary = this.rst.getNodeId(this.rst.getPrimary());
    this.rst.stop(this.originalPrimary);
};

ElectionTimingTest.prototype.stopPrimaryReset = function() {
    this.rst.restart(this.originalPrimary);
};

ElectionTimingTest.prototype.stepDownPrimary = function() {
    var adminDB = this.rst.getPrimary().getDB("admin");
    adminDB.runCommand({replSetStepDown: this.stepDownGuardTime, force: true});
};

ElectionTimingTest.prototype.stepDownPrimaryReset = function() {
    sleep(this.stepDownGuardTime * 1000);
};

ElectionTimingTest.prototype.waitForNewPrimary = function(rst, secondary) {
    assert.commandWorked(
        secondary.adminCommand({
            replSetTest: 1,
            waitForMemberState: ReplSetTest.State.PRIMARY,
            timeoutMillis: 60 * 1000
        }),
        "node " + secondary.host + " failed to become primary"
    );
};

/**
 * Calculates upper limit for actual failover time in milliseconds.
 */
ElectionTimingTest.calculateElectionTimeoutLimitMillis = function(primary) {
    var configResult = assert.commandWorked(primary.adminCommand({replSetGetConfig: 1}));
    var config = configResult.config;
    // Protocol version is 0 if missing from config.
    var protocolVersion = config.hasOwnProperty("protocolVersion") ? config.protocolVersion : 0;
    var electionTimeoutMillis = 0;
    var electionTimeoutOffsetLimitFraction = 0;
    if (protocolVersion == 0) {
        electionTimeoutMillis = 30000;  // from TopologyCoordinatorImpl::VoteLease::leaseTime
        electionTimeoutOffsetLimitFraction = 0;
    } else {
        electionTimeoutMillis = config.settings.electionTimeoutMillis;
        var getParameterResult = assert.commandWorked(primary.adminCommand({
            getParameter: 1,
            replElectionTimeoutOffsetLimitFraction: 1,
        }));
        electionTimeoutOffsetLimitFraction =
            getParameterResult.replElectionTimeoutOffsetLimitFraction;
    }
    var assertSoonIntervalMillis = 200;  // from assert.js
    var applierDrainWaitMillis = 1000;  // from SyncTail::tryPopAndWaitForMore()
    var electionTimeoutLimitMillis =
        (1 + electionTimeoutOffsetLimitFraction) * electionTimeoutMillis +
        applierDrainWaitMillis +
        assertSoonIntervalMillis;
    return electionTimeoutLimitMillis;
}