mongo/jstests/multiVersion/libs/multi_rs.js

//
// Utility functions for multi-version replica sets
//

/**
 * @param options {Object} see ReplSetTest.start & MongoRunner.runMongod.
 * @param user {string} optional, user name for authentication.
 * @param pwd {string} optional, password for authentication. Must be set if user is set.
 */
ReplSetTest.prototype.upgradeSet = function(options, user, pwd) {
    let primary = this.getPrimary();

    // Upgrade secondaries first.
    this.upgradeSecondaries(primary, options, user, pwd);

    // Then upgrade the primary after stepping down.
    this.upgradePrimary(primary, options, user, pwd);

};

ReplSetTest.prototype.upgradeSecondaries = function(primary, options, user, pwd) {
    const noDowntimePossible = this.nodes.length > 2;

    // Merge new options into node settings.
    for (let nodeName in this.nodeOptions) {
        this.nodeOptions[nodeName] = Object.merge(this.nodeOptions[nodeName], options);
    }

    for (let secondary of this.getSecondaries()) {
        this.upgradeNode(secondary, options, user, pwd);

        if (noDowntimePossible)
            assert.eq(this.getPrimary(), primary);
    }
};

ReplSetTest.prototype.upgradePrimary = function(primary, options, user, pwd) {
    const noDowntimePossible = this.nodes.length > 2;

    // Merge new options into node settings.
    for (let nodeName in this.nodeOptions) {
        this.nodeOptions[nodeName] = Object.merge(this.nodeOptions[nodeName], options);
    }

    let oldPrimary = this.stepdown(primary);
    this.waitForState(oldPrimary, ReplSetTest.State.SECONDARY);

    // stepping down the node can close the connection and lose the authentication state, so
    // re-authenticate here before calling awaitNodesAgreeOnPrimary().
    if (user != undefined) {
        oldPrimary.getDB('admin').auth(user, pwd);
    }
    jsTest.authenticate(oldPrimary);

    this.awaitNodesAgreeOnPrimary();
    primary = this.getPrimary();

    this.upgradeNode(oldPrimary, options, user, pwd);

    let newPrimary = this.getPrimary();

    if (noDowntimePossible)
        assert.eq(newPrimary, primary);

    return newPrimary;
};

ReplSetTest.prototype.upgradeNode = function(node, opts = {}, user, pwd) {
    if (user != undefined) {
        assert.eq(1, node.getDB("admin").auth(user, pwd));
    }
    jsTest.authenticate(node);

    var isMaster = node.getDB('admin').runCommand({isMaster: 1});

    if (!isMaster.arbiterOnly) {
        // Must retry this command, as it might return "currently running for election" and fail.
        // Node might still be running for an election that will fail because it lost the election
        // race with another node, at test initialization.  See SERVER-23133.
        assert.soonNoExcept(function() {
            assert.commandWorked(node.adminCommand("replSetMaintenance"));
            return true;
        });
        this.waitForState(node, ReplSetTest.State.RECOVERING);
    }

    var newNode = this.restart(node, opts);
    if (user != undefined) {
        newNode.getDB("admin").auth(user, pwd);
    }

    var waitForStates =
        [ReplSetTest.State.PRIMARY, ReplSetTest.State.SECONDARY, ReplSetTest.State.ARBITER];
    this.waitForState(newNode, waitForStates);

    return newNode;
};

ReplSetTest.prototype.stepdown = function(nodeId) {
    nodeId = this.getNodeId(nodeId);
    assert.eq(this.getNodeId(this.getPrimary()), nodeId);
    var node = this.nodes[nodeId];

    assert.soonNoExcept(function() {
        // Due to a rare race condition in stepdown, it's possible the secondary just replicated
        // the most recent write and sent replSetUpdatePosition to the primary, and that
        // replSetUpdatePosition command gets interrupted by the stepdown.  In that case,
        // the secondary will clear its sync source, but will be unable to re-connect to the
        // primary that is trying to step down, because they are at the same OpTime.  The primary
        // will then get stuck waiting forever for the secondary to catch up so it can complete
        // stepdown.  Adding a garbage write here ensures that the secondary will be able to
        // resume syncing from the primary in this case, which in turn will let the primary
        // finish stepping down successfully.
        node.getDB('admin').garbageWriteToAdvanceOpTime.insert({a: 1});
        assert.adminCommandWorkedAllowingNetworkError(
            node, {replSetStepDown: 5 * 60, secondaryCatchUpPeriodSecs: 60});
        return true;
    });

    return this.reconnect(node);
};

ReplSetTest.prototype.reconnect = function(node) {
    var nodeId = this.getNodeId(node);
    this.nodes[nodeId] = new Mongo(node.host);
    var except = {};
    for (var i in node) {
        if (typeof(node[i]) == "function")
            continue;
        this.nodes[nodeId][i] = node[i];
    }

    return this.nodes[nodeId];
};

ReplSetTest.prototype.conf = function() {
    var admin = this.getPrimary().getDB('admin');

    var resp = admin.runCommand({replSetGetConfig: 1});

    if (resp.ok && !(resp.errmsg) && resp.config)
        return resp.config;

    else if (resp.errmsg && resp.errmsg.startsWith("no such cmd"))
        return admin.getSisterDB("local").system.replset.findOne();

    throw new Error("Could not retrieve replica set config: " + tojson(resp));
};