mongo/jstests/replsets/libs/initial_sync_update_missing_doc.js

"use strict";
/**
 * Initial sync runs in several phases - the first 3 are as follows:
 * 1) fetches the last oplog entry (op_start1) on the source;
 * 2) copies all non-local databases from the source; and
 * 3) fetches and applies operations from the source after op_start1.
 *
 * This library is used to delete documents on the sync source between the first two phases so
 * that the secondary will fail to apply the update operation in phase three.
 */

// reInitiate the replica set with a secondary node, which will go through initial sync. This
// function will hand the secondary in initial sync. turnOffHangBeforeCopyingDatabasesFailPoint
// must be called after reInitiateSetWithSecondary, followed by
// turnOffHangBeforeGettingMissingDocFailPoint.
var reInitiateSetWithSecondary = function(replSet, secondaryConfig) {

    const secondary = replSet.add(secondaryConfig);
    secondary.setSlaveOk();

    // Make the secondary hang after retrieving the last op on the sync source but before
    // copying databases.
    assert.commandWorked(secondary.getDB('admin').runCommand(
        {configureFailPoint: 'initialSyncHangBeforeCopyingDatabases', mode: 'alwaysOn'}));
    assert.commandWorked(secondary.getDB('admin').runCommand(
        {configureFailPoint: 'initialSyncHangBeforeGettingMissingDocument', mode: 'alwaysOn'}));

    // Skip clearing initial sync progress after a successful initial sync attempt so that we
    // can check initialSyncStatus fields after initial sync is complete.
    assert.commandWorked(secondary.adminCommand(
        {configureFailPoint: 'skipClearInitialSyncState', mode: 'alwaysOn'}));

    replSet.reInitiate();

    // Wait for fail point message to be logged.
    checkLog.contains(secondary,
                      'initial sync - initialSyncHangBeforeCopyingDatabases fail point enabled');

    return secondary;

};

// Must be called after reInitiateSetWithSecondary. Turns off the
// initialSyncHangBeforeCopyingDatabases fail point so that the secondary will start copying all
// non-local databases.
var turnOffHangBeforeCopyingDatabasesFailPoint = function(secondary) {

    assert.commandWorked(secondary.getDB('admin').runCommand(
        {configureFailPoint: 'initialSyncHangBeforeCopyingDatabases', mode: 'off'}));

    // The following checks assume that we have updated and deleted a document on the sync source
    // that the secondary will try to update in phase 3.
    checkLog.contains(secondary, 'update of non-mod failed');
    checkLog.contains(secondary, 'Fetching missing document');
    checkLog.contains(
        secondary, 'initial sync - initialSyncHangBeforeGettingMissingDocument fail point enabled');
};

// Must be called after turnOffHangBeforeCopyingDatabasesFailPoint. Turns off the
// initialSyncHangBeforeGettingMissingDocument fail point so that the secondary can check if the
// sync source has the missing document.
var turnOffHangBeforeGettingMissingDocFailPoint = function(primary, secondary, name, numInserted) {

    if (numInserted === 0) {
        // If we did not re-insert the missing document, insert an arbitrary document to move
        // forward minValid even though the document was not found.
        assert.commandWorked(
            primary.getDB('test').getCollection(name + 'b').insert({_id: 1, y: 1}));
    }

    assert.commandWorked(secondary.getDB('admin').runCommand(
        {configureFailPoint: 'initialSyncHangBeforeGettingMissingDocument', mode: 'off'}));

    // If we've re-inserted the missing document between secondaryHangsBeforeGettingMissingDoc and
    // this function, the secondary will insert the missing document after it fails the update.
    // Otherwise, it will fail to fetch anything from the sync source because the document was
    // deleted.
    if (numInserted > 0) {
        checkLog.contains(secondary, 'Inserted missing document');
    } else {
        checkLog.contains(
            secondary, 'Missing document not found on source; presumably deleted later in oplog.');
    }
    checkLog.contains(secondary, 'initial sync done');

};

var finishAndValidate = function(replSet, name, firstOplogEnd, numInserted, numCollections) {

    replSet.awaitReplication();
    replSet.awaitSecondaryNodes();
    const dbName = 'test';
    const secondary = replSet.getSecondary();

    assert.eq(numCollections,
              secondary.getDB(dbName).getCollection(name).find().itcount(),
              'collection successfully synced to secondary');

    const res = assert.commandWorked(secondary.adminCommand({replSetGetStatus: 1}));

    // If we haven't re-inserted any documents after deleting them, the fetch count is 0 because we
    // are unable to get the document from the sync source.
    assert.eq(res.initialSyncStatus.fetchedMissingDocs, numInserted);

    const finalOplogEnd = res.initialSyncStatus.initialSyncOplogEnd;

    if (numInserted > 0) {
        assert.neq(firstOplogEnd,
                   finalOplogEnd,
                   "minValid was not moved forward when missing document was fetched");
    } else {
        assert.eq(firstOplogEnd,
                  finalOplogEnd,
                  "minValid was moved forward when missing document was not fetched");
    }

    assert.eq(0,
              secondary.getDB('local')['temp_oplog_buffer'].find().itcount(),
              "Oplog buffer was not dropped after initial sync");

};