post 2.2.7

Conflicts:
	src/mongo/db/ops/update.cpp

doxygenConfig

@@ -3,7 +3,7 @@
 #---------------------------------------------------------------------------
 DOXYFILE_ENCODING      = UTF-8
 PROJECT_NAME           = MongoDB
-PROJECT_NUMBER         = 2.2.4
+PROJECT_NUMBER         = 2.2.8-pre-
 OUTPUT_DIRECTORY       = docs/doxygen
 CREATE_SUBDIRS         = NO
 OUTPUT_LANGUAGE        = English

jstests/replsets/rollback5.js

@@ -0,0 +1,102 @@
+// test that a rollback directory is created during a replica set rollback
+// this also tests that updates are recorded in the rollback file
+//  (this test does no delete rollbacks)
+
+var replTest = new ReplSetTest({ name: 'rollback5', nodes: 3 });
+var nodes = replTest.nodeList();
+
+var conns = replTest.startSet();
+var r = replTest.initiate({ "_id": "rollback5",
+                            "members": [
+                                { "_id": 0, "host": nodes[0] },
+                                { "_id": 1, "host": nodes[1] },
+                                { "_id": 2, "host": nodes[2], arbiterOnly: true}]
+                          });
+
+// Make sure we have a master
+var master = replTest.getMaster();
+var a_conn = conns[0];
+var b_conn = conns[1];
+a_conn.setSlaveOk();
+b_conn.setSlaveOk();
+var A = a_conn.getDB("test");
+var B = b_conn.getDB("test");
+var AID = replTest.getNodeId(a_conn);
+var BID = replTest.getNodeId(b_conn);
+var Apath = "/data/db/rollback5-0/";
+var Bpath = "/data/db/rollback5-1/";
+assert(master == conns[0], "conns[0] assumed to be master");
+assert(a_conn.host == master.host);
+
+// Make sure we have an arbiter
+assert.soon(function () {
+    res = conns[2].getDB("admin").runCommand({ replSetGetStatus: 1 });
+    return res.myState == 7;
+}, "Arbiter failed to initialize.");
+
+A.foo.update({key:'value1'}, {$set: {req: 'req'}}, true);
+A.foo.runCommand({getLastError : 1, w : 2, wtimeout : 60000});
+replTest.stop(AID);
+
+master = replTest.getMaster();
+assert(b_conn.host == master.host);
+B.foo.update({key:'value1'}, {$set: {res: 'res'}}, true);
+B.foo.runCommand({getLastError : 1, w : 1, wtimeout : 60000});
+replTest.stop(BID);
+replTest.restart(AID);
+master = replTest.getMaster();
+assert(a_conn.host == master.host);
+A.foo.update({key:'value2'}, {$set: {req: 'req'}}, true);
+A.foo.runCommand({getLastError : 1, w : 1, wtimeout : 60000});
+replTest.restart(BID); // should rollback
+reconnect(B);
+
+print("BEFORE------------------");
+printjson(A.foo.find().toArray());
+
+replTest.awaitReplication();
+replTest.awaitSecondaryNodes();
+
+print("AFTER------------------");
+printjson(A.foo.find().toArray());
+
+assert.eq(2, A.foo.count());
+assert.eq('req', A.foo.findOne({key:'value1'}).req);
+assert.eq(null, A.foo.findOne({key:'value1'}).res);
+assert.eq(2, B.foo.count());
+assert.eq('req', B.foo.findOne({key:'value1'}).req);
+assert.eq(null, B.foo.findOne({key:'value1'}).res);
+
+// check here for rollback files
+var rollbackDir = Bpath + "rollback/";
+assert(pathExists(rollbackDir), "rollback directory was not created!");
+
+print("rollback5.js SUCCESS");
+replTest.stopSet(15);
+
+
+function wait(f) {
+    var n = 0;
+    while (!f()) {
+        if (n % 4 == 0)
+            print("rollback5.js waiting");
+        if (++n == 4) {
+            print("" + f);
+        }
+        assert(n < 200, 'tried 200 times, giving up');
+        sleep(1000);
+    }
+}
+
+function reconnect(a) {
+  wait(function() {
+      try {
+        a.bar.stats();
+        return true;
+      } catch(e) {
+        print(e);
+        return false;
+      }
+    });
+};
+

jstests/replsets/stepdown3.js

@@ -0,0 +1,48 @@
+// Test that GLE asserts when the primary steps down while we're waiting for w:
+
+var replTest = new ReplSetTest({ name: 'testSet', nodes: 2 });
+var nodes = replTest.startSet();
+replTest.initiate();
+var master = replTest.getMaster();
+
+// do a write to allow stepping down of the primary;
+// otherwise, the primary will refuse to step down
+print("\ndo a write");
+master.getDB("test").foo.insert({x:1});
+replTest.awaitReplication();
+
+// do another write, because the first one might be longer than 10 seconds ago
+// on the secondary (due to starting up), and we need to be within 10 seconds
+// to step down.
+master.getDB("test").foo.insert({x:2});
+master.getDB("test").runCommand({getLastError : 1, w : 2, wtimeout : 30000 });
+// lock secondary, to pause replication
+print("\nlock secondary");
+var locked = replTest.liveNodes.slaves[0];
+printjson( locked.getDB("admin").runCommand({fsync : 1, lock : 1}) );
+
+// do a write
+print("\ndo a write");
+master.getDB("test").foo.insert({x:3});
+
+// step down the primary asyncronously
+print("stepdown");
+var command = "sleep(4000); tojson(db.adminCommand( { replSetStepDown : 60, force : 1 } ));"
+var waitfunc = startParallelShell(command, master.port);
+
+print("getlasterror; should assert or return an error, depending on timing");
+var gleFunction = function() { 
+    var result = master.getDB("test").runCommand({getLastError : 1, w: 2 , wtimeout :30000 });
+    if (result.errmsg === "not master") {
+        throw new Error("satisfy assert.throws()");
+    }
+    print("failed to throw exception; GLE returned: ");
+    printjson(result);
+};
+var result = assert.throws(gleFunction);
+print("result of gle:");
+printjson(result);
+
+// unlock and shut down
+printjson(locked.getDB("admin").$cmd.sys.unlock.findOne());
+replTest.stopSet();

jstests/sharding/gle_error_message.js

@@ -111,7 +111,11 @@ jsTest.log( "Testing stale version GLE when host goes down..." )
 var staleColl = st.s1.getCollection( coll + "" )
 staleColl.findOne()
 
-printjson( admin.runCommand({ moveChunk : "" + coll, find : { _id : 0 }, to : shards[2]._id }) )
+// As it turns out, on the *second* auto-reconnect attempt we need to wait at least 2 secs,
+// otherwise reconnect fails with FAILED_STATE 
+sleep( 2000 );
+
+assert( admin.runCommand({ moveChunk : "" + coll, find : { _id : 0 }, to : shards[2]._id }).ok );
 
 MongoRunner.stopMongod( st.shard2 )
 
@@ -124,4 +128,4 @@ assert.neq( null, staleColl.getDB().getLastError() )
 
 jsTest.log( "Done!" )
 
-st.stop()
+st.stop()

jstests/slowNightly/sharding_migrate_large_docs.js

@@ -0,0 +1,63 @@
+//
+// Tests migration behavior of large documents
+//
+
+var st = new ShardingTest({ shards : 2, mongos : 1,
+                            other : { separateConfig : true,
+                                      mongosOptions : { noAutoSplit : "" },
+                                      shardOptions : { /* binVersion : "latest" */ } } });
+st.stopBalancer()
+
+var mongos = st.s0;
+var coll = mongos.getCollection( "foo.bar" );
+var admin = mongos.getDB( "admin" );
+var shards = mongos.getCollection( "config.shards" ).find().toArray();
+var shardAdmin = st.shard0.getDB( "admin" );
+
+assert( admin.runCommand({ enableSharding : coll.getDB() + "" }).ok );
+printjson( admin.runCommand({ movePrimary : coll.getDB() + "", to : shards[0]._id }) );
+assert( admin.runCommand({ shardCollection : coll + "", key : { _id : 1 } }).ok );
+assert( admin.runCommand({ split : coll + "", middle : { _id : 0 } }).ok );
+
+jsTest.log( "Preparing large insert..." );
+
+var data1MB = "x"
+while ( data1MB.length < 1024 * 1024 )
+    data1MB += data1MB;
+
+var data15MB = "";
+for ( var i = 0; i < 15; i++ ) data15MB += data1MB;
+
+var data15PlusMB = data15MB;
+for ( var i = 0; i < 1023 * 1024; i++ ) data15PlusMB += "x";
+
+print("~15MB object size is : " + Object.bsonsize({ _id : 0, d : data15PlusMB }));
+
+jsTest.log( "Inserting docs of large and small sizes..." );
+
+// Two large docs next to each other
+coll.insert({ _id : -2, d : data15PlusMB });
+coll.insert({ _id : -1, d : data15PlusMB });
+
+// Docs of assorted sizes
+coll.insert({ _id : 0, d : "x" });
+coll.insert({ _id : 1, d : data15PlusMB });
+coll.insert({ _id : 2, d : "x" });
+coll.insert({ _id : 3, d : data15MB });
+coll.insert({ _id : 4, d : "x" });
+coll.insert({ _id : 5, d : data1MB });
+coll.insert({ _id : 6, d : "x" });
+
+assert.eq( null, coll.getDB().getLastError() );
+assert.eq( 9, coll.find().itcount() );
+
+jsTest.log( "Starting migration..." );
+
+assert( admin.runCommand({ moveChunk : coll + "", find : { _id : 0 }, to : shards[1]._id }).ok );
+assert( admin.runCommand({ moveChunk : coll + "", find : { _id : -1 }, to : shards[1]._id }).ok );
+
+assert.eq( 9, coll.find().itcount() );
+
+jsTest.log( "DONE!" );
+
+st.stop();

rpm/mongo.spec

@@ -1,7 +1,7 @@
 Name: mongo-10gen
 Conflicts: mongo, mongo-10gen-unstable
 Obsoletes: mongo-stable
-Version: 2.2.3
+Version: 2.2.6
 Release: mongodb_1%{?dist}
 Summary: mongodb client shell and tools
 License: AGPL 3.0

src/mongo/bson/bsonelement.h

@@ -17,6 +17,7 @@
 
 #pragma once
 
+#include <boost/cstdint.hpp>
 #include <string.h> // strlen
 #include <string>
 #include <vector>
@@ -240,8 +241,8 @@ namespace mongo {
         }
 
         // for objects the size *includes* the size of the size field
-        int objsize() const {
-            return *reinterpret_cast< const int* >( value() );
+        size_t objsize() const {
+            return static_cast< const size_t >( *reinterpret_cast< const uint32_t* >( value() ) );
         }
 
         /** Get a string's value.  Also gives you start of the real data for an embedded object.

src/mongo/db/cmdline.cpp

@@ -520,13 +520,8 @@ namespace mongo {
 
     void ignoreSignal( int sig ) {}
 
-    static void rotateLogsOrDie(int sig) {
-        fassert(16176, rotateLogs());
-    }
-
     void setupCoreSignals() {
 #if !defined(_WIN32)
-        verify( signal(SIGUSR1 , rotateLogsOrDie ) != SIG_ERR );
         verify( signal(SIGHUP , ignoreSignal ) != SIG_ERR );
 #endif
     }

src/mongo/db/curop.h

@@ -101,8 +101,8 @@ namespace mongo {
 
         void set( const BSONObj& o ) {
             scoped_spinlock lk(_lock);
-            int sz = o.objsize();
-            if ( sz > (int) sizeof(_buf) ) {
+            size_t sz = o.objsize();
+            if ( sz > sizeof(_buf) ) {
                 _reset(TOO_BIG_SENTINEL);
             }
             else {

src/mongo/db/database.cpp

@@ -323,6 +323,7 @@ namespace mongo {
                 warning() << "quota exceeded, but can't assert, probably going over quota for: " << ns << endl;
             }
             else {
+                log() << "quota exceeded for namespace: " << ns << endl;
                 uasserted(12501, "quota exceeded");
             }
         }

src/mongo/db/db.cpp

@@ -1205,12 +1205,24 @@ namespace mongo {
     // The above signals will be processed by this thread only, in order to
     // ensure the db and log mutexes aren't held.
     void interruptThread() {
-        int actualSignal;
-        sigwait( &asyncSignals, &actualSignal );
-        log() << "got signal " << actualSignal << " (" << strsignal( actualSignal )
-              << "), will terminate after current cmd ends" << endl;
-        Client::initThread( "interruptThread" );
-        exitCleanly( EXIT_CLEAN );
+        while (true) {
+            int actualSignal = 0;
+            int status = sigwait( &asyncSignals, &actualSignal );
+            fassert(16781, status == 0);
+            switch (actualSignal) {
+            case SIGUSR1:
+                // log rotate signal
+                fassert(16782, rotateLogs());
+                break;
+            default:
+                // interrupt/terminate signal
+                Client::initThread( "signalProcessingThread" );
+                log() << "got signal " << actualSignal << " (" << strsignal( actualSignal )
+                      << "), will terminate after current cmd ends" << endl;
+                exitCleanly( EXIT_CLEAN );
+                break;
+            }
+        }
     }
 
     // this will be called in certain c++ error cases, for example if there are two active
@@ -1256,6 +1268,7 @@ namespace mongo {
         sigemptyset( &asyncSignals );
         sigaddset( &asyncSignals, SIGINT );
         sigaddset( &asyncSignals, SIGTERM );
+        sigaddset( &asyncSignals, SIGUSR1 );
         verify( pthread_sigmask( SIG_SETMASK, &asyncSignals, 0 ) == 0 );
         boost::thread it( interruptThread );
     }

src/mongo/db/dbcommands.cpp

@@ -241,6 +241,13 @@ namespace mongo {
                         break;
                     }
 
+                    if ( !_isMaster() ) {
+                        // this should be in the while loop in case we step down
+                        errmsg = "not master";
+                        result.append( "wnote", "no longer primary" );
+                        return false;
+                    }
+
                     // check this first for w=0 or w=1
                     if ( opReplicatedEnough( op, e ) ) {
                         break;

src/mongo/db/dbhelpers.cpp

@@ -380,7 +380,6 @@ namespace mongo {
         stringstream ss;
         ss << why << "." << terseCurrentTime(false) << "." << NUM++ << ".bson";
         _file /= ss.str();
-
     }
 
     RemoveSaver::~RemoveSaver() {
@@ -397,7 +396,8 @@ namespace mongo {
             _out = new ofstream();
             _out->open( _file.string().c_str() , ios_base::out | ios_base::binary );
             if ( ! _out->good() ) {
-                LOG( LL_WARNING ) << "couldn't create file: " << _file.string() << " for remove saving" << endl;
+                error() << "couldn't create file: " << _file.string() << 
+                    " for remove saving" << endl;
                 delete _out;
                 _out = 0;
                 return;

src/mongo/db/matcher.cpp

@@ -465,6 +465,8 @@ namespace mongo {
         uassert( 10066 , "$where may only appear once in query", _where == 0 );
         uassert( 10067 , "$where query, but no script engine", globalScriptEngine );
         massert( 13089 , "no current client needed for $where" , haveClient() );
+        uassert( 17126 , "no valid context found for $where", cc().getContext());
+
         _where = new Where( cc().ns() );
 
         if ( e.type() == CodeWScope ) {

src/mongo/db/ops/delete.cpp

@@ -129,9 +129,6 @@ namespace mongo {
                 }
             }
 
-            if ( rs )
-                rs->goingToDelete( rloc.obj() /*cc->c->current()*/ );
-
             theDataFileMgr.deleteRecord(ns, rloc.rec(), rloc);
             nDeleted++;
             if ( foundAllResults ) {

src/mongo/db/ops/update.cpp

@@ -364,9 +364,6 @@ namespace mongo {
                         d->paddingFits();
                     }
                     else {
-                        if ( rs )
-                            rs->goingToDelete( onDisk );
-
                         BSONObj newObj = mss->createNewFromMods();
                         checkTooLarge(newObj);
                         DiskLoc newLoc = theDataFileMgr.updateRecord(ns,

src/mongo/db/repl.cpp

@@ -1141,7 +1141,9 @@ namespace mongo {
 
     bool OplogReader::commonConnect(const string& hostName) {
         if( conn() == 0 ) {
-            _conn = shared_ptr<DBClientConnection>(new DBClientConnection( false, 0, 60*10 /* tcp timeout */));
+            _conn = shared_ptr<DBClientConnection>(new DBClientConnection(false,
+                                                                          0,
+                                                                          30 /* tcp timeout */));
             string errmsg;
             ReplInfo r("trying to connect to sync source");
             if ( !_conn->connect(hostName.c_str(), errmsg) ||

src/mongo/db/repl/consensus.cpp

@@ -207,16 +207,14 @@ namespace mongo {
             log() << "replSet electCmdReceived couldn't find member with id " << whoid << rsLog;
             vote = -10000;
         }
-        else if( primary && primary == rs._self && rs.lastOpTimeWritten >= hopeful->hbinfo().opTime ) {
-            // hbinfo is not updated, so we have to check the primary's last optime separately
+        else if( primary && primary == rs._self) {
             log() << "I am already primary, " << hopeful->fullName()
                   << " can try again once I've stepped down" << rsLog;
             vote = -10000;
         }
-        else if( primary && primary->hbinfo().opTime >= hopeful->hbinfo().opTime ) {
-            // other members might be aware of more up-to-date nodes
+        else if (primary) {
             log() << hopeful->fullName() << " is trying to elect itself but " <<
-                  primary->fullName() << " is already primary and more up-to-date" << rsLog;
+                  primary->fullName() << " is already primary" << rsLog;
             vote = -10000;
         }
         else if( highestPriority && highestPriority->config().priority > hopeful->config().priority) {

src/mongo/db/repl/rs_rollback.cpp

@@ -465,6 +465,16 @@ namespace mongo {
 
                 // todo: lots of overhead in context, this can be faster
                 Client::Context c(d.ns);
+
+                // Add the doc to our rollback file
+                BSONObj obj;
+                bool found = Helpers::findOne(d.ns, pattern, obj, false);
+                if ( found ) {
+                    rs->goingToDelete( obj );
+                } else {
+                    error() << "rollback cannot find object by id" << endl;
+                }
+
                 if( i->second.isEmpty() ) {
                     // wasn't on the primary; delete.
                     /* TODO1.6 : can't delete from a capped collection.  need to handle that here. */

src/mongo/db/repl/rs_sync.cpp

@@ -818,6 +818,9 @@ namespace replset {
         }
 
         try {
+            // haveCursor() does not necessarily tell us if we have a non-dead cursor, so we check
+            // tailCheck() as well; see SERVER-8420
+            slave->reader.tailCheck();
             if (!slave->reader.haveCursor()) {
                 if (!slave->reader.connect(id, slave->slave->id(), target->fullName())) {
                     // error message logged in OplogReader::connect

src/mongo/db/repl_block.cpp

@@ -168,7 +168,9 @@ namespace mongo {
         }
 
         bool replicatedToNum(OpTime& op, int w) {
-            if ( w <= 1 || ! _isMaster() )
+            massert( 16805, "replicatedToNum called but not master anymore", _isMaster() );
+
+            if ( w <= 1 )
                 return true;
 
             w--; // now this is the # of slaves i need
@@ -177,7 +179,11 @@ namespace mongo {
         }
 
         bool waitForReplication(OpTime& op, int w, int maxSecondsToWait) {
-            if ( w <= 1 || ! _isMaster() )
+            static const int noLongerMasterAssertCode = 16806;
+            massert(noLongerMasterAssertCode, 
+                    "waitForReplication called but not master anymore", _isMaster() );
+
+            if ( w <= 1 )
                 return true;
 
             w--; // now this is the # of slaves i need
@@ -188,8 +194,13 @@ namespace mongo {
             
             scoped_lock mylk(_mutex);
             while ( ! _replicatedToNum_slaves_locked( op, w ) ) {
-                if ( ! _threadsWaitingForReplication.timed_wait( mylk.boost() , xt ) )
+                if ( ! _threadsWaitingForReplication.timed_wait( mylk.boost() , xt ) ) {
+                    massert(noLongerMasterAssertCode,
+                            "waitForReplication called but not master anymore", _isMaster());
                     return false;
+                }
+                massert(noLongerMasterAssertCode, 
+                        "waitForReplication called but not master anymore", _isMaster());
             }
             return true;
         }

src/mongo/dbtests/sharding.cpp

@@ -167,6 +167,11 @@ namespace ShardingTests {
             _shard = Shard( "shard0000", "$hostFooBar:27017" );
             // Need to run this to ensure the shard is in the global lookup table
             _shard.setAddress( _shard.getAddress() );
+
+            // Create an index so that diffing works correctly, otherwise no cursors from S&O
+            client().ensureIndex( "config.chunks", // br
+                                  BSON( "ns" << 1 << // br
+                                          "lastmod" << 1 ) );
         }
 
         virtual ~ChunkManagerTest() {

src/mongo/s/chunk_diff.hpp

@@ -284,9 +284,6 @@ namespace mongo {
 
         BSONObj query = queryB.obj();
 
-        LOG(2) << "major version query from " << *_maxVersion << " and over "
-               << _maxShardVersions->size() << " shards is " << query << endl;
-
         //
         // NOTE: IT IS IMPORTANT FOR CONSISTENCY THAT WE SORT BY ASC VERSION, TO HANDLE
         // CURSOR YIELDING BETWEEN CHUNKS BEING MIGRATED.
@@ -295,7 +292,10 @@ namespace mongo {
         Query queryObj(query);
         queryObj.sort(BSON( "lastmod" << 1 ));
 
-        return Query( query );
+        LOG(2) << "major version query from " << *_maxVersion << " and over "
+               << _maxShardVersions->size() << " shards is " << queryObj << endl;
+
+        return queryObj;
     }
 
 } // namespace mongo

src/mongo/s/client_info.cpp

@@ -227,41 +227,39 @@ namespace mongo {
             }
             clearSinceLastGetError();
 
-            LOG(4) << "checking " << writebacks.size() << " writebacks for"
-                   << " gle (" << theShard << ")" << endl;
+            // We never need to handle writebacks if we're coming from the wbl itself
+            if ( writebacks.size() && !fromWriteBackListener ){
 
-            if ( writebacks.size() ){
-                vector<BSONObj> v = _handleWriteBacks( writebacks , fromWriteBackListener );
-                if ( v.size() == 0 && fromWriteBackListener ) {
-                    // ok
+                LOG(4) << "checking " << writebacks.size() << " writebacks for"
+                       << " gle (" << theShard << ")" << endl;
+
+                vector<BSONObj> v = _handleWriteBacks( writebacks , false );
+
+                // this will usually be 1
+                // it can be greater than 1 if a write to a different shard
+                // than the last write op had a writeback
+                // all we're going to report is the first
+                // since that's the current write
+                // but we block for all
+                verify( v.size() >= 1 );
+
+                if ( res["writebackSince"].numberInt() > 0 ) {
+                    // got writeback from older op
+                    // ignore the result from it, just needed to wait
+                    result.appendElements( res );
+                }
+                else if ( writebacks[0].fromLastOperation ) {
+                    result.appendElements( v[0] );
+                    result.appendElementsUnique( res );
+                    result.append( "writebackGLE" , v[0] );
+                    result.append( "initialGLEHost" , theShard );
+                    result.append( "initialGLE", res );
                 }
                 else {
-                    // this will usually be 1
-                    // it can be greater than 1 if a write to a different shard
-                    // than the last write op had a writeback
-                    // all we're going to report is the first
-                    // since that's the current write
-                    // but we block for all
-                    verify( v.size() >= 1 );
-
-                    if ( res["writebackSince"].numberInt() > 0 ) {
-                        // got writeback from older op
-                        // ignore the result from it, just needed to wait
-                        result.appendElements( res );
-                    }
-                    else if ( writebacks[0].fromLastOperation ) {
-                        result.appendElements( v[0] );
-                        result.appendElementsUnique( res );
-                        result.append( "writebackGLE" , v[0] );
-                        result.append( "initialGLEHost" , theShard );
-                        result.append( "initialGLE", res );
-                    }
-                    else {
-                        // there was a writeback
-                        // but its from an old operations
-                        // so all that's important is that we block, not that we return stats
-                        result.appendElements( res );
-                    }
+                    // there was a writeback
+                    // but its from an old operations
+                    // so all that's important is that we block, not that we return stats
+                    result.appendElements( res );
                 }
             }
             else {
@@ -365,6 +363,10 @@ namespace mongo {
         LOG(4) << "checking " << writebacks.size() << " writebacks for"
                 << " gle (" << shards->size() << " shards)" << endl;
 
+        // Multi-shard results from the writeback listener implicitly means that:
+        // A) no versioning was used (multi-update/delete)
+        // B) internal GLE was used (bulk insert)
+
         if ( errors.size() == 0 ) {
             result.appendNull( "err" );
             _handleWriteBacks( writebacks , fromWriteBackListener );

src/mongo/s/d_migrate.cpp

@@ -40,6 +40,7 @@
 #include "../db/clientcursor.h"
 #include "../db/pagefault.h"
 #include "../db/repl.h"
+#include "../db/repl/rs.h"
 
 #include "../client/connpool.h"
 #include "../client/distlock.h"
@@ -560,10 +561,11 @@ namespace mongo {
                         }
                         
                         BSONObj o = dl.obj();
-                        
+
                         // use the builder size instead of accumulating 'o's size so that we take into consideration
-                        // the overhead of BSONArray indices
-                        if ( a.len() + o.objsize() + 1024 > BSONObjMaxUserSize ) {
+                        // the overhead of BSONArray indices, and *always* append one doc
+                        if ( a.arrSize() != 0 &&
+                             a.len() + o.objsize() + 1024 > BSONObjMaxUserSize ) {
                             filledBuffer = true; // break out of outer while loop
                             break;
                         }
@@ -609,6 +611,11 @@ namespace mongo {
             _cloneLocs.erase( dl );
         }
 
+        std::size_t cloneLocsRemaining() {
+            scoped_spinlock lk( _trackerLocks );
+            return _cloneLocs.size();
+        }
+
         long long mbUsed() const { return _memoryUsed / ( 1024 * 1024 ); }
 
         bool getInCriticalSection() const { scoped_lock l(_m); return _inCriticalSection; }
@@ -1052,13 +1059,19 @@ namespace mongo {
             timing.done( 3 );
 
             // 4.
+
+            // Track last result from TO shard for sanity check
+            BSONObj res;
             for ( int i=0; i<86400; i++ ) { // don't want a single chunk move to take more than a day
                 verify( !Lock::isLocked() );
+
                 sleepsecs( 1 );
+
                 scoped_ptr<ScopedDbConnection> conn(
                         ScopedDbConnection::getScopedDbConnection( toShard.getConnString() ) );
-                BSONObj res;
+
                 bool ok;
+                res = BSONObj();
                 try {
                     ok = conn->get()->runCommand( "admin" , BSON( "_recvChunkStatus" << 1 ) , res );
                     res = res.getOwned();
@@ -1104,6 +1117,26 @@ namespace mongo {
             timing.done(4);
 
             // 5.
+
+            // Before we get into the critical section of the migration, let's double check
+            // that the docs have been cloned
+            log() << "About to check if it is safe to enter critical section" << endl;
+
+            // Ensure all cloned docs have actually been transferred
+            std::size_t locsRemaining = migrateFromStatus.cloneLocsRemaining();
+            if ( locsRemaining != 0 ) {
+
+                errmsg =
+                    str::stream() << "moveChunk cannot enter critical section before all data is"
+                                  << " cloned, " << locsRemaining << " locs were not transferred"
+                                  << " but to-shard reported " << res;
+
+                // Should never happen, but safe to abort before critical section
+                error() << errmsg << migrateLog;
+                dassert( false );
+                return false;
+            }
+
             {
                 // 5.a
                 // we're under the collection lock here, so no other migrate can change maxVersion or ShardChunkManager state
@@ -1437,11 +1470,10 @@ namespace mongo {
             verify( ! min.isEmpty() );
             verify( ! max.isEmpty() );
             
-            slaveCount = ( getSlaveCount() / 2 ) + 1;
+            replSetMajorityCount = theReplSet ? theReplSet->config().getMajority() : 0;
 
             log() << "starting receiving-end of migration of chunk " << min << " -> " << max <<
-                    " for collection " << ns << " from " << from <<
-                    " (" << getSlaveCount() << " slaves detected)" << endl;
+                    " for collection " << ns << " from " << from << endl;
 
             string errmsg;
             MoveTimingHelper timing( "to" , ns , min , max , 5 /* steps */ , errmsg );
@@ -1627,7 +1659,15 @@ namespace mongo {
                 // 5. wait for commit
 
                 state = STEADY;
+                bool transferAfterCommit = false;
                 while ( state == STEADY || state == COMMIT_START ) {
+
+                    // Make sure we do at least one transfer after recv'ing the commit message
+                    // If we aren't sure that at least one transfer happens *after* our state
+                    // changes to COMMIT_START, there could be mods still on the FROM shard that
+                    // got logged *after* our _transferMods but *before* the critical section.
+                    if ( state == COMMIT_START ) transferAfterCommit = true;
+
                     BSONObj res;
                     if ( ! conn->runCommand( "admin" , BSON( "_transferMods" << 1 ) , res ) ) {
                         log() << "_transferMods failed in STEADY state: " << res << migrateLog;
@@ -1645,12 +1685,16 @@ namespace mongo {
                         return;
                     }
                     
-                    if ( state == COMMIT_START ) {
+                    // We know we're finished when:
+                    // 1) The from side has told us that it has locked writes (COMMIT_START)
+                    // 2) We've checked at least one more time for un-transmitted mods
+                    if ( state == COMMIT_START && transferAfterCommit == true ) {
                         if ( flushPendingWrites( lastOpApplied ) )
                             break;
                     }
                     
-                    sleepmillis( 10 );
+                    // Only sleep if we aren't committing
+                    if ( state == STEADY ) sleepmillis( 10 );
                 }
 
                 if ( state == FAIL ) {
@@ -1757,13 +1801,13 @@ namespace mongo {
             // if replication is on, try to force enough secondaries to catch up
             // TODO opReplicatedEnough should eventually honor priorities and geo-awareness
             //      for now, we try to replicate to a sensible number of secondaries
-            return mongo::opReplicatedEnough( lastOpApplied , slaveCount );
+            return mongo::opReplicatedEnough( lastOpApplied , replSetMajorityCount );
         }
 
         bool flushPendingWrites( const ReplTime& lastOpApplied ) {
             if ( ! opReplicatedEnough( lastOpApplied ) ) {
                 OpTime op( lastOpApplied );
-                OCCASIONALLY warning() << "migrate commit waiting for " << slaveCount 
+                OCCASIONALLY warning() << "migrate commit waiting for " << replSetMajorityCount 
                                        << " slaves for '" << ns << "' " << min << " -> " << max 
                                        << " waiting for: " << op
                                        << migrateLog;
@@ -1841,7 +1885,7 @@ namespace mongo {
         long long numSteady;
         bool secondaryThrottle;
 
-        int slaveCount;
+        int replSetMajorityCount;
 
         enum State { READY , CLONE , CATCHUP , STEADY , COMMIT_START , DONE , FAIL , ABORT } state;
         string errmsg;

src/mongo/s/server.cpp

@@ -45,6 +45,7 @@
 #include "../util/processinfo.h"
 #include "mongo/db/lasterror.h"
 #include "mongo/util/stacktrace.h"
+#include "mongo/util/log.h"
 
 #if defined(_WIN32)
 # include "../util/ntservice.h"
@@ -151,6 +152,33 @@ namespace mongo {
         ::_exit(EXIT_ABRUPT);
     }
 
+#ifndef _WIN32
+    sigset_t asyncSignals;
+
+    void signalProcessingThread() {
+        while (true) {
+            int actualSignal = 0;
+            int status = sigwait( &asyncSignals, &actualSignal );
+            fassert(16779, status == 0);
+            switch (actualSignal) {
+            case SIGUSR1:
+                // log rotate signal
+                fassert(16780, rotateLogs());
+                break;
+            default:
+                // no one else should be here
+                fassertFailed(16778);
+                break;
+            }
+        }
+    }
+
+    void startSignalProcessingThread() {
+        verify( pthread_sigmask( SIG_SETMASK, &asyncSignals, 0 ) == 0 );
+        boost::thread it( signalProcessingThread );
+    }
+#endif
+
     void setupSignals( bool inFork ) {
         signal(SIGTERM, sighandler);
         signal(SIGINT, sighandler);
@@ -167,7 +195,11 @@ namespace mongo {
 #if defined(SIGPIPE)
         signal( SIGPIPE , SIG_IGN );
 #endif
-
+#ifndef _WIN32
+        sigemptyset( &asyncSignals );
+        sigaddset( &asyncSignals, SIGUSR1 );
+        startSignalProcessingThread();
+#endif
         set_new_handler( my_new_handler );
     }
 
@@ -175,7 +207,7 @@ namespace mongo {
         serverID.init();
         setupSIGTRAPforGDB();
         setupCoreSignals();
-        setupSignals( false );
+
         Logstream::get().addGlobalTee( new RamLog("global") );
     }
 
@@ -218,7 +250,7 @@ namespace mongo {
 using namespace mongo;
 
 static bool runMongosServer( bool doUpgrade ) {
-
+    setupSignals( false );
     setThreadName( "mongosMain" );
     printShardingVersionInfo( false );
 

src/mongo/s/writeback_listener.cpp

@@ -140,6 +140,7 @@ namespace mongo {
         int secsToSleep = 0;
         scoped_ptr<ShardChunkVersion> lastNeededVersion;
         int lastNeededCount = 0;
+        bool needsToReloadShardInfo = false;
 
         while ( ! inShutdown() ) {
 
@@ -150,6 +151,12 @@ namespace mongo {
             }
 
             try {
+                if (needsToReloadShardInfo) {
+                    // It's possible this shard was removed
+                    Shard::reloadShardInfo();
+                    needsToReloadShardInfo = false;
+                }
+
                 scoped_ptr<ScopedDbConnection> conn(
                         ScopedDbConnection::getInternalScopedDbConnection( _addr ) );
 
@@ -337,6 +344,9 @@ namespace mongo {
                                 gle = b.obj();
                             }
 
+                            dassert( !gle.isEmpty() );
+                            verify( !gle.isEmpty() );
+
                             if ( gle["code"].numberInt() == 9517 ) {
 
                                 log() << "new version change detected, "
@@ -401,6 +411,8 @@ namespace mongo {
                 continue;
             }
             catch ( std::exception& e ) {
+                // Attention! Do not call any method that would throw an exception
+                // (or assert) in this block.
 
                 if ( inShutdown() ) {
                     // we're shutting down, so just clean up
@@ -409,8 +421,7 @@ namespace mongo {
 
                 log() << "WriteBackListener exception : " << e.what() << endl;
 
-                // It's possible this shard was removed
-                Shard::reloadShardInfo();
+                needsToReloadShardInfo = true;
             }
             catch ( ... ) {
                 log() << "WriteBackListener uncaught exception!" << endl;

src/mongo/shell/db.js

@@ -178,11 +178,15 @@ DB.prototype.auth = function( username , pass ){
 */
 DB.prototype.createCollection = function(name, opt) {
     var options = opt || {};
-    var cmd = { create: name, capped: options.capped, size: options.size };
+    var cmd = { create: name };
     if (options.max != undefined)
         cmd.max = options.max;
     if (options.autoIndexId != undefined)
         cmd.autoIndexId = options.autoIndexId;
+    if (options.capped != undefined)
+        cmd.capped = options.capped;
+    if (options.size != undefined)
+        cmd.size = options.size;
     var res = this._dbCommand(cmd);
     return res;
 }

src/mongo/shell/shell_utils_launcher.cpp

@@ -281,8 +281,10 @@ namespace mongo {
                         break;
                     }
                     if ( last != buf ) {
-                        strcpy( temp, last );
-                        strcpy( buf, temp );
+                        strncpy( temp, last, bufSize );
+                        temp[ bufSize-1 ] = '\0';
+                        strncpy( buf, temp, bufSize );
+                        buf[ bufSize-1 ] = '\0';
                     }
                     else {
                         verify( strlen( buf ) < bufSize );
@@ -531,6 +533,14 @@ namespace mongo {
             return undefinedReturn;
         }
 
+        BSONObj PathExists( const BSONObj &a, void* data ) {
+            verify( a.nFields() == 1 );
+            string path = a.firstElement().valuestrsafe();
+            verify( !path.empty() );
+            bool exists = boost::filesystem::exists(path);
+            return BSON( string( "" ) << exists );
+        }
+
         void copyDir( const boost::filesystem::path &from, const boost::filesystem::path &to ) {
             boost::filesystem::directory_iterator end;
             boost::filesystem::directory_iterator i( from );
@@ -746,6 +756,7 @@ namespace mongo {
             scope.injectNative( "clearRawMongoProgramOutput", ClearRawMongoProgramOutput );
             scope.injectNative( "waitProgram" , WaitProgram );
             scope.injectNative( "resetDbpath", ResetDbpath );
+            scope.injectNative( "pathExists", PathExists );
             scope.injectNative( "copyDbpath", CopyDbpath );
         }
     }

src/mongo/shell/utils.js

@@ -412,9 +412,29 @@ ISODate = function(isoDateStr){
     var date = parseInt(res[3],10) || 0;
     var hour = parseInt(res[5],10) || 0;
     var min = parseInt(res[7],10) || 0;
-    var sec = parseFloat(res[9]) || 0;
-    var ms = Math.round((sec%1) * 1000)
-    sec -= ms/1000
+    var sec = parseInt((res[9] && res[9].substr(0,2)),10) || 0;
+    var ms = Math.round((parseFloat(res[10]) || 0) * 1000);
+    if (ms == 1000) {
+        ms = 0;
+        ++sec;
+    }
+    if (sec == 60) {
+        sec = 0;
+        ++min;
+    }
+    if (min == 60) {
+        min = 0;
+        ++hour;
+    }
+    if (hour == 24) {
+        hour = 0;   // the day wrapped, let JavaScript figure out the rest
+        var tempTime = Date.UTC(year, month, date, hour, min, sec, ms);
+        tempTime += 24 * 60 * 60 * 1000;    // milliseconds in a day
+        var tempDate = new Date(tempTime);
+        year = tempDate.getUTCFullYear();
+       month = tempDate.getUTCMonth();
+        date = tempDate.getUTCDate();
+    }
 
     var time = Date.UTC(year, month, date, hour, min, sec, ms);
 

src/mongo/util/processinfo_linux2.cpp

@@ -49,7 +49,7 @@ namespace mongo {
                 msgassertedNoTrace( 13538 , s.c_str() );
             }
             int found = fscanf(f,
-                               "%d %s %c "
+                               "%d %127s %c "
                                "%d %d %d %d %d "
                                "%lu %lu %lu %lu %lu "
                                "%lu %lu %ld %ld "  /* utime stime cutime cstime */

src/mongo/util/version.cpp

@@ -43,7 +43,7 @@ namespace mongo {
      *      1.2.3-rc4-pre-
      * If you really need to do something else you'll need to fix _versionArray()
      */
-    const char versionString[] = "2.2.4";
+    const char versionString[] = "2.2.8-pre-";
 
     // See unit test for example outputs
     static BSONArray _versionArray(const char* version){