post 2.2.7

Conflicts:
	src/mongo/db/ops/update.cpp

doxygenConfig

@@ -3,7 +3,7 @@
 #---------------------------------------------------------------------------
 DOXYFILE_ENCODING      = UTF-8
 PROJECT_NAME           = MongoDB
-PROJECT_NUMBER         = 2.2.5
+PROJECT_NUMBER         = 2.2.8-pre-
 OUTPUT_DIRECTORY       = docs/doxygen
 CREATE_SUBDIRS         = NO
 OUTPUT_LANGUAGE        = English

jstests/replsets/rollback5.js

@@ -0,0 +1,102 @@
+// test that a rollback directory is created during a replica set rollback
+// this also tests that updates are recorded in the rollback file
+//  (this test does no delete rollbacks)
+
+var replTest = new ReplSetTest({ name: 'rollback5', nodes: 3 });
+var nodes = replTest.nodeList();
+
+var conns = replTest.startSet();
+var r = replTest.initiate({ "_id": "rollback5",
+                            "members": [
+                                { "_id": 0, "host": nodes[0] },
+                                { "_id": 1, "host": nodes[1] },
+                                { "_id": 2, "host": nodes[2], arbiterOnly: true}]
+                          });
+
+// Make sure we have a master
+var master = replTest.getMaster();
+var a_conn = conns[0];
+var b_conn = conns[1];
+a_conn.setSlaveOk();
+b_conn.setSlaveOk();
+var A = a_conn.getDB("test");
+var B = b_conn.getDB("test");
+var AID = replTest.getNodeId(a_conn);
+var BID = replTest.getNodeId(b_conn);
+var Apath = "/data/db/rollback5-0/";
+var Bpath = "/data/db/rollback5-1/";
+assert(master == conns[0], "conns[0] assumed to be master");
+assert(a_conn.host == master.host);
+
+// Make sure we have an arbiter
+assert.soon(function () {
+    res = conns[2].getDB("admin").runCommand({ replSetGetStatus: 1 });
+    return res.myState == 7;
+}, "Arbiter failed to initialize.");
+
+A.foo.update({key:'value1'}, {$set: {req: 'req'}}, true);
+A.foo.runCommand({getLastError : 1, w : 2, wtimeout : 60000});
+replTest.stop(AID);
+
+master = replTest.getMaster();
+assert(b_conn.host == master.host);
+B.foo.update({key:'value1'}, {$set: {res: 'res'}}, true);
+B.foo.runCommand({getLastError : 1, w : 1, wtimeout : 60000});
+replTest.stop(BID);
+replTest.restart(AID);
+master = replTest.getMaster();
+assert(a_conn.host == master.host);
+A.foo.update({key:'value2'}, {$set: {req: 'req'}}, true);
+A.foo.runCommand({getLastError : 1, w : 1, wtimeout : 60000});
+replTest.restart(BID); // should rollback
+reconnect(B);
+
+print("BEFORE------------------");
+printjson(A.foo.find().toArray());
+
+replTest.awaitReplication();
+replTest.awaitSecondaryNodes();
+
+print("AFTER------------------");
+printjson(A.foo.find().toArray());
+
+assert.eq(2, A.foo.count());
+assert.eq('req', A.foo.findOne({key:'value1'}).req);
+assert.eq(null, A.foo.findOne({key:'value1'}).res);
+assert.eq(2, B.foo.count());
+assert.eq('req', B.foo.findOne({key:'value1'}).req);
+assert.eq(null, B.foo.findOne({key:'value1'}).res);
+
+// check here for rollback files
+var rollbackDir = Bpath + "rollback/";
+assert(pathExists(rollbackDir), "rollback directory was not created!");
+
+print("rollback5.js SUCCESS");
+replTest.stopSet(15);
+
+
+function wait(f) {
+    var n = 0;
+    while (!f()) {
+        if (n % 4 == 0)
+            print("rollback5.js waiting");
+        if (++n == 4) {
+            print("" + f);
+        }
+        assert(n < 200, 'tried 200 times, giving up');
+        sleep(1000);
+    }
+}
+
+function reconnect(a) {
+  wait(function() {
+      try {
+        a.bar.stats();
+        return true;
+      } catch(e) {
+        print(e);
+        return false;
+      }
+    });
+};
+

jstests/sharding/gle_error_message.js

@@ -111,7 +111,11 @@ jsTest.log( "Testing stale version GLE when host goes down..." )
 var staleColl = st.s1.getCollection( coll + "" )
 staleColl.findOne()
 
-printjson( admin.runCommand({ moveChunk : "" + coll, find : { _id : 0 }, to : shards[2]._id }) )
+// As it turns out, on the *second* auto-reconnect attempt we need to wait at least 2 secs,
+// otherwise reconnect fails with FAILED_STATE 
+sleep( 2000 );
+
+assert( admin.runCommand({ moveChunk : "" + coll, find : { _id : 0 }, to : shards[2]._id }).ok );
 
 MongoRunner.stopMongod( st.shard2 )
 
@@ -124,4 +128,4 @@ assert.neq( null, staleColl.getDB().getLastError() )
 
 jsTest.log( "Done!" )
 
-st.stop()
+st.stop()

jstests/slowNightly/sharding_migrate_large_docs.js

@@ -0,0 +1,63 @@
+//
+// Tests migration behavior of large documents
+//
+
+var st = new ShardingTest({ shards : 2, mongos : 1,
+                            other : { separateConfig : true,
+                                      mongosOptions : { noAutoSplit : "" },
+                                      shardOptions : { /* binVersion : "latest" */ } } });
+st.stopBalancer()
+
+var mongos = st.s0;
+var coll = mongos.getCollection( "foo.bar" );
+var admin = mongos.getDB( "admin" );
+var shards = mongos.getCollection( "config.shards" ).find().toArray();
+var shardAdmin = st.shard0.getDB( "admin" );
+
+assert( admin.runCommand({ enableSharding : coll.getDB() + "" }).ok );
+printjson( admin.runCommand({ movePrimary : coll.getDB() + "", to : shards[0]._id }) );
+assert( admin.runCommand({ shardCollection : coll + "", key : { _id : 1 } }).ok );
+assert( admin.runCommand({ split : coll + "", middle : { _id : 0 } }).ok );
+
+jsTest.log( "Preparing large insert..." );
+
+var data1MB = "x"
+while ( data1MB.length < 1024 * 1024 )
+    data1MB += data1MB;
+
+var data15MB = "";
+for ( var i = 0; i < 15; i++ ) data15MB += data1MB;
+
+var data15PlusMB = data15MB;
+for ( var i = 0; i < 1023 * 1024; i++ ) data15PlusMB += "x";
+
+print("~15MB object size is : " + Object.bsonsize({ _id : 0, d : data15PlusMB }));
+
+jsTest.log( "Inserting docs of large and small sizes..." );
+
+// Two large docs next to each other
+coll.insert({ _id : -2, d : data15PlusMB });
+coll.insert({ _id : -1, d : data15PlusMB });
+
+// Docs of assorted sizes
+coll.insert({ _id : 0, d : "x" });
+coll.insert({ _id : 1, d : data15PlusMB });
+coll.insert({ _id : 2, d : "x" });
+coll.insert({ _id : 3, d : data15MB });
+coll.insert({ _id : 4, d : "x" });
+coll.insert({ _id : 5, d : data1MB });
+coll.insert({ _id : 6, d : "x" });
+
+assert.eq( null, coll.getDB().getLastError() );
+assert.eq( 9, coll.find().itcount() );
+
+jsTest.log( "Starting migration..." );
+
+assert( admin.runCommand({ moveChunk : coll + "", find : { _id : 0 }, to : shards[1]._id }).ok );
+assert( admin.runCommand({ moveChunk : coll + "", find : { _id : -1 }, to : shards[1]._id }).ok );
+
+assert.eq( 9, coll.find().itcount() );
+
+jsTest.log( "DONE!" );
+
+st.stop();

rpm/mongo.spec

@@ -1,7 +1,7 @@
 Name: mongo-10gen
 Conflicts: mongo, mongo-10gen-unstable
 Obsoletes: mongo-stable
-Version: 2.2.3
+Version: 2.2.6
 Release: mongodb_1%{?dist}
 Summary: mongodb client shell and tools
 License: AGPL 3.0

src/mongo/db/dbhelpers.cpp

@@ -380,7 +380,6 @@ namespace mongo {
         stringstream ss;
         ss << why << "." << terseCurrentTime(false) << "." << NUM++ << ".bson";
         _file /= ss.str();
-
     }
 
     RemoveSaver::~RemoveSaver() {
@@ -397,7 +396,8 @@ namespace mongo {
             _out = new ofstream();
             _out->open( _file.string().c_str() , ios_base::out | ios_base::binary );
             if ( ! _out->good() ) {
-                LOG( LL_WARNING ) << "couldn't create file: " << _file.string() << " for remove saving" << endl;
+                error() << "couldn't create file: " << _file.string() << 
+                    " for remove saving" << endl;
                 delete _out;
                 _out = 0;
                 return;

src/mongo/db/matcher.cpp

@@ -465,6 +465,8 @@ namespace mongo {
         uassert( 10066 , "$where may only appear once in query", _where == 0 );
         uassert( 10067 , "$where query, but no script engine", globalScriptEngine );
         massert( 13089 , "no current client needed for $where" , haveClient() );
+        uassert( 17126 , "no valid context found for $where", cc().getContext());
+
         _where = new Where( cc().ns() );
 
         if ( e.type() == CodeWScope ) {

src/mongo/db/ops/delete.cpp

@@ -129,9 +129,6 @@ namespace mongo {
                 }
             }
 
-            if ( rs )
-                rs->goingToDelete( rloc.obj() /*cc->c->current()*/ );
-
             theDataFileMgr.deleteRecord(ns, rloc.rec(), rloc);
             nDeleted++;
             if ( foundAllResults ) {

src/mongo/db/ops/update.cpp

@@ -364,9 +364,6 @@ namespace mongo {
                         d->paddingFits();
                     }
                     else {
-                        if ( rs )
-                            rs->goingToDelete( onDisk );
-
                         BSONObj newObj = mss->createNewFromMods();
                         checkTooLarge(newObj);
                         DiskLoc newLoc = theDataFileMgr.updateRecord(ns,

src/mongo/db/repl/rs_rollback.cpp

@@ -465,6 +465,16 @@ namespace mongo {
 
                 // todo: lots of overhead in context, this can be faster
                 Client::Context c(d.ns);
+
+                // Add the doc to our rollback file
+                BSONObj obj;
+                bool found = Helpers::findOne(d.ns, pattern, obj, false);
+                if ( found ) {
+                    rs->goingToDelete( obj );
+                } else {
+                    error() << "rollback cannot find object by id" << endl;
+                }
+
                 if( i->second.isEmpty() ) {
                     // wasn't on the primary; delete.
                     /* TODO1.6 : can't delete from a capped collection.  need to handle that here. */

src/mongo/dbtests/sharding.cpp

@@ -167,6 +167,11 @@ namespace ShardingTests {
             _shard = Shard( "shard0000", "$hostFooBar:27017" );
             // Need to run this to ensure the shard is in the global lookup table
             _shard.setAddress( _shard.getAddress() );
+
+            // Create an index so that diffing works correctly, otherwise no cursors from S&O
+            client().ensureIndex( "config.chunks", // br
+                                  BSON( "ns" << 1 << // br
+                                          "lastmod" << 1 ) );
         }
 
         virtual ~ChunkManagerTest() {

src/mongo/s/chunk_diff.hpp

@@ -284,9 +284,6 @@ namespace mongo {
 
         BSONObj query = queryB.obj();
 
-        LOG(2) << "major version query from " << *_maxVersion << " and over "
-               << _maxShardVersions->size() << " shards is " << query << endl;
-
         //
         // NOTE: IT IS IMPORTANT FOR CONSISTENCY THAT WE SORT BY ASC VERSION, TO HANDLE
         // CURSOR YIELDING BETWEEN CHUNKS BEING MIGRATED.
@@ -295,7 +292,10 @@ namespace mongo {
         Query queryObj(query);
         queryObj.sort(BSON( "lastmod" << 1 ));
 
-        return Query( query );
+        LOG(2) << "major version query from " << *_maxVersion << " and over "
+               << _maxShardVersions->size() << " shards is " << queryObj << endl;
+
+        return queryObj;
     }
 
 } // namespace mongo

src/mongo/s/client_info.cpp

@@ -227,41 +227,39 @@ namespace mongo {
             }
             clearSinceLastGetError();
 
-            LOG(4) << "checking " << writebacks.size() << " writebacks for"
-                   << " gle (" << theShard << ")" << endl;
+            // We never need to handle writebacks if we're coming from the wbl itself
+            if ( writebacks.size() && !fromWriteBackListener ){
 
-            if ( writebacks.size() ){
-                vector<BSONObj> v = _handleWriteBacks( writebacks , fromWriteBackListener );
-                if ( v.size() == 0 && fromWriteBackListener ) {
-                    // ok
+                LOG(4) << "checking " << writebacks.size() << " writebacks for"
+                       << " gle (" << theShard << ")" << endl;
+
+                vector<BSONObj> v = _handleWriteBacks( writebacks , false );
+
+                // this will usually be 1
+                // it can be greater than 1 if a write to a different shard
+                // than the last write op had a writeback
+                // all we're going to report is the first
+                // since that's the current write
+                // but we block for all
+                verify( v.size() >= 1 );
+
+                if ( res["writebackSince"].numberInt() > 0 ) {
+                    // got writeback from older op
+                    // ignore the result from it, just needed to wait
+                    result.appendElements( res );
+                }
+                else if ( writebacks[0].fromLastOperation ) {
+                    result.appendElements( v[0] );
+                    result.appendElementsUnique( res );
+                    result.append( "writebackGLE" , v[0] );
+                    result.append( "initialGLEHost" , theShard );
+                    result.append( "initialGLE", res );
                 }
                 else {
-                    // this will usually be 1
-                    // it can be greater than 1 if a write to a different shard
-                    // than the last write op had a writeback
-                    // all we're going to report is the first
-                    // since that's the current write
-                    // but we block for all
-                    verify( v.size() >= 1 );
-
-                    if ( res["writebackSince"].numberInt() > 0 ) {
-                        // got writeback from older op
-                        // ignore the result from it, just needed to wait
-                        result.appendElements( res );
-                    }
-                    else if ( writebacks[0].fromLastOperation ) {
-                        result.appendElements( v[0] );
-                        result.appendElementsUnique( res );
-                        result.append( "writebackGLE" , v[0] );
-                        result.append( "initialGLEHost" , theShard );
-                        result.append( "initialGLE", res );
-                    }
-                    else {
-                        // there was a writeback
-                        // but its from an old operations
-                        // so all that's important is that we block, not that we return stats
-                        result.appendElements( res );
-                    }
+                    // there was a writeback
+                    // but its from an old operations
+                    // so all that's important is that we block, not that we return stats
+                    result.appendElements( res );
                 }
             }
             else {
@@ -365,6 +363,10 @@ namespace mongo {
         LOG(4) << "checking " << writebacks.size() << " writebacks for"
                 << " gle (" << shards->size() << " shards)" << endl;
 
+        // Multi-shard results from the writeback listener implicitly means that:
+        // A) no versioning was used (multi-update/delete)
+        // B) internal GLE was used (bulk insert)
+
         if ( errors.size() == 0 ) {
             result.appendNull( "err" );
             _handleWriteBacks( writebacks , fromWriteBackListener );

src/mongo/s/d_migrate.cpp

@@ -561,10 +561,11 @@ namespace mongo {
                         }
                         
                         BSONObj o = dl.obj();
-                        
+
                         // use the builder size instead of accumulating 'o's size so that we take into consideration
-                        // the overhead of BSONArray indices
-                        if ( a.len() + o.objsize() + 1024 > BSONObjMaxUserSize ) {
+                        // the overhead of BSONArray indices, and *always* append one doc
+                        if ( a.arrSize() != 0 &&
+                             a.len() + o.objsize() + 1024 > BSONObjMaxUserSize ) {
                             filledBuffer = true; // break out of outer while loop
                             break;
                         }
@@ -610,6 +611,11 @@ namespace mongo {
             _cloneLocs.erase( dl );
         }
 
+        std::size_t cloneLocsRemaining() {
+            scoped_spinlock lk( _trackerLocks );
+            return _cloneLocs.size();
+        }
+
         long long mbUsed() const { return _memoryUsed / ( 1024 * 1024 ); }
 
         bool getInCriticalSection() const { scoped_lock l(_m); return _inCriticalSection; }
@@ -1053,13 +1059,19 @@ namespace mongo {
             timing.done( 3 );
 
             // 4.
+
+            // Track last result from TO shard for sanity check
+            BSONObj res;
             for ( int i=0; i<86400; i++ ) { // don't want a single chunk move to take more than a day
                 verify( !Lock::isLocked() );
+
                 sleepsecs( 1 );
+
                 scoped_ptr<ScopedDbConnection> conn(
                         ScopedDbConnection::getScopedDbConnection( toShard.getConnString() ) );
-                BSONObj res;
+
                 bool ok;
+                res = BSONObj();
                 try {
                     ok = conn->get()->runCommand( "admin" , BSON( "_recvChunkStatus" << 1 ) , res );
                     res = res.getOwned();
@@ -1105,6 +1117,26 @@ namespace mongo {
             timing.done(4);
 
             // 5.
+
+            // Before we get into the critical section of the migration, let's double check
+            // that the docs have been cloned
+            log() << "About to check if it is safe to enter critical section" << endl;
+
+            // Ensure all cloned docs have actually been transferred
+            std::size_t locsRemaining = migrateFromStatus.cloneLocsRemaining();
+            if ( locsRemaining != 0 ) {
+
+                errmsg =
+                    str::stream() << "moveChunk cannot enter critical section before all data is"
+                                  << " cloned, " << locsRemaining << " locs were not transferred"
+                                  << " but to-shard reported " << res;
+
+                // Should never happen, but safe to abort before critical section
+                error() << errmsg << migrateLog;
+                dassert( false );
+                return false;
+            }
+
             {
                 // 5.a
                 // we're under the collection lock here, so no other migrate can change maxVersion or ShardChunkManager state

src/mongo/s/writeback_listener.cpp

@@ -344,6 +344,9 @@ namespace mongo {
                                 gle = b.obj();
                             }
 
+                            dassert( !gle.isEmpty() );
+                            verify( !gle.isEmpty() );
+
                             if ( gle["code"].numberInt() == 9517 ) {
 
                                 log() << "new version change detected, "

src/mongo/shell/shell_utils_launcher.cpp

@@ -533,6 +533,14 @@ namespace mongo {
             return undefinedReturn;
         }
 
+        BSONObj PathExists( const BSONObj &a, void* data ) {
+            verify( a.nFields() == 1 );
+            string path = a.firstElement().valuestrsafe();
+            verify( !path.empty() );
+            bool exists = boost::filesystem::exists(path);
+            return BSON( string( "" ) << exists );
+        }
+
         void copyDir( const boost::filesystem::path &from, const boost::filesystem::path &to ) {
             boost::filesystem::directory_iterator end;
             boost::filesystem::directory_iterator i( from );
@@ -748,6 +756,7 @@ namespace mongo {
             scope.injectNative( "clearRawMongoProgramOutput", ClearRawMongoProgramOutput );
             scope.injectNative( "waitProgram" , WaitProgram );
             scope.injectNative( "resetDbpath", ResetDbpath );
+            scope.injectNative( "pathExists", PathExists );
             scope.injectNative( "copyDbpath", CopyDbpath );
         }
     }

src/mongo/shell/utils.js

@@ -412,9 +412,29 @@ ISODate = function(isoDateStr){
     var date = parseInt(res[3],10) || 0;
     var hour = parseInt(res[5],10) || 0;
     var min = parseInt(res[7],10) || 0;
-    var sec = parseFloat(res[9]) || 0;
-    var ms = Math.round((sec%1) * 1000)
-    sec -= ms/1000
+    var sec = parseInt((res[9] && res[9].substr(0,2)),10) || 0;
+    var ms = Math.round((parseFloat(res[10]) || 0) * 1000);
+    if (ms == 1000) {
+        ms = 0;
+        ++sec;
+    }
+    if (sec == 60) {
+        sec = 0;
+        ++min;
+    }
+    if (min == 60) {
+        min = 0;
+        ++hour;
+    }
+    if (hour == 24) {
+        hour = 0;   // the day wrapped, let JavaScript figure out the rest
+        var tempTime = Date.UTC(year, month, date, hour, min, sec, ms);
+        tempTime += 24 * 60 * 60 * 1000;    // milliseconds in a day
+        var tempDate = new Date(tempTime);
+        year = tempDate.getUTCFullYear();
+       month = tempDate.getUTCMonth();
+        date = tempDate.getUTCDate();
+    }
 
     var time = Date.UTC(year, month, date, hour, min, sec, ms);
 

src/mongo/util/version.cpp

@@ -43,7 +43,7 @@ namespace mongo {
      *      1.2.3-rc4-pre-
      * If you really need to do something else you'll need to fix _versionArray()
      */
-    const char versionString[] = "2.2.5";
+    const char versionString[] = "2.2.8-pre-";
 
     // See unit test for example outputs
     static BSONArray _versionArray(const char* version){