post 2.2.1-rc0

Even with a byte cap on replication batches, it is still possible to
batch up a large number of writes which can take a long time to apply.
This commit ensures that batches do not get too large.  5 seconds' worth
of replicated ops over the network will take much less than 5 seconds to
write to the database.

Conflicts:

	src/mongo/db/repl/rs_sync.cpp

SConstruct

@@ -861,7 +861,7 @@ def doConfigure(myenv):
 
     # discover modules (subdirectories of db/modules/), and
     # load the (python) module for each module's build.py
-    modules = moduleconfig.discover_modules('.')
+    modules = moduleconfig.discover_modules('src/mongo/')
 
     # ask each module to configure itself, and return a
     # dictionary of name => list_of_sources for each module.
@@ -939,6 +939,9 @@ def getSystemInstallName():
     if nix and os.uname()[2].startswith( "8." ):
         n += "-tiger"
 
+    if len(env.get("MONGO_MODULES", None)):
+            n += "-" + "-".join(env["MONGO_MODULES"].keys())
+
     try:
         findSettingsSetup()
         import settings

debian/changelog

@@ -1,3 +1,9 @@
+mongodb (2.2.0) unstable; urgency=low
+
+  * see http://docs.mongodb.org/manual/release-notes/2.2/
+
+ -- Richard Kreuter <richard@10gen.com>  Wed, 29 Aug 2012 16:56:28 -0500
+
 mongodb (2.1.2) unstable; urgency=low
 
   * see http://jira.mongodb.org/browse/SERVER/fixforversion/10894

distsrc/THIRD-PARTY-NOTICES

@@ -228,6 +228,39 @@ ghost@aladdin.com
  using BMDiff and then compressing the output of BMDiff with
  Snappy.
 
+6) License notice for Google Perftools (TCMalloc utility)
+---------------------------------
+New BSD License
 
+Copyright (c) 1998-2006, Google Inc.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or 
+without modification, are permitted provided that the following 
+conditions are met:
+
+    * Redistributions of source code must retain the above 
+      copyright notice, this list of conditions and the following 
+      disclaimer.
+    * Redistributions in binary form must reproduce the above
+      copyright notice, this list of conditions and the following 
+      disclaimer in the documentation and/or other materials provided 
+      with the distribution.
+    * Neither the name of Google Inc. nor the names of its
+      contributors may be used to endorse or promote products 
+      derived from this software without specific prior written 
+      permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 End

doxygenConfig

@@ -3,7 +3,7 @@
 #---------------------------------------------------------------------------
 DOXYFILE_ENCODING      = UTF-8
 PROJECT_NAME           = MongoDB
-PROJECT_NUMBER         = 2.2.0-rc1-pre-
+PROJECT_NUMBER         = 2.2.1-rc1-pre-
 OUTPUT_DIRECTORY       = docs/doxygen
 CREATE_SUBDIRS         = NO
 OUTPUT_LANGUAGE        = English

jstests/aggregation/bugs/server6779.js

@@ -0,0 +1,18 @@
+// server 6779: serializing ExpressionCoerceToBool
+// This test only fails in debug mode with the bug since that tests round-tripping
+function test(op, val) {
+    t = db.server6779;
+    t.drop();
+
+    t.insert({a:true});
+    t.insert({a:false});
+
+    obj = {};
+    obj[op] = ['$a', val];
+    result = t.aggregate({$project: {_id: 0, bool: obj}});
+
+    assert.commandWorked(result);
+    assert.eq(result.result, [{bool:true}, {bool:false}]);
+}
+test('$and', true);
+test('$or', false);

jstests/cursorb.js

@@ -0,0 +1,17 @@
+// The 'cursor not found in map -1' warning is not logged when get more exhausts a client cursor.
+// SERVER-6931
+
+t = db.jstests_cursorb;
+t.drop();
+
+// Exhaust a client cursor in get more.
+for( i = 0; i < 200; ++i ) {
+    t.save( { a:i } );
+}
+t.find().itcount();
+
+// Check that the 'cursor not found in map -1' message is not printed.  This message indicates an
+// attempt to look up a cursor with an invalid id and should never appear in the log.
+log = db.adminCommand( { getLog:'global' } ).log
+log.forEach( function( line ) { assert( !line.match( /cursor not found in map -1 / ),
+                                        'Cursor map lookup with id -1.' ); } );

jstests/distinct3.js

@@ -16,8 +16,15 @@ for( i = 0; i < 1000; ++i ) {
 }
 db.getLastError();
 
-// The idea here is to try and remove the last match for the {a:1} index scan while distinct is yielding.
-p = startParallelShell( 'for( i = 0; i < 2500; ++i ) { db.jstests_distinct3.remove({a:49}); for( j = 0; j < 20; ++j ) { db.jstests_distinct3.save({a:49,c:49,d:j}) } }' );
+// Attempt to remove the last match for the {a:1} index scan while distinct is yielding.
+p = startParallelShell( 'for( i = 0; i < 2500; ++i ) {                             ' +
+                        '    db.jstests_distinct3.remove( { a:49 } );              ' +
+                        '    for( j = 0; j < 20; ++j ) {                           ' +
+                        '        db.jstests_distinct3.save( { a:49, c:49, d:j } ); ' +
+                        '    }                                                     ' +
+                        '}                                                         ' +
+                        '// Wait for the above writes to complete.                 ' +
+                        'db.getLastError();                                        ' );
 
 for( i = 0; i < 100; ++i ) {
     count = t.distinct( 'c', {$or:[{a:{$gte:0},d:0},{b:{$gte:0}}]} ).length;

jstests/evalb.js

@@ -1,17 +1,40 @@
+// Check the return value of a db.eval function running a database query, and ensure the function's
+// contents are logged in the profile log.
 
-t = db.evalb;
-t.drop();
+// Use a reserved database name to avoid a conflict in the parallel test suite.
+var stddb = db;
+var db = db.getSisterDB( 'evalb' );
 
-t.save( { x : 3 } );
+function profileCursor() {
+    return db.system.profile.find( { user:username } );
+}
 
-assert.eq( 3, db.eval( function(){ return db.evalb.findOne().x; } ) , "A" );
+function lastOp() {
+    return profileCursor().sort( { $natural:-1 } ).next();
+}
 
-db.setProfilingLevel( 2 );
+try {
 
-assert.eq( 3, db.eval( function(){ return db.evalb.findOne().x; } ) , "B" );
+    username = 'jstests_evalb_user';
+    db.addUser( username, 'password', false, 1 );
+    db.auth( username, 'password' );
 
-o = db.system.profile.find( { "command.$eval" : { $exists : true } } ).sort( { $natural : -1 } ).limit(1).next();
-assert( tojson(o).indexOf( "findOne().x" ) > 0 , "C : " + tojson( o ) )
+    t = db.evalb;
+    t.drop();
 
-db.setProfilingLevel( 0 );
+    t.save( { x:3 } );
 
+    assert.eq( 3, db.eval( function() { return db.evalb.findOne().x; } ), 'A' );
+
+    db.setProfilingLevel( 2 );
+
+    assert.eq( 3, db.eval( function() { return db.evalb.findOne().x; } ), 'B' );
+
+    o = lastOp();
+    assert( tojson( o ).indexOf( 'findOne().x' ) > 0, 'C : ' + tojson( o ) );
+}
+finally {
+
+    db.setProfilingLevel(0);
+    db = stddb;
+}

jstests/find_and_modify_server6909.js

@@ -0,0 +1,21 @@
+c = db.find_and_modify_server6906;
+
+
+c.drop();
+
+c.insert( { _id : 5 , a:{ b:1 } } );
+ret = c.findAndModify( { query:{ 'a.b':1 },
+                         update:{ $set:{ 'a.b':2 } }, // Ensure the query on 'a.b' no longer matches.          
+                         new:true } );
+assert.eq( 5, ret._id );
+assert.eq( 2, ret.a.b );
+
+
+c.drop();
+
+c.insert( { _id : null , a:{ b:1 } } );
+ret = c.findAndModify( { query:{ 'a.b':1 },
+                         update:{ $set:{ 'a.b':2 } }, // Ensure the query on 'a.b' no longer matches.          
+                         new:true } );
+assert.eq( 2, ret.a.b );
+

jstests/find_and_modify_server6993.js

@@ -0,0 +1,9 @@
+
+c = db.find_and_modify_server6993;
+c.drop();
+ 
+c.insert( { a:[ 1, 2 ] } );
+ 
+c.findAndModify( { query:{ a:1 }, update:{ $set:{ 'a.$':5 } } } );
+ 
+assert.eq( 5, c.findOne().a[ 0 ] );

jstests/sharding/authCommands.js

@@ -16,9 +16,12 @@ assert.eq(0, db.runCommand({dbStats : 1}).ok);
 
 assert( db.getSiblingDB('local').auth('__system', 'foopdedoop'), "Failed to authenticate as system user" );
 
-assert.eq(0, db.runCommand({dbStats : 1}).ok);
+// Because of SERVER-6897, commands sent without an $auth table are assumed to have full access
+// to preserve compatibility with 2.0
+// assert.eq(0, db.runCommand({dbStats : 1}).ok); // SERVER-6897
 assert.eq(1, db.runCommand({dbStats : 1, $auth : { test : { userName : NumberInt(1) } } } ).ok );
-assert.eq(0, db.runCommand({dbStats : 1}).ok); // Make sure the credentials are temporary.
+ // SERVER-6897
+// assert.eq(0, db.runCommand({dbStats : 1}).ok); // Make sure the credentials are temporary.
 assert.eq(0, db.runCommand({dropDatabase : 1, $auth : { test : { userName : NumberInt(1) } } } ).ok );
 assert.eq(1, db.runCommand({dropDatabase : 1, $auth : { test : { userName : NumberInt(2) } } } ).ok );
 

jstests/sharding/delete_during_migrate.js

@@ -0,0 +1,42 @@
+// Test migrating a big chunk while deletions are happening within that chunk.
+// Test is slightly non-deterministic, since removes could happen before migrate
+// starts. Protect against that by making chunk very large.
+
+// start up a new sharded cluster
+var st = new ShardingTest({ shards : 2, mongos : 1 });
+
+// stop balancer since we want manual control for this
+st.stopBalancer();
+
+var dbname = "testDB";
+var coll = "foo";
+var ns = dbname + "." + coll;
+var s = st.s0;
+var t = s.getDB( dbname ).getCollection( coll );
+
+// Create fresh collection with lots of docs
+t.drop();
+for ( i=0; i<200000; i++ ){
+    t.insert( { a : i  } );
+}
+
+// enable sharding of the collection. Only 1 chunk.
+t.ensureIndex( { a : 1 } );
+s.adminCommand( { enablesharding : dbname } );
+s.adminCommand( { shardcollection : ns , key: { a : 1 } } );
+
+// start a parallel shell that deletes things
+startMongoProgramNoConnect( "mongo" ,
+                            "--host" , getHostName() ,
+                            "--port" , st.s0.port ,
+                            "--eval" , "db." + coll + ".remove({});" ,
+                            dbname );
+
+// migrate while deletions are happening
+var moveResult =  s.adminCommand( { moveChunk : ns ,
+                                    find : { a : 1 } ,
+                                    to : st.getOther( st.getServer( dbname ) ).name } );
+// check if migration worked
+assert( moveResult.ok , "migration didn't work while doing deletes" );
+
+st.stop();

jstests/sharding/deletion_range.js

@@ -0,0 +1,43 @@
+//
+// Tests deletion ranges for a sharded system when using prefix shard key
+//
+
+var st = new ShardingTest({ shards : 2, mongos : 2 });
+
+st.stopBalancer();
+
+var mongos = st.s0;
+var config = mongos.getDB( "config" );
+var admin = mongos.getDB( "admin" );
+var shards = config.shards.find().toArray();
+var shard0 = new Mongo( shards[0].host );
+var shard1 = new Mongo( shards[1].host );
+
+var coll = mongos.getCollection( "foo.bar" );
+
+printjson( admin.runCommand({ enableSharding : coll.getDB() + "" }) );
+printjson( admin.runCommand({ movePrimary : coll.getDB() + "", to : shards[0]._id }) );
+printjson( coll.ensureIndex({ skey : 1, extra : 1 }) );
+printjson( admin.runCommand({ shardCollection : coll + "", key : { skey : 1 } }) );
+
+for( var i = 0; i < 5; i++ ){
+    coll.insert({ skey : 0, extra : i });
+}
+assert.eq( null, coll.getDB().getLastError() );
+
+printjson( admin.runCommand({ split : coll + "", middle : { skey : 0 } }) );
+printjson( admin.runCommand({ moveChunk : coll + "", find : { skey : 0 }, to : shards[1]._id }) );
+
+printjson( shard0.getCollection( coll + "" ).find().toArray() );
+printjson( shard1.getCollection( coll + "" ).find().toArray() );
+
+assert( coll.find().itcount() == 5 );
+
+printjson( admin.runCommand({ moveChunk : coll + "", find : { skey : -1 }, to : shards[1]._id }) );
+
+assert.eq( 0 , shard0.getCollection( coll + "" ).find().itcount() );
+assert.eq( 5 , shard1.getCollection( coll + "" ).find().itcount() );
+
+assert( coll.find().itcount() == 5 );
+
+st.stop()

jstests/sharding/findandmodify2.js

@@ -103,10 +103,10 @@ s.printChunks();
 
 
 print("---------- Verifying that both codepaths resulted in splits...");
-assert.gt( s.config.chunks.count({ "ns": "test." + col_fam }), minChunks, "findAndModify update code path didn't result in splits" );
-assert.gt( s.config.chunks.count({ "ns": "test." + col_fam_upsert }), minChunks, "findAndModify upsert code path didn't result in splits" );
-assert.gt( s.config.chunks.count({ "ns": "test." + col_update }), minChunks, "update code path didn't result in splits" );
-assert.gt( s.config.chunks.count({ "ns": "test." + col_update_upsert }), minChunks, "upsert code path didn't result in splits" );
+assert.gte( s.config.chunks.count({ "ns": "test." + col_fam }), minChunks, "findAndModify update code path didn't result in splits" );
+assert.gte( s.config.chunks.count({ "ns": "test." + col_fam_upsert }), minChunks, "findAndModify upsert code path didn't result in splits" );
+assert.gte( s.config.chunks.count({ "ns": "test." + col_update }), minChunks, "update code path didn't result in splits" );
+assert.gte( s.config.chunks.count({ "ns": "test." + col_update_upsert }), minChunks, "upsert code path didn't result in splits" );
 
 printjson( db[col_update].stats() );
 

jstests/sharding/read_pref.js

@@ -112,12 +112,14 @@ assert.eq( primaryNode.name, explain.server );
 assert.eq( 1, explain.n );
 
 // Kill all members except one
+var stoppedNodes = [];
 for ( var x = 0; x < NODES - 1; x++ ){
     replTest.stop( x );
+    stoppedNodes.push( replTest.nodes[x] );
 }
 
 // Wait for ReplicaSetMonitor to realize nodes are down
-ReplSetTest.awaitRSClientHosts( conn, replTest.nodes[0], { ok: false }, replTest.name );
+ReplSetTest.awaitRSClientHosts( conn, stoppedNodes, { ok: false }, replTest.name );
 
 // Wait for the last node to be in steady state -> secondary (not recovering)
 var lastNode = replTest.nodes[NODES - 1];

jstests/sharding/writeback_bulk_insert.js

@@ -0,0 +1,115 @@
+//
+// Tests whether a writeback error during bulk insert hangs GLE
+//
+
+jsTest.log("Starting sharded cluster...")
+
+var st = new ShardingTest({shards : 1,
+                           mongos : 3,
+                           verbose : 2,
+                           separateConfig : 1})
+
+st.stopBalancer()
+
+var mongosA = st.s0
+var mongosB = st.s1
+var mongosC = st.s2
+
+jsTest.log("Adding new collection...")
+
+var collA = mongosA.getCollection(jsTestName() + ".coll")
+collA.insert({hello : "world"})
+assert.eq(null, collA.getDB().getLastError())
+
+var collB = mongosB.getCollection("" + collA)
+collB.insert({hello : "world"})
+assert.eq(null, collB.getDB().getLastError())
+
+var collC = mongosB.getCollection("" + collA)
+collC.insert({hello : "world"})
+assert.eq(null, collC.getDB().getLastError())
+
+jsTest.log("Enabling sharding...")
+
+printjson(mongosA.getDB("admin").runCommand({enableSharding : collA.getDB()
+                                                              + ""}))
+printjson(mongosA.getDB("admin").runCommand({shardCollection : collA + "",
+                                             key : {_id : 1}}))
+
+// MongoD doesn't know about the config shard version *until* MongoS tells it
+collA.findOne()
+
+// Preparing insert of exactly 16MB
+
+jsTest.log("Preparing bulk insert...")
+
+var data1MB = "x"
+while (data1MB.length < 1024 * 1024)
+    data1MB += data1MB;
+
+var data7MB = ""
+// Data now at 7MB
+for ( var i = 0; i < 7; i++)
+    data7MB += data1MB;
+
+print("7MB object size is : " + Object.bsonsize({_id : 0,
+                                                 d : data7MB}))
+
+var dataCloseTo8MB = data7MB;
+// WARNING - MAGIC NUMBERS HERE
+// The idea is to exceed the 16MB limit by just enough so that the message gets passed in the
+// shell, but adding additional writeback information fails.
+for ( var i = 0; i < 1031 * 1024 + 862; i++) {
+    dataCloseTo8MB += "x"
+}
+
+print("Object size is: " + Object.bsonsize([{_id : 0,
+                                             d : dataCloseTo8MB},
+                                            {_id : 1,
+                                             d : dataCloseTo8MB}]))
+
+jsTest.log("Trigger wbl for mongosB...")
+
+collB.insert([{_id : 0,
+               d : dataCloseTo8MB},
+              {_id : 1,
+               d : dataCloseTo8MB}])
+
+// Will hang if overflow is not detected correctly
+jsTest.log("Waiting for GLE...")
+
+assert.neq(null, collB.getDB().getLastError())
+
+print("GLE correctly returned error...")
+
+assert.eq(3, collA.find().itcount())
+assert.eq(3, collB.find().itcount())
+
+var data8MB = data8MB;
+for ( var i = 0; i < 1024 * 1024; i++) {
+    data8MB += "x"
+}
+
+print("Object size is: " + Object.bsonsize([{_id : 0,
+                                             d : data8MB},
+                                            {_id : 1,
+                                             d : data8MB}]))
+
+jsTest.log("Trigger wbl for mongosC...")
+
+collC.insert([{_id : 0,
+               d : data8MB},
+              {_id : 1,
+               d : data8MB}])
+
+// Should succeed since our insert size is 16MB (plus very small overhead)              
+jsTest.log("Waiting for GLE...")
+
+assert.eq(null, collC.getDB().getLastError())
+
+print("GLE Successful...")
+
+assert.eq(5, collA.find().itcount())
+assert.eq(5, collB.find().itcount())
+
+st.stop()

jstests/slowNightly/balance_tags1.js

@@ -1,5 +1,5 @@
 
-s = new ShardingTest( "balance_tags1" , 3 , 1 , 1 , { chunksize : 1 , nopreallocj : true } )
+s = new ShardingTest( "balance_tags1" , 3 , 1 , 1 , { sync:true, chunksize : 1 , nopreallocj : true } )
 s.config.settings.update( { _id: "balancer" }, { $set : { stopped: false, _nosleep: true } } , true );
 
 db = s.getDB( "test" );
@@ -11,9 +11,13 @@ db.getLastError();
 s.adminCommand( { enablesharding : "test" } )
 s.adminCommand( { shardcollection : "test.foo" , key : { _id : 1 } } );
 
+s.stopBalancer();
+
 for ( i=0; i<20; i++ )
     s.adminCommand( { split : "test.foo" , middle : { _id : i } } );
 
+s.startBalancer();
+
 sh.status( true )
 assert.soon( function() {
     counts = s.chunkCounts( "foo" );
@@ -42,6 +46,7 @@ assert.soon( function() {
     return counts["shard0002"] == 0;
 } , "balance 2 didn't happen" , 1000 * 60 * 10 , 1000 )
 
+printjson(sh.status());
 
 s.stop();
 

jstests/splitvector.js

@@ -28,6 +28,22 @@ assertChunkSizes = function ( splitVec , numDocs , maxChunkSize , msg ){
     }
 }
 
+// Takes two documents and asserts that both contain exactly the same set of field names.
+// This is useful for checking that splitPoints have the same format as the original key pattern,
+// even when sharding on a prefix key.
+// Not very efficient, so only call when # of field names is small
+var assertFieldNamesMatch = function( splitPoint , keyPattern ){
+    for ( var p in splitPoint ) {
+        if( splitPoint.hasOwnProperty( p ) ) {
+            assert( keyPattern.hasOwnProperty( p ) , "property " + p + " not in keyPattern" );
+        }
+    }
+    for ( var p in keyPattern ) {
+        if( keyPattern.hasOwnProperty( p ) ){
+            assert( splitPoint.hasOwnProperty( p ) , "property " + p + " not in splitPoint" );
+        }
+    }
+}
 
 // -------------------------
 //  TESTS START HERE
@@ -84,6 +100,9 @@ var case4 = function() {
     assert.eq( true , res.ok , "4b" );
     assert.close( numDocs*docSize / ((1<<20) * factor), res.splitKeys.length , "num split keys" , -1 );
     assertChunkSizes( res.splitKeys , numDocs, (1<<20) * factor , "4d" );
+    for( i=0; i < res.splitKeys.length; i++ ){
+        assertFieldNamesMatch( res.splitKeys[i] , {x : 1} );
+    }
 }
 case4();
 
@@ -104,6 +123,9 @@ var case5 = function() {
 
     assert.eq( true , res.ok , "5a" );
     assert.eq( 1 , res.splitKeys.length , "5b" );
+    for( i=0; i < res.splitKeys.length; i++ ){
+        assertFieldNamesMatch( res.splitKeys[i] , {x : 1} );
+    }
 }
 case5();
 
@@ -124,6 +146,9 @@ var case6 = function() {
 
     assert.eq( true , res.ok , "6a" );
     assert.eq( 19 , res.splitKeys.length , "6b" );
+    for( i=0; i < res.splitKeys.length; i++ ){
+        assertFieldNamesMatch( res.splitKeys[i] , {x : 1} );
+    }
 }
 case6();
 
@@ -149,6 +174,9 @@ var case7 = function() {
 
     assert.eq( true , res.ok , "7a" );
     assert.eq( 2 , res.splitKeys[0].x, "7b");
+    for( i=0; i < res.splitKeys.length; i++ ){
+        assertFieldNamesMatch( res.splitKeys[i] , {x : 1} );
+    }
 }
 case7();
 
@@ -180,6 +208,9 @@ var case8 = function() {
     assert.eq( 2 , res.splitKeys.length , "8b" );
     assert.eq( 2 , res.splitKeys[0].x , "8c" );
     assert.eq( 3 , res.splitKeys[1].x , "8d" );
+    for( i=0; i < res.splitKeys.length; i++ ){
+        assertFieldNamesMatch( res.splitKeys[i] , {x : 1} );
+    }
 }
 case8();
 
@@ -211,6 +242,9 @@ var case9 = function() {
         assert.eq( true , res.ok , "9a: " + tojson(res) );
         assert.eq( 1 , res.splitKeys.length , "9b: " + tojson(res) );
         assert.eq( 2 , res.splitKeys[0].x , "9c: " + tojson(res) );
+        for( i=0; i < res.splitKeys.length; i++ ){
+            assertFieldNamesMatch( res.splitKeys[i] , {x : 1} );
+        }
     }
 }
 case9();

rpm/mongo.spec

@@ -1,7 +1,7 @@
 Name: mongo-10gen
 Conflicts: mongo, mongo-10gen-unstable
 Obsoletes: mongo-stable
-Version: 2.1.2
+Version: 2.2.0
 Release: mongodb_1%{?dist}
 Summary: mongodb client shell and tools
 License: AGPL 3.0

site_scons/libdeps.py

@@ -138,13 +138,13 @@ def get_libdeps(source, target, env, for_signature):
     """
 
     target = env.Flatten([target])
-    return list(__get_libdeps(target[0], 'LIBDEPS'))
+    return sorted_by_str(__get_libdeps(target[0], 'LIBDEPS'))
 
 def get_libdeps_objs(source, target, env, for_signature):
     objs = set()
     for lib in get_libdeps(source, target, env, for_signature):
         objs.update(lib.sources_set)
-    return list(objs)
+    return sorted_by_str(objs)
 
 def get_libdeps_special_sun(source, target, env, for_signature):
     x = get_libdeps(source, target, env, for_signature )

src/mongo/client/dbclient_rs.h

@@ -448,6 +448,12 @@ namespace mongo {
 
         // ---- access raw connections ----
 
+        /**
+         * WARNING: this method is very dangerous - this object can decide to free the
+         *     returned master connection any time.
+         *
+         * @return the reference to the address that points to the master connection.
+         */
         DBClientConnection& masterConn();
         DBClientConnection& slaveConn();
 

src/mongo/db/btreecursor.cpp

@@ -221,20 +221,20 @@ namespace mongo {
         NamespaceDetails *d, int idxNo, const IndexDetails& id, 
         const BSONObj &startKey, const BSONObj &endKey, bool endKeyInclusive, int direction) 
     { 
-        BtreeCursor *c = make( d , idxNo , id );
+        auto_ptr<BtreeCursor> c( make( d , idxNo , id ) );
         c->init(startKey,endKey,endKeyInclusive,direction);
         c->initWithoutIndependentFieldRanges();
         dassert( c->_dups.size() == 0 );
-        return c;
+        return c.release();
     }
 
     BtreeCursor* BtreeCursor::make(
         NamespaceDetails *d, int idxNo, const IndexDetails& id, 
         const shared_ptr< FieldRangeVector > &bounds, int singleIntervalLimit, int direction )
     {
-        BtreeCursor *c = make( d , idxNo , id );
+        auto_ptr<BtreeCursor> c( make( d , idxNo , id ) );
         c->init(bounds,singleIntervalLimit,direction);
-        return c;
+        return c.release();
     }
 
     BtreeCursor::BtreeCursor( NamespaceDetails* nsd , int theIndexNo, const IndexDetails& id ) 

src/mongo/db/clientcursor.cpp

@@ -792,7 +792,7 @@ namespace mongo {
         if ( ! cc().getAuthenticationInfo()->isAuthorizedReads( nsToDatabase( cursor->ns() ) ) )
             return false;
 
-        // mustn't have an active ClientCursor::Pointer
+        // Must not have an active ClientCursor::Pin.
         massert( 16089,
                 str::stream() << "Cannot kill active cursor " << id,
                 cursor->_pinValue < 100 );

src/mongo/db/clientcursor.h

@@ -100,6 +100,9 @@ namespace mongo {
                 }
             }
             void release() {
+                if ( _cursorid == INVALID_CURSOR_ID ) {
+                    return;
+                }
                 ClientCursor *cursor = c();
                 _cursorid = INVALID_CURSOR_ID;
                 if ( cursor ) {

src/mongo/db/commands/find_and_modify.cpp

@@ -78,7 +78,7 @@ namespace mongo {
                     return runNoDirectClient( ns , 
                                               query , fields , update , 
                                               upsert , returnNew , remove , 
-                                              result );
+                                              result , errmsg );
                 }
                 catch ( PageFaultException& e ) {
                     e.touch();
@@ -108,7 +108,7 @@ namespace mongo {
         bool runNoDirectClient( const string& ns , 
                                 const BSONObj& queryOriginal , const BSONObj& fields , const BSONObj& update , 
                                 bool upsert , bool returnNew , bool remove ,
-                                BSONObjBuilder& result ) {
+                                BSONObjBuilder& result , string& errmsg ) {
             
             
             Lock::DBWrite lk( ns );
@@ -123,6 +123,30 @@ namespace mongo {
                 // we're going to re-write the query to be more efficient
                 // we have to be a little careful because of positional operators
                 // maybe we can pass this all through eventually, but right now isn't an easy way
+                
+                bool hasPositionalUpdate = false;
+                {
+                    // if the update has a positional piece ($)
+                    // then we need to pull all query parts in
+                    // so here we check for $
+                    // a little hacky
+                    BSONObjIterator i( update );
+                    while ( i.more() ) {
+                        const BSONElement& elem = i.next();
+                        
+                        if ( elem.fieldName()[0] != '$' || elem.type() != Object )
+                            continue;
+
+                        BSONObjIterator j( elem.Obj() );
+                        while ( j.more() ) {
+                            if ( str::contains( j.next().fieldName(), ".$" ) ) {
+                                hasPositionalUpdate = true;
+                                break;
+                            }
+                        }
+                    }
+                }
+
                 BSONObjBuilder b( queryOriginal.objsize() + 10 );
                 b.append( doc["_id"] );
                 
@@ -137,7 +161,7 @@ namespace mongo {
                         continue;
                     }
                     
-                    if ( ! str::contains( elem.fieldName() , '.' ) ) {
+                    if ( ! hasPositionalUpdate ) {
                         // if there is a dotted field, accept we may need more query parts
                         continue;
                     }
@@ -177,11 +201,21 @@ namespace mongo {
                     UpdateResult res = updateObjects( ns.c_str() , update , queryModified , upsert , false , true , cc().curop()->debug() );
                     
                     if ( returnNew ) {
-                        if ( ! res.existing && res.upserted.isSet() ) {
+                        if ( res.upserted.isSet() ) {
                             queryModified = BSON( "_id" << res.upserted );
                         }
-                        log() << "queryModified: " << queryModified << endl;
-                        verify( Helpers::findOne( ns.c_str() , queryModified , doc ) );
+                        else if ( queryModified["_id"].type() ) {
+                            // we do this so that if the update changes the fields, it still matches
+                            queryModified = queryModified["_id"].wrap();
+                        }
+                        if ( ! Helpers::findOne( ns.c_str() , queryModified , doc ) ) {
+                            errmsg = str::stream() << "can't find object after modification  " 
+                                                   << " ns: " << ns 
+                                                   << " queryModified: " << queryModified 
+                                                   << " queryOriginal: " << queryOriginal;
+                            log() << errmsg << endl;
+                            return false;
+                        }
                         _appendHelper( result , doc , true , fields );
                     }
                     

src/mongo/db/commands/mr.cpp

@@ -1024,7 +1024,7 @@ namespace mongo {
 
                 uassert( 16149 , "cannot run map reduce without the js engine", globalScriptEngine );
 
-                auto_ptr<ClientCursor> holdCursor;
+                ClientCursor::Holder holdCursor;
                 ShardChunkManagerPtr chunkManager;
 
                 {
@@ -1040,8 +1040,8 @@ namespace mongo {
                     // Get a very basic cursor, prevents deletion of migrated data while we m/r
                     shared_ptr<Cursor> temp = NamespaceDetailsTransient::getCursor( config.ns.c_str(), BSONObj(), BSONObj() );
                     uassert( 15876, str::stream() << "could not create cursor over " << config.ns << " to hold data while prepping m/r", temp.get() );
-                    holdCursor = auto_ptr<ClientCursor>( new ClientCursor( QueryOption_NoCursorTimeout , temp , config.ns.c_str() ) );
-                    uassert( 15877, str::stream() << "could not create m/r holding client cursor over " << config.ns, holdCursor.get() );
+                    holdCursor.reset( new ClientCursor( QueryOption_NoCursorTimeout , temp , config.ns.c_str() ) );
+                    uassert( 15877, str::stream() << "could not create m/r holding client cursor over " << config.ns, holdCursor );
 
                 }
 

src/mongo/db/d_concurrency.cpp

@@ -278,11 +278,24 @@ namespace mongo {
     void Lock::ParallelBatchWriterMode::iAmABatchParticipant() {
         lockState()._batchWriter = true;
     }
-    Lock::ParallelBatchWriterSupport::ParallelBatchWriterSupport() :
-        _lk( lockState()._batchWriter ? 0 : new RWLockRecursive::Shared(ParallelBatchWriterMode::_batchLock) )
-    {
+
+    Lock::ParallelBatchWriterSupport::ParallelBatchWriterSupport() {
+        relock();
     }
 
+    void Lock::ParallelBatchWriterSupport::tempRelease() {
+        _lk.reset( 0 );
+    }
+
+    void Lock::ParallelBatchWriterSupport::relock() {
+        LockState& ls = lockState();
+        if ( ! ls._batchWriter ) {
+            AcquiringParallelWriter a(ls);
+            _lk.reset( new RWLockRecursive::Shared(ParallelBatchWriterMode::_batchLock) );
+        }
+    }
+
+
     Lock::ScopedLock::ScopedLock( char type ) 
         : _type(type), _stat(0) {
         LockState& ls = lockState();
@@ -306,6 +319,7 @@ namespace mongo {
     void Lock::ScopedLock::tempRelease() {
         long long micros = _timer.micros();
         _tempRelease();
+        _pbws_lk.tempRelease();
         _recordTime( micros ); // might as well do after we unlock
     }
 
@@ -324,6 +338,7 @@ namespace mongo {
     }
     
     void Lock::ScopedLock::relock() {
+        _pbws_lk.relock();
         _relock();
         resetTime();
     }

src/mongo/db/d_concurrency.h

@@ -78,9 +78,15 @@ namespace mongo {
 
     private:
         class ParallelBatchWriterSupport : boost::noncopyable {
-            scoped_ptr<RWLockRecursive::Shared> _lk;
         public:
             ParallelBatchWriterSupport();
+
+        private:
+            void tempRelease();
+            void relock();
+
+            scoped_ptr<RWLockRecursive::Shared> _lk;
+            friend class ScopedLock;
         };
 
     public:

src/mongo/db/dbcommands.cpp

@@ -1249,8 +1249,9 @@ namespace mongo {
                     errmsg = "couldn't find valid index containing key pattern";
                     return false;
                 }
+                // If both min and max non-empty, append MinKey's to make them fit chosen index
                 min = Helpers::modifiedRangeBound( min , idx->keyPattern() , -1 );
-                max = Helpers::modifiedRangeBound( max , idx->keyPattern() , 1 );
+                max = Helpers::modifiedRangeBound( max , idx->keyPattern() , -1 );
 
                 c.reset( BtreeCursor::make( d, d->idxNo(*idx), *idx, min, max, false, 1 ) );
             }
@@ -1908,12 +1909,11 @@ namespace mongo {
                 BSONObj authObj = cmdObj[AuthenticationTable::fieldName].Obj();
                 ai->setTemporaryAuthorization( authObj );
             } else {
-                result.append( "errmsg" ,
-                               "unauthorized: no auth credentials provided for command and "
-                               "authenticated using internal user.  This is most likely because "
-                               "you are using an old version of mongos" );
-                log() << "command denied: " << cmdObj.toString() << endl;
-                return false;
+                SOMETIMES ( noAuthTableCounter, 1000 ) {
+                    warning() << "Received command without $auth table.  This is probably because "
+                        "you are running with 1 or more mongod or mongos nodes that are running a "
+                        "version prior to 2.2.  Command object: " << cmdObj.toString() << endl;
+                }
             }
         }
 

src/mongo/db/dbhelpers.cpp

@@ -293,8 +293,12 @@ namespace mongo {
                     
                     IndexDetails& i = nsd->idx( ii );
 
+                    // Extend min to get (min, MinKey, MinKey, ....)
                     BSONObj newMin = Helpers::modifiedRangeBound( min , keyPattern , -1 );
-                    BSONObj newMax = Helpers::modifiedRangeBound( max , keyPattern , 1 );
+                    // If upper bound is included, extend max to get (max, MaxKey, MaxKey, ...)
+                    // If not included, extend max to get (max, MinKey, MinKey, ....)
+                    int minOrMax = maxInclusive ? 1 : -1;
+                    BSONObj newMax = Helpers::modifiedRangeBound( max , keyPattern , minOrMax );
                     
                     c.reset( BtreeCursor::make( nsd , ii , i , newMin , newMax , maxInclusive , 1 ) );
                 }

src/mongo/db/dbwebserver.cpp

@@ -91,7 +91,7 @@ namespace mongo {
                 pcrecpp::StringPiece input( auth );
 
                 string name, val;
-                pcrecpp::RE re("(\\w+)=\"?(.*?)\"?, ");
+                pcrecpp::RE re("(\\w+)=\"?(.*?)\"?,\\s*");
                 while ( re.Consume( &input, &name, &val) ) {
                     parms[name] = val;
                 }

src/mongo/db/dur.cpp

@@ -262,42 +262,51 @@ namespace mongo {
         }
 
         bool NOINLINE_DECL DurableImpl::_aCommitIsNeeded() {
-            if( !Lock::isLocked() ) {
-                DEV log() << "commitIfNeeded but we are unlocked that is ok but why do we get here" << endl;
-                Lock::GlobalRead r;
-                if( commitJob.bytes() < UncommittedBytesLimit ) {
-                    // someone else beat us to it
-                    return false;
+            switch (Lock::isLocked()) {
+                case '\0': {
+                    DEV log() << "commitIfNeeded but we are unlocked that is ok but why do we get here" << endl;
+                    Lock::GlobalRead r;
+                    if( commitJob.bytes() < UncommittedBytesLimit ) {
+                        // someone else beat us to it
+                        return false;
+                    }
+                    commitNow();
+                    return true;
                 }
-                commitNow();
-            }
-            else if( Lock::isLocked() == 'w' ) {
-                if( Lock::atLeastReadLocked("local") ) { 
-                    error() << "can't commitNow from commitIfNeeded, as we are in local db lock" << endl;
-                    printStackTrace();
-                    dassert(false); // this will make _DEBUG builds terminate. so we will notice in buildbot.
-                    return false;
-                }
-                else if( Lock::atLeastReadLocked("admin") ) { 
-                    error() << "can't commitNow from commitIfNeeded, as we are in admin db lock" << endl;
-                    printStackTrace();
-                    dassert(false);
-                    return false;
-                }
-                else {
+                case 'w': {
+                    if( Lock::atLeastReadLocked("local") ) {
+                        error() << "can't commitNow from commitIfNeeded, as we are in local db lock" << endl;
+                        printStackTrace();
+                        dassert(false); // this will make _DEBUG builds terminate. so we will notice in buildbot.
+                        return false;
+                    }
+                    if( Lock::atLeastReadLocked("admin") ) {
+                        error() << "can't commitNow from commitIfNeeded, as we are in admin db lock" << endl;
+                        printStackTrace();
+                        dassert(false);
+                        return false;
+                    }
+
                     log(1) << "commitIfNeeded upgrading from shared write to exclusive write state"
                            << endl;
                     Lock::DBWrite::UpgradeToExclusive ex;
                     if (ex.gotUpgrade()) {
                         commitNow();
                     }
+                    return true;
                 }
+
+                case 'W':
+                case 'R':
+                    commitNow();
+                    return true;
+
+                case 'r':
+                    return false;
+
+                default:
+                    fassertFailed(16434); // unknown lock type
             }
-            else { 
-                // 'W'
-                commitNow();
-            }
-            return true;
         }
 
         /** we may need to commit earlier than normal if data are being written at 

src/mongo/db/dur_recover.cpp

@@ -214,9 +214,55 @@ namespace mongo {
             _mmfs.clear();
         }
 
-        void RecoveryJob::write(const ParsedJournalEntry& entry) {
+        void RecoveryJob::write(const ParsedJournalEntry& entry, MongoMMF* mmf) {
             //TODO(mathias): look into making some of these dasserts
             verify(entry.e);
+
+            if ((entry.e->ofs + entry.e->len) <= mmf->length()) {
+                verify(mmf->view_write());
+                verify(entry.e->srcData());
+
+                void* dest = (char*)mmf->view_write() + entry.e->ofs;
+                memcpy(dest, entry.e->srcData(), entry.e->len);
+                stats.curr->_writeToDataFilesBytes += entry.e->len;
+            }
+            else {
+                massert(13622, "Trying to write past end of file in WRITETODATAFILES", _recovering);
+            }
+        }
+
+        void RecoveryJob::applyEntry(const ParsedJournalEntry& entry, bool apply, bool dump, MongoMMF* mmf) {
+            if( entry.e ) {
+                if( dump ) {
+                    stringstream ss;
+                    ss << "  BASICWRITE " << setw(20) << entry.dbName << '.';
+                    if( entry.e->isNsSuffix() )
+                        ss << "ns";
+                    else
+                        ss << setw(2) << entry.e->getFileNo();
+                    ss << ' ' << setw(6) << entry.e->len << ' ' << /*hex << setw(8) << (size_t) fqe.srcData << dec <<*/
+                       "  " << hexdump(entry.e->srcData(), entry.e->len);
+                    log() << ss.str() << endl;
+                }
+                if( apply ) {
+                    write(entry, mmf);
+                }
+            }
+            else if(entry.op) {
+                // a DurOp subclass operation
+                if( dump ) {
+                    log() << "  OP " << entry.op->toString() << endl;
+                }
+                if( apply ) {
+                    if( entry.op->needFilesClosed() ) {
+                        _close(); // locked in processSection
+                    }
+                    entry.op->replay();
+                }
+            }
+        }
+
+        MongoMMF* RecoveryJob::getMongoMMF(const ParsedJournalEntry& entry) {
             verify(entry.dbName);
             verify(strnlen(entry.dbName, MaxDatabaseNameLen) < MaxDatabaseNameLen);
 
@@ -243,48 +289,7 @@ namespace mongo {
                 mmf = sp.get();
             }
 
-            if ((entry.e->ofs + entry.e->len) <= mmf->length()) {
-                verify(mmf->view_write());
-                verify(entry.e->srcData());
-
-                void* dest = (char*)mmf->view_write() + entry.e->ofs;
-                memcpy(dest, entry.e->srcData(), entry.e->len);
-                stats.curr->_writeToDataFilesBytes += entry.e->len;
-            }
-            else {
-                massert(13622, "Trying to write past end of file in WRITETODATAFILES", _recovering);
-            }
-        }
-
-        void RecoveryJob::applyEntry(const ParsedJournalEntry& entry, bool apply, bool dump) {
-            if( entry.e ) {
-                if( dump ) {
-                    stringstream ss;
-                    ss << "  BASICWRITE " << setw(20) << entry.dbName << '.';
-                    if( entry.e->isNsSuffix() )
-                        ss << "ns";
-                    else
-                        ss << setw(2) << entry.e->getFileNo();
-                    ss << ' ' << setw(6) << entry.e->len << ' ' << /*hex << setw(8) << (size_t) fqe.srcData << dec <<*/
-                       "  " << hexdump(entry.e->srcData(), entry.e->len);
-                    log() << ss.str() << endl;
-                }
-                if( apply ) {
-                    write(entry);
-                }
-            }
-            else if(entry.op) {
-                // a DurOp subclass operation
-                if( dump ) {
-                    log() << "  OP " << entry.op->toString() << endl;
-                }
-                if( apply ) {
-                    if( entry.op->needFilesClosed() ) {
-                        _close(); // locked in processSection
-                    }
-                    entry.op->replay();
-                }
-            }
+            return mmf;
         }
 
         void RecoveryJob::applyEntries(const vector<ParsedJournalEntry> &entries) {
@@ -293,8 +298,18 @@ namespace mongo {
             if( dump )
                 log() << "BEGIN section" << endl;
 
+            const char* lastDbName = NULL;
+            int lastFileNo = 0;
+            MongoMMF* mmf = NULL;
             for( vector<ParsedJournalEntry>::const_iterator i = entries.begin(); i != entries.end(); ++i ) {
-                applyEntry(*i, apply, dump);
+                if (i->e && (i->dbName != lastDbName || i->e->getFileNo() != lastFileNo)) {
+                    mmf = getMongoMMF(*i);
+                    lastDbName = i->dbName;
+                    lastFileNo = i->e->getFileNo();
+                }
+                fassert(16429, !i->e || mmf);
+
+                applyEntry(*i, apply, dump, mmf);
             }
 
             if( dump )

src/mongo/db/dur_recover.h

@@ -28,12 +28,13 @@ namespace mongo {
 
             static RecoveryJob & get() { return _instance; }
         private:
-            void write(const ParsedJournalEntry& entry); // actually writes to the file
-            void applyEntry(const ParsedJournalEntry& entry, bool apply, bool dump);
+            void write(const ParsedJournalEntry& entry, MongoMMF* mmf); // actually writes to the file
+            void applyEntry(const ParsedJournalEntry& entry, bool apply, bool dump, MongoMMF* mmf);
             void applyEntries(const vector<ParsedJournalEntry> &entries);
             bool processFileBuffer(const void *, unsigned len);
             bool processFile(boost::filesystem::path journalfile);
             void _close(); // doesn't lock
+            MongoMMF* getMongoMMF(const ParsedJournalEntry& entry);
 
             list<boost::shared_ptr<MongoMMF> > _mmfs;
 

src/mongo/db/lockstate.cpp

@@ -35,7 +35,8 @@ namespace mongo {
           _otherCount(0), 
           _otherLock(NULL),
           _scopedLk(NULL),
-          _lockPending(false)
+          _lockPending(false),
+          _lockPendingParallelWriter(false)
     {
     }
 
@@ -241,4 +242,13 @@ namespace mongo {
             stat->recordAcquireTimeMicros( _ls.threadState(), _lock->acquireFinished( stat ) );
     }
     
+    AcquiringParallelWriter::AcquiringParallelWriter( LockState& ls )
+        : _ls( ls ) {
+        _ls._lockPendingParallelWriter = true;
+    }
+    
+    AcquiringParallelWriter::~AcquiringParallelWriter() {
+        _ls._lockPendingParallelWriter = false;
+    }
+
 }

src/mongo/db/lockstate.h

@@ -50,7 +50,7 @@ namespace mongo {
         bool isLocked( const StringData& ns ); // rwRW
 
         /** pending means we are currently trying to get a lock */
-        bool hasLockPending() const { return _lockPending; }
+        bool hasLockPending() const { return _lockPending || _lockPendingParallelWriter; }
 
         // ----
 
@@ -105,8 +105,10 @@ namespace mongo {
         Lock::ScopedLock* _scopedLk;   
         
         bool _lockPending;
+        bool _lockPendingParallelWriter;
 
         friend class Acquiring;
+        friend class AcquiringParallelWriter;
     };
 
     class WrapperForRWLock : boost::noncopyable { 
@@ -132,7 +134,14 @@ namespace mongo {
         LockState& _ls;
     };
         
+    class AcquiringParallelWriter {
+    public:
+        AcquiringParallelWriter( LockState& ls );
+        ~AcquiringParallelWriter();
 
+    private:
+        LockState& _ls;
+    };
 
 
 }

src/mongo/db/oplog.cpp

@@ -720,7 +720,7 @@ namespace mongo {
     /** @param fromRepl false if from ApplyOpsCmd
         @return true if was and update should have happened and the document DNE.  see replset initial sync code.
      */
-    bool applyOperation_inlock(const BSONObj& op , bool fromRepl ) {
+    bool applyOperation_inlock(const BSONObj& op, bool fromRepl, bool convertUpdateToUpsert) {
         LOG(6) << "applying op: " << op << endl;
         bool failedUpdate = false;
 
@@ -789,7 +789,7 @@ namespace mongo {
 
             OpDebug debug;
             BSONObj updateCriteria = op.getObjectField("o2");
-            bool upsert = fields[3].booleanSafe();
+            bool upsert = fields[3].booleanSafe() || convertUpdateToUpsert;
             UpdateResult ur = updateObjects(ns, o, updateCriteria, upsert, /*multi*/ false,
                                             /*logop*/ false , debug, /*fromMigrate*/ false,
                                             QueryPlanSelectionPolicy::idElseNatural() );

src/mongo/db/oplog.h

@@ -156,5 +156,5 @@ namespace mongo {
      * @param fromRepl really from replication or for testing/internal/command/etc...
      * Returns if the op was an update that could not be applied (true on failure)
      */
-    bool applyOperation_inlock(const BSONObj& op , bool fromRepl = true );
+    bool applyOperation_inlock(const BSONObj& op, bool fromRepl = true, bool convertUpdateToUpsert = false);
 }

src/mongo/db/pipeline/accumulator.cpp

@@ -38,17 +38,17 @@ namespace mongo {
 
     void Accumulator::opToBson(
         BSONObjBuilder *pBuilder, string opName,
-        string fieldName) const {
+        string fieldName, bool requireExpression) const {
         verify(vpOperand.size() == 1);
         BSONObjBuilder builder;
-        vpOperand[0]->addToBsonObj(&builder, opName, false);
+        vpOperand[0]->addToBsonObj(&builder, opName, requireExpression);
         pBuilder->append(fieldName, builder.done());
     }
 
     void Accumulator::addToBsonObj(
         BSONObjBuilder *pBuilder, string fieldName,
         bool requireExpression) const {
-        opToBson(pBuilder, getOpName(), fieldName);
+        opToBson(pBuilder, getOpName(), fieldName, requireExpression);
     }
 
     void Accumulator::addToBsonArray(BSONArrayBuilder *pBuilder) const {

src/mongo/db/pipeline/accumulator.h

@@ -56,7 +56,8 @@ namespace mongo {
           @param opName the operator name
          */
         void opToBson(
-            BSONObjBuilder *pBuilder, string fieldName, string opName) const;
+            BSONObjBuilder *pBuilder, string fieldName, string opName,
+            bool requireExpression) const;
     };
 
 

src/mongo/db/pipeline/document_source_group.cpp

@@ -72,14 +72,14 @@ namespace mongo {
         BSONObjBuilder insides;
 
         /* add the _id */
-        pIdExpression->addToBsonObj(&insides, Document::idName.c_str(), false);
+        pIdExpression->addToBsonObj(&insides, Document::idName.c_str(), true);
 
         /* add the remaining fields */
         const size_t n = vFieldName.size();
         for(size_t i = 0; i < n; ++i) {
             intrusive_ptr<Accumulator> pA((*vpAccumulatorFactory[i])(pExpCtx));
             pA->addOperand(vpExpression[i]);
-            pA->addToBsonObj(&insides, vFieldName[i], false);
+            pA->addToBsonObj(&insides, vFieldName[i], true);
         }
 
         pBuilder->append(groupName, insides.done());

src/mongo/db/pipeline/expression.cpp

@@ -526,14 +526,24 @@ namespace mongo {
     }
 
     void ExpressionCoerceToBool::addToBsonObj(
-        BSONObjBuilder *pBuilder, string fieldName,
-        bool requireExpression) const {
-        verify(false && "not possible"); // no equivalent of this
+            BSONObjBuilder *pBuilder, string fieldName,
+            bool requireExpression) const {
+        // Serializing as an $and expression which will become a CoerceToBool
+        BSONObjBuilder sub (pBuilder->subobjStart(fieldName));
+        BSONArrayBuilder arr (sub.subarrayStart("$and"));
+        pExpression->addToBsonArray(&arr);
+        arr.doneFast();
+        sub.doneFast();
     }
 
     void ExpressionCoerceToBool::addToBsonArray(
-        BSONArrayBuilder *pBuilder) const {
-        verify(false && "not possible"); // no equivalent of this
+            BSONArrayBuilder *pBuilder) const {
+        // Serializing as an $and expression which will become a CoerceToBool
+        BSONObjBuilder sub (pBuilder->subobjStart());
+        BSONArrayBuilder arr (sub.subarrayStart("$and"));
+        pExpression->addToBsonArray(&arr);
+        arr.doneFast();
+        sub.doneFast();
     }
 
     /* ----------------------- ExpressionCompare --------------------------- */
@@ -606,10 +616,10 @@ namespace mongo {
         /*             -1      0      1      reverse          name   */
         /* EQ  */ { { false, true,  false }, Expression::EQ,  "$eq"  },
         /* NE  */ { { true,  false, true },  Expression::NE,  "$ne"  },
-        /* GT  */ { { false, false, true },  Expression::LTE, "$gt"  },
-        /* GTE */ { { false, true,  true },  Expression::LT,  "$gte" },
-        /* LT  */ { { true,  false, false }, Expression::GTE, "$lt"  },
-        /* LTE */ { { true,  true,  false }, Expression::GT,  "$lte" },
+        /* GT  */ { { false, false, true },  Expression::LT,  "$gt"  },
+        /* GTE */ { { false, true,  true },  Expression::LTE, "$gte" },
+        /* LT  */ { { true,  false, false }, Expression::GT,  "$lt"  },
+        /* LTE */ { { true,  true,  false }, Expression::GTE, "$lte" },
         /* CMP */ { { false, false, false }, Expression::CMP, "$cmp" },
     };
 

src/mongo/db/prefetch.cpp

@@ -135,7 +135,7 @@ namespace mongo {
                         _dummy_char += *(result.objdata() + i); 
                     }
                     // hit the last page, in case we missed it above
-                    _dummy_char += *(result.objdata() + result.objsize());
+                    _dummy_char += *(result.objdata() + result.objsize() - 1);
                 }
             }
             catch(const DBException& e) {

src/mongo/db/repl/rs.cpp

@@ -84,10 +84,7 @@ namespace mongo {
         log() << "replSet See http://dochub.mongodb.org/core/resyncingaverystalereplicasetmember" << rsLog;
 
         // reset minvalid so that we can't become primary prematurely
-        {
-            Lock::DBWrite lk("local.replset.minvalid");
-            Helpers::putSingleton("local.replset.minvalid", oldest);
-        }
+        setMinValid(oldest);
 
         sethbmsg("error RS102 too stale to catch up");
         changeState(MemberState::RS_RECOVERING);
@@ -838,5 +835,13 @@ namespace mongo {
         cc().getAuthenticationInfo()->authorize("local","_repl");
     }
     
+    void ReplSetImpl::setMinValid(BSONObj obj) {
+        BSONObjBuilder builder;
+        builder.appendTimestamp("ts", obj["ts"].date());
+        builder.append("h", obj["h"]);
+        Lock::DBWrite cx( "local" );
+        Helpers::putSingleton("local.replset.minvalid", builder.obj());
+    }
+
 }
 

src/mongo/db/repl/rs.h

@@ -538,7 +538,7 @@ namespace mongo {
         void syncRollback(OplogReader& r);
         void syncThread();
         const OpTime lastOtherOpTime() const;
-
+        static void setMinValid(BSONObj obj);
     private:
         IndexPrefetchConfig _indexPrefetchConfig;
     };

src/mongo/db/repl/rs_initialsync.cpp

@@ -419,7 +419,9 @@ namespace mongo {
                 log() << "replSet set minValid=" << minValid["ts"]._opTime().toString() << rsLog;
             }
             catch(...) { }
-            Helpers::putSingleton("local.replset.minvalid", minValid);
+
+            theReplSet->setMinValid(minValid);
+
             cx.ctx().db()->flushFiles(true);
         }
 

src/mongo/db/repl/rs_rollback.cpp

@@ -304,18 +304,6 @@ namespace mongo {
         bson::bo goodVersionOfObject;
     };
 
-    static void setMinValid(bo newMinValid) {
-        try {
-            log() << "replSet minvalid=" << newMinValid["ts"]._opTime().toStringLong() << rsLog;
-        }
-        catch(...) { }
-        {
-            Helpers::putSingleton("local.replset.minvalid", newMinValid);
-            Client::Context cx( "local." );
-            cx.db()->flushFiles(true);
-        }
-    }
-
     void ReplSetImpl::syncFixUp(HowToFixUp& h, OplogReader& r) {
         DBClientConnection *them = r.conn();
 
@@ -378,6 +366,7 @@ namespace mongo {
 
         /* we have items we are writing that aren't from a point-in-time.  thus best not to come online
            until we get to that point in freshness. */
+        log() << "replSet minvalid=" << newMinValid["ts"]._opTime().toStringLong() << rsLog;
         setMinValid(newMinValid);
 
         /** any full collection resyncs required? */
@@ -411,6 +400,7 @@ namespace mongo {
                         err = "can't get minvalid from primary";
                     }
                     else {
+                        log() << "replSet minvalid=" << newMinValid["ts"]._opTime().toStringLong() << rsLog;
                         setMinValid(newMinValid);
                     }
                 }

src/mongo/db/repl/rs_sync.cpp

@@ -50,7 +50,7 @@ namespace replset {
     /* apply the log op that is in param o
        @return bool success (true) or failure (false)
     */
-    bool SyncTail::syncApply(const BSONObj &op) {
+    bool SyncTail::syncApply(const BSONObj &op, bool convertUpdateToUpsert) {
         const char *ns = op.getStringField("ns");
         verify(ns);
 
@@ -79,7 +79,9 @@ namespace replset {
 
         Client::Context ctx(ns, dbpath, false);
         ctx.getClient()->curop()->reset();
-        bool ok = !applyOperation_inlock(op);
+        // For non-initial-sync, we convert updates to upserts
+        // to suppress errors when replaying oplog entries.
+        bool ok = !applyOperation_inlock(op, true, convertUpdateToUpsert);
         getDur().commitIfNeeded();
 
         return ok;
@@ -111,7 +113,7 @@ namespace replset {
              it != ops.end();
              ++it) {
             try {
-                fassert(16359, st->syncApply(*it));
+                fassert(16359, st->syncApply(*it, true));
             } catch (DBException& e) {
                 error() << "writer worker caught exception: " << e.what() 
                         << " on: " << it->toString() << endl;
@@ -275,7 +277,7 @@ namespace replset {
         while( ts < minValid ) {
             OpQueue ops;
 
-            while (ops.getSize() < replBatchSizeBytes) {
+            while (ops.getSize() < replBatchLimitBytes) {
                 if (tryPopAndWaitForMore(&ops)) {
                     break;
                 }
@@ -310,20 +312,30 @@ namespace replset {
     void SyncTail::oplogApplication() {
         while( 1 ) {
             OpQueue ops;
-            time_t lastTimeChecked = time(0);
 
             verify( !Lock::isLocked() );
 
+            Timer batchTimer;
+            int lastTimeChecked = 0;
+
             // always fetch a few ops first
-            
             // tryPopAndWaitForMore returns true when we need to end a batch early
             while (!tryPopAndWaitForMore(&ops) && 
-                   (ops.getSize() < replBatchSizeBytes)) {
+                   (ops.getSize() < replBatchLimitBytes)) {
 
                 if (theReplSet->isPrimary()) {
                     return;
                 }
-                time_t now = time(0);
+
+                int now = batchTimer.seconds();
+
+                // apply replication batch limits
+                if (!ops.empty()) {
+                    if (now > replBatchLimitSeconds)
+                        break;
+                    if (ops.getDeque().size() > replBatchLimitOperations)
+                        break;
+                }
                 // occasionally check some things
                 if (ops.empty() || now > lastTimeChecked) {
                     lastTimeChecked = now;
@@ -347,17 +359,30 @@ namespace replset {
                         return;
                     }
                 }
+
+                const int slaveDelaySecs = theReplSet->myConfig().slaveDelay;
+                if (!ops.empty() && slaveDelaySecs > 0) {
+                    const BSONObj& lastOp = ops.getDeque().back();
+                    const unsigned int opTimestampSecs = lastOp["ts"]._opTime().getSecs();
+
+                    // Stop the batch as the lastOp is too new to be applied. If we continue
+                    // on, we can get ops that are way ahead of the delay and this will
+                    // make this thread sleep longer when handleSlaveDelay is called
+                    // and apply ops much sooner than we like.
+                    if (opTimestampSecs > static_cast<unsigned int>(time(0) - slaveDelaySecs)) {
+                        break;
+                    }
+                }
             }
+
             const BSONObj& lastOp = ops.getDeque().back();
             handleSlaveDelay(lastOp);
 
             // Set minValid to the last op to be applied in this next batch.
             // This will cause this node to go into RECOVERING state
             // if we should crash and restart before updating the oplog
-            { 
-                Client::WriteContext cx( "local" );
-                Helpers::putSingleton("local.replset.minvalid", lastOp);
-            }
+            theReplSet->setMinValid(lastOp);
+
             multiApply(ops.getDeque(), multiSyncApply);
 
             applyOpsToOplog(&ops.getDeque());

src/mongo/db/repl/rs_sync.h

@@ -20,6 +20,7 @@
 #include <vector>
 
 #include "mongo/db/client.h"
+#include "mongo/db/dur.h"
 #include "mongo/db/jsobj.h"
 #include "mongo/db/oplog.h"
 #include "mongo/util/concurrency/thread_pool.h"
@@ -37,7 +38,7 @@ namespace replset {
     public:
         SyncTail(BackgroundSyncInterface *q);
         virtual ~SyncTail();
-        virtual bool syncApply(const BSONObj &o);
+        virtual bool syncApply(const BSONObj &o, bool convertUpdateToUpsert = false);
         void oplogApplication();
         bool peek(BSONObj* obj);
 
@@ -69,7 +70,11 @@ namespace replset {
         void applyOpsToOplog(std::deque<BSONObj>* ops);
 
     protected:
-        static const unsigned int replBatchSizeBytes = 1024 * 1024 * 256 ;
+        // Cap the batches using the limit on journal commits.
+        // This works out to be 100 MB (64 bit) or 50 MB (32 bit)
+        static const unsigned int replBatchLimitBytes = dur::UncommittedBytesLimit;
+        static const int replBatchLimitSeconds = 1;
+        static const unsigned int replBatchLimitOperations = 5000;
 
         // Prefetch and write a deque of operations, using the supplied function.
         // Initial Sync and Sync Tail each use a different function.

src/mongo/db/security_commands.cpp

@@ -88,8 +88,8 @@ namespace mongo {
 
         {
             bool reject = false;
-            nonce64 *ln = lastNonce.release();
-            if ( ln == 0 ) {
+            scoped_ptr<nonce64> ln(lastNonce.release());
+            if ( !ln ) {
                 reject = true;
                 log(1) << "auth: no lastNonce" << endl;
             }

src/mongo/dbtests/documentsourcetests.cpp

@@ -31,6 +31,12 @@ namespace DocumentSourceTests {
     static const char* const ns = "unittests.documentsourcetests";
     static DBDirectClient client;
 
+    BSONObj toBson( const intrusive_ptr<DocumentSource>& source ) {
+        BSONArrayBuilder bab;
+        source->addToBsonArray( &bab );
+        return bab.arr()[ 0 ].Obj().getOwned();
+    }
+
     class CollectionBase {
     public:
         ~CollectionBase() {
@@ -315,14 +321,16 @@ namespace DocumentSourceTests {
 
         class Base : public DocumentSourceCursor::Base {
         protected:
-            void createGroup( const BSONObj &spec ) {
+            void createGroup( const BSONObj &spec, bool inShard = false ) {
                 BSONObj namedSpec = BSON( "$group" << spec );
                 BSONElement specElement = namedSpec.firstElement();
-                _group = DocumentSourceGroup::createFromBson( &specElement, ctx() );
-                BSONArrayBuilder bab;
-                _group->addToBsonArray( &bab );
-                // The $group spec round trips.
-                ASSERT_EQUALS( namedSpec, bab.arr()[ 0 ].Obj().getOwned() );
+                intrusive_ptr<ExpressionContext> expressionContext =
+                        ExpressionContext::create( &InterruptStatusMongod::status );
+                if ( inShard ) {
+                    expressionContext->setInShard( true );
+                }
+                _group = DocumentSourceGroup::createFromBson( &specElement, expressionContext );
+                assertRoundTrips( _group );
                 _group->setSource( source() );
             }
             DocumentSource* group() { return _group.get(); }
@@ -336,6 +344,16 @@ namespace DocumentSourceTests {
                 ASSERT( !source->getCurrent() );
             }
         private:
+            /** Check that the group's spec round trips. */
+            void assertRoundTrips( const intrusive_ptr<DocumentSource>& group ) {
+                // We don't check against the spec that generated 'group' originally, because
+                // $const operators may be introduced in the first serialization.
+                BSONObj spec = toBson( group );
+                BSONElement specElement = spec.firstElement();
+                intrusive_ptr<DocumentSource> generated =
+                        DocumentSourceGroup::createFromBson( &specElement, ctx() );
+                ASSERT_EQUALS( spec, toBson( generated ) );
+            }
             intrusive_ptr<DocumentSource> _group;
         };
 
@@ -540,6 +558,8 @@ namespace DocumentSourceTests {
                 intrusive_ptr<DocumentSource> sink = group();
                 if ( sharded ) {
                     sink = createMerger();
+                    // Serialize and re-parse the shard stage.
+                    createGroup( toBson( group() )[ "$group" ].Obj(), true );
                     sink->setSource( group() );
                 }
 
@@ -562,18 +582,10 @@ namespace DocumentSourceTests {
                 // case only one shard is in use.
                 SplittableDocumentSource *splittable =
                         dynamic_cast<SplittableDocumentSource*>( group() );
+                ASSERT( splittable );
                 intrusive_ptr<DocumentSource> routerSource = splittable->getRouterSource();
                 ASSERT_NOT_EQUALS( group(), routerSource.get() );
-                BSONArrayBuilder bab;
-                routerSource->addToBsonArray( &bab );
-                BSONObj routerSourceSpec = bab.arr()[ 0 ].Obj().getOwned();
-                BSONElement routerSourceSpecElement = routerSourceSpec.firstElement();
-                intrusive_ptr<ExpressionContext> routerContext =
-                        ExpressionContext::create( &InterruptStatusMongod::status );
-                routerContext->setDoingMerge(true);
-                routerContext->setInShard(false);
-                return DocumentSourceGroup::createFromBson( &routerSourceSpecElement,
-                                                            routerContext );
+                return routerSource;
             }
             void checkResultSet( const intrusive_ptr<DocumentSource> &sink ) {
                 // Load the results from the DocumentSourceGroup and sort them by _id.
@@ -800,6 +812,36 @@ namespace DocumentSourceTests {
             }
         };
 
+        /**
+         * A string constant (not a field path) as an _id expression and passed to an accumulator.
+         * SERVER-6766
+         */
+        class StringConstantIdAndAccumulatorExpressions : public CheckResultsBase {
+            void populateData() { client.insert( ns, BSONObj() ); }
+            BSONObj groupSpec() {
+                return fromjson( "{_id:{$const:'$_id...'},a:{$push:{$const:'$a...'}}}" );
+            }
+            string expectedResultSetString() { return "[{_id:'$_id...',a:['$a...']}]"; }
+        };
+
+        /** An array constant passed to an accumulator. */
+        class ArrayConstantAccumulatorExpression : public CheckResultsBase {
+        public:
+            void run() {
+                // A parse exception is thrown when a raw array is provided to an accumulator.
+                ASSERT_THROWS( createGroup( fromjson( "{_id:1,a:{$push:[4,5,6]}}" ) ),
+                               UserException );
+                // Run standard base tests.
+                CheckResultsBase::run();
+            }
+            void populateData() { client.insert( ns, BSONObj() ); }
+            BSONObj groupSpec() {
+                // An array can be specified using $const.
+                return fromjson( "{_id:[1,2,3],a:{$push:{$const:[4,5,6]}}}" );
+            }
+            string expectedResultSetString() { return "[{_id:[1,2,3],a:[[4,5,6]]}]"; }
+        };
+
     } // namespace DocumentSourceGroup
 
     namespace DocumentSourceProject {
@@ -1694,6 +1736,8 @@ namespace DocumentSourceTests {
             add<DocumentSourceGroup::UndefinedAccumulatorValue>();
             add<DocumentSourceGroup::RouterMerger>();
             add<DocumentSourceGroup::Dependencies>();
+            add<DocumentSourceGroup::StringConstantIdAndAccumulatorExpressions>();
+            add<DocumentSourceGroup::ArrayConstantAccumulatorExpression>();
 
             add<DocumentSourceProject::EofInit>();
             add<DocumentSourceProject::AdvanceInit>();

src/mongo/dbtests/expressiontests.cpp

@@ -39,6 +39,13 @@ namespace ExpressionTests {
         return bob.obj();
     }
 
+    /** Convert Expression to BSON. */
+    static BSONObj expressionToBson( const intrusive_ptr<Expression>& expression ) {
+        BSONObjBuilder bob;
+        expression->addToBsonObj( &bob, "", true );
+        return bob.obj().firstElement().embeddedObject().getOwned();
+    }
+
     /** Create a Document from a BSONObj. */
     intrusive_ptr<Document> fromBson( BSONObj obj ) {
         return Document::createFromBsonObj( &obj );
@@ -81,10 +88,364 @@ namespace ExpressionTests {
             }
         };
 
+        /** Output to BSONObj. */
+        class AddToBsonObj {
+        public:
+            void run() {
+                intrusive_ptr<Expression> expression = ExpressionCoerceToBool::create(
+                        ExpressionFieldPath::create("foo"));
+
+                // serialized as $and because CoerceToBool isn't an ExpressionNary
+                assertBinaryEqual(fromjson("{field:{$and:['$foo']}}"), toBsonObj(expression));
+            }
+        private:
+            static BSONObj toBsonObj(const intrusive_ptr<Expression>& expression) {
+
+                BSONObjBuilder bob;
+                expression->addToBsonObj(&bob, "field", false);
+                return bob.obj();
+            }
+        };
+
+        /** Output to BSONArray. */
+        class AddToBsonArray {
+        public:
+            void run() {
+                intrusive_ptr<Expression> expression = ExpressionCoerceToBool::create(
+                        ExpressionFieldPath::create("foo"));
+
+                // serialized as $and because CoerceToBool isn't an ExpressionNary
+                assertBinaryEqual(BSON_ARRAY(fromjson("{$and:['$foo']}")), toBsonArray(expression));
+            }
+        private:
+            static BSONArray toBsonArray(const intrusive_ptr<Expression>& expression) {
+                BSONArrayBuilder bab;
+                expression->addToBsonArray(&bab);
+                return bab.arr();
+            }
+        };
+
+
         // TODO Test optimize(), difficult because a CoerceToBool cannot be output as BSON.
         
     } // namespace CoerceToBool
 
+    namespace Compare {
+
+        class OptimizeBase {
+        public:
+            virtual ~OptimizeBase() {
+            }
+            void run() {
+                BSONObj specObject = BSON( "" << spec() );
+                BSONElement specElement = specObject.firstElement();
+                intrusive_ptr<Expression> expression = Expression::parseOperand( &specElement );
+                intrusive_ptr<Expression> optimized = expression->optimize();
+                ASSERT_EQUALS( expectedFieldRange(),
+                               (bool)dynamic_pointer_cast<ExpressionFieldRange>( optimized ) );
+                ASSERT_EQUALS( expectedOptimized(), expressionToBson( optimized ) );
+            }
+        protected:
+            virtual BSONObj spec() = 0;
+            virtual BSONObj expectedOptimized() = 0;
+            virtual bool expectedFieldRange() = 0;
+        };
+        
+        class FieldRangeOptimize : public OptimizeBase {
+            BSONObj expectedOptimized() { return spec(); }
+            bool expectedFieldRange() { return true; }
+        };
+        
+        class NoOptimize : public OptimizeBase {
+            BSONObj expectedOptimized() { return spec(); }
+            bool expectedFieldRange() { return false; }
+        };
+
+        /** Check expected result for expressions depending on constants. */
+        class ExpectedResultBase : public OptimizeBase {
+        public:
+            void run() {
+                OptimizeBase::run();
+                BSONObj specObject = BSON( "" << spec() );
+                BSONElement specElement = specObject.firstElement();
+                intrusive_ptr<Expression> expression = Expression::parseOperand( &specElement );
+                // Check expression spec round trip.
+                ASSERT_EQUALS( spec(), expressionToBson( expression ) );
+                // Check evaluation result.
+                ASSERT_EQUALS( expectedResult(),
+                               toBson( expression->evaluate( Document::create() ) ) );
+                // Check that the result is the same after optimizing.
+                intrusive_ptr<Expression> optimized = expression->optimize();
+                ASSERT_EQUALS( expectedResult(),
+                               toBson( optimized->evaluate( Document::create() ) ) );
+            }
+        protected:
+            virtual BSONObj spec() = 0;
+            virtual BSONObj expectedResult() = 0;
+        private:
+            virtual BSONObj expectedOptimized() {
+                return BSON( "$const" << expectedResult().firstElement() );
+            }
+            virtual bool expectedFieldRange() { return false; }
+        };
+
+        class ExpectedTrue : public ExpectedResultBase {
+            BSONObj expectedResult() { return BSON( "" << true ); }            
+        };
+        
+        class ExpectedFalse : public ExpectedResultBase {
+            BSONObj expectedResult() { return BSON( "" << false ); }            
+        };
+        
+        class ParseError {
+        public:
+            virtual ~ParseError() {
+            }
+            void run() {
+                BSONObj specObject = BSON( "" << spec() );
+                BSONElement specElement = specObject.firstElement();
+                ASSERT_THROWS( Expression::parseOperand( &specElement ), UserException );
+            }
+        protected:
+            virtual BSONObj spec() = 0;
+        };
+
+        /** $eq with first < second. */
+        class EqLt : public ExpectedFalse {
+            BSONObj spec() { return BSON( "$eq" << BSON_ARRAY( 1 << 2 ) ); }
+        };
+
+        /** $eq with first == second. */
+        class EqEq : public ExpectedTrue {
+            BSONObj spec() { return BSON( "$eq" << BSON_ARRAY( 1 << 1 ) ); }
+        };
+        
+        /** $eq with first > second. */
+        class EqGt : public ExpectedFalse {
+            BSONObj spec() { return BSON( "$eq" << BSON_ARRAY( 1 << 0 ) ); }
+        };
+        
+        /** $ne with first < second. */
+        class NeLt : public ExpectedTrue {
+            BSONObj spec() { return BSON( "$ne" << BSON_ARRAY( 1 << 2 ) ); }
+        };
+        
+        /** $ne with first == second. */
+        class NeEq : public ExpectedFalse {
+            BSONObj spec() { return BSON( "$ne" << BSON_ARRAY( 1 << 1 ) ); }
+        };
+        
+        /** $ne with first > second. */
+        class NeGt : public ExpectedTrue {
+            BSONObj spec() { return BSON( "$ne" << BSON_ARRAY( 1 << 0 ) ); }
+        };
+        
+        /** $gt with first < second. */
+        class GtLt : public ExpectedFalse {
+            BSONObj spec() { return BSON( "$gt" << BSON_ARRAY( 1 << 2 ) ); }
+        };
+        
+        /** $gt with first == second. */
+        class GtEq : public ExpectedFalse {
+            BSONObj spec() { return BSON( "$gt" << BSON_ARRAY( 1 << 1 ) ); }
+        };
+        
+        /** $gt with first > second. */
+        class GtGt : public ExpectedTrue {
+            BSONObj spec() { return BSON( "$gt" << BSON_ARRAY( 1 << 0 ) ); }
+        };
+        
+        /** $gte with first < second. */
+        class GteLt : public ExpectedFalse {
+            BSONObj spec() { return BSON( "$gte" << BSON_ARRAY( 1 << 2 ) ); }
+        };
+        
+        /** $gte with first == second. */
+        class GteEq : public ExpectedTrue {
+            BSONObj spec() { return BSON( "$gte" << BSON_ARRAY( 1 << 1 ) ); }
+        };
+        
+        /** $gte with first > second. */
+        class GteGt : public ExpectedTrue {
+            BSONObj spec() { return BSON( "$gte" << BSON_ARRAY( 1 << 0 ) ); }
+        };
+        
+        /** $lt with first < second. */
+        class LtLt : public ExpectedTrue {
+            BSONObj spec() { return BSON( "$lt" << BSON_ARRAY( 1 << 2 ) ); }
+        };
+        
+        /** $lt with first == second. */
+        class LtEq : public ExpectedFalse {
+            BSONObj spec() { return BSON( "$lt" << BSON_ARRAY( 1 << 1 ) ); }
+        };
+        
+        /** $lt with first > second. */
+        class LtGt : public ExpectedFalse {
+            BSONObj spec() { return BSON( "$lt" << BSON_ARRAY( 1 << 0 ) ); }
+        };
+        
+        /** $lte with first < second. */
+        class LteLt : public ExpectedTrue {
+            BSONObj spec() { return BSON( "$lte" << BSON_ARRAY( 1 << 2 ) ); }
+        };
+        
+        /** $lte with first == second. */
+        class LteEq : public ExpectedTrue {
+            BSONObj spec() { return BSON( "$lte" << BSON_ARRAY( 1 << 1 ) ); }
+        };
+        
+        /** $lte with first > second. */
+        class LteGt : public ExpectedFalse {
+            BSONObj spec() { return BSON( "$lte" << BSON_ARRAY( 1 << 0 ) ); }
+        };
+        
+        /** $cmp with first < second. */
+        class CmpLt : public ExpectedResultBase {
+            BSONObj spec() { return BSON( "$cmp" << BSON_ARRAY( 1 << 2 ) ); }
+            BSONObj expectedResult() { return BSON( "" << -1 ); }
+        };
+        
+        /** $cmp with first == second. */
+        class CmpEq : public ExpectedResultBase {
+            BSONObj spec() { return BSON( "$cmp" << BSON_ARRAY( 1 << 1 ) ); }
+            BSONObj expectedResult() { return BSON( "" << 0 ); }
+        };
+        
+        /** $cmp with first > second. */
+        class CmpGt : public ExpectedResultBase {
+            BSONObj spec() { return BSON( "$cmp" << BSON_ARRAY( 1 << 0 ) ); }
+            BSONObj expectedResult() { return BSON( "" << 1 ); }
+        };
+
+        /** $cmp results are bracketed to an absolute value of 1. */
+        class CmpBracketed : public ExpectedResultBase {
+            BSONObj spec() { return BSON( "$cmp" << BSON_ARRAY( "z" << "a" ) ); }
+            BSONObj expectedResult() { return BSON( "" << 1 ); }
+        };
+
+        /** Zero operands provided. */
+        class ZeroOperands : public ParseError {
+            BSONObj spec() { return BSON( "$ne" << BSONArray() ); }
+        };
+
+        /** One operand provided. */
+        class OneOperand : public ParseError {
+            BSONObj spec() { return BSON( "$eq" << BSON_ARRAY( 1 ) ); }
+        };
+        
+        /** Three operands provided. */
+        class ThreeOperands : public ParseError {
+            BSONObj spec() { return BSON( "$gt" << BSON_ARRAY( 2 << 3 << 4 ) ); }
+        };
+        
+        /** Incompatible types cannot be compared. */
+        class IncompatibleTypes {
+        public:
+            void run() {
+                BSONObj specObject = BSON( "" << BSON( "$ne" << BSON_ARRAY( "a" << 1 ) ) );
+                BSONElement specElement = specObject.firstElement();
+                intrusive_ptr<Expression> expression = Expression::parseOperand( &specElement );
+                ASSERT_THROWS( expression->evaluate( Document::create() ), UserException );
+            }
+        };
+
+        /**
+         * An expression depending on constants is optimized to a constant via
+         * ExpressionNary::optimize().
+         */
+        class OptimizeConstants : public OptimizeBase {
+            BSONObj spec() { return BSON( "$eq" << BSON_ARRAY( 1 << 1 ) ); }
+            BSONObj expectedOptimized() { return BSON( "$const" << true ); }
+            bool expectedFieldRange() { return false; }
+        };
+
+        /** $cmp is not optimized. */
+        class NoOptimizeCmp : public NoOptimize {
+            BSONObj spec() { return BSON( "$cmp" << BSON_ARRAY( 1 << "$a" ) ); }
+        };
+        
+        /** $ne is not optimized. */
+        class NoOptimizeNe : public NoOptimize {
+            BSONObj spec() { return BSON( "$ne" << BSON_ARRAY( 1 << "$a" ) ); }
+        };
+        
+        /** No optimization is performend without a constant. */
+        class NoOptimizeNoConstant : public NoOptimize {
+            BSONObj spec() { return BSON( "$ne" << BSON_ARRAY( "$a" << "$b" ) ); }
+        };
+        
+        /** No optimization is performend without an immediate field path. */
+        class NoOptimizeWithoutFieldPath : public NoOptimize {
+            BSONObj spec() {
+                return BSON( "$eq" << BSON_ARRAY( BSON( "$and" << BSON_ARRAY( "$a" ) ) << 1 ) );
+            }
+        };
+        
+        /** No optimization is performend without an immediate field path. */
+        class NoOptimizeWithoutFieldPathReverse : public NoOptimize {
+            BSONObj spec() {
+                return BSON( "$eq" << BSON_ARRAY( 1 << BSON( "$and" << BSON_ARRAY( "$a" ) ) ) );
+            }
+        };
+        
+        /** An equality expression is optimized. */
+        class OptimizeEq : public FieldRangeOptimize {
+            BSONObj spec() { return BSON( "$eq" << BSON_ARRAY( "$a" << 1 ) ); }
+        };
+        
+        /** A reverse sense equality expression is optimized. */
+        class OptimizeEqReverse : public FieldRangeOptimize {
+            BSONObj spec() { return BSON( "$eq" << BSON_ARRAY( 1 << "$a" ) ); }
+            BSONObj expectedOptimized() { return BSON( "$eq" << BSON_ARRAY( "$a" << 1 ) ); }
+        };
+        
+        /** A $lt expression is optimized. */
+        class OptimizeLt : public FieldRangeOptimize {
+            BSONObj spec() { return BSON( "$lt" << BSON_ARRAY( "$a" << 1 ) ); }
+        };
+        
+        /** A reverse sense $lt expression is optimized. */
+        class OptimizeLtReverse : public FieldRangeOptimize {
+            BSONObj spec() { return BSON( "$lt" << BSON_ARRAY( 1 << "$a" ) ); }
+            BSONObj expectedOptimized() { return BSON( "$gt" << BSON_ARRAY( "$a" << 1 ) ); }
+        };
+        
+        /** A $lte expression is optimized. */
+        class OptimizeLte : public FieldRangeOptimize {
+            BSONObj spec() { return BSON( "$lte" << BSON_ARRAY( "$b" << 2 ) ); }
+        };
+        
+        /** A reverse sense $lte expression is optimized. */
+        class OptimizeLteReverse : public FieldRangeOptimize {
+            BSONObj spec() { return BSON( "$lte" << BSON_ARRAY( 2 << "$b" ) ); }
+            BSONObj expectedOptimized() { return BSON( "$gte" << BSON_ARRAY( "$b" << 2 ) ); }
+        };
+        
+        /** A $gt expression is optimized. */
+        class OptimizeGt : public FieldRangeOptimize {
+            BSONObj spec() { return BSON( "$gt" << BSON_ARRAY( "$b" << 2 ) ); }
+        };
+        
+        /** A reverse sense $gt expression is optimized. */
+        class OptimizeGtReverse : public FieldRangeOptimize {
+            BSONObj spec() { return BSON( "$gt" << BSON_ARRAY( 2 << "$b" ) ); }
+            BSONObj expectedOptimized() { return BSON( "$lt" << BSON_ARRAY( "$b" << 2 ) ); }
+        };
+        
+        /** A $gte expression is optimized. */
+        class OptimizeGte : public FieldRangeOptimize {
+            BSONObj spec() { return BSON( "$gte" << BSON_ARRAY( "$b" << 2 ) ); }
+        };
+        
+        /** A reverse sense $gte expression is optimized. */
+        class OptimizeGteReverse : public FieldRangeOptimize {
+            BSONObj spec() { return BSON( "$gte" << BSON_ARRAY( 2 << "$b" ) ); }
+            BSONObj expectedOptimized() { return BSON( "$lte" << BSON_ARRAY( "$b" << 2 ) ); }
+        };
+        
+    } // namespace Compare
+    
     namespace Constant {
 
         /** Create an ExpressionConstant from a Value. */
@@ -399,6 +760,51 @@ namespace ExpressionTests {
             add<CoerceToBool::EvaluateTrue>();
             add<CoerceToBool::EvaluateFalse>();
             add<CoerceToBool::Dependencies>();
+            add<CoerceToBool::AddToBsonObj>();
+            add<CoerceToBool::AddToBsonArray>();
+
+            add<Compare::EqLt>();
+            add<Compare::EqEq>();
+            add<Compare::EqGt>();
+            add<Compare::NeLt>();
+            add<Compare::NeEq>();
+            add<Compare::NeGt>();
+            add<Compare::GtLt>();
+            add<Compare::GtEq>();
+            add<Compare::GtGt>();
+            add<Compare::GteLt>();
+            add<Compare::GteEq>();
+            add<Compare::GteGt>();
+            add<Compare::LtLt>();
+            add<Compare::LtEq>();
+            add<Compare::LtGt>();
+            add<Compare::LteLt>();
+            add<Compare::LteEq>();
+            add<Compare::LteGt>();
+            add<Compare::CmpLt>();
+            add<Compare::CmpEq>();
+            add<Compare::CmpGt>();
+            add<Compare::CmpBracketed>();
+            add<Compare::ZeroOperands>();
+            add<Compare::OneOperand>();
+            add<Compare::ThreeOperands>();
+            add<Compare::IncompatibleTypes>();
+            add<Compare::OptimizeConstants>();
+            add<Compare::NoOptimizeCmp>();
+            add<Compare::NoOptimizeNe>();
+            add<Compare::NoOptimizeNoConstant>();
+            add<Compare::NoOptimizeWithoutFieldPath>();
+            add<Compare::NoOptimizeWithoutFieldPathReverse>();
+            add<Compare::OptimizeEq>();
+            add<Compare::OptimizeEqReverse>();
+            add<Compare::OptimizeLt>();
+            add<Compare::OptimizeLtReverse>();
+            add<Compare::OptimizeLte>();
+            add<Compare::OptimizeLteReverse>();
+            add<Compare::OptimizeGt>();
+            add<Compare::OptimizeGtReverse>();
+            add<Compare::OptimizeGte>();
+            add<Compare::OptimizeGteReverse>();
 
             add<Constant::Create>();
             add<Constant::CreateFromBsonElement>();

src/mongo/dbtests/replsettests.cpp

@@ -178,7 +178,7 @@ namespace ReplSetTests {
         bool retry;
 
         // instead of actually applying operations, we return success or failure
-        virtual bool syncApply(const BSONObj& o) {
+        virtual bool syncApply(const BSONObj& o, bool convertUpdateToUpsert) {
             step++;
 
             if ((failOnStep == FAIL_FIRST_APPLY && step == 1) ||

src/mongo/s/d_logic.cpp

@@ -100,7 +100,7 @@ namespace mongo {
             dbresponse->responseTo = m.header()->id;
             return true;
         }
-        
+
         uassert( 9517 , "writeback" , ( d.reservedField() & Reserved_FromWriteback ) == 0 );
 
         OID writebackID;
@@ -109,6 +109,13 @@ namespace mongo {
         const OID& clientID = ShardedConnectionInfo::get(false)->getID();
         massert( 10422 ,  "write with bad shard config and no server id!" , clientID.isSet() );
 
+        // We need to check this here, since otherwise we'll get errors wrapping the writeback -
+        // not just here, but also when returning as a command result.
+        // We choose 1/2 the overhead of the internal maximum so that we can still handle ops of
+        // 16MB exactly.
+        massert( 16437, "data size of operation is too large to queue for writeback",
+                 m.dataSize() < BSONObjMaxInternalSize - (8 * 1024));
+
         LOG(1) << "writeback queued for " << m.toString() << endl;
 
         BSONObjBuilder b;
@@ -123,11 +130,14 @@ namespace mongo {
         b.appendBinData( "msg" , m.header()->len , bdtCustom , (char*)(m.singleData()) );
         LOG(2) << "writing back msg with len: " << m.header()->len << " op: " << m.operation() << endl;
 
+        // Convert to new BSONObj here just to be safe
+        BSONObj wbObj = b.obj();
+
         // Don't register the writeback until immediately before we queue it -
         // after this line, mongos will wait for an hour if we don't queue correctly
         lastError.getSafe()->writeback( writebackID );
 
-        writeBackManager.queueWriteBack( clientID.str() , b.obj() );
+        writeBackManager.queueWriteBack( clientID.str() , wbObj );
 
         return true;
     }

src/mongo/s/d_migrate.cpp

@@ -437,9 +437,9 @@ namespace mongo {
                 errmsg = (string)"can't find index in storeCurrentLocs" + causedBy( errmsg );
                 return false;
             }
-
+            // Assume both min and max non-empty, append MinKey's to make them fit chosen index
             BSONObj min = Helpers::modifiedRangeBound( _min , idx->keyPattern() , -1 );
-            BSONObj max = Helpers::modifiedRangeBound( _max , idx->keyPattern() , 1 );
+            BSONObj max = Helpers::modifiedRangeBound( _max , idx->keyPattern() , -1 );
 
             BtreeCursor* btreeCursor = BtreeCursor::make( d , *idx , min , max , false , 1 );
             auto_ptr<ClientCursor> cc(
@@ -1678,10 +1678,16 @@ namespace mongo {
                         }
                     }
 
+                    // id object most likely has form { _id : ObjectId(...) }
+                    // infer from that correct index to use, e.g. { _id : 1 }
+                    BSONObj idIndexPattern;
+                    Helpers::toKeyFormat( id , idIndexPattern );
+
+                    // TODO: create a better interface to remove objects directly
                     Helpers::removeRange( ns ,
                                           id ,
                                           id,
-                                          findShardKeyIndexPattern_locked( ns , shardKeyPattern ), 
+                                          idIndexPattern ,
                                           true , /*maxInclusive*/
                                           false , /* secondaryThrottle */
                                           cmdLine.moveParanoia ? &rs : 0 , /*callback*/
@@ -1843,9 +1849,26 @@ namespace mongo {
             migrateStatus.from = cmdObj["from"].String();
             migrateStatus.min = cmdObj["min"].Obj().getOwned();
             migrateStatus.max = cmdObj["max"].Obj().getOwned();
-            migrateStatus.shardKeyPattern = cmdObj["shardKeyPattern"].Obj().getOwned();
             migrateStatus.secondaryThrottle = cmdObj["secondaryThrottle"].trueValue();
-            
+            if (cmdObj.hasField("shardKeyPattern")) {
+                migrateStatus.shardKeyPattern = cmdObj["shardKeyPattern"].Obj().getOwned();
+            } else {
+                // shardKeyPattern may not be provided if another shard is from pre 2.2
+                // In that case, assume the shard key pattern is the same as the range
+                // specifiers provided.
+                BSONObj keya , keyb;
+                Helpers::toKeyFormat( migrateStatus.min , keya );
+                Helpers::toKeyFormat( migrateStatus.max , keyb );
+                verify( keya == keyb );
+
+                warning() << "No shard key pattern provided by source shard for migration."
+                    " This is likely because the source shard is running a version prior to 2.2."
+                    " Falling back to assuming the shard key matches the pattern of the min and max"
+                    " chunk range specifiers.  Inferred shard key: " << keya << endl;
+
+                migrateStatus.shardKeyPattern = keya.getOwned();
+            }
+
             if ( migrateStatus.secondaryThrottle && ! anyReplEnabled() ) {
                 warning() << "secondaryThrottle asked for, but not replication" << endl;
                 migrateStatus.secondaryThrottle = false;

src/mongo/s/d_split.cpp

@@ -97,9 +97,15 @@ namespace mongo {
                 errmsg = "couldn't find valid index for shard key";
                 return false;
             }
-            //has the effect of extending empty min/max to match selected index
+            // extend min to get (min, MinKey, MinKey, ....)
             min = Helpers::modifiedRangeBound( min , idx->keyPattern() , -1 );
-            max = Helpers::modifiedRangeBound( max , idx->keyPattern() , 1 );
+            if  ( max.isEmpty() ) {
+                // if max not specified, make it (MaxKey, Maxkey, MaxKey...)
+                max = Helpers::modifiedRangeBound( max , idx->keyPattern() , 1 );
+            } else {
+                // otherwise make it (max,MinKey,MinKey...) so that bound is non-inclusive
+                max = Helpers::modifiedRangeBound( max , idx->keyPattern() , -1 );
+            }
 
             BtreeCursor * bc = BtreeCursor::make( d , d->idxNo(*idx) , *idx , min , max , false , 1 );
             shared_ptr<Cursor> c( bc );
@@ -196,17 +202,7 @@ namespace mongo {
             // If min and max are not provided use the "minKey" and "maxKey" for the sharding key pattern.
             BSONObj min = jsobj.getObjectField( "min" );
             BSONObj max = jsobj.getObjectField( "max" );
-            if ( min.isEmpty() && max.isEmpty() ) {
-                BSONObjBuilder minBuilder;
-                BSONObjBuilder maxBuilder;
-                BSONForEach(key, keyPattern) {
-                    minBuilder.appendMinKey( key.fieldName() );
-                    maxBuilder.appendMaxKey( key.fieldName() );
-                }
-                min = minBuilder.obj();
-                max = maxBuilder.obj();
-            }
-            else if ( min.isEmpty() || max.isEmpty() ) {
+            if ( min.isEmpty() != max.isEmpty() ){
                 errmsg = "either provide both min and max or leave both empty";
                 return false;
             }
@@ -241,8 +237,15 @@ namespace mongo {
                              keyPattern.clientReadable().toString();
                     return false;
                 }
+                // extend min to get (min, MinKey, MinKey, ....)
                 min = Helpers::modifiedRangeBound( min , idx->keyPattern() , -1 );
-                max = Helpers::modifiedRangeBound( max , idx->keyPattern() , 1 );
+                if  ( max.isEmpty() ) {
+                    // if max not specified, make it (MaxKey, Maxkey, MaxKey...)
+                    max = Helpers::modifiedRangeBound( max , idx->keyPattern() , 1 );
+                } else {
+                    // otherwise make it (max,MinKey,MinKey...) so that bound is non-inclusive
+                    max = Helpers::modifiedRangeBound( max , idx->keyPattern() , -1 );
+                }
                 
                 const long long recCount = d->stats.nrecords;
                 const long long dataSize = d->stats.datasize;
@@ -324,15 +327,13 @@ namespace mongo {
                 // at the end. If a key appears more times than entries allowed on a chunk, we issue a warning and
                 // split on the following key.
                 set<BSONObj> tooFrequentKeys;
-                splitKeys.push_back( c->currKey().getOwned() );
+                splitKeys.push_back( bc->prettyKey( c->currKey().getOwned() ).extractFields( keyPattern ) );
                 while ( 1 ) {
                     while ( cc->ok() ) {
                         currCount++;
-                        BSONObj currKey = c->currKey();
-                        
-                        DEV verify( currKey.woCompare( max ) <= 0 );
                         
                         if ( currCount > keyCount ) {
+                            BSONObj currKey = bc->prettyKey( c->currKey() ).extractFields(keyPattern);
                             // Do not use this split key if it is the same used in the previous split point.
                             if ( currKey.woCompare( splitKeys.back() ) == 0 ) {
                                 tooFrequentKeys.insert( currKey.getOwned() );
@@ -343,7 +344,7 @@ namespace mongo {
                                 currCount = 0;
                                 numChunks++;
                                 
-                                LOG(4) << "picked a split key: " << bc->prettyKey( currKey ) << endl;
+                                LOG(4) << "picked a split key: " << currKey << endl;
                             }
                             
                         }
@@ -393,12 +394,9 @@ namespace mongo {
                               << " bytes because of key " << bc->prettyKey( *it ) << endl;
                 }
                 
-                // Remove the sentinel at the beginning before returning and add fieldnames.
+                // Remove the sentinel at the beginning before returning
                 splitKeys.erase( splitKeys.begin() );
                 verify( c.get() );
-                for ( vector<BSONObj>::iterator it = splitKeys.begin(); it != splitKeys.end() ; ++it ) {
-                    *it = bc->prettyKey( *it );
-                }
                 
                 if ( timer.millis() > cmdLine.slowMS ) {
                     warning() << "Finding the split vector for " <<  ns << " over "<< keyPattern
@@ -751,16 +749,38 @@ namespace mongo {
 
             if (newChunks.size() == 2){
                 // If one of the chunks has only one object in it we should move it
-                static const BSONObj fields = BSON("_id" << 1 );
-                DBDirectClient conn;
                 for (int i=1; i >= 0 ; i--){ // high chunk more likely to have only one obj
-                    ChunkInfo chunk = newChunks[i];
-                    Query q = Query().minKey(chunk.min).maxKey(chunk.max);
-                    scoped_ptr<DBClientCursor> c (conn.query(ns, q, /*limit*/-2, 0, &fields));
-                    if (c && c->itcount() == 1) {
-                        result.append("shouldMigrate", BSON("min" << chunk.min << "max" << chunk.max));
+
+                    Client::ReadContext ctx( ns );
+                    NamespaceDetails *d = nsdetails( ns.c_str() );
+
+                    const IndexDetails *idx = d->findIndexByPrefix( keyPattern ,
+                                                                    true ); /* exclude multikeys */
+                    if ( idx == NULL ) {
                         break;
                     }
+
+                    ChunkInfo chunk = newChunks[i];
+                    BSONObj newmin = Helpers::modifiedRangeBound(chunk.min, idx->keyPattern(), -1);
+                    BSONObj newmax = Helpers::modifiedRangeBound(chunk.max , idx->keyPattern(), -1);
+
+                    scoped_ptr<BtreeCursor> bc( BtreeCursor::make( d ,
+                                                                   d->idxNo(*idx) ,
+                                                                   *idx ,
+                                                                   newmin , /* lower */
+                                                                   newmax , /* upper */
+                                                                   false , /* upper noninclusive */
+                                                                   1 ) ); /* direction */
+
+                    // check if exactly one document found
+                    if ( bc->ok() ) {
+                        bc->advance();
+                        if ( bc->eof() ) {
+                            result.append( "shouldMigrate",
+                                           BSON("min" << chunk.min << "max" << chunk.max) );
+                            break;
+                        }
+                    }
                 }
             }
 

src/mongo/s/shard_version.cpp

@@ -232,6 +232,8 @@ namespace mongo {
                << " version: " << version << " manager: " << manager.get()
                << endl;
 
+        const string versionableServerAddress(conn->getServerAddress());
+
         BSONObj result;
         if ( setShardVersion( *conn , ns , version , authoritative , result ) ) {
             // success!
@@ -246,7 +248,9 @@ namespace mongo {
             massert( 10428 ,  "need_authoritative set but in authoritative mode already" , ! authoritative );
 
         if ( ! authoritative ) {
-            checkShardVersion( conn , ns , refManager, 1 , tryNumber + 1 );
+            // use the original connection and get a fresh versionable connection
+            // since conn can be invalidated (or worse, freed) after the failure
+            checkShardVersion(conn_in, ns, refManager, 1, tryNumber + 1);
             return true;
         }
         
@@ -268,13 +272,15 @@ namespace mongo {
         const int maxNumTries = 7;
         if ( tryNumber < maxNumTries ) {
             LOG( tryNumber < ( maxNumTries / 2 ) ? 1 : 0 ) 
-                << "going to retry checkShardVersion host: " << conn->getServerAddress() << " " << result << endl;
+                << "going to retry checkShardVersion host: " << versionableServerAddress << " " << result << endl;
             sleepmillis( 10 * tryNumber );
-            checkShardVersion( conn , ns , refManager, true , tryNumber + 1 );
+            // use the original connection and get a fresh versionable connection
+            // since conn can be invalidated (or worse, freed) after the failure
+            checkShardVersion(conn_in, ns, refManager, true, tryNumber + 1);
             return true;
         }
         
-        string errmsg = str::stream() << "setShardVersion failed host: " << conn->getServerAddress() << " " << result;
+        string errmsg = str::stream() << "setShardVersion failed host: " << versionableServerAddress << " " << result;
         log() << "     " << errmsg << endl;
         massert( 10429 , errmsg , 0 );
         return true;

src/mongo/s/strategy_shard.cpp

@@ -125,11 +125,20 @@ namespace mongo {
                         startFrom, hasMore ? cc->getId() : 0 );
             }
             else{
+                // Remote cursors are stored remotely, we shouldn't need this around.
+                // TODO: we should probably just make cursor an auto_ptr
+                scoped_ptr<ParallelSortClusteredCursor> cursorDeleter( cursor );
+
                 // TODO:  Better merge this logic.  We potentially can now use the same cursor logic for everything.
                 ShardPtr primary = cursor->getPrimary();
                 verify( primary.get() );
                 DBClientCursorPtr shardCursor = cursor->getShardCursor( *primary );
+
+                // Implicitly stores the cursor in the cache
                 r.reply( *(shardCursor->getMessage()) , shardCursor->originalHost() );
+
+                // We don't want to kill the cursor remotely if there's still data left
+                shardCursor->decouple();
             }
         }
 

src/mongo/shell/utils_sh.js

@@ -341,8 +341,8 @@ sh.removeShardTag = function( shard, tag ) {
 
 sh.addTagRange = function( ns, min, max, tag ) {
     var config = db.getSisterDB( "config" );
-    config.tags.update( { ns : ns , min : min } , 
-                        { ns : ns , min : min , max : max , tag : tag } , 
-                        true );
+    config.tags.update( {_id: { ns : ns , min : min } } , 
+            {_id: { ns : ns , min : min }, ns : ns , min : min , max : max , tag : tag } , 
+            true );
     sh._checkLastError( config );    
 }

src/mongo/util/assert_util.cpp

@@ -147,7 +147,7 @@ namespace mongo {
 
     NOINLINE_DECL void msgasserted(int msgid, const char *msg) {
         assertionCount.condrollover( ++assertionCount.warning );
-        tlog() << "Assertion: " << msgid << ":" << msg << endl;
+        log() << "Assertion: " << msgid << ":" << msg << endl;
         setLastError(msgid,msg && *msg ? msg : "massert failure");
         //breakpoint();
         logContext();

src/mongo/util/processinfo_darwin.cpp

@@ -114,12 +114,25 @@ namespace mongo {
     typedef long long NumberVal;
     template <typename Variant>
     Variant getSysctlByName( const char * sysctlName ) {
-        char value[256];
-        size_t len = sizeof(value);
-        if ( sysctlbyname(sysctlName, &value, &len, NULL, 0) < 0 ) {
-            log() << "Unable to resolve sysctl " << sysctlName << " (string) " << endl;
+        string value;
+        size_t len;
+        int status;
+        // NB: sysctlbyname is called once to determine the buffer length, and once to copy
+        //     the sysctl value.  Retry if the buffer length grows between calls.
+        do {
+            status = sysctlbyname(sysctlName, NULL, &len, NULL, 0);
+            if (status == -1)
+                break;
+            value.resize(len);
+            status = sysctlbyname(sysctlName, &*value.begin(), &len, NULL, 0);
+        } while (status == -1 && errno == ENOMEM);
+        if (status == -1) {
+            // unrecoverable error from sysctlbyname
+            log() << sysctlName << " unavailable" << endl;
+            return "";
         }
-        return string(value, len - 1);        
+        value.resize(len);
+        return value;
     }
 
     /**

src/mongo/util/stacktrace.cpp

@@ -9,10 +9,13 @@
 #include <vector>
 
 #ifdef _WIN32
+#include <sstream>
 #include <stdio.h>
 #include <boost/smart_ptr/scoped_array.hpp>
 #include "mongo/platform/windows_basic.h"
 #include <DbgHelp.h>
+#include "mongo/util/assert_util.h"
+#include "mongo/util/log.h"
 #endif
 
 #ifdef MONGO_HAVE_EXECINFO_BACKTRACE
@@ -169,7 +172,7 @@ namespace mongo {
         BOOL ret = SymInitialize( process, NULL, TRUE );
         if ( ret == FALSE ) {
             DWORD dosError = GetLastError();
-            os << "Stack trace failed, SymInitialize failed with error " <<
+            log() << "Stack trace failed, SymInitialize failed with error " <<
                     std::dec << dosError << std::endl;
             return;
         }
@@ -244,20 +247,21 @@ namespace mongo {
         ++sourceWidth;
         size_t frameCount = traceList.size();
         for ( size_t i = 0; i < frameCount; ++i ) {
-            os << traceList[i].moduleName << " ";
+            std::stringstream ss;
+            ss << traceList[i].moduleName << " ";
             size_t width = traceList[i].moduleName.length();
             while ( width < moduleWidth ) {
-                os << " ";
+                ss << " ";
                 ++width;
             }
-            os << traceList[i].sourceAndLine << " ";
+            ss << traceList[i].sourceAndLine << " ";
             width = traceList[i].sourceAndLine.length();
             while ( width < sourceWidth ) {
-                os << " ";
+                ss << " ";
                 ++width;
             }
-            os << traceList[i].symbolAndOffset;
-            os << std::endl;
+            ss << traceList[i].symbolAndOffset;
+            log() << ss.str() << std::endl;
         }
     }
 }

src/mongo/util/version.cpp

@@ -43,7 +43,7 @@ namespace mongo {
      *      1.2.3-rc4-pre-
      * If you really need to do something else you'll need to fix _versionArray()
      */
-    const char versionString[] = "2.2.0-rc1-pre-";
+    const char versionString[] = "2.2.1-rc1-pre-";
 
     // See unit test for example outputs
     static BSONArray _versionArray(const char* version){

src/third_party/gperftools-2.0/src/sampler.cc

@@ -56,6 +56,8 @@ DEFINE_int64(tcmalloc_sample_parameter,
 
 namespace tcmalloc {
 
+#if !defined(NO_TCMALLOC_SAMPLES) // 10gen
+
 // Statics for Sampler
 double Sampler::log_table_[1<<kFastlogNumBits];
 
@@ -126,5 +128,6 @@ size_t Sampler::PickNextSamplingPoint() {
   return static_cast<size_t>(min(0.0, (FastLog2(q) - 26)) * (-log(2.0)
                              * FLAGS_tcmalloc_sample_parameter) + 1);
 }
+#endif // !defined(NO_TCMALLOC_SAMPLES) // 10gen
 
 }  // namespace tcmalloc

src/third_party/gperftools-2.0/src/sampler.h

@@ -99,6 +99,27 @@ namespace tcmalloc {
 // only result in a single call to PickNextSamplingPoint.
 //-------------------------------------------------------------------
 
+#ifdef NO_TCMALLOC_SAMPLES // 10gen
+
+// Dummy class with same public interface as below
+class PERFTOOLS_DLL_DECL Sampler {
+ public:
+  void Init(uint32_t seed) {}
+  bool SampleAllocation(size_t k) { return false; }
+  static void InitStatics() {}
+  int GetSamplePeriod() { return 0; }
+
+# if 0
+  // "public" functions only used in the Sampler and tests of the Sampler
+  void Cleanup();
+  size_t PickNextSamplingPoint();
+  static uint64_t NextRandom(uint64_t rnd_);
+  static double FastLog2(const double & d);
+  static void PopulateFastLog2Table();
+# endif
+};
+
+#else
 class PERFTOOLS_DLL_DECL Sampler {
  public:
   // Initialize this sampler.
@@ -173,6 +194,7 @@ inline double Sampler::FastLog2(const double & d) {
   const int32_t exponent = ((x_high >> 20) & 0x7FF) - 1023;
   return exponent + log_table_[y];
 }
+#endif // NO_TCMALLOC_SAMPLES // 10gen
 
 }  // namespace tcmalloc