BUMP 2.2.0

During initial sync, you must apply updates as updates and upserts as upserts,
since it is possible that an in-place update will move a document backwards and
cause the cloner to miss it.  We need to detect this situation on the secondary.

However, during normal replication, we need all updates to be upserts.
Consider the situation where the oplog is replayed (possibly due to a server
crash).  If an update and subsequent delete are played and then replyed,
the reply will hit an error on the update because the document will not exist.
Converting the update to an upsert will allow oplog application to proceed;
while it may result in an incomplete document, this document will be deleted
soon afterward.

debian/changelog

@@ -1,3 +1,9 @@
+mongodb (2.2.0) unstable; urgency=low
+
+  * see http://docs.mongodb.org/manual/release-notes/2.2/
+
+ -- Richard Kreuter <richard@10gen.com>  Wed, 29 Aug 2012 16:56:28 -0500
+
 mongodb (2.1.2) unstable; urgency=low
 
   * see http://jira.mongodb.org/browse/SERVER/fixforversion/10894

distsrc/THIRD-PARTY-NOTICES

@@ -228,6 +228,39 @@ ghost@aladdin.com
  using BMDiff and then compressing the output of BMDiff with
  Snappy.
 
+6) License notice for Google Perftools (TCMalloc utility)
+---------------------------------
+New BSD License
 
+Copyright (c) 1998-2006, Google Inc.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or 
+without modification, are permitted provided that the following 
+conditions are met:
+
+    * Redistributions of source code must retain the above 
+      copyright notice, this list of conditions and the following 
+      disclaimer.
+    * Redistributions in binary form must reproduce the above
+      copyright notice, this list of conditions and the following 
+      disclaimer in the documentation and/or other materials provided 
+      with the distribution.
+    * Neither the name of Google Inc. nor the names of its
+      contributors may be used to endorse or promote products 
+      derived from this software without specific prior written 
+      permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 End

doxygenConfig

@@ -3,7 +3,7 @@
 #---------------------------------------------------------------------------
 DOXYFILE_ENCODING      = UTF-8
 PROJECT_NAME           = MongoDB
-PROJECT_NUMBER         = 2.2.0-rc1-pre-
+PROJECT_NUMBER         = 2.2.0
 OUTPUT_DIRECTORY       = docs/doxygen
 CREATE_SUBDIRS         = NO
 OUTPUT_LANGUAGE        = English

jstests/aggregation/bugs/server6779.js

@@ -0,0 +1,18 @@
+// server 6779: serializing ExpressionCoerceToBool
+// This test only fails in debug mode with the bug since that tests round-tripping
+function test(op, val) {
+    t = db.server6779;
+    t.drop();
+
+    t.insert({a:true});
+    t.insert({a:false});
+
+    obj = {};
+    obj[op] = ['$a', val];
+    result = t.aggregate({$project: {_id: 0, bool: obj}});
+
+    assert.commandWorked(result);
+    assert.eq(result.result, [{bool:true}, {bool:false}]);
+}
+test('$and', true);
+test('$or', false);

jstests/sharding/deletion_range.js

@@ -0,0 +1,43 @@
+//
+// Tests deletion ranges for a sharded system when using prefix shard key
+//
+
+var st = new ShardingTest({ shards : 2, mongos : 2 });
+
+st.stopBalancer();
+
+var mongos = st.s0;
+var config = mongos.getDB( "config" );
+var admin = mongos.getDB( "admin" );
+var shards = config.shards.find().toArray();
+var shard0 = new Mongo( shards[0].host );
+var shard1 = new Mongo( shards[1].host );
+
+var coll = mongos.getCollection( "foo.bar" );
+
+printjson( admin.runCommand({ enableSharding : coll.getDB() + "" }) );
+printjson( admin.runCommand({ movePrimary : coll.getDB() + "", to : shards[0]._id }) );
+printjson( coll.ensureIndex({ skey : 1, extra : 1 }) );
+printjson( admin.runCommand({ shardCollection : coll + "", key : { skey : 1 } }) );
+
+for( var i = 0; i < 5; i++ ){
+    coll.insert({ skey : 0, extra : i });
+}
+assert.eq( null, coll.getDB().getLastError() );
+
+printjson( admin.runCommand({ split : coll + "", middle : { skey : 0 } }) );
+printjson( admin.runCommand({ moveChunk : coll + "", find : { skey : 0 }, to : shards[1]._id }) );
+
+printjson( shard0.getCollection( coll + "" ).find().toArray() );
+printjson( shard1.getCollection( coll + "" ).find().toArray() );
+
+assert( coll.find().itcount() == 5 );
+
+printjson( admin.runCommand({ moveChunk : coll + "", find : { skey : -1 }, to : shards[1]._id }) );
+
+assert.eq( 0 , shard0.getCollection( coll + "" ).find().itcount() );
+assert.eq( 5 , shard1.getCollection( coll + "" ).find().itcount() );
+
+assert( coll.find().itcount() == 5 );
+
+st.stop()

jstests/splitvector.js

@@ -28,6 +28,22 @@ assertChunkSizes = function ( splitVec , numDocs , maxChunkSize , msg ){
     }
 }
 
+// Takes two documents and asserts that both contain exactly the same set of field names.
+// This is useful for checking that splitPoints have the same format as the original key pattern,
+// even when sharding on a prefix key.
+// Not very efficient, so only call when # of field names is small
+var assertFieldNamesMatch = function( splitPoint , keyPattern ){
+    for ( var p in splitPoint ) {
+        if( splitPoint.hasOwnProperty( p ) ) {
+            assert( keyPattern.hasOwnProperty( p ) , "property " + p + " not in keyPattern" );
+        }
+    }
+    for ( var p in keyPattern ) {
+        if( keyPattern.hasOwnProperty( p ) ){
+            assert( splitPoint.hasOwnProperty( p ) , "property " + p + " not in splitPoint" );
+        }
+    }
+}
 
 // -------------------------
 //  TESTS START HERE
@@ -84,6 +100,9 @@ var case4 = function() {
     assert.eq( true , res.ok , "4b" );
     assert.close( numDocs*docSize / ((1<<20) * factor), res.splitKeys.length , "num split keys" , -1 );
     assertChunkSizes( res.splitKeys , numDocs, (1<<20) * factor , "4d" );
+    for( i=0; i < res.splitKeys.length; i++ ){
+        assertFieldNamesMatch( res.splitKeys[i] , {x : 1} );
+    }
 }
 case4();
 
@@ -104,6 +123,9 @@ var case5 = function() {
 
     assert.eq( true , res.ok , "5a" );
     assert.eq( 1 , res.splitKeys.length , "5b" );
+    for( i=0; i < res.splitKeys.length; i++ ){
+        assertFieldNamesMatch( res.splitKeys[i] , {x : 1} );
+    }
 }
 case5();
 
@@ -124,6 +146,9 @@ var case6 = function() {
 
     assert.eq( true , res.ok , "6a" );
     assert.eq( 19 , res.splitKeys.length , "6b" );
+    for( i=0; i < res.splitKeys.length; i++ ){
+        assertFieldNamesMatch( res.splitKeys[i] , {x : 1} );
+    }
 }
 case6();
 
@@ -149,6 +174,9 @@ var case7 = function() {
 
     assert.eq( true , res.ok , "7a" );
     assert.eq( 2 , res.splitKeys[0].x, "7b");
+    for( i=0; i < res.splitKeys.length; i++ ){
+        assertFieldNamesMatch( res.splitKeys[i] , {x : 1} );
+    }
 }
 case7();
 
@@ -180,6 +208,9 @@ var case8 = function() {
     assert.eq( 2 , res.splitKeys.length , "8b" );
     assert.eq( 2 , res.splitKeys[0].x , "8c" );
     assert.eq( 3 , res.splitKeys[1].x , "8d" );
+    for( i=0; i < res.splitKeys.length; i++ ){
+        assertFieldNamesMatch( res.splitKeys[i] , {x : 1} );
+    }
 }
 case8();
 
@@ -211,6 +242,9 @@ var case9 = function() {
         assert.eq( true , res.ok , "9a: " + tojson(res) );
         assert.eq( 1 , res.splitKeys.length , "9b: " + tojson(res) );
         assert.eq( 2 , res.splitKeys[0].x , "9c: " + tojson(res) );
+        for( i=0; i < res.splitKeys.length; i++ ){
+            assertFieldNamesMatch( res.splitKeys[i] , {x : 1} );
+        }
     }
 }
 case9();

rpm/mongo.spec

@@ -1,7 +1,7 @@
 Name: mongo-10gen
 Conflicts: mongo, mongo-10gen-unstable
 Obsoletes: mongo-stable
-Version: 2.1.2
+Version: 2.2.0
 Release: mongodb_1%{?dist}
 Summary: mongodb client shell and tools
 License: AGPL 3.0

src/mongo/db/btreecursor.cpp

@@ -221,20 +221,20 @@ namespace mongo {
         NamespaceDetails *d, int idxNo, const IndexDetails& id, 
         const BSONObj &startKey, const BSONObj &endKey, bool endKeyInclusive, int direction) 
     { 
-        BtreeCursor *c = make( d , idxNo , id );
+        auto_ptr<BtreeCursor> c( make( d , idxNo , id ) );
         c->init(startKey,endKey,endKeyInclusive,direction);
         c->initWithoutIndependentFieldRanges();
         dassert( c->_dups.size() == 0 );
-        return c;
+        return c.release();
     }
 
     BtreeCursor* BtreeCursor::make(
         NamespaceDetails *d, int idxNo, const IndexDetails& id, 
         const shared_ptr< FieldRangeVector > &bounds, int singleIntervalLimit, int direction )
     {
-        BtreeCursor *c = make( d , idxNo , id );
+        auto_ptr<BtreeCursor> c( make( d , idxNo , id ) );
         c->init(bounds,singleIntervalLimit,direction);
-        return c;
+        return c.release();
     }
 
     BtreeCursor::BtreeCursor( NamespaceDetails* nsd , int theIndexNo, const IndexDetails& id ) 

src/mongo/db/commands/mr.cpp

@@ -1024,7 +1024,7 @@ namespace mongo {
 
                 uassert( 16149 , "cannot run map reduce without the js engine", globalScriptEngine );
 
-                auto_ptr<ClientCursor> holdCursor;
+                ClientCursor::Holder holdCursor;
                 ShardChunkManagerPtr chunkManager;
 
                 {
@@ -1040,8 +1040,8 @@ namespace mongo {
                     // Get a very basic cursor, prevents deletion of migrated data while we m/r
                     shared_ptr<Cursor> temp = NamespaceDetailsTransient::getCursor( config.ns.c_str(), BSONObj(), BSONObj() );
                     uassert( 15876, str::stream() << "could not create cursor over " << config.ns << " to hold data while prepping m/r", temp.get() );
-                    holdCursor = auto_ptr<ClientCursor>( new ClientCursor( QueryOption_NoCursorTimeout , temp , config.ns.c_str() ) );
-                    uassert( 15877, str::stream() << "could not create m/r holding client cursor over " << config.ns, holdCursor.get() );
+                    holdCursor.reset( new ClientCursor( QueryOption_NoCursorTimeout , temp , config.ns.c_str() ) );
+                    uassert( 15877, str::stream() << "could not create m/r holding client cursor over " << config.ns, holdCursor );
 
                 }
 

src/mongo/db/d_concurrency.cpp

@@ -278,11 +278,24 @@ namespace mongo {
     void Lock::ParallelBatchWriterMode::iAmABatchParticipant() {
         lockState()._batchWriter = true;
     }
-    Lock::ParallelBatchWriterSupport::ParallelBatchWriterSupport() :
-        _lk( lockState()._batchWriter ? 0 : new RWLockRecursive::Shared(ParallelBatchWriterMode::_batchLock) )
-    {
+
+    Lock::ParallelBatchWriterSupport::ParallelBatchWriterSupport() {
+        relock();
     }
 
+    void Lock::ParallelBatchWriterSupport::tempRelease() {
+        _lk.reset( 0 );
+    }
+
+    void Lock::ParallelBatchWriterSupport::relock() {
+        LockState& ls = lockState();
+        if ( ! ls._batchWriter ) {
+            AcquiringParallelWriter a(ls);
+            _lk.reset( new RWLockRecursive::Shared(ParallelBatchWriterMode::_batchLock) );
+        }
+    }
+
+
     Lock::ScopedLock::ScopedLock( char type ) 
         : _type(type), _stat(0) {
         LockState& ls = lockState();
@@ -306,6 +319,7 @@ namespace mongo {
     void Lock::ScopedLock::tempRelease() {
         long long micros = _timer.micros();
         _tempRelease();
+        _pbws_lk.tempRelease();
         _recordTime( micros ); // might as well do after we unlock
     }
 
@@ -324,6 +338,7 @@ namespace mongo {
     }
     
     void Lock::ScopedLock::relock() {
+        _pbws_lk.relock();
         _relock();
         resetTime();
     }

src/mongo/db/d_concurrency.h

@@ -78,9 +78,15 @@ namespace mongo {
 
     private:
         class ParallelBatchWriterSupport : boost::noncopyable {
-            scoped_ptr<RWLockRecursive::Shared> _lk;
         public:
             ParallelBatchWriterSupport();
+
+        private:
+            void tempRelease();
+            void relock();
+
+            scoped_ptr<RWLockRecursive::Shared> _lk;
+            friend class ScopedLock;
         };
 
     public:

src/mongo/db/dbcommands.cpp

@@ -1249,8 +1249,9 @@ namespace mongo {
                     errmsg = "couldn't find valid index containing key pattern";
                     return false;
                 }
+                // If both min and max non-empty, append MinKey's to make them fit chosen index
                 min = Helpers::modifiedRangeBound( min , idx->keyPattern() , -1 );
-                max = Helpers::modifiedRangeBound( max , idx->keyPattern() , 1 );
+                max = Helpers::modifiedRangeBound( max , idx->keyPattern() , -1 );
 
                 c.reset( BtreeCursor::make( d, d->idxNo(*idx), *idx, min, max, false, 1 ) );
             }

src/mongo/db/dbhelpers.cpp

@@ -293,8 +293,12 @@ namespace mongo {
                     
                     IndexDetails& i = nsd->idx( ii );
 
+                    // Extend min to get (min, MinKey, MinKey, ....)
                     BSONObj newMin = Helpers::modifiedRangeBound( min , keyPattern , -1 );
-                    BSONObj newMax = Helpers::modifiedRangeBound( max , keyPattern , 1 );
+                    // If upper bound is included, extend max to get (max, MaxKey, MaxKey, ...)
+                    // If not included, extend max to get (max, MinKey, MinKey, ....)
+                    int minOrMax = maxInclusive ? 1 : -1;
+                    BSONObj newMax = Helpers::modifiedRangeBound( max , keyPattern , minOrMax );
                     
                     c.reset( BtreeCursor::make( nsd , ii , i , newMin , newMax , maxInclusive , 1 ) );
                 }

src/mongo/db/lockstate.cpp

@@ -35,7 +35,8 @@ namespace mongo {
           _otherCount(0), 
           _otherLock(NULL),
           _scopedLk(NULL),
-          _lockPending(false)
+          _lockPending(false),
+          _lockPendingParallelWriter(false)
     {
     }
 
@@ -241,4 +242,13 @@ namespace mongo {
             stat->recordAcquireTimeMicros( _ls.threadState(), _lock->acquireFinished( stat ) );
     }
     
+    AcquiringParallelWriter::AcquiringParallelWriter( LockState& ls )
+        : _ls( ls ) {
+        _ls._lockPendingParallelWriter = true;
+    }
+    
+    AcquiringParallelWriter::~AcquiringParallelWriter() {
+        _ls._lockPendingParallelWriter = false;
+    }
+
 }

src/mongo/db/lockstate.h

@@ -50,7 +50,7 @@ namespace mongo {
         bool isLocked( const StringData& ns ); // rwRW
 
         /** pending means we are currently trying to get a lock */
-        bool hasLockPending() const { return _lockPending; }
+        bool hasLockPending() const { return _lockPending || _lockPendingParallelWriter; }
 
         // ----
 
@@ -105,8 +105,10 @@ namespace mongo {
         Lock::ScopedLock* _scopedLk;   
         
         bool _lockPending;
+        bool _lockPendingParallelWriter;
 
         friend class Acquiring;
+        friend class AcquiringParallelWriter;
     };
 
     class WrapperForRWLock : boost::noncopyable { 
@@ -132,7 +134,14 @@ namespace mongo {
         LockState& _ls;
     };
         
+    class AcquiringParallelWriter {
+    public:
+        AcquiringParallelWriter( LockState& ls );
+        ~AcquiringParallelWriter();
 
+    private:
+        LockState& _ls;
+    };
 
 
 }

src/mongo/db/oplog.cpp

@@ -720,7 +720,7 @@ namespace mongo {
     /** @param fromRepl false if from ApplyOpsCmd
         @return true if was and update should have happened and the document DNE.  see replset initial sync code.
      */
-    bool applyOperation_inlock(const BSONObj& op , bool fromRepl ) {
+    bool applyOperation_inlock(const BSONObj& op, bool fromRepl, bool convertUpdateToUpsert) {
         LOG(6) << "applying op: " << op << endl;
         bool failedUpdate = false;
 
@@ -789,7 +789,7 @@ namespace mongo {
 
             OpDebug debug;
             BSONObj updateCriteria = op.getObjectField("o2");
-            bool upsert = fields[3].booleanSafe();
+            bool upsert = fields[3].booleanSafe() || convertUpdateToUpsert;
             UpdateResult ur = updateObjects(ns, o, updateCriteria, upsert, /*multi*/ false,
                                             /*logop*/ false , debug, /*fromMigrate*/ false,
                                             QueryPlanSelectionPolicy::idElseNatural() );

src/mongo/db/oplog.h

@@ -156,5 +156,5 @@ namespace mongo {
      * @param fromRepl really from replication or for testing/internal/command/etc...
      * Returns if the op was an update that could not be applied (true on failure)
      */
-    bool applyOperation_inlock(const BSONObj& op , bool fromRepl = true );
+    bool applyOperation_inlock(const BSONObj& op, bool fromRepl = true, bool convertUpdateToUpsert = false);
 }

src/mongo/db/pipeline/accumulator.cpp

@@ -38,17 +38,17 @@ namespace mongo {
 
     void Accumulator::opToBson(
         BSONObjBuilder *pBuilder, string opName,
-        string fieldName) const {
+        string fieldName, bool requireExpression) const {
         verify(vpOperand.size() == 1);
         BSONObjBuilder builder;
-        vpOperand[0]->addToBsonObj(&builder, opName, false);
+        vpOperand[0]->addToBsonObj(&builder, opName, requireExpression);
         pBuilder->append(fieldName, builder.done());
     }
 
     void Accumulator::addToBsonObj(
         BSONObjBuilder *pBuilder, string fieldName,
         bool requireExpression) const {
-        opToBson(pBuilder, getOpName(), fieldName);
+        opToBson(pBuilder, getOpName(), fieldName, requireExpression);
     }
 
     void Accumulator::addToBsonArray(BSONArrayBuilder *pBuilder) const {

src/mongo/db/pipeline/accumulator.h

@@ -56,7 +56,8 @@ namespace mongo {
           @param opName the operator name
          */
         void opToBson(
-            BSONObjBuilder *pBuilder, string fieldName, string opName) const;
+            BSONObjBuilder *pBuilder, string fieldName, string opName,
+            bool requireExpression) const;
     };
 
 

src/mongo/db/pipeline/document_source_group.cpp

@@ -72,14 +72,14 @@ namespace mongo {
         BSONObjBuilder insides;
 
         /* add the _id */
-        pIdExpression->addToBsonObj(&insides, Document::idName.c_str(), false);
+        pIdExpression->addToBsonObj(&insides, Document::idName.c_str(), true);
 
         /* add the remaining fields */
         const size_t n = vFieldName.size();
         for(size_t i = 0; i < n; ++i) {
             intrusive_ptr<Accumulator> pA((*vpAccumulatorFactory[i])(pExpCtx));
             pA->addOperand(vpExpression[i]);
-            pA->addToBsonObj(&insides, vFieldName[i], false);
+            pA->addToBsonObj(&insides, vFieldName[i], true);
         }
 
         pBuilder->append(groupName, insides.done());

src/mongo/db/pipeline/expression.cpp

@@ -526,14 +526,24 @@ namespace mongo {
     }
 
     void ExpressionCoerceToBool::addToBsonObj(
-        BSONObjBuilder *pBuilder, string fieldName,
-        bool requireExpression) const {
-        verify(false && "not possible"); // no equivalent of this
+            BSONObjBuilder *pBuilder, string fieldName,
+            bool requireExpression) const {
+        // Serializing as an $and expression which will become a CoerceToBool
+        BSONObjBuilder sub (pBuilder->subobjStart(fieldName));
+        BSONArrayBuilder arr (sub.subarrayStart("$and"));
+        pExpression->addToBsonArray(&arr);
+        arr.doneFast();
+        sub.doneFast();
     }
 
     void ExpressionCoerceToBool::addToBsonArray(
-        BSONArrayBuilder *pBuilder) const {
-        verify(false && "not possible"); // no equivalent of this
+            BSONArrayBuilder *pBuilder) const {
+        // Serializing as an $and expression which will become a CoerceToBool
+        BSONObjBuilder sub (pBuilder->subobjStart());
+        BSONArrayBuilder arr (sub.subarrayStart("$and"));
+        pExpression->addToBsonArray(&arr);
+        arr.doneFast();
+        sub.doneFast();
     }
 
     /* ----------------------- ExpressionCompare --------------------------- */
@@ -606,10 +616,10 @@ namespace mongo {
         /*             -1      0      1      reverse          name   */
         /* EQ  */ { { false, true,  false }, Expression::EQ,  "$eq"  },
         /* NE  */ { { true,  false, true },  Expression::NE,  "$ne"  },
-        /* GT  */ { { false, false, true },  Expression::LTE, "$gt"  },
-        /* GTE */ { { false, true,  true },  Expression::LT,  "$gte" },
-        /* LT  */ { { true,  false, false }, Expression::GTE, "$lt"  },
-        /* LTE */ { { true,  true,  false }, Expression::GT,  "$lte" },
+        /* GT  */ { { false, false, true },  Expression::LT,  "$gt"  },
+        /* GTE */ { { false, true,  true },  Expression::LTE, "$gte" },
+        /* LT  */ { { true,  false, false }, Expression::GT,  "$lt"  },
+        /* LTE */ { { true,  true,  false }, Expression::GTE, "$lte" },
         /* CMP */ { { false, false, false }, Expression::CMP, "$cmp" },
     };
 

src/mongo/db/repl/rs_sync.cpp

@@ -50,7 +50,7 @@ namespace replset {
     /* apply the log op that is in param o
        @return bool success (true) or failure (false)
     */
-    bool SyncTail::syncApply(const BSONObj &op) {
+    bool SyncTail::syncApply(const BSONObj &op, bool convertUpdateToUpsert) {
         const char *ns = op.getStringField("ns");
         verify(ns);
 
@@ -79,7 +79,9 @@ namespace replset {
 
         Client::Context ctx(ns, dbpath, false);
         ctx.getClient()->curop()->reset();
-        bool ok = !applyOperation_inlock(op);
+        // For non-initial-sync, we convert updates to upserts
+        // to suppress errors when replaying oplog entries.
+        bool ok = !applyOperation_inlock(op, true, convertUpdateToUpsert);
         getDur().commitIfNeeded();
 
         return ok;
@@ -111,7 +113,7 @@ namespace replset {
              it != ops.end();
              ++it) {
             try {
-                fassert(16359, st->syncApply(*it));
+                fassert(16359, st->syncApply(*it, true));
             } catch (DBException& e) {
                 error() << "writer worker caught exception: " << e.what() 
                         << " on: " << it->toString() << endl;

src/mongo/db/repl/rs_sync.h

@@ -37,7 +37,7 @@ namespace replset {
     public:
         SyncTail(BackgroundSyncInterface *q);
         virtual ~SyncTail();
-        virtual bool syncApply(const BSONObj &o);
+        virtual bool syncApply(const BSONObj &o, bool convertUpdateToUpsert = false);
         void oplogApplication();
         bool peek(BSONObj* obj);
 

src/mongo/dbtests/documentsourcetests.cpp

@@ -31,6 +31,12 @@ namespace DocumentSourceTests {
     static const char* const ns = "unittests.documentsourcetests";
     static DBDirectClient client;
 
+    BSONObj toBson( const intrusive_ptr<DocumentSource>& source ) {
+        BSONArrayBuilder bab;
+        source->addToBsonArray( &bab );
+        return bab.arr()[ 0 ].Obj().getOwned();
+    }
+
     class CollectionBase {
     public:
         ~CollectionBase() {
@@ -315,14 +321,16 @@ namespace DocumentSourceTests {
 
         class Base : public DocumentSourceCursor::Base {
         protected:
-            void createGroup( const BSONObj &spec ) {
+            void createGroup( const BSONObj &spec, bool inShard = false ) {
                 BSONObj namedSpec = BSON( "$group" << spec );
                 BSONElement specElement = namedSpec.firstElement();
-                _group = DocumentSourceGroup::createFromBson( &specElement, ctx() );
-                BSONArrayBuilder bab;
-                _group->addToBsonArray( &bab );
-                // The $group spec round trips.
-                ASSERT_EQUALS( namedSpec, bab.arr()[ 0 ].Obj().getOwned() );
+                intrusive_ptr<ExpressionContext> expressionContext =
+                        ExpressionContext::create( &InterruptStatusMongod::status );
+                if ( inShard ) {
+                    expressionContext->setInShard( true );
+                }
+                _group = DocumentSourceGroup::createFromBson( &specElement, expressionContext );
+                assertRoundTrips( _group );
                 _group->setSource( source() );
             }
             DocumentSource* group() { return _group.get(); }
@@ -336,6 +344,16 @@ namespace DocumentSourceTests {
                 ASSERT( !source->getCurrent() );
             }
         private:
+            /** Check that the group's spec round trips. */
+            void assertRoundTrips( const intrusive_ptr<DocumentSource>& group ) {
+                // We don't check against the spec that generated 'group' originally, because
+                // $const operators may be introduced in the first serialization.
+                BSONObj spec = toBson( group );
+                BSONElement specElement = spec.firstElement();
+                intrusive_ptr<DocumentSource> generated =
+                        DocumentSourceGroup::createFromBson( &specElement, ctx() );
+                ASSERT_EQUALS( spec, toBson( generated ) );
+            }
             intrusive_ptr<DocumentSource> _group;
         };
 
@@ -540,6 +558,8 @@ namespace DocumentSourceTests {
                 intrusive_ptr<DocumentSource> sink = group();
                 if ( sharded ) {
                     sink = createMerger();
+                    // Serialize and re-parse the shard stage.
+                    createGroup( toBson( group() )[ "$group" ].Obj(), true );
                     sink->setSource( group() );
                 }
 
@@ -562,18 +582,10 @@ namespace DocumentSourceTests {
                 // case only one shard is in use.
                 SplittableDocumentSource *splittable =
                         dynamic_cast<SplittableDocumentSource*>( group() );
+                ASSERT( splittable );
                 intrusive_ptr<DocumentSource> routerSource = splittable->getRouterSource();
                 ASSERT_NOT_EQUALS( group(), routerSource.get() );
-                BSONArrayBuilder bab;
-                routerSource->addToBsonArray( &bab );
-                BSONObj routerSourceSpec = bab.arr()[ 0 ].Obj().getOwned();
-                BSONElement routerSourceSpecElement = routerSourceSpec.firstElement();
-                intrusive_ptr<ExpressionContext> routerContext =
-                        ExpressionContext::create( &InterruptStatusMongod::status );
-                routerContext->setDoingMerge(true);
-                routerContext->setInShard(false);
-                return DocumentSourceGroup::createFromBson( &routerSourceSpecElement,
-                                                            routerContext );
+                return routerSource;
             }
             void checkResultSet( const intrusive_ptr<DocumentSource> &sink ) {
                 // Load the results from the DocumentSourceGroup and sort them by _id.
@@ -800,6 +812,36 @@ namespace DocumentSourceTests {
             }
         };
 
+        /**
+         * A string constant (not a field path) as an _id expression and passed to an accumulator.
+         * SERVER-6766
+         */
+        class StringConstantIdAndAccumulatorExpressions : public CheckResultsBase {
+            void populateData() { client.insert( ns, BSONObj() ); }
+            BSONObj groupSpec() {
+                return fromjson( "{_id:{$const:'$_id...'},a:{$push:{$const:'$a...'}}}" );
+            }
+            string expectedResultSetString() { return "[{_id:'$_id...',a:['$a...']}]"; }
+        };
+
+        /** An array constant passed to an accumulator. */
+        class ArrayConstantAccumulatorExpression : public CheckResultsBase {
+        public:
+            void run() {
+                // A parse exception is thrown when a raw array is provided to an accumulator.
+                ASSERT_THROWS( createGroup( fromjson( "{_id:1,a:{$push:[4,5,6]}}" ) ),
+                               UserException );
+                // Run standard base tests.
+                CheckResultsBase::run();
+            }
+            void populateData() { client.insert( ns, BSONObj() ); }
+            BSONObj groupSpec() {
+                // An array can be specified using $const.
+                return fromjson( "{_id:[1,2,3],a:{$push:{$const:[4,5,6]}}}" );
+            }
+            string expectedResultSetString() { return "[{_id:[1,2,3],a:[[4,5,6]]}]"; }
+        };
+
     } // namespace DocumentSourceGroup
 
     namespace DocumentSourceProject {
@@ -1694,6 +1736,8 @@ namespace DocumentSourceTests {
             add<DocumentSourceGroup::UndefinedAccumulatorValue>();
             add<DocumentSourceGroup::RouterMerger>();
             add<DocumentSourceGroup::Dependencies>();
+            add<DocumentSourceGroup::StringConstantIdAndAccumulatorExpressions>();
+            add<DocumentSourceGroup::ArrayConstantAccumulatorExpression>();
 
             add<DocumentSourceProject::EofInit>();
             add<DocumentSourceProject::AdvanceInit>();

src/mongo/dbtests/expressiontests.cpp

@@ -39,6 +39,13 @@ namespace ExpressionTests {
         return bob.obj();
     }
 
+    /** Convert Expression to BSON. */
+    static BSONObj expressionToBson( const intrusive_ptr<Expression>& expression ) {
+        BSONObjBuilder bob;
+        expression->addToBsonObj( &bob, "", true );
+        return bob.obj().firstElement().embeddedObject().getOwned();
+    }
+
     /** Create a Document from a BSONObj. */
     intrusive_ptr<Document> fromBson( BSONObj obj ) {
         return Document::createFromBsonObj( &obj );
@@ -81,10 +88,364 @@ namespace ExpressionTests {
             }
         };
 
+        /** Output to BSONObj. */
+        class AddToBsonObj {
+        public:
+            void run() {
+                intrusive_ptr<Expression> expression = ExpressionCoerceToBool::create(
+                        ExpressionFieldPath::create("foo"));
+
+                // serialized as $and because CoerceToBool isn't an ExpressionNary
+                assertBinaryEqual(fromjson("{field:{$and:['$foo']}}"), toBsonObj(expression));
+            }
+        private:
+            static BSONObj toBsonObj(const intrusive_ptr<Expression>& expression) {
+
+                BSONObjBuilder bob;
+                expression->addToBsonObj(&bob, "field", false);
+                return bob.obj();
+            }
+        };
+
+        /** Output to BSONArray. */
+        class AddToBsonArray {
+        public:
+            void run() {
+                intrusive_ptr<Expression> expression = ExpressionCoerceToBool::create(
+                        ExpressionFieldPath::create("foo"));
+
+                // serialized as $and because CoerceToBool isn't an ExpressionNary
+                assertBinaryEqual(BSON_ARRAY(fromjson("{$and:['$foo']}")), toBsonArray(expression));
+            }
+        private:
+            static BSONArray toBsonArray(const intrusive_ptr<Expression>& expression) {
+                BSONArrayBuilder bab;
+                expression->addToBsonArray(&bab);
+                return bab.arr();
+            }
+        };
+
+
         // TODO Test optimize(), difficult because a CoerceToBool cannot be output as BSON.
         
     } // namespace CoerceToBool
 
+    namespace Compare {
+
+        class OptimizeBase {
+        public:
+            virtual ~OptimizeBase() {
+            }
+            void run() {
+                BSONObj specObject = BSON( "" << spec() );
+                BSONElement specElement = specObject.firstElement();
+                intrusive_ptr<Expression> expression = Expression::parseOperand( &specElement );
+                intrusive_ptr<Expression> optimized = expression->optimize();
+                ASSERT_EQUALS( expectedFieldRange(),
+                               (bool)dynamic_pointer_cast<ExpressionFieldRange>( optimized ) );
+                ASSERT_EQUALS( expectedOptimized(), expressionToBson( optimized ) );
+            }
+        protected:
+            virtual BSONObj spec() = 0;
+            virtual BSONObj expectedOptimized() = 0;
+            virtual bool expectedFieldRange() = 0;
+        };
+        
+        class FieldRangeOptimize : public OptimizeBase {
+            BSONObj expectedOptimized() { return spec(); }
+            bool expectedFieldRange() { return true; }
+        };
+        
+        class NoOptimize : public OptimizeBase {
+            BSONObj expectedOptimized() { return spec(); }
+            bool expectedFieldRange() { return false; }
+        };
+
+        /** Check expected result for expressions depending on constants. */
+        class ExpectedResultBase : public OptimizeBase {
+        public:
+            void run() {
+                OptimizeBase::run();
+                BSONObj specObject = BSON( "" << spec() );
+                BSONElement specElement = specObject.firstElement();
+                intrusive_ptr<Expression> expression = Expression::parseOperand( &specElement );
+                // Check expression spec round trip.
+                ASSERT_EQUALS( spec(), expressionToBson( expression ) );
+                // Check evaluation result.
+                ASSERT_EQUALS( expectedResult(),
+                               toBson( expression->evaluate( Document::create() ) ) );
+                // Check that the result is the same after optimizing.
+                intrusive_ptr<Expression> optimized = expression->optimize();
+                ASSERT_EQUALS( expectedResult(),
+                               toBson( optimized->evaluate( Document::create() ) ) );
+            }
+        protected:
+            virtual BSONObj spec() = 0;
+            virtual BSONObj expectedResult() = 0;
+        private:
+            virtual BSONObj expectedOptimized() {
+                return BSON( "$const" << expectedResult().firstElement() );
+            }
+            virtual bool expectedFieldRange() { return false; }
+        };
+
+        class ExpectedTrue : public ExpectedResultBase {
+            BSONObj expectedResult() { return BSON( "" << true ); }            
+        };
+        
+        class ExpectedFalse : public ExpectedResultBase {
+            BSONObj expectedResult() { return BSON( "" << false ); }            
+        };
+        
+        class ParseError {
+        public:
+            virtual ~ParseError() {
+            }
+            void run() {
+                BSONObj specObject = BSON( "" << spec() );
+                BSONElement specElement = specObject.firstElement();
+                ASSERT_THROWS( Expression::parseOperand( &specElement ), UserException );
+            }
+        protected:
+            virtual BSONObj spec() = 0;
+        };
+
+        /** $eq with first < second. */
+        class EqLt : public ExpectedFalse {
+            BSONObj spec() { return BSON( "$eq" << BSON_ARRAY( 1 << 2 ) ); }
+        };
+
+        /** $eq with first == second. */
+        class EqEq : public ExpectedTrue {
+            BSONObj spec() { return BSON( "$eq" << BSON_ARRAY( 1 << 1 ) ); }
+        };
+        
+        /** $eq with first > second. */
+        class EqGt : public ExpectedFalse {
+            BSONObj spec() { return BSON( "$eq" << BSON_ARRAY( 1 << 0 ) ); }
+        };
+        
+        /** $ne with first < second. */
+        class NeLt : public ExpectedTrue {
+            BSONObj spec() { return BSON( "$ne" << BSON_ARRAY( 1 << 2 ) ); }
+        };
+        
+        /** $ne with first == second. */
+        class NeEq : public ExpectedFalse {
+            BSONObj spec() { return BSON( "$ne" << BSON_ARRAY( 1 << 1 ) ); }
+        };
+        
+        /** $ne with first > second. */
+        class NeGt : public ExpectedTrue {
+            BSONObj spec() { return BSON( "$ne" << BSON_ARRAY( 1 << 0 ) ); }
+        };
+        
+        /** $gt with first < second. */
+        class GtLt : public ExpectedFalse {
+            BSONObj spec() { return BSON( "$gt" << BSON_ARRAY( 1 << 2 ) ); }
+        };
+        
+        /** $gt with first == second. */
+        class GtEq : public ExpectedFalse {
+            BSONObj spec() { return BSON( "$gt" << BSON_ARRAY( 1 << 1 ) ); }
+        };
+        
+        /** $gt with first > second. */
+        class GtGt : public ExpectedTrue {
+            BSONObj spec() { return BSON( "$gt" << BSON_ARRAY( 1 << 0 ) ); }
+        };
+        
+        /** $gte with first < second. */
+        class GteLt : public ExpectedFalse {
+            BSONObj spec() { return BSON( "$gte" << BSON_ARRAY( 1 << 2 ) ); }
+        };
+        
+        /** $gte with first == second. */
+        class GteEq : public ExpectedTrue {
+            BSONObj spec() { return BSON( "$gte" << BSON_ARRAY( 1 << 1 ) ); }
+        };
+        
+        /** $gte with first > second. */
+        class GteGt : public ExpectedTrue {
+            BSONObj spec() { return BSON( "$gte" << BSON_ARRAY( 1 << 0 ) ); }
+        };
+        
+        /** $lt with first < second. */
+        class LtLt : public ExpectedTrue {
+            BSONObj spec() { return BSON( "$lt" << BSON_ARRAY( 1 << 2 ) ); }
+        };
+        
+        /** $lt with first == second. */
+        class LtEq : public ExpectedFalse {
+            BSONObj spec() { return BSON( "$lt" << BSON_ARRAY( 1 << 1 ) ); }
+        };
+        
+        /** $lt with first > second. */
+        class LtGt : public ExpectedFalse {
+            BSONObj spec() { return BSON( "$lt" << BSON_ARRAY( 1 << 0 ) ); }
+        };
+        
+        /** $lte with first < second. */
+        class LteLt : public ExpectedTrue {
+            BSONObj spec() { return BSON( "$lte" << BSON_ARRAY( 1 << 2 ) ); }
+        };
+        
+        /** $lte with first == second. */
+        class LteEq : public ExpectedTrue {
+            BSONObj spec() { return BSON( "$lte" << BSON_ARRAY( 1 << 1 ) ); }
+        };
+        
+        /** $lte with first > second. */
+        class LteGt : public ExpectedFalse {
+            BSONObj spec() { return BSON( "$lte" << BSON_ARRAY( 1 << 0 ) ); }
+        };
+        
+        /** $cmp with first < second. */
+        class CmpLt : public ExpectedResultBase {
+            BSONObj spec() { return BSON( "$cmp" << BSON_ARRAY( 1 << 2 ) ); }
+            BSONObj expectedResult() { return BSON( "" << -1 ); }
+        };
+        
+        /** $cmp with first == second. */
+        class CmpEq : public ExpectedResultBase {
+            BSONObj spec() { return BSON( "$cmp" << BSON_ARRAY( 1 << 1 ) ); }
+            BSONObj expectedResult() { return BSON( "" << 0 ); }
+        };
+        
+        /** $cmp with first > second. */
+        class CmpGt : public ExpectedResultBase {
+            BSONObj spec() { return BSON( "$cmp" << BSON_ARRAY( 1 << 0 ) ); }
+            BSONObj expectedResult() { return BSON( "" << 1 ); }
+        };
+
+        /** $cmp results are bracketed to an absolute value of 1. */
+        class CmpBracketed : public ExpectedResultBase {
+            BSONObj spec() { return BSON( "$cmp" << BSON_ARRAY( "z" << "a" ) ); }
+            BSONObj expectedResult() { return BSON( "" << 1 ); }
+        };
+
+        /** Zero operands provided. */
+        class ZeroOperands : public ParseError {
+            BSONObj spec() { return BSON( "$ne" << BSONArray() ); }
+        };
+
+        /** One operand provided. */
+        class OneOperand : public ParseError {
+            BSONObj spec() { return BSON( "$eq" << BSON_ARRAY( 1 ) ); }
+        };
+        
+        /** Three operands provided. */
+        class ThreeOperands : public ParseError {
+            BSONObj spec() { return BSON( "$gt" << BSON_ARRAY( 2 << 3 << 4 ) ); }
+        };
+        
+        /** Incompatible types cannot be compared. */
+        class IncompatibleTypes {
+        public:
+            void run() {
+                BSONObj specObject = BSON( "" << BSON( "$ne" << BSON_ARRAY( "a" << 1 ) ) );
+                BSONElement specElement = specObject.firstElement();
+                intrusive_ptr<Expression> expression = Expression::parseOperand( &specElement );
+                ASSERT_THROWS( expression->evaluate( Document::create() ), UserException );
+            }
+        };
+
+        /**
+         * An expression depending on constants is optimized to a constant via
+         * ExpressionNary::optimize().
+         */
+        class OptimizeConstants : public OptimizeBase {
+            BSONObj spec() { return BSON( "$eq" << BSON_ARRAY( 1 << 1 ) ); }
+            BSONObj expectedOptimized() { return BSON( "$const" << true ); }
+            bool expectedFieldRange() { return false; }
+        };
+
+        /** $cmp is not optimized. */
+        class NoOptimizeCmp : public NoOptimize {
+            BSONObj spec() { return BSON( "$cmp" << BSON_ARRAY( 1 << "$a" ) ); }
+        };
+        
+        /** $ne is not optimized. */
+        class NoOptimizeNe : public NoOptimize {
+            BSONObj spec() { return BSON( "$ne" << BSON_ARRAY( 1 << "$a" ) ); }
+        };
+        
+        /** No optimization is performend without a constant. */
+        class NoOptimizeNoConstant : public NoOptimize {
+            BSONObj spec() { return BSON( "$ne" << BSON_ARRAY( "$a" << "$b" ) ); }
+        };
+        
+        /** No optimization is performend without an immediate field path. */
+        class NoOptimizeWithoutFieldPath : public NoOptimize {
+            BSONObj spec() {
+                return BSON( "$eq" << BSON_ARRAY( BSON( "$and" << BSON_ARRAY( "$a" ) ) << 1 ) );
+            }
+        };
+        
+        /** No optimization is performend without an immediate field path. */
+        class NoOptimizeWithoutFieldPathReverse : public NoOptimize {
+            BSONObj spec() {
+                return BSON( "$eq" << BSON_ARRAY( 1 << BSON( "$and" << BSON_ARRAY( "$a" ) ) ) );
+            }
+        };
+        
+        /** An equality expression is optimized. */
+        class OptimizeEq : public FieldRangeOptimize {
+            BSONObj spec() { return BSON( "$eq" << BSON_ARRAY( "$a" << 1 ) ); }
+        };
+        
+        /** A reverse sense equality expression is optimized. */
+        class OptimizeEqReverse : public FieldRangeOptimize {
+            BSONObj spec() { return BSON( "$eq" << BSON_ARRAY( 1 << "$a" ) ); }
+            BSONObj expectedOptimized() { return BSON( "$eq" << BSON_ARRAY( "$a" << 1 ) ); }
+        };
+        
+        /** A $lt expression is optimized. */
+        class OptimizeLt : public FieldRangeOptimize {
+            BSONObj spec() { return BSON( "$lt" << BSON_ARRAY( "$a" << 1 ) ); }
+        };
+        
+        /** A reverse sense $lt expression is optimized. */
+        class OptimizeLtReverse : public FieldRangeOptimize {
+            BSONObj spec() { return BSON( "$lt" << BSON_ARRAY( 1 << "$a" ) ); }
+            BSONObj expectedOptimized() { return BSON( "$gt" << BSON_ARRAY( "$a" << 1 ) ); }
+        };
+        
+        /** A $lte expression is optimized. */
+        class OptimizeLte : public FieldRangeOptimize {
+            BSONObj spec() { return BSON( "$lte" << BSON_ARRAY( "$b" << 2 ) ); }
+        };
+        
+        /** A reverse sense $lte expression is optimized. */
+        class OptimizeLteReverse : public FieldRangeOptimize {
+            BSONObj spec() { return BSON( "$lte" << BSON_ARRAY( 2 << "$b" ) ); }
+            BSONObj expectedOptimized() { return BSON( "$gte" << BSON_ARRAY( "$b" << 2 ) ); }
+        };
+        
+        /** A $gt expression is optimized. */
+        class OptimizeGt : public FieldRangeOptimize {
+            BSONObj spec() { return BSON( "$gt" << BSON_ARRAY( "$b" << 2 ) ); }
+        };
+        
+        /** A reverse sense $gt expression is optimized. */
+        class OptimizeGtReverse : public FieldRangeOptimize {
+            BSONObj spec() { return BSON( "$gt" << BSON_ARRAY( 2 << "$b" ) ); }
+            BSONObj expectedOptimized() { return BSON( "$lt" << BSON_ARRAY( "$b" << 2 ) ); }
+        };
+        
+        /** A $gte expression is optimized. */
+        class OptimizeGte : public FieldRangeOptimize {
+            BSONObj spec() { return BSON( "$gte" << BSON_ARRAY( "$b" << 2 ) ); }
+        };
+        
+        /** A reverse sense $gte expression is optimized. */
+        class OptimizeGteReverse : public FieldRangeOptimize {
+            BSONObj spec() { return BSON( "$gte" << BSON_ARRAY( 2 << "$b" ) ); }
+            BSONObj expectedOptimized() { return BSON( "$lte" << BSON_ARRAY( "$b" << 2 ) ); }
+        };
+        
+    } // namespace Compare
+    
     namespace Constant {
 
         /** Create an ExpressionConstant from a Value. */
@@ -399,6 +760,51 @@ namespace ExpressionTests {
             add<CoerceToBool::EvaluateTrue>();
             add<CoerceToBool::EvaluateFalse>();
             add<CoerceToBool::Dependencies>();
+            add<CoerceToBool::AddToBsonObj>();
+            add<CoerceToBool::AddToBsonArray>();
+
+            add<Compare::EqLt>();
+            add<Compare::EqEq>();
+            add<Compare::EqGt>();
+            add<Compare::NeLt>();
+            add<Compare::NeEq>();
+            add<Compare::NeGt>();
+            add<Compare::GtLt>();
+            add<Compare::GtEq>();
+            add<Compare::GtGt>();
+            add<Compare::GteLt>();
+            add<Compare::GteEq>();
+            add<Compare::GteGt>();
+            add<Compare::LtLt>();
+            add<Compare::LtEq>();
+            add<Compare::LtGt>();
+            add<Compare::LteLt>();
+            add<Compare::LteEq>();
+            add<Compare::LteGt>();
+            add<Compare::CmpLt>();
+            add<Compare::CmpEq>();
+            add<Compare::CmpGt>();
+            add<Compare::CmpBracketed>();
+            add<Compare::ZeroOperands>();
+            add<Compare::OneOperand>();
+            add<Compare::ThreeOperands>();
+            add<Compare::IncompatibleTypes>();
+            add<Compare::OptimizeConstants>();
+            add<Compare::NoOptimizeCmp>();
+            add<Compare::NoOptimizeNe>();
+            add<Compare::NoOptimizeNoConstant>();
+            add<Compare::NoOptimizeWithoutFieldPath>();
+            add<Compare::NoOptimizeWithoutFieldPathReverse>();
+            add<Compare::OptimizeEq>();
+            add<Compare::OptimizeEqReverse>();
+            add<Compare::OptimizeLt>();
+            add<Compare::OptimizeLtReverse>();
+            add<Compare::OptimizeLte>();
+            add<Compare::OptimizeLteReverse>();
+            add<Compare::OptimizeGt>();
+            add<Compare::OptimizeGtReverse>();
+            add<Compare::OptimizeGte>();
+            add<Compare::OptimizeGteReverse>();
 
             add<Constant::Create>();
             add<Constant::CreateFromBsonElement>();

src/mongo/dbtests/replsettests.cpp

@@ -178,7 +178,7 @@ namespace ReplSetTests {
         bool retry;
 
         // instead of actually applying operations, we return success or failure
-        virtual bool syncApply(const BSONObj& o) {
+        virtual bool syncApply(const BSONObj& o, bool convertUpdateToUpsert) {
             step++;
 
             if ((failOnStep == FAIL_FIRST_APPLY && step == 1) ||

src/mongo/s/d_migrate.cpp

@@ -437,9 +437,9 @@ namespace mongo {
                 errmsg = (string)"can't find index in storeCurrentLocs" + causedBy( errmsg );
                 return false;
             }
-
+            // Assume both min and max non-empty, append MinKey's to make them fit chosen index
             BSONObj min = Helpers::modifiedRangeBound( _min , idx->keyPattern() , -1 );
-            BSONObj max = Helpers::modifiedRangeBound( _max , idx->keyPattern() , 1 );
+            BSONObj max = Helpers::modifiedRangeBound( _max , idx->keyPattern() , -1 );
 
             BtreeCursor* btreeCursor = BtreeCursor::make( d , *idx , min , max , false , 1 );
             auto_ptr<ClientCursor> cc(

src/mongo/s/d_split.cpp

@@ -97,9 +97,15 @@ namespace mongo {
                 errmsg = "couldn't find valid index for shard key";
                 return false;
             }
-            //has the effect of extending empty min/max to match selected index
+            // extend min to get (min, MinKey, MinKey, ....)
             min = Helpers::modifiedRangeBound( min , idx->keyPattern() , -1 );
-            max = Helpers::modifiedRangeBound( max , idx->keyPattern() , 1 );
+            if  ( max.isEmpty() ) {
+                // if max not specified, make it (MaxKey, Maxkey, MaxKey...)
+                max = Helpers::modifiedRangeBound( max , idx->keyPattern() , 1 );
+            } else {
+                // otherwise make it (max,MinKey,MinKey...) so that bound is non-inclusive
+                max = Helpers::modifiedRangeBound( max , idx->keyPattern() , -1 );
+            }
 
             BtreeCursor * bc = BtreeCursor::make( d , d->idxNo(*idx) , *idx , min , max , false , 1 );
             shared_ptr<Cursor> c( bc );
@@ -196,17 +202,7 @@ namespace mongo {
             // If min and max are not provided use the "minKey" and "maxKey" for the sharding key pattern.
             BSONObj min = jsobj.getObjectField( "min" );
             BSONObj max = jsobj.getObjectField( "max" );
-            if ( min.isEmpty() && max.isEmpty() ) {
-                BSONObjBuilder minBuilder;
-                BSONObjBuilder maxBuilder;
-                BSONForEach(key, keyPattern) {
-                    minBuilder.appendMinKey( key.fieldName() );
-                    maxBuilder.appendMaxKey( key.fieldName() );
-                }
-                min = minBuilder.obj();
-                max = maxBuilder.obj();
-            }
-            else if ( min.isEmpty() || max.isEmpty() ) {
+            if ( min.isEmpty() != max.isEmpty() ){
                 errmsg = "either provide both min and max or leave both empty";
                 return false;
             }
@@ -241,8 +237,15 @@ namespace mongo {
                              keyPattern.clientReadable().toString();
                     return false;
                 }
+                // extend min to get (min, MinKey, MinKey, ....)
                 min = Helpers::modifiedRangeBound( min , idx->keyPattern() , -1 );
-                max = Helpers::modifiedRangeBound( max , idx->keyPattern() , 1 );
+                if  ( max.isEmpty() ) {
+                    // if max not specified, make it (MaxKey, Maxkey, MaxKey...)
+                    max = Helpers::modifiedRangeBound( max , idx->keyPattern() , 1 );
+                } else {
+                    // otherwise make it (max,MinKey,MinKey...) so that bound is non-inclusive
+                    max = Helpers::modifiedRangeBound( max , idx->keyPattern() , -1 );
+                }
                 
                 const long long recCount = d->stats.nrecords;
                 const long long dataSize = d->stats.datasize;
@@ -324,15 +327,13 @@ namespace mongo {
                 // at the end. If a key appears more times than entries allowed on a chunk, we issue a warning and
                 // split on the following key.
                 set<BSONObj> tooFrequentKeys;
-                splitKeys.push_back( c->currKey().getOwned() );
+                splitKeys.push_back( bc->prettyKey( c->currKey().getOwned() ).extractFields( keyPattern ) );
                 while ( 1 ) {
                     while ( cc->ok() ) {
                         currCount++;
-                        BSONObj currKey = c->currKey();
-                        
-                        DEV verify( currKey.woCompare( max ) <= 0 );
                         
                         if ( currCount > keyCount ) {
+                            BSONObj currKey = bc->prettyKey( c->currKey() ).extractFields(keyPattern);
                             // Do not use this split key if it is the same used in the previous split point.
                             if ( currKey.woCompare( splitKeys.back() ) == 0 ) {
                                 tooFrequentKeys.insert( currKey.getOwned() );
@@ -343,7 +344,7 @@ namespace mongo {
                                 currCount = 0;
                                 numChunks++;
                                 
-                                LOG(4) << "picked a split key: " << bc->prettyKey( currKey ) << endl;
+                                LOG(4) << "picked a split key: " << currKey << endl;
                             }
                             
                         }
@@ -393,12 +394,9 @@ namespace mongo {
                               << " bytes because of key " << bc->prettyKey( *it ) << endl;
                 }
                 
-                // Remove the sentinel at the beginning before returning and add fieldnames.
+                // Remove the sentinel at the beginning before returning
                 splitKeys.erase( splitKeys.begin() );
                 verify( c.get() );
-                for ( vector<BSONObj>::iterator it = splitKeys.begin(); it != splitKeys.end() ; ++it ) {
-                    *it = bc->prettyKey( *it );
-                }
                 
                 if ( timer.millis() > cmdLine.slowMS ) {
                     warning() << "Finding the split vector for " <<  ns << " over "<< keyPattern

src/mongo/s/strategy_shard.cpp

@@ -125,11 +125,20 @@ namespace mongo {
                         startFrom, hasMore ? cc->getId() : 0 );
             }
             else{
+                // Remote cursors are stored remotely, we shouldn't need this around.
+                // TODO: we should probably just make cursor an auto_ptr
+                scoped_ptr<ParallelSortClusteredCursor> cursorDeleter( cursor );
+
                 // TODO:  Better merge this logic.  We potentially can now use the same cursor logic for everything.
                 ShardPtr primary = cursor->getPrimary();
                 verify( primary.get() );
                 DBClientCursorPtr shardCursor = cursor->getShardCursor( *primary );
+
+                // Implicitly stores the cursor in the cache
                 r.reply( *(shardCursor->getMessage()) , shardCursor->originalHost() );
+
+                // We don't want to kill the cursor remotely if there's still data left
+                shardCursor->decouple();
             }
         }
 

src/mongo/util/stacktrace.cpp

@@ -9,10 +9,13 @@
 #include <vector>
 
 #ifdef _WIN32
+#include <sstream>
 #include <stdio.h>
 #include <boost/smart_ptr/scoped_array.hpp>
 #include "mongo/platform/windows_basic.h"
 #include <DbgHelp.h>
+#include "mongo/util/assert_util.h"
+#include "mongo/util/log.h"
 #endif
 
 #ifdef MONGO_HAVE_EXECINFO_BACKTRACE
@@ -169,7 +172,7 @@ namespace mongo {
         BOOL ret = SymInitialize( process, NULL, TRUE );
         if ( ret == FALSE ) {
             DWORD dosError = GetLastError();
-            os << "Stack trace failed, SymInitialize failed with error " <<
+            log() << "Stack trace failed, SymInitialize failed with error " <<
                     std::dec << dosError << std::endl;
             return;
         }
@@ -244,20 +247,21 @@ namespace mongo {
         ++sourceWidth;
         size_t frameCount = traceList.size();
         for ( size_t i = 0; i < frameCount; ++i ) {
-            os << traceList[i].moduleName << " ";
+            std::stringstream ss;
+            ss << traceList[i].moduleName << " ";
             size_t width = traceList[i].moduleName.length();
             while ( width < moduleWidth ) {
-                os << " ";
+                ss << " ";
                 ++width;
             }
-            os << traceList[i].sourceAndLine << " ";
+            ss << traceList[i].sourceAndLine << " ";
             width = traceList[i].sourceAndLine.length();
             while ( width < sourceWidth ) {
-                os << " ";
+                ss << " ";
                 ++width;
             }
-            os << traceList[i].symbolAndOffset;
-            os << std::endl;
+            ss << traceList[i].symbolAndOffset;
+            log() << ss.str() << std::endl;
         }
     }
 }

src/mongo/util/version.cpp

@@ -43,7 +43,7 @@ namespace mongo {
      *      1.2.3-rc4-pre-
      * If you really need to do something else you'll need to fix _versionArray()
      */
-    const char versionString[] = "2.2.0-rc1-pre-";
+    const char versionString[] = "2.2.0";
 
     // See unit test for example outputs
     static BSONArray _versionArray(const char* version){

src/third_party/gperftools-2.0/src/sampler.cc

@@ -56,6 +56,8 @@ DEFINE_int64(tcmalloc_sample_parameter,
 
 namespace tcmalloc {
 
+#if !defined(NO_TCMALLOC_SAMPLES) // 10gen
+
 // Statics for Sampler
 double Sampler::log_table_[1<<kFastlogNumBits];
 
@@ -126,5 +128,6 @@ size_t Sampler::PickNextSamplingPoint() {
   return static_cast<size_t>(min(0.0, (FastLog2(q) - 26)) * (-log(2.0)
                              * FLAGS_tcmalloc_sample_parameter) + 1);
 }
+#endif // !defined(NO_TCMALLOC_SAMPLES) // 10gen
 
 }  // namespace tcmalloc

src/third_party/gperftools-2.0/src/sampler.h

@@ -99,6 +99,27 @@ namespace tcmalloc {
 // only result in a single call to PickNextSamplingPoint.
 //-------------------------------------------------------------------
 
+#ifdef NO_TCMALLOC_SAMPLES // 10gen
+
+// Dummy class with same public interface as below
+class PERFTOOLS_DLL_DECL Sampler {
+ public:
+  void Init(uint32_t seed) {}
+  bool SampleAllocation(size_t k) { return false; }
+  static void InitStatics() {}
+  int GetSamplePeriod() { return 0; }
+
+# if 0
+  // "public" functions only used in the Sampler and tests of the Sampler
+  void Cleanup();
+  size_t PickNextSamplingPoint();
+  static uint64_t NextRandom(uint64_t rnd_);
+  static double FastLog2(const double & d);
+  static void PopulateFastLog2Table();
+# endif
+};
+
+#else
 class PERFTOOLS_DLL_DECL Sampler {
  public:
   // Initialize this sampler.
@@ -173,6 +194,7 @@ inline double Sampler::FastLog2(const double & d) {
   const int32_t exponent = ((x_high >> 20) & 0x7FF) - 1023;
   return exponent + log_table_[y];
 }
+#endif // NO_TCMALLOC_SAMPLES // 10gen
 
 }  // namespace tcmalloc